gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-13 00:26:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: improve legacy metadata and admin compatibility (#2202)

Browse Source
This commit is contained in:
weisd
2026-03-18 21:05:09 +08:00
committed by GitHub
parent 84077adf17
commit b9b7d86ae4
133 changed files with 11707 additions and 1945 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/bitrot.rs
+4 -4
View File
@@ -119,7 +119,7 @@ mod tests {
async fn test_create_bitrot_reader_with_inline_data() {
let test_data = b"hello world test data";
let shard_size = 16;
let checksum_algo = HashAlgorithm::HighwayHash256;
let checksum_algo = HashAlgorithm::HighwayHash256S;
let result =
create_bitrot_reader(Some(test_data), None, "test-bucket", "test-path", 0, 0, shard_size, checksum_algo, false).await;
@@ -131,7 +131,7 @@ mod tests {
#[tokio::test]
async fn test_create_bitrot_reader_without_data_or_disk() {
let shard_size = 16;
let checksum_algo = HashAlgorithm::HighwayHash256;
let checksum_algo = HashAlgorithm::HighwayHash256S;
let result =
create_bitrot_reader(None, None, "test-bucket", "test-path", 0, 1024, shard_size, checksum_algo, false).await;
@@ -151,7 +151,7 @@ mod tests {
"test-path",
1024, // length
1024, // shard_size
HashAlgorithm::HighwayHash256,
HashAlgorithm::HighwayHash256S,
)
.await;
@@ -183,7 +183,7 @@ mod tests {
"test-path",
1024, // length
1024, // shard_size
HashAlgorithm::HighwayHash256,
HashAlgorithm::HighwayHash256S,
)
.await;
crates/ecstore/src/bucket/bucket_target_sys.rs
+22 -31
View File
@@ -43,11 +43,13 @@ use rustfs_config::{DEFAULT_TRUST_LEAF_CERT_AS_CA, ENV_TRUST_LEAF_CERT_AS_CA, RU
use rustfs_filemeta::{ReplicationStatusType, ReplicationType};
use rustfs_utils::http::{
AMZ_BUCKET_REPLICATION_STATUS, AMZ_OBJECT_LOCK_BYPASS_GOVERNANCE, AMZ_OBJECT_LOCK_LEGAL_HOLD, AMZ_OBJECT_LOCK_MODE,
AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, AMZ_STORAGE_CLASS, AMZ_WEBSITE_REDIRECT_LOCATION, RUSTFS_BUCKET_REPLICATION_CHECK,
RUSTFS_BUCKET_REPLICATION_DELETE_MARKER, RUSTFS_BUCKET_REPLICATION_REQUEST, RUSTFS_BUCKET_SOURCE_ETAG,
RUSTFS_BUCKET_SOURCE_MTIME, RUSTFS_BUCKET_SOURCE_VERSION_ID, RUSTFS_FORCE_DELETE, is_amz_header, is_minio_header,
AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, AMZ_STORAGE_CLASS, AMZ_WEBSITE_REDIRECT_LOCATION, is_amz_header, is_minio_header,
is_rustfs_header, is_standard_header, is_storageclass_header,
};
use rustfs_utils::http::{
SUFFIX_FORCE_DELETE, SUFFIX_SOURCE_DELETEMARKER, SUFFIX_SOURCE_ETAG, SUFFIX_SOURCE_MTIME, SUFFIX_SOURCE_REPLICATION_CHECK,
SUFFIX_SOURCE_REPLICATION_REQUEST, SUFFIX_SOURCE_VERSION_ID, insert_header,
};
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::error::Error;
@@ -68,8 +70,6 @@ use uuid::Uuid;
const DEFAULT_HEALTH_CHECK_DURATION: Duration = Duration::from_secs(5);
const DEFAULT_HEALTH_CHECK_RELOAD_DURATION: Duration = Duration::from_secs(30 * 60);
const REPLICATION_REQUEST_TRUE: HeaderValue = HeaderValue::from_static("true");
pub static GLOBAL_BUCKET_TARGET_SYS: OnceLock<BucketTargetSys> = OnceLock::new();
#[derive(Debug, Clone)]
@@ -1081,23 +1081,21 @@ impl PutObjectOptions {
}
if !self.internal.source_version_id.is_empty() {
header.insert(
RUSTFS_BUCKET_SOURCE_VERSION_ID,
HeaderValue::from_str(&self.internal.source_version_id).expect("err"),
);
insert_header(&mut header, SUFFIX_SOURCE_VERSION_ID, &self.internal.source_version_id);
}
if self.internal.source_etag.is_empty() {
header.insert(RUSTFS_BUCKET_SOURCE_ETAG, HeaderValue::from_str(&self.internal.source_etag).expect("err"));
insert_header(&mut header, SUFFIX_SOURCE_ETAG, &self.internal.source_etag);
}
if self.internal.source_mtime.unix_timestamp() != 0 {
header.insert(
RUSTFS_BUCKET_SOURCE_MTIME,
HeaderValue::from_str(&self.internal.source_mtime.format(&Rfc3339).unwrap_or_default()).expect("err"),
insert_header(
&mut header,
SUFFIX_SOURCE_MTIME,
self.internal.source_mtime.format(&Rfc3339).unwrap_or_default(),
);
}
if self.internal.replication_request {
header.insert(RUSTFS_BUCKET_REPLICATION_REQUEST, REPLICATION_REQUEST_TRUE);
insert_header(&mut header, SUFFIX_SOURCE_REPLICATION_REQUEST, "true");
}
header
@@ -1266,10 +1264,8 @@ impl TargetClient {
let builder = self.client.put_object();
let version_id = opts.internal.source_version_id.clone();
if !version_id.is_empty()
&& let Ok(header_value) = HeaderValue::from_str(&version_id)
{
headers.insert(RUSTFS_BUCKET_SOURCE_VERSION_ID, header_value);
if !version_id.is_empty() {
insert_header(&mut headers, SUFFIX_SOURCE_VERSION_ID, &version_id);
}
match builder
@@ -1303,13 +1299,11 @@ impl TargetClient {
) -> Result<String, S3ClientError> {
let mut headers = HeaderMap::new();
let version_id = opts.internal.source_version_id.clone();
if !version_id.is_empty()
&& let Ok(header_value) = HeaderValue::from_str(&version_id)
{
headers.insert(RUSTFS_BUCKET_SOURCE_VERSION_ID, header_value);
if !version_id.is_empty() {
insert_header(&mut headers, SUFFIX_SOURCE_VERSION_ID, &version_id);
}
if opts.internal.replication_request {
headers.insert(RUSTFS_BUCKET_REPLICATION_REQUEST, REPLICATION_REQUEST_TRUE);
insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_REQUEST, "true");
}
match self
@@ -1418,21 +1412,18 @@ impl TargetClient {
) -> Result<(), S3ClientError> {
let mut headers = HeaderMap::new();
if opts.force_delete {
headers.insert(RUSTFS_FORCE_DELETE, "true".parse().unwrap());
insert_header(&mut headers, SUFFIX_FORCE_DELETE, "true");
}
if opts.governance_bypass {
headers.insert(AMZ_OBJECT_LOCK_BYPASS_GOVERNANCE, "true".parse().unwrap());
}
if opts.replication_delete_marker {
headers.insert(RUSTFS_BUCKET_REPLICATION_DELETE_MARKER, "true".parse().unwrap());
insert_header(&mut headers, SUFFIX_SOURCE_DELETEMARKER, "true");
}
if let Some(t) = opts.replication_mtime {
headers.insert(
RUSTFS_BUCKET_SOURCE_MTIME,
t.format(&Rfc3339).unwrap_or_default().as_str().parse().unwrap(),
);
insert_header(&mut headers, SUFFIX_SOURCE_MTIME, t.format(&Rfc3339).unwrap_or_default());
}
if !opts.replication_status.is_empty() {
@@ -1440,10 +1431,10 @@ impl TargetClient {
}
if opts.replication_request {
headers.insert(RUSTFS_BUCKET_REPLICATION_REQUEST, "true".parse().unwrap());
insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_REQUEST, "true");
}
if opts.replication_validity_check {
headers.insert(RUSTFS_BUCKET_REPLICATION_CHECK, "true".parse().unwrap());
insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_CHECK, "true");
}
match self
crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs
+14 -24
View File
@@ -45,8 +45,7 @@ use rustfs_common::heal_channel::rep_has_active_rules;
use rustfs_common::metrics::{IlmAction, Metrics};
use rustfs_filemeta::{NULL_VERSION_ID, RestoreStatusOps, is_restored_object_on_disk};
use rustfs_s3_common::EventName;
use rustfs_utils::path::encode_dir_object;
use rustfs_utils::string::strings_has_prefix_fold;
use rustfs_utils::{get_env_i64, get_env_usize, path::encode_dir_object, string::strings_has_prefix_fold};
use s3s::Body;
use s3s::dto::{
BucketLifecycleConfiguration, DefaultRetention, ReplicationConfiguration, RestoreRequest, RestoreRequestType, RestoreStatus,
@@ -97,8 +96,14 @@ impl LifecycleSys {
}
pub async fn get(&self, bucket: &str) -> Option<BucketLifecycleConfiguration> {
let lc = get_lifecycle_config(bucket).await.expect("get_lifecycle_config err!").0;
Some(lc)
match get_lifecycle_config(bucket).await {
Ok((lc, _)) => Some(lc),
Err(err) if err == Error::ConfigNotFound => None,
Err(err) => {
warn!(bucket, error = ?err, "failed to load lifecycle config");
None
}
}
}
pub fn trace(_oi: &ObjectInfo) -> TraceFn {
@@ -471,10 +476,7 @@ impl TransitionState {
}
pub async fn init(api: Arc<ECStore>) {
let max_workers = env::var("RUSTFS_MAX_TRANSITION_WORKERS")
.ok()
.and_then(|s| s.parse::<i64>().ok())
.unwrap_or_else(|| std::cmp::min(num_cpus::get() as i64, 16));
let max_workers = get_env_i64("RUSTFS_MAX_TRANSITION_WORKERS", std::cmp::min(num_cpus::get() as i64, 16));
let mut n = max_workers;
let tw = 8; //globalILMConfig.getTransitionWorkers();
if tw > 0 {
@@ -569,17 +571,11 @@ impl TransitionState {
pub async fn update_workers_inner(api: Arc<ECStore>, n: i64) {
let mut n = n;
if n == 0 {
let max_workers = env::var("RUSTFS_MAX_TRANSITION_WORKERS")
.ok()
.and_then(|s| s.parse::<i64>().ok())
.unwrap_or_else(|| std::cmp::min(num_cpus::get() as i64, 16));
let max_workers = get_env_i64("RUSTFS_MAX_TRANSITION_WORKERS", std::cmp::min(num_cpus::get() as i64, 16));
n = max_workers;
}
// Allow environment override of maximum workers
let absolute_max = env::var("RUSTFS_ABSOLUTE_MAX_WORKERS")
.ok()
.and_then(|s| s.parse::<i64>().ok())
.unwrap_or(32);
let absolute_max = get_env_i64("RUSTFS_ABSOLUTE_MAX_WORKERS", 32);
n = std::cmp::min(n, absolute_max);
let mut num_workers = GLOBAL_TransitionState.num_workers.load(Ordering::SeqCst);
@@ -603,10 +599,7 @@ impl TransitionState {
}
pub async fn init_background_expiry(api: Arc<ECStore>) {
let mut workers = env::var("RUSTFS_MAX_EXPIRY_WORKERS")
.ok()
.and_then(|s| s.parse::<usize>().ok())
.unwrap_or_else(|| std::cmp::min(num_cpus::get(), 16));
let mut workers = get_env_usize("RUSTFS_MAX_EXPIRY_WORKERS", std::cmp::min(num_cpus::get(), 16));
//globalILMConfig.getExpirationWorkers()
if let Ok(env_expiration_workers) = env::var("_RUSTFS_ILM_EXPIRATION_WORKERS") {
if let Ok(num_expirations) = env_expiration_workers.parse::<usize>() {
@@ -615,10 +608,7 @@ pub async fn init_background_expiry(api: Arc<ECStore>) {
}
if workers == 0 {
workers = env::var("RUSTFS_DEFAULT_EXPIRY_WORKERS")
.ok()
.and_then(|s| s.parse::<usize>().ok())
.unwrap_or(8);
workers = get_env_usize("RUSTFS_DEFAULT_EXPIRY_WORKERS", 8);
}
//let expiry_state = GLOBAL_ExpiryStSate.write().await;
crates/ecstore/src/bucket/lifecycle/lifecycle.rs
+128 -12
View File
@@ -49,6 +49,8 @@ const ERR_LIFECYCLE_INVALID_EXPIRATION_DAYS: &str = "Lifecycle expiration days m
const ERR_LIFECYCLE_INVALID_EXPIRATION_DATE_NOT_MIDNIGHT: &str = "Expiration.Date must be at midnight UTC";
const ERR_LIFECYCLE_INVALID_RULE_ID_TOO_LONG: &str = "Rule ID must be at most 255 characters";
const ERR_LIFECYCLE_INVALID_RULE_STATUS: &str = "Rule status must be either Enabled or Disabled";
const ERR_LIFECYCLE_DEL_MARKER_WITH_TAGS: &str = "Rule with DelMarkerExpiration cannot have tags based filtering";
const ERR_LIFECYCLE_RULE_MUST_HAVE_ACTION: &str = "Rule must have at least one of Expiration, Transition, NoncurrentVersionExpiration, NoncurrentVersionTransition, or DelMarkerExpiration";
pub use rustfs_common::metrics::IlmAction;
@@ -117,19 +119,43 @@ impl RuleValidate for LifecycleRule {
}*/
fn validate(&self) -> Result<(), std::io::Error> {
/*self.validate_id()?;
self.validate_status()?;
self.validate_expiration()?;
self.validate_noncurrent_expiration()?;
self.validate_prefix_and_filter()?;
self.validate_transition()?;
self.validate_noncurrent_transition()?;
if (!self.Filter.Tag.IsEmpty() || len(self.Filter.And.Tags) != 0) && !self.delmarker_expiration.Empty() {
return errInvalidRuleDelMarkerExpiration
// Rule with DelMarkerExpiration cannot have tags based filtering
let has_tag_filter = self
.filter
.as_ref()
.map_or(false, |f| f.tag.is_some() || f.and.as_ref().and_then(|a| a.tags.as_ref()).is_some());
if has_tag_filter && self.del_marker_expiration.is_some() {
return Err(std::io::Error::other(ERR_LIFECYCLE_DEL_MARKER_WITH_TAGS));
}
// Rule must have at least one action
let has_expiration = self.expiration.is_some();
let has_transition = self.transitions.as_ref().map_or(false, |t| !t.is_empty());
let has_noncurrent_expiration = self
.noncurrent_version_expiration
.as_ref()
.and_then(|e| e.noncurrent_days)
.map_or(false, |d| d != 0);
let has_noncurrent_transition = self
.noncurrent_version_transitions
.as_ref()
.and_then(|t| t.first())
.and_then(|t| t.storage_class.as_ref())
.is_some();
let has_abort_incomplete_multipart_upload = self.abort_incomplete_multipart_upload.is_some();
let has_del_marker_expiration = self
.del_marker_expiration
.as_ref()
.and_then(|d| d.days)
.map_or(false, |d| d > 0);
if !has_expiration
&& !has_transition
&& !has_noncurrent_expiration
&& !has_noncurrent_transition
&& !has_abort_incomplete_multipart_upload
&& !has_del_marker_expiration
{
return Err(std::io::Error::other(ERR_LIFECYCLE_RULE_MUST_HAVE_ACTION));
}
if !self.expiration.set && !self.transition.set && !self.noncurrent_version_expiration.set && !self.noncurrent_version_transitions.unwrap()[0].set && self.delmarker_expiration.Empty() {
return errXMLNotWellFormed
}*/
Ok(())
}
}
@@ -456,6 +482,27 @@ impl Lifecycle for BucketLifecycleConfiguration {
}
}
}
// DelMarkerExpiration: expire delete marker after N days from mod_time
if obj.delete_marker {
if let Some(ref dme) = rule.del_marker_expiration {
if let Some(days) = dme.days {
if days > 0 {
let due = expected_expiry_time(mod_time, days);
if now.unix_timestamp() >= due.unix_timestamp() {
events.push(Event {
action: IlmAction::DelMarkerDeleteAllVersionsAction,
rule_id: rule.id.clone().unwrap_or_default(),
due: Some(due),
noncurrent_days: 0,
newer_noncurrent_versions: 0,
storage_class: "".into(),
});
}
continue;
}
}
}
}
}
if !obj.is_latest {
@@ -866,6 +913,7 @@ mod tests {
#[serial]
async fn validate_rejects_non_positive_expiration_days() {
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
expiration: Some(LifecycleExpiration {
@@ -873,6 +921,7 @@ mod tests {
..Default::default()
}),
abort_incomplete_multipart_upload: None,
del_marker_expiration: None,
filter: None,
id: None,
noncurrent_version_expiration: None,
@@ -894,6 +943,7 @@ mod tests {
#[serial]
async fn validate_accepts_positive_expiration_days() {
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
expiration: Some(LifecycleExpiration {
@@ -901,6 +951,7 @@ mod tests {
..Default::default()
}),
abort_incomplete_multipart_upload: None,
del_marker_expiration: None,
filter: None,
id: None,
noncurrent_version_expiration: None,
@@ -915,10 +966,37 @@ mod tests {
.expect("expected validation to pass");
}
#[tokio::test]
#[serial]
async fn validate_accepts_abort_incomplete_multipart_upload_only_rule() {
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
expiration: None,
abort_incomplete_multipart_upload: Some(s3s::dto::AbortIncompleteMultipartUpload {
days_after_initiation: Some(2),
}),
del_marker_expiration: None,
filter: None,
id: Some("abort-only".to_string()),
noncurrent_version_expiration: None,
noncurrent_version_transitions: None,
prefix: Some("test/".to_string()),
transitions: None,
}],
};
lc.validate(&ObjectLockConfiguration::default())
.await
.expect("expected validation to pass");
}
#[tokio::test]
#[serial]
async fn validate_rejects_non_midnight_expiration_date() {
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
expiration: Some(LifecycleExpiration {
@@ -926,6 +1004,7 @@ mod tests {
..Default::default()
}),
abort_incomplete_multipart_upload: None,
del_marker_expiration: None,
filter: None,
id: None,
noncurrent_version_expiration: None,
@@ -945,6 +1024,7 @@ mod tests {
async fn predict_expiration_selects_closest_expiry_for_put_object() {
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![
LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
@@ -953,6 +1033,7 @@ mod tests {
..Default::default()
}),
abort_incomplete_multipart_upload: None,
del_marker_expiration: None,
filter: None,
id: Some("rule-days".to_string()),
noncurrent_version_expiration: None,
@@ -967,6 +1048,7 @@ mod tests {
..Default::default()
}),
abort_incomplete_multipart_upload: None,
del_marker_expiration: None,
filter: None,
id: Some("rule-date".to_string()),
noncurrent_version_expiration: None,
@@ -996,6 +1078,7 @@ mod tests {
#[serial]
async fn validate_accepts_multiple_rules_without_ids() {
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![
LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
@@ -1004,6 +1087,7 @@ mod tests {
..Default::default()
}),
abort_incomplete_multipart_upload: None,
del_marker_expiration: None,
filter: None,
id: None,
noncurrent_version_expiration: None,
@@ -1018,6 +1102,7 @@ mod tests {
..Default::default()
}),
abort_incomplete_multipart_upload: None,
del_marker_expiration: None,
filter: None,
id: None,
noncurrent_version_expiration: None,
@@ -1037,6 +1122,7 @@ mod tests {
#[serial]
async fn validate_rejects_rule_id_too_long() {
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
expiration: Some(LifecycleExpiration {
@@ -1044,6 +1130,7 @@ mod tests {
..Default::default()
}),
abort_incomplete_multipart_upload: None,
del_marker_expiration: None,
filter: None,
id: Some("a".repeat(256)),
noncurrent_version_expiration: None,
@@ -1062,6 +1149,7 @@ mod tests {
#[serial]
async fn validate_rejects_duplicate_rule_ids() {
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![
LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
@@ -1070,6 +1158,7 @@ mod tests {
..Default::default()
}),
abort_incomplete_multipart_upload: None,
del_marker_expiration: None,
filter: None,
id: Some("dup-rule".to_string()),
noncurrent_version_expiration: None,
@@ -1090,6 +1179,7 @@ mod tests {
noncurrent_version_transitions: None,
prefix: None,
transitions: None,
del_marker_expiration: None,
},
],
};
@@ -1103,6 +1193,7 @@ mod tests {
async fn eval_inner_expires_latest_object_after_days_due() {
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
expiration: Some(LifecycleExpiration {
@@ -1110,6 +1201,7 @@ mod tests {
..Default::default()
}),
abort_incomplete_multipart_upload: None,
del_marker_expiration: None,
filter: None,
id: Some("expire-days".to_string()),
noncurrent_version_expiration: None,
@@ -1137,6 +1229,7 @@ mod tests {
async fn eval_inner_keeps_latest_object_before_days_due() {
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
expiration: Some(LifecycleExpiration {
@@ -1144,6 +1237,7 @@ mod tests {
..Default::default()
}),
abort_incomplete_multipart_upload: None,
del_marker_expiration: None,
filter: None,
id: Some("expire-days".to_string()),
noncurrent_version_expiration: None,
@@ -1169,10 +1263,12 @@ mod tests {
async fn eval_inner_transitions_latest_object_after_days_due() {
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
expiration: None,
abort_incomplete_multipart_upload: None,
del_marker_expiration: None,
filter: None,
id: Some("transition-days".to_string()),
noncurrent_version_expiration: None,
@@ -1205,10 +1301,12 @@ mod tests {
async fn eval_inner_expires_noncurrent_version_after_due() {
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
expiration: None,
abort_incomplete_multipart_upload: None,
del_marker_expiration: None,
filter: None,
id: Some("noncurrent-expire".to_string()),
noncurrent_version_expiration: Some(s3s::dto::NoncurrentVersionExpiration {
@@ -1241,10 +1339,12 @@ mod tests {
async fn eval_inner_transitions_noncurrent_version_after_due() {
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
expiration: None,
abort_incomplete_multipart_upload: None,
del_marker_expiration: None,
filter: None,
id: Some("noncurrent-transition".to_string()),
noncurrent_version_expiration: None,
@@ -1278,10 +1378,12 @@ mod tests {
#[serial]
async fn noncurrent_versions_expiration_limit_returns_configured_limits() {
let lc = Arc::new(BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
expiration: None,
abort_incomplete_multipart_upload: None,
del_marker_expiration: None,
filter: None,
id: Some("noncurrent-limit".to_string()),
noncurrent_version_expiration: Some(s3s::dto::NoncurrentVersionExpiration {
@@ -1313,6 +1415,7 @@ mod tests {
#[serial]
async fn validate_rejects_invalid_status_case_sensitive() {
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![LifecycleRule {
status: ExpirationStatus::from_static("enabled"),
expiration: Some(LifecycleExpiration {
@@ -1320,6 +1423,7 @@ mod tests {
..Default::default()
}),
abort_incomplete_multipart_upload: None,
del_marker_expiration: None,
filter: None,
id: None,
noncurrent_version_expiration: None,
@@ -1340,6 +1444,7 @@ mod tests {
let mut filter = LifecycleRuleFilter::default();
filter.prefix = Some("prefix".to_string());
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
expiration: Some(LifecycleExpiration {
@@ -1353,6 +1458,7 @@ mod tests {
noncurrent_version_transitions: None,
prefix: None,
transitions: None,
del_marker_expiration: None,
}],
};
@@ -1385,6 +1491,7 @@ mod tests {
filter.and = Some(and);
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
expiration: Some(LifecycleExpiration {
@@ -1398,6 +1505,7 @@ mod tests {
noncurrent_version_transitions: None,
prefix: None,
transitions: None,
del_marker_expiration: None,
}],
};
@@ -1425,6 +1533,7 @@ mod tests {
async fn expired_object_delete_marker_requires_single_version() {
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
expiration: Some(LifecycleExpiration {
@@ -1439,6 +1548,7 @@ mod tests {
noncurrent_version_transitions: None,
prefix: None,
transitions: None,
del_marker_expiration: None,
}],
};
@@ -1462,6 +1572,7 @@ mod tests {
async fn expired_object_delete_marker_deletes_only_delete_marker_after_due() {
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
expiration: Some(LifecycleExpiration {
@@ -1476,6 +1587,7 @@ mod tests {
noncurrent_version_transitions: None,
prefix: None,
transitions: None,
del_marker_expiration: None,
}],
};
@@ -1500,6 +1612,7 @@ mod tests {
async fn expired_object_delete_marker_without_date_or_days_deletes_immediately() {
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
expiration: Some(LifecycleExpiration {
@@ -1513,6 +1626,7 @@ mod tests {
noncurrent_version_transitions: None,
prefix: None,
transitions: None,
del_marker_expiration: None,
}],
};
@@ -1539,6 +1653,7 @@ mod tests {
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
let future_date = base_time + Duration::days(10);
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
expiration: Some(LifecycleExpiration {
@@ -1553,6 +1668,7 @@ mod tests {
noncurrent_version_transitions: None,
prefix: None,
transitions: None,
del_marker_expiration: None,
}],
};
crates/ecstore/src/bucket/metadata.rs
+218 -56
View File
@@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use super::msgp_decode::{read_msgp_ext8_time, skip_msgp_value, write_msgp_time};
use super::object_lock::ObjectLockApi;
use super::versioning::VersioningApi;
use super::{quota::BucketQuota, target::BucketTargets};
@@ -22,7 +23,6 @@ use crate::error::{Error, Result};
use crate::new_object_layer_fn;
use crate::store::ECStore;
use byteorder::{BigEndian, ByteOrder, LittleEndian};
use rmp_serde::Serializer as rmpSerializer;
use rustfs_policy::policy::BucketPolicy;
use s3s::dto::{
BucketLifecycleConfiguration, CORSConfiguration, NotificationConfiguration, ObjectLockConfiguration,
@@ -30,12 +30,41 @@ use s3s::dto::{
VersioningConfiguration,
};
use serde::Serializer;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::io::{Read, Write};
use std::sync::Arc;
use time::OffsetDateTime;
use tracing::error;
fn read_msgp_str<R: Read>(rd: &mut R) -> Result<String> {
let len = rmp::decode::read_str_len(rd)? as usize;
let mut buf = vec![0u8; len];
rd.read_exact(&mut buf)?;
Ok(String::from_utf8(buf)?)
}
fn read_msgp_time_value<R: Read>(rd: &mut R) -> Result<OffsetDateTime> {
let marker = rmp::decode::read_marker(rd).map_err(|e| Error::other(format!("{e:?}")))?;
match marker {
rmp::Marker::Null => Ok(OffsetDateTime::UNIX_EPOCH),
rmp::Marker::Ext8 => read_msgp_ext8_time(rd),
_ => Err(Error::other(format!("expected time ext or nil, got marker: {marker:?}"))),
}
}
fn read_msgp_bin<R: Read>(rd: &mut R) -> Result<Vec<u8>> {
let len = rmp::decode::read_bin_len(rd)? as usize;
let mut buf = vec![0u8; len];
rd.read_exact(&mut buf)?;
Ok(buf)
}
fn write_bin_field<W: Write>(wr: &mut W, key: &str, val: &[u8]) -> Result<()> {
rmp::encode::write_str(wr, key)?;
rmp::encode::write_bin(wr, val)?;
Ok(())
}
pub const BUCKET_METADATA_FILE: &str = ".metadata.bin";
pub const BUCKET_METADATA_FORMAT: u16 = 1;
pub const BUCKET_METADATA_VERSION: u16 = 1;
@@ -54,8 +83,7 @@ pub const BUCKET_CORS_CONFIG: &str = "cors.xml";
pub const BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG: &str = "public-access-block.xml";
pub const BUCKET_ACL_CONFIG: &str = "bucket-acl.json";
#[derive(Debug, Deserialize, Serialize, Clone)]
#[serde(rename_all = "PascalCase", default)]
#[derive(Debug, Clone)]
pub struct BucketMetadata {
pub name: String,
pub created: OffsetDateTime,
@@ -90,36 +118,21 @@ pub struct BucketMetadata {
pub public_access_block_config_updated_at: OffsetDateTime,
pub bucket_acl_config_updated_at: OffsetDateTime,
#[serde(skip)]
pub new_field_updated_at: OffsetDateTime,
#[serde(skip)]
pub policy_config: Option<BucketPolicy>,
#[serde(skip)]
pub notification_config: Option<NotificationConfiguration>,
#[serde(skip)]
pub lifecycle_config: Option<BucketLifecycleConfiguration>,
#[serde(skip)]
pub object_lock_config: Option<ObjectLockConfiguration>,
#[serde(skip)]
pub versioning_config: Option<VersioningConfiguration>,
#[serde(skip)]
pub sse_config: Option<ServerSideEncryptionConfiguration>,
#[serde(skip)]
pub tagging_config: Option<Tagging>,
#[serde(skip)]
pub quota_config: Option<BucketQuota>,
#[serde(skip)]
pub replication_config: Option<ReplicationConfiguration>,
#[serde(skip)]
pub bucket_target_config: Option<BucketTargets>,
#[serde(skip)]
pub bucket_target_config_meta: Option<HashMap<String, String>>,
#[serde(skip)]
pub cors_config: Option<CORSConfiguration>,
#[serde(skip)]
pub public_access_block_config: Option<PublicAccessBlockConfiguration>,
#[serde(skip)]
pub bucket_acl_config: Option<String>,
}
@@ -198,17 +211,137 @@ impl BucketMetadata {
self.lock_enabled || (self.versioning_config.as_ref().is_some_and(|v| v.enabled()))
}
/// Decode from msgp bytes. Field order follows MinIO BucketMetadata for compatibility.
pub fn decode_from<R: Read>(&mut self, rd: &mut R) -> Result<()> {
let mut fields = rmp::decode::read_map_len(rd)?;
*self = Self::default();
while fields > 0 {
fields -= 1;
let key_len = rmp::decode::read_str_len(rd)?;
let mut key_buf = vec![0u8; key_len as usize];
rd.read_exact(&mut key_buf)?;
let key = String::from_utf8(key_buf)?;
match key.as_str() {
"Name" => self.name = read_msgp_str(rd)?,
"Created" => self.created = read_msgp_time_value(rd)?,
"LockEnabled" => self.lock_enabled = rmp::decode::read_bool(rd)?,
"PolicyConfigJSON" => self.policy_config_json = read_msgp_bin(rd)?,
"NotificationConfigXML" => self.notification_config_xml = read_msgp_bin(rd)?,
"LifecycleConfigXML" => self.lifecycle_config_xml = read_msgp_bin(rd)?,
"ObjectLockConfigXML" => self.object_lock_config_xml = read_msgp_bin(rd)?,
"VersioningConfigXML" => self.versioning_config_xml = read_msgp_bin(rd)?,
"EncryptionConfigXML" => self.encryption_config_xml = read_msgp_bin(rd)?,
"TaggingConfigXML" => self.tagging_config_xml = read_msgp_bin(rd)?,
"QuotaConfigJSON" => self.quota_config_json = read_msgp_bin(rd)?,
"ReplicationConfigXML" => self.replication_config_xml = read_msgp_bin(rd)?,
"BucketTargetsConfigJSON" => self.bucket_targets_config_json = read_msgp_bin(rd)?,
"BucketTargetsConfigMetaJSON" => self.bucket_targets_config_meta_json = read_msgp_bin(rd)?,
"PolicyConfigUpdatedAt" => self.policy_config_updated_at = read_msgp_time_value(rd)?,
"ObjectLockConfigUpdatedAt" => self.object_lock_config_updated_at = read_msgp_time_value(rd)?,
"EncryptionConfigUpdatedAt" => self.encryption_config_updated_at = read_msgp_time_value(rd)?,
"TaggingConfigUpdatedAt" => self.tagging_config_updated_at = read_msgp_time_value(rd)?,
"QuotaConfigUpdatedAt" => self.quota_config_updated_at = read_msgp_time_value(rd)?,
"ReplicationConfigUpdatedAt" => self.replication_config_updated_at = read_msgp_time_value(rd)?,
"VersioningConfigUpdatedAt" => self.versioning_config_updated_at = read_msgp_time_value(rd)?,
"LifecycleConfigUpdatedAt" => self.lifecycle_config_updated_at = read_msgp_time_value(rd)?,
"NotificationConfigUpdatedAt" => self.notification_config_updated_at = read_msgp_time_value(rd)?,
"BucketTargetsConfigUpdatedAt" => self.bucket_targets_config_updated_at = read_msgp_time_value(rd)?,
"BucketTargetsConfigMetaUpdatedAt" => self.bucket_targets_config_meta_updated_at = read_msgp_time_value(rd)?,
"CorsConfigXML" => self.cors_config_xml = read_msgp_bin(rd)?,
"PublicAccessBlockConfigXML" => self.public_access_block_config_xml = read_msgp_bin(rd)?,
"BucketAclConfigJSON" => self.bucket_acl_config_json = read_msgp_bin(rd)?,
"CorsConfigUpdatedAt" => self.cors_config_updated_at = read_msgp_time_value(rd)?,
"PublicAccessBlockConfigUpdatedAt" => self.public_access_block_config_updated_at = read_msgp_time_value(rd)?,
"BucketAclConfigUpdatedAt" => self.bucket_acl_config_updated_at = read_msgp_time_value(rd)?,
other => {
tracing::debug!(field = %other, "BucketMetadata decode_from: skipping unknown field");
skip_msgp_value(rd)?;
}
}
}
Ok(())
}
/// Encode to msgp bytes. Field order follows MinIO BucketMetadata for compatibility.
pub fn encode_to<W: Write>(&self, wr: &mut W) -> Result<()> {
// Map size: MinIO fields (25) + RustFS extensions (6)
let map_len: u32 = 31;
rmp::encode::write_map_len(wr, map_len)?;
// MinIO field order (same as Go struct)
rmp::encode::write_str(wr, "Name")?;
rmp::encode::write_str(wr, &self.name)?;
rmp::encode::write_str(wr, "Created")?;
write_msgp_time(wr, self.created)?;
rmp::encode::write_str(wr, "LockEnabled")?;
rmp::encode::write_bool(wr, self.lock_enabled)?;
write_bin_field(wr, "PolicyConfigJSON", &self.policy_config_json)?;
write_bin_field(wr, "NotificationConfigXML", &self.notification_config_xml)?;
write_bin_field(wr, "LifecycleConfigXML", &self.lifecycle_config_xml)?;
write_bin_field(wr, "ObjectLockConfigXML", &self.object_lock_config_xml)?;
write_bin_field(wr, "VersioningConfigXML", &self.versioning_config_xml)?;
write_bin_field(wr, "EncryptionConfigXML", &self.encryption_config_xml)?;
write_bin_field(wr, "TaggingConfigXML", &self.tagging_config_xml)?;
write_bin_field(wr, "QuotaConfigJSON", &self.quota_config_json)?;
write_bin_field(wr, "ReplicationConfigXML", &self.replication_config_xml)?;
write_bin_field(wr, "BucketTargetsConfigJSON", &self.bucket_targets_config_json)?;
write_bin_field(wr, "BucketTargetsConfigMetaJSON", &self.bucket_targets_config_meta_json)?;
rmp::encode::write_str(wr, "PolicyConfigUpdatedAt")?;
write_msgp_time(wr, self.policy_config_updated_at)?;
rmp::encode::write_str(wr, "ObjectLockConfigUpdatedAt")?;
write_msgp_time(wr, self.object_lock_config_updated_at)?;
rmp::encode::write_str(wr, "EncryptionConfigUpdatedAt")?;
write_msgp_time(wr, self.encryption_config_updated_at)?;
rmp::encode::write_str(wr, "TaggingConfigUpdatedAt")?;
write_msgp_time(wr, self.tagging_config_updated_at)?;
rmp::encode::write_str(wr, "QuotaConfigUpdatedAt")?;
write_msgp_time(wr, self.quota_config_updated_at)?;
rmp::encode::write_str(wr, "ReplicationConfigUpdatedAt")?;
write_msgp_time(wr, self.replication_config_updated_at)?;
rmp::encode::write_str(wr, "VersioningConfigUpdatedAt")?;
write_msgp_time(wr, self.versioning_config_updated_at)?;
rmp::encode::write_str(wr, "LifecycleConfigUpdatedAt")?;
write_msgp_time(wr, self.lifecycle_config_updated_at)?;
rmp::encode::write_str(wr, "NotificationConfigUpdatedAt")?;
write_msgp_time(wr, self.notification_config_updated_at)?;
rmp::encode::write_str(wr, "BucketTargetsConfigUpdatedAt")?;
write_msgp_time(wr, self.bucket_targets_config_updated_at)?;
rmp::encode::write_str(wr, "BucketTargetsConfigMetaUpdatedAt")?;
write_msgp_time(wr, self.bucket_targets_config_meta_updated_at)?;
// RustFS extensions
write_bin_field(wr, "CorsConfigXML", &self.cors_config_xml)?;
write_bin_field(wr, "PublicAccessBlockConfigXML", &self.public_access_block_config_xml)?;
write_bin_field(wr, "BucketAclConfigJSON", &self.bucket_acl_config_json)?;
rmp::encode::write_str(wr, "CorsConfigUpdatedAt")?;
write_msgp_time(wr, self.cors_config_updated_at)?;
rmp::encode::write_str(wr, "PublicAccessBlockConfigUpdatedAt")?;
write_msgp_time(wr, self.public_access_block_config_updated_at)?;
rmp::encode::write_str(wr, "BucketAclConfigUpdatedAt")?;
write_msgp_time(wr, self.bucket_acl_config_updated_at)?;
Ok(())
}
pub fn marshal_msg(&self) -> Result<Vec<u8>> {
let mut buf = Vec::new();
self.serialize(&mut rmpSerializer::new(&mut buf).with_struct_map())?;
self.encode_to(&mut buf)?;
Ok(buf)
}
pub fn unmarshal(buf: &[u8]) -> Result<Self> {
let t: BucketMetadata = rmp_serde::from_slice(buf)?;
Ok(t)
let mut bm = Self::default();
let mut cur = std::io::Cursor::new(buf);
bm.decode_from(&mut cur)?;
Ok(bm)
}
pub fn check_header(buf: &[u8]) -> Result<()> {
@@ -382,50 +515,80 @@ impl BucketMetadata {
}
fn parse_all_configs(&mut self, _api: Arc<ECStore>) -> Result<()> {
self.parse_policy_config()?;
if !self.notification_config_xml.is_empty() {
self.notification_config = Some(deserialize::<NotificationConfiguration>(&self.notification_config_xml)?);
if let Err(e) = self.parse_policy_config() {
tracing::warn!(bucket = %self.name, config = "policy", error = %e, "parse_all_configs: failed to parse");
}
if !self.lifecycle_config_xml.is_empty() {
self.lifecycle_config = Some(deserialize::<BucketLifecycleConfiguration>(&self.lifecycle_config_xml)?);
if !self.notification_config_xml.is_empty()
&& let Err(e) = deserialize::<NotificationConfiguration>(&self.notification_config_xml)
.map(|c| self.notification_config = Some(c))
{
tracing::warn!(bucket = %self.name, config = "notification", error = %e, "parse_all_configs: failed to parse");
}
if !self.object_lock_config_xml.is_empty() {
self.object_lock_config = Some(deserialize::<ObjectLockConfiguration>(&self.object_lock_config_xml)?);
if !self.lifecycle_config_xml.is_empty()
&& let Err(e) =
deserialize::<BucketLifecycleConfiguration>(&self.lifecycle_config_xml).map(|c| self.lifecycle_config = Some(c))
{
tracing::warn!(bucket = %self.name, config = "lifecycle", error = %e, "parse_all_configs: failed to parse");
}
if !self.versioning_config_xml.is_empty() {
self.versioning_config = Some(deserialize::<VersioningConfiguration>(&self.versioning_config_xml)?);
if !self.object_lock_config_xml.is_empty()
&& let Err(e) =
deserialize::<ObjectLockConfiguration>(&self.object_lock_config_xml).map(|c| self.object_lock_config = Some(c))
{
tracing::warn!(bucket = %self.name, config = "object_lock", error = %e, "parse_all_configs: failed to parse");
}
if !self.encryption_config_xml.is_empty() {
self.sse_config = Some(deserialize::<ServerSideEncryptionConfiguration>(&self.encryption_config_xml)?);
if !self.versioning_config_xml.is_empty()
&& let Err(e) =
deserialize::<VersioningConfiguration>(&self.versioning_config_xml).map(|c| self.versioning_config = Some(c))
{
tracing::warn!(bucket = %self.name, config = "versioning", error = %e, "parse_all_configs: failed to parse");
}
if !self.tagging_config_xml.is_empty() {
self.tagging_config = Some(deserialize::<Tagging>(&self.tagging_config_xml)?);
if !self.encryption_config_xml.is_empty()
&& let Err(e) =
deserialize::<ServerSideEncryptionConfiguration>(&self.encryption_config_xml).map(|c| self.sse_config = Some(c))
{
tracing::warn!(bucket = %self.name, config = "encryption", error = %e, "parse_all_configs: failed to parse");
}
if !self.quota_config_json.is_empty() {
self.quota_config = Some(serde_json::from_slice(&self.quota_config_json)?);
if !self.tagging_config_xml.is_empty()
&& let Err(e) = deserialize::<Tagging>(&self.tagging_config_xml).map(|c| self.tagging_config = Some(c))
{
tracing::warn!(bucket = %self.name, config = "tagging", error = %e, "parse_all_configs: failed to parse");
}
if !self.replication_config_xml.is_empty() {
self.replication_config = Some(deserialize::<ReplicationConfiguration>(&self.replication_config_xml)?);
if !self.quota_config_json.is_empty()
&& let Err(e) = serde_json::from_slice(&self.quota_config_json).map(|c| self.quota_config = Some(c))
{
tracing::warn!(bucket = %self.name, config = "quota", error = %e, "parse_all_configs: failed to parse");
}
if !self.replication_config_xml.is_empty()
&& let Err(e) =
deserialize::<ReplicationConfiguration>(&self.replication_config_xml).map(|c| self.replication_config = Some(c))
{
tracing::warn!(bucket = %self.name, config = "replication", error = %e, "parse_all_configs: failed to parse");
}
//let temp = self.bucket_targets_config_json.clone();
if !self.bucket_targets_config_json.is_empty() {
let bucket_targets: BucketTargets = serde_json::from_slice(&self.bucket_targets_config_json)?;
self.bucket_target_config = Some(bucket_targets);
if let Err(e) = serde_json::from_slice::<BucketTargets>(&self.bucket_targets_config_json)
.map(|t| self.bucket_target_config = Some(t))
{
tracing::warn!(bucket = %self.name, config = "bucket_targets", error = %e, "parse_all_configs: failed to parse");
self.bucket_target_config = Some(BucketTargets::default());
}
} else {
self.bucket_target_config = Some(BucketTargets::default())
self.bucket_target_config = Some(BucketTargets::default());
}
if !self.cors_config_xml.is_empty() {
self.cors_config = Some(deserialize::<CORSConfiguration>(&self.cors_config_xml)?);
if !self.cors_config_xml.is_empty()
&& let Err(e) = deserialize::<CORSConfiguration>(&self.cors_config_xml).map(|c| self.cors_config = Some(c))
{
tracing::warn!(bucket = %self.name, config = "cors", error = %e, "parse_all_configs: failed to parse");
}
if !self.public_access_block_config_xml.is_empty() {
self.public_access_block_config =
Some(deserialize::<PublicAccessBlockConfiguration>(&self.public_access_block_config_xml)?);
if !self.public_access_block_config_xml.is_empty()
&& let Err(e) = deserialize::<PublicAccessBlockConfiguration>(&self.public_access_block_config_xml)
.map(|c| self.public_access_block_config = Some(c))
{
tracing::warn!(bucket = %self.name, config = "public_access_block", error = %e, "parse_all_configs: failed to parse");
}
if !self.bucket_acl_config_json.is_empty() {
let acl = String::from_utf8(self.bucket_acl_config_json.clone())
.map_err(|e| Error::other(format!("invalid UTF-8 in bucket ACL: {}", e)))?;
self.bucket_acl_config = Some(acl);
if !self.bucket_acl_config_json.is_empty()
&& let Err(e) = String::from_utf8(self.bucket_acl_config_json.clone()).map(|acl| self.bucket_acl_config = Some(acl))
{
tracing::warn!(bucket = %self.name, config = "bucket_acl", error = %e, "parse_all_configs: failed to parse");
}
Ok(())
@@ -478,7 +641,6 @@ async fn read_bucket_metadata(api: Arc<ECStore>, bucket: &str) -> Result<BucketM
Ok(bm)
}
fn _write_time<S>(t: &OffsetDateTime, s: S) -> std::result::Result<S::Ok, S::Error>
where
S: Serializer,
crates/ecstore/src/bucket/metadata_sys.rs
+8 -6
View File
@@ -360,12 +360,14 @@ impl BucketMetadataSys {
};
if !meta.lifecycle_config_xml.is_empty() {
let cfg = deserialize::<BucketLifecycleConfiguration>(&meta.lifecycle_config_xml)?;
// TODO: FIXME:
// for _v in cfg.rules.iter() {
// break;
// }
if let Some(_v) = cfg.rules.first() {}
if let Ok(cfg) = deserialize::<BucketLifecycleConfiguration>(&meta.lifecycle_config_xml) {
if let Some(_v) = cfg.rules.first() {}
} else {
tracing::warn!(
bucket = %bucket,
"delete: failed to parse lifecycle config XML"
);
}
}
// TODO: other lifecycle handle
crates/ecstore/src/bucket/metadata_test.rs
+261
View File
@@ -0,0 +1,261 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::metadata::BucketMetadata;
use time::OffsetDateTime;
/// Full BucketMetadata hex (all fields populated).
const TEST_BUCKET_METADATA_HEX: &str = "de0019a44e616d65b27275737466732d636f6d7061742d74657374a743726561746564c70c050000000065920080075bcd15ab4c6f636b456e61626c6564c3b0506f6c696379436f6e6669674a534f4ec4907b2256657273696f6e223a22323031322d31302d3137222c2253746174656d656e74223a5b7b22456666656374223a22416c6c6f77222c225072696e636970616c223a222a222c22416374696f6e223a2273333a4765744f626a656374222c225265736f75726365223a2261726e3a6177733a73333a3a3a7275737466732d636f6d7061742d746573742f2a227d5d7db54e6f74696669636174696f6e436f6e666967584d4cc4963c4e6f74696669636174696f6e436f6e66696775726174696f6e3e3c436c6f75645761746368436f6e66696775726174696f6e3e3c49643e6e313c2f49643e3c4576656e743e73333a4f626a656374437265617465643a2a3c2f4576656e743e3c2f436c6f75645761746368436f6e66696775726174696f6e3e3c2f4e6f74696669636174696f6e436f6e66696775726174696f6e3eb24c6966656379636c65436f6e666967584d4cc48c3c4c6966656379636c65436f6e66696775726174696f6e3e3c52756c653e3c49443e72756c65313c2f49443e3c5374617475733e456e61626c65643c2f5374617475733e3c45787069726174696f6e3e3c446179733e33303c2f446179733e3c2f45787069726174696f6e3e3c2f52756c653e3c2f4c6966656379636c65436f6e66696775726174696f6e3eb34f626a6563744c6f636b436f6e666967584d4cc4b83c4f626a6563744c6f636b436f6e66696775726174696f6e3e3c4f626a6563744c6f636b456e61626c65643e456e61626c65643c2f4f626a6563744c6f636b456e61626c65643e3c52756c653e3c44656661756c74526574656e74696f6e3e3c4d6f64653e474f5645524e414e43453c2f4d6f64653e3c446179733e373c2f446179733e3c2f44656661756c74526574656e74696f6e3e3c2f52756c653e3c2f4f626a6563744c6f636b436f6e66696775726174696f6e3eb356657273696f6e696e67436f6e666967584d4cc44b3c56657273696f6e696e67436f6e66696775726174696f6e3e3c5374617475733e456e61626c65643c2f5374617475733e3c2f56657273696f6e696e67436f6e66696775726174696f6e3eb3456e6372797074696f6e436f6e666967584d4cc4c03c53657276657253696465456e6372797074696f6e436f6e66696775726174696f6e3e3c52756c653e3c4170706c7953657276657253696465456e6372797074696f6e427944656661756c743e3c535345416c676f726974686d3e4145533235363c2f535345416c676f726974686d3e3c2f4170706c7953657276657253696465456e6372797074696f6e427944656661756c743e3c2f52756c653e3c2f53657276657253696465456e6372797074696f6e436f6e66696775726174696f6e3eb054616767696e67436f6e666967584d4cc4503c54616767696e673e3c5461675365743e3c5461673e3c4b65793e456e763c2f4b65793e3c56616c75653e546573743c2f56616c75653e3c2f5461673e3c2f5461675365743e3c2f54616767696e673eaf51756f7461436f6e6669674a534f4ec4707b2271756f7461223a313037333734313832342c2271756f74615f74797065223a2248617264222c22637265617465645f6174223a22323032342d30312d30315430303a30303a30305a222c22757064617465645f6174223a22323032342d30312d30315430303a30303a30305a227db45265706c69636174696f6e436f6e666967584d4cc4e73c5265706c69636174696f6e436f6e66696775726174696f6e3e3c526f6c653e61726e3a6177733a69616d3a3a3132333435363738393031323a726f6c652f7265706c3c2f526f6c653e3c52756c653e3c49443e72313c2f49443e3c5374617475733e456e61626c65643c2f5374617475733e3c5072656669783e646f632f3c2f5072656669783e3c44657374696e6174696f6e3e3c4275636b65743e61726e3a6177733a73333a3a3a646573743c2f4275636b65743e3c2f44657374696e6174696f6e3e3c2f52756c653e3c2f5265706c69636174696f6e436f6e66696775726174696f6e3eb74275636b657454617267657473436f6e6669674a534f4ec4535b7b22656e64706f696e74223a22687474703a2f2f7461726765742e6578616d706c652e636f6d222c227461726765744275636b6574223a227462222c22726567696f6e223a2275732d656173742d31227d5dbb4275636b657454617267657473436f6e6669674d6574614a534f4ec42d7b227265706c69636174696f6e4964223a227265706c2d31222c2273796e634d6f6465223a226173796e63227db5506f6c696379436f6e666967557064617465644174c70c050000000065a5022000000000b94f626a6563744c6f636b436f6e666967557064617465644174c70c050000000065a5022000000000b9456e6372797074696f6e436f6e666967557064617465644174c70c050000000065a5022000000000b654616767696e67436f6e666967557064617465644174c70c050000000065a5022000000000b451756f7461436f6e666967557064617465644174c70c050000000065a5022000000000ba5265706c69636174696f6e436f6e666967557064617465644174c70c050000000065a5022000000000b956657273696f6e696e67436f6e666967557064617465644174c70c050000000065a5022000000000b84c6966656379636c65436f6e666967557064617465644174c70c050000000065a5022000000000bb4e6f74696669636174696f6e436f6e666967557064617465644174c70c050000000065a5022000000000bc4275636b657454617267657473436f6e666967557064617465644174c70c050000000065a5022000000000d9204275636b657454617267657473436f6e6669674d657461557064617465644174c70c050000000065a5022000000000";
#[tokio::test]
async fn marshal_msg() {
let bm = BucketMetadata::new("dada");
let buf = bm.marshal_msg().unwrap();
let new = BucketMetadata::unmarshal(&buf).unwrap();
assert_eq!(bm.name, new.name);
}
/// Verifies that serialized time uses msgp ext type 5.
#[tokio::test]
async fn marshal_msg_uses_time_format() {
let mut bm = BucketMetadata::new("test-bucket");
bm.created = OffsetDateTime::from_unix_timestamp(1704067200).unwrap(); // 2024-01-01 00:00:00 UTC
let buf = bm.marshal_msg().unwrap();
// msgp uses ext8 (0xc7), len 12, type 5 for time
assert!(
buf.windows(3).any(|w| w == [0xc7, 0x0c, 0x05]),
"serialized data should contain msgp time ext (0xc7 0x0c 0x05)"
);
}
#[tokio::test]
async fn unmarshal_test_bucket_metadata() {
use faster_hex::hex_decode;
let mut bytes = vec![0u8; TEST_BUCKET_METADATA_HEX.len() / 2];
hex_decode(TEST_BUCKET_METADATA_HEX.as_bytes(), &mut bytes).expect("valid hex");
let bm = BucketMetadata::unmarshal(&bytes).expect("RustFS must unmarshal MinIO format");
assert_eq!(bm.name, "rustfs-compat-test");
assert_eq!(bm.created.unix_timestamp(), 1704067200);
assert_eq!(bm.created.nanosecond(), 123456789);
assert!(bm.lock_enabled);
assert!(!bm.policy_config_json.is_empty());
assert!(bm.policy_config_json.starts_with(b"{\"Version\""));
assert!(!bm.notification_config_xml.is_empty());
assert!(bm.notification_config_xml.starts_with(b"<Notification"));
assert!(!bm.lifecycle_config_xml.is_empty());
assert!(bm.lifecycle_config_xml.starts_with(b"<Lifecycle"));
assert!(!bm.object_lock_config_xml.is_empty());
assert!(bm.object_lock_config_xml.starts_with(b"<ObjectLock"));
assert!(!bm.versioning_config_xml.is_empty());
assert!(bm.versioning_config_xml.starts_with(b"<Versioning"));
assert!(!bm.encryption_config_xml.is_empty());
assert!(bm.encryption_config_xml.starts_with(b"<ServerSide"));
assert!(!bm.tagging_config_xml.is_empty());
assert!(bm.tagging_config_xml.starts_with(b"<Tagging"));
assert!(!bm.quota_config_json.is_empty());
assert!(bm.quota_config_json.starts_with(b"{\"quota\""));
assert!(!bm.replication_config_xml.is_empty());
assert!(bm.replication_config_xml.starts_with(b"<Replication"));
assert!(!bm.bucket_targets_config_json.is_empty());
assert!(bm.bucket_targets_config_json.starts_with(b"[{"));
assert!(!bm.bucket_targets_config_meta_json.is_empty());
assert!(bm.bucket_targets_config_meta_json.starts_with(b"{\"replication"));
let updated_sec = 1705312800; // 2024-01-15 12:00:00 UTC
assert_eq!(bm.policy_config_updated_at.unix_timestamp(), updated_sec);
assert_eq!(bm.object_lock_config_updated_at.unix_timestamp(), updated_sec);
assert_eq!(bm.encryption_config_updated_at.unix_timestamp(), updated_sec);
assert_eq!(bm.tagging_config_updated_at.unix_timestamp(), updated_sec);
assert_eq!(bm.quota_config_updated_at.unix_timestamp(), updated_sec);
assert_eq!(bm.replication_config_updated_at.unix_timestamp(), updated_sec);
assert_eq!(bm.versioning_config_updated_at.unix_timestamp(), updated_sec);
assert_eq!(bm.lifecycle_config_updated_at.unix_timestamp(), updated_sec);
assert_eq!(bm.notification_config_updated_at.unix_timestamp(), updated_sec);
assert_eq!(bm.bucket_targets_config_updated_at.unix_timestamp(), updated_sec);
assert_eq!(bm.bucket_targets_config_meta_updated_at.unix_timestamp(), updated_sec);
assert!(bm.cors_config_xml.is_empty());
assert!(bm.public_access_block_config_xml.is_empty());
assert!(bm.bucket_acl_config_json.is_empty());
}
#[tokio::test]
async fn marshal_msg_complete_example() {
// Create a complete BucketMetadata with various configurations
let mut bm = BucketMetadata::new("test-bucket");
// Set creation time to current time
bm.created = OffsetDateTime::now_utc();
bm.lock_enabled = true;
// Add policy configuration
let policy_json = r#"{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":"s3:GetObject","Resource":"arn:aws:s3:::test-bucket/*"}]}"#;
bm.policy_config_json = policy_json.as_bytes().to_vec();
bm.policy_config_updated_at = OffsetDateTime::now_utc();
// Add lifecycle configuration
let lifecycle_xml = r#"<LifecycleConfiguration><Rule><ID>rule1</ID><Status>Enabled</Status><Expiration><Days>30</Days></Expiration></Rule></LifecycleConfiguration>"#;
bm.lifecycle_config_xml = lifecycle_xml.as_bytes().to_vec();
bm.lifecycle_config_updated_at = OffsetDateTime::now_utc();
// Add versioning configuration
let versioning_xml = r#"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>"#;
bm.versioning_config_xml = versioning_xml.as_bytes().to_vec();
bm.versioning_config_updated_at = OffsetDateTime::now_utc();
// Add encryption configuration
let encryption_xml = r#"<ServerSideEncryptionConfiguration><Rule><ApplyServerSideEncryptionByDefault><SSEAlgorithm>AES256</SSEAlgorithm></ApplyServerSideEncryptionByDefault></Rule></ServerSideEncryptionConfiguration>"#;
bm.encryption_config_xml = encryption_xml.as_bytes().to_vec();
bm.encryption_config_updated_at = OffsetDateTime::now_utc();
// Add tagging configuration
let tagging_xml = r#"<Tagging><TagSet><Tag><Key>Environment</Key><Value>Test</Value></Tag><Tag><Key>Owner</Key><Value>RustFS</Value></Tag></TagSet></Tagging>"#;
bm.tagging_config_xml = tagging_xml.as_bytes().to_vec();
bm.tagging_config_updated_at = OffsetDateTime::now_utc();
// Add quota configuration
let quota_json =
r#"{"quota":1073741824,"quota_type":"Hard","created_at":"2024-01-01T00:00:00Z","updated_at":"2024-01-01T00:00:00Z"}"#; // 1GB quota
bm.quota_config_json = quota_json.as_bytes().to_vec();
bm.quota_config_updated_at = OffsetDateTime::now_utc();
// Add object lock configuration
let object_lock_xml = r#"<ObjectLockConfiguration><ObjectLockEnabled>Enabled</ObjectLockEnabled><Rule><DefaultRetention><Mode>GOVERNANCE</Mode><Days>7</Days></DefaultRetention></Rule></ObjectLockConfiguration>"#;
bm.object_lock_config_xml = object_lock_xml.as_bytes().to_vec();
bm.object_lock_config_updated_at = OffsetDateTime::now_utc();
// Add notification configuration
let notification_xml = r#"<NotificationConfiguration><CloudWatchConfiguration><Id>notification1</Id><Event>s3:ObjectCreated:*</Event><CloudWatchConfiguration><LogGroupName>test-log-group</LogGroupName></CloudWatchConfiguration></CloudWatchConfiguration></NotificationConfiguration>"#;
bm.notification_config_xml = notification_xml.as_bytes().to_vec();
bm.notification_config_updated_at = OffsetDateTime::now_utc();
// Add replication configuration
let replication_xml = r#"<ReplicationConfiguration><Role>arn:aws:iam::123456789012:role/replication-role</Role><Rule><ID>rule1</ID><Status>Enabled</Status><Prefix>documents/</Prefix><Destination><Bucket>arn:aws:s3:::destination-bucket</Bucket></Destination></Rule></ReplicationConfiguration>"#;
bm.replication_config_xml = replication_xml.as_bytes().to_vec();
bm.replication_config_updated_at = OffsetDateTime::now_utc();
// Add bucket targets configuration
let bucket_targets_json = r#"[{"endpoint":"http://target1.example.com","credentials":{"accessKey":"key1","secretKey":"secret1"},"targetBucket":"target-bucket-1","region":"us-east-1"},{"endpoint":"http://target2.example.com","credentials":{"accessKey":"key2","secretKey":"secret2"},"targetBucket":"target-bucket-2","region":"us-west-2"}]"#;
bm.bucket_targets_config_json = bucket_targets_json.as_bytes().to_vec();
bm.bucket_targets_config_updated_at = OffsetDateTime::now_utc();
// Add bucket targets meta configuration
let bucket_targets_meta_json = r#"{"replicationId":"repl-123","syncMode":"async","bandwidth":"100MB"}"#;
bm.bucket_targets_config_meta_json = bucket_targets_meta_json.as_bytes().to_vec();
bm.bucket_targets_config_meta_updated_at = OffsetDateTime::now_utc();
// Add public access block configuration
let public_access_block_xml = r#"<PublicAccessBlockConfiguration><BlockPublicAcls>true</BlockPublicAcls><IgnorePublicAcls>true</IgnorePublicAcls><BlockPublicPolicy>true</BlockPublicPolicy><RestrictPublicBuckets>false</RestrictPublicBuckets></PublicAccessBlockConfiguration>"#;
bm.public_access_block_config_xml = public_access_block_xml.as_bytes().to_vec();
bm.public_access_block_config_updated_at = OffsetDateTime::now_utc();
let bucket_acl = r#"{"owner":{"id":"rustfsadmin","display_name":"RustFS Tester"},"grants":[{"grantee":{"grantee_type":"CanonicalUser","id":"rustfsadmin","display_name":"RustFS Tester","uri":null,"email_address":null},"permission":"FULL_CONTROL"}]}"#;
bm.bucket_acl_config_json = bucket_acl.as_bytes().to_vec();
bm.bucket_acl_config_updated_at = OffsetDateTime::now_utc();
// Test serialization
let buf = bm.marshal_msg().unwrap();
assert!(!buf.is_empty(), "Serialized buffer should not be empty");
// Test deserialization
let deserialized_bm = BucketMetadata::unmarshal(&buf).unwrap();
// Verify all fields are correctly serialized and deserialized
assert_eq!(bm.name, deserialized_bm.name);
assert_eq!(bm.created.unix_timestamp(), deserialized_bm.created.unix_timestamp());
assert_eq!(bm.lock_enabled, deserialized_bm.lock_enabled);
// Verify configuration data
assert_eq!(bm.policy_config_json, deserialized_bm.policy_config_json);
assert_eq!(bm.lifecycle_config_xml, deserialized_bm.lifecycle_config_xml);
assert_eq!(bm.versioning_config_xml, deserialized_bm.versioning_config_xml);
assert_eq!(bm.encryption_config_xml, deserialized_bm.encryption_config_xml);
assert_eq!(bm.tagging_config_xml, deserialized_bm.tagging_config_xml);
assert_eq!(bm.quota_config_json, deserialized_bm.quota_config_json);
assert_eq!(bm.public_access_block_config_xml, deserialized_bm.public_access_block_config_xml);
assert_eq!(bm.bucket_acl_config_json, deserialized_bm.bucket_acl_config_json);
assert_eq!(bm.object_lock_config_xml, deserialized_bm.object_lock_config_xml);
assert_eq!(bm.notification_config_xml, deserialized_bm.notification_config_xml);
assert_eq!(bm.replication_config_xml, deserialized_bm.replication_config_xml);
assert_eq!(bm.bucket_targets_config_json, deserialized_bm.bucket_targets_config_json);
assert_eq!(bm.bucket_targets_config_meta_json, deserialized_bm.bucket_targets_config_meta_json);
// Verify timestamps (comparing unix timestamps to avoid precision issues)
assert_eq!(
bm.policy_config_updated_at.unix_timestamp(),
deserialized_bm.policy_config_updated_at.unix_timestamp()
);
assert_eq!(
bm.lifecycle_config_updated_at.unix_timestamp(),
deserialized_bm.lifecycle_config_updated_at.unix_timestamp()
);
assert_eq!(
bm.versioning_config_updated_at.unix_timestamp(),
deserialized_bm.versioning_config_updated_at.unix_timestamp()
);
assert_eq!(
bm.encryption_config_updated_at.unix_timestamp(),
deserialized_bm.encryption_config_updated_at.unix_timestamp()
);
assert_eq!(
bm.tagging_config_updated_at.unix_timestamp(),
deserialized_bm.tagging_config_updated_at.unix_timestamp()
);
assert_eq!(
bm.quota_config_updated_at.unix_timestamp(),
deserialized_bm.quota_config_updated_at.unix_timestamp()
);
assert_eq!(
bm.object_lock_config_updated_at.unix_timestamp(),
deserialized_bm.object_lock_config_updated_at.unix_timestamp()
);
assert_eq!(
bm.notification_config_updated_at.unix_timestamp(),
deserialized_bm.notification_config_updated_at.unix_timestamp()
);
assert_eq!(
bm.replication_config_updated_at.unix_timestamp(),
deserialized_bm.replication_config_updated_at.unix_timestamp()
);
assert_eq!(
bm.bucket_targets_config_updated_at.unix_timestamp(),
deserialized_bm.bucket_targets_config_updated_at.unix_timestamp()
);
assert_eq!(
bm.bucket_targets_config_meta_updated_at.unix_timestamp(),
deserialized_bm.bucket_targets_config_meta_updated_at.unix_timestamp()
);
// Test that the serialized data contains expected content
let buf_str = String::from_utf8_lossy(&buf);
assert!(buf_str.contains("test-bucket"), "Serialized data should contain bucket name");
// Verify the buffer size is reasonable (should be larger due to all the config data)
assert!(buf.len() > 1000, "Buffer should be substantial in size due to all configurations");
println!("✅ Complete BucketMetadata serialization test passed");
println!(" - Bucket name: {}", deserialized_bm.name);
println!(" - Lock enabled: {}", deserialized_bm.lock_enabled);
println!(" - Policy config size: {} bytes", deserialized_bm.policy_config_json.len());
println!(" - Lifecycle config size: {} bytes", deserialized_bm.lifecycle_config_xml.len());
println!(" - Serialized buffer size: {} bytes", buf.len());
}
crates/ecstore/src/bucket/migration.rs
+420
View File
@@ -0,0 +1,420 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Migration of bucket metadata and IAM config from legacy format to RustFS format.
use crate::bucket::metadata::BUCKET_METADATA_FILE;
use crate::bucket::replication::{decode_resync_file, encode_resync_file};
use crate::disk::{BUCKET_META_PREFIX, MIGRATING_META_BUCKET, RUSTFS_META_BUCKET};
use crate::store_api::{BucketOptions, ObjectOptions, PutObjReader, StorageAPI};
use http::HeaderMap;
use rustfs_policy::auth::UserIdentity;
use rustfs_policy::policy::PolicyDoc;
use rustfs_utils::path::SLASH_SEPARATOR;
use serde::{Deserialize, Serialize};
use std::sync::Arc;
use time::OffsetDateTime;
use tracing::{debug, info, warn};
/// IAM config prefix under meta bucket (e.g. config/iam/).
const IAM_CONFIG_PREFIX: &str = "config/iam";
const IAM_FORMAT_FILE_PATH: &str = "config/iam/format.json";
const IAM_USERS_PREFIX: &str = "config/iam/users/";
const IAM_SERVICE_ACCOUNTS_PREFIX: &str = "config/iam/service-accounts/";
const IAM_STS_PREFIX: &str = "config/iam/sts/";
const IAM_GROUPS_PREFIX: &str = "config/iam/groups/";
const IAM_POLICIES_PREFIX: &str = "config/iam/policies/";
const IAM_POLICY_DB_PREFIX: &str = "config/iam/policydb/";
const REPLICATION_META_DIR: &str = ".replication";
const RESYNC_META_FILE: &str = "resync.bin";
#[derive(Debug, Serialize, Deserialize)]
struct CompatIamFormat {
#[serde(default)]
version: i64,
}
#[derive(Debug, Serialize, Deserialize)]
struct CompatGroupInfo {
#[serde(default)]
version: i64,
#[serde(default = "default_group_status")]
status: String,
#[serde(default)]
members: Vec<String>,
#[serde(
rename = "updatedAt",
alias = "update_at",
default,
with = "rustfs_policy::serde_datetime::option"
)]
update_at: Option<OffsetDateTime>,
}
#[derive(Debug, Serialize, Deserialize)]
struct CompatMappedPolicy {
#[serde(default)]
version: i64,
#[serde(rename = "policy", alias = "policies", default)]
policy: String,
#[serde(
rename = "updatedAt",
alias = "update_at",
default,
with = "rustfs_policy::serde_datetime::option"
)]
update_at: Option<OffsetDateTime>,
}
fn default_group_status() -> String {
"enabled".to_string()
}
fn normalize_iam_config_blob(path: &str, data: &[u8]) -> std::result::Result<Option<Vec<u8>>, String> {
if path == IAM_FORMAT_FILE_PATH {
let mut format: CompatIamFormat =
serde_json::from_slice(data).map_err(|err| format!("parse IAM format failed: {err}"))?;
if format.version <= 0 {
format.version = 1;
}
return serde_json::to_vec(&format)
.map(Some)
.map_err(|err| format!("serialize IAM format failed: {err}"));
}
if is_identity_path(path) {
let mut identity: UserIdentity =
serde_json::from_slice(data).map_err(|err| format!("parse IAM identity failed: {err}"))?;
if identity.update_at.is_none() {
identity.update_at = Some(OffsetDateTime::now_utc());
}
return serde_json::to_vec(&identity)
.map(Some)
.map_err(|err| format!("serialize IAM identity failed: {err}"));
}
if is_group_path(path) {
let mut group: CompatGroupInfo = serde_json::from_slice(data).map_err(|err| format!("parse IAM group failed: {err}"))?;
if group.update_at.is_none() {
group.update_at = Some(OffsetDateTime::now_utc());
}
return serde_json::to_vec(&group)
.map(Some)
.map_err(|err| format!("serialize IAM group failed: {err}"));
}
if is_policy_doc_path(path) {
let mut doc = PolicyDoc::try_from(data.to_vec()).map_err(|err| format!("parse IAM policy doc failed: {err}"))?;
if doc.create_date.is_none() {
doc.create_date = doc.update_date;
}
if doc.update_date.is_none() {
doc.update_date = doc.create_date;
}
return serde_json::to_vec(&doc)
.map(Some)
.map_err(|err| format!("serialize IAM policy doc failed: {err}"));
}
if is_policy_mapping_path(path) {
let mut mapped: CompatMappedPolicy =
serde_json::from_slice(data).map_err(|err| format!("parse IAM policy mapping failed: {err}"))?;
if mapped.update_at.is_none() {
mapped.update_at = Some(OffsetDateTime::now_utc());
}
return serde_json::to_vec(&mapped)
.map(Some)
.map_err(|err| format!("serialize IAM policy mapping failed: {err}"));
}
Ok(None)
}
fn is_identity_path(path: &str) -> bool {
(path.starts_with(IAM_USERS_PREFIX) || path.starts_with(IAM_SERVICE_ACCOUNTS_PREFIX) || path.starts_with(IAM_STS_PREFIX))
&& path.ends_with("/identity.json")
}
fn is_group_path(path: &str) -> bool {
path.starts_with(IAM_GROUPS_PREFIX) && path.ends_with("/members.json")
}
fn is_policy_doc_path(path: &str) -> bool {
path.starts_with(IAM_POLICIES_PREFIX) && path.ends_with("/policy.json")
}
fn is_policy_mapping_path(path: &str) -> bool {
path.starts_with(IAM_POLICY_DB_PREFIX) && path.ends_with(".json")
}
fn is_resync_meta_path(path: &str) -> bool {
path.ends_with(&format!("{REPLICATION_META_DIR}/{RESYNC_META_FILE}"))
}
fn normalize_bucket_meta_blob(path: &str, data: &[u8]) -> std::result::Result<Option<Vec<u8>>, String> {
if !is_resync_meta_path(path) {
return Ok(None);
}
let status = decode_resync_file(data).map_err(|err| format!("decode resync meta failed: {err}"))?;
encode_resync_file(&status)
.map(Some)
.map_err(|err| format!("encode resync meta failed: {err}"))
}
/// Migrates bucket metadata from legacy format to RustFS.
/// Uses list_bucket (from disk volumes) to get bucket names, since list_objects_v2 on the legacy
/// meta bucket may not work (legacy format differs from object layer expectations).
/// Skips buckets that already exist in RustFS (idempotent).
pub async fn try_migrate_bucket_metadata<S: StorageAPI>(store: Arc<S>) {
let buckets_list = match store
.list_bucket(&BucketOptions {
no_metadata: true,
..Default::default()
})
.await
{
Ok(b) => b,
Err(e) => {
warn!("list buckets failed (skip migration): {e}");
return;
}
};
let buckets: Vec<String> = buckets_list.into_iter().map(|b| b.name).collect();
if buckets.is_empty() {
debug!("No migrating bucket metadata found");
return;
}
debug!("Found {} migrating bucket metadata, migrating...", buckets.len());
let opts = ObjectOptions {
max_parity: true,
no_lock: true,
..Default::default()
};
let h = HeaderMap::new();
for bucket in buckets {
let meta_path = format!("{BUCKET_META_PREFIX}{SLASH_SEPARATOR}{bucket}{SLASH_SEPARATOR}{BUCKET_METADATA_FILE}");
migrate_one_if_missing(store.clone(), &opts, &h, &meta_path, &format!("bucket metadata: {bucket}")).await;
let resync_path = format!(
"{BUCKET_META_PREFIX}{SLASH_SEPARATOR}{bucket}{SLASH_SEPARATOR}{REPLICATION_META_DIR}{SLASH_SEPARATOR}{RESYNC_META_FILE}"
);
migrate_one_if_missing(store.clone(), &opts, &h, &resync_path, &format!("bucket replication resync: {bucket}")).await;
}
}
async fn migrate_one_if_missing<S: StorageAPI>(
store: Arc<S>,
opts: &ObjectOptions,
headers: &HeaderMap,
path: &str,
label: &str,
) {
if store
.get_object_info(RUSTFS_META_BUCKET, path, &ObjectOptions::default())
.await
.is_ok()
{
debug!("{label} already exists in RustFS, skip");
return;
}
let mut rd = match store
.get_object_reader(MIGRATING_META_BUCKET, path, None, headers.clone(), opts)
.await
{
Ok(r) => r,
Err(e) => {
debug!("read migrating {label}: {e}");
return;
}
};
let data = match rd.read_all().await {
Ok(d) if !d.is_empty() => d,
Ok(_) => return,
Err(e) => {
debug!("read migrating {label} body: {e}");
return;
}
};
let data = match normalize_bucket_meta_blob(path, &data) {
Ok(Some(normalized)) => normalized,
Ok(None) => data,
Err(e) => {
warn!("skip {label} migration due to incompatible format: {e}");
return;
}
};
if let Err(e) = store
.put_object(RUSTFS_META_BUCKET, path, &mut PutObjReader::from_vec(data), opts)
.await
{
warn!("write {label}: {e}");
} else {
info!("Migrated {label}");
}
}
/// Migrates IAM config from legacy meta bucket `config/iam/` to RustFS meta bucket.
/// Lists all objects under the IAM prefix in the source, copies each to the target if not present.
/// Skips objects that already exist in RustFS (idempotent).
/// If list_objects_v2 on the legacy bucket fails (e.g. format differs), migration is skipped.
pub async fn try_migrate_iam_config<S: StorageAPI>(store: Arc<S>) {
let opts = ObjectOptions {
max_parity: true,
no_lock: true,
..Default::default()
};
let h = HeaderMap::new();
let prefix = format!("{IAM_CONFIG_PREFIX}/");
let mut continuation: Option<String> = None;
let mut total_migrated = 0usize;
loop {
let list_result = match store
.clone()
.list_objects_v2(MIGRATING_META_BUCKET, &prefix, continuation, None, 500, false, None, false)
.await
{
Ok(r) => r,
Err(e) => {
debug!("list IAM config from legacy bucket failed (skip migration): {e}");
return;
}
};
for obj in list_result.objects {
let path = &obj.name;
if path.is_empty() || path.ends_with('/') {
continue;
}
if store
.get_object_info(RUSTFS_META_BUCKET, path, &ObjectOptions::default())
.await
.is_ok()
{
debug!("IAM config already exists in RustFS, skip: {path}");
continue;
}
let mut rd = match store
.get_object_reader(MIGRATING_META_BUCKET, path, None, h.clone(), &opts)
.await
{
Ok(r) => r,
Err(e) => {
debug!("read migrating IAM config {path}: {e}");
continue;
}
};
let data = match rd.read_all().await {
Ok(d) if !d.is_empty() => d,
Ok(_) => continue,
Err(e) => {
debug!("read migrating IAM config {path} body: {e}");
continue;
}
};
let data = match normalize_iam_config_blob(path, &data) {
Ok(Some(normalized)) => normalized,
Ok(None) => {
debug!("skip unsupported IAM config path during migration: {path}");
continue;
}
Err(e) => {
warn!("skip IAM config migration due to incompatible format, path: {path}, err: {e}");
continue;
}
};
if let Err(e) = store
.put_object(RUSTFS_META_BUCKET, path, &mut PutObjReader::from_vec(data), &opts)
.await
{
warn!("write IAM config {path}: {e}");
} else {
info!("Migrated IAM config: {path}");
total_migrated += 1;
}
}
continuation = list_result.next_continuation_token.or(list_result.continuation_token);
if !list_result.is_truncated || continuation.is_none() {
break;
}
}
if total_migrated > 0 {
info!("IAM migration complete: {} object(s) migrated", total_migrated);
}
}
#[cfg(test)]
mod tests {
use super::{normalize_bucket_meta_blob, normalize_iam_config_blob};
use crate::bucket::replication::{
BucketReplicationResyncStatus, ResyncStatusType, TargetReplicationResyncStatus, decode_resync_file, encode_resync_file,
};
use std::collections::HashMap;
#[test]
fn test_normalize_policy_mapping_legacy_timestamp_and_fields() {
let path = "config/iam/policydb/users/alice.json";
let input = r#"{"version":1,"policies":"readwrite","update_at":"2026-03-09 02:22:44.998954 +00:00:00"}"#;
let output = normalize_iam_config_blob(path, input.as_bytes())
.expect("normalize should succeed")
.expect("path should be supported");
let v: serde_json::Value = serde_json::from_slice(&output).expect("output should be valid JSON");
assert_eq!(v.get("policy").and_then(|x| x.as_str()), Some("readwrite"));
assert!(v.get("policies").is_none(), "legacy field should be normalized");
let updated_at = v
.get("updatedAt")
.and_then(|x| x.as_str())
.expect("updatedAt should exist as string");
assert!(updated_at.contains('T'), "updatedAt should be RFC3339-like");
}
#[test]
fn test_normalize_bucket_meta_blob_resync_reencode() {
let path = ".buckets/test/.replication/resync.bin";
let mut status = BucketReplicationResyncStatus::new();
status.id = 123;
status.targets_map = HashMap::from([(
"arn:replication::1:dest".to_string(),
TargetReplicationResyncStatus {
resync_id: "reset-1".to_string(),
resync_status: ResyncStatusType::ResyncStarted,
replicated_count: 1,
..Default::default()
},
)]);
let input = encode_resync_file(&status).expect("encode should succeed");
let output = normalize_bucket_meta_blob(path, &input)
.expect("normalize should succeed")
.expect("resync path should be normalized");
let decoded = decode_resync_file(&output).expect("decode should succeed");
assert_eq!(decoded.id, 123);
assert_eq!(decoded.targets_map["arn:replication::1:dest"].resync_id, "reset-1");
}
}
crates/ecstore/src/bucket/mod.rs
+4
View File
@@ -18,6 +18,10 @@ pub mod error;
pub mod lifecycle;
pub mod metadata;
pub mod metadata_sys;
#[cfg(test)]
mod metadata_test;
pub mod migration;
mod msgp_decode;
pub mod object_lock;
pub mod policy_sys;
pub mod quota;
crates/ecstore/src/bucket/msgp_decode.rs
+189
View File
@@ -0,0 +1,189 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! MessagePack decode helpers for bucket metadata, aligned with msgp format.
use crate::error::{Error, Result};
use byteorder::{BigEndian, ByteOrder};
use rmp::Marker;
use std::io::{Read, Write};
use time::OffsetDateTime;
/// Skip a single MessagePack value. Used for unknown map keys.
pub(crate) fn skip_msgp_value<R: Read>(rd: &mut R) -> Result<()> {
let marker = rmp::decode::read_marker(rd).map_err(|e| Error::other(format!("{e:?}")))?;
let skip_len: usize = match marker {
Marker::Null | Marker::False | Marker::True => 0,
Marker::FixPos(_) | Marker::FixNeg(_) => 0,
Marker::U8 => 1,
Marker::U16 => 2,
Marker::U32 => 4,
Marker::U64 => 8,
Marker::I8 => 1,
Marker::I16 => 2,
Marker::I32 => 4,
Marker::I64 => 8,
Marker::F32 => 4,
Marker::F64 => 8,
Marker::FixStr(n) => n as usize,
Marker::Str8 => {
let mut b = [0u8; 1];
rd.read_exact(&mut b).map_err(Error::other)?;
b[0] as usize
}
Marker::Str16 => {
let mut b = [0u8; 2];
rd.read_exact(&mut b).map_err(Error::other)?;
u16::from_be_bytes(b) as usize
}
Marker::Str32 => {
let mut b = [0u8; 4];
rd.read_exact(&mut b).map_err(Error::other)?;
u32::from_be_bytes(b) as usize
}
Marker::Bin8 => {
let mut b = [0u8; 1];
rd.read_exact(&mut b).map_err(Error::other)?;
b[0] as usize
}
Marker::Bin16 => {
let mut b = [0u8; 2];
rd.read_exact(&mut b).map_err(Error::other)?;
u16::from_be_bytes(b) as usize
}
Marker::Bin32 => {
let mut b = [0u8; 4];
rd.read_exact(&mut b).map_err(Error::other)?;
u32::from_be_bytes(b) as usize
}
Marker::FixArray(n) => {
for _ in 0..n {
skip_msgp_value(rd)?;
}
return Ok(());
}
Marker::Array16 => {
let mut b = [0u8; 2];
rd.read_exact(&mut b).map_err(Error::other)?;
let n = u16::from_be_bytes(b);
for _ in 0..n {
skip_msgp_value(rd)?;
}
return Ok(());
}
Marker::Array32 => {
let mut b = [0u8; 4];
rd.read_exact(&mut b).map_err(Error::other)?;
let n = u32::from_be_bytes(b);
for _ in 0..n {
skip_msgp_value(rd)?;
}
return Ok(());
}
Marker::FixMap(n) => {
for _ in 0..n {
skip_msgp_value(rd)?;
skip_msgp_value(rd)?;
}
return Ok(());
}
Marker::Map16 => {
let mut b = [0u8; 2];
rd.read_exact(&mut b).map_err(Error::other)?;
let n = u16::from_be_bytes(b);
for _ in 0..n {
skip_msgp_value(rd)?;
skip_msgp_value(rd)?;
}
return Ok(());
}
Marker::Map32 => {
let mut b = [0u8; 4];
rd.read_exact(&mut b).map_err(Error::other)?;
let n = u32::from_be_bytes(b);
for _ in 0..n {
skip_msgp_value(rd)?;
skip_msgp_value(rd)?;
}
return Ok(());
}
Marker::FixExt1 => 1,
Marker::FixExt2 => 2,
Marker::FixExt4 => 4,
Marker::FixExt8 => 8,
Marker::FixExt16 => 16,
Marker::Ext8 => {
let mut b = [0u8; 1];
rd.read_exact(&mut b).map_err(Error::other)?;
let len = b[0] as usize;
1 + len // type byte + data
}
Marker::Ext16 => {
let mut b = [0u8; 2];
rd.read_exact(&mut b).map_err(Error::other)?;
let len = u16::from_be_bytes(b) as usize;
2 + len
}
Marker::Ext32 => {
let mut b = [0u8; 4];
rd.read_exact(&mut b).map_err(Error::other)?;
let len = u32::from_be_bytes(b) as usize;
4 + len
}
Marker::Reserved => 0,
};
if skip_len > 0 {
let mut buf = vec![0u8; skip_len];
rd.read_exact(&mut buf).map_err(Error::other)?;
}
Ok(())
}
/// msgp time format: ext8 (0xc7), len 12, type 5, 8 bytes sec (BE) + 4 bytes nsec (BE).
pub(crate) const MSGP_TIME_EXT_TYPE: i8 = 5;
pub(crate) const MSGP_TIME_LEN: u8 = 12;
/// Read msgp ext8 time - caller must have already read the marker and verified it's ext8.
/// Ext8 format: 1 byte len, 1 byte type, then data bytes.
pub(crate) fn read_msgp_ext8_time<R: Read>(rd: &mut R) -> Result<OffsetDateTime> {
let mut len_buf = [0u8; 1];
rd.read_exact(&mut len_buf).map_err(Error::other)?;
let len = len_buf[0] as usize;
if len != MSGP_TIME_LEN as usize {
return Err(Error::other(format!("invalid msgp time len: {len}")));
}
let mut type_buf = [0u8; 1];
rd.read_exact(&mut type_buf).map_err(Error::other)?;
if type_buf[0] != MSGP_TIME_EXT_TYPE as u8 {
return Err(Error::other(format!("invalid msgp time type: {}", type_buf[0])));
}
let mut buf = [0u8; 12];
rd.read_exact(&mut buf).map_err(Error::other)?;
let sec = BigEndian::read_i64(&buf[0..8]);
let nsec = BigEndian::read_u32(&buf[8..12]);
OffsetDateTime::from_unix_timestamp(sec)
.map_err(|_| Error::other("invalid timestamp"))?
.replace_nanosecond(nsec)
.map_err(|_| Error::other("invalid nanosecond"))
}
/// Write msgp time as ext8 (0xc7), len 12, type 5. Always uses ext format (never nil).
pub(crate) fn write_msgp_time<W: Write>(wr: &mut W, t: OffsetDateTime) -> Result<()> {
wr.write_all(&[0xc7, MSGP_TIME_LEN, MSGP_TIME_EXT_TYPE as u8])
.map_err(Error::other)?;
let mut buf = [0u8; 12];
BigEndian::write_i64(&mut buf[0..8], t.unix_timestamp());
BigEndian::write_u32(&mut buf[8..12], t.nanosecond());
wr.write_all(&buf).map_err(Error::other)
}
crates/ecstore/src/bucket/quota/mod.rs
+68 -6
View File
@@ -15,7 +15,6 @@
pub mod checker;
use crate::error::Result;
use rmp_serde::Serializer as rmpSerializer;
use rustfs_config::{
QUOTA_API_PATH, QUOTA_EXCEEDED_ERROR_CODE, QUOTA_INTERNAL_ERROR_CODE, QUOTA_INVALID_CONFIG_ERROR_CODE,
QUOTA_NOT_FOUND_ERROR_CODE,
@@ -28,27 +27,35 @@ use time::OffsetDateTime;
pub enum QuotaType {
/// Hard quota: reject immediately when exceeded
#[default]
#[serde(alias = "HARD", alias = "hard")]
Hard,
}
/// Bucket quota configuration. quota_type defaults to Hard when omitted.
#[derive(Debug, Deserialize, Serialize, Default, Clone, PartialEq)]
pub struct BucketQuota {
#[serde(default)]
pub quota: Option<u64>,
/// Defaults to Hard when missing.
#[serde(default)]
pub quota_type: QuotaType,
/// Timestamp when this quota configuration was set (for audit purposes)
#[serde(default, with = "time::serde::rfc3339::option")]
pub created_at: Option<OffsetDateTime>,
/// Accept updated_at for compatibility; not used.
#[serde(default, with = "time::serde::rfc3339::option", skip_serializing_if = "Option::is_none")]
pub updated_at: Option<OffsetDateTime>,
}
impl BucketQuota {
/// Serialize to JSON bytes. Same format as parse_all_configs.
pub fn marshal_msg(&self) -> Result<Vec<u8>> {
let mut buf = Vec::new();
self.serialize(&mut rmpSerializer::new(&mut buf).with_struct_map())?;
Ok(buf)
serde_json::to_vec(self).map_err(Into::into)
}
/// Deserialize from JSON bytes. Same format as parse_all_configs.
pub fn unmarshal(buf: &[u8]) -> Result<Self> {
let t: BucketQuota = rmp_serde::from_slice(buf)?;
Ok(t)
serde_json::from_slice(buf).map_err(Into::into)
}
pub fn new(quota: Option<u64>) -> Self {
@@ -57,6 +64,7 @@ impl BucketQuota {
quota,
quota_type: QuotaType::Hard,
created_at: Some(now),
updated_at: None,
}
}
@@ -156,3 +164,57 @@ impl QuotaErrorResponse {
}
}
}
#[cfg(test)]
mod tests {
use super::*;
/// Legacy format: quota, created_at, updated_at (no quota_type)
#[test]
fn deserialize_format_without_quota_type() {
let json = r#"{"quota":1073741824,"created_at":"2024-01-01T00:00:00Z","updated_at":"2024-01-01T00:00:00Z"}"#;
let q: BucketQuota = serde_json::from_slice(json.as_bytes()).expect("should parse");
assert_eq!(q.quota, Some(1073741824));
assert_eq!(q.quota_type, QuotaType::Hard);
assert!(q.created_at.is_some());
assert!(q.updated_at.is_some());
}
/// RustFS format: quota, quota_type, created_at
#[test]
fn deserialize_rustfs_format() {
let json = r#"{"quota":1073741824,"quota_type":"Hard","created_at":"2024-01-01T00:00:00Z"}"#;
let q: BucketQuota = serde_json::from_slice(json.as_bytes()).expect("should parse");
assert_eq!(q.quota, Some(1073741824));
assert_eq!(q.quota_type, QuotaType::Hard);
assert!(q.created_at.is_some());
assert!(q.created_at.is_some_and(|t| t.unix_timestamp() == 1704067200));
}
/// E2E format uses "HARD" (uppercase)
#[test]
fn deserialize_quota_type_hard_uppercase() {
let json = r#"{"quota":2048,"quota_type":"HARD"}"#;
let q: BucketQuota = serde_json::from_slice(json.as_bytes()).expect("should parse");
assert_eq!(q.quota_type, QuotaType::Hard);
}
/// marshal_msg/unmarshal use JSON, same as parse_all_configs
#[test]
fn marshal_unmarshal_roundtrip() {
let q = BucketQuota::new(Some(1073741824));
let buf = q.marshal_msg().expect("marshal");
let restored = BucketQuota::unmarshal(&buf).expect("unmarshal");
assert_eq!(q.quota, restored.quota);
assert_eq!(q.quota_type, restored.quota_type);
}
/// unmarshal accepts format without quota_type
#[test]
fn unmarshal_format_without_quota_type() {
let json = r#"{"quota":1073741824,"created_at":"2024-01-01T00:00:00Z","updated_at":"2024-01-01T00:00:00Z"}"#;
let q = BucketQuota::unmarshal(json.as_bytes()).expect("should parse");
assert_eq!(q.quota, Some(1073741824));
assert_eq!(q.quota_type, QuotaType::Hard);
}
}
crates/ecstore/src/bucket/replication/replication_pool.rs
+6 -37
View File
@@ -20,8 +20,8 @@ use crate::bucket::replication::ResyncStatusType;
use crate::bucket::replication::replicate_delete;
use crate::bucket::replication::replicate_object;
use crate::bucket::replication::replication_resyncer::{
BucketReplicationResyncStatus, DeletedObjectReplicationInfo, ReplicationConfig, ReplicationResyncer,
get_heal_replicate_object_info,
BucketReplicationResyncStatus, DeletedObjectReplicationInfo, REPLICATION_DIR, RESYNC_FILE_NAME, ReplicationConfig,
ReplicationResyncer, decode_resync_file, get_heal_replicate_object_info,
};
use crate::bucket::replication::replication_state::ReplicationStats;
use crate::config::com::read_config;
@@ -41,7 +41,7 @@ use rustfs_filemeta::VersionPurgeStatusType;
use rustfs_filemeta::replication_statuses_map;
use rustfs_filemeta::version_purge_statuses_map;
use rustfs_filemeta::{REPLICATE_EXISTING, REPLICATE_HEAL, REPLICATE_HEAL_DELETE};
use rustfs_utils::http::RESERVED_METADATA_PREFIX_LOWER;
use rustfs_utils::http::{SUFFIX_REPLICATION_TIMESTAMP, get_str};
use std::any::Any;
use std::sync::Arc;
use std::sync::atomic::AtomicI32;
@@ -861,17 +861,8 @@ async fn load_bucket_resync_metadata<S: StorageAPI>(
bucket: &str,
obj_api: Arc<S>,
) -> Result<BucketReplicationResyncStatus, EcstoreError> {
use std::convert::TryInto;
let mut brs = BucketReplicationResyncStatus::new();
// Constants that would be defined elsewhere
const REPLICATION_DIR: &str = "replication";
const RESYNC_FILE_NAME: &str = "resync.bin";
const RESYNC_META_FORMAT: u16 = 1;
const RESYNC_META_VERSION: u16 = 1;
const RESYNC_META_VERSION_V1: u16 = 1;
let resync_dir_path = format!("{BUCKET_META_PREFIX}/{bucket}/{REPLICATION_DIR}");
let resync_file_path = format!("{resync_dir_path}/{RESYNC_FILE_NAME}");
@@ -886,27 +877,7 @@ async fn load_bucket_resync_metadata<S: StorageAPI>(
return Ok(brs);
}
if data.len() <= 4 {
return Err(EcstoreError::CorruptedFormat);
}
// Read resync meta header
let format = u16::from_le_bytes(data[0..2].try_into().unwrap());
if format != RESYNC_META_FORMAT {
return Err(EcstoreError::CorruptedFormat);
}
let version = u16::from_le_bytes(data[2..4].try_into().unwrap());
if version != RESYNC_META_VERSION {
return Err(EcstoreError::CorruptedFormat);
}
// Parse data
brs = BucketReplicationResyncStatus::unmarshal_msg(&data[4..])?;
if brs.version != RESYNC_META_VERSION_V1 {
return Err(EcstoreError::CorruptedFormat);
}
brs = decode_resync_file(&data)?;
Ok(brs)
}
@@ -984,10 +955,8 @@ pub fn get_global_replication_pool() -> Option<Arc<DynReplicationPool>> {
pub async fn schedule_replication<S: StorageAPI>(oi: ObjectInfo, o: Arc<S>, dsc: ReplicateDecision, op_type: ReplicationType) {
let tgt_statuses = replication_statuses_map(&oi.replication_status_internal.clone().unwrap_or_default());
let purge_statuses = version_purge_statuses_map(&oi.version_purge_status_internal.clone().unwrap_or_default());
let tm = oi
.user_defined
.get(&format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, "replication-timestamp"))
.map(|v| OffsetDateTime::parse(v, &Rfc3339).unwrap_or(OffsetDateTime::UNIX_EPOCH));
let tm = get_str(&oi.user_defined, SUFFIX_REPLICATION_TIMESTAMP)
.map(|v| OffsetDateTime::parse(&v, &Rfc3339).unwrap_or(OffsetDateTime::UNIX_EPOCH));
let mut rstate = oi.replication_state();
rstate.replicate_decision_str = dsc.to_string();
let asz = oi.get_actual_size().unwrap_or_default();
crates/ecstore/src/bucket/replication/replication_resyncer.rs
+416 -68
View File
@@ -17,6 +17,7 @@ use crate::bucket::bucket_target_sys::{
AdvancedPutOptions, BucketTargetSys, PutObjectOptions, PutObjectPartOptions, RemoveObjectOptions, TargetClient,
};
use crate::bucket::metadata_sys;
use crate::bucket::msgp_decode::{read_msgp_ext8_time, skip_msgp_value, write_msgp_time};
use crate::bucket::replication::ResyncStatusType;
use crate::bucket::replication::replication_pool::GLOBAL_REPLICATION_STATS;
use crate::bucket::replication::{ObjectOpts, ReplicationConfigurationExt as _};
@@ -50,7 +51,7 @@ use http_body::Frame;
use http_body_util::StreamBody;
use regex::Regex;
use rustfs_filemeta::{
MrfReplicateEntry, REPLICATE_EXISTING, REPLICATE_EXISTING_DELETE, REPLICATION_RESET, ReplicateDecision, ReplicateObjectInfo,
MrfReplicateEntry, REPLICATE_EXISTING, REPLICATE_EXISTING_DELETE, ReplicateDecision, ReplicateObjectInfo,
ReplicateTargetDecision, ReplicatedInfos, ReplicatedTargetInfo, ReplicationAction, ReplicationState, ReplicationStatusType,
ReplicationType, ReplicationWorkerOperation, ResyncDecision, ResyncTargetDecision, VersionPurgeStatusType,
get_replication_state, parse_replicate_decision, replication_statuses_map, target_reset_header, version_purge_statuses_map,
@@ -58,8 +59,13 @@ use rustfs_filemeta::{
use rustfs_s3_common::EventName;
use rustfs_utils::http::{
AMZ_BUCKET_REPLICATION_STATUS, AMZ_OBJECT_TAGGING, AMZ_TAGGING_DIRECTIVE, CONTENT_ENCODING, HeaderExt as _,
RESERVED_METADATA_PREFIX, RESERVED_METADATA_PREFIX_LOWER, RUSTFS_REPLICATION_ACTUAL_OBJECT_SIZE,
RUSTFS_REPLICATION_RESET_STATUS, SSEC_ALGORITHM_HEADER, SSEC_KEY_HEADER, SSEC_KEY_MD5_HEADER, headers,
SSEC_ALGORITHM_HEADER, SSEC_KEY_HEADER, SSEC_KEY_MD5_HEADER, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP,
SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, SUFFIX_REPLICATION_RESET, SUFFIX_REPLICATION_RESET_ARN_PREFIX,
SUFFIX_REPLICATION_STATUS, SUFFIX_TAGGING_TIMESTAMP, headers,
};
use rustfs_utils::http::{
SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE, SUFFIX_REPLICATION_RESET_STATUS, SUFFIX_REPLICATION_SSEC_CRC, get_header_map, get_str,
has_internal_suffix, insert_header_map, insert_str, internal_key_strip_suffix_prefix, is_internal_key,
};
use rustfs_utils::path::path_join_buf;
use rustfs_utils::string::strings_has_prefix_fold;
@@ -69,7 +75,8 @@ use serde::Deserialize;
use serde::Serialize;
use std::any::Any;
use std::collections::HashMap;
use std::sync::Arc;
use std::io::{Cursor, Read};
use std::sync::{Arc, LazyLock};
use time::OffsetDateTime;
use time::format_description::well_known::Rfc3339;
use tokio::io::AsyncRead;
@@ -80,14 +87,29 @@ use tokio_util::io::ReaderStream;
use tokio_util::sync::CancellationToken;
use tracing::{error, info, instrument, warn};
const REPLICATION_DIR: &str = ".replication";
const RESYNC_FILE_NAME: &str = "resync.bin";
const RESYNC_META_FORMAT: u16 = 1;
const RESYNC_META_VERSION: u16 = 1;
pub(crate) const REPLICATION_DIR: &str = ".replication";
pub(crate) const RESYNC_FILE_NAME: &str = "resync.bin";
pub(crate) const RESYNC_META_FORMAT: u16 = 1;
pub(crate) const RESYNC_META_VERSION: u16 = 1;
const RESYNC_TIME_INTERVAL: TokioDuration = TokioDuration::from_secs(60);
const WIRE_ZERO_TIME_UNIX: i64 = -62_135_596_800;
static WIRE_ZERO_TIME: LazyLock<OffsetDateTime> =
LazyLock::new(|| OffsetDateTime::from_unix_timestamp(WIRE_ZERO_TIME_UNIX).unwrap_or(OffsetDateTime::UNIX_EPOCH));
static WARNED_MONITOR_UNINIT: std::sync::Once = std::sync::Once::new();
fn wire_time_or_default(value: Option<OffsetDateTime>) -> OffsetDateTime {
value.unwrap_or(*WIRE_ZERO_TIME)
}
fn normalize_wire_time(value: Option<OffsetDateTime>) -> Option<OffsetDateTime> {
match value {
Some(v) if v == *WIRE_ZERO_TIME || v == OffsetDateTime::UNIX_EPOCH => None,
other => other,
}
}
#[derive(Debug, Clone, Default)]
pub struct ResyncOpts {
pub bucket: String,
@@ -139,14 +161,199 @@ impl BucketReplicationResyncStatus {
}
pub fn marshal_msg(&self) -> Result<Vec<u8>> {
Ok(rmp_serde::to_vec(&self)?)
let mut wr = Vec::new();
rmp::encode::write_map_len(&mut wr, 4)?;
rmp::encode::write_str(&mut wr, "v")?;
rmp::encode::write_i32(&mut wr, i32::from(self.version))?;
rmp::encode::write_str(&mut wr, "brs")?;
rmp::encode::write_map_len(&mut wr, self.targets_map.len() as u32)?;
for (arn, status) in &self.targets_map {
rmp::encode::write_str(&mut wr, arn)?;
status.marshal_wire_msg(&mut wr)?;
}
rmp::encode::write_str(&mut wr, "id")?;
rmp::encode::write_i32(&mut wr, self.id)?;
rmp::encode::write_str(&mut wr, "lu")?;
write_msgp_time(&mut wr, wire_time_or_default(self.last_update))?;
Ok(wr)
}
pub fn unmarshal_msg(data: &[u8]) -> Result<Self> {
let mut rd = Cursor::new(data);
let mut out = Self::new();
let mut fields = rmp::decode::read_map_len(&mut rd)?;
while fields > 0 {
fields -= 1;
let key = read_msgp_str(&mut rd)?;
match key.as_str() {
"v" => {
let v: i32 = rmp::decode::read_int(&mut rd)?;
out.version = u16::try_from(v).map_err(|_| Error::other("invalid resync version"))?;
}
"brs" => {
let map_len = rmp::decode::read_map_len(&mut rd)?;
let mut targets = HashMap::with_capacity(map_len as usize);
for _ in 0..map_len {
let arn = read_msgp_str(&mut rd)?;
let status = TargetReplicationResyncStatus::unmarshal_wire_msg(&mut rd)?;
targets.insert(arn, status);
}
out.targets_map = targets;
}
"id" => {
out.id = rmp::decode::read_int::<i32, _>(&mut rd)?;
}
"lu" => {
out.last_update = normalize_wire_time(read_msgp_time_or_nil(&mut rd)?);
}
_ => skip_msgp_value(&mut rd)?,
}
}
Ok(out)
}
pub fn unmarshal_legacy_msg(data: &[u8]) -> Result<Self> {
Ok(rmp_serde::from_slice(data)?)
}
}
pub(crate) fn encode_resync_file(status: &BucketReplicationResyncStatus) -> Result<Vec<u8>> {
let payload = status.marshal_msg()?;
let mut data = Vec::with_capacity(4 + payload.len());
let mut major = [0u8; 2];
byteorder::LittleEndian::write_u16(&mut major, RESYNC_META_FORMAT);
data.extend_from_slice(&major);
let mut minor = [0u8; 2];
byteorder::LittleEndian::write_u16(&mut minor, RESYNC_META_VERSION);
data.extend_from_slice(&minor);
data.extend_from_slice(&payload);
Ok(data)
}
pub(crate) fn decode_resync_file(data: &[u8]) -> Result<BucketReplicationResyncStatus> {
if data.len() <= 4 {
return Err(Error::CorruptedFormat);
}
let mut major = [0u8; 2];
major.copy_from_slice(&data[0..2]);
if byteorder::LittleEndian::read_u16(&major) != RESYNC_META_FORMAT {
return Err(Error::CorruptedFormat);
}
let mut minor = [0u8; 2];
minor.copy_from_slice(&data[2..4]);
if byteorder::LittleEndian::read_u16(&minor) != RESYNC_META_VERSION {
return Err(Error::CorruptedFormat);
}
let status = match BucketReplicationResyncStatus::unmarshal_msg(&data[4..]) {
Ok(v) => v,
Err(_) => BucketReplicationResyncStatus::unmarshal_legacy_msg(&data[4..])?,
};
if status.version != RESYNC_META_VERSION {
return Err(Error::CorruptedFormat);
}
Ok(status)
}
impl TargetReplicationResyncStatus {
fn marshal_wire_msg(&self, wr: &mut Vec<u8>) -> Result<()> {
rmp::encode::write_map_len(wr, 11)?;
rmp::encode::write_str(wr, "st")?;
write_msgp_time(wr, wire_time_or_default(self.start_time))?;
rmp::encode::write_str(wr, "lst")?;
write_msgp_time(wr, wire_time_or_default(self.last_update))?;
rmp::encode::write_str(wr, "id")?;
rmp::encode::write_str(wr, &self.resync_id)?;
rmp::encode::write_str(wr, "rdt")?;
write_msgp_time(wr, wire_time_or_default(self.resync_before_date))?;
rmp::encode::write_str(wr, "rst")?;
rmp::encode::write_i32(wr, resync_status_to_i32(self.resync_status))?;
rmp::encode::write_str(wr, "fs")?;
rmp::encode::write_i64(wr, self.failed_size)?;
rmp::encode::write_str(wr, "frc")?;
rmp::encode::write_i64(wr, self.failed_count)?;
rmp::encode::write_str(wr, "rs")?;
rmp::encode::write_i64(wr, self.replicated_size)?;
rmp::encode::write_str(wr, "rrc")?;
rmp::encode::write_i64(wr, self.replicated_count)?;
rmp::encode::write_str(wr, "bkt")?;
rmp::encode::write_str(wr, &self.bucket)?;
rmp::encode::write_str(wr, "obj")?;
rmp::encode::write_str(wr, &self.object)?;
Ok(())
}
fn unmarshal_wire_msg<R: Read>(rd: &mut R) -> Result<Self> {
let mut out = Self::new();
let mut fields = rmp::decode::read_map_len(rd)?;
while fields > 0 {
fields -= 1;
let key = read_msgp_str(rd)?;
match key.as_str() {
"st" => out.start_time = normalize_wire_time(read_msgp_time_or_nil(rd)?),
"lst" => out.last_update = normalize_wire_time(read_msgp_time_or_nil(rd)?),
"id" => out.resync_id = read_msgp_str(rd)?,
"rdt" => out.resync_before_date = normalize_wire_time(read_msgp_time_or_nil(rd)?),
"rst" => {
let v: i32 = rmp::decode::read_int(rd)?;
out.resync_status = resync_status_from_i32(v)?;
}
"fs" => out.failed_size = rmp::decode::read_int(rd)?,
"frc" => out.failed_count = rmp::decode::read_int(rd)?,
"rs" => out.replicated_size = rmp::decode::read_int(rd)?,
"rrc" => out.replicated_count = rmp::decode::read_int(rd)?,
"bkt" => out.bucket = read_msgp_str(rd)?,
"obj" => out.object = read_msgp_str(rd)?,
_ => skip_msgp_value(rd)?,
}
}
Ok(out)
}
}
fn read_msgp_str<R: Read>(rd: &mut R) -> Result<String> {
let len = rmp::decode::read_str_len(rd)? as usize;
let mut buf = vec![0u8; len];
rd.read_exact(&mut buf)?;
Ok(String::from_utf8(buf)?)
}
fn read_msgp_time_or_nil<R: Read>(rd: &mut R) -> Result<Option<OffsetDateTime>> {
let marker = rmp::decode::read_marker(rd).map_err(|e| Error::other(format!("{e:?}")))?;
match marker {
rmp::Marker::Null => Ok(None),
rmp::Marker::Ext8 => Ok(Some(read_msgp_ext8_time(rd)?)),
other => Err(Error::other(format!("expected time ext or nil, got marker: {other:?}"))),
}
}
fn resync_status_to_i32(status: ResyncStatusType) -> i32 {
match status {
ResyncStatusType::NoResync => 0,
ResyncStatusType::ResyncPending => 1,
ResyncStatusType::ResyncCanceled => 2,
ResyncStatusType::ResyncStarted => 3,
ResyncStatusType::ResyncCompleted => 4,
ResyncStatusType::ResyncFailed => 5,
}
}
fn resync_status_from_i32(code: i32) -> Result<ResyncStatusType> {
match code {
0 => Ok(ResyncStatusType::NoResync),
1 => Ok(ResyncStatusType::ResyncPending),
2 => Ok(ResyncStatusType::ResyncCanceled),
3 => Ok(ResyncStatusType::ResyncStarted),
4 => Ok(ResyncStatusType::ResyncCompleted),
5 => Ok(ResyncStatusType::ResyncFailed),
_ => Err(Error::other(format!("invalid resync status code: {code}"))),
}
}
static RESYNC_WORKER_COUNT: usize = 10;
#[derive(Debug)]
@@ -617,7 +824,7 @@ pub async fn get_heal_replicate_object_info(oi: &ObjectInfo, rcfg: &ReplicationC
let keys_to_update: Vec<_> = user_defined
.iter()
.filter(|(k, _)| k.eq_ignore_ascii_case(format!("{RESERVED_METADATA_PREFIX_LOWER}{REPLICATION_RESET}").as_str()))
.filter(|(k, _)| has_internal_suffix(k, SUFFIX_REPLICATION_RESET))
.map(|(k, v)| (k.clone(), v.clone()))
.collect();
@@ -695,19 +902,7 @@ pub async fn get_heal_replicate_object_info(oi: &ObjectInfo, rcfg: &ReplicationC
}
async fn save_resync_status<S: StorageAPI>(bucket: &str, status: &BucketReplicationResyncStatus, api: Arc<S>) -> Result<()> {
let buf = status.marshal_msg()?;
let mut data = Vec::new();
let mut major = [0u8; 2];
byteorder::LittleEndian::write_u16(&mut major, RESYNC_META_FORMAT);
data.extend_from_slice(&major);
let mut minor = [0u8; 2];
byteorder::LittleEndian::write_u16(&mut minor, RESYNC_META_VERSION);
data.extend_from_slice(&minor);
data.extend_from_slice(&buf);
let data = encode_resync_file(status)?;
let config_file = path_join_buf(&[BUCKET_META_PREFIX, bucket, REPLICATION_DIR, RESYNC_FILE_NAME]);
save_config(api, &config_file, data).await?;
@@ -900,8 +1095,8 @@ pub fn resync_target(
let rs = oi
.user_defined
.get(target_reset_header(arn).as_str())
.or(oi.user_defined.get(RUSTFS_REPLICATION_RESET_STATUS))
.map(|s| s.to_string());
.cloned()
.or_else(|| get_header_map(&oi.user_defined, SUFFIX_REPLICATION_RESET_STATUS));
let mut dec = ResyncTargetDecision::default();
@@ -1132,15 +1327,7 @@ impl ObjectInfoExt for ObjectInfo {
.user_defined
.iter()
.filter_map(|(k, v)| {
if k.starts_with(&format!("{RESERVED_METADATA_PREFIX_LOWER}-{REPLICATION_RESET}")) {
Some((
k.trim_start_matches(&format!("{RESERVED_METADATA_PREFIX_LOWER}-{REPLICATION_RESET}"))
.to_string(),
v.clone(),
))
} else {
None
}
internal_key_strip_suffix_prefix(k, SUFFIX_REPLICATION_RESET_ARN_PREFIX).map(|arn| (arn, v.clone()))
})
.collect(),
..Default::default()
@@ -1893,7 +2080,7 @@ pub async fn replicate_object<S: StorageAPI>(roi: ReplicateObjectInfo, storage:
if roi.replication_status_internal != new_replication_internal || rinfos.replication_resynced() {
let mut eval_metadata = HashMap::new();
if let Some(ref s) = new_replication_internal {
eval_metadata.insert(format!("{RESERVED_METADATA_PREFIX_LOWER}replication-status"), s.clone());
insert_str(&mut eval_metadata, SUFFIX_REPLICATION_STATUS, s.clone());
}
let popts = ObjectOptions {
version_id: roi.version_id.map(|v| v.to_string()),
@@ -2549,8 +2736,6 @@ static VALID_SSE_REPLICATION_HEADERS: &[(&str, &str)] = &[
("X-Rustfs-Internal-Actual-Object-Size", "X-Rustfs-Replication-Actual-Object-Size"),
];
const REPLICATION_SSEC_CHECKSUM_HEADER: &str = "X-Rustfs-Replication-Ssec-Crc";
fn is_valid_sse_header(k: &str) -> Option<&str> {
VALID_SSE_REPLICATION_HEADERS
.iter()
@@ -2574,7 +2759,7 @@ fn put_replication_opts(sc: &str, object_info: &ObjectInfo) -> Result<(PutObject
// In case of SSE-C objects copy the allowed internal headers as well
if !is_ssec || !has_valid_sse_header {
if strings_has_prefix_fold(k, RESERVED_METADATA_PREFIX) {
if is_internal_key(k) {
continue;
}
if is_standard_header(k) {
@@ -2598,7 +2783,7 @@ fn put_replication_opts(sc: &str, object_info: &ObjectInfo) -> Result<(PutObject
// Add encrypted CRC to metadata for SSE-C objects
if is_ssec {
let encoded = BASE64_STANDARD.encode(checksum_data);
meta.insert(REPLICATION_SSEC_CHECKSUM_HEADER.to_string(), encoded);
insert_header_map(&mut meta, SUFFIX_REPLICATION_SSEC_CRC, encoded);
} else {
// Get checksum metadata for non-SSE-C objects
let (cs_meta, is_mp) = object_info.decrypt_checksums(0, &HeaderMap::new())?;
@@ -2658,11 +2843,8 @@ fn put_replication_opts(sc: &str, object_info: &ObjectInfo) -> Result<(PutObject
if !tags.is_empty() {
put_op.user_tags = tags;
// set tag timestamp in opts
put_op.internal.tagging_timestamp = if let Some(ts) = object_info
.user_defined
.get(&format!("{RESERVED_METADATA_PREFIX_LOWER}tagging-timestamp"))
{
OffsetDateTime::parse(ts, &Rfc3339)
put_op.internal.tagging_timestamp = if let Some(ts) = get_str(&object_info.user_defined, SUFFIX_TAGGING_TIMESTAMP) {
OffsetDateTime::parse(&ts, &Rfc3339)
.map_err(|e| Error::other(format!("Failed to parse tagging timestamp: {}", e)))?
} else {
object_info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH)
@@ -2694,28 +2876,24 @@ fn put_replication_opts(sc: &str, object_info: &ObjectInfo) -> Result<(PutObject
put_op.retain_until_date =
OffsetDateTime::parse(v, &Rfc3339).map_err(|e| Error::other(format!("Failed to parse retain until date: {}", e)))?;
// set retention timestamp in opts
put_op.internal.retention_timestamp = if let Some(v) = object_info
.user_defined
.get(&format!("{RESERVED_METADATA_PREFIX_LOWER}objectlock-retention-timestamp"))
{
OffsetDateTime::parse(v, &Rfc3339).unwrap_or(OffsetDateTime::UNIX_EPOCH)
} else {
object_info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH)
};
put_op.internal.retention_timestamp =
if let Some(v) = get_str(&object_info.user_defined, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP) {
OffsetDateTime::parse(&v, &Rfc3339).unwrap_or(OffsetDateTime::UNIX_EPOCH)
} else {
object_info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH)
};
}
if let Some(v) = lk_map.lookup(AMZ_OBJECT_LOCK_LEGAL_HOLD) {
let hold = v.to_uppercase();
put_op.legalhold = Some(ObjectLockLegalHoldStatus::from(hold.as_str()));
// set legalhold timestamp in opts
put_op.internal.legalhold_timestamp = if let Some(v) = object_info
.user_defined
.get(&format!("{RESERVED_METADATA_PREFIX_LOWER}objectlock-legalhold-timestamp"))
{
OffsetDateTime::parse(v, &Rfc3339).unwrap_or(OffsetDateTime::UNIX_EPOCH)
} else {
object_info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH)
};
put_op.internal.legalhold_timestamp =
if let Some(v) = get_str(&object_info.user_defined, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP) {
OffsetDateTime::parse(&v, &Rfc3339).unwrap_or(OffsetDateTime::UNIX_EPOCH)
} else {
object_info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH)
};
}
// Handle SSE-S3 encryption
@@ -2736,7 +2914,7 @@ fn put_replication_opts(sc: &str, object_info: &ObjectInfo) -> Result<(PutObject
// If KMS key ID replication is enabled (as by default)
// we include the object's KMS key ID. In any case, we
// always set the SSE-KMS header. If no KMS key ID is
// specified, MinIO is supposed to use whatever default
// specified, the server uses the default applicable
// config applies on the site or bucket.
// TODO: Implement SSE-KMS support with key ID replication
// let key_id = if kms::replicate_key_id() {
@@ -2859,13 +3037,10 @@ async fn replicate_object_with_multipart<S: StorageAPI>(ctx: MultipartReplicatio
let mut user_metadata = HashMap::new();
user_metadata.insert(
RUSTFS_REPLICATION_ACTUAL_OBJECT_SIZE.to_string(),
object_info
.user_defined
.get(&format!("{RESERVED_METADATA_PREFIX}actual-size"))
.map(|v| v.to_string())
.unwrap_or_default(),
insert_header_map(
&mut user_metadata,
SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE,
rustfs_utils::http::get_str(&object_info.user_defined, rustfs_utils::http::SUFFIX_ACTUAL_SIZE).unwrap_or_default(),
);
cli.complete_multipart_upload(
@@ -2994,6 +3169,9 @@ fn get_replication_action(oi1: &ObjectInfo, oi2: &HeadObjectOutput, op_type: Rep
#[cfg(test)]
mod tests {
use super::*;
use crate::bucket::msgp_decode::write_msgp_time;
use std::collections::HashMap;
use time::OffsetDateTime;
use uuid::Uuid;
#[test]
@@ -3010,6 +3188,176 @@ mod tests {
assert!(part_range_spec_from_actual_size(0, -1).is_err());
}
#[test]
fn test_unmarshal_resync_payload() {
let start = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid ts");
let last = OffsetDateTime::from_unix_timestamp(1_700_000_123).expect("valid ts");
let before = OffsetDateTime::from_unix_timestamp(1_699_000_000).expect("valid ts");
let bucket_last = OffsetDateTime::from_unix_timestamp(1_700_111_111).expect("valid ts");
let mut payload = Vec::new();
rmp::encode::write_map_len(&mut payload, 4).expect("write map");
rmp::encode::write_str(&mut payload, "v").expect("write key");
rmp::encode::write_i32(&mut payload, 1).expect("write version");
rmp::encode::write_str(&mut payload, "brs").expect("write key");
rmp::encode::write_map_len(&mut payload, 1).expect("write target map");
rmp::encode::write_str(&mut payload, "arn:replication::1:dest").expect("write arn");
rmp::encode::write_map_len(&mut payload, 11).expect("write target");
rmp::encode::write_str(&mut payload, "st").expect("write key");
write_msgp_time(&mut payload, start).expect("write time");
rmp::encode::write_str(&mut payload, "lst").expect("write key");
write_msgp_time(&mut payload, last).expect("write time");
rmp::encode::write_str(&mut payload, "id").expect("write key");
rmp::encode::write_str(&mut payload, "resync-1").expect("write id");
rmp::encode::write_str(&mut payload, "rdt").expect("write key");
write_msgp_time(&mut payload, before).expect("write time");
rmp::encode::write_str(&mut payload, "rst").expect("write key");
rmp::encode::write_i32(&mut payload, 3).expect("write status");
rmp::encode::write_str(&mut payload, "fs").expect("write key");
rmp::encode::write_i64(&mut payload, 11).expect("write fs");
rmp::encode::write_str(&mut payload, "frc").expect("write key");
rmp::encode::write_i64(&mut payload, 2).expect("write frc");
rmp::encode::write_str(&mut payload, "rs").expect("write key");
rmp::encode::write_i64(&mut payload, 101).expect("write rs");
rmp::encode::write_str(&mut payload, "rrc").expect("write key");
rmp::encode::write_i64(&mut payload, 9).expect("write rrc");
rmp::encode::write_str(&mut payload, "bkt").expect("write key");
rmp::encode::write_str(&mut payload, "bucket-a").expect("write bucket");
rmp::encode::write_str(&mut payload, "obj").expect("write key");
rmp::encode::write_str(&mut payload, "object-a").expect("write obj");
rmp::encode::write_str(&mut payload, "id").expect("write key");
rmp::encode::write_i32(&mut payload, 42).expect("write id");
rmp::encode::write_str(&mut payload, "lu").expect("write key");
write_msgp_time(&mut payload, bucket_last).expect("write lu");
let got = BucketReplicationResyncStatus::unmarshal_msg(&payload).expect("decode");
assert_eq!(got.version, 1);
assert_eq!(got.id, 42);
assert_eq!(got.last_update, Some(bucket_last));
let tgt = got.targets_map.get("arn:replication::1:dest").expect("target exists");
assert_eq!(tgt.resync_id, "resync-1");
assert_eq!(tgt.resync_status, ResyncStatusType::ResyncStarted);
assert_eq!(tgt.bucket, "bucket-a");
assert_eq!(tgt.object, "object-a");
assert_eq!(tgt.start_time, Some(start));
assert_eq!(tgt.last_update, Some(last));
assert_eq!(tgt.resync_before_date, Some(before));
}
#[test]
fn test_unmarshal_legacy_resync_payload() {
let mut status = BucketReplicationResyncStatus::new();
status.id = 7;
status.version = 1;
status.last_update = Some(OffsetDateTime::from_unix_timestamp(1_700_222_222).expect("valid ts"));
status.targets_map = HashMap::from([(
"legacy-arn".to_string(),
TargetReplicationResyncStatus {
resync_id: "legacy-1".to_string(),
resync_status: ResyncStatusType::ResyncCompleted,
..Default::default()
},
)]);
let old_payload = rmp_serde::to_vec(&status).expect("legacy encode");
let got = BucketReplicationResyncStatus::unmarshal_legacy_msg(&old_payload).expect("legacy decode");
assert_eq!(got.id, 7);
assert_eq!(got.version, 1);
assert_eq!(got.targets_map["legacy-arn"].resync_id, "legacy-1");
assert_eq!(got.targets_map["legacy-arn"].resync_status, ResyncStatusType::ResyncCompleted);
}
#[test]
fn test_resync_file_roundtrip_wire_format() {
let mut status = BucketReplicationResyncStatus::new();
status.id = 19;
status.last_update = Some(OffsetDateTime::from_unix_timestamp(1_700_333_333).expect("valid ts"));
status.targets_map = HashMap::from([(
"arn:replication::1:dest".to_string(),
TargetReplicationResyncStatus {
resync_id: "wire-1".to_string(),
resync_status: ResyncStatusType::ResyncStarted,
replicated_count: 5,
..Default::default()
},
)]);
let bytes = encode_resync_file(&status).expect("encode file");
assert_eq!(&bytes[0..2], &RESYNC_META_FORMAT.to_le_bytes());
assert_eq!(&bytes[2..4], &RESYNC_META_VERSION.to_le_bytes());
let got = decode_resync_file(&bytes).expect("decode file");
assert_eq!(got.version, RESYNC_META_VERSION);
assert_eq!(got.id, 19);
assert_eq!(got.targets_map["arn:replication::1:dest"].resync_id, "wire-1");
assert_eq!(got.targets_map["arn:replication::1:dest"].replicated_count, 5);
}
#[test]
fn test_resync_file_decodes_legacy_payload() {
let mut status = BucketReplicationResyncStatus::new();
status.id = 7;
status.version = RESYNC_META_VERSION;
status.targets_map = HashMap::from([(
"legacy-arn".to_string(),
TargetReplicationResyncStatus {
resync_id: "legacy-v1".to_string(),
resync_status: ResyncStatusType::ResyncCompleted,
..Default::default()
},
)]);
let legacy_payload = rmp_serde::to_vec(&status).expect("legacy encode");
let mut file_bytes = Vec::new();
file_bytes.extend_from_slice(&RESYNC_META_FORMAT.to_le_bytes());
file_bytes.extend_from_slice(&RESYNC_META_VERSION.to_le_bytes());
file_bytes.extend_from_slice(&legacy_payload);
let got = decode_resync_file(&file_bytes).expect("decode legacy");
assert_eq!(got.id, 7);
assert_eq!(got.targets_map["legacy-arn"].resync_id, "legacy-v1");
assert_eq!(got.targets_map["legacy-arn"].resync_status, ResyncStatusType::ResyncCompleted);
}
#[test]
fn test_resync_none_time_encodes_as_wire_zero_and_decodes_to_none() {
let wire_zero = OffsetDateTime::from_unix_timestamp(WIRE_ZERO_TIME_UNIX).expect("valid wire zero timestamp");
let mut with_none = BucketReplicationResyncStatus::new();
with_none.id = 77;
with_none.targets_map = HashMap::from([(
"arn:replication::1:dest".to_string(),
TargetReplicationResyncStatus {
resync_id: "wire-none".to_string(),
resync_status: ResyncStatusType::ResyncStarted,
replicated_count: 1,
..Default::default()
},
)]);
let mut with_zero = with_none.clone();
with_zero.last_update = Some(wire_zero);
if let Some(target) = with_zero.targets_map.get_mut("arn:replication::1:dest") {
target.start_time = Some(wire_zero);
target.last_update = Some(wire_zero);
target.resync_before_date = Some(wire_zero);
}
let encoded_none = encode_resync_file(&with_none).expect("encode with none");
let encoded_zero = encode_resync_file(&with_zero).expect("encode with zero");
assert_eq!(encoded_none, encoded_zero);
let decoded = decode_resync_file(&encoded_none).expect("decode");
let target = decoded
.targets_map
.get("arn:replication::1:dest")
.expect("target should exist");
assert_eq!(decoded.last_update, None);
assert_eq!(target.start_time, None);
assert_eq!(target.last_update, None);
assert_eq!(target.resync_before_date, None);
}
#[test]
fn test_heal_should_use_check_replicate_delete_failed_non_delete_marker() {
let oi = ObjectInfo {
crates/ecstore/src/bucket/utils.rs
+2 -2
View File
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::disk::RUSTFS_META_BUCKET;
use crate::disk::{MIGRATING_META_BUCKET, RUSTFS_META_BUCKET};
use crate::error::{Error, Result, StorageError};
use regex::Regex;
use rustfs_utils::path::SLASH_SEPARATOR;
@@ -20,7 +20,7 @@ use s3s::xml;
use tracing::instrument;
pub fn is_meta_bucketname(name: &str) -> bool {
name.starts_with(RUSTFS_META_BUCKET)
name.starts_with(RUSTFS_META_BUCKET) || name.starts_with(MIGRATING_META_BUCKET)
}
lazy_static::lazy_static! {
crates/ecstore/src/config/com.rs
+331 -8
View File
@@ -13,16 +13,17 @@
// limitations under the License.
use crate::config::{Config, GLOBAL_STORAGE_CLASS, storageclass};
use crate::disk::RUSTFS_META_BUCKET;
use crate::disk::{MIGRATING_META_BUCKET, RUSTFS_META_BUCKET};
use crate::error::{Error, Result};
use crate::store_api::{ObjectInfo, ObjectOptions, PutObjReader, StorageAPI};
use http::HeaderMap;
use rustfs_config::DEFAULT_DELIMITER;
use rustfs_config::{DEFAULT_DELIMITER, RUSTFS_REGION};
use rustfs_utils::path::SLASH_SEPARATOR;
use std::collections::HashSet;
use serde_json::{Map, Value};
use std::collections::{HashMap, HashSet};
use std::sync::Arc;
use std::sync::LazyLock;
use tracing::{error, instrument, warn};
use tracing::{debug, error, info, instrument, warn};
pub const CONFIG_PREFIX: &str = "config";
const CONFIG_FILE: &str = "config.json";
@@ -136,6 +137,226 @@ fn get_config_file() -> String {
format!("{CONFIG_PREFIX}{SLASH_SEPARATOR}{CONFIG_FILE}")
}
fn storage_class_kvs_mut(cfg: &mut Config) -> &mut crate::config::KVS {
let sub_cfg = cfg.0.entry(STORAGE_CLASS_SUB_SYS.to_string()).or_insert_with(|| {
let mut section = HashMap::new();
section.insert(DEFAULT_DELIMITER.to_string(), storageclass::DEFAULT_KVS.clone());
section
});
sub_cfg
.entry(DEFAULT_DELIMITER.to_string())
.or_insert_with(|| storageclass::DEFAULT_KVS.clone())
}
fn parse_storage_class_value(value: &Value) -> Option<String> {
match value {
Value::String(v) => Some(v.trim().to_string()),
Value::Object(m) => m
.get("parity")
.and_then(Value::as_u64)
.map(|parity| if parity == 0 { String::new() } else { format!("EC:{parity}") }),
_ => None,
}
}
fn parse_inline_block_value(value: &Value) -> Option<String> {
match value {
Value::String(v) if !v.trim().is_empty() => Some(v.trim().to_string()),
Value::Number(v) => Some(v.to_string()),
_ => None,
}
}
fn apply_external_storage_class_map(cfg: &mut Config, root: &Map<String, Value>) -> bool {
let sc = root.get("storageclass").or_else(|| root.get("storage_class"));
let Some(Value::Object(sc_obj)) = sc else {
return false;
};
let mut applied = false;
let kvs = storage_class_kvs_mut(cfg);
if let Some(v) = sc_obj.get("standard").and_then(parse_storage_class_value) {
kvs.insert(storageclass::CLASS_STANDARD.to_string(), v);
applied = true;
}
if let Some(v) = sc_obj.get("rrs").and_then(parse_storage_class_value) {
kvs.insert(storageclass::CLASS_RRS.to_string(), v);
applied = true;
}
if let Some(Value::String(v)) = sc_obj.get("optimize")
&& !v.trim().is_empty()
{
kvs.insert(storageclass::OPTIMIZE.to_string(), v.clone());
applied = true;
}
if let Some(v) = sc_obj.get("inline_block").and_then(parse_inline_block_value) {
kvs.insert(storageclass::INLINE_BLOCK.to_string(), v);
applied = true;
}
applied
}
fn decode_server_config_blob(data: &[u8]) -> Result<Config> {
if let Ok(cfg) = Config::unmarshal(data) {
return Ok(cfg);
}
let value: Value = serde_json::from_slice(data)?;
let Value::Object(root) = value else {
return Err(Error::other("unrecognized external server config shape"));
};
let mut cfg = Config::new();
let has_storage = apply_external_storage_class_map(&mut cfg, &root);
let has_header = root.contains_key("version") || root.contains_key("region") || root.contains_key("credential");
if !has_storage && !has_header {
return Err(Error::other("unrecognized external server config shape"));
}
Ok(cfg)
}
fn parse_object_seed(data: &[u8]) -> Option<Map<String, Value>> {
let value: Value = serde_json::from_slice(data).ok()?;
value.as_object().cloned()
}
fn build_storageclass_object(cfg: &Config) -> Map<String, Value> {
let kvs = cfg.get_value(STORAGE_CLASS_SUB_SYS, DEFAULT_DELIMITER).unwrap_or_default();
let mut sc_obj = Map::new();
sc_obj.insert(
"standard".to_string(),
Value::String(kvs.lookup(storageclass::CLASS_STANDARD).unwrap_or_default()),
);
sc_obj.insert("rrs".to_string(), Value::String(kvs.lookup(storageclass::CLASS_RRS).unwrap_or_default()));
let optimize = kvs
.lookup(storageclass::OPTIMIZE)
.filter(|v| !v.trim().is_empty())
.unwrap_or_else(|| "availability".to_string());
sc_obj.insert("optimize".to_string(), Value::String(optimize));
if let Some(v) = kvs.lookup(storageclass::INLINE_BLOCK).filter(|v| !v.trim().is_empty()) {
sc_obj.insert("inline_block".to_string(), Value::String(v));
}
sc_obj
}
fn encode_server_config_blob(cfg: &Config, seed: Option<&[u8]>) -> Result<Vec<u8>> {
let mut root = seed.and_then(parse_object_seed).unwrap_or_default();
if !matches!(root.get("version"), Some(Value::String(v)) if !v.trim().is_empty()) {
root.insert("version".to_string(), Value::String("33".to_string()));
}
if !matches!(root.get("region"), Some(Value::String(v)) if !v.trim().is_empty()) {
root.insert("region".to_string(), Value::String(RUSTFS_REGION.to_string()));
}
let mut sc_obj = match root.remove("storageclass") {
Some(Value::Object(v)) => v,
_ => Map::new(),
};
for (k, v) in build_storageclass_object(cfg) {
sc_obj.insert(k, v);
}
root.insert("storageclass".to_string(), Value::Object(sc_obj));
root.remove("storage_class");
Ok(serde_json::to_vec(&Value::Object(root))?)
}
fn is_standard_object_server_config(data: &[u8]) -> bool {
let Ok(value) = serde_json::from_slice::<Value>(data) else {
return false;
};
let Value::Object(root) = value else {
return false;
};
matches!(root.get("version"), Some(Value::String(v)) if !v.trim().is_empty())
&& matches!(root.get("storageclass"), Some(Value::Object(_)))
&& !root.contains_key("storage_class")
}
fn configs_semantically_equal(lhs: &Config, rhs: &Config) -> bool {
build_storageclass_object(lhs) == build_storageclass_object(rhs)
}
fn is_object_not_found(err: &Error) -> bool {
*err == Error::FileNotFound || matches!(err, Error::ObjectNotFound(_, _) | Error::BucketNotFound(_))
}
pub async fn try_migrate_server_config<S: StorageAPI>(api: Arc<S>) {
let config_file = get_config_file();
match api
.get_object_info(RUSTFS_META_BUCKET, &config_file, &ObjectOptions::default())
.await
{
Ok(_) => {
debug!("server config already exists in RustFS metadata bucket, skip migration");
return;
}
Err(err) if is_object_not_found(&err) => {}
Err(err) => {
warn!("check target server config failed, skip migration: {:?}", err);
return;
}
}
let opts = ObjectOptions {
max_parity: true,
no_lock: true,
..Default::default()
};
let mut rd = match api
.get_object_reader(MIGRATING_META_BUCKET, &config_file, None, HeaderMap::new(), &opts)
.await
{
Ok(v) => v,
Err(err) => {
if !is_object_not_found(&err) {
warn!("read legacy server config failed: {:?}", err);
}
return;
}
};
let data = match rd.read_all().await {
Ok(v) if !v.is_empty() => v,
Ok(_) => {
debug!("legacy server config is empty, skip migration");
return;
}
Err(err) => {
warn!("read legacy server config body failed: {:?}", err);
return;
}
};
let cfg = match decode_server_config_blob(&data) {
Ok(v) => v,
Err(err) => {
warn!("legacy server config format is incompatible, skip migration: {:?}", err);
return;
}
};
let normalized = match encode_server_config_blob(&cfg, Some(&data)) {
Ok(v) => v,
Err(err) => {
warn!("serialize migrated server config failed, skip migration: {:?}", err);
return;
}
};
match save_config(api, &config_file, normalized).await {
Ok(()) => {
info!("Migrated compatible server config from legacy metadata bucket");
}
Err(err) => {
warn!("write migrated server config failed: {:?}", err);
}
}
}
/// Handle the situation where the configuration file does not exist, create and save a new configuration
async fn handle_missing_config<S: StorageAPI>(api: Arc<S>, context: &str) -> Result<Config> {
warn!("Configuration not found ({}): Start initializing new configuration", context);
@@ -171,7 +392,7 @@ async fn read_server_config<S: StorageAPI>(api: Arc<S>, data: &[u8]) -> Result<C
match read_config(api.clone(), &config_file).await {
Ok(cfg_data) => {
// TODO: decrypt
let cfg = Config::unmarshal(&cfg_data)?;
let cfg = decode_server_config_blob(&cfg_data)?;
return Ok(cfg.merge());
}
Err(Error::ConfigNotFound) => return handle_missing_config(api, "Read alternate configuration").await,
@@ -180,14 +401,35 @@ async fn read_server_config<S: StorageAPI>(api: Arc<S>, data: &[u8]) -> Result<C
}
// Process non-empty configuration data
let cfg = Config::unmarshal(data)?;
let cfg = decode_server_config_blob(data)?;
Ok(cfg.merge())
}
pub async fn save_server_config<S: StorageAPI>(api: Arc<S>, cfg: &Config) -> Result<()> {
let data = cfg.marshal()?;
let config_file = get_config_file();
let existing = match read_config(api.clone(), &config_file).await {
Ok(v) => Some(v),
Err(Error::ConfigNotFound) => None,
Err(err) => {
warn!("read existing server config before save failed, continue with clean output: {:?}", err);
None
}
};
if let Some(current) = existing.as_deref()
&& is_standard_object_server_config(current)
&& let Ok(decoded_current) = decode_server_config_blob(current)
&& configs_semantically_equal(&decoded_current, cfg)
{
debug!("server config unchanged and already in standard object shape, skip write");
return Ok(());
}
let data = encode_server_config_blob(cfg, existing.as_deref())?;
if existing.as_deref().is_some_and(|current| current == data.as_slice()) {
debug!("server config bytes unchanged after encode, skip write");
return Ok(());
}
save_config(api, &config_file, data).await
}
@@ -232,3 +474,84 @@ async fn apply_dynamic_config_for_sub_sys<S: StorageAPI>(cfg: &mut Config, api:
Ok(())
}
#[cfg(test)]
mod tests {
use super::{
configs_semantically_equal, decode_server_config_blob, encode_server_config_blob, is_standard_object_server_config,
storage_class_kvs_mut,
};
use crate::config::Config;
use serde_json::Value;
#[test]
fn test_decode_server_config_accepts_legacy_hidden_if_empty_alias() {
let input = r#"{"storage_class":{"_":[{"key":"standard","value":"EC:2","hiddenIfEmpty":true}]}}"#;
let cfg = decode_server_config_blob(input.as_bytes()).expect("decode should succeed");
let kvs = cfg.get_value("storage_class", "_").expect("storage_class should exist");
assert!(kvs.0[0].hidden_if_empty);
}
#[test]
fn test_decode_server_config_accepts_missing_hidden_if_empty() {
let input = r#"{"storage_class":{"_":[{"key":"standard","value":"EC:2"}]}}"#;
let cfg = decode_server_config_blob(input.as_bytes()).expect("decode should succeed");
let kvs = cfg.get_value("storage_class", "_").expect("storage_class should exist");
assert!(!kvs.0[0].hidden_if_empty);
}
#[test]
fn test_decode_server_config_accepts_v33_object_shape() {
let input = r#"{
"version":"33",
"credential":{"accessKey":"test","secretKey":"testtesttest"},
"region":"us-east-1",
"worm":"off",
"storageclass":{"standard":"EC:2","rrs":"EC:1"},
"notify":{},
"logger":{},
"compress":{"enabled":false},
"openid":{},
"policy":{"opa":{}},
"ldapserverconfig":{}
}"#;
let cfg = decode_server_config_blob(input.as_bytes()).expect("decode should succeed");
let kvs = cfg.get_value("storage_class", "_").expect("storage_class should exist");
assert_eq!(kvs.get("standard"), "EC:2");
assert_eq!(kvs.get("rrs"), "EC:1");
assert_eq!(kvs.get("optimize"), "availability");
}
#[test]
fn test_encode_server_config_writes_external_object_shape() {
let mut cfg = Config::new();
let kvs = storage_class_kvs_mut(&mut cfg);
kvs.insert("standard".to_string(), "EC:2".to_string());
kvs.insert("rrs".to_string(), "EC:1".to_string());
let out = encode_server_config_blob(&cfg, None).expect("encode should succeed");
let v: Value = serde_json::from_slice(&out).expect("output should be json");
assert!(v.get("version").is_some(), "external object should have version");
assert!(v.get("storageclass").is_some(), "external object should have storageclass");
assert!(v.get("storage_class").is_none(), "should not write rustfs map shape");
}
#[test]
fn test_is_standard_object_server_config_detection() {
let external = br#"{"version":"33","storageclass":{"standard":"EC:2","rrs":"EC:1"}}"#;
assert!(is_standard_object_server_config(external));
let legacy = br#"{"storage_class":{"_":[{"key":"standard","value":"EC:2"}]}}"#;
assert!(!is_standard_object_server_config(legacy));
}
#[test]
fn test_configs_semantically_equal_for_equivalent_shapes() {
let external = br#"{"version":"33","storageclass":{"standard":"EC:2","rrs":"EC:1","optimize":"availability"}}"#;
let legacy = br#"{"storage_class":{"_":[{"key":"standard","value":"EC:2"},{"key":"rrs","value":"EC:1"},{"key":"optimize","value":"availability"}]}}"#;
let lhs = decode_server_config_blob(external).expect("decode external");
let rhs = decode_server_config_blob(legacy).expect("decode legacy");
assert!(configs_semantically_equal(&lhs, &rhs));
}
}
crates/ecstore/src/config/mod.rs
+5
View File
@@ -75,10 +75,15 @@ pub async fn init_global_config_sys(api: Arc<ECStore>) -> Result<()> {
GLOBAL_CONFIG_SYS.init(api).await
}
pub async fn try_migrate_server_config(api: Arc<ECStore>) {
com::try_migrate_server_config(api).await
}
#[derive(Debug, Deserialize, Serialize, Clone)]
pub struct KV {
pub key: String,
pub value: String,
#[serde(default, alias = "hiddenIfEmpty")]
pub hidden_if_empty: bool,
}
crates/ecstore/src/disk/local.rs
+7 -3
View File
@@ -697,7 +697,6 @@ impl LocalDisk {
match self.read_metadata_with_dmtime(meta_path).await {
Ok(res) => Ok(res),
Err(err) => {
warn!("read_raw: error: {:?}", err);
if err == Error::FileNotFound
&& !skip_access_checks(volume_dir.as_ref().to_string_lossy().to_string().as_str())
&& let Err(e) = access(volume_dir.as_ref()).await
@@ -1493,6 +1492,12 @@ impl DiskAPI for LocalDisk {
let erasure = &fi.erasure;
for (i, part) in fi.parts.iter().enumerate() {
let checksum_info = erasure.get_checksum_info(part.number);
let checksum_algo =
if fi.uses_legacy_checksum && checksum_info.algorithm == rustfs_utils::HashAlgorithm::HighwayHash256S {
rustfs_utils::HashAlgorithm::HighwayHash256SLegacy
} else {
checksum_info.algorithm
};
let part_path = self.get_object_path(
volume,
path_join_buf(&[
@@ -1506,7 +1511,7 @@ impl DiskAPI for LocalDisk {
.bitrot_verify(
&part_path,
erasure.shard_file_size(part.size as i64) as usize,
checksum_info.algorithm,
checksum_algo,
&checksum_info.hash,
erasure.shard_size(),
)
@@ -2058,7 +2063,6 @@ impl DiskAPI for LocalDisk {
let search_version_id = fi.version_id.or(Some(Uuid::nil()));
// Check if there's an existing version with the same version_id that has a data_dir to clean up
// Note: For non-versioned buckets, fi.version_id is None, but in xl.meta it's stored as Some(Uuid::nil())
let has_old_data_dir = {
xlmeta.find_version(search_version_id).ok().and_then(|(_, ver)| {
// shard_count == 0 means no other version shares this data_dir
crates/ecstore/src/disk/mod.rs
+1
View File
@@ -23,6 +23,7 @@ pub mod local;
pub mod os;
pub const RUSTFS_META_BUCKET: &str = ".rustfs.sys";
pub const MIGRATING_META_BUCKET: &str = ".minio.sys";
pub const RUSTFS_META_MULTIPART_BUCKET: &str = ".rustfs.sys/multipart";
pub const RUSTFS_META_TMP_BUCKET: &str = ".rustfs.sys/tmp";
pub const RUSTFS_META_TMP_DELETED_BUCKET: &str = ".rustfs.sys/tmp/.trash";
crates/ecstore/src/erasure_coding/bitrot.rs
+1 -1
View File
@@ -173,7 +173,7 @@ where
}
pub fn bitrot_shard_file_size(size: usize, shard_size: usize, algo: HashAlgorithm) -> usize {
if algo != HashAlgorithm::HighwayHash256S {
if algo != HashAlgorithm::HighwayHash256S && algo != HashAlgorithm::HighwayHash256SLegacy {
return size;
}
size.div_ceil(shard_size) * algo.size() + size
crates/ecstore/src/erasure_coding/erasure.rs
+274 -117
View File
@@ -12,31 +12,12 @@
// See the License for the specific language governing permissions and
// limitations under the License.
//! Erasure coding implementation using Reed-Solomon SIMD backend.
//! Erasure coding implementation using reed-solomon-erasure (GF(2^8)).
//! Supports legacy (reed-solomon-simd) for reading/healing old-version files.
//!
//! This module provides erasure coding functionality with high-performance SIMD
//! Reed-Solomon implementation:
//!
//! ## Reed-Solomon Implementation
//!
//! ### SIMD Mode (Only)
//! - **Performance**: Uses SIMD optimization for high-performance encoding/decoding
//! - **Compatibility**: Works with any shard size through SIMD implementation
//! - **Reliability**: High-performance SIMD implementation for large data processing
//! - **Use case**: Optimized for maximum performance in large data processing scenarios
//!
//! ## Example
//!
//! ```ignore
//! use rustfs_ecstore::erasure_coding::Erasure;
//!
//! let erasure = Erasure::new(4, 2, 1024); // 4 data shards, 2 parity shards, 1KB block size
//! let data = b"hello world";
//! let shards = erasure.encode_data(data).unwrap();
//! // Simulate loss and recovery...
//! ```
use bytes::{Bytes, BytesMut};
use reed_solomon_erasure::galois_8::ReedSolomon;
use reed_solomon_simd;
use smallvec::SmallVec;
use std::io;
@@ -44,132 +25,88 @@ use tokio::io::AsyncRead;
use tracing::warn;
use uuid::Uuid;
/// Reed-Solomon encoder using SIMD implementation.
pub struct ReedSolomonEncoder {
/// Legacy calc_shard_size formula: (block_size.div_ceil(data_shards) + 1) & !1
/// Matches main branch and filemeta::ErasureInfo for old-version files.
pub fn calc_shard_size_legacy(block_size: usize, data_shards: usize) -> usize {
(block_size.div_ceil(data_shards) + 1) & !1
}
/// Reed-Solomon encoder for legacy (main branch) format using reed-solomon-simd.
/// Used when decoding/encoding files with uses_legacy_checksum == true.
struct LegacyReedSolomonEncoder {
data_shards: usize,
parity_shards: usize,
// Use RwLock to ensure thread safety, implementing Send + Sync
encoder_cache: std::sync::RwLock<Option<reed_solomon_simd::ReedSolomonEncoder>>,
decoder_cache: std::sync::RwLock<Option<reed_solomon_simd::ReedSolomonDecoder>>,
}
impl Clone for ReedSolomonEncoder {
impl Clone for LegacyReedSolomonEncoder {
fn clone(&self) -> Self {
Self {
data_shards: self.data_shards,
parity_shards: self.parity_shards,
// Create an empty cache for the new instance instead of sharing one
encoder_cache: std::sync::RwLock::new(None),
decoder_cache: std::sync::RwLock::new(None),
}
}
}
impl ReedSolomonEncoder {
/// Create a new Reed-Solomon encoder with specified data and parity shards.
pub fn new(data_shards: usize, parity_shards: usize) -> io::Result<Self> {
Ok(ReedSolomonEncoder {
data_shards,
parity_shards,
impl LegacyReedSolomonEncoder {
fn new(_data_shards: usize, _parity_shards: usize) -> io::Result<Self> {
Ok(Self {
data_shards: _data_shards,
parity_shards: _parity_shards,
encoder_cache: std::sync::RwLock::new(None),
decoder_cache: std::sync::RwLock::new(None),
})
}
/// Encode data shards with parity.
pub fn encode(&self, shards: SmallVec<[&mut [u8]; 16]>) -> io::Result<()> {
fn encode(&self, shards: SmallVec<[&mut [u8]; 16]>) -> io::Result<()> {
let mut shards_vec: Vec<&mut [u8]> = shards.into_vec();
if shards_vec.is_empty() {
return Ok(());
}
let simd_result = self.encode_with_simd(&mut shards_vec);
match simd_result {
Ok(()) => Ok(()),
Err(simd_error) => {
warn!("SIMD encoding failed: {}", simd_error);
Err(simd_error)
}
}
}
fn encode_with_simd(&self, shards_vec: &mut [&mut [u8]]) -> io::Result<()> {
let shard_len = shards_vec[0].len();
// Get or create encoder
let mut encoder = {
let mut cache_guard = self
.encoder_cache
.write()
.map_err(|_| io::Error::other("Failed to acquire encoder cache lock"))?;
match cache_guard.take() {
Some(mut cached_encoder) => {
// Use reset method to reset existing encoder to adapt to new parameters
if let Err(e) = cached_encoder.reset(self.data_shards, self.parity_shards, shard_len) {
warn!("Failed to reset SIMD encoder: {:?}, creating new one", e);
// If reset fails, create new encoder
Some(mut cached) => {
if cached.reset(self.data_shards, self.parity_shards, shard_len).is_err() {
reed_solomon_simd::ReedSolomonEncoder::new(self.data_shards, self.parity_shards, shard_len)
.map_err(|e| io::Error::other(format!("Failed to create SIMD encoder: {e:?}")))?
} else {
cached_encoder
cached
}
}
None => {
// First use, create new encoder
reed_solomon_simd::ReedSolomonEncoder::new(self.data_shards, self.parity_shards, shard_len)
.map_err(|e| io::Error::other(format!("Failed to create SIMD encoder: {e:?}")))?
}
None => reed_solomon_simd::ReedSolomonEncoder::new(self.data_shards, self.parity_shards, shard_len)
.map_err(|e| io::Error::other(format!("Failed to create SIMD encoder: {e:?}")))?,
}
};
// Add original shards
for (i, shard) in shards_vec.iter().enumerate().take(self.data_shards) {
encoder
.add_original_shard(shard)
.map_err(|e| io::Error::other(format!("Failed to add shard {i}: {e:?}")))?;
}
// Encode and get recovery shards
let result = encoder
.encode()
.map_err(|e| io::Error::other(format!("SIMD encoding failed: {e:?}")))?;
// Copy recovery shards to output buffer
for (i, recovery_shard) in result.recovery_iter().enumerate() {
if i + self.data_shards < shards_vec.len() {
shards_vec[i + self.data_shards].copy_from_slice(recovery_shard);
}
}
// Return encoder to cache (encoder is automatically reset after result is dropped, can be reused)
drop(result); // Explicitly drop result to ensure encoder is reset
drop(result);
*self
.encoder_cache
.write()
.map_err(|_| io::Error::other("Failed to return encoder to cache"))? = Some(encoder);
Ok(())
}
/// Reconstruct missing shards.
pub fn reconstruct(&self, shards: &mut [Option<Vec<u8>>]) -> io::Result<()> {
// Use SIMD for reconstruction
let simd_result = self.reconstruct_with_simd(shards);
match simd_result {
Ok(()) => Ok(()),
Err(simd_error) => {
warn!("SIMD reconstruction failed: {}", simd_error);
Err(simd_error)
}
}
}
fn reconstruct_with_simd(&self, shards: &mut [Option<Vec<u8>>]) -> io::Result<()> {
// Find a valid shard to determine length
fn reconstruct(&self, shards: &mut [Option<Vec<u8>>]) -> io::Result<()> {
let shard_len = shards
.iter()
.find_map(|s| s.as_ref().map(|v| v.len()))
@@ -185,7 +122,6 @@ impl ReedSolomonEncoder {
Some(mut cached_decoder) => {
if let Err(e) = cached_decoder.reset(self.data_shards, self.parity_shards, shard_len) {
warn!("Failed to reset SIMD decoder: {:?}, creating new one", e);
reed_solomon_simd::ReedSolomonDecoder::new(self.data_shards, self.parity_shards, shard_len)
.map_err(|e| io::Error::other(format!("Failed to create SIMD decoder: {e:?}")))?
} else {
@@ -197,7 +133,6 @@ impl ReedSolomonEncoder {
}
};
// Add available shards (both data and parity)
for (i, shard_opt) in shards.iter().enumerate() {
if let Some(shard) = shard_opt {
if i < self.data_shards {
@@ -217,7 +152,6 @@ impl ReedSolomonEncoder {
.decode()
.map_err(|e| io::Error::other(format!("SIMD decode error: {e:?}")))?;
// Fill in missing data shards from reconstruction result
for (i, shard_opt) in shards.iter_mut().enumerate() {
if shard_opt.is_none() && i < self.data_shards {
for (restored_index, restored_data) in result.restored_original_iter() {
@@ -240,6 +174,67 @@ impl ReedSolomonEncoder {
}
}
/// Reed-Solomon encoder using reed-solomon-erasure
pub struct ReedSolomonEncoder {
data_shards: usize,
parity_shards: usize,
encoder: Option<ReedSolomon>,
}
impl Clone for ReedSolomonEncoder {
fn clone(&self) -> Self {
Self {
data_shards: self.data_shards,
parity_shards: self.parity_shards,
encoder: self.encoder.clone(),
}
}
}
impl ReedSolomonEncoder {
/// Create a new Reed-Solomon encoder with specified data and parity shards.
pub fn new(data_shards: usize, parity_shards: usize) -> io::Result<Self> {
let encoder = if parity_shards > 0 {
ReedSolomon::new(data_shards, parity_shards)
.map_err(|e| io::Error::other(format!("Failed to create Reed-Solomon encoder: {e:?}")))
.map(Some)?
} else {
None
};
Ok(ReedSolomonEncoder {
data_shards,
parity_shards,
encoder,
})
}
/// Encode data shards with parity.
pub fn encode(&self, shards: SmallVec<[&mut [u8]; 16]>) -> io::Result<()> {
let mut shards_vec: Vec<&mut [u8]> = shards.into_vec();
if shards_vec.is_empty() {
return Ok(());
}
if let Some(ref rs) = self.encoder {
rs.encode(&mut shards_vec)
.map_err(|e| io::Error::other(format!("Reed-Solomon encode failed: {e:?}")))
} else {
Ok(())
}
}
/// Reconstruct missing shards.
pub fn reconstruct(&self, shards: &mut [Option<Vec<u8>>]) -> io::Result<()> {
if let Some(ref rs) = self.encoder {
rs.reconstruct_data(shards)
.map_err(|e| io::Error::other(format!("Reed-Solomon reconstruct failed: {e:?}")))
} else {
Ok(())
}
}
}
/// Erasure coding utility for data reliability using Reed-Solomon codes.
///
/// This struct provides encoding and decoding of data into data and parity shards.
@@ -262,24 +257,41 @@ impl ReedSolomonEncoder {
/// let shards = erasure.encode_data(data).unwrap();
/// // Simulate loss and recovery...
/// ```
#[derive(Default)]
pub struct Erasure {
pub data_shards: usize,
pub parity_shards: usize,
encoder: Option<ReedSolomonEncoder>,
legacy_encoder: Option<LegacyReedSolomonEncoder>,
pub block_size: usize,
uses_legacy: bool,
_id: Uuid,
_buf: Vec<u8>,
}
impl Default for Erasure {
fn default() -> Self {
Self {
data_shards: 0,
parity_shards: 0,
encoder: None,
legacy_encoder: None,
block_size: 0,
uses_legacy: false,
_id: Uuid::nil(),
_buf: vec![],
}
}
}
impl Clone for Erasure {
fn clone(&self) -> Self {
Self {
data_shards: self.data_shards,
parity_shards: self.parity_shards,
encoder: self.encoder.clone(),
legacy_encoder: self.legacy_encoder.clone(),
block_size: self.block_size,
uses_legacy: self.uses_legacy,
_id: Uuid::new_v4(), // Generate new ID for clone
_buf: vec![0u8; self.block_size],
}
@@ -287,28 +299,44 @@ impl Clone for Erasure {
}
pub fn calc_shard_size(block_size: usize, data_shards: usize) -> usize {
(block_size.div_ceil(data_shards) + 1) & !1
block_size.div_ceil(data_shards)
}
impl Erasure {
/// Create a new Erasure instance.
/// Create a new Erasure instance
///
/// # Arguments
/// * `data_shards` - Number of data shards.
/// * `parity_shards` - Number of parity shards.
/// * `block_size` - Block size for each shard.
pub fn new(data_shards: usize, parity_shards: usize, block_size: usize) -> Self {
let encoder = if parity_shards > 0 {
Self::new_with_options(data_shards, parity_shards, block_size, false)
}
/// Create a new Erasure instance with legacy format support.
///
/// When `uses_legacy` is true, uses main-branch shard_size formula and reed-solomon-simd
/// for decode/reconstruct (for reading and healing old-version files).
pub fn new_with_options(data_shards: usize, parity_shards: usize, block_size: usize, uses_legacy: bool) -> Self {
let encoder = if !uses_legacy && parity_shards > 0 {
Some(ReedSolomonEncoder::new(data_shards, parity_shards).unwrap())
} else {
None
};
let legacy_encoder = if uses_legacy && parity_shards > 0 {
Some(LegacyReedSolomonEncoder::new(data_shards, parity_shards).unwrap())
} else {
None
};
Erasure {
data_shards,
parity_shards,
block_size,
encoder,
legacy_encoder,
uses_legacy,
_id: Uuid::new_v4(),
_buf: vec![0u8; block_size],
}
@@ -323,28 +351,29 @@ impl Erasure {
/// A vector of encoded shards as `Bytes`.
#[tracing::instrument(level = "debug", skip_all, fields(data_len=data.len()))]
pub fn encode_data(&self, data: &[u8]) -> io::Result<Vec<Bytes>> {
// let shard_size = self.shard_size();
// let total_size = shard_size * self.total_shard_count();
// Data shard count
let per_shard_size = calc_shard_size(data.len(), self.data_shards);
// Total required size
let shard_size_fn = if self.uses_legacy {
calc_shard_size_legacy
} else {
calc_shard_size
};
let per_shard_size = shard_size_fn(data.len(), self.data_shards);
let need_total_size = per_shard_size * self.total_shard_count();
// Create a new buffer with the required total length for all shards
let mut data_buffer = BytesMut::with_capacity(need_total_size);
// Copy source data
data_buffer.extend_from_slice(data);
data_buffer.resize(need_total_size, 0u8);
{
// EC encode, the result will be written into data_buffer
let data_slices: SmallVec<[&mut [u8]; 16]> = data_buffer.chunks_exact_mut(per_shard_size).collect();
// Only do EC if parity_shards > 0
if self.parity_shards > 0 {
if let Some(encoder) = self.encoder.as_ref() {
if self.uses_legacy {
if let Some(encoder) = self.legacy_encoder.as_ref() {
encoder.encode(data_slices)?;
} else {
warn!("parity_shards > 0, uses_legacy but legacy_encoder is None");
}
} else if let Some(encoder) = self.encoder.as_ref() {
encoder.encode(data_slices)?;
} else {
warn!("parity_shards > 0, but encoder is None");
@@ -372,7 +401,13 @@ impl Erasure {
/// Ok if reconstruction succeeds, error otherwise.
pub fn decode_data(&self, shards: &mut [Option<Vec<u8>>]) -> io::Result<()> {
if self.parity_shards > 0 {
if let Some(encoder) = self.encoder.as_ref() {
if self.uses_legacy {
if let Some(encoder) = self.legacy_encoder.as_ref() {
encoder.reconstruct(shards)?;
} else {
warn!("parity_shards > 0, uses_legacy but legacy_encoder is None");
}
} else if let Some(encoder) = self.encoder.as_ref() {
encoder.reconstruct(shards)?;
} else {
warn!("parity_shards > 0, but encoder is None");
@@ -395,7 +430,11 @@ impl Erasure {
/// Calculate the size of each shard.
pub fn shard_size(&self) -> usize {
calc_shard_size(self.block_size, self.data_shards)
if self.uses_legacy {
calc_shard_size_legacy(self.block_size, self.data_shards)
} else {
calc_shard_size(self.block_size, self.data_shards)
}
}
/// Calculate the total erasure file size for a given original size.
// Returns the final erasure size from the original size
@@ -408,10 +447,15 @@ impl Erasure {
}
let total_length = total_length as usize;
let shard_size_fn = if self.uses_legacy {
calc_shard_size_legacy
} else {
calc_shard_size
};
let num_shards = total_length / self.block_size;
let last_block_size = total_length % self.block_size;
let last_shard_size = calc_shard_size(last_block_size, self.data_shards);
let last_shard_size = shard_size_fn(last_block_size, self.data_shards);
(num_shards * self.shard_size() + last_shard_size) as i64
}
@@ -494,8 +538,7 @@ mod tests {
#[test]
fn test_shard_file_size_cases2() {
let erasure = Erasure::new(12, 4, 1024 * 1024);
assert_eq!(erasure.shard_file_size(1572864), 131074);
assert_eq!(erasure.shard_file_size(1572864), 131073);
}
#[test]
@@ -517,11 +560,14 @@ mod tests {
// Case 5: total_length > block_size, aligned
assert_eq!(erasure.shard_file_size(16), 4); // 16/8=2, last=0, 2*2+0=4
assert_eq!(erasure.shard_file_size(1248739), 312186); // 1248739/8=156092, last=3, 3 div_ceil 4=1, 156092*2+1=312185
// MinIO-compatible: 1248739/8=156092, last=3, ceil(3/4)=1, 156092*2+1=312185
assert_eq!(erasure.shard_file_size(1248739), 312185);
assert_eq!(erasure.shard_file_size(43), 12); // 43/8=5, last=3, 3 div_ceil 4=1, 5*2+1=11
// MinIO-compatible: 43/8=5, last=3, ceil(3/4)=1, 5*2+1=11
assert_eq!(erasure.shard_file_size(43), 11);
assert_eq!(erasure.shard_file_size(1572864), 393216); // 43/8=5, last=3, 3 div_ceil 4=1, 5*2+1=11
// 1572864 with block_size=8: 196608 full blocks, last=0, 196608*2+0=393216
assert_eq!(erasure.shard_file_size(1572864), 393216);
}
#[test]
@@ -601,10 +647,70 @@ mod tests {
#[test]
fn test_shard_size_and_file_size() {
let erasure = Erasure::new(4, 2, 8);
assert_eq!(erasure.shard_file_size(33), 9);
assert_eq!(erasure.shard_file_size(0), 0);
}
#[test]
fn test_legacy_shard_size_and_file_size() {
let erasure = Erasure::new_with_options(4, 2, 8, true);
assert_eq!(erasure.shard_size(), 2);
assert_eq!(calc_shard_size_legacy(8, 4), 2);
assert_eq!(calc_shard_size_legacy(1, 4), 2);
assert_eq!(erasure.shard_file_size(33), 10);
assert_eq!(erasure.shard_file_size(0), 0);
}
#[test]
fn test_legacy_encode_decode_roundtrip() {
let data_shards = 4;
let parity_shards = 2;
let block_size = 1024;
let erasure = Erasure::new_with_options(data_shards, parity_shards, block_size, true);
let data = b"Legacy encode/decode roundtrip test data with sufficient length.".repeat(20);
let encoded_shards = erasure.encode_data(&data).unwrap();
assert_eq!(encoded_shards.len(), data_shards + parity_shards);
let mut decode_input: Vec<Option<Vec<u8>>> = vec![None; data_shards + parity_shards];
for i in 0..data_shards {
decode_input[i] = Some(encoded_shards[i].to_vec());
}
erasure.decode_data(&mut decode_input).unwrap();
let mut recovered = Vec::new();
for shard in decode_input.iter().take(data_shards) {
recovered.extend_from_slice(shard.as_ref().unwrap());
}
recovered.truncate(data.len());
assert_eq!(&recovered, &data);
}
#[test]
fn test_legacy_decode_with_missing_shards() {
let data_shards = 4;
let parity_shards = 2;
let block_size = 256;
let erasure = Erasure::new_with_options(data_shards, parity_shards, block_size, true);
let data = b"Legacy decode with missing shards test.".repeat(10);
let encoded_shards = erasure.encode_data(&data).unwrap();
let mut shards_opt: Vec<Option<Vec<u8>>> = encoded_shards.iter().map(|s| Some(s.to_vec())).collect();
shards_opt[1] = None;
shards_opt[5] = None;
erasure.decode_data(&mut shards_opt).unwrap();
let mut recovered = Vec::new();
for shard in shards_opt.iter().take(data_shards) {
recovered.extend_from_slice(shard.as_ref().unwrap());
}
recovered.truncate(data.len());
assert_eq!(&recovered, &data);
}
#[test]
fn test_shard_file_offset() {
let erasure = Erasure::new(8, 8, 1024 * 1024);
@@ -887,6 +993,57 @@ mod tests {
assert_eq!(&recovered, &data);
}
/// Generates 7557 bytes identical to MinIO generateCompatTestData.
fn generate_compat_test_data(size: usize) -> Vec<u8> {
(0..size).map(|i| ((i * 7 + 13) % 256) as u8).collect()
}
/// Verifies reed-solomon-simd produces same shards.
/// Data shards (0-3) must match for MinIO to read RustFS part files.
/// Parity shards (4-5) differ: reed-solomon-simd vs klauspost use different RS encoding.
/// Run: cargo test -p rustfs-ecstore test_reed_solomon_compat
#[test]
fn test_reed_solomon_compat() {
let data = generate_compat_test_data(7557);
let erasure = Erasure::new(4, 2, 7557);
let shards = erasure.encode_data(&data).unwrap();
assert_eq!(shards.len(), 6, "expected 6 shards (4 data + 2 parity)");
// Per-shard HighwayHash
let expected_hashes: [&str; 6] = [
"fb3db9338e610cec541504ddae4b0bfd54445bcbd45318cf21f35f024240914d", // data 0
"a545269a3196e18e77ef9f5ec6e735a4f4ebe82d342db666b11a5256eb305720", // data 1
"2adbf0058f36c4cbcb5c9c16c38a6530c54198dfe504179a6f92d2349f245318", // data 2
"898e6d060b0cb4f0e830add7e1f936bc8b78442bf582283ee244a3a058602db8", // data 3
"4a20460bca044b3a777b26f2b0bcd371e3eab2f156f84778be3ccd8edd521ef2", // parity 4
"eb8ba4c0db15ca910d58d031f74e4601ba2fed62ad03ec29cadde3367ab0d415", // parity 5
];
let mut data_shards_match = true;
let mut parity_shards_match = true;
for (i, shard) in shards.iter().enumerate() {
let hash = rustfs_utils::HashAlgorithm::HighwayHash256S.hash_encode(shard);
let got = hex_simd::encode_to_string(hash.as_ref(), hex_simd::AsciiCase::Lower);
let matches = got == expected_hashes[i];
if i < 4 {
data_shards_match &= matches;
} else {
parity_shards_match &= matches;
}
if !matches {
eprintln!(
"Shard {} ({}): got {} want {}",
i,
if i < 4 { "data" } else { "parity" },
got,
expected_hashes[i]
);
}
}
assert!(data_shards_match, "Data shards (0-3) must match");
assert!(parity_shards_match, "Parity shards (4-5): reed-solomon-simd differs");
}
#[test]
fn test_simd_small_data_handling() {
let data_shards = 4;
crates/ecstore/src/erasure_coding/mod.rs
+1 -1
View File
@@ -19,4 +19,4 @@ pub mod erasure;
pub mod heal;
pub use bitrot::*;
pub use erasure::{Erasure, ReedSolomonEncoder, calc_shard_size};
pub use erasure::{Erasure, ReedSolomonEncoder, calc_shard_size, calc_shard_size_legacy};
crates/ecstore/src/pools.rs
+698 -62
View File
@@ -13,6 +13,17 @@
// limitations under the License.
use crate::bucket::versioning_sys::BucketVersioningSys;
use crate::bucket::{
lifecycle::{
bucket_lifecycle_audit::LcEventSrc,
bucket_lifecycle_ops::{
LifecycleOps, apply_expiry_on_transitioned_object, apply_expiry_rule, eval_action_from_lifecycle,
},
lifecycle::IlmAction,
},
metadata_sys,
object_lock::objectlock_sys::BucketObjectLockSys,
};
use crate::cache_value::metacache_set::{ListPathRawOptions, list_path_raw};
use crate::config::com::{CONFIG_PREFIX, read_config, save_config};
use crate::data_usage::DATA_USAGE_CACHE_NAME;
@@ -30,25 +41,32 @@ use crate::store_api::{
BucketOperations, BucketOptions, CompletePart, GetObjectReader, HealOperations, MakeBucketOptions, MultipartOperations,
ObjectIO, ObjectOperations, ObjectOptions, PutObjReader, StorageAPI,
};
use crate::{sets::Sets, store::ECStore};
use crate::{global::GLOBAL_LifecycleSys, sets::Sets, store::ECStore};
use byteorder::{ByteOrder, LittleEndian, WriteBytesExt};
use bytes::Bytes;
use futures::future::BoxFuture;
use http::HeaderMap;
use rmp_serde::{Deserializer, Serializer};
use rustfs_common::defer;
use rustfs_common::heal_channel::HealOpts;
use rustfs_filemeta::{MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams};
use rustfs_rio::{HashReader, WarpReader};
use rustfs_filemeta::{FileInfoVersions, MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams};
use rustfs_rio::{EtagResolvable, HashReader, HashReaderDetector, Index, Reader, TryGetIndex, WarpReader};
use rustfs_utils::path::{SLASH_SEPARATOR, encode_dir_object, path_join};
use rustfs_workers::workers::Workers;
use s3s::dto::{BucketLifecycleConfiguration, DefaultRetention, ReplicationConfiguration};
use serde::{Deserialize, Serialize};
use std::collections::{HashMap, HashSet};
use std::fmt::Display;
use std::io::{Cursor, Write};
use std::path::PathBuf;
use std::sync::Arc;
use std::pin::Pin;
use std::sync::{
Arc,
atomic::{AtomicUsize, Ordering},
};
use std::task::{Context, Poll};
use time::{Duration, OffsetDateTime};
use tokio::io::{AsyncReadExt, BufReader};
use tokio::io::{AsyncRead, AsyncReadExt, BufReader, ReadBuf};
use tokio_util::sync::CancellationToken;
use tracing::{debug, error, info, warn};
@@ -75,6 +93,147 @@ pub struct PoolMeta {
pub dont_save: bool,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
struct PersistedPoolMeta {
pub version: u16,
pub pools: Vec<PersistedPoolStatus>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
struct PersistedPoolStatus {
#[serde(rename = "id")]
pub id: usize,
#[serde(rename = "cmdline")]
pub cmd_line: String,
#[serde(rename = "lastUpdate", with = "time::serde::rfc3339")]
pub last_update: OffsetDateTime,
#[serde(rename = "decommissionInfo")]
pub decommission: Option<PersistedPoolDecommissionInfo>,
}
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
struct PersistedPoolDecommissionInfo {
#[serde(rename = "startTime", with = "time::serde::rfc3339::option")]
pub start_time: Option<OffsetDateTime>,
#[serde(rename = "startSize")]
pub start_size: usize,
#[serde(rename = "totalSize")]
pub total_size: usize,
#[serde(rename = "currentSize")]
pub current_size: usize,
#[serde(rename = "complete")]
pub complete: bool,
#[serde(rename = "failed")]
pub failed: bool,
#[serde(rename = "canceled")]
pub canceled: bool,
#[serde(rename = "queuedBuckets", default)]
pub queued_buckets: Vec<String>,
#[serde(rename = "decommissionedBuckets", default)]
pub decommissioned_buckets: Vec<String>,
#[serde(rename = "bucket", default)]
pub bucket: String,
#[serde(rename = "prefix", default)]
pub prefix: String,
#[serde(rename = "object", default)]
pub object: String,
#[serde(rename = "objectsDecommissioned")]
pub items_decommissioned: usize,
#[serde(rename = "objectsDecommissionedFailed")]
pub items_decommission_failed: usize,
#[serde(rename = "bytesDecommissioned")]
pub bytes_done: usize,
#[serde(rename = "bytesDecommissionedFailed")]
pub bytes_failed: usize,
}
impl From<PersistedPoolMeta> for PoolMeta {
fn from(value: PersistedPoolMeta) -> Self {
Self {
version: value.version,
pools: value.pools.into_iter().map(Into::into).collect(),
dont_save: false,
}
}
}
impl From<PersistedPoolStatus> for PoolStatus {
fn from(value: PersistedPoolStatus) -> Self {
Self {
id: value.id,
cmd_line: value.cmd_line,
last_update: value.last_update,
decommission: value.decommission.map(Into::into),
}
}
}
impl From<PersistedPoolDecommissionInfo> for PoolDecommissionInfo {
fn from(value: PersistedPoolDecommissionInfo) -> Self {
Self {
start_time: value.start_time,
start_size: value.start_size,
total_size: value.total_size,
current_size: value.current_size,
complete: value.complete,
failed: value.failed,
canceled: value.canceled,
queued_buckets: value.queued_buckets,
decommissioned_buckets: value.decommissioned_buckets,
bucket: value.bucket,
prefix: value.prefix,
object: value.object,
items_decommissioned: value.items_decommissioned,
items_decommission_failed: value.items_decommission_failed,
bytes_done: value.bytes_done,
bytes_failed: value.bytes_failed,
}
}
}
impl From<&PoolMeta> for PersistedPoolMeta {
fn from(value: &PoolMeta) -> Self {
Self {
version: value.version,
pools: value.pools.iter().map(Into::into).collect(),
}
}
}
impl From<&PoolStatus> for PersistedPoolStatus {
fn from(value: &PoolStatus) -> Self {
Self {
id: value.id,
cmd_line: value.cmd_line.clone(),
last_update: value.last_update,
decommission: value.decommission.as_ref().map(Into::into),
}
}
}
impl From<&PoolDecommissionInfo> for PersistedPoolDecommissionInfo {
fn from(value: &PoolDecommissionInfo) -> Self {
Self {
start_time: value.start_time,
start_size: value.start_size,
total_size: value.total_size,
current_size: value.current_size,
complete: value.complete,
failed: value.failed,
canceled: value.canceled,
queued_buckets: value.queued_buckets.clone(),
decommissioned_buckets: value.decommissioned_buckets.clone(),
bucket: value.bucket.clone(),
prefix: value.prefix.clone(),
object: value.object.clone(),
items_decommissioned: value.items_decommissioned,
items_decommission_failed: value.items_decommission_failed,
bytes_done: value.bytes_done,
bytes_failed: value.bytes_failed,
}
}
}
impl PoolMeta {
pub fn new(pools: &[Arc<Sets>], prev_meta: &PoolMeta) -> Self {
let mut new_meta = Self {
@@ -144,8 +303,8 @@ impl PoolMeta {
}
let mut buf = Deserializer::new(Cursor::new(&data[4..]));
let meta: PoolMeta = Deserialize::deserialize(&mut buf)?;
*self = meta;
let meta: PersistedPoolMeta = Deserialize::deserialize(&mut buf)?;
*self = meta.into();
if self.version != POOL_META_VERSION {
return Err(Error::other(format!("unexpected PoolMeta version: {}", self.version)));
@@ -161,7 +320,7 @@ impl PoolMeta {
data.write_u16::<LittleEndian>(POOL_META_FORMAT)?;
data.write_u16::<LittleEndian>(POOL_META_VERSION)?;
let mut buf = Vec::new();
self.serialize(&mut Serializer::new(&mut buf))?;
PersistedPoolMeta::from(self).serialize(&mut Serializer::new(&mut buf))?;
data.write_all(&buf)?;
for pool in pools {
@@ -178,7 +337,6 @@ impl PoolMeta {
stats.last_update = OffsetDateTime::now_utc();
let mut pd = d.clone();
pd.start_time = None;
pd.canceled = true;
pd.failed = false;
pd.complete = false;
@@ -202,7 +360,6 @@ impl PoolMeta {
stats.last_update = OffsetDateTime::now_utc();
let mut pd = d.clone();
pd.start_time = None;
pd.canceled = false;
pd.failed = true;
pd.complete = false;
@@ -226,7 +383,6 @@ impl PoolMeta {
stats.last_update = OffsetDateTime::now_utc();
let mut pd = d.clone();
pd.start_time = None;
pd.canceled = false;
pd.failed = false;
pd.complete = true;
@@ -576,6 +732,132 @@ impl Display for DecomBucketInfo {
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum DecommissionFinalState {
Complete,
Failed,
}
fn determine_decommission_final_state(items_failed: usize, was_cancelled: bool) -> DecommissionFinalState {
if items_failed > 0 || was_cancelled {
DecommissionFinalState::Failed
} else {
DecommissionFinalState::Complete
}
}
fn remaining_versions_after_decommission(fivs: &FileInfoVersions) -> usize {
fivs.versions.iter().filter(|version| !version.deleted).count()
}
fn decommission_delete_marker_opts(
version: &rustfs_filemeta::FileInfo,
version_id: Option<String>,
src_pool_idx: usize,
) -> ObjectOptions {
ObjectOptions {
versioned: true,
version_id,
mod_time: version.mod_time,
src_pool_idx,
data_movement: true,
delete_marker: true,
skip_decommissioned: true,
delete_replication: version.replication_state_internal.clone(),
..Default::default()
}
}
async fn should_skip_lifecycle_for_decommission(
store: Arc<ECStore>,
bucket: &str,
version: &rustfs_filemeta::FileInfo,
lifecycle_config: Option<&BucketLifecycleConfiguration>,
lock_retention: Option<DefaultRetention>,
replication_config: Option<(ReplicationConfiguration, OffsetDateTime)>,
apply_actions: bool,
) -> bool {
let Some(lifecycle_config) = lifecycle_config else {
return false;
};
let versioned = BucketVersioningSys::prefix_enabled(bucket, &version.name).await;
let object_info = crate::store_api::ObjectInfo::from_file_info(version, bucket, &version.name, versioned);
let event = eval_action_from_lifecycle(lifecycle_config, lock_retention, replication_config, &object_info).await;
match event.action {
IlmAction::DeleteRestoredAction | IlmAction::DeleteRestoredVersionAction => {
if apply_actions && object_info.is_remote() {
let _ = apply_expiry_on_transitioned_object(store, &object_info, &event, &LcEventSrc::Decom).await;
}
false
}
IlmAction::DeleteAction
| IlmAction::DeleteVersionAction
| IlmAction::DeleteAllVersionsAction
| IlmAction::DelMarkerDeleteAllVersionsAction => {
if apply_actions {
let _ = apply_expiry_rule(&event, &LcEventSrc::Decom, &object_info).await;
}
true
}
_ => false,
}
}
struct IndexedDecommissionReader<R> {
inner: R,
index: Option<Index>,
}
impl<R> IndexedDecommissionReader<R> {
fn new(inner: R, index: Option<Index>) -> Self {
Self { inner, index }
}
}
impl<R: AsyncRead + Unpin + Send + Sync> AsyncRead for IndexedDecommissionReader<R> {
fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
Pin::new(&mut self.inner).poll_read(cx, buf)
}
}
impl<R: AsyncRead + Unpin + Send + Sync> EtagResolvable for IndexedDecommissionReader<R> {}
impl<R: AsyncRead + Unpin + Send + Sync> HashReaderDetector for IndexedDecommissionReader<R> {}
impl<R: AsyncRead + Unpin + Send + Sync> TryGetIndex for IndexedDecommissionReader<R> {
fn try_get_index(&self) -> Option<&Index> {
self.index.as_ref()
}
}
impl<R: AsyncRead + Unpin + Send + Sync> Reader for IndexedDecommissionReader<R> {}
fn decode_part_index(index: Option<&Bytes>) -> Option<Index> {
let bytes = index?;
let mut decoded = Index::new();
if decoded.load(bytes.as_ref()).is_ok() {
Some(decoded)
} else {
None
}
}
fn put_obj_reader_from_chunk(chunk: Vec<u8>, size: i64, actual_size: i64, index: Option<Index>) -> Result<PutObjReader> {
use sha2::{Digest, Sha256};
let sha256hex = if !chunk.is_empty() {
Some(hex_simd::encode_to_string(Sha256::digest(&chunk), hex_simd::AsciiCase::Lower))
} else {
None
};
let reader = IndexedDecommissionReader::new(WarpReader::new(Cursor::new(chunk)), index);
let hash_reader = HashReader::new(Box::new(reader), size, actual_size, None, sha256hex, false)?;
Ok(PutObjReader::new(hash_reader))
}
impl ECStore {
pub async fn status(&self, idx: usize) -> Result<PoolStatus> {
let space_info = self.get_decommission_pool_space_info(idx).await?;
@@ -621,13 +903,18 @@ impl ECStore {
return Err(Error::other("InvalidArgument"));
}
let Some(has_canceler) = self.decommission_cancelers.get(idx) else {
return Err(Error::other("InvalidArgument"));
};
let canceler = {
let mut cancelers = self.decommission_cancelers.write().await;
let Some(slot) = cancelers.get_mut(idx) else {
return Err(Error::other("InvalidArgument"));
};
if has_canceler.is_none() {
return Err(StorageError::DecommissionNotStarted);
}
let Some(canceler) = slot.take() else {
return Err(StorageError::DecommissionNotStarted);
};
canceler
};
let mut lock = self.pool_meta.write().await;
if lock.decommission_cancel(idx) {
@@ -640,6 +927,8 @@ impl ECStore {
}
}
canceler.cancel();
Ok(())
}
pub async fn is_decommission_running(&self) -> bool {
@@ -684,8 +973,8 @@ impl ECStore {
Ok(())
}
#[allow(unused_assignments)]
#[tracing::instrument(skip(self, set, wk, rcfg))]
#[allow(unused_assignments, clippy::too_many_arguments)]
#[tracing::instrument(skip(self, set, wk, lifecycle_config, lock_retention, replication_config))]
async fn decommission_entry(
self: &Arc<Self>,
idx: usize,
@@ -693,7 +982,9 @@ impl ECStore {
bucket: String,
set: Arc<SetDisks>,
wk: Arc<Workers>,
rcfg: Option<String>,
lifecycle_config: Option<BucketLifecycleConfiguration>,
lock_retention: Option<DefaultRetention>,
replication_config: Option<(ReplicationConfiguration, OffsetDateTime)>,
) {
warn!("decommission_entry: {} {}", &bucket, &entry.name);
wk.give().await;
@@ -713,12 +1004,27 @@ impl ECStore {
fivs.versions.sort_by(|a, b| b.mod_time.cmp(&a.mod_time));
let mut decommissioned: usize = 0;
let expired: usize = 0;
let mut expired: usize = 0;
for version in fivs.versions.iter() {
// TODO: filterLifecycle
if should_skip_lifecycle_for_decommission(
self.clone(),
&bucket,
version,
lifecycle_config.as_ref(),
lock_retention.clone(),
replication_config.clone(),
true,
)
.await
{
expired += 1;
decommissioned += 1;
continue;
}
let remaining_versions = fivs.versions.len() - expired;
if version.deleted && remaining_versions == 1 && rcfg.is_none() {
if version.deleted && remaining_versions == 1 && replication_config.is_none() {
//
decommissioned += 1;
info!("decommission_pool: DELETE marked object with no other non-current versions will be skipped");
@@ -731,25 +1037,19 @@ impl ECStore {
let mut failure = false;
let mut error = None;
if version.deleted {
// TODO: other params
if let Err(err) = self
.delete_object(
bucket.as_str(),
&version.name,
ObjectOptions {
versioned: true,
version_id: version_id.clone(),
mod_time: version.mod_time,
src_pool_idx: idx,
data_movement: true,
delete_marker: true,
skip_decommissioned: true,
..Default::default()
},
decommission_delete_marker_opts(version, version_id.clone(), idx),
)
.await
{
if is_err_object_not_found(&err) || is_err_version_not_found(&err) || is_err_data_movement_overwrite(&err) {
warn!(
"decommission_pool: ignore delete-marker copy for {}/{} version {:?}: {:?}",
&bucket, &version.name, &version_id, &err
);
ignore = true;
continue;
}
@@ -776,7 +1076,33 @@ impl ECStore {
for _i in 0..3 {
if version.is_remote() {
// TODO: DecomTieredObject
if let Err(err) = self
.decommission_tiered_object(
bucket.as_str(),
&version.name,
version,
&ObjectOptions {
version_id: version_id.clone(),
mod_time: version.mod_time,
user_defined: version.metadata.clone(),
src_pool_idx: idx,
data_movement: true,
..Default::default()
},
)
.await
{
if is_err_object_not_found(&err) || is_err_version_not_found(&err) || is_err_data_movement_overwrite(&err)
{
ignore = true;
break;
}
failure = true;
error!("decommission_pool: decommission_tiered_object err {:?}", &err);
error = Some(err);
}
break;
}
let bucket = bucket.clone();
@@ -847,7 +1173,8 @@ impl ECStore {
}
{
self.pool_meta.write().await.count_item(idx, decommissioned, failure);
let size = usize::try_from(version.size).unwrap_or_default();
self.pool_meta.write().await.count_item(idx, size, failure);
}
if failure {
@@ -872,6 +1199,14 @@ impl ECStore {
.await
{
error!("decommission_pool: delete_object err {:?}", &err);
} else if decommissioned != fivs.versions.len() {
warn!(
"decommission_pool: source object retained for {}/{} because only {}/{} versions were decommissioned",
&bucket,
&entry.name,
decommissioned,
fivs.versions.len()
);
}
{
@@ -903,16 +1238,19 @@ impl ECStore {
) -> Result<()> {
let wk = Workers::new(pool.disk_set.len() * 2).map_err(Error::other)?;
// let mut vc = None;
// replication
let rcfg: Option<String> = None;
let mut lifecycle_config = None;
let mut lock_retention = None;
let mut replication_config = None;
if bi.name != RUSTFS_META_BUCKET {
let _versioning = BucketVersioningSys::get(&bi.name).await?;
// vc = Some(versioning);
// TODO: LifecycleSys
// TODO: BucketObjectLockSys
// TODO: ReplicationConfig
let _ = BucketVersioningSys::get(&bi.name).await?;
lifecycle_config = GLOBAL_LifecycleSys.get(&bi.name).await;
lock_retention = BucketObjectLockSys::get(&bi.name).await;
replication_config = match metadata_sys::get_replication_config(&bi.name).await {
Ok(config) => Some(config),
Err(Error::ConfigNotFound) => None,
Err(err) => return Err(err),
};
}
for (set_idx, set) in pool.disk_set.iter().enumerate() {
@@ -925,17 +1263,22 @@ impl ECStore {
let bucket = bi.name.clone();
let wk = wk.clone();
let set = set.clone();
let rcfg = rcfg.clone();
let lifecycle_config = lifecycle_config.clone();
let lock_retention = lock_retention.clone();
let replication_config = replication_config.clone();
move |entry: MetaCacheEntry| {
let this = this.clone();
let bucket = bucket.clone();
let wk = wk.clone();
let set = set.clone();
let rcfg = rcfg.clone();
let lifecycle_config = lifecycle_config.clone();
let lock_retention = lock_retention.clone();
let replication_config = replication_config.clone();
Box::pin(async move {
wk.take().await;
this.decommission_entry(idx, entry, bucket, set, wk, rcfg).await
this.decommission_entry(idx, entry, bucket, set, wk, lifecycle_config, lock_retention, replication_config)
.await
})
}
});
@@ -988,7 +1331,15 @@ impl ECStore {
#[tracing::instrument(skip(self, rx))]
pub async fn do_decommission_in_routine(self: &Arc<Self>, rx: CancellationToken, idx: usize) {
if let Err(err) = self.decommission_in_background(rx, idx).await {
let decommission_token = rx.child_token();
{
let mut cancelers = self.decommission_cancelers.write().await;
if let Some(slot) = cancelers.get_mut(idx) {
*slot = Some(decommission_token.clone());
}
}
if let Err(err) = self.decommission_in_background(decommission_token.clone(), idx).await {
error!("decom err {:?}", &err);
if let Err(er) = self.decommission_failed(idx).await {
error!("decom failed err {:?}", &er);
@@ -1001,29 +1352,49 @@ impl ECStore {
warn!("decommission: decommission_in_background complete {}", idx);
let (failed, cmd_line) = {
let (final_state, cmd_line) = {
let pool_meta = self.pool_meta.read().await;
let failed = {
let final_state = {
if let Some(info) = &pool_meta.pools[idx].decommission {
info.items_decommission_failed > 0
determine_decommission_final_state(info.items_decommission_failed, info.canceled)
} else {
false
DecommissionFinalState::Failed
}
};
let cmd_line = pool_meta.pools[idx].cmd_line.clone();
(failed, cmd_line)
(final_state, cmd_line)
};
if !failed {
let mut completed_successfully = false;
if final_state == DecommissionFinalState::Complete {
warn!("Decommissioning complete for pool {}, verifying for any pending objects", cmd_line);
if let Err(er) = self.decommission_failed(idx).await {
error!("decom failed err {:?}", &er);
if let Err(err) = self.check_after_decommission(idx).await {
error!("decom post-check err {:?}", &err);
if let Err(er) = self.decommission_failed(idx).await {
error!("decom failed err {:?}", &er);
}
} else if let Err(er) = self.complete_decommission(idx).await {
error!("decom complete err {:?}", &er);
} else {
completed_successfully = true;
}
} else if let Err(er) = self.complete_decommission(idx).await {
error!("decom complete err {:?}", &er);
} else if let Err(er) = self.decommission_failed(idx).await {
error!("decom failed err {:?}", &er);
}
warn!("Decommissioning complete for pool {}", cmd_line);
{
let mut cancelers = self.decommission_cancelers.write().await;
if let Some(slot) = cancelers.get_mut(idx) {
*slot = None;
}
}
if completed_successfully {
warn!("Decommissioning complete for pool {}", cmd_line);
} else {
warn!("Decommissioning finished in failed state for pool {}", cmd_line);
}
}
#[tracing::instrument(skip(self))]
@@ -1043,6 +1414,14 @@ impl ECStore {
}
}
let canceler = {
let mut cancelers = self.decommission_cancelers.write().await;
cancelers.get_mut(idx).and_then(Option::take)
};
if let Some(canceler) = canceler {
canceler.cancel();
}
Ok(())
}
@@ -1061,6 +1440,14 @@ impl ECStore {
}
}
let canceler = {
let mut cancelers = self.decommission_cancelers.write().await;
cancelers.get_mut(idx).and_then(Option::take)
};
if let Some(canceler) = canceler {
canceler.cancel();
}
Ok(())
}
@@ -1190,6 +1577,94 @@ impl ECStore {
Ok(ret)
}
async fn check_after_decommission(self: &Arc<Self>, idx: usize) -> Result<()> {
let buckets = self.get_buckets_to_decommission().await?;
let pool = self.pools[idx].clone();
for set in &pool.disk_set {
for bucket_info in &buckets {
let mut lifecycle_config = None;
let mut lock_retention = None;
let mut replication_config = None;
if bucket_info.name != RUSTFS_META_BUCKET {
lifecycle_config = GLOBAL_LifecycleSys.get(&bucket_info.name).await;
lock_retention = BucketObjectLockSys::get(&bucket_info.name).await;
replication_config = match metadata_sys::get_replication_config(&bucket_info.name).await {
Ok(config) => Some(config),
Err(Error::ConfigNotFound) => None,
Err(err) => return Err(err),
};
}
let versions_found = Arc::new(AtomicUsize::new(0));
let versions_found_cb = versions_found.clone();
let bucket_name = bucket_info.name.clone();
let lifecycle_config_cb = lifecycle_config.clone();
let lock_retention_cb = lock_retention.clone();
let replication_config_cb = replication_config.clone();
let store = Arc::clone(self);
let callback: ListCallback = Arc::new(move |entry: MetaCacheEntry| {
let versions_found = versions_found_cb.clone();
let bucket_name = bucket_name.clone();
let lifecycle_config = lifecycle_config_cb.clone();
let lock_retention = lock_retention_cb.clone();
let replication_config = replication_config_cb.clone();
let store = Arc::clone(&store);
Box::pin(async move {
if !entry.is_object() {
return;
}
if bucket_name == RUSTFS_META_BUCKET && entry.name.contains(DATA_USAGE_CACHE_NAME) {
return;
}
let Ok(fivs) = entry.file_info_versions(&bucket_name) else {
return;
};
let mut remaining = 0;
for version in &fivs.versions {
if version.deleted {
continue;
}
if should_skip_lifecycle_for_decommission(
Arc::clone(&store),
&bucket_name,
version,
lifecycle_config.as_ref(),
lock_retention.clone(),
replication_config.clone(),
false,
)
.await
{
continue;
}
remaining += 1;
}
versions_found.fetch_add(remaining, Ordering::Relaxed);
})
});
set.list_objects_to_decommission(CancellationToken::new(), bucket_info.clone(), callback)
.await?;
let versions_found = versions_found.load(Ordering::Relaxed);
if versions_found > 0 {
return Err(Error::other(format!(
"at least {versions_found} object(s)/version(s) were found in bucket `{}` after decommissioning",
bucket_info.name
)));
}
}
}
Ok(())
}
#[tracing::instrument(skip(self, rd))]
async fn decommission_object(self: Arc<Self>, pool_idx: usize, bucket: String, rd: GetObjectReader) -> Result<()> {
warn!("decommission_object: start {} {}", &bucket, &rd.object_info.name);
@@ -1238,7 +1713,10 @@ impl ECStore {
reader.read_exact(&mut chunk).await?;
let mut data = PutObjReader::from_vec(chunk);
let part_size = i64::try_from(part.size).map_err(|_| Error::other("part size overflow"))?;
let part_actual_size = if part.actual_size > 0 { part.actual_size } else { part_size };
let index = decode_part_index(part.index.as_ref());
let mut data = put_obj_reader_from_chunk(chunk, part_size, part_actual_size, index)?;
let pi = match self
.put_object_part(
@@ -1294,8 +1772,13 @@ impl ECStore {
return Ok(());
}
let reader = BufReader::new(rd.stream);
let hrd = HashReader::new(Box::new(WarpReader::new(reader)), object_info.size, object_info.size, None, None, false)?;
let actual_size = object_info.get_actual_size()?;
let index = object_info
.parts
.first()
.and_then(|part| decode_part_index(part.index.as_ref()));
let reader = IndexedDecommissionReader::new(WarpReader::new(BufReader::new(rd.stream)), index);
let hrd = HashReader::new(Box::new(reader), object_info.size, actual_size, object_info.etag.clone(), None, false)?;
let mut data = PutObjReader::new(hrd);
if let Err(err) = self
@@ -1325,6 +1808,159 @@ impl ECStore {
}
}
#[cfg(test)]
#[allow(clippy::items_after_test_module)]
mod tests {
use super::*;
#[test]
fn determine_decommission_final_state_marks_failures_and_cancellations() {
assert_eq!(determine_decommission_final_state(0, false), DecommissionFinalState::Complete);
assert_eq!(determine_decommission_final_state(1, false), DecommissionFinalState::Failed);
assert_eq!(determine_decommission_final_state(0, true), DecommissionFinalState::Failed);
}
#[test]
fn remaining_versions_after_decommission_ignores_delete_markers() {
let fivs = FileInfoVersions {
versions: vec![
rustfs_filemeta::FileInfo {
deleted: false,
size: 128,
..Default::default()
},
rustfs_filemeta::FileInfo {
deleted: true,
size: 0,
..Default::default()
},
],
..Default::default()
};
assert_eq!(remaining_versions_after_decommission(&fivs), 1);
}
#[test]
fn decommission_delete_marker_opts_preserves_replication_state() {
let mod_time = OffsetDateTime::now_utc();
let version = rustfs_filemeta::FileInfo {
mod_time: Some(mod_time),
replication_state_internal: Some(rustfs_filemeta::ReplicationState {
replica_status: rustfs_filemeta::ReplicationStatusType::Replica,
delete_marker: true,
replicate_decision_str: "existing".to_string(),
..Default::default()
}),
..Default::default()
};
let opts = decommission_delete_marker_opts(&version, Some("version-id".to_string()), 7);
let replication = opts.delete_replication.expect("replication state should be preserved");
assert!(opts.versioned);
assert!(opts.data_movement);
assert!(opts.delete_marker);
assert!(opts.skip_decommissioned);
assert_eq!(opts.src_pool_idx, 7);
assert_eq!(opts.version_id.as_deref(), Some("version-id"));
assert_eq!(opts.mod_time, Some(mod_time));
assert_eq!(replication.replica_status, rustfs_filemeta::ReplicationStatusType::Replica);
assert!(replication.delete_marker);
assert_eq!(replication.replicate_decision_str, "existing");
}
#[test]
fn decommission_state_transitions_preserve_start_time() {
let start_time = OffsetDateTime::now_utc();
let mut pool_meta = PoolMeta {
version: POOL_META_VERSION,
pools: vec![PoolStatus {
id: 0,
cmd_line: "/tmp/pool".to_string(),
last_update: start_time,
decommission: Some(PoolDecommissionInfo {
start_time: Some(start_time),
..Default::default()
}),
}],
dont_save: true,
};
assert!(pool_meta.decommission_failed(0));
assert_eq!(
pool_meta.pools[0].decommission.as_ref().and_then(|info| info.start_time),
Some(start_time)
);
assert!(pool_meta.decommission_complete(0));
assert_eq!(
pool_meta.pools[0].decommission.as_ref().and_then(|info| info.start_time),
Some(start_time)
);
assert!(pool_meta.decommission_cancel(0));
assert_eq!(
pool_meta.pools[0].decommission.as_ref().and_then(|info| info.start_time),
Some(start_time)
);
}
#[test]
fn pool_meta_persists_decommission_resume_queues() {
let start_time = OffsetDateTime::now_utc();
let pool_meta = PoolMeta {
version: POOL_META_VERSION,
pools: vec![PoolStatus {
id: 1,
cmd_line: "/data/pool1/disk{1...4}".to_string(),
last_update: start_time,
decommission: Some(PoolDecommissionInfo {
start_time: Some(start_time),
queued_buckets: vec!["bucket-a".to_string(), "bucket-b/prefix".to_string()],
decommissioned_buckets: vec!["bucket-done".to_string()],
bucket: "bucket-b".to_string(),
prefix: "prefix".to_string(),
object: "object.txt".to_string(),
items_decommissioned: 7,
items_decommission_failed: 1,
bytes_done: 1024,
bytes_failed: 128,
..Default::default()
}),
}],
dont_save: false,
};
let mut buf = Vec::new();
PersistedPoolMeta::from(&pool_meta)
.serialize(&mut Serializer::new(&mut buf))
.expect("pool meta should serialize");
let mut deserializer = Deserializer::new(Cursor::new(&buf));
let restored: PoolMeta = PersistedPoolMeta::deserialize(&mut deserializer)
.expect("pool meta should deserialize")
.into();
let restored_decommission = restored.pools[0]
.decommission
.as_ref()
.expect("decommission info should survive round-trip");
assert_eq!(
restored_decommission.queued_buckets,
vec!["bucket-a".to_string(), "bucket-b/prefix".to_string()]
);
assert_eq!(restored_decommission.decommissioned_buckets, vec!["bucket-done".to_string()]);
assert_eq!(restored_decommission.bucket, "bucket-b");
assert_eq!(restored_decommission.prefix, "prefix");
assert_eq!(restored_decommission.object, "object.txt");
assert_eq!(restored_decommission.items_decommissioned, 7);
assert_eq!(restored_decommission.items_decommission_failed, 1);
assert_eq!(restored_decommission.bytes_done, 1024);
assert_eq!(restored_decommission.bytes_failed, 128);
}
}
// impl Fn(MetaCacheEntry) -> impl Future<Output = Result<(), Error>>
pub type ListCallback = Arc<dyn Fn(MetaCacheEntry) -> BoxFuture<'static, ()> + Send + Sync + 'static>;
crates/ecstore/src/set_disk.rs
+147 -25
View File
@@ -80,9 +80,12 @@ use rustfs_lock::{FastLockGuard, NamespaceLock, NamespaceLockGuard, NamespaceLoc
use rustfs_madmin::heal_commands::{HealDriveInfo, HealResultItem};
use rustfs_rio::{EtagResolvable, HashReader, HashReaderMut, TryGetIndex as _, WarpReader};
use rustfs_s3_common::EventName;
use rustfs_utils::http::RUSTFS_BUCKET_REPLICATION_SSEC_CHECKSUM;
use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING;
use rustfs_utils::http::headers::AMZ_STORAGE_CLASS;
use rustfs_utils::http::headers::{AMZ_OBJECT_TAGGING, RESERVED_METADATA_PREFIX, RESERVED_METADATA_PREFIX_LOWER};
use rustfs_utils::http::{
SUFFIX_ACTUAL_OBJECT_SIZE_CAP, SUFFIX_ACTUAL_SIZE, SUFFIX_COMPRESSION, SUFFIX_COMPRESSION_SIZE, SUFFIX_REPLICATION_SSEC_CRC,
contains_key_str, get_header_map, get_str, insert_str, remove_header_map,
};
use rustfs_utils::{
HashAlgorithm,
crypto::hex,
@@ -136,6 +139,30 @@ pub fn get_lock_acquire_timeout() -> Duration {
Duration::from_secs(rustfs_utils::get_env_u64("RUSTFS_LOCK_ACQUIRE_TIMEOUT", 5))
}
fn build_tiered_decommission_file_info(
bucket: &str,
object: &str,
fi: &FileInfo,
disk_count: usize,
default_parity_count: usize,
storage_class: Option<&str>,
) -> (FileInfo, usize) {
let parity_drives = GLOBAL_STORAGE_CLASS
.get()
.and_then(|sc| sc.get_parity_for_sc(storage_class.unwrap_or_default()))
.unwrap_or(default_parity_count);
let data_drives = disk_count - parity_drives;
let mut write_quorum = data_drives;
if data_drives == parity_drives {
write_quorum += 1;
}
let mut updated = fi.clone();
updated.erasure = FileInfo::new([bucket, object].join("/").as_str(), data_drives, parity_drives).erasure;
(updated, write_quorum)
}
#[derive(Clone, Debug)]
pub struct SetDisks {
pub locker_owner: String,
@@ -620,7 +647,7 @@ impl ObjectIO for SetDisks {
&tmp_object,
erasure.shard_file_size(data.size()),
erasure.shard_size(),
HashAlgorithm::HighwayHash256,
HashAlgorithm::HighwayHash256S,
)
.await
{
@@ -678,8 +705,8 @@ impl ObjectIO for SetDisks {
)));
}
if user_defined.contains_key(&format!("{RESERVED_METADATA_PREFIX_LOWER}compression")) {
user_defined.insert(format!("{RESERVED_METADATA_PREFIX_LOWER}compression-size"), w_size.to_string());
if contains_key_str(&user_defined, SUFFIX_COMPRESSION) {
insert_str(&mut user_defined, SUFFIX_COMPRESSION_SIZE, w_size.to_string());
}
let index_op = data.stream.try_get_index().map(|v| v.clone().into_vec());
@@ -765,7 +792,7 @@ impl ObjectIO for SetDisks {
.await?;
if let Some(old_dir) = op_old_dir {
self.commit_rename_data_dir(&shuffle_disks, bucket, object, &old_dir.to_string(), write_quorum)
self.commit_rename_data_dir(&online_disks, bucket, object, &old_dir.to_string(), write_quorum)
.await?;
}
@@ -1336,6 +1363,12 @@ impl ObjectOperations for SetDisks {
let mut delete_marker = opts.versioned;
if opts.version_id.is_some() {
// Decommission/rebalance may recreate a delete marker on a new pool before that
// exact version exists there, so we must still treat it as a mark-delete write.
if opts.data_movement && opts.delete_marker && !version_found {
mark_delete = true;
}
if version_found && opts.delete_marker_replication_status() == ReplicationStatusType::Replica {
mark_delete = false;
}
@@ -1889,6 +1922,68 @@ impl ObjectOperations for SetDisks {
}
}
impl SetDisks {
#[tracing::instrument(skip(self, fi, opts))]
pub(crate) async fn decommission_tiered_object(
&self,
bucket: &str,
object: &str,
fi: &FileInfo,
opts: &ObjectOptions,
) -> Result<()> {
let _lock_guard = if !opts.no_lock {
Some(
self.new_ns_lock(bucket, object)
.await?
.get_write_lock(get_lock_acquire_timeout())
.await
.map_err(|e| {
Error::other(format!(
"Failed to acquire write lock: {}",
self.format_lock_error_from_error(bucket, object, "write", &e)
))
})?,
)
} else {
None
};
let disks = self.disks.read().await.clone();
let storage_class = opts.user_defined.get(AMZ_STORAGE_CLASS).map(String::as_str);
let (fi, write_quorum) =
build_tiered_decommission_file_info(bucket, object, fi, disks.len(), self.default_parity_count, storage_class);
let parts_metadata = vec![fi.clone(); disks.len()];
let (shuffle_disks, parts_metadata) = Self::shuffle_disks_and_parts_metadata(&disks, &parts_metadata, &fi);
let mut errs = Vec::with_capacity(shuffle_disks.len());
let mut futures = Vec::with_capacity(shuffle_disks.len());
for (index, disk) in shuffle_disks.iter().enumerate() {
let mut file_info = parts_metadata[index].clone();
file_info.erasure.index = index + 1;
futures.push(async move {
if let Some(disk) = disk {
disk.write_metadata("", bucket, object, file_info).await
} else {
Err(DiskError::DiskNotFound)
}
});
}
for result in join_all(futures).await {
match result {
Ok(_) => errs.push(None),
Err(err) => errs.push(Some(err)),
}
}
if let Some(err) = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, write_quorum) {
return Err(to_object_err(err.into(), vec![bucket, object]));
}
Ok(())
}
}
#[async_trait::async_trait]
impl ListOperations for SetDisks {
#[tracing::instrument(skip(self))]
@@ -2007,7 +2102,7 @@ impl MultipartOperations for SetDisks {
&tmp_part_path,
erasure.shard_file_size(data.size()),
erasure.shard_size(),
HashAlgorithm::HighwayHash256,
HashAlgorithm::HighwayHash256S,
)
.await
{
@@ -2454,11 +2549,11 @@ impl MultipartOperations for SetDisks {
fi.data_dir = Some(Uuid::new_v4());
if let Some(cssum) = user_defined.get(RUSTFS_BUCKET_REPLICATION_SSEC_CHECKSUM)
if let Some(cssum) = get_header_map(&user_defined, SUFFIX_REPLICATION_SSEC_CRC)
&& !cssum.is_empty()
{
fi.checksum = base64_simd::STANDARD.decode_to_vec(cssum).ok().map(Bytes::from);
user_defined.remove(RUSTFS_BUCKET_REPLICATION_SSEC_CHECKSUM);
fi.checksum = base64_simd::STANDARD.decode_to_vec(&cssum).ok().map(Bytes::from);
remove_header_map(&mut user_defined, SUFFIX_REPLICATION_SSEC_CRC);
}
let parts_metadata = vec![fi.clone(); disks.len()];
@@ -2809,8 +2904,8 @@ impl MultipartOperations for SetDisks {
}
}
if let Some(rc_crc) = opts.user_defined.get(RUSTFS_BUCKET_REPLICATION_SSEC_CHECKSUM) {
if let Ok(rc_crc_bytes) = base64_simd::STANDARD.decode_to_vec(rc_crc) {
if let Some(rc_crc) = get_header_map(&opts.user_defined, SUFFIX_REPLICATION_SSEC_CRC) {
if let Ok(rc_crc_bytes) = base64_simd::STANDARD.decode_to_vec(&rc_crc) {
fi.checksum = Some(Bytes::from(rc_crc_bytes));
} else {
error!("complete_multipart_upload decode rc_crc failed rc_crc={}", rc_crc);
@@ -2849,25 +2944,19 @@ impl MultipartOperations for SetDisks {
fi.metadata.insert("etag".to_owned(), etag);
if opts.replication_request {
if let Some(actual_size) = opts
.user_defined
.get(format!("{RESERVED_METADATA_PREFIX_LOWER}Actual-Object-Size").as_str())
{
if let Some(actual_size) = get_str(&opts.user_defined, SUFFIX_ACTUAL_OBJECT_SIZE_CAP) {
insert_str(&mut fi.metadata, SUFFIX_ACTUAL_SIZE, actual_size.clone());
fi.metadata
.insert(format!("{RESERVED_METADATA_PREFIX}actual-size"), actual_size.clone());
fi.metadata
.insert("x-rustfs-encryption-original-size".to_string(), actual_size.to_string());
.insert("x-rustfs-encryption-original-size".to_string(), actual_size);
}
} else {
fi.metadata
.insert(format!("{RESERVED_METADATA_PREFIX}actual-size"), object_actual_size.to_string());
insert_str(&mut fi.metadata, SUFFIX_ACTUAL_SIZE, object_actual_size.to_string());
fi.metadata
.insert("x-rustfs-encryption-original-size".to_string(), object_actual_size.to_string());
}
if fi.is_compressed() {
fi.metadata
.insert(format!("{RESERVED_METADATA_PREFIX_LOWER}compression-size"), object_size.to_string());
insert_str(&mut fi.metadata, SUFFIX_COMPRESSION_SIZE, object_size.to_string());
}
if opts.data_movement {
@@ -2927,7 +3016,7 @@ impl MultipartOperations for SetDisks {
.await?;
if let Some(old_dir) = op_old_dir {
self.commit_rename_data_dir(&shuffle_disks, bucket, object, &old_dir.to_string(), write_quorum)
self.commit_rename_data_dir(&online_disks, bucket, object, &old_dir.to_string(), write_quorum)
.await?;
}
@@ -3356,12 +3445,18 @@ async fn disks_with_all_parts(
if (meta.data.is_some() || meta.size == 0) && !meta.parts.is_empty() {
if let Some(data) = &meta.data {
let checksum_info = meta.erasure.get_checksum_info(meta.parts[0].number);
let checksum_algo =
if meta.uses_legacy_checksum && checksum_info.algorithm == rustfs_utils::HashAlgorithm::HighwayHash256S {
rustfs_utils::HashAlgorithm::HighwayHash256SLegacy
} else {
checksum_info.algorithm
};
let data_len = data.len();
let verify_err = bitrot_verify(
Box::new(Cursor::new(data.clone())),
data_len,
meta.erasure.shard_file_size(meta.size) as usize,
checksum_info.algorithm,
checksum_algo,
checksum_info.hash,
meta.erasure.shard_size(),
)
@@ -4161,6 +4256,33 @@ mod tests {
assert!(e_tag_matches("\"abc\"", "*"));
}
#[test]
fn test_build_tiered_decommission_file_info_preserves_transition_metadata() {
let version_id = Uuid::new_v4();
let transition_version_id = Uuid::new_v4();
let original = FileInfo {
version_id: Some(version_id),
transition_status: TRANSITION_COMPLETE.to_string(),
transitioned_objname: "remote/object".to_string(),
transition_tier: "WARM-TIER".to_string(),
transition_version_id: Some(transition_version_id),
erasure: FileInfo::new("old-bucket/old-object", 8, 8).erasure,
..Default::default()
};
let (updated, write_quorum) = build_tiered_decommission_file_info("bucket", "object", &original, 16, 4, None);
assert_eq!(updated.version_id, original.version_id);
assert_eq!(updated.transition_status, original.transition_status);
assert_eq!(updated.transitioned_objname, original.transitioned_objname);
assert_eq!(updated.transition_tier, original.transition_tier);
assert_eq!(updated.transition_version_id, original.transition_version_id);
assert_eq!(updated.erasure.data_blocks, 12);
assert_eq!(updated.erasure.parity_blocks, 4);
assert_eq!(write_quorum, 12);
assert_ne!(updated.erasure.distribution, original.erasure.distribution);
}
#[test]
fn test_should_prevent_write() {
let oi = ObjectInfo {
crates/ecstore/src/set_disk/heal.rs
+12 -4
View File
@@ -124,11 +124,12 @@ impl SetDisks {
);
let erasure = if !latest_meta.deleted && !latest_meta.is_remote() {
// Initialize erasure coding
erasure_coding::Erasure::new(
// Initialize erasure coding; use legacy mode for old-version files
erasure_coding::Erasure::new_with_options(
latest_meta.erasure.data_blocks,
latest_meta.erasure.parity_blocks,
latest_meta.erasure.block_size,
latest_meta.uses_legacy_checksum,
)
} else {
erasure_coding::Erasure::default()
@@ -347,7 +348,14 @@ impl SetDisks {
for (part_index, part) in latest_meta.parts.iter().enumerate() {
let till_offset = erasure.shard_file_offset(0, part.size, part.size);
let checksum_algo = erasure_info.get_checksum_info(part.number).algorithm;
let checksum_info = erasure_info.get_checksum_info(part.number);
let checksum_algo = if latest_meta.uses_legacy_checksum
&& checksum_info.algorithm == rustfs_utils::HashAlgorithm::HighwayHash256S
{
rustfs_utils::HashAlgorithm::HighwayHash256SLegacy
} else {
checksum_info.algorithm
};
let mut readers = Vec::with_capacity(latest_disks.len());
let mut writers = Vec::with_capacity(out_dated_disks.len());
// let mut errors = Vec::with_capacity(out_dated_disks.len());
@@ -420,7 +428,7 @@ impl SetDisks {
]),
erasure.shard_file_size(part.size as i64),
erasure.shard_size(),
HashAlgorithm::HighwayHash256,
HashAlgorithm::HighwayHash256S,
)
.await
{
crates/ecstore/src/set_disk/read.rs
+15 -2
View File
@@ -603,7 +603,12 @@ impl SetDisks {
object, offset, length, end_offset, part_index, last_part_index, last_part_relative_offset, "Multipart read bounds"
);
let erasure = erasure_coding::Erasure::new(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size);
let erasure = erasure_coding::Erasure::new_with_options(
fi.erasure.data_blocks,
fi.erasure.parity_blocks,
fi.erasure.block_size,
fi.uses_legacy_checksum,
);
let part_indices: Vec<usize> = (part_index..=last_part_index).collect();
debug!(bucket, object, ?part_indices, "Multipart part indices to stream");
@@ -648,6 +653,14 @@ impl SetDisks {
"Streaming multipart part"
);
let checksum_info = fi.erasure.get_checksum_info(part_number);
let checksum_algo =
if fi.uses_legacy_checksum && checksum_info.algorithm == rustfs_utils::HashAlgorithm::HighwayHash256S {
rustfs_utils::HashAlgorithm::HighwayHash256SLegacy
} else {
checksum_info.algorithm
};
let mut readers = Vec::with_capacity(disks.len());
let mut errors = Vec::with_capacity(disks.len());
for (idx, disk_op) in disks.iter().enumerate() {
@@ -659,7 +672,7 @@ impl SetDisks {
read_offset,
till_offset,
erasure.shard_size(),
HashAlgorithm::HighwayHash256,
checksum_algo.clone(),
skip_verify_bitrot,
)
.await
crates/ecstore/src/store.rs
+1 -1
View File
@@ -156,7 +156,7 @@ pub struct ECStore {
// pub local_disks: Vec<DiskStore>,
pub pool_meta: RwLock<PoolMeta>,
pub rebalance_meta: RwLock<Option<RebalanceMeta>>,
pub decommission_cancelers: Vec<Option<usize>>,
pub decommission_cancelers: RwLock<Vec<Option<CancellationToken>>>,
}
// impl Clone for ECStore {
crates/ecstore/src/store/init.rs
+2 -1
View File
@@ -149,7 +149,7 @@ impl ECStore {
let mut pool_meta = PoolMeta::new(&pools, &PoolMeta::default());
pool_meta.dont_save = true;
let decommission_cancelers = vec![None; pools.len()];
let decommission_cancelers = RwLock::new(vec![None; pools.len()]);
let ec = Arc::new(ECStore {
id: deployment_id.unwrap(),
disk_map,
@@ -265,6 +265,7 @@ impl ECStore {
init_background_expiry(self.clone()).await;
TransitionState::init(self.clone()).await;
crate::tier::tier::try_migrate_tiering_config(self.clone()).await;
if let Err(err) = GLOBAL_TierConfigMgr.write().await.init(self.clone()).await {
info!("TierConfigMgr init error: {}", err);
crates/ecstore/src/store/object.rs
+105 -6
View File
@@ -14,7 +14,64 @@
use super::*;
fn select_data_movement_target_pool(
existing_pool_idx: Result<usize>,
src_pool_idx: usize,
delete_marker: bool,
) -> Result<Option<usize>> {
match existing_pool_idx {
Ok(pool_idx) => {
if delete_marker && pool_idx == src_pool_idx {
Ok(None)
} else {
Ok(Some(pool_idx))
}
}
Err(err) => {
if is_err_read_quorum(&err) {
return Err(StorageError::ErasureWriteQuorum);
}
if delete_marker && (is_err_object_not_found(&err) || is_err_version_not_found(&err)) {
Ok(None)
} else {
Err(err)
}
}
}
}
impl ECStore {
#[instrument(skip(self, fi, opts))]
pub(crate) async fn decommission_tiered_object(
&self,
bucket: &str,
object: &str,
fi: &rustfs_filemeta::FileInfo,
opts: &ObjectOptions,
) -> Result<()> {
check_put_object_args(bucket, object)?;
let object = encode_dir_object(object);
if self.single_pool() {
return Err(Error::other(format!("error decommissioning {bucket}/{object}")));
}
let idx = self.get_pool_idx_no_lock(bucket, &object, fi.size).await?;
if opts.data_movement && idx == opts.src_pool_idx {
return Err(StorageError::DataMovementOverwriteErr(
bucket.to_owned(),
object.to_owned(),
opts.version_id.clone().unwrap_or_default(),
));
}
self.pools[idx]
.get_disks_by_key(&object)
.decommission_tiered_object(bucket, &object, fi, opts)
.await
}
#[instrument(level = "debug", skip(self))]
pub(super) async fn handle_get_object_reader(
&self,
@@ -179,6 +236,30 @@ impl ECStore {
let mut gopts = opts.clone();
gopts.no_lock = true;
if opts.data_movement {
let existing_pool_idx = self
.get_pool_info_existing_with_opts(bucket, object, &gopts)
.await
.map(|(pinfo, _)| pinfo.index);
let target_pool_idx =
match select_data_movement_target_pool(existing_pool_idx, opts.src_pool_idx, opts.delete_marker)? {
Some(pool_idx) => pool_idx,
None => self.get_pool_idx_no_lock(bucket, object, 0).await?,
};
if opts.src_pool_idx == target_pool_idx {
return Err(StorageError::DataMovementOverwriteErr(
bucket.to_owned(),
object.to_owned(),
opts.version_id.unwrap_or_default(),
));
}
let mut obj = self.pools[target_pool_idx].delete_object(bucket, object, opts).await?;
obj.name = decode_dir_object(obj.name.as_str());
return Ok(obj);
}
// Determine which pool contains it
let (mut pinfo, errs) = self
.get_pool_info_existing_with_opts(bucket, object, &gopts)
@@ -204,12 +285,6 @@ impl ECStore {
));
}
if opts.data_movement {
let mut obj = self.pools[pinfo.index].delete_object(bucket, object, opts).await?;
obj.name = decode_dir_object(obj.name.as_str());
return Ok(obj);
}
if !errs.is_empty() && !opts.versioned && !opts.version_suspended {
return self.delete_object_from_all_pools(bucket, object, &opts, errs).await;
}
@@ -565,3 +640,27 @@ impl ECStore {
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn delete_marker_data_movement_falls_back_when_only_source_pool_has_object() {
let target = select_data_movement_target_pool(Ok(1), 1, true).unwrap();
assert_eq!(target, None);
}
#[test]
fn delete_marker_data_movement_falls_back_when_version_does_not_exist_yet() {
let err = StorageError::ObjectNotFound("bucket".to_string(), "object".to_string());
let target = select_data_movement_target_pool(Err(err), 1, true).unwrap();
assert_eq!(target, None);
}
#[test]
fn non_delete_marker_data_movement_keeps_existing_pool() {
let target = select_data_movement_target_pool(Ok(0), 1, false).unwrap();
assert_eq!(target, Some(0));
}
}
crates/ecstore/src/store_api.rs
+3 -3
View File
@@ -27,8 +27,8 @@ use bytes::Bytes;
use http::{HeaderMap, HeaderValue};
use rustfs_common::heal_channel::HealOpts;
use rustfs_filemeta::{
FileInfo, MetaCacheEntriesSorted, ObjectPartInfo, REPLICATION_RESET, REPLICATION_STATUS, ReplicateDecision, ReplicationState,
ReplicationStatusType, RestoreStatusOps as _, VersionPurgeStatusType, parse_restore_obj_status, replication_statuses_map,
FileInfo, MetaCacheEntriesSorted, ObjectPartInfo, ReplicateDecision, ReplicationState, ReplicationStatusType,
RestoreStatusOps as _, VersionPurgeStatusType, parse_restore_obj_status, replication_statuses_map,
version_purge_statuses_map,
};
use rustfs_lock::NamespaceLockWrapper;
@@ -36,7 +36,7 @@ use rustfs_madmin::heal_commands::HealResultItem;
use rustfs_rio::Checksum;
use rustfs_rio::{DecompressReader, HashReader, LimitReader, WarpReader};
use rustfs_utils::CompressionAlgorithm;
use rustfs_utils::http::headers::{AMZ_OBJECT_TAGGING, RESERVED_METADATA_PREFIX_LOWER};
use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING;
use rustfs_utils::http::{AMZ_BUCKET_REPLICATION_STATUS, AMZ_RESTORE, AMZ_STORAGE_CLASS};
use rustfs_utils::path::decode_dir_object;
use serde::{Deserialize, Serialize};
crates/ecstore/src/store_api/types.rs
+14 -25
View File
@@ -118,11 +118,8 @@ impl ObjectOptions {
}
pub fn put_replication_state(&self) -> ReplicationState {
let rs = match self
.user_defined
.get(format!("{RESERVED_METADATA_PREFIX_LOWER}{REPLICATION_STATUS}").as_str())
{
Some(v) => v.to_string(),
let rs = match rustfs_utils::http::get_str(&self.user_defined, rustfs_utils::http::SUFFIX_REPLICATION_STATUS) {
Some(v) => v,
None => return ReplicationState::default(),
};
@@ -341,15 +338,11 @@ impl Clone for ObjectInfo {
impl ObjectInfo {
pub fn is_compressed(&self) -> bool {
self.user_defined
.contains_key(&format!("{RESERVED_METADATA_PREFIX_LOWER}compression"))
rustfs_utils::http::contains_key_str(&self.user_defined, rustfs_utils::http::SUFFIX_COMPRESSION)
}
pub fn is_compressed_ok(&self) -> Result<(CompressionAlgorithm, bool)> {
let scheme = self
.user_defined
.get(&format!("{RESERVED_METADATA_PREFIX_LOWER}compression"))
.cloned();
let scheme = rustfs_utils::http::get_str(&self.user_defined, rustfs_utils::http::SUFFIX_COMPRESSION);
if let Some(scheme) = scheme {
let algorithm = CompressionAlgorithm::from_str(&scheme)?;
@@ -369,7 +362,7 @@ impl ObjectInfo {
}
if self.is_compressed() {
if let Some(size_str) = self.user_defined.get(&format!("{RESERVED_METADATA_PREFIX_LOWER}actual-size"))
if let Some(size_str) = rustfs_utils::http::get_str(&self.user_defined, rustfs_utils::http::SUFFIX_ACTUAL_SIZE)
&& !size_str.is_empty()
{
// Todo: deal with error
@@ -745,15 +738,11 @@ impl ObjectInfo {
.user_defined
.iter()
.filter_map(|(k, v)| {
if k.starts_with(&format!("{RESERVED_METADATA_PREFIX_LOWER}{REPLICATION_RESET}")) {
Some((
k.trim_start_matches(&format!("{RESERVED_METADATA_PREFIX_LOWER}{REPLICATION_RESET}-"))
.to_string(),
v.clone(),
))
} else {
None
}
rustfs_utils::http::internal_key_strip_suffix_prefix(
k,
rustfs_utils::http::SUFFIX_REPLICATION_RESET_ARN_PREFIX,
)
.map(|arn| (arn, v.clone()))
})
.collect(),
..Default::default()
@@ -1035,8 +1024,8 @@ mod tests {
fn get_actual_size_uses_compressed_metadata_size() {
let user_defined = {
let mut map = HashMap::new();
map.insert(format!("{RESERVED_METADATA_PREFIX_LOWER}compression"), "zstd".to_string());
map.insert(format!("{RESERVED_METADATA_PREFIX_LOWER}actual-size"), "42".to_string());
rustfs_utils::http::insert_str(&mut map, rustfs_utils::http::SUFFIX_COMPRESSION, "zstd".to_string());
rustfs_utils::http::insert_str(&mut map, rustfs_utils::http::SUFFIX_ACTUAL_SIZE, "42".to_string());
map
};
@@ -1072,7 +1061,7 @@ mod tests {
fn get_actual_size_uses_compressed_parts_actual_size_when_metadata_missing() {
let user_defined = {
let mut map = HashMap::new();
map.insert(format!("{RESERVED_METADATA_PREFIX_LOWER}compression"), "zstd".to_string());
rustfs_utils::http::insert_str(&mut map, rustfs_utils::http::SUFFIX_COMPRESSION, "zstd".to_string());
map
};
@@ -1100,7 +1089,7 @@ mod tests {
fn get_actual_size_returns_error_when_compressed_parts_missing_and_size_mismatch() {
let user_defined = {
let mut map = HashMap::new();
map.insert(format!("{RESERVED_METADATA_PREFIX_LOWER}compression"), "zstd".to_string());
rustfs_utils::http::insert_str(&mut map, rustfs_utils::http::SUFFIX_COMPRESSION, "zstd".to_string());
map
};
crates/ecstore/src/store_init.rs
+60 -4
View File
@@ -18,7 +18,7 @@ use crate::disk::{self, DiskAPI};
use crate::error::{Error, Result};
use crate::{
disk::{
DiskInfoOptions, DiskOption, DiskStore, FORMAT_CONFIG_FILE, RUSTFS_META_BUCKET,
DiskInfoOptions, DiskOption, DiskStore, FORMAT_CONFIG_FILE, MIGRATING_META_BUCKET, RUSTFS_META_BUCKET,
error::DiskError,
format::{FormatErasureVersion, FormatMetaVersion, FormatV3},
new_disk,
@@ -71,11 +71,13 @@ pub async fn connect_load_init_formats(
check_format_erasure_values(&formats, set_drive_count)?;
if first_disk && should_init_erasure_disks(&errs) {
// UnformattedDisk, not format file create
// UnformattedDisk, try migrate from MinIO format first, else create new format
info!("first_disk && should_init_erasure_disks");
// new format and save
if let Ok(fm) = try_migrate_format(disks, set_count, set_drive_count).await {
info!("Migrated format from MinIO config");
return Ok(fm);
}
let fm = init_format_erasure(disks, set_count, set_drive_count, deployment_id).await?;
return Ok(fm);
}
@@ -149,6 +151,60 @@ async fn init_format_erasure(
get_format_erasure_in_quorum(&fms)
}
/// Tries to migrate format
/// Returns Ok(FormatV3) if migration succeeds, Err otherwise.
async fn try_migrate_format(disks: &[Option<DiskStore>], set_count: usize, set_drive_count: usize) -> Result<FormatV3> {
for disk in disks.iter().flatten() {
let data = match disk.read_all(MIGRATING_META_BUCKET, FORMAT_CONFIG_FILE).await {
Ok(d) if !d.is_empty() => d,
_ => continue,
};
let fm = FormatV3::try_from(data.as_ref()).map_err(|e| Error::other(format!("parse MinIO format: {e}")))?;
let first_set = fm
.erasure
.sets
.first()
.ok_or_else(|| Error::other("MinIO format: erasure.sets is empty"))?;
if fm.erasure.sets.len() != set_count || first_set.len() != set_drive_count {
debug!(
"MinIO format set count mismatch: got {}x{}, expected {}x{}",
fm.erasure.sets.len(),
first_set.len(),
set_count,
set_drive_count
);
continue;
}
if fm.erasure.version != FormatErasureVersion::V3 {
debug!("MinIO format erasure version not V3: {:?}", fm.erasure.version);
continue;
}
let mut fms = vec![None; disks.len()];
for (idx, disk_opt) in disks.iter().enumerate() {
if disk_opt.is_none() {
continue;
}
let set_idx = idx / set_drive_count;
let disk_idx = idx % set_drive_count;
if set_idx >= fm.erasure.sets.len() || disk_idx >= fm.erasure.sets[set_idx].len() {
continue;
}
let mut newfm = fm.clone();
newfm.erasure.this = fm.erasure.sets[set_idx][disk_idx];
fms[idx] = Some(newfm);
}
save_format_file_all(disks, &fms).await?;
return get_format_erasure_in_quorum(&fms);
}
Err(Error::other("no MinIO format to migrate"))
}
pub fn get_format_erasure_in_quorum(formats: &[Option<FormatV3>]) -> Result<FormatV3> {
let mut countmap = HashMap::new();
crates/ecstore/src/store_list_objects.rs
+1 -1
View File
@@ -492,7 +492,7 @@ impl ECStore {
}
pub async fn list_path(self: Arc<Self>, o: &ListPathOptions) -> Result<MetaCacheEntriesSortedResult> {
// warn!("list_path opt {:?}", &o);
// tracing::warn!("list_path opt {:?}", &o);
check_list_objs_args(&o.bucket, &o.prefix, &o.marker)?;
// if opts.prefix.ends_with(SLASH_SEPARATOR) {
crates/ecstore/src/store_utils.rs
+4 -2
View File
@@ -13,7 +13,7 @@
// limitations under the License.
use crate::config::storageclass::STANDARD;
use crate::disk::RUSTFS_META_BUCKET;
use crate::disk::{MIGRATING_META_BUCKET, RUSTFS_META_BUCKET};
use regex::Regex;
use rustfs_utils::http::headers::{AMZ_OBJECT_TAGGING, AMZ_STORAGE_CLASS};
use std::collections::HashMap;
@@ -45,7 +45,7 @@ pub fn clean_metadata_keys(metadata: &mut HashMap<String, String>, key_names: &[
// Check whether the bucket is the metadata bucket
fn is_meta_bucket(bucket_name: &str) -> bool {
bucket_name == RUSTFS_META_BUCKET
bucket_name == RUSTFS_META_BUCKET || bucket_name == MIGRATING_META_BUCKET
}
// Check whether the bucket is reserved
@@ -164,6 +164,8 @@ mod tests {
fn test_meta_bucket_is_invalid() {
assert!(is_reserved_or_invalid_bucket(RUSTFS_META_BUCKET, false));
assert!(is_reserved_or_invalid_bucket(RUSTFS_META_BUCKET, true));
assert!(is_reserved_or_invalid_bucket(MIGRATING_META_BUCKET, false));
assert!(is_reserved_or_invalid_bucket(MIGRATING_META_BUCKET, true));
}
#[test]
crates/ecstore/src/tier/tier.rs
+839 -37
View File
@@ -18,14 +18,16 @@
#![allow(unused_must_use)]
#![allow(clippy::all)]
use byteorder::{ByteOrder, LittleEndian};
use bytes::Bytes;
use http::HeaderMap;
use http::status::StatusCode;
use lazy_static::lazy_static;
use rand::{Rng, RngExt};
use serde::{Deserialize, Serialize};
use std::{
collections::{HashMap, hash_map::Entry},
io::Cursor,
io::{self, Cursor},
sync::Arc,
time::Duration,
};
@@ -46,9 +48,9 @@ use crate::tier::{
use crate::{
StorageAPI,
config::com::{CONFIG_PREFIX, read_config},
disk::RUSTFS_META_BUCKET,
disk::{MIGRATING_META_BUCKET, RUSTFS_META_BUCKET},
store::ECStore,
store_api::{ObjectOptions, PutObjReader},
store_api::{ObjectIO as _, ObjectOptions, PutObjReader},
};
use rustfs_rio::HashReader;
use rustfs_utils::path::{SLASH_SEPARATOR, path_join};
@@ -61,10 +63,17 @@ use super::{
const TIER_CFG_REFRESH: Duration = Duration::from_secs(15 * 60);
pub const TIER_CONFIG_FILE: &str = "tier-config.json";
const TIER_CONFIG_LEGACY_FILE: &str = "tier-config.json";
pub const TIER_CONFIG_FILE: &str = "tier-config.bin";
pub const TIER_CONFIG_FORMAT: u16 = 1;
pub const TIER_CONFIG_V1: u16 = 1;
pub const TIER_CONFIG_VERSION: u16 = 1;
pub const TIER_CONFIG_VERSION: u16 = 2;
const EXTERNAL_TIER_TYPE_UNSUPPORTED: i32 = 0;
const EXTERNAL_TIER_TYPE_S3: i32 = 1;
const EXTERNAL_TIER_TYPE_AZURE: i32 = 2;
const EXTERNAL_TIER_TYPE_GCS: i32 = 3;
const EXTERNAL_TIER_TYPE_MINIO: i32 = 4;
const _TIER_CFG_REFRESH_AT_HDR: &str = "X-RustFS-TierCfg-RefreshedAt";
@@ -104,6 +113,609 @@ pub struct TierConfigMgr {
pub last_refreshed_at: OffsetDateTime,
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
#[serde(default)]
struct ExternalTierConfigMgr {
#[serde(rename = "Tiers")]
tiers: HashMap<String, ExternalTierConfig>,
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
#[serde(default)]
struct ExternalTierConfig {
#[serde(rename = "Version")]
version: String,
#[serde(rename = "Type")]
tier_type: i32,
#[serde(rename = "Name")]
name: String,
#[serde(rename = "S3")]
s3: Option<ExternalTierS3>,
#[serde(rename = "Azure")]
azure: Option<ExternalTierAzure>,
#[serde(rename = "GCS")]
gcs: Option<ExternalTierGcs>,
#[serde(rename = "MinIO", alias = "Compatible")]
compatible_backend: Option<ExternalTierCompatible>,
#[serde(rename = "XTierType", skip_serializing_if = "Option::is_none")]
tier_type_hint: Option<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
#[serde(default)]
struct ExternalTierS3 {
#[serde(rename = "Endpoint")]
endpoint: String,
#[serde(rename = "AccessKey")]
access_key: String,
#[serde(rename = "SecretKey")]
secret_key: String,
#[serde(rename = "Bucket")]
bucket: String,
#[serde(rename = "Prefix")]
prefix: String,
#[serde(rename = "Region")]
region: String,
#[serde(rename = "StorageClass")]
storage_class: String,
#[serde(rename = "AWSRole")]
aws_role: bool,
#[serde(rename = "AWSRoleWebIdentityTokenFile")]
aws_role_web_identity_token_file: String,
#[serde(rename = "AWSRoleARN")]
aws_role_arn: String,
#[serde(rename = "AWSRoleSessionName")]
aws_role_session_name: String,
#[serde(rename = "AWSRoleDurationSeconds")]
aws_role_duration_seconds: i32,
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
#[serde(default)]
struct ExternalServicePrincipalAuth {
#[serde(rename = "TenantID")]
tenant_id: String,
#[serde(rename = "ClientID")]
client_id: String,
#[serde(rename = "ClientSecret")]
client_secret: String,
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
#[serde(default)]
struct ExternalTierAzure {
#[serde(rename = "Endpoint")]
endpoint: String,
#[serde(rename = "AccountName")]
account_name: String,
#[serde(rename = "AccountKey")]
account_key: String,
#[serde(rename = "Bucket")]
bucket: String,
#[serde(rename = "Prefix")]
prefix: String,
#[serde(rename = "Region")]
region: String,
#[serde(rename = "StorageClass")]
storage_class: String,
#[serde(rename = "SPAuth")]
sp_auth: ExternalServicePrincipalAuth,
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
#[serde(default)]
struct ExternalTierGcs {
#[serde(rename = "Endpoint")]
endpoint: String,
#[serde(rename = "Creds")]
creds: String,
#[serde(rename = "Bucket")]
bucket: String,
#[serde(rename = "Prefix")]
prefix: String,
#[serde(rename = "Region")]
region: String,
#[serde(rename = "StorageClass")]
storage_class: String,
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
#[serde(default)]
struct ExternalTierCompatible {
#[serde(rename = "Endpoint")]
endpoint: String,
#[serde(rename = "AccessKey")]
access_key: String,
#[serde(rename = "SecretKey")]
secret_key: String,
#[serde(rename = "Bucket")]
bucket: String,
#[serde(rename = "Prefix")]
prefix: String,
#[serde(rename = "Region")]
region: String,
}
fn tier_config_path(file: &str) -> String {
format!("{}{}{}", CONFIG_PREFIX, SLASH_SEPARATOR, file)
}
fn tier_hint_for_type(tier_type: TierType) -> Option<&'static str> {
match tier_type {
TierType::RustFS => Some("rustfs"),
TierType::Aliyun => Some("aliyun"),
TierType::Tencent => Some("tencent"),
TierType::Huaweicloud => Some("huaweicloud"),
TierType::R2 => Some("r2"),
_ => None,
}
}
fn tier_type_from_hint(hint: Option<&str>) -> Option<TierType> {
match hint {
Some("rustfs") => Some(TierType::RustFS),
Some("aliyun") => Some(TierType::Aliyun),
Some("tencent") => Some(TierType::Tencent),
Some("huaweicloud") => Some(TierType::Huaweicloud),
Some("r2") => Some(TierType::R2),
_ => None,
}
}
fn external_tier_s3_from_internal(s3: &crate::tier::tier_config::TierS3) -> ExternalTierS3 {
ExternalTierS3 {
endpoint: s3.endpoint.clone(),
access_key: s3.access_key.clone(),
secret_key: s3.secret_key.clone(),
bucket: s3.bucket.clone(),
prefix: s3.prefix.clone(),
region: s3.region.clone(),
storage_class: s3.storage_class.clone(),
aws_role: s3.aws_role,
aws_role_web_identity_token_file: s3.aws_role_web_identity_token_file.clone(),
aws_role_arn: s3.aws_role_arn.clone(),
aws_role_session_name: s3.aws_role_session_name.clone(),
aws_role_duration_seconds: s3.aws_role_duration_seconds,
}
}
fn external_tier_s3_from_compatible_payload(
endpoint: String,
access_key: String,
secret_key: String,
bucket: String,
prefix: String,
region: String,
) -> ExternalTierS3 {
ExternalTierS3 {
endpoint,
access_key,
secret_key,
bucket,
prefix,
region,
storage_class: String::new(),
aws_role: false,
aws_role_web_identity_token_file: String::new(),
aws_role_arn: String::new(),
aws_role_session_name: String::new(),
aws_role_duration_seconds: 0,
}
}
fn external_tier_alias_from_compatible_payload(
endpoint: String,
access_key: String,
secret_key: String,
bucket: String,
prefix: String,
region: String,
) -> ExternalTierCompatible {
ExternalTierCompatible {
endpoint,
access_key,
secret_key,
bucket,
prefix,
region,
}
}
fn to_external_tier_config(name: &str, tier: &TierConfig) -> io::Result<ExternalTierConfig> {
let mut out = ExternalTierConfig {
version: if tier.version.is_empty() {
"v1".to_string()
} else {
tier.version.clone()
},
name: if tier.name.is_empty() {
name.to_string()
} else {
tier.name.clone()
},
..Default::default()
};
match tier.tier_type {
TierType::S3 => {
let s3 = tier
.s3
.as_ref()
.ok_or_else(|| io::Error::other("tier config missing s3 backend payload"))?;
out.tier_type = EXTERNAL_TIER_TYPE_S3;
out.s3 = Some(external_tier_s3_from_internal(s3));
}
TierType::Azure => {
let az = tier
.azure
.as_ref()
.ok_or_else(|| io::Error::other("tier config missing azure backend payload"))?;
out.tier_type = EXTERNAL_TIER_TYPE_AZURE;
out.azure = Some(ExternalTierAzure {
endpoint: az.endpoint.clone(),
account_name: az.access_key.clone(),
account_key: az.secret_key.clone(),
bucket: az.bucket.clone(),
prefix: az.prefix.clone(),
region: az.region.clone(),
storage_class: az.storage_class.clone(),
sp_auth: ExternalServicePrincipalAuth {
tenant_id: az.sp_auth.tenant_id.clone(),
client_id: az.sp_auth.client_id.clone(),
client_secret: az.sp_auth.client_secret.clone(),
},
});
}
TierType::GCS => {
let gcs = tier
.gcs
.as_ref()
.ok_or_else(|| io::Error::other("tier config missing gcs backend payload"))?;
out.tier_type = EXTERNAL_TIER_TYPE_GCS;
out.gcs = Some(ExternalTierGcs {
endpoint: gcs.endpoint.clone(),
creds: gcs.creds.clone(),
bucket: gcs.bucket.clone(),
prefix: gcs.prefix.clone(),
region: gcs.region.clone(),
storage_class: gcs.storage_class.clone(),
});
}
TierType::MinIO => {
let backend = tier
.minio
.as_ref()
.ok_or_else(|| io::Error::other("tier config missing compatible backend payload"))?;
out.tier_type = EXTERNAL_TIER_TYPE_MINIO;
out.compatible_backend = Some(external_tier_alias_from_compatible_payload(
backend.endpoint.clone(),
backend.access_key.clone(),
backend.secret_key.clone(),
backend.bucket.clone(),
backend.prefix.clone(),
backend.region.clone(),
));
}
TierType::RustFS => {
let backend = tier
.rustfs
.as_ref()
.ok_or_else(|| io::Error::other("tier config missing compatible backend payload"))?;
out.tier_type = EXTERNAL_TIER_TYPE_S3;
out.tier_type_hint = tier_hint_for_type(tier.tier_type.clone()).map(ToString::to_string);
out.s3 = Some(external_tier_s3_from_compatible_payload(
backend.endpoint.clone(),
backend.access_key.clone(),
backend.secret_key.clone(),
backend.bucket.clone(),
backend.prefix.clone(),
backend.region.clone(),
));
}
TierType::Aliyun => {
let backend = tier
.aliyun
.as_ref()
.ok_or_else(|| io::Error::other("tier config missing compatible backend payload"))?;
out.tier_type = EXTERNAL_TIER_TYPE_S3;
out.tier_type_hint = tier_hint_for_type(tier.tier_type.clone()).map(ToString::to_string);
out.s3 = Some(external_tier_s3_from_compatible_payload(
backend.endpoint.clone(),
backend.access_key.clone(),
backend.secret_key.clone(),
backend.bucket.clone(),
backend.prefix.clone(),
backend.region.clone(),
));
}
TierType::Tencent => {
let backend = tier
.tencent
.as_ref()
.ok_or_else(|| io::Error::other("tier config missing compatible backend payload"))?;
out.tier_type = EXTERNAL_TIER_TYPE_S3;
out.tier_type_hint = tier_hint_for_type(tier.tier_type.clone()).map(ToString::to_string);
out.s3 = Some(external_tier_s3_from_compatible_payload(
backend.endpoint.clone(),
backend.access_key.clone(),
backend.secret_key.clone(),
backend.bucket.clone(),
backend.prefix.clone(),
backend.region.clone(),
));
}
TierType::Huaweicloud => {
let backend = tier
.huaweicloud
.as_ref()
.ok_or_else(|| io::Error::other("tier config missing compatible backend payload"))?;
out.tier_type = EXTERNAL_TIER_TYPE_S3;
out.tier_type_hint = tier_hint_for_type(tier.tier_type.clone()).map(ToString::to_string);
out.s3 = Some(external_tier_s3_from_compatible_payload(
backend.endpoint.clone(),
backend.access_key.clone(),
backend.secret_key.clone(),
backend.bucket.clone(),
backend.prefix.clone(),
backend.region.clone(),
));
}
TierType::R2 => {
let backend = tier
.r2
.as_ref()
.ok_or_else(|| io::Error::other("tier config missing compatible backend payload"))?;
out.tier_type = EXTERNAL_TIER_TYPE_S3;
out.tier_type_hint = tier_hint_for_type(tier.tier_type.clone()).map(ToString::to_string);
out.s3 = Some(external_tier_s3_from_compatible_payload(
backend.endpoint.clone(),
backend.access_key.clone(),
backend.secret_key.clone(),
backend.bucket.clone(),
backend.prefix.clone(),
backend.region.clone(),
));
}
TierType::Unsupported => {
out.tier_type = EXTERNAL_TIER_TYPE_UNSUPPORTED;
}
}
Ok(out)
}
fn decode_legacy_s3_like(name: &str, ext: &ExternalTierConfig) -> io::Result<ExternalTierS3> {
if let Some(s3) = ext.s3.as_ref() {
return Ok(s3.clone());
}
if let Some(m) = ext.compatible_backend.as_ref() {
return Ok(external_tier_s3_from_compatible_payload(
m.endpoint.clone(),
m.access_key.clone(),
m.secret_key.clone(),
m.bucket.clone(),
m.prefix.clone(),
m.region.clone(),
));
}
Err(io::Error::other(format!("tier config '{name}' missing compatible backend payload")))
}
fn from_external_tier_config(name: String, ext: ExternalTierConfig) -> io::Result<TierConfig> {
let mut cfg = TierConfig {
version: if ext.version.is_empty() {
"v1".to_string()
} else {
ext.version.clone()
},
name: if ext.name.is_empty() { name.clone() } else { ext.name.clone() },
..Default::default()
};
let hinted = tier_type_from_hint(ext.tier_type_hint.as_deref());
let tier_type = if let Some(h) = hinted {
h
} else {
match ext.tier_type {
EXTERNAL_TIER_TYPE_S3 => TierType::S3,
EXTERNAL_TIER_TYPE_AZURE => TierType::Azure,
EXTERNAL_TIER_TYPE_GCS => TierType::GCS,
EXTERNAL_TIER_TYPE_MINIO => TierType::MinIO,
_ => TierType::Unsupported,
}
};
cfg.tier_type = tier_type.clone();
match tier_type {
TierType::S3 => {
let s3 = ext
.s3
.as_ref()
.ok_or_else(|| io::Error::other(format!("tier config '{}' missing s3 backend payload", cfg.name)))?;
cfg.s3 = Some(crate::tier::tier_config::TierS3 {
name: cfg.name.clone(),
endpoint: s3.endpoint.clone(),
access_key: s3.access_key.clone(),
secret_key: s3.secret_key.clone(),
bucket: s3.bucket.clone(),
prefix: s3.prefix.clone(),
region: s3.region.clone(),
storage_class: s3.storage_class.clone(),
aws_role: s3.aws_role,
aws_role_web_identity_token_file: s3.aws_role_web_identity_token_file.clone(),
aws_role_arn: s3.aws_role_arn.clone(),
aws_role_session_name: s3.aws_role_session_name.clone(),
aws_role_duration_seconds: s3.aws_role_duration_seconds,
});
}
TierType::Azure => {
let az = ext
.azure
.as_ref()
.ok_or_else(|| io::Error::other(format!("tier config '{}' missing azure backend payload", cfg.name)))?;
cfg.azure = Some(crate::tier::tier_config::TierAzure {
name: cfg.name.clone(),
endpoint: az.endpoint.clone(),
access_key: az.account_name.clone(),
secret_key: az.account_key.clone(),
bucket: az.bucket.clone(),
prefix: az.prefix.clone(),
region: az.region.clone(),
storage_class: az.storage_class.clone(),
sp_auth: crate::tier::tier_config::ServicePrincipalAuth {
tenant_id: az.sp_auth.tenant_id.clone(),
client_id: az.sp_auth.client_id.clone(),
client_secret: az.sp_auth.client_secret.clone(),
},
});
}
TierType::GCS => {
let gcs = ext
.gcs
.as_ref()
.ok_or_else(|| io::Error::other(format!("tier config '{}' missing gcs backend payload", cfg.name)))?;
cfg.gcs = Some(crate::tier::tier_config::TierGCS {
name: cfg.name.clone(),
endpoint: gcs.endpoint.clone(),
creds: gcs.creds.clone(),
bucket: gcs.bucket.clone(),
prefix: gcs.prefix.clone(),
region: gcs.region.clone(),
storage_class: gcs.storage_class.clone(),
});
}
TierType::MinIO => {
let m = ext
.compatible_backend
.as_ref()
.ok_or_else(|| io::Error::other(format!("tier config '{}' missing compatible backend payload", cfg.name)))?;
cfg.minio = Some(crate::tier::tier_config::TierMinIO {
name: cfg.name.clone(),
endpoint: m.endpoint.clone(),
access_key: m.access_key.clone(),
secret_key: m.secret_key.clone(),
bucket: m.bucket.clone(),
prefix: m.prefix.clone(),
region: m.region.clone(),
});
}
TierType::RustFS => {
let m = decode_legacy_s3_like(&cfg.name, &ext)?;
cfg.rustfs = Some(crate::tier::tier_config::TierRustFS {
name: cfg.name.clone(),
endpoint: m.endpoint,
access_key: m.access_key,
secret_key: m.secret_key,
bucket: m.bucket,
prefix: m.prefix,
region: m.region,
storage_class: m.storage_class,
});
}
TierType::Aliyun => {
let m = decode_legacy_s3_like(&cfg.name, &ext)?;
cfg.aliyun = Some(crate::tier::tier_config::TierAliyun {
name: cfg.name.clone(),
endpoint: m.endpoint,
access_key: m.access_key,
secret_key: m.secret_key,
bucket: m.bucket,
prefix: m.prefix,
region: m.region,
});
}
TierType::Tencent => {
let m = decode_legacy_s3_like(&cfg.name, &ext)?;
cfg.tencent = Some(crate::tier::tier_config::TierTencent {
name: cfg.name.clone(),
endpoint: m.endpoint,
access_key: m.access_key,
secret_key: m.secret_key,
bucket: m.bucket,
prefix: m.prefix,
region: m.region,
});
}
TierType::Huaweicloud => {
let m = decode_legacy_s3_like(&cfg.name, &ext)?;
cfg.huaweicloud = Some(crate::tier::tier_config::TierHuaweicloud {
name: cfg.name.clone(),
endpoint: m.endpoint,
access_key: m.access_key,
secret_key: m.secret_key,
bucket: m.bucket,
prefix: m.prefix,
region: m.region,
});
}
TierType::R2 => {
let m = decode_legacy_s3_like(&cfg.name, &ext)?;
cfg.r2 = Some(crate::tier::tier_config::TierR2 {
name: cfg.name.clone(),
endpoint: m.endpoint,
access_key: m.access_key,
secret_key: m.secret_key,
bucket: m.bucket,
prefix: m.prefix,
region: m.region,
});
}
TierType::Unsupported => {}
}
Ok(cfg)
}
fn encode_external_tiering_config_blob(cfg: &TierConfigMgr) -> io::Result<Bytes> {
let mut tiers = HashMap::with_capacity(cfg.tiers.len());
for (name, tier_cfg) in &cfg.tiers {
tiers.insert(name.clone(), to_external_tier_config(name, tier_cfg)?);
}
let payload = rmp_serde::to_vec(&ExternalTierConfigMgr { tiers })
.map_err(|err| io::Error::other(format!("serialize tier config payload failed: {err}")))?;
let mut data = Vec::with_capacity(4 + payload.len());
let mut format = [0u8; 2];
LittleEndian::write_u16(&mut format, TIER_CONFIG_FORMAT);
data.extend_from_slice(&format);
let mut version = [0u8; 2];
LittleEndian::write_u16(&mut version, TIER_CONFIG_VERSION);
data.extend_from_slice(&version);
data.extend_from_slice(&payload);
Ok(Bytes::from(data))
}
fn decode_external_tiering_config_blob(data: &[u8]) -> io::Result<TierConfigMgr> {
if data.len() <= 4 {
return Err(io::Error::other("tierConfigInit: no data"));
}
let format = LittleEndian::read_u16(&data[0..2]);
if format != TIER_CONFIG_FORMAT {
return Err(io::Error::other(format!("tierConfigInit: unknown format: {format}")));
}
let version = LittleEndian::read_u16(&data[2..4]);
if version != TIER_CONFIG_V1 && version != TIER_CONFIG_VERSION {
return Err(io::Error::other(format!("tierConfigInit: unknown version: {version}")));
}
let external: ExternalTierConfigMgr =
rmp_serde::from_slice(&data[4..]).map_err(|err| io::Error::other(format!("decode tier config payload failed: {err}")))?;
let mut tiers = HashMap::with_capacity(external.tiers.len());
for (name, ext_cfg) in external.tiers {
tiers.insert(name.clone(), from_external_tier_config(name, ext_cfg)?);
}
Ok(TierConfigMgr {
driver_cache: HashMap::new(),
tiers,
last_refreshed_at: OffsetDateTime::now_utc(),
})
}
fn decode_tiering_config_blob(data: &[u8]) -> io::Result<TierConfigMgr> {
if let Ok(cfg) = TierConfigMgr::unmarshal(data) {
return Ok(cfg);
}
decode_external_tiering_config_blob(data)
}
impl TierConfigMgr {
pub fn new() -> Arc<RwLock<Self>> {
Arc::new(RwLock::new(Self {
@@ -285,12 +897,12 @@ impl TierConfigMgr {
rustfs.secret_key = creds.secret_key;
}
TierType::MinIO => {
let mut minio = tier_config.minio.as_mut().expect("err");
let compatible_backend = tier_config.minio.as_mut().expect("err");
if creds.access_key == "" || creds.secret_key == "" {
return Err(ERR_TIER_MISSING_CREDENTIALS.clone());
}
minio.access_key = creds.access_key;
minio.secret_key = creds.secret_key;
compatible_backend.access_key = creds.access_key;
compatible_backend.secret_key = creds.secret_key;
}
TierType::Aliyun => {
let mut aliyun = tier_config.aliyun.as_mut().expect("err");
@@ -401,9 +1013,8 @@ impl TierConfigMgr {
}
pub async fn save_tiering_config<S: StorageAPI>(&self, api: Arc<S>) -> std::result::Result<(), std::io::Error> {
let data = self.marshal()?;
let config_file = format!("{}{}{}", CONFIG_PREFIX, SLASH_SEPARATOR, TIER_CONFIG_FILE);
let data = encode_external_tiering_config_blob(self)?;
let config_file = tier_config_path(TIER_CONFIG_FILE);
self.save_config(api, &config_file, data).await
}
@@ -483,38 +1094,229 @@ async fn new_and_save_tiering_config<S: StorageAPI>(api: Arc<S>) -> Result<TierC
#[tracing::instrument(level = "debug", name = "load_tier_config", skip(api))]
async fn load_tier_config(api: Arc<ECStore>) -> std::result::Result<TierConfigMgr, std::io::Error> {
let config_file = format!("{}{}{}", CONFIG_PREFIX, SLASH_SEPARATOR, TIER_CONFIG_FILE);
let data = read_config(api.clone(), config_file.as_str()).await;
if let Err(err) = data {
if is_err_config_not_found(&err) {
warn!("config not found, start to init");
let cfg = new_and_save_tiering_config(api).await?;
return Ok(cfg);
} else {
error!("read config err {:?}", &err);
return Err(std::io::Error::other(err));
}
}
let cfg;
let version = 1; //LittleEndian::read_u16(&data[2..4]);
match version {
TIER_CONFIG_V1/* | TIER_CONFIG_VERSION */ => {
cfg = match TierConfigMgr::unmarshal(&data.unwrap()) {
Ok(cfg) => cfg,
Err(err) => {
return Err(std::io::Error::other(err.to_string()));
let config_file = tier_config_path(TIER_CONFIG_FILE);
match read_config(api.clone(), config_file.as_str()).await {
Ok(data) => decode_tiering_config_blob(&data),
Err(err) if is_err_config_not_found(&err) => {
let legacy_file = tier_config_path(TIER_CONFIG_LEGACY_FILE);
match read_config(api.clone(), legacy_file.as_str()).await {
Ok(data) => {
let cfg = TierConfigMgr::unmarshal(&data)?;
let normalized = encode_external_tiering_config_blob(&cfg)?;
let _ = api
.put_object(
RUSTFS_META_BUCKET,
&config_file,
&mut PutObjReader::from_vec(normalized.to_vec()),
&ObjectOptions {
max_parity: true,
..Default::default()
},
)
.await;
Ok(cfg)
}
};
Err(legacy_err) if is_err_config_not_found(&legacy_err) => {
warn!("config not found, start to init");
new_and_save_tiering_config(api).await.map_err(io::Error::other)
}
Err(legacy_err) => Err(io::Error::other(legacy_err)),
}
}
_ => {
return Err(std::io::Error::other(format!("tierConfigInit: unknown version: {}", version)));
Err(err) => {
error!("read config err {:?}", &err);
Err(io::Error::other(err))
}
}
}
Ok(cfg)
async fn read_tier_config_from_bucket<S: StorageAPI>(
api: Arc<S>,
bucket: &str,
path: &str,
opts: &ObjectOptions,
) -> io::Result<Option<Vec<u8>>> {
let mut rd = match api.get_object_reader(bucket, path, None, HeaderMap::new(), opts).await {
Ok(v) => v,
Err(err) if is_err_config_not_found(&err) => return Ok(None),
Err(err) => return Err(io::Error::other(err)),
};
let data = rd.read_all().await.map_err(io::Error::other)?;
if data.is_empty() {
return Ok(None);
}
Ok(Some(data))
}
async fn write_tier_config_to_rustfs<S: StorageAPI>(api: Arc<S>, path: &str, data: Bytes) -> io::Result<()> {
api.put_object(
RUSTFS_META_BUCKET,
path,
&mut PutObjReader::from_vec(data.to_vec()),
&ObjectOptions {
max_parity: true,
..Default::default()
},
)
.await
.map_err(io::Error::other)?;
Ok(())
}
pub async fn try_migrate_tiering_config<S: StorageAPI>(api: Arc<S>) {
let target_path = tier_config_path(TIER_CONFIG_FILE);
if api
.get_object_info(RUSTFS_META_BUCKET, &target_path, &ObjectOptions::default())
.await
.is_ok()
{
debug!("tier config already exists in RustFS metadata bucket, skip migration");
return;
}
let opts = ObjectOptions {
max_parity: true,
no_lock: true,
..Default::default()
};
let legacy_path = tier_config_path(TIER_CONFIG_LEGACY_FILE);
match read_tier_config_from_bucket(api.clone(), RUSTFS_META_BUCKET, &legacy_path, &opts).await {
Ok(Some(data)) => match TierConfigMgr::unmarshal(&data)
.and_then(|cfg| encode_external_tiering_config_blob(&cfg).map_err(io::Error::other))
{
Ok(out) => {
if write_tier_config_to_rustfs(api.clone(), &target_path, out).await.is_ok() {
info!("Migrated tier config from legacy RustFS metadata format");
return;
}
}
Err(err) => warn!("legacy tier config is incompatible, skip local migration: {}", err),
},
Ok(None) => {}
Err(err) => warn!("read legacy local tier config failed: {}", err),
}
match read_tier_config_from_bucket(api.clone(), MIGRATING_META_BUCKET, &target_path, &opts).await {
Ok(Some(data)) => match decode_tiering_config_blob(&data).and_then(|cfg| encode_external_tiering_config_blob(&cfg)) {
Ok(out) => {
if write_tier_config_to_rustfs(api.clone(), &target_path, out).await.is_ok() {
info!("Migrated compatible tier config from migrating metadata bucket");
}
}
Err(err) => warn!("migrating tier config is incompatible, skip migration: {}", err),
},
Ok(None) => {}
Err(err) => warn!("read migrating tier config failed: {}", err),
}
}
pub fn is_err_config_not_found(err: &StorageError) -> bool {
matches!(err, StorageError::ObjectNotFound(_, _)) || err == &StorageError::ConfigNotFound
matches!(err, StorageError::ObjectNotFound(_, _) | StorageError::BucketNotFound(_)) || err == &StorageError::ConfigNotFound
}
#[cfg(test)]
mod tests {
use super::*;
fn build_s3_tier(name: &str) -> TierConfig {
TierConfig {
version: "v1".to_string(),
tier_type: TierType::S3,
name: name.to_string(),
s3: Some(crate::tier::tier_config::TierS3 {
name: name.to_string(),
endpoint: "https://example-s3.invalid".to_string(),
access_key: "ak".to_string(),
secret_key: "sk".to_string(),
bucket: "bucket-a".to_string(),
prefix: "prefix-a".to_string(),
region: "us-east-1".to_string(),
storage_class: "STANDARD".to_string(),
aws_role: false,
aws_role_web_identity_token_file: String::new(),
aws_role_arn: String::new(),
aws_role_session_name: String::new(),
aws_role_duration_seconds: 0,
}),
..Default::default()
}
}
#[test]
fn test_tiering_external_blob_roundtrip_for_standard_type() {
let mut cfg = TierConfigMgr {
driver_cache: HashMap::new(),
tiers: HashMap::new(),
last_refreshed_at: OffsetDateTime::now_utc(),
};
cfg.tiers.insert("COLD-A".to_string(), build_s3_tier("COLD-A"));
let bytes = encode_external_tiering_config_blob(&cfg).expect("encode should succeed");
assert_eq!(&bytes[0..2], &TIER_CONFIG_FORMAT.to_le_bytes());
assert_eq!(&bytes[2..4], &TIER_CONFIG_VERSION.to_le_bytes());
let decoded = decode_external_tiering_config_blob(&bytes).expect("decode should succeed");
let tier = decoded.tiers.get("COLD-A").expect("tier should exist");
assert_eq!(tier.tier_type.as_lowercase(), "s3");
assert_eq!(tier.s3.as_ref().expect("s3 should exist").endpoint, "https://example-s3.invalid");
}
#[test]
fn test_tiering_external_blob_roundtrip_for_extended_type_hint() {
let mut cfg = TierConfigMgr {
driver_cache: HashMap::new(),
tiers: HashMap::new(),
last_refreshed_at: OffsetDateTime::now_utc(),
};
cfg.tiers.insert(
"COLD-B".to_string(),
TierConfig {
version: "v1".to_string(),
tier_type: TierType::RustFS,
name: "COLD-B".to_string(),
rustfs: Some(crate::tier::tier_config::TierRustFS {
name: "COLD-B".to_string(),
endpoint: "https://example-compat.invalid".to_string(),
access_key: "ak".to_string(),
secret_key: "sk".to_string(),
bucket: "bucket-b".to_string(),
prefix: "prefix-b".to_string(),
region: "us-east-1".to_string(),
storage_class: "STANDARD".to_string(),
}),
..Default::default()
},
);
let bytes = encode_external_tiering_config_blob(&cfg).expect("encode should succeed");
let decoded = decode_external_tiering_config_blob(&bytes).expect("decode should succeed");
let tier = decoded.tiers.get("COLD-B").expect("tier should exist");
assert_eq!(tier.tier_type.as_lowercase(), "rustfs");
assert_eq!(
tier.rustfs.as_ref().expect("backend should exist").endpoint,
"https://example-compat.invalid"
);
}
#[test]
fn test_decode_tiering_config_blob_accepts_legacy_json() {
let mut cfg = TierConfigMgr {
driver_cache: HashMap::new(),
tiers: HashMap::new(),
last_refreshed_at: OffsetDateTime::now_utc(),
};
cfg.tiers.insert("COLD-A".to_string(), build_s3_tier("COLD-A"));
let data = serde_json::to_vec(&cfg).expect("legacy json should encode");
let decoded = decode_tiering_config_blob(&data).expect("legacy json should decode");
assert_eq!(
decoded
.tiers
.get("COLD-A")
.and_then(|tier| tier.s3.as_ref())
.map(|s3| s3.bucket.as_str()),
Some("bucket-a")
);
}
}
crates/ecstore/src/tier/tier_config.rs
+8 -8
View File
@@ -146,7 +146,7 @@ impl Clone for TierConfig {
fn clone(&self) -> TierConfig {
let mut s3 = None;
let mut r = None;
let mut m = None;
let mut compatible_backend = None;
let mut aliyun = None;
let mut tencent = None;
let mut huaweicloud = None;
@@ -165,9 +165,9 @@ impl Clone for TierConfig {
r = Some(r_);
}
TierType::MinIO => {
let mut m_ = self.minio.as_ref().expect("err").clone();
m_.secret_key = "REDACTED".to_string();
m = Some(m_);
let mut compatible_backend_ = self.minio.as_ref().expect("err").clone();
compatible_backend_.secret_key = "REDACTED".to_string();
compatible_backend = Some(compatible_backend_);
}
TierType::Aliyun => {
let mut aliyun_ = self.aliyun.as_ref().expect("err").clone();
@@ -207,7 +207,7 @@ impl Clone for TierConfig {
name: self.name.clone(),
s3,
rustfs: r,
minio: m,
minio: compatible_backend,
aliyun,
tencent,
huaweicloud,
@@ -408,7 +408,7 @@ impl TierMinIO {
if name.is_empty() {
return Err(std::io::Error::other(ERR_TIER_NAME_EMPTY));
}
let m = TierMinIO {
let backend = TierMinIO {
access_key: access_key.to_string(),
secret_key: secret_key.to_string(),
bucket: bucket.to_string(),
@@ -417,7 +417,7 @@ impl TierMinIO {
};
for option in options {
let option = option(m.clone());
let option = option(backend.clone());
let option = *option;
option?;
}
@@ -426,7 +426,7 @@ impl TierMinIO {
version: C_TIER_CONFIG_VER.to_string(),
tier_type: TierType::MinIO,
name: name.to_string(),
minio: Some(m),
minio: Some(backend),
..Default::default()
})
}