mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-28 16:07:05 +00:00
feat(s3): advance parity coverage (#2278)
This commit is contained in:
@@ -1134,11 +1134,20 @@ pub struct S3ClientError {
|
||||
}
|
||||
impl S3ClientError {
|
||||
pub fn new(value: impl Into<String>) -> Self {
|
||||
Self::with_metadata(value, None, None, None)
|
||||
}
|
||||
|
||||
pub fn with_metadata(
|
||||
error: impl Into<String>,
|
||||
status_code: Option<StatusCode>,
|
||||
code: Option<String>,
|
||||
message: Option<String>,
|
||||
) -> Self {
|
||||
S3ClientError {
|
||||
error: value.into(),
|
||||
status_code: None,
|
||||
code: None,
|
||||
message: None,
|
||||
error: error.into(),
|
||||
status_code,
|
||||
code,
|
||||
message,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1154,16 +1163,16 @@ impl S3ClientError {
|
||||
|
||||
impl<T: aws_sdk_s3::error::ProvideErrorMetadata> From<T> for S3ClientError {
|
||||
fn from(value: T) -> Self {
|
||||
S3ClientError {
|
||||
error: format!(
|
||||
"{}: {}",
|
||||
value.code().map(String::from).unwrap_or("unknown code".into()),
|
||||
value.message().map(String::from).unwrap_or("missing reason".into()),
|
||||
),
|
||||
status_code: None,
|
||||
code: None,
|
||||
message: None,
|
||||
}
|
||||
let code = value.code().map(String::from);
|
||||
let message = value.message().map(String::from);
|
||||
let error = match (code.as_deref(), message.as_deref()) {
|
||||
(Some(code), Some(message)) => format!("{code}: {message}"),
|
||||
(Some(code), None) => code.to_string(),
|
||||
(None, Some(message)) => message.to_string(),
|
||||
(None, None) => "unknown remote error".to_string(),
|
||||
};
|
||||
|
||||
S3ClientError::with_metadata(error, None, code, message)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1207,10 +1216,15 @@ impl TargetClient {
|
||||
other
|
||||
);
|
||||
let message = other.meta().meta();
|
||||
Err(S3ClientError::new(format!(
|
||||
"failed to check bucket exists for bucket:{bucket} please check the bucket name and credentials, error:{:?}",
|
||||
message
|
||||
)))
|
||||
Err(S3ClientError::with_metadata(
|
||||
format!(
|
||||
"failed to check bucket exists for bucket:{bucket} please check the bucket name and credentials, error:{:?}",
|
||||
message
|
||||
),
|
||||
None,
|
||||
message.code().map(ToOwned::to_owned),
|
||||
message.message().map(ToOwned::to_owned),
|
||||
))
|
||||
}
|
||||
},
|
||||
SdkError::DispatchFailure(e) => Err(S3ClientError::new(format!(
|
||||
|
||||
@@ -1337,6 +1337,7 @@ mod tests {
|
||||
assert_eq!(event.action, IlmAction::TransitionAction);
|
||||
assert_eq!(event.rule_id, "transition-date");
|
||||
assert_eq!(event.storage_class, "WARM");
|
||||
assert_eq!(event.due, Some(transition_date));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -25,9 +25,9 @@ use crate::store::ECStore;
|
||||
use byteorder::{BigEndian, ByteOrder, LittleEndian};
|
||||
use rustfs_policy::policy::BucketPolicy;
|
||||
use s3s::dto::{
|
||||
BucketLifecycleConfiguration, CORSConfiguration, NotificationConfiguration, ObjectLockConfiguration,
|
||||
PublicAccessBlockConfiguration, ReplicationConfiguration, ServerSideEncryptionConfiguration, Tagging,
|
||||
VersioningConfiguration,
|
||||
AccelerateConfiguration, BucketLifecycleConfiguration, BucketLoggingStatus, CORSConfiguration, NotificationConfiguration,
|
||||
ObjectLockConfiguration, PublicAccessBlockConfiguration, ReplicationConfiguration, RequestPaymentConfiguration,
|
||||
ServerSideEncryptionConfiguration, Tagging, VersioningConfiguration, WebsiteConfiguration,
|
||||
};
|
||||
use serde::Serializer;
|
||||
use std::collections::HashMap;
|
||||
@@ -238,6 +238,10 @@ pub const BUCKET_VERSIONING_CONFIG: &str = "versioning.xml";
|
||||
pub const BUCKET_REPLICATION_CONFIG: &str = "replication.xml";
|
||||
pub const BUCKET_TARGETS_FILE: &str = "bucket-targets.json";
|
||||
pub const BUCKET_CORS_CONFIG: &str = "cors.xml";
|
||||
pub const BUCKET_LOGGING_CONFIG: &str = "logging.xml";
|
||||
pub const BUCKET_WEBSITE_CONFIG: &str = "website.xml";
|
||||
pub const BUCKET_ACCELERATE_CONFIG: &str = "accelerate.xml";
|
||||
pub const BUCKET_REQUEST_PAYMENT_CONFIG: &str = "request-payment.xml";
|
||||
pub const BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG: &str = "public-access-block.xml";
|
||||
pub const BUCKET_ACL_CONFIG: &str = "bucket-acl.json";
|
||||
|
||||
@@ -258,6 +262,10 @@ pub struct BucketMetadata {
|
||||
pub bucket_targets_config_json: Vec<u8>,
|
||||
pub bucket_targets_config_meta_json: Vec<u8>,
|
||||
pub cors_config_xml: Vec<u8>,
|
||||
pub logging_config_xml: Vec<u8>,
|
||||
pub website_config_xml: Vec<u8>,
|
||||
pub accelerate_config_xml: Vec<u8>,
|
||||
pub request_payment_config_xml: Vec<u8>,
|
||||
pub public_access_block_config_xml: Vec<u8>,
|
||||
pub bucket_acl_config_json: Vec<u8>,
|
||||
|
||||
@@ -273,6 +281,10 @@ pub struct BucketMetadata {
|
||||
pub bucket_targets_config_updated_at: OffsetDateTime,
|
||||
pub bucket_targets_config_meta_updated_at: OffsetDateTime,
|
||||
pub cors_config_updated_at: OffsetDateTime,
|
||||
pub logging_config_updated_at: OffsetDateTime,
|
||||
pub website_config_updated_at: OffsetDateTime,
|
||||
pub accelerate_config_updated_at: OffsetDateTime,
|
||||
pub request_payment_config_updated_at: OffsetDateTime,
|
||||
pub public_access_block_config_updated_at: OffsetDateTime,
|
||||
pub bucket_acl_config_updated_at: OffsetDateTime,
|
||||
|
||||
@@ -290,6 +302,10 @@ pub struct BucketMetadata {
|
||||
pub bucket_target_config: Option<BucketTargets>,
|
||||
pub bucket_target_config_meta: Option<HashMap<String, String>>,
|
||||
pub cors_config: Option<CORSConfiguration>,
|
||||
pub logging_config: Option<BucketLoggingStatus>,
|
||||
pub website_config: Option<WebsiteConfiguration>,
|
||||
pub accelerate_config: Option<AccelerateConfiguration>,
|
||||
pub request_payment_config: Option<RequestPaymentConfiguration>,
|
||||
pub public_access_block_config: Option<PublicAccessBlockConfiguration>,
|
||||
pub bucket_acl_config: Option<String>,
|
||||
}
|
||||
@@ -312,6 +328,10 @@ impl Default for BucketMetadata {
|
||||
bucket_targets_config_json: Default::default(),
|
||||
bucket_targets_config_meta_json: Default::default(),
|
||||
cors_config_xml: Default::default(),
|
||||
logging_config_xml: Default::default(),
|
||||
website_config_xml: Default::default(),
|
||||
accelerate_config_xml: Default::default(),
|
||||
request_payment_config_xml: Default::default(),
|
||||
public_access_block_config_xml: Default::default(),
|
||||
bucket_acl_config_json: Default::default(),
|
||||
policy_config_updated_at: OffsetDateTime::UNIX_EPOCH,
|
||||
@@ -326,6 +346,10 @@ impl Default for BucketMetadata {
|
||||
bucket_targets_config_updated_at: OffsetDateTime::UNIX_EPOCH,
|
||||
bucket_targets_config_meta_updated_at: OffsetDateTime::UNIX_EPOCH,
|
||||
cors_config_updated_at: OffsetDateTime::UNIX_EPOCH,
|
||||
logging_config_updated_at: OffsetDateTime::UNIX_EPOCH,
|
||||
website_config_updated_at: OffsetDateTime::UNIX_EPOCH,
|
||||
accelerate_config_updated_at: OffsetDateTime::UNIX_EPOCH,
|
||||
request_payment_config_updated_at: OffsetDateTime::UNIX_EPOCH,
|
||||
public_access_block_config_updated_at: OffsetDateTime::UNIX_EPOCH,
|
||||
bucket_acl_config_updated_at: OffsetDateTime::UNIX_EPOCH,
|
||||
new_field_updated_at: OffsetDateTime::UNIX_EPOCH,
|
||||
@@ -341,6 +365,10 @@ impl Default for BucketMetadata {
|
||||
bucket_target_config: Default::default(),
|
||||
bucket_target_config_meta: Default::default(),
|
||||
cors_config: Default::default(),
|
||||
logging_config: Default::default(),
|
||||
website_config: Default::default(),
|
||||
accelerate_config: Default::default(),
|
||||
request_payment_config: Default::default(),
|
||||
public_access_block_config: Default::default(),
|
||||
bucket_acl_config: Default::default(),
|
||||
}
|
||||
@@ -411,11 +439,19 @@ impl BucketMetadata {
|
||||
"BucketTargetsConfigUpdatedAt" => self.bucket_targets_config_updated_at = read_msgp_time_value(rd)?,
|
||||
"BucketTargetsConfigMetaUpdatedAt" => self.bucket_targets_config_meta_updated_at = read_msgp_time_value(rd)?,
|
||||
"CorsConfigXML" | "CorsConfigXml" => self.cors_config_xml = read_msgp_bin(rd)?,
|
||||
"LoggingConfigXML" | "LoggingConfigXml" => self.logging_config_xml = read_msgp_bin(rd)?,
|
||||
"WebsiteConfigXML" | "WebsiteConfigXml" => self.website_config_xml = read_msgp_bin(rd)?,
|
||||
"AccelerateConfigXML" | "AccelerateConfigXml" => self.accelerate_config_xml = read_msgp_bin(rd)?,
|
||||
"RequestPaymentConfigXML" | "RequestPaymentConfigXml" => self.request_payment_config_xml = read_msgp_bin(rd)?,
|
||||
"PublicAccessBlockConfigXML" | "PublicAccessBlockConfigXml" => {
|
||||
self.public_access_block_config_xml = read_msgp_bin(rd)?
|
||||
}
|
||||
"BucketAclConfigJSON" | "BucketAclConfigJson" => self.bucket_acl_config_json = read_msgp_bin(rd)?,
|
||||
"CorsConfigUpdatedAt" => self.cors_config_updated_at = read_msgp_time_value(rd)?,
|
||||
"LoggingConfigUpdatedAt" => self.logging_config_updated_at = read_msgp_time_value(rd)?,
|
||||
"WebsiteConfigUpdatedAt" => self.website_config_updated_at = read_msgp_time_value(rd)?,
|
||||
"AccelerateConfigUpdatedAt" => self.accelerate_config_updated_at = read_msgp_time_value(rd)?,
|
||||
"RequestPaymentConfigUpdatedAt" => self.request_payment_config_updated_at = read_msgp_time_value(rd)?,
|
||||
"PublicAccessBlockConfigUpdatedAt" => self.public_access_block_config_updated_at = read_msgp_time_value(rd)?,
|
||||
"BucketAclConfigUpdatedAt" => self.bucket_acl_config_updated_at = read_msgp_time_value(rd)?,
|
||||
other => {
|
||||
@@ -430,8 +466,8 @@ impl BucketMetadata {
|
||||
|
||||
/// Encode to msgp bytes. Field order follows MinIO BucketMetadata for compatibility.
|
||||
pub fn encode_to<W: Write>(&self, wr: &mut W) -> Result<()> {
|
||||
// Map size: MinIO fields (25) + RustFS extensions (6)
|
||||
let map_len: u32 = 31;
|
||||
// Map size: MinIO fields (25) + RustFS extensions (14)
|
||||
let map_len: u32 = 39;
|
||||
rmp::encode::write_map_len(wr, map_len)?;
|
||||
|
||||
// MinIO field order (same as Go struct)
|
||||
@@ -481,10 +517,22 @@ impl BucketMetadata {
|
||||
|
||||
// RustFS extensions
|
||||
write_bin_field(wr, "CorsConfigXML", &self.cors_config_xml)?;
|
||||
write_bin_field(wr, "LoggingConfigXML", &self.logging_config_xml)?;
|
||||
write_bin_field(wr, "WebsiteConfigXML", &self.website_config_xml)?;
|
||||
write_bin_field(wr, "AccelerateConfigXML", &self.accelerate_config_xml)?;
|
||||
write_bin_field(wr, "RequestPaymentConfigXML", &self.request_payment_config_xml)?;
|
||||
write_bin_field(wr, "PublicAccessBlockConfigXML", &self.public_access_block_config_xml)?;
|
||||
write_bin_field(wr, "BucketAclConfigJSON", &self.bucket_acl_config_json)?;
|
||||
rmp::encode::write_str(wr, "CorsConfigUpdatedAt")?;
|
||||
write_msgp_time(wr, self.cors_config_updated_at)?;
|
||||
rmp::encode::write_str(wr, "LoggingConfigUpdatedAt")?;
|
||||
write_msgp_time(wr, self.logging_config_updated_at)?;
|
||||
rmp::encode::write_str(wr, "WebsiteConfigUpdatedAt")?;
|
||||
write_msgp_time(wr, self.website_config_updated_at)?;
|
||||
rmp::encode::write_str(wr, "AccelerateConfigUpdatedAt")?;
|
||||
write_msgp_time(wr, self.accelerate_config_updated_at)?;
|
||||
rmp::encode::write_str(wr, "RequestPaymentConfigUpdatedAt")?;
|
||||
write_msgp_time(wr, self.request_payment_config_updated_at)?;
|
||||
rmp::encode::write_str(wr, "PublicAccessBlockConfigUpdatedAt")?;
|
||||
write_msgp_time(wr, self.public_access_block_config_updated_at)?;
|
||||
rmp::encode::write_str(wr, "BucketAclConfigUpdatedAt")?;
|
||||
@@ -569,6 +617,18 @@ impl BucketMetadata {
|
||||
if self.public_access_block_config_updated_at == OffsetDateTime::UNIX_EPOCH {
|
||||
self.public_access_block_config_updated_at = self.created
|
||||
}
|
||||
if self.logging_config_updated_at == OffsetDateTime::UNIX_EPOCH {
|
||||
self.logging_config_updated_at = self.created
|
||||
}
|
||||
if self.website_config_updated_at == OffsetDateTime::UNIX_EPOCH {
|
||||
self.website_config_updated_at = self.created
|
||||
}
|
||||
if self.accelerate_config_updated_at == OffsetDateTime::UNIX_EPOCH {
|
||||
self.accelerate_config_updated_at = self.created
|
||||
}
|
||||
if self.request_payment_config_updated_at == OffsetDateTime::UNIX_EPOCH {
|
||||
self.request_payment_config_updated_at = self.created
|
||||
}
|
||||
if self.bucket_acl_config_updated_at == OffsetDateTime::UNIX_EPOCH {
|
||||
self.bucket_acl_config_updated_at = self.created
|
||||
}
|
||||
@@ -625,6 +685,22 @@ impl BucketMetadata {
|
||||
self.cors_config_xml = data;
|
||||
self.cors_config_updated_at = updated;
|
||||
}
|
||||
BUCKET_LOGGING_CONFIG => {
|
||||
self.logging_config_xml = data;
|
||||
self.logging_config_updated_at = updated;
|
||||
}
|
||||
BUCKET_WEBSITE_CONFIG => {
|
||||
self.website_config_xml = data;
|
||||
self.website_config_updated_at = updated;
|
||||
}
|
||||
BUCKET_ACCELERATE_CONFIG => {
|
||||
self.accelerate_config_xml = data;
|
||||
self.accelerate_config_updated_at = updated;
|
||||
}
|
||||
BUCKET_REQUEST_PAYMENT_CONFIG => {
|
||||
self.request_payment_config_xml = data;
|
||||
self.request_payment_config_updated_at = updated;
|
||||
}
|
||||
BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG => {
|
||||
self.public_access_block_config_xml = data;
|
||||
self.public_access_block_config_updated_at = updated;
|
||||
@@ -741,6 +817,33 @@ impl BucketMetadata {
|
||||
{
|
||||
tracing::warn!(bucket = %self.name, config = "cors", error = %e, "parse_all_configs: failed to parse");
|
||||
}
|
||||
if !self.logging_config_xml.is_empty()
|
||||
&& let Err(e) = deserialize::<BucketLoggingStatus>(&self.logging_config_xml).map(|c| self.logging_config = Some(c))
|
||||
{
|
||||
tracing::warn!(bucket = %self.name, config = "logging", error = %e, "parse_all_configs: failed to parse");
|
||||
}
|
||||
if !self.website_config_xml.is_empty()
|
||||
&& let Err(e) = deserialize::<WebsiteConfiguration>(&self.website_config_xml).map(|c| self.website_config = Some(c))
|
||||
{
|
||||
tracing::warn!(bucket = %self.name, config = "website", error = %e, "parse_all_configs: failed to parse");
|
||||
}
|
||||
if !self.accelerate_config_xml.is_empty()
|
||||
&& let Err(e) =
|
||||
deserialize::<AccelerateConfiguration>(&self.accelerate_config_xml).map(|c| self.accelerate_config = Some(c))
|
||||
{
|
||||
tracing::warn!(bucket = %self.name, config = "accelerate", error = %e, "parse_all_configs: failed to parse");
|
||||
}
|
||||
if !self.request_payment_config_xml.is_empty()
|
||||
&& let Err(e) = deserialize::<RequestPaymentConfiguration>(&self.request_payment_config_xml)
|
||||
.map(|c| self.request_payment_config = Some(c))
|
||||
{
|
||||
tracing::warn!(
|
||||
bucket = %self.name,
|
||||
config = "request_payment",
|
||||
error = %e,
|
||||
"parse_all_configs: failed to parse"
|
||||
);
|
||||
}
|
||||
if !self.public_access_block_config_xml.is_empty()
|
||||
&& let Err(e) = deserialize::<PublicAccessBlockConfiguration>(&self.public_access_block_config_xml)
|
||||
.map(|c| self.public_access_block_config = Some(c))
|
||||
|
||||
@@ -28,8 +28,9 @@ use rustfs_common::heal_channel::HealOpts;
|
||||
use rustfs_policy::policy::BucketPolicy;
|
||||
use s3s::dto::ReplicationConfiguration;
|
||||
use s3s::dto::{
|
||||
BucketLifecycleConfiguration, CORSConfiguration, NotificationConfiguration, ObjectLockConfiguration,
|
||||
PublicAccessBlockConfiguration, ServerSideEncryptionConfiguration, Tagging, VersioningConfiguration,
|
||||
AccelerateConfiguration, BucketLifecycleConfiguration, BucketLoggingStatus, CORSConfiguration, NotificationConfiguration,
|
||||
ObjectLockConfiguration, PublicAccessBlockConfiguration, RequestPaymentConfiguration, ServerSideEncryptionConfiguration,
|
||||
Tagging, VersioningConfiguration, WebsiteConfiguration,
|
||||
};
|
||||
use std::collections::HashSet;
|
||||
use std::sync::OnceLock;
|
||||
@@ -193,6 +194,34 @@ pub async fn get_versioning_config(bucket: &str) -> Result<(VersioningConfigurat
|
||||
bucket_meta_sys.get_versioning_config(bucket).await
|
||||
}
|
||||
|
||||
pub async fn get_website_config(bucket: &str) -> Result<(WebsiteConfiguration, OffsetDateTime)> {
|
||||
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
|
||||
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
|
||||
|
||||
bucket_meta_sys.get_website_config(bucket).await
|
||||
}
|
||||
|
||||
pub async fn get_logging_config(bucket: &str) -> Result<(BucketLoggingStatus, OffsetDateTime)> {
|
||||
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
|
||||
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
|
||||
|
||||
bucket_meta_sys.get_logging_config(bucket).await
|
||||
}
|
||||
|
||||
pub async fn get_accelerate_config(bucket: &str) -> Result<(AccelerateConfiguration, OffsetDateTime)> {
|
||||
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
|
||||
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
|
||||
|
||||
bucket_meta_sys.get_accelerate_config(bucket).await
|
||||
}
|
||||
|
||||
pub async fn get_request_payment_config(bucket: &str) -> Result<(RequestPaymentConfiguration, OffsetDateTime)> {
|
||||
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
|
||||
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
|
||||
|
||||
bucket_meta_sys.get_request_payment_config(bucket).await
|
||||
}
|
||||
|
||||
pub async fn get_config_from_disk(bucket: &str) -> Result<BucketMetadata> {
|
||||
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
|
||||
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
|
||||
@@ -587,6 +616,46 @@ impl BucketMetadataSys {
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn get_website_config(&self, bucket: &str) -> Result<(WebsiteConfiguration, OffsetDateTime)> {
|
||||
let (bm, _) = self.get_config(bucket).await?;
|
||||
|
||||
if let Some(config) = &bm.website_config {
|
||||
Ok((config.clone(), bm.website_config_updated_at))
|
||||
} else {
|
||||
Err(Error::ConfigNotFound)
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn get_logging_config(&self, bucket: &str) -> Result<(BucketLoggingStatus, OffsetDateTime)> {
|
||||
let (bm, _) = self.get_config(bucket).await?;
|
||||
|
||||
if let Some(config) = &bm.logging_config {
|
||||
Ok((config.clone(), bm.logging_config_updated_at))
|
||||
} else {
|
||||
Err(Error::ConfigNotFound)
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn get_accelerate_config(&self, bucket: &str) -> Result<(AccelerateConfiguration, OffsetDateTime)> {
|
||||
let (bm, _) = self.get_config(bucket).await?;
|
||||
|
||||
if let Some(config) = &bm.accelerate_config {
|
||||
Ok((config.clone(), bm.accelerate_config_updated_at))
|
||||
} else {
|
||||
Err(Error::ConfigNotFound)
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn get_request_payment_config(&self, bucket: &str) -> Result<(RequestPaymentConfiguration, OffsetDateTime)> {
|
||||
let (bm, _) = self.get_config(bucket).await?;
|
||||
|
||||
if let Some(config) = &bm.request_payment_config {
|
||||
Ok((config.clone(), bm.request_payment_config_updated_at))
|
||||
} else {
|
||||
Err(Error::ConfigNotFound)
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn created_at(&self, bucket: &str) -> Result<OffsetDateTime> {
|
||||
let bm = match self.get_config(bucket).await {
|
||||
Ok((bm, _)) => bm.created,
|
||||
|
||||
@@ -21,7 +21,7 @@ use crate::bucket::replication::replicate_delete;
|
||||
use crate::bucket::replication::replicate_object;
|
||||
use crate::bucket::replication::replication_resyncer::{
|
||||
BucketReplicationResyncStatus, DeletedObjectReplicationInfo, REPLICATION_DIR, RESYNC_FILE_NAME, ReplicationConfig,
|
||||
ReplicationResyncer, decode_resync_file, get_heal_replicate_object_info,
|
||||
ReplicationResyncer, TargetReplicationResyncStatus, decode_resync_file, get_heal_replicate_object_info, save_resync_status,
|
||||
};
|
||||
use crate::bucket::replication::replication_state::ReplicationStats;
|
||||
use crate::config::com::read_config;
|
||||
@@ -763,6 +763,63 @@ impl<S: StorageAPI> ReplicationPool<S> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn get_bucket_resync_status(&self, bucket: &str) -> Result<BucketReplicationResyncStatus, EcstoreError> {
|
||||
if let Some(status) = self.resyncer.status_map.read().await.get(bucket).cloned() {
|
||||
return Ok(status);
|
||||
}
|
||||
|
||||
let status = load_bucket_resync_metadata(bucket, self.storage.clone()).await?;
|
||||
self.resyncer
|
||||
.status_map
|
||||
.write()
|
||||
.await
|
||||
.insert(bucket.to_string(), status.clone());
|
||||
Ok(status)
|
||||
}
|
||||
|
||||
pub async fn start_bucket_resync(self: Arc<Self>, opts: ResyncOpts) -> Result<(), EcstoreError> {
|
||||
let now = OffsetDateTime::now_utc();
|
||||
let bucket_status = {
|
||||
let mut status_map = self.resyncer.status_map.write().await;
|
||||
let bucket_status = status_map.entry(opts.bucket.clone()).or_insert_with(|| {
|
||||
let mut status = BucketReplicationResyncStatus::new();
|
||||
status.id = 0;
|
||||
status
|
||||
});
|
||||
|
||||
bucket_status.last_update = Some(now);
|
||||
bucket_status.targets_map.insert(
|
||||
opts.arn.clone(),
|
||||
TargetReplicationResyncStatus {
|
||||
start_time: Some(now),
|
||||
last_update: Some(now),
|
||||
resync_id: opts.resync_id.clone(),
|
||||
resync_before_date: opts.resync_before,
|
||||
resync_status: ResyncStatusType::ResyncPending,
|
||||
failed_size: 0,
|
||||
failed_count: 0,
|
||||
replicated_size: 0,
|
||||
replicated_count: 0,
|
||||
bucket: opts.bucket.clone(),
|
||||
object: String::new(),
|
||||
error: None,
|
||||
},
|
||||
);
|
||||
|
||||
bucket_status.clone()
|
||||
};
|
||||
|
||||
save_resync_status(&opts.bucket, &bucket_status, self.storage.clone()).await?;
|
||||
|
||||
let resyncer = self.resyncer.clone();
|
||||
let storage = self.storage.clone();
|
||||
tokio::spawn(async move {
|
||||
resyncer.resync_bucket(CancellationToken::new(), storage, false, opts).await;
|
||||
});
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Start the resync routine that runs in a loop
|
||||
async fn start_resync_routine(self: Arc<Self>, buckets: Vec<String>, cancellation_token: CancellationToken) {
|
||||
// Run the replication resync in a loop
|
||||
@@ -891,6 +948,8 @@ pub trait ReplicationPoolTrait: std::fmt::Debug {
|
||||
async fn queue_replica_task(&self, ri: ReplicateObjectInfo);
|
||||
async fn queue_replica_delete_task(&self, ri: DeletedObjectReplicationInfo);
|
||||
async fn resize(&self, priority: ReplicationPriority, max_workers: usize, max_l_workers: usize);
|
||||
async fn get_bucket_resync_status(&self, bucket: &str) -> Result<BucketReplicationResyncStatus, EcstoreError>;
|
||||
async fn start_bucket_resync(self: Arc<Self>, opts: ResyncOpts) -> Result<(), EcstoreError>;
|
||||
async fn init_resync(
|
||||
self: Arc<Self>,
|
||||
cancellation_token: CancellationToken,
|
||||
@@ -913,6 +972,14 @@ impl<S: StorageAPI> ReplicationPoolTrait for ReplicationPool<S> {
|
||||
self.resize(priority, max_workers, max_l_workers).await;
|
||||
}
|
||||
|
||||
async fn get_bucket_resync_status(&self, bucket: &str) -> Result<BucketReplicationResyncStatus, EcstoreError> {
|
||||
self.get_bucket_resync_status(bucket).await
|
||||
}
|
||||
|
||||
async fn start_bucket_resync(self: Arc<Self>, opts: ResyncOpts) -> Result<(), EcstoreError> {
|
||||
self.start_bucket_resync(opts).await
|
||||
}
|
||||
|
||||
async fn init_resync(
|
||||
self: Arc<Self>,
|
||||
cancellation_token: CancellationToken,
|
||||
|
||||
@@ -901,7 +901,11 @@ pub async fn get_heal_replicate_object_info(oi: &ObjectInfo, rcfg: &ReplicationC
|
||||
}
|
||||
}
|
||||
|
||||
async fn save_resync_status<S: StorageAPI>(bucket: &str, status: &BucketReplicationResyncStatus, api: Arc<S>) -> Result<()> {
|
||||
pub(crate) async fn save_resync_status<S: StorageAPI>(
|
||||
bucket: &str,
|
||||
status: &BucketReplicationResyncStatus,
|
||||
api: Arc<S>,
|
||||
) -> Result<()> {
|
||||
let data = encode_resync_file(status)?;
|
||||
|
||||
let config_file = path_join_buf(&[BUCKET_META_PREFIX, bucket, REPLICATION_DIR, RESYNC_FILE_NAME]);
|
||||
|
||||
@@ -21,6 +21,7 @@ use crate::store::ECStore;
|
||||
use crate::store_api::ObjectInfo;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use std::sync::OnceLock;
|
||||
use std::sync::atomic::Ordering;
|
||||
use tokio::sync::RwLock;
|
||||
use tracing::warn;
|
||||
@@ -82,4 +83,50 @@ pub struct EventArgs {
|
||||
|
||||
impl EventArgs {}
|
||||
|
||||
pub fn send_event(args: EventArgs) {}
|
||||
type EventDispatchHook = Arc<dyn Fn(EventArgs) + Send + Sync + 'static>;
|
||||
|
||||
static EVENT_DISPATCH_HOOK: OnceLock<EventDispatchHook> = OnceLock::new();
|
||||
|
||||
pub fn register_event_dispatch_hook<F>(hook: F) -> bool
|
||||
where
|
||||
F: Fn(EventArgs) + Send + Sync + 'static,
|
||||
{
|
||||
EVENT_DISPATCH_HOOK.set(Arc::new(hook)).is_ok()
|
||||
}
|
||||
|
||||
pub fn send_event(args: EventArgs) {
|
||||
if let Some(hook) = EVENT_DISPATCH_HOOK.get() {
|
||||
hook(args);
|
||||
return;
|
||||
}
|
||||
|
||||
warn!(
|
||||
event_name = args.event_name,
|
||||
bucket = args.bucket_name,
|
||||
"event send() dropped because no event dispatch hook is registered"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
static DISPATCH_COUNT: AtomicUsize = AtomicUsize::new(0);
|
||||
|
||||
#[test]
|
||||
fn send_event_dispatches_to_registered_hook() {
|
||||
let _ = register_event_dispatch_hook(|_args| {
|
||||
DISPATCH_COUNT.fetch_add(1, Ordering::Relaxed);
|
||||
});
|
||||
let before = DISPATCH_COUNT.load(Ordering::Relaxed);
|
||||
|
||||
send_event(EventArgs {
|
||||
event_name: "s3:ObjectCreated:Put".to_string(),
|
||||
bucket_name: "demo".to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
assert_eq!(DISPATCH_COUNT.load(Ordering::Relaxed), before + 1);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -29,9 +29,9 @@ use rustfs_madmin::{
|
||||
use rustfs_protos::evict_failed_connection;
|
||||
use rustfs_protos::proto_gen::node_service::{
|
||||
DeleteBucketMetadataRequest, DeletePolicyRequest, DeleteServiceAccountRequest, DeleteUserRequest, GetCpusRequest,
|
||||
GetMemInfoRequest, GetMetricsRequest, GetNetInfoRequest, GetOsInfoRequest, GetPartitionsRequest, GetProcInfoRequest,
|
||||
GetSeLinuxInfoRequest, GetSysConfigRequest, GetSysErrorsRequest, LoadBucketMetadataRequest, LoadGroupRequest,
|
||||
LoadPolicyMappingRequest, LoadPolicyRequest, LoadRebalanceMetaRequest, LoadServiceAccountRequest,
|
||||
GetLiveEventsRequest, GetMemInfoRequest, GetMetricsRequest, GetNetInfoRequest, GetOsInfoRequest, GetPartitionsRequest,
|
||||
GetProcInfoRequest, GetSeLinuxInfoRequest, GetSysConfigRequest, GetSysErrorsRequest, LoadBucketMetadataRequest,
|
||||
LoadGroupRequest, LoadPolicyMappingRequest, LoadPolicyRequest, LoadRebalanceMetaRequest, LoadServiceAccountRequest,
|
||||
LoadTransitionTierConfigRequest, LoadUserRequest, LocalStorageInfoRequest, Mss, ReloadPoolMetaRequest,
|
||||
ReloadSiteReplicationConfigRequest, ServerInfoRequest, SignalServiceRequest, StartProfilingRequest, StopRebalanceRequest,
|
||||
node_service_client::NodeServiceClient,
|
||||
@@ -48,6 +48,13 @@ pub const PEER_RESTSIGNAL: &str = "signal";
|
||||
pub const PEER_RESTSUB_SYS: &str = "sub-sys";
|
||||
pub const PEER_RESTDRY_RUN: &str = "dry-run";
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct PeerLiveEventsBatch {
|
||||
pub events: Vec<u8>,
|
||||
pub next_sequence: u64,
|
||||
pub truncated: bool,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct PeerRestClient {
|
||||
pub host: XHost,
|
||||
@@ -333,6 +340,25 @@ impl PeerRestClient {
|
||||
Ok(realtime_metrics)
|
||||
}
|
||||
|
||||
pub async fn get_live_events(&self, after_sequence: u64, limit: u32) -> Result<PeerLiveEventsBatch> {
|
||||
let mut client = self.get_client().await?;
|
||||
let request = Request::new(GetLiveEventsRequest { after_sequence, limit });
|
||||
|
||||
let response = client.get_live_events(request).await?.into_inner();
|
||||
if !response.success {
|
||||
if let Some(msg) = response.error_info {
|
||||
return Err(Error::other(msg));
|
||||
}
|
||||
return Err(Error::other(""));
|
||||
}
|
||||
|
||||
Ok(PeerLiveEventsBatch {
|
||||
events: response.events.to_vec(),
|
||||
next_sequence: response.next_sequence,
|
||||
truncated: response.truncated,
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn get_proc_info(&self) -> Result<ProcInfo> {
|
||||
let mut client = self.get_client().await?;
|
||||
let request = Request::new(GetProcInfoRequest {});
|
||||
|
||||
@@ -701,6 +701,11 @@ impl ObjectIO for SetDisks {
|
||||
}
|
||||
|
||||
let mut user_defined = opts.user_defined.clone();
|
||||
if let Some(eval_metadata) = &opts.eval_metadata {
|
||||
for (key, value) in eval_metadata {
|
||||
user_defined.insert(key.clone(), value.clone());
|
||||
}
|
||||
}
|
||||
|
||||
let sc_parity_drives = {
|
||||
if let Some(sc) = GLOBAL_STORAGE_CLASS.get() {
|
||||
|
||||
@@ -14,7 +14,8 @@
|
||||
|
||||
#![allow(clippy::map_entry)]
|
||||
|
||||
use crate::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry;
|
||||
use crate::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc;
|
||||
use crate::bucket::lifecycle::bucket_lifecycle_ops::{enqueue_transition_immediate, init_background_expiry};
|
||||
use crate::bucket::metadata_sys::{self, set_bucket_metadata};
|
||||
use crate::bucket::utils::check_abort_multipart_args;
|
||||
use crate::bucket::utils::check_complete_multipart_args;
|
||||
@@ -27,7 +28,7 @@ use crate::bucket::utils::check_new_multipart_args;
|
||||
use crate::bucket::utils::check_object_args;
|
||||
use crate::bucket::utils::check_put_object_args;
|
||||
use crate::bucket::utils::check_put_object_part_args;
|
||||
use crate::bucket::utils::{check_valid_bucket_name, check_valid_bucket_name_strict};
|
||||
use crate::bucket::utils::{check_valid_bucket_name, check_valid_bucket_name_strict, is_meta_bucketname};
|
||||
use crate::config::GLOBAL_STORAGE_CLASS;
|
||||
use crate::config::storageclass;
|
||||
use crate::disk::endpoint::{Endpoint, EndpointType};
|
||||
@@ -129,6 +130,22 @@ async fn has_xlmeta_files(path: &std::path::Path) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
async fn enqueue_transition_after_write(result: Result<ObjectInfo>, src: LcEventSrc) -> Result<ObjectInfo> {
|
||||
match result {
|
||||
Ok(oi) => {
|
||||
if should_enqueue_transition_immediately(&oi) {
|
||||
enqueue_transition_immediate(&oi, src).await;
|
||||
}
|
||||
Ok(oi)
|
||||
}
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
fn should_enqueue_transition_immediately(oi: &ObjectInfo) -> bool {
|
||||
!is_meta_bucketname(&oi.bucket)
|
||||
}
|
||||
|
||||
const MAX_UPLOADS_LIST: usize = 10000;
|
||||
|
||||
mod bucket;
|
||||
@@ -243,7 +260,7 @@ impl ObjectIO for ECStore {
|
||||
}
|
||||
#[instrument(level = "debug", skip(self, data))]
|
||||
async fn put_object(&self, bucket: &str, object: &str, data: &mut PutObjReader, opts: &ObjectOptions) -> Result<ObjectInfo> {
|
||||
self.handle_put_object(bucket, object, data, opts).await
|
||||
enqueue_transition_after_write(self.handle_put_object(bucket, object, data, opts).await, LcEventSrc::S3PutObject).await
|
||||
}
|
||||
}
|
||||
|
||||
@@ -301,8 +318,12 @@ impl ObjectOperations for ECStore {
|
||||
src_opts: &ObjectOptions,
|
||||
dst_opts: &ObjectOptions,
|
||||
) -> Result<ObjectInfo> {
|
||||
self.handle_copy_object(src_bucket, src_object, dst_bucket, dst_object, src_info, src_opts, dst_opts)
|
||||
.await
|
||||
enqueue_transition_after_write(
|
||||
self.handle_copy_object(src_bucket, src_object, dst_bucket, dst_object, src_info, src_opts, dst_opts)
|
||||
.await,
|
||||
LcEventSrc::S3CopyObject,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[instrument(skip(self))]
|
||||
@@ -520,8 +541,12 @@ impl MultipartOperations for ECStore {
|
||||
uploaded_parts: Vec<CompletePart>,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<ObjectInfo> {
|
||||
self.handle_complete_multipart_upload(bucket, object, upload_id, uploaded_parts, opts)
|
||||
.await
|
||||
enqueue_transition_after_write(
|
||||
self.handle_complete_multipart_upload(bucket, object, upload_id, uploaded_parts, opts)
|
||||
.await,
|
||||
LcEventSrc::S3CompleteMultipartUpload,
|
||||
)
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
@@ -709,6 +734,17 @@ mod tests {
|
||||
assert!(disks.is_empty() || !disks.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_should_not_enqueue_transition_for_internal_metadata_bucket() {
|
||||
let oi = ObjectInfo {
|
||||
bucket: RUSTFS_META_BUCKET.to_string(),
|
||||
name: format!("{BUCKET_META_PREFIX}/bucket/.metadata.bin"),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert!(!should_enqueue_transition_immediately(&oi));
|
||||
}
|
||||
|
||||
// Test that we can create the basic structures without global state
|
||||
#[test]
|
||||
fn test_pool_available_space_creation() {
|
||||
|
||||
Reference in New Issue
Block a user