mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-26 16:28:15 +00:00
refactor(replication): move heal queue routing contracts (#4176)
* refactor(replication): move heal queue routing contracts * fix(replication): satisfy feature clippy
This commit is contained in:
Zhengchao An
@@ -14,8 +14,8 @@
|
||||
|
||||
pub(crate) use rustfs_replication::{MrfOpKind, MrfReplicateEntry};
|
||||
pub(crate) use rustfs_replication::{
|
||||
REPLICATE_EXISTING, REPLICATE_EXISTING_DELETE, REPLICATE_HEAL, REPLICATE_HEAL_DELETE, REPLICATE_INCOMING_DELETE,
|
||||
ReplicateDecision, ReplicateObjectInfo, ReplicateTargetDecision, ReplicatedInfos, ReplicatedTargetInfo, ReplicationAction,
|
||||
ReplicationState, ReplicationStatusType, ReplicationType, ReplicationWorkerOperation, ResyncDecision, VersionPurgeStatusType,
|
||||
REPLICATE_EXISTING, REPLICATE_EXISTING_DELETE, REPLICATE_HEAL_DELETE, REPLICATE_INCOMING_DELETE, ReplicateDecision,
|
||||
ReplicateObjectInfo, ReplicateTargetDecision, ReplicatedInfos, ReplicatedTargetInfo, ReplicationAction, ReplicationState,
|
||||
ReplicationStatusType, ReplicationType, ReplicationWorkerOperation, ResyncDecision, VersionPurgeStatusType,
|
||||
get_replication_state, parse_replicate_decision, replication_statuses_map, target_reset_header, version_purge_statuses_map,
|
||||
};
|
||||
|
||||
@@ -16,9 +16,9 @@ use super::datatypes::ResyncStatusType;
|
||||
use super::replication_config_store::ReplicationConfigStore;
|
||||
use super::replication_error_boundary::Error as EcstoreError;
|
||||
use super::replication_filemeta_boundary::{
|
||||
MrfOpKind, MrfReplicateEntry, REPLICATE_EXISTING, REPLICATE_HEAL, REPLICATE_HEAL_DELETE, ReplicateDecision,
|
||||
ReplicateObjectInfo, ReplicatedTargetInfo, ReplicationStatusType, ReplicationType, ReplicationWorkerOperation,
|
||||
ResyncDecision, VersionPurgeStatusType, replication_statuses_map, version_purge_statuses_map,
|
||||
MrfOpKind, MrfReplicateEntry, REPLICATE_HEAL_DELETE, ReplicateDecision, ReplicateObjectInfo, ReplicatedTargetInfo,
|
||||
ReplicationStatusType, ReplicationType, ReplicationWorkerOperation, ResyncDecision, replication_statuses_map,
|
||||
version_purge_statuses_map,
|
||||
};
|
||||
use super::replication_metadata_boundary::ReplicationMetadataStore;
|
||||
use super::replication_resyncer::{
|
||||
@@ -32,10 +32,11 @@ use super::runtime_boundary as runtime_sources;
|
||||
use super::{BucketReplicationResyncStatus, ResyncOpts, TargetReplicationResyncStatus};
|
||||
use lazy_static::lazy_static;
|
||||
use rustfs_replication::{
|
||||
DeletedObjectReplicationInfo, LARGE_WORKER_COUNT, ReplicationHealQueueResult, ReplicationOperation, ReplicationPoolOpts,
|
||||
ReplicationPriority, ReplicationQueueAdmission, WORKER_MAX_LIMIT, initial_worker_counts, mrf_worker_size_to_count,
|
||||
next_large_worker_count, next_mrf_worker_count, next_regular_worker_count, resized_worker_counts, should_grow_large_workers,
|
||||
should_queue_large_object,
|
||||
DeletedObjectReplicationInfo, LARGE_WORKER_COUNT, ReplicationHealQueueAction, ReplicationHealQueueResult,
|
||||
ReplicationHealResyncDeletes, ReplicationOperation, ReplicationPoolOpts, ReplicationPriority, ReplicationQueueAdmission,
|
||||
WORKER_MAX_LIMIT, initial_worker_counts, mrf_worker_size_to_count, next_large_worker_count, next_mrf_worker_count,
|
||||
next_regular_worker_count, replication_heal_queue_action, resized_worker_counts, should_auto_resume_resync,
|
||||
should_grow_large_workers, should_queue_large_object,
|
||||
};
|
||||
use rustfs_utils::http::{SUFFIX_REPLICATION_TIMESTAMP, get_str};
|
||||
use std::sync::Arc;
|
||||
@@ -63,10 +64,6 @@ const EVENT_REPLICATION_RESYNC_RECOVERED: &str = "replication_resync_recovered";
|
||||
const EVENT_REPLICATION_CONFIG_LOOKUP_SKIPPED: &str = "replication_config_lookup_skipped";
|
||||
const EVENT_REPLICATION_MRF_QUEUE_OVERFLOW: &str = "replication_mrf_queue_overflow";
|
||||
|
||||
fn should_auto_resume_resync(status: ResyncStatusType) -> bool {
|
||||
matches!(status, ResyncStatusType::ResyncPending | ResyncStatusType::ResyncStarted)
|
||||
}
|
||||
|
||||
/// Main replication pool structure
|
||||
#[derive(Debug)]
|
||||
pub struct ReplicationPool<S: ReplicationStorage> {
|
||||
@@ -1383,140 +1380,52 @@ pub(crate) async fn queue_replication_heal_internal(
|
||||
roi = get_heal_replicate_object_info(&oi, &rcfg).await;
|
||||
roi.retry_count = retry_count;
|
||||
|
||||
if !roi.dsc.replicate_any() {
|
||||
return ReplicationHealQueueResult {
|
||||
match replication_heal_queue_action(&mut roi) {
|
||||
ReplicationHealQueueAction::Skip => ReplicationHealQueueResult {
|
||||
object_info: roi,
|
||||
admission: ReplicationQueueAdmission::Skipped,
|
||||
};
|
||||
}
|
||||
|
||||
// early return if replication already done, otherwise we need to determine if this
|
||||
// version is an existing object that needs healing.
|
||||
if roi.replication_status == ReplicationStatusType::Completed
|
||||
&& roi.version_purge_status.is_empty()
|
||||
&& !roi.existing_obj_resync.must_resync()
|
||||
{
|
||||
return ReplicationHealQueueResult {
|
||||
object_info: roi,
|
||||
admission: ReplicationQueueAdmission::Skipped,
|
||||
};
|
||||
}
|
||||
|
||||
if roi.delete_marker || !roi.version_purge_status.is_empty() {
|
||||
let (version_id, dm_version_id) = if roi.version_purge_status.is_empty() {
|
||||
(None, roi.version_id)
|
||||
} else {
|
||||
(roi.version_id, None)
|
||||
};
|
||||
|
||||
let dv = DeletedObjectReplicationInfo {
|
||||
delete_object: DeletedObject {
|
||||
object_name: roi.name.clone(),
|
||||
delete_marker_version_id: dm_version_id,
|
||||
version_id,
|
||||
replication_state: roi.replication_state.clone(),
|
||||
delete_marker_mtime: roi.mod_time,
|
||||
delete_marker: roi.delete_marker,
|
||||
..Default::default()
|
||||
},
|
||||
bucket: roi.bucket.clone(),
|
||||
op_type: ReplicationType::Heal,
|
||||
event_type: REPLICATE_HEAL_DELETE.to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
// heal delete marker replication failure or versioned delete replication failure
|
||||
if roi.replication_status == ReplicationStatusType::Pending
|
||||
|| roi.replication_status == ReplicationStatusType::Failed
|
||||
|| roi.version_purge_status == VersionPurgeStatusType::Failed
|
||||
|| roi.version_purge_status == VersionPurgeStatusType::Pending
|
||||
{
|
||||
let admission = if let Some(pool) = runtime_sources::replication_pool() {
|
||||
pool.queue_replica_delete_task(dv).await
|
||||
} else {
|
||||
ReplicationQueueAdmission::Missed
|
||||
};
|
||||
return ReplicationHealQueueResult {
|
||||
object_info: roi,
|
||||
admission,
|
||||
};
|
||||
}
|
||||
|
||||
// if replication status is Complete on DeleteMarker and existing object resync required
|
||||
let existing_obj_resync = roi.existing_obj_resync.clone();
|
||||
if existing_obj_resync.must_resync()
|
||||
&& (roi.replication_status == ReplicationStatusType::Completed || roi.replication_status.is_empty())
|
||||
{
|
||||
let admission = queue_replicate_deletes_wrapper(dv, existing_obj_resync).await;
|
||||
return ReplicationHealQueueResult {
|
||||
object_info: roi,
|
||||
admission,
|
||||
};
|
||||
}
|
||||
|
||||
return ReplicationHealQueueResult {
|
||||
object_info: roi,
|
||||
admission: ReplicationQueueAdmission::Skipped,
|
||||
};
|
||||
}
|
||||
|
||||
if roi.existing_obj_resync.must_resync() {
|
||||
roi.op_type = ReplicationType::ExistingObject;
|
||||
}
|
||||
|
||||
match roi.replication_status {
|
||||
ReplicationStatusType::Pending | ReplicationStatusType::Failed => {
|
||||
roi.event_type = REPLICATE_HEAL.to_string();
|
||||
},
|
||||
ReplicationHealQueueAction::QueueObject => {
|
||||
let admission = if let Some(pool) = runtime_sources::replication_pool() {
|
||||
pool.queue_replica_task(roi.clone()).await
|
||||
} else {
|
||||
ReplicationQueueAdmission::Missed
|
||||
};
|
||||
return ReplicationHealQueueResult {
|
||||
ReplicationHealQueueResult {
|
||||
object_info: roi,
|
||||
admission,
|
||||
};
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
|
||||
if roi.existing_obj_resync.must_resync() {
|
||||
roi.event_type = REPLICATE_EXISTING.to_string();
|
||||
let admission = if let Some(pool) = runtime_sources::replication_pool() {
|
||||
pool.queue_replica_task(roi.clone()).await
|
||||
} else {
|
||||
ReplicationQueueAdmission::Missed
|
||||
};
|
||||
return ReplicationHealQueueResult {
|
||||
object_info: roi,
|
||||
admission,
|
||||
};
|
||||
}
|
||||
|
||||
ReplicationHealQueueResult {
|
||||
object_info: roi,
|
||||
admission: ReplicationQueueAdmission::Skipped,
|
||||
}
|
||||
}
|
||||
|
||||
/// Wrapper function for queueing replicate deletes with resync decision
|
||||
async fn queue_replicate_deletes_wrapper(
|
||||
doi: DeletedObjectReplicationInfo,
|
||||
existing_obj_resync: ResyncDecision,
|
||||
) -> ReplicationQueueAdmission {
|
||||
let mut admission = ReplicationQueueAdmission::Skipped;
|
||||
for (k, v) in existing_obj_resync.targets.iter() {
|
||||
if v.replicate {
|
||||
let mut dv = doi.clone();
|
||||
dv.reset_id = v.reset_id.clone();
|
||||
dv.target_arn = k.clone();
|
||||
let target_admission = if let Some(pool) = runtime_sources::replication_pool() {
|
||||
ReplicationHealQueueAction::QueueDelete(dv) => {
|
||||
let admission = if let Some(pool) = runtime_sources::replication_pool() {
|
||||
pool.queue_replica_delete_task(dv).await
|
||||
} else {
|
||||
ReplicationQueueAdmission::Missed
|
||||
};
|
||||
admission.merge(target_admission);
|
||||
ReplicationHealQueueResult {
|
||||
object_info: roi,
|
||||
admission,
|
||||
}
|
||||
}
|
||||
ReplicationHealQueueAction::QueueResyncDeletes(batch) => {
|
||||
let admission = queue_replicate_deletes(batch).await;
|
||||
ReplicationHealQueueResult {
|
||||
object_info: roi,
|
||||
admission,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn queue_replicate_deletes(batch: ReplicationHealResyncDeletes) -> ReplicationQueueAdmission {
|
||||
let mut admission = ReplicationQueueAdmission::Skipped;
|
||||
for dv in batch.target_delete_infos() {
|
||||
let target_admission = if let Some(pool) = runtime_sources::replication_pool() {
|
||||
pool.queue_replica_delete_task(dv).await
|
||||
} else {
|
||||
ReplicationQueueAdmission::Missed
|
||||
};
|
||||
admission.merge(target_admission);
|
||||
}
|
||||
admission
|
||||
}
|
||||
@@ -1538,16 +1447,6 @@ mod tests {
|
||||
assert_eq!(admission, ReplicationQueueAdmission::Missed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn auto_resume_resync_only_for_inflight_states() {
|
||||
assert!(should_auto_resume_resync(ResyncStatusType::ResyncPending));
|
||||
assert!(should_auto_resume_resync(ResyncStatusType::ResyncStarted));
|
||||
assert!(!should_auto_resume_resync(ResyncStatusType::NoResync));
|
||||
assert!(!should_auto_resume_resync(ResyncStatusType::ResyncCanceled));
|
||||
assert!(!should_auto_resume_resync(ResyncStatusType::ResyncCompleted));
|
||||
assert!(!should_auto_resume_resync(ResyncStatusType::ResyncFailed));
|
||||
}
|
||||
|
||||
// ── MrfReplicateEntry encode/decode roundtrips ────────────────────────────
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -38,10 +38,14 @@ pub use operation::{
|
||||
MustReplicateOptions, ReplicationDeleteSource, ReplicationResyncTargetObject, delete_replication_missing_source_decision,
|
||||
delete_replication_object_opts, heal_uses_delete_replication_path, is_ssec_encrypted, resync_target_for_object,
|
||||
};
|
||||
pub use queue::{ReplicationHealQueueResult, ReplicationOperation, ReplicationPriority, ReplicationQueueAdmission};
|
||||
pub use queue::{
|
||||
ReplicationHealQueueAction, ReplicationHealQueueResult, ReplicationHealResyncDeletes, ReplicationOperation,
|
||||
ReplicationPriority, ReplicationQueueAdmission, replication_heal_queue_action,
|
||||
};
|
||||
pub use resync::{
|
||||
BucketReplicationResyncStatus, Error, Result, ResyncOpts, ResyncStatusType, TargetReplicationResyncStatus,
|
||||
decode_resync_file, encode_resync_file, is_version_id_mismatch, resync_state_accepts_update, should_count_head_proxy_failure,
|
||||
decode_resync_file, encode_resync_file, is_version_id_mismatch, resync_state_accepts_update, should_auto_resume_resync,
|
||||
should_count_head_proxy_failure,
|
||||
};
|
||||
pub use rule::ReplicationRuleExt;
|
||||
pub use runtime::{
|
||||
|
||||
@@ -140,10 +140,7 @@ fn tag_metadata_differs(source: &ReplicationSourceObject<'_>, target: &Replicati
|
||||
.and_then(|metadata| metadata.get(AMZ_OBJECT_TAGGING).map(String::as_str))
|
||||
.unwrap_or_default();
|
||||
let target_tags = ReplicationTagFilter::decode_tags_to_map(target_tagging);
|
||||
let source_tag_count = match i32::try_from(source_tags.len()) {
|
||||
Ok(count) => count,
|
||||
Err(_) => i32::MAX,
|
||||
};
|
||||
let source_tag_count = i32::try_from(source_tags.len()).unwrap_or(i32::MAX);
|
||||
|
||||
(target.tag_count > 0 && source_tags != target_tags) || target.tag_count != source_tag_count
|
||||
}
|
||||
|
||||
@@ -14,7 +14,13 @@
|
||||
|
||||
use std::any::Any;
|
||||
|
||||
use crate::{DeletedObjectReplicationInfo, MrfReplicateEntry, ReplicateObjectInfo, ReplicationType, ReplicationWorkerOperation};
|
||||
use rustfs_storage_api::DeletedObject;
|
||||
|
||||
use crate::{
|
||||
DeletedObjectReplicationInfo, MrfReplicateEntry, REPLICATE_EXISTING, REPLICATE_HEAL, REPLICATE_HEAL_DELETE,
|
||||
ReplicateObjectInfo, ReplicationStatusType, ReplicationType, ReplicationWorkerOperation, ResyncDecision,
|
||||
VersionPurgeStatusType,
|
||||
};
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
pub enum ReplicationQueueAdmission {
|
||||
@@ -40,6 +46,116 @@ pub struct ReplicationHealQueueResult {
|
||||
pub admission: ReplicationQueueAdmission,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub enum ReplicationHealQueueAction {
|
||||
#[default]
|
||||
Skip,
|
||||
QueueObject,
|
||||
QueueDelete(DeletedObjectReplicationInfo),
|
||||
QueueResyncDeletes(ReplicationHealResyncDeletes),
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct ReplicationHealResyncDeletes {
|
||||
pub delete_info: DeletedObjectReplicationInfo,
|
||||
pub existing_obj_resync: ResyncDecision,
|
||||
}
|
||||
|
||||
impl ReplicationHealResyncDeletes {
|
||||
pub fn target_delete_infos(&self) -> impl Iterator<Item = DeletedObjectReplicationInfo> + '_ {
|
||||
self.existing_obj_resync
|
||||
.targets
|
||||
.iter()
|
||||
.filter(|(_, target)| target.replicate)
|
||||
.map(|(target_arn, target)| {
|
||||
let mut delete_info = self.delete_info.clone();
|
||||
delete_info.reset_id = target.reset_id.clone();
|
||||
delete_info.target_arn = target_arn.clone();
|
||||
delete_info
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
pub fn replication_heal_queue_action(roi: &mut ReplicateObjectInfo) -> ReplicationHealQueueAction {
|
||||
if !roi.dsc.replicate_any() {
|
||||
return ReplicationHealQueueAction::Skip;
|
||||
}
|
||||
|
||||
if roi.replication_status == ReplicationStatusType::Completed
|
||||
&& roi.version_purge_status.is_empty()
|
||||
&& !roi.existing_obj_resync.must_resync()
|
||||
{
|
||||
return ReplicationHealQueueAction::Skip;
|
||||
}
|
||||
|
||||
if roi.delete_marker || !roi.version_purge_status.is_empty() {
|
||||
let delete_info = heal_deleted_object_replication_info(roi);
|
||||
|
||||
if is_pending_or_failed_object_heal(roi) || is_pending_or_failed_version_purge(roi) {
|
||||
return ReplicationHealQueueAction::QueueDelete(delete_info);
|
||||
}
|
||||
|
||||
if roi.existing_obj_resync.must_resync()
|
||||
&& (roi.replication_status == ReplicationStatusType::Completed || roi.replication_status.is_empty())
|
||||
{
|
||||
return ReplicationHealQueueAction::QueueResyncDeletes(ReplicationHealResyncDeletes {
|
||||
delete_info,
|
||||
existing_obj_resync: roi.existing_obj_resync.clone(),
|
||||
});
|
||||
}
|
||||
|
||||
return ReplicationHealQueueAction::Skip;
|
||||
}
|
||||
|
||||
if roi.existing_obj_resync.must_resync() {
|
||||
roi.op_type = ReplicationType::ExistingObject;
|
||||
}
|
||||
|
||||
if is_pending_or_failed_object_heal(roi) {
|
||||
roi.event_type = REPLICATE_HEAL.to_string();
|
||||
return ReplicationHealQueueAction::QueueObject;
|
||||
}
|
||||
|
||||
if roi.existing_obj_resync.must_resync() {
|
||||
roi.event_type = REPLICATE_EXISTING.to_string();
|
||||
return ReplicationHealQueueAction::QueueObject;
|
||||
}
|
||||
|
||||
ReplicationHealQueueAction::Skip
|
||||
}
|
||||
|
||||
fn heal_deleted_object_replication_info(roi: &ReplicateObjectInfo) -> DeletedObjectReplicationInfo {
|
||||
let (version_id, delete_marker_version_id) = if roi.version_purge_status.is_empty() {
|
||||
(None, roi.version_id)
|
||||
} else {
|
||||
(roi.version_id, None)
|
||||
};
|
||||
|
||||
DeletedObjectReplicationInfo {
|
||||
delete_object: DeletedObject {
|
||||
object_name: roi.name.clone(),
|
||||
delete_marker_version_id,
|
||||
version_id,
|
||||
replication_state: roi.replication_state.clone(),
|
||||
delete_marker_mtime: roi.mod_time,
|
||||
delete_marker: roi.delete_marker,
|
||||
..Default::default()
|
||||
},
|
||||
bucket: roi.bucket.clone(),
|
||||
op_type: ReplicationType::Heal,
|
||||
event_type: REPLICATE_HEAL_DELETE.to_string(),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn is_pending_or_failed_object_heal(roi: &ReplicateObjectInfo) -> bool {
|
||||
matches!(roi.replication_status, ReplicationStatusType::Pending | ReplicationStatusType::Failed)
|
||||
}
|
||||
|
||||
fn is_pending_or_failed_version_purge(roi: &ReplicateObjectInfo) -> bool {
|
||||
matches!(roi.version_purge_status, VersionPurgeStatusType::Pending | VersionPurgeStatusType::Failed)
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub enum ReplicationPriority {
|
||||
Fast,
|
||||
@@ -129,9 +245,14 @@ mod tests {
|
||||
use std::str::FromStr;
|
||||
|
||||
use rustfs_storage_api::DeletedObject;
|
||||
use uuid::Uuid;
|
||||
|
||||
use super::{ReplicationOperation, ReplicationPriority, ReplicationQueueAdmission};
|
||||
use crate::{DeletedObjectReplicationInfo, ReplicateObjectInfo, ReplicationType, ReplicationWorkerOperation};
|
||||
use super::{ReplicationHealQueueAction, ReplicationOperation, ReplicationPriority, ReplicationQueueAdmission};
|
||||
use crate::{
|
||||
DeletedObjectReplicationInfo, REPLICATE_EXISTING, REPLICATE_HEAL, REPLICATE_HEAL_DELETE, ReplicateDecision,
|
||||
ReplicateObjectInfo, ReplicateTargetDecision, ReplicationStatusType, ReplicationType, ReplicationWorkerOperation,
|
||||
ResyncTargetDecision, VersionPurgeStatusType, replication_heal_queue_action,
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn replication_queue_admission_combines_target_results() {
|
||||
@@ -147,6 +268,102 @@ mod tests {
|
||||
assert_eq!(admission, ReplicationQueueAdmission::Missed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn heal_queue_action_routes_failed_objects_to_heal_queue() {
|
||||
let mut roi = replicate_object_info(ReplicationStatusType::Failed);
|
||||
|
||||
let action = replication_heal_queue_action(&mut roi);
|
||||
|
||||
assert!(matches!(action, ReplicationHealQueueAction::QueueObject));
|
||||
assert_eq!(roi.event_type, REPLICATE_HEAL);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn heal_queue_action_routes_existing_object_resync() {
|
||||
let mut roi = replicate_object_info(ReplicationStatusType::Completed);
|
||||
roi.existing_obj_resync.targets.insert(
|
||||
"arn:target".to_string(),
|
||||
ResyncTargetDecision {
|
||||
replicate: true,
|
||||
reset_id: "reset-1".to_string(),
|
||||
reset_before_date: None,
|
||||
},
|
||||
);
|
||||
|
||||
let action = replication_heal_queue_action(&mut roi);
|
||||
|
||||
assert!(matches!(action, ReplicationHealQueueAction::QueueObject));
|
||||
assert_eq!(roi.op_type, ReplicationType::ExistingObject);
|
||||
assert_eq!(roi.event_type, REPLICATE_EXISTING);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn heal_queue_action_routes_pending_delete_marker() {
|
||||
let version_id = Uuid::new_v4();
|
||||
let mut roi = replicate_object_info(ReplicationStatusType::Pending);
|
||||
roi.delete_marker = true;
|
||||
roi.version_id = Some(version_id);
|
||||
|
||||
let action = replication_heal_queue_action(&mut roi);
|
||||
|
||||
let ReplicationHealQueueAction::QueueDelete(delete_info) = action else {
|
||||
panic!("expected delete queue action");
|
||||
};
|
||||
assert_eq!(delete_info.bucket, "bucket");
|
||||
assert_eq!(delete_info.event_type, REPLICATE_HEAL_DELETE);
|
||||
assert_eq!(delete_info.delete_object.object_name, "object");
|
||||
assert_eq!(delete_info.delete_object.delete_marker_version_id, Some(version_id));
|
||||
assert_eq!(delete_info.delete_object.version_id, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn heal_queue_action_expands_resync_delete_targets() {
|
||||
let mut roi = replicate_object_info(ReplicationStatusType::Completed);
|
||||
roi.delete_marker = true;
|
||||
roi.existing_obj_resync.targets.insert(
|
||||
"arn:replicate".to_string(),
|
||||
ResyncTargetDecision {
|
||||
replicate: true,
|
||||
reset_id: "reset-1".to_string(),
|
||||
reset_before_date: None,
|
||||
},
|
||||
);
|
||||
roi.existing_obj_resync.targets.insert(
|
||||
"arn:skip".to_string(),
|
||||
ResyncTargetDecision {
|
||||
replicate: false,
|
||||
reset_id: "reset-2".to_string(),
|
||||
reset_before_date: None,
|
||||
},
|
||||
);
|
||||
|
||||
let action = replication_heal_queue_action(&mut roi);
|
||||
|
||||
let ReplicationHealQueueAction::QueueResyncDeletes(batch) = action else {
|
||||
panic!("expected resync delete queue action");
|
||||
};
|
||||
let target_deletes = batch.target_delete_infos().collect::<Vec<_>>();
|
||||
assert_eq!(target_deletes.len(), 1);
|
||||
assert_eq!(target_deletes[0].target_arn, "arn:replicate");
|
||||
assert_eq!(target_deletes[0].reset_id, "reset-1");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn heal_queue_action_routes_pending_version_purge() {
|
||||
let version_id = Uuid::new_v4();
|
||||
let mut roi = replicate_object_info(ReplicationStatusType::Completed);
|
||||
roi.version_id = Some(version_id);
|
||||
roi.version_purge_status = VersionPurgeStatusType::Pending;
|
||||
|
||||
let action = replication_heal_queue_action(&mut roi);
|
||||
|
||||
let ReplicationHealQueueAction::QueueDelete(delete_info) = action else {
|
||||
panic!("expected version purge delete queue action");
|
||||
};
|
||||
assert_eq!(delete_info.delete_object.version_id, Some(version_id));
|
||||
assert_eq!(delete_info.delete_object.delete_marker_version_id, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replication_priority_parses_known_values_and_defaults_unknown() {
|
||||
assert_eq!(ReplicationPriority::from_str("fast"), Ok(ReplicationPriority::Fast));
|
||||
@@ -194,4 +411,17 @@ mod tests {
|
||||
assert_eq!(operation.get_size(), 128);
|
||||
assert_eq!(operation.get_op_type(), ReplicationType::Object);
|
||||
}
|
||||
|
||||
fn replicate_object_info(replication_status: ReplicationStatusType) -> ReplicateObjectInfo {
|
||||
let mut dsc = ReplicateDecision::new();
|
||||
dsc.set(ReplicateTargetDecision::new("arn:target".to_string(), true, false));
|
||||
|
||||
ReplicateObjectInfo {
|
||||
bucket: "bucket".to_string(),
|
||||
name: "object".to_string(),
|
||||
replication_status,
|
||||
dsc,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -106,6 +106,10 @@ impl ResyncStatusType {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn should_auto_resume_resync(status: ResyncStatusType) -> bool {
|
||||
matches!(status, ResyncStatusType::ResyncPending | ResyncStatusType::ResyncStarted)
|
||||
}
|
||||
|
||||
impl fmt::Display for ResyncStatusType {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
let s = match self {
|
||||
@@ -543,6 +547,16 @@ mod tests {
|
||||
assert_eq!(ResyncStatusType::NoResync.to_string(), "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn auto_resume_resync_only_for_inflight_states() {
|
||||
assert!(should_auto_resume_resync(ResyncStatusType::ResyncPending));
|
||||
assert!(should_auto_resume_resync(ResyncStatusType::ResyncStarted));
|
||||
assert!(!should_auto_resume_resync(ResyncStatusType::NoResync));
|
||||
assert!(!should_auto_resume_resync(ResyncStatusType::ResyncCanceled));
|
||||
assert!(!should_auto_resume_resync(ResyncStatusType::ResyncCompleted));
|
||||
assert!(!should_auto_resume_resync(ResyncStatusType::ResyncFailed));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resync_state_accepts_update_only_for_matching_run() {
|
||||
let current = TargetReplicationResyncStatus {
|
||||
|
||||
@@ -2598,7 +2598,7 @@ fi
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
replication_queue_status=0
|
||||
rg -n --with-filename '^\s*(?:pub(?:\([^)]*\))?\s+)?(?:enum\s+ReplicationQueueAdmission|struct\s+ReplicationHealQueueResult|enum\s+ReplicationPriority|enum\s+ReplicationOperation)\b' \
|
||||
rg -n --with-filename '^\s*(?:pub(?:\([^)]*\))?\s+)?(?:(?:enum)\s+(?:ReplicationQueueAdmission|ReplicationHealQueueAction|ReplicationPriority|ReplicationOperation)|(?:struct)\s+(?:ReplicationHealQueueResult|ReplicationHealResyncDeletes)|fn\s+(?:replication_heal_queue_action|heal_deleted_object_replication_info|is_pending_or_failed_object_heal|is_pending_or_failed_version_purge))\b' \
|
||||
crates/ecstore/src/bucket/replication \
|
||||
--glob '*.rs' >"$REPLICATION_QUEUE_CONTRACT_BACKSLIDE_HITS_FILE" || replication_queue_status=$?
|
||||
if [[ "$replication_queue_status" -ne 0 && "$replication_queue_status" -ne 1 ]]; then
|
||||
@@ -2643,7 +2643,7 @@ fi
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
replication_resync_status=0
|
||||
rg -n --with-filename '^\s*(?:pub(?:\([^)]*\))?\s+)?(?:(?:struct|enum)\s+(?:ResyncOpts|TargetReplicationResyncStatus|BucketReplicationResyncStatus|ResyncStatusType)|fn\s+(?:resync_state_accepts_update|should_count_head_proxy_failure|is_version_id_mismatch))\b' \
|
||||
rg -n --with-filename '^\s*(?:pub(?:\([^)]*\))?\s+)?(?:(?:struct|enum)\s+(?:ResyncOpts|TargetReplicationResyncStatus|BucketReplicationResyncStatus|ResyncStatusType)|fn\s+(?:resync_state_accepts_update|should_count_head_proxy_failure|should_auto_resume_resync|is_version_id_mismatch))\b' \
|
||||
crates/ecstore/src/bucket/replication \
|
||||
--glob '*.rs' >"$REPLICATION_RESYNC_CONTRACT_BACKSLIDE_HITS_FILE" || replication_resync_status=$?
|
||||
if [[ "$replication_resync_status" -ne 0 && "$replication_resync_status" -ne 1 ]]; then
|
||||
|
||||
Reference in New Issue
Block a user