mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-14 00:53:14 +00:00
fix(replication): report authoritative backlog metrics (#5209)
This commit is contained in:
cxymds
@@ -149,18 +149,19 @@ pub mod bucket {
|
||||
|
||||
pub mod replication {
|
||||
pub use crate::bucket::replication::{
|
||||
BucketReplicationResyncStatus, BucketStats, DeletedObjectReplicationInfo, DynReplicationPool, MustReplicateOptions,
|
||||
ObjectOpts, REPLICATE_INCOMING_DELETE, ReplicateDecision, ReplicateObjectInfo, ReplicationConfig,
|
||||
ReplicationConfigurationExt, ReplicationDeleteScheduleInput, ReplicationDeleteStateSource,
|
||||
ReplicationHealQueueResult, ReplicationObjectBridge, ReplicationObjectIO, ReplicationOperation, ReplicationPoolTrait,
|
||||
ReplicationPriority, ReplicationQueueAdmission, ReplicationScannerBridge, ReplicationState, ReplicationStats,
|
||||
ReplicationStatusType, ReplicationStorage, ReplicationTargetValidationError, ReplicationType, ResyncOpts,
|
||||
ResyncStatusType, TargetReplicationResyncStatus, VersionPurgeStatusType, delete_replication_state_from_config,
|
||||
delete_replication_version_id, get_global_replication_pool, get_global_replication_stats,
|
||||
init_background_replication, replication_state_to_filemeta, replication_status_to_filemeta, replication_statuses_map,
|
||||
replication_target_arns, should_remove_replication_target, should_schedule_delete_replication,
|
||||
should_use_existing_delete_replication_info, should_use_existing_delete_replication_source,
|
||||
validate_replication_config_target_arns, version_purge_status_to_filemeta,
|
||||
BucketReplicationResyncStatus, BucketStats, DeletedObjectReplicationInfo, DurableMrfBacklog, DynReplicationPool,
|
||||
MrfOpKind, MrfReplicateEntry, MustReplicateOptions, ObjectOpts, REPLICATE_INCOMING_DELETE, ReplicateDecision,
|
||||
ReplicateObjectInfo, ReplicationConfig, ReplicationConfigurationExt, ReplicationDeleteScheduleInput,
|
||||
ReplicationDeleteStateSource, ReplicationHealQueueResult, ReplicationObjectBridge, ReplicationObjectIO,
|
||||
ReplicationOperation, ReplicationPoolTrait, ReplicationPriority, ReplicationQueueAdmission, ReplicationScannerBridge,
|
||||
ReplicationState, ReplicationStats, ReplicationStatusType, ReplicationStorage, ReplicationTargetValidationError,
|
||||
ReplicationType, ResyncOpts, ResyncStatusType, TargetReplicationResyncStatus, VersionPurgeStatusType,
|
||||
delete_replication_state_from_config, delete_replication_version_id, get_global_replication_pool,
|
||||
get_global_replication_stats, init_background_replication, read_durable_mrf_backlog, replication_state_to_filemeta,
|
||||
replication_status_to_filemeta, replication_statuses_map, replication_target_arns, should_remove_replication_target,
|
||||
should_schedule_delete_replication, should_use_existing_delete_replication_info,
|
||||
should_use_existing_delete_replication_source, validate_replication_config_target_arns,
|
||||
version_purge_status_to_filemeta,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -52,9 +52,9 @@ pub use replication_config_boundary::{
|
||||
pub(crate) use replication_filemeta_boundary::ReplicateTargetDecision;
|
||||
pub(crate) use replication_filemeta_boundary::version_purge_statuses_map;
|
||||
pub use replication_filemeta_boundary::{
|
||||
REPLICATE_INCOMING_DELETE, ReplicateDecision, ReplicateObjectInfo, ReplicationState, ReplicationStatusType, ReplicationType,
|
||||
VersionPurgeStatusType, replication_state_to_filemeta, replication_status_to_filemeta, replication_statuses_map,
|
||||
version_purge_status_to_filemeta,
|
||||
MrfOpKind, MrfReplicateEntry, REPLICATE_INCOMING_DELETE, ReplicateDecision, ReplicateObjectInfo, ReplicationState,
|
||||
ReplicationStatusType, ReplicationType, VersionPurgeStatusType, replication_state_to_filemeta,
|
||||
replication_status_to_filemeta, replication_statuses_map, version_purge_status_to_filemeta,
|
||||
};
|
||||
pub(crate) use replication_filemeta_boundary::{
|
||||
replication_state_from_filemeta, replication_status_from_filemeta, version_purge_status_from_filemeta,
|
||||
@@ -69,8 +69,8 @@ pub use replication_object_decision_boundary::{
|
||||
should_use_existing_delete_replication_source,
|
||||
};
|
||||
pub use replication_pool::{
|
||||
DynReplicationPool, ReplicationPoolTrait, get_global_replication_pool, get_global_replication_stats,
|
||||
init_background_replication,
|
||||
DurableMrfBacklog, DynReplicationPool, ReplicationPoolTrait, get_global_replication_pool, get_global_replication_stats,
|
||||
init_background_replication, read_durable_mrf_backlog,
|
||||
};
|
||||
pub use replication_queue_boundary::{
|
||||
DeletedObjectReplicationInfo, ReplicationHealQueueResult, ReplicationOperation, ReplicationPriority,
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub(crate) use rustfs_replication::{MrfOpKind, MrfReplicateEntry};
|
||||
pub use rustfs_replication::{MrfOpKind, MrfReplicateEntry};
|
||||
pub(crate) use rustfs_replication::{
|
||||
REPLICATE_EXISTING, REPLICATE_EXISTING_DELETE, REPLICATE_HEAL_DELETE, ReplicateTargetDecision, ReplicatedInfos,
|
||||
ReplicatedTargetInfo, ReplicationAction, ReplicationWorkerOperation, ResyncDecision, get_replication_state,
|
||||
|
||||
@@ -68,6 +68,33 @@ const EVENT_REPLICATION_RESYNC_LOAD_SKIPPED: &str = "replication_resync_load_ski
|
||||
const EVENT_REPLICATION_RESYNC_RECOVERED: &str = "replication_resync_recovered";
|
||||
const EVENT_REPLICATION_MRF_QUEUE_UNAVAILABLE: &str = "replication_mrf_queue_unavailable";
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct DurableMrfBacklog {
|
||||
pub available: bool,
|
||||
pub entries: Vec<MrfReplicateEntry>,
|
||||
}
|
||||
|
||||
fn durable_mrf_backlog_from_read(result: Result<Vec<u8>, EcstoreError>) -> DurableMrfBacklog {
|
||||
match result {
|
||||
Ok(data) => match decode_mrf_file(&data) {
|
||||
Ok(entries) if entries.iter().all(|entry| entry.size >= 0) => DurableMrfBacklog {
|
||||
available: true,
|
||||
entries,
|
||||
},
|
||||
Ok(_) | Err(_) => DurableMrfBacklog::default(),
|
||||
},
|
||||
Err(EcstoreError::ConfigNotFound) => DurableMrfBacklog {
|
||||
available: true,
|
||||
entries: Vec::new(),
|
||||
},
|
||||
Err(_) => DurableMrfBacklog::default(),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn read_durable_mrf_backlog<S: ReplicationObjectIO>(storage: Arc<S>) -> DurableMrfBacklog {
|
||||
durable_mrf_backlog_from_read(ReplicationConfigStore::read(storage, ReplicationMetadataStore::MRF_REPLICATION_FILE).await)
|
||||
}
|
||||
|
||||
/// Main replication pool structure
|
||||
#[derive(Debug)]
|
||||
pub struct ReplicationPool<S: ReplicationStorage> {
|
||||
@@ -2233,4 +2260,53 @@ mod tests {
|
||||
// None so replay falls back to the current time (backlog#867 backward compatibility).
|
||||
assert_eq!(entry.delete_marker_mtime, None, "missing deleteMarkerMtime key must default to None");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn durable_mrf_snapshot_reads_restart_backlog_and_valid_empty_state() {
|
||||
let entries = vec![MrfReplicateEntry {
|
||||
bucket: "restart-bucket".to_string(),
|
||||
object: "object".to_string(),
|
||||
version_id: None,
|
||||
retry_count: 1,
|
||||
size: 512,
|
||||
op: MrfOpKind::Object,
|
||||
delete_marker_version_id: None,
|
||||
delete_marker: false,
|
||||
delete_marker_mtime: None,
|
||||
}];
|
||||
let encoded = encode_mrf_file(&entries).expect("durable MRF backlog should encode");
|
||||
|
||||
let recovered = durable_mrf_backlog_from_read(Ok(encoded));
|
||||
assert!(recovered.available);
|
||||
assert_eq!(recovered.entries.len(), 1);
|
||||
assert_eq!(recovered.entries[0].bucket, "restart-bucket");
|
||||
assert_eq!(recovered.entries[0].size, 512);
|
||||
|
||||
let missing_file = durable_mrf_backlog_from_read(Err(EcstoreError::ConfigNotFound));
|
||||
assert!(missing_file.available);
|
||||
assert!(missing_file.entries.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn durable_mrf_snapshot_marks_corrupt_or_invalid_data_unavailable() {
|
||||
let corrupt = durable_mrf_backlog_from_read(Ok(vec![0, 1, 2]));
|
||||
assert!(!corrupt.available);
|
||||
assert!(corrupt.entries.is_empty());
|
||||
|
||||
let negative = encode_mrf_file(&[MrfReplicateEntry {
|
||||
bucket: "bucket".to_string(),
|
||||
object: "object".to_string(),
|
||||
version_id: None,
|
||||
retry_count: 0,
|
||||
size: -1,
|
||||
op: MrfOpKind::Object,
|
||||
delete_marker_version_id: None,
|
||||
delete_marker: false,
|
||||
delete_marker_mtime: None,
|
||||
}])
|
||||
.expect("invalid persisted entry should still encode for boundary testing");
|
||||
let invalid = durable_mrf_backlog_from_read(Ok(negative));
|
||||
assert!(!invalid.available);
|
||||
assert!(invalid.entries.is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,9 +15,11 @@
|
||||
use super::replication_error_boundary::Error;
|
||||
use super::replication_filemeta_boundary::{ReplicatedTargetInfo, ReplicationStatusType, ReplicationType};
|
||||
use super::replication_resync_boundary::ResyncStatusType;
|
||||
#[cfg(test)]
|
||||
use super::replication_stats_boundary::FailStats;
|
||||
use super::replication_stats_boundary::{
|
||||
ActiveWorkerStat, BucketReplicationStat, BucketReplicationStats, BucketStats, InQueueMetric, ProxyMetric, ProxyStatsCache,
|
||||
QueueCache, SRMetricsSummary, XferStats,
|
||||
QueueCache, ReplicationMetricScope, SRMetricsSummary, XferStats,
|
||||
};
|
||||
use super::runtime_boundary as runtime_sources;
|
||||
use std::collections::HashMap;
|
||||
@@ -361,10 +363,12 @@ impl ReplicationStats {
|
||||
if rs.transfer_duration > Duration::default() {
|
||||
stat.latency.update(rs.transfer_size, rs.transfer_duration);
|
||||
stat.update_xfer_rate(rs.transfer_size, rs.transfer_duration);
|
||||
stat.latency_scope = ReplicationMetricScope::NodeLocal;
|
||||
}
|
||||
}
|
||||
(false, true, false) => {
|
||||
stat.fail_stats.add_size(rs.transfer_size, rs.err.as_ref());
|
||||
stat.failed = stat.fail_stats.to_metric();
|
||||
}
|
||||
(false, false, true) => {
|
||||
// Pending status, no processing for now
|
||||
@@ -379,7 +383,10 @@ impl ReplicationStats {
|
||||
let mut result = HashMap::with_capacity(cache.len());
|
||||
|
||||
for (bucket, stats) in cache.iter() {
|
||||
result.insert(bucket.clone(), stats.clone_stats());
|
||||
let mut snapshot = stats.clone_stats();
|
||||
snapshot.mark_node_local_provider_available();
|
||||
snapshot.queue_scope = ReplicationMetricScope::NodeLocal;
|
||||
result.insert(bucket.clone(), snapshot);
|
||||
}
|
||||
drop(cache);
|
||||
|
||||
@@ -388,6 +395,8 @@ impl ReplicationStats {
|
||||
for (bucket, queue_stats) in &q_cache.bucket_stats {
|
||||
let bucket_stats = result.entry(bucket.clone()).or_insert_with(BucketReplicationStats::new);
|
||||
bucket_stats.q_stat = queue_stats.snapshot();
|
||||
bucket_stats.mark_node_local_provider_available();
|
||||
bucket_stats.queue_scope = ReplicationMetricScope::NodeLocal;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -405,9 +414,13 @@ impl ReplicationStats {
|
||||
pub async fn get(&self, bucket: &str) -> BucketReplicationStats {
|
||||
let cache = self.cache.read().await;
|
||||
if let Some(stats) = cache.get(bucket) {
|
||||
stats.clone_stats()
|
||||
let mut snapshot = stats.clone_stats();
|
||||
snapshot.mark_node_local_provider_available();
|
||||
snapshot
|
||||
} else {
|
||||
BucketReplicationStats::new()
|
||||
let mut snapshot = BucketReplicationStats::new();
|
||||
snapshot.mark_node_local_provider_available();
|
||||
snapshot
|
||||
}
|
||||
}
|
||||
|
||||
@@ -453,11 +466,15 @@ impl ReplicationStats {
|
||||
let mut tq = InQueueMetric::default();
|
||||
|
||||
for bucket_stat in &bucket_stats {
|
||||
tot_replica_size += bucket_stat.replication_stats.replica_size;
|
||||
tot_replica_count += bucket_stat.replication_stats.replica_count;
|
||||
tot_replica_size = tot_replica_size.saturating_add(bucket_stat.replication_stats.replica_size);
|
||||
tot_replica_count = tot_replica_count.saturating_add(bucket_stat.replication_stats.replica_count);
|
||||
|
||||
for q in &bucket_stat.queue_stats.nodes {
|
||||
tq = tq.merge(&q.q_stats);
|
||||
if bucket_stat.replication_stats.queue_scope != ReplicationMetricScope::Unavailable {
|
||||
tq = tq.merge(&bucket_stat.replication_stats.q_stat);
|
||||
} else {
|
||||
for q in &bucket_stat.queue_stats.nodes {
|
||||
tq = tq.merge(&q.q_stats);
|
||||
}
|
||||
}
|
||||
|
||||
for (arn, stat) in &bucket_stat.replication_stats.stats {
|
||||
@@ -470,22 +487,38 @@ impl ReplicationStats {
|
||||
let f_stats = stat.fail_stats.merge(&old_stat.fail_stats);
|
||||
let lrg = old_stat.xfer_rate_lrg.merge(&stat.xfer_rate_lrg);
|
||||
let sml = old_stat.xfer_rate_sml.merge(&stat.xfer_rate_sml);
|
||||
let latency_available = stat.latency_scope != ReplicationMetricScope::Unavailable
|
||||
|| old_stat.latency_scope != ReplicationMetricScope::Unavailable;
|
||||
let bandwidth_available = stat.bandwidth_scope != ReplicationMetricScope::Unavailable
|
||||
|| old_stat.bandwidth_scope != ReplicationMetricScope::Unavailable;
|
||||
|
||||
*old_stat = BucketReplicationStat {
|
||||
failed: f_stats.to_metric(),
|
||||
fail_stats: f_stats,
|
||||
replicated_size: stat.replicated_size + old_stat.replicated_size,
|
||||
replicated_count: stat.replicated_count + old_stat.replicated_count,
|
||||
replicated_size: stat.replicated_size.saturating_add(old_stat.replicated_size),
|
||||
replicated_count: stat.replicated_count.saturating_add(old_stat.replicated_count),
|
||||
latency: stat.latency.merge(&old_stat.latency),
|
||||
xfer_rate_lrg: lrg,
|
||||
xfer_rate_sml: sml,
|
||||
bandwidth_limit_bytes_per_sec: stat.bandwidth_limit_bytes_per_sec,
|
||||
bandwidth_limit_bytes_per_sec: stat
|
||||
.bandwidth_limit_bytes_per_sec
|
||||
.saturating_add(old_stat.bandwidth_limit_bytes_per_sec),
|
||||
current_bandwidth_bytes_per_sec: stat.current_bandwidth_bytes_per_sec
|
||||
+ old_stat.current_bandwidth_bytes_per_sec,
|
||||
latency_scope: if latency_available {
|
||||
ReplicationMetricScope::ClusterAggregated
|
||||
} else {
|
||||
ReplicationMetricScope::Unavailable
|
||||
},
|
||||
bandwidth_scope: if bandwidth_available {
|
||||
ReplicationMetricScope::ClusterAggregated
|
||||
} else {
|
||||
ReplicationMetricScope::Unavailable
|
||||
},
|
||||
};
|
||||
|
||||
tot_replicated_size += stat.replicated_size;
|
||||
tot_replicated_count += stat.replicated_count;
|
||||
tot_replicated_size = tot_replicated_size.saturating_add(stat.replicated_size);
|
||||
tot_replicated_count = tot_replicated_count.saturating_add(stat.replicated_count);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -499,23 +532,28 @@ impl ReplicationStats {
|
||||
resync_started_count: bucket_stats
|
||||
.iter()
|
||||
.map(|stats| stats.replication_stats.resync_started_count)
|
||||
.sum(),
|
||||
.fold(0i64, i64::saturating_add),
|
||||
resync_completed_count: bucket_stats
|
||||
.iter()
|
||||
.map(|stats| stats.replication_stats.resync_completed_count)
|
||||
.sum(),
|
||||
.fold(0i64, i64::saturating_add),
|
||||
resync_failed_count: bucket_stats
|
||||
.iter()
|
||||
.map(|stats| stats.replication_stats.resync_failed_count)
|
||||
.sum(),
|
||||
.fold(0i64, i64::saturating_add),
|
||||
resync_canceled_count: bucket_stats
|
||||
.iter()
|
||||
.map(|stats| stats.replication_stats.resync_canceled_count)
|
||||
.sum(),
|
||||
.fold(0i64, i64::saturating_add),
|
||||
resync_duration_ms: bucket_stats
|
||||
.iter()
|
||||
.map(|stats| stats.replication_stats.resync_duration_ms)
|
||||
.sum(),
|
||||
.fold(0i64, i64::saturating_add),
|
||||
provider_available: true,
|
||||
cluster_complete: true,
|
||||
observed_node_count: u32::try_from(bucket_stats.len()).unwrap_or(u32::MAX),
|
||||
expected_node_count: u32::try_from(bucket_stats.len()).unwrap_or(u32::MAX),
|
||||
queue_scope: ReplicationMetricScope::ClusterAggregated,
|
||||
};
|
||||
|
||||
let qs = Default::default();
|
||||
@@ -547,6 +585,33 @@ impl ReplicationStats {
|
||||
bs
|
||||
}
|
||||
|
||||
pub async fn aggregate_bucket_replication_stats(
|
||||
&self,
|
||||
bucket: &str,
|
||||
bucket_stats: Vec<BucketStats>,
|
||||
expected_node_count: u32,
|
||||
) -> BucketStats {
|
||||
let mut aggregated = self.calculate_bucket_replication_stats(bucket, bucket_stats).await;
|
||||
let observed_node_count = aggregated.replication_stats.observed_node_count;
|
||||
let complete = observed_node_count == expected_node_count;
|
||||
aggregated.replication_stats.expected_node_count = expected_node_count;
|
||||
aggregated.replication_stats.cluster_complete = complete;
|
||||
aggregated.replication_stats.queue_scope = if complete {
|
||||
ReplicationMetricScope::ClusterAggregated
|
||||
} else {
|
||||
ReplicationMetricScope::PartialCluster
|
||||
};
|
||||
for stat in aggregated.replication_stats.stats.values_mut() {
|
||||
if stat.latency_scope != ReplicationMetricScope::Unavailable {
|
||||
stat.latency_scope = aggregated.replication_stats.queue_scope;
|
||||
}
|
||||
if stat.bandwidth_scope != ReplicationMetricScope::Unavailable {
|
||||
stat.bandwidth_scope = aggregated.replication_stats.queue_scope;
|
||||
}
|
||||
}
|
||||
aggregated
|
||||
}
|
||||
|
||||
/// Get latest replication statistics
|
||||
pub async fn get_latest_replication_stats(&self, bucket: &str) -> BucketStats {
|
||||
// In actual implementation, statistics would be obtained from cluster
|
||||
@@ -567,6 +632,15 @@ impl ReplicationStats {
|
||||
};
|
||||
drop(cache);
|
||||
|
||||
{
|
||||
let q_cache = self.q_cache.lock().await;
|
||||
if let Some(queue_stats) = q_cache.bucket_stats.get(bucket) {
|
||||
replication_stats.q_stat = queue_stats.snapshot();
|
||||
}
|
||||
}
|
||||
replication_stats.mark_node_local_provider_available();
|
||||
replication_stats.queue_scope = ReplicationMetricScope::NodeLocal;
|
||||
|
||||
if let Some(monitor) = runtime_sources::bucket_monitor() {
|
||||
let bw_report = monitor.get_report(|name| name == bucket);
|
||||
for (opts, bw) in bw_report.bucket_stats {
|
||||
@@ -578,8 +652,7 @@ impl ReplicationStats {
|
||||
xfer_rate_sml: XferStats::new(),
|
||||
..Default::default()
|
||||
});
|
||||
stat.bandwidth_limit_bytes_per_sec = bw.limit_bytes_per_sec;
|
||||
stat.current_bandwidth_bytes_per_sec = bw.current_bandwidth_bytes_per_sec;
|
||||
stat.set_node_local_bandwidth(bw.limit_bytes_per_sec, bw.current_bandwidth_bytes_per_sec);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -724,6 +797,132 @@ mod tests {
|
||||
assert_eq!(stat.replicated_count, 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn latest_stats_include_queue_until_drained() {
|
||||
let stats = ReplicationStats::new();
|
||||
|
||||
stats.inc_q("queued-bucket", 4096, false, ReplicationType::Object).await;
|
||||
let queued = stats.get_latest_replication_stats("queued-bucket").await;
|
||||
assert!(queued.replication_stats.provider_available);
|
||||
assert_eq!(queued.replication_stats.q_stat.curr.count, 1);
|
||||
assert_eq!(queued.replication_stats.q_stat.curr.bytes, 4096);
|
||||
assert_eq!(queued.replication_stats.queue_scope, ReplicationMetricScope::NodeLocal);
|
||||
|
||||
stats.dec_q("queued-bucket", 4096, false, ReplicationType::Object).await;
|
||||
let drained = stats.get_latest_replication_stats("queued-bucket").await;
|
||||
assert_eq!(drained.replication_stats.q_stat.curr.count, 0);
|
||||
assert_eq!(drained.replication_stats.q_stat.curr.bytes, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn failed_metric_matches_authoritative_fail_stats() {
|
||||
let stats = ReplicationStats::new();
|
||||
let target_info = ReplicatedTargetInfo {
|
||||
arn: "failed-arn".to_string(),
|
||||
size: 2048,
|
||||
duration: Duration::from_millis(25),
|
||||
op_type: ReplicationType::Object,
|
||||
error: Some("target unavailable".to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
stats
|
||||
.update(
|
||||
"failed-bucket",
|
||||
&target_info,
|
||||
ReplicationStatusType::Failed,
|
||||
ReplicationStatusType::Pending,
|
||||
)
|
||||
.await;
|
||||
|
||||
let snapshot = stats.get_latest_replication_stats("failed-bucket").await;
|
||||
let target = &snapshot.replication_stats.stats["failed-arn"];
|
||||
assert_eq!(target.failed.count, target.fail_stats.count);
|
||||
assert_eq!(target.failed.size, target.fail_stats.size);
|
||||
assert_eq!(target.failed.count, 1);
|
||||
assert_eq!(target.failed.size, 2048);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn valid_empty_provider_is_not_reported_as_unavailable() {
|
||||
let stats = ReplicationStats::new();
|
||||
|
||||
let snapshot = stats.get_latest_replication_stats("empty-bucket").await;
|
||||
|
||||
assert!(snapshot.replication_stats.provider_available);
|
||||
assert!(snapshot.replication_stats.cluster_complete);
|
||||
assert_eq!(snapshot.replication_stats.observed_node_count, 1);
|
||||
assert_eq!(snapshot.replication_stats.expected_node_count, 1);
|
||||
assert!(snapshot.replication_stats.stats.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cluster_aggregation_counts_each_node_once_and_marks_partial() {
|
||||
let stats = ReplicationStats::new();
|
||||
let node = |failed_count, failed_size, queued_count, queued_size| {
|
||||
let mut fail_stats = FailStats::new();
|
||||
fail_stats.count = failed_count;
|
||||
fail_stats.size = failed_size;
|
||||
let mut targets = HashMap::new();
|
||||
targets.insert(
|
||||
"arn".to_string(),
|
||||
BucketReplicationStat {
|
||||
fail_stats,
|
||||
latency_scope: ReplicationMetricScope::NodeLocal,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
let q_stat = InQueueMetric::default();
|
||||
q_stat.curr.now_count.store(queued_count, Ordering::Relaxed);
|
||||
q_stat.curr.now_bytes.store(queued_size, Ordering::Relaxed);
|
||||
let q_stat = q_stat.snapshot();
|
||||
BucketStats {
|
||||
replication_stats: BucketReplicationStats {
|
||||
stats: targets,
|
||||
q_stat,
|
||||
provider_available: true,
|
||||
queue_scope: ReplicationMetricScope::NodeLocal,
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
}
|
||||
};
|
||||
|
||||
let aggregated = stats
|
||||
.aggregate_bucket_replication_stats("bucket", vec![node(1, 10, 2, 20), node(3, 30, 4, 40)], 3)
|
||||
.await;
|
||||
|
||||
let target = &aggregated.replication_stats.stats["arn"];
|
||||
assert_eq!(target.failed.count, 4);
|
||||
assert_eq!(target.failed.size, 40);
|
||||
assert_eq!(aggregated.replication_stats.q_stat.curr.count, 6);
|
||||
assert_eq!(aggregated.replication_stats.q_stat.curr.bytes, 60);
|
||||
assert_eq!(aggregated.replication_stats.observed_node_count, 2);
|
||||
assert_eq!(aggregated.replication_stats.expected_node_count, 3);
|
||||
assert!(!aggregated.replication_stats.cluster_complete);
|
||||
assert_eq!(aggregated.replication_stats.queue_scope, ReplicationMetricScope::PartialCluster);
|
||||
assert_eq!(target.latency_scope, ReplicationMetricScope::PartialCluster);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn concurrent_queue_updates_are_visible_without_lost_counts() {
|
||||
let stats = Arc::new(ReplicationStats::new());
|
||||
let mut tasks = Vec::with_capacity(32);
|
||||
for _ in 0..32 {
|
||||
let stats = Arc::clone(&stats);
|
||||
tasks.push(tokio::spawn(async move {
|
||||
stats.inc_q("concurrent-bucket", 7, false, ReplicationType::Object).await;
|
||||
}));
|
||||
}
|
||||
for task in tasks {
|
||||
task.await.expect("queue update task should complete");
|
||||
}
|
||||
|
||||
let snapshot = stats.get_latest_replication_stats("concurrent-bucket").await;
|
||||
assert_eq!(snapshot.replication_stats.q_stat.curr.count, 32);
|
||||
assert_eq!(snapshot.replication_stats.q_stat.curr.bytes, 224);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_get_all_includes_proxy_only_bucket() {
|
||||
let stats = ReplicationStats::new();
|
||||
|
||||
@@ -13,7 +13,9 @@
|
||||
// limitations under the License.
|
||||
|
||||
pub use rustfs_replication::BucketStats;
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_replication::FailStats;
|
||||
pub(crate) use rustfs_replication::{
|
||||
ActiveWorkerStat, BucketReplicationStat, BucketReplicationStats, InQueueMetric, ProxyMetric, ProxyStatsCache, QueueCache,
|
||||
SRMetricsSummary, XferStats,
|
||||
ReplicationMetricScope, SRMetricsSummary, XferStats,
|
||||
};
|
||||
|
||||
@@ -19,6 +19,7 @@ use crate::cluster::rpc::client::{
|
||||
use crate::cluster::rpc::{set_tonic_canonical_body_digest, verify_tonic_rpc_response_proof};
|
||||
use crate::error::{Error, Result};
|
||||
use crate::{
|
||||
bucket::replication::BucketStats,
|
||||
disk::disk_store::{get_drive_active_check_interval, get_drive_active_check_timeout},
|
||||
layout::endpoints::EndpointServerPools,
|
||||
runtime::sources as runtime_sources,
|
||||
@@ -34,14 +35,15 @@ use rustfs_madmin::{
|
||||
};
|
||||
use rustfs_protos::proto_gen::node_service::{
|
||||
BackgroundHealStatusRequest, CancelDecommissionRequest, ClearDecommissionRequest, DeleteBucketMetadataRequest,
|
||||
DeletePolicyRequest, DeleteServiceAccountRequest, DeleteUserRequest, GetCpusRequest, GetLiveEventsRequest, GetMemInfoRequest,
|
||||
GetMetricsRequest, GetNetInfoRequest, GetOsInfoRequest, GetPartitionsRequest, GetProcInfoRequest, GetSeLinuxInfoRequest,
|
||||
GetSysConfigRequest, GetSysErrorsRequest, HealControlRequest, LoadBucketMetadataRequest, LoadGroupRequest,
|
||||
LoadPolicyMappingRequest, LoadPolicyRequest, LoadRebalanceMetaRequest, LoadServiceAccountRequest,
|
||||
LoadTransitionTierConfigRequest, LoadUserRequest, LocalStorageInfoRequest, Mss, ReloadPoolMetaRequest,
|
||||
ReloadSiteReplicationConfigRequest, ScannerActivityRequest, ScannerActivityResponse, ServerInfoRequest, SignalServiceRequest,
|
||||
StartDecommissionRequest, StartProfilingRequest, StopRebalanceRequest, TierMutationAbortRequest, TierMutationCommitRequest,
|
||||
TierMutationControlResponse, TierMutationPeerState, TierMutationPrepareRequest, node_service_client::NodeServiceClient,
|
||||
DeletePolicyRequest, DeleteServiceAccountRequest, DeleteUserRequest, GetBucketStatsDataRequest, GetBucketStatsDataResponse,
|
||||
GetCpusRequest, GetLiveEventsRequest, GetMemInfoRequest, GetMetricsRequest, GetNetInfoRequest, GetOsInfoRequest,
|
||||
GetPartitionsRequest, GetProcInfoRequest, GetSeLinuxInfoRequest, GetSysConfigRequest, GetSysErrorsRequest,
|
||||
HealControlRequest, LoadBucketMetadataRequest, LoadGroupRequest, LoadPolicyMappingRequest, LoadPolicyRequest,
|
||||
LoadRebalanceMetaRequest, LoadServiceAccountRequest, LoadTransitionTierConfigRequest, LoadUserRequest,
|
||||
LocalStorageInfoRequest, Mss, ReloadPoolMetaRequest, ReloadSiteReplicationConfigRequest, ScannerActivityRequest,
|
||||
ScannerActivityResponse, ServerInfoRequest, SignalServiceRequest, StartDecommissionRequest, StartProfilingRequest,
|
||||
StopRebalanceRequest, TierMutationAbortRequest, TierMutationCommitRequest, TierMutationControlResponse,
|
||||
TierMutationPeerState, TierMutationPrepareRequest, node_service_client::NodeServiceClient,
|
||||
tier_mutation_control_service_client::TierMutationControlServiceClient,
|
||||
};
|
||||
use rustfs_protos::{TierMutationRpcPhase, evict_failed_connection};
|
||||
@@ -74,6 +76,26 @@ const HEAL_CONTROL_PAYLOAD_MAX_SIZE: usize = 64 * 1024;
|
||||
const PEER_REST_RECOVERY_MAX_ATTEMPTS: u32 = 60;
|
||||
const PEER_REST_RECOVERY_MAX_BACKOFF: Duration = Duration::from_secs(30);
|
||||
const SCANNER_ACTIVITY_MAX_MESSAGE_SIZE: usize = 1024;
|
||||
const REPLICATION_STATS_MAX_MESSAGE_SIZE: usize = 8 * 1024 * 1024;
|
||||
|
||||
fn decode_bucket_stats_response(response: GetBucketStatsDataResponse) -> Result<BucketStats> {
|
||||
if !response.success {
|
||||
return Err(Error::other(
|
||||
response
|
||||
.error_info
|
||||
.unwrap_or_else(|| "peer replication statistics provider is unavailable".to_string()),
|
||||
));
|
||||
}
|
||||
if response.bucket_stats.len() > REPLICATION_STATS_MAX_MESSAGE_SIZE {
|
||||
return Err(Error::other("peer replication statistics response exceeds size limit"));
|
||||
}
|
||||
let mut buf = Deserializer::new(Cursor::new(response.bucket_stats));
|
||||
let stats = BucketStats::deserialize(&mut buf).map_err(Error::from)?;
|
||||
if !stats.replication_stats.provider_available {
|
||||
return Err(Error::other("peer replication statistics provider is unavailable"));
|
||||
}
|
||||
Ok(stats)
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct ScannerPeerActivity {
|
||||
@@ -819,9 +841,26 @@ impl PeerRestClient {
|
||||
Err(Error::NotImplemented)
|
||||
}
|
||||
|
||||
pub async fn get_bucket_stats(&self) -> Result<()> {
|
||||
warn!("get_bucket_stats is not implemented in PeerRestClient");
|
||||
Err(Error::NotImplemented)
|
||||
pub async fn get_bucket_stats(&self, bucket: &str) -> Result<BucketStats> {
|
||||
let response = self
|
||||
.finalize_result(
|
||||
async {
|
||||
let mut client = self
|
||||
.get_client()
|
||||
.await?
|
||||
.max_decoding_message_size(REPLICATION_STATS_MAX_MESSAGE_SIZE);
|
||||
let response = client
|
||||
.get_bucket_stats(Request::new(GetBucketStatsDataRequest {
|
||||
bucket: bucket.to_string(),
|
||||
}))
|
||||
.await?
|
||||
.into_inner();
|
||||
Ok(response)
|
||||
}
|
||||
.await,
|
||||
)
|
||||
.await?;
|
||||
decode_bucket_stats_response(response)
|
||||
}
|
||||
|
||||
pub async fn get_sr_metrics(&self) -> Result<()> {
|
||||
@@ -1508,6 +1547,50 @@ mod tests {
|
||||
use std::sync::{Arc, Mutex};
|
||||
use tracing_subscriber::{Registry, fmt::MakeWriter, layer::SubscriberExt};
|
||||
|
||||
#[test]
|
||||
fn replication_stats_response_decodes_valid_empty_provider() {
|
||||
let mut stats = BucketStats::default();
|
||||
stats.replication_stats.provider_available = true;
|
||||
let payload = rmp_serde::to_vec_named(&stats).expect("bucket statistics should encode");
|
||||
|
||||
let decoded = decode_bucket_stats_response(GetBucketStatsDataResponse {
|
||||
success: true,
|
||||
bucket_stats: payload.into(),
|
||||
error_info: None,
|
||||
})
|
||||
.expect("valid bucket statistics should decode");
|
||||
|
||||
assert!(decoded.replication_stats.provider_available);
|
||||
assert!(decoded.replication_stats.stats.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replication_stats_response_rejects_unavailable_malformed_and_oversized_payloads() {
|
||||
let unavailable = decode_bucket_stats_response(GetBucketStatsDataResponse {
|
||||
success: false,
|
||||
bucket_stats: Bytes::new(),
|
||||
error_info: Some("provider unavailable".to_string()),
|
||||
})
|
||||
.expect_err("unavailable provider must not become a zero snapshot");
|
||||
assert!(unavailable.to_string().contains("provider unavailable"));
|
||||
|
||||
let malformed = decode_bucket_stats_response(GetBucketStatsDataResponse {
|
||||
success: true,
|
||||
bucket_stats: Bytes::from_static(b"not-msgpack"),
|
||||
error_info: None,
|
||||
})
|
||||
.expect_err("malformed peer statistics must fail closed");
|
||||
assert!(!malformed.to_string().is_empty());
|
||||
|
||||
let oversized = decode_bucket_stats_response(GetBucketStatsDataResponse {
|
||||
success: true,
|
||||
bucket_stats: Bytes::from(vec![0; REPLICATION_STATS_MAX_MESSAGE_SIZE + 1]),
|
||||
error_info: None,
|
||||
})
|
||||
.expect_err("oversized peer statistics must fail closed");
|
||||
assert!(oversized.to_string().contains("size limit"));
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct CapturedLogs {
|
||||
buffer: Arc<Mutex<Vec<u8>>>,
|
||||
|
||||
@@ -82,7 +82,7 @@ pub use runtime::{
|
||||
pub use stats::{
|
||||
ActiveWorkerStat, BucketReplicationStat, BucketReplicationStats, BucketStats, ExponentialMovingAverage, FailStats,
|
||||
FailedMetric, InQueueMetric, InQueueStats, LatencyStats, ProxyMetric, ProxyStatsCache, QueueCache, QueueNode, QueueStats,
|
||||
SRMetricsSummary, XferStats,
|
||||
ReplicationMetricScope, SRMetricsSummary, XferStats,
|
||||
};
|
||||
pub use storage_api::{DeletedObject, ObjectToDelete};
|
||||
pub use tagging::{ReplicationTagFilter, decode_tags_to_map};
|
||||
|
||||
@@ -313,18 +313,24 @@ impl InQueueMetric {
|
||||
pub fn merge(&self, other: &InQueueMetric) -> Self {
|
||||
Self {
|
||||
curr: InQueueStats {
|
||||
bytes: self.curr.bytes + other.curr.bytes,
|
||||
count: self.curr.count + other.curr.count,
|
||||
bytes: self.curr.bytes.saturating_add(other.curr.bytes),
|
||||
count: self.curr.count.saturating_add(other.curr.count),
|
||||
now_bytes: AtomicI64::new(
|
||||
self.curr.now_bytes.load(Ordering::Relaxed) + other.curr.now_bytes.load(Ordering::Relaxed),
|
||||
self.curr
|
||||
.now_bytes
|
||||
.load(Ordering::Relaxed)
|
||||
.saturating_add(other.curr.now_bytes.load(Ordering::Relaxed)),
|
||||
),
|
||||
now_count: AtomicI64::new(
|
||||
self.curr.now_count.load(Ordering::Relaxed) + other.curr.now_count.load(Ordering::Relaxed),
|
||||
self.curr
|
||||
.now_count
|
||||
.load(Ordering::Relaxed)
|
||||
.saturating_add(other.curr.now_count.load(Ordering::Relaxed)),
|
||||
),
|
||||
},
|
||||
avg: InQueueStats {
|
||||
bytes: (self.avg.bytes + other.avg.bytes) / 2,
|
||||
count: (self.avg.count + other.avg.count) / 2,
|
||||
bytes: self.avg.bytes.saturating_add(other.avg.bytes) / 2,
|
||||
count: self.avg.count.saturating_add(other.avg.count) / 2,
|
||||
..Default::default()
|
||||
},
|
||||
max: InQueueStats {
|
||||
@@ -333,8 +339,8 @@ impl InQueueMetric {
|
||||
..Default::default()
|
||||
},
|
||||
last_minute: InQueueStats {
|
||||
bytes: self.last_minute.bytes + other.last_minute.bytes,
|
||||
count: self.last_minute.count + other.last_minute.count,
|
||||
bytes: self.last_minute.bytes.saturating_add(other.last_minute.bytes),
|
||||
count: self.last_minute.count.saturating_add(other.last_minute.count),
|
||||
..Default::default()
|
||||
},
|
||||
samples: VecDeque::new(),
|
||||
@@ -495,8 +501,8 @@ impl FailStats {
|
||||
|
||||
pub fn add_size<E>(&mut self, size: i64, _err: Option<&E>) {
|
||||
let observed_at = Instant::now();
|
||||
self.count += 1;
|
||||
self.size += size;
|
||||
self.count = self.count.saturating_add(1);
|
||||
self.size = self.size.saturating_add(size);
|
||||
self.recent.push_back(FailureSample { observed_at, size });
|
||||
self.prune(observed_at);
|
||||
}
|
||||
@@ -527,8 +533,8 @@ impl FailStats {
|
||||
|
||||
pub fn merge(&self, other: &FailStats) -> Self {
|
||||
Self {
|
||||
count: self.count + other.count,
|
||||
size: self.size + other.size,
|
||||
count: self.count.saturating_add(other.count),
|
||||
size: self.size.saturating_add(other.size),
|
||||
recent: VecDeque::new(),
|
||||
}
|
||||
}
|
||||
@@ -588,6 +594,10 @@ pub struct BucketReplicationStat {
|
||||
pub xfer_rate_sml: XferStats,
|
||||
pub bandwidth_limit_bytes_per_sec: i64,
|
||||
pub current_bandwidth_bytes_per_sec: f64,
|
||||
#[serde(default)]
|
||||
pub latency_scope: ReplicationMetricScope,
|
||||
#[serde(default)]
|
||||
pub bandwidth_scope: ReplicationMetricScope,
|
||||
}
|
||||
|
||||
impl BucketReplicationStat {
|
||||
@@ -602,6 +612,22 @@ impl BucketReplicationStat {
|
||||
self.xfer_rate_sml.add_size(size, duration);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn set_node_local_bandwidth(&mut self, limit_bytes_per_sec: i64, current_bytes_per_sec: f64) {
|
||||
self.bandwidth_limit_bytes_per_sec = limit_bytes_per_sec;
|
||||
self.current_bandwidth_bytes_per_sec = current_bytes_per_sec;
|
||||
self.bandwidth_scope = ReplicationMetricScope::NodeLocal;
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum ReplicationMetricScope {
|
||||
#[default]
|
||||
Unavailable,
|
||||
NodeLocal,
|
||||
ClusterAggregated,
|
||||
PartialCluster,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
||||
@@ -632,6 +658,16 @@ pub struct BucketReplicationStats {
|
||||
pub resync_canceled_count: i64,
|
||||
#[serde(default)]
|
||||
pub resync_duration_ms: i64,
|
||||
#[serde(default)]
|
||||
pub provider_available: bool,
|
||||
#[serde(default)]
|
||||
pub cluster_complete: bool,
|
||||
#[serde(default)]
|
||||
pub observed_node_count: u32,
|
||||
#[serde(default)]
|
||||
pub expected_node_count: u32,
|
||||
#[serde(default)]
|
||||
pub queue_scope: ReplicationMetricScope,
|
||||
}
|
||||
|
||||
impl BucketReplicationStats {
|
||||
@@ -662,7 +698,18 @@ impl BucketReplicationStats {
|
||||
}
|
||||
|
||||
pub fn clone_stats(&self) -> Self {
|
||||
self.clone()
|
||||
let mut snapshot = self.clone();
|
||||
for stat in snapshot.stats.values_mut() {
|
||||
stat.failed = stat.fail_stats.to_metric();
|
||||
}
|
||||
snapshot
|
||||
}
|
||||
|
||||
pub fn mark_node_local_provider_available(&mut self) {
|
||||
self.provider_available = true;
|
||||
self.cluster_complete = true;
|
||||
self.observed_node_count = 1;
|
||||
self.expected_node_count = 1;
|
||||
}
|
||||
|
||||
pub fn record_resync_status(&mut self, status: ResyncStatusType, duration: Option<Duration>) {
|
||||
@@ -791,6 +838,27 @@ mod tests {
|
||||
assert_eq!(last_hour.size, 96);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fail_stats_saturate_instead_of_wrapping() {
|
||||
let mut stats = FailStats {
|
||||
count: i64::MAX,
|
||||
size: i64::MAX,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
stats.add_size(1, None::<&()>);
|
||||
let merged = stats.merge(&FailStats {
|
||||
count: 1,
|
||||
size: 1,
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
assert_eq!(stats.count, i64::MAX);
|
||||
assert_eq!(stats.size, i64::MAX);
|
||||
assert_eq!(merged.count, i64::MAX);
|
||||
assert_eq!(merged.size, i64::MAX);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn active_worker_stat_update_tracks_rolling_avg_and_max() {
|
||||
let mut stats = ActiveWorkerStat::default();
|
||||
|
||||
Reference in New Issue
Block a user