mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-11 07:36:53 +00:00
refactor(s3): consolidate semantic boundaries and remove s3-common (#3012)
* refactor(common): introduce rustfs-data-usage core crate * refactor(concurrency): migrate workers crate into concurrency * refactor(crypto): migrate appauth token APIs into crypto * fix docs urls * remove unused crate * refactor(data-usage): switch consumers to rustfs-data-usage * chore(fmt): apply cargo fmt and lockfile sync * refactor(common): remove data_usage compatibility re-export * refactor(capacity): move capacity_scope to object-capacity * refactor(io-metrics): relocate internode metrics from common * refactor(common): decouple scanner report from madmin * chore(fmt): normalize import ordering after pre-commit * refactor(s3): split s3 types and ops crates * refactor(s3): centralize event version and safe parsing * refactor(s3): add op-event compatibility guardrails * refactor(s3): add runtime op-event mismatch observability * refactor(s3): extract delete event mapping helper * refactor(s3): extract put event mapping helper * refactor(s3): consolidate remaining event semantic helpers * refactor(s3): add op-event coverage checks and observability alerts * refactor(s3-ops): consolidate op-event semantic mapping * refactor(scanner): remove last_minute wrapper module * refactor(scanner): consolidate duplicated data usage models
This commit is contained in:
houseme
@@ -1,308 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::sync::{Mutex, OnceLock};
|
||||
use std::time::{Duration, Instant};
|
||||
use uuid::Uuid;
|
||||
|
||||
const CAPACITY_SCOPE_REGISTRY_SOFT_LIMIT: usize = 2_048;
|
||||
const CAPACITY_SCOPE_REGISTRY_HARD_LIMIT: usize = 4_096;
|
||||
const CAPACITY_SCOPE_TTL: Duration = Duration::from_secs(300);
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
|
||||
pub struct CapacityScopeDisk {
|
||||
pub endpoint: String,
|
||||
pub drive_path: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Default)]
|
||||
pub struct CapacityScope {
|
||||
pub disks: Vec<CapacityScopeDisk>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct CapacityScopeEntry {
|
||||
scope: CapacityScope,
|
||||
recorded_at: Instant,
|
||||
}
|
||||
|
||||
fn capacity_scope_registry() -> &'static Mutex<HashMap<Uuid, CapacityScopeEntry>> {
|
||||
static REGISTRY: OnceLock<Mutex<HashMap<Uuid, CapacityScopeEntry>>> = OnceLock::new();
|
||||
REGISTRY.get_or_init(|| Mutex::new(HashMap::new()))
|
||||
}
|
||||
|
||||
fn global_dirty_scope_registry() -> &'static Mutex<HashSet<CapacityScopeDisk>> {
|
||||
static REGISTRY: OnceLock<Mutex<HashSet<CapacityScopeDisk>>> = OnceLock::new();
|
||||
REGISTRY.get_or_init(|| Mutex::new(HashSet::new()))
|
||||
}
|
||||
|
||||
fn prune_expired_entries(entries: &mut HashMap<Uuid, CapacityScopeEntry>, now: Instant) {
|
||||
entries.retain(|_, entry| now.duration_since(entry.recorded_at) <= CAPACITY_SCOPE_TTL);
|
||||
}
|
||||
|
||||
fn enforce_hard_limit(entries: &mut HashMap<Uuid, CapacityScopeEntry>, max_len: usize) {
|
||||
if entries.len() < max_len {
|
||||
return;
|
||||
}
|
||||
|
||||
let evict_count = entries.len() - max_len + 1;
|
||||
let mut eviction_order: Vec<_> = entries.iter().map(|(token, entry)| (*token, entry.recorded_at)).collect();
|
||||
eviction_order.sort_unstable_by_key(|(_, recorded_at)| *recorded_at);
|
||||
|
||||
for (token, _) in eviction_order.into_iter().take(evict_count) {
|
||||
entries.remove(&token);
|
||||
}
|
||||
}
|
||||
|
||||
fn merge_capacity_scopes(existing: &mut CapacityScope, incoming: CapacityScope) {
|
||||
let mut seen: HashSet<CapacityScopeDisk> = existing.disks.iter().cloned().collect();
|
||||
for disk in incoming.disks {
|
||||
if seen.insert(disk.clone()) {
|
||||
existing.disks.push(disk);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn record_capacity_scope(token: Uuid, scope: CapacityScope) {
|
||||
let now = Instant::now();
|
||||
let mut entries = capacity_scope_registry().lock().unwrap_or_else(|p| p.into_inner());
|
||||
if !entries.contains_key(&token) && entries.len() >= CAPACITY_SCOPE_REGISTRY_SOFT_LIMIT {
|
||||
prune_expired_entries(&mut entries, now);
|
||||
enforce_hard_limit(&mut entries, CAPACITY_SCOPE_REGISTRY_HARD_LIMIT);
|
||||
}
|
||||
if let Some(entry) = entries.get_mut(&token) {
|
||||
merge_capacity_scopes(&mut entry.scope, scope);
|
||||
entry.recorded_at = now;
|
||||
} else {
|
||||
entries.insert(token, CapacityScopeEntry { scope, recorded_at: now });
|
||||
}
|
||||
}
|
||||
|
||||
pub fn take_capacity_scope(token: Uuid) -> Option<CapacityScope> {
|
||||
let now = Instant::now();
|
||||
let mut entries = capacity_scope_registry().lock().unwrap_or_else(|p| p.into_inner());
|
||||
let entry = entries.remove(&token)?;
|
||||
if now.duration_since(entry.recorded_at) > CAPACITY_SCOPE_TTL {
|
||||
return None;
|
||||
}
|
||||
Some(entry.scope)
|
||||
}
|
||||
|
||||
pub fn record_global_dirty_scope(scope: CapacityScope) {
|
||||
if scope.disks.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
let mut dirty_scopes = global_dirty_scope_registry().lock().unwrap_or_else(|p| p.into_inner());
|
||||
dirty_scopes.extend(scope.disks);
|
||||
}
|
||||
|
||||
pub fn drain_global_dirty_scopes() -> Vec<CapacityScopeDisk> {
|
||||
let mut dirty_scopes = global_dirty_scope_registry().lock().unwrap_or_else(|p| p.into_inner());
|
||||
dirty_scopes.drain().collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::sync::Mutex;
|
||||
use std::thread;
|
||||
|
||||
fn test_lock() -> &'static Mutex<()> {
|
||||
static LOCK: OnceLock<Mutex<()>> = OnceLock::new();
|
||||
LOCK.get_or_init(|| Mutex::new(()))
|
||||
}
|
||||
|
||||
fn clear_capacity_scope_registry_for_test() {
|
||||
capacity_scope_registry()
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
.clear();
|
||||
global_dirty_scope_registry()
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
.clear();
|
||||
}
|
||||
|
||||
fn poison_capacity_scope_registry_for_test() {
|
||||
let _ = thread::spawn(|| {
|
||||
let _guard = capacity_scope_registry()
|
||||
.lock()
|
||||
.expect("capacity scope registry lock should succeed");
|
||||
panic!("poison capacity scope registry");
|
||||
})
|
||||
.join();
|
||||
}
|
||||
|
||||
fn poison_global_dirty_scope_registry_for_test() {
|
||||
let _ = thread::spawn(|| {
|
||||
let _guard = global_dirty_scope_registry()
|
||||
.lock()
|
||||
.expect("global dirty scope registry lock should succeed");
|
||||
panic!("poison global dirty scope registry");
|
||||
})
|
||||
.join();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn record_and_take_capacity_scope_round_trips() {
|
||||
let _guard = test_lock().lock().expect("test lock poisoned");
|
||||
clear_capacity_scope_registry_for_test();
|
||||
let token = Uuid::new_v4();
|
||||
let scope = CapacityScope {
|
||||
disks: vec![CapacityScopeDisk {
|
||||
endpoint: "node-a".to_string(),
|
||||
drive_path: "/tmp/disk-a".to_string(),
|
||||
}],
|
||||
};
|
||||
|
||||
record_capacity_scope(token, scope.clone());
|
||||
|
||||
assert_eq!(take_capacity_scope(token), Some(scope));
|
||||
assert_eq!(take_capacity_scope(token), None);
|
||||
clear_capacity_scope_registry_for_test();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn record_capacity_scope_merges_disks_for_same_token() {
|
||||
let _guard = test_lock().lock().expect("test lock poisoned");
|
||||
clear_capacity_scope_registry_for_test();
|
||||
let token = Uuid::new_v4();
|
||||
record_capacity_scope(
|
||||
token,
|
||||
CapacityScope {
|
||||
disks: vec![CapacityScopeDisk {
|
||||
endpoint: "node-a".to_string(),
|
||||
drive_path: "/tmp/disk-a".to_string(),
|
||||
}],
|
||||
},
|
||||
);
|
||||
record_capacity_scope(
|
||||
token,
|
||||
CapacityScope {
|
||||
disks: vec![
|
||||
CapacityScopeDisk {
|
||||
endpoint: "node-b".to_string(),
|
||||
drive_path: "/tmp/disk-b".to_string(),
|
||||
},
|
||||
CapacityScopeDisk {
|
||||
endpoint: "node-a".to_string(),
|
||||
drive_path: "/tmp/disk-a".to_string(),
|
||||
},
|
||||
],
|
||||
},
|
||||
);
|
||||
|
||||
let scope = take_capacity_scope(token).expect("scope should exist");
|
||||
assert_eq!(scope.disks.len(), 2);
|
||||
assert!(scope.disks.iter().any(|disk| disk.endpoint == "node-a"));
|
||||
assert!(scope.disks.iter().any(|disk| disk.endpoint == "node-b"));
|
||||
clear_capacity_scope_registry_for_test();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn record_capacity_scope_enforces_hard_limit() {
|
||||
let _guard = test_lock().lock().expect("test lock poisoned");
|
||||
clear_capacity_scope_registry_for_test();
|
||||
|
||||
for _ in 0..(CAPACITY_SCOPE_REGISTRY_HARD_LIMIT + 32) {
|
||||
record_capacity_scope(
|
||||
Uuid::new_v4(),
|
||||
CapacityScope {
|
||||
disks: vec![CapacityScopeDisk {
|
||||
endpoint: "node-a".to_string(),
|
||||
drive_path: "/tmp/disk-a".to_string(),
|
||||
}],
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
let entries = capacity_scope_registry()
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
assert!(entries.len() <= CAPACITY_SCOPE_REGISTRY_HARD_LIMIT);
|
||||
drop(entries);
|
||||
clear_capacity_scope_registry_for_test();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn record_capacity_scope_recovers_from_poisoned_registry() {
|
||||
let _guard = test_lock().lock().expect("test lock poisoned");
|
||||
clear_capacity_scope_registry_for_test();
|
||||
poison_capacity_scope_registry_for_test();
|
||||
let token = Uuid::new_v4();
|
||||
let scope = CapacityScope {
|
||||
disks: vec![CapacityScopeDisk {
|
||||
endpoint: "node-a".to_string(),
|
||||
drive_path: "/tmp/disk-a".to_string(),
|
||||
}],
|
||||
};
|
||||
|
||||
record_capacity_scope(token, scope.clone());
|
||||
|
||||
assert_eq!(take_capacity_scope(token), Some(scope));
|
||||
clear_capacity_scope_registry_for_test();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn record_and_drain_global_dirty_scope_round_trips() {
|
||||
let _guard = test_lock().lock().expect("test lock poisoned");
|
||||
clear_capacity_scope_registry_for_test();
|
||||
record_global_dirty_scope(CapacityScope {
|
||||
disks: vec![CapacityScopeDisk {
|
||||
endpoint: "node-a".to_string(),
|
||||
drive_path: "/tmp/disk-a".to_string(),
|
||||
}],
|
||||
});
|
||||
record_global_dirty_scope(CapacityScope {
|
||||
disks: vec![
|
||||
CapacityScopeDisk {
|
||||
endpoint: "node-b".to_string(),
|
||||
drive_path: "/tmp/disk-b".to_string(),
|
||||
},
|
||||
CapacityScopeDisk {
|
||||
endpoint: "node-a".to_string(),
|
||||
drive_path: "/tmp/disk-a".to_string(),
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
let drained = drain_global_dirty_scopes();
|
||||
assert_eq!(drained.len(), 2);
|
||||
assert!(drained.iter().any(|disk| disk.endpoint == "node-a"));
|
||||
assert!(drained.iter().any(|disk| disk.endpoint == "node-b"));
|
||||
assert!(drain_global_dirty_scopes().is_empty());
|
||||
clear_capacity_scope_registry_for_test();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn record_global_dirty_scope_recovers_from_poisoned_registry() {
|
||||
let _guard = test_lock().lock().expect("test lock poisoned");
|
||||
clear_capacity_scope_registry_for_test();
|
||||
poison_global_dirty_scope_registry_for_test();
|
||||
|
||||
record_global_dirty_scope(CapacityScope {
|
||||
disks: vec![CapacityScopeDisk {
|
||||
endpoint: "node-a".to_string(),
|
||||
drive_path: "/tmp/disk-a".to_string(),
|
||||
}],
|
||||
});
|
||||
|
||||
let drained = drain_global_dirty_scopes();
|
||||
assert_eq!(drained.len(), 1);
|
||||
assert_eq!(drained[0].endpoint, "node-a");
|
||||
clear_capacity_scope_registry_for_test();
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,232 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use metrics::{counter, gauge};
|
||||
use std::sync::{
|
||||
Arc, LazyLock,
|
||||
atomic::{AtomicU64, Ordering},
|
||||
};
|
||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
|
||||
pub const INTERNODE_OPERATION_READ_FILE_STREAM: &str = "read_file_stream";
|
||||
pub const INTERNODE_OPERATION_PUT_FILE_STREAM: &str = "put_file_stream";
|
||||
pub const INTERNODE_OPERATION_WALK_DIR: &str = "walk_dir";
|
||||
pub const INTERNODE_OPERATION_GRPC_READ_ALL: &str = "grpc_read_all";
|
||||
pub const INTERNODE_OPERATION_GRPC_WRITE_ALL: &str = "grpc_write_all";
|
||||
|
||||
const OPERATION_LABEL: &str = "operation";
|
||||
const INTERNODE_OPERATION_SENT_BYTES_TOTAL: &str = "rustfs_system_network_internode_operation_sent_bytes_total";
|
||||
const INTERNODE_OPERATION_RECV_BYTES_TOTAL: &str = "rustfs_system_network_internode_operation_recv_bytes_total";
|
||||
const INTERNODE_OPERATION_REQUESTS_OUTGOING_TOTAL: &str = "rustfs_system_network_internode_operation_requests_outgoing_total";
|
||||
const INTERNODE_OPERATION_REQUESTS_INCOMING_TOTAL: &str = "rustfs_system_network_internode_operation_requests_incoming_total";
|
||||
const INTERNODE_OPERATION_ERRORS_TOTAL: &str = "rustfs_system_network_internode_operation_errors_total";
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
pub struct InternodeMetricsSnapshot {
|
||||
pub sent_bytes_total: u64,
|
||||
pub recv_bytes_total: u64,
|
||||
pub outgoing_requests_total: u64,
|
||||
pub incoming_requests_total: u64,
|
||||
pub errors_total: u64,
|
||||
pub dial_errors_total: u64,
|
||||
pub dial_avg_time_nanos: u64,
|
||||
pub last_dial_unix_millis: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct InternodeMetrics {
|
||||
sent_bytes_total: AtomicU64,
|
||||
recv_bytes_total: AtomicU64,
|
||||
outgoing_requests_total: AtomicU64,
|
||||
incoming_requests_total: AtomicU64,
|
||||
errors_total: AtomicU64,
|
||||
dial_errors_total: AtomicU64,
|
||||
dial_total_time_nanos: AtomicU64,
|
||||
dial_samples_total: AtomicU64,
|
||||
last_dial_unix_millis: AtomicU64,
|
||||
}
|
||||
|
||||
impl InternodeMetrics {
|
||||
pub fn record_sent_bytes(&self, bytes: usize) {
|
||||
let bytes = bytes as u64;
|
||||
if bytes == 0 {
|
||||
return;
|
||||
}
|
||||
self.sent_bytes_total.fetch_add(bytes, Ordering::Relaxed);
|
||||
counter!("rustfs_system_network_internode_sent_bytes_total").increment(bytes);
|
||||
}
|
||||
|
||||
pub fn record_sent_bytes_for_operation(&self, operation: &'static str, bytes: usize) {
|
||||
self.record_sent_bytes(bytes);
|
||||
|
||||
let bytes = bytes as u64;
|
||||
if bytes == 0 {
|
||||
return;
|
||||
}
|
||||
counter!(INTERNODE_OPERATION_SENT_BYTES_TOTAL, OPERATION_LABEL => operation).increment(bytes);
|
||||
}
|
||||
|
||||
pub fn record_recv_bytes(&self, bytes: usize) {
|
||||
let bytes = bytes as u64;
|
||||
if bytes == 0 {
|
||||
return;
|
||||
}
|
||||
self.recv_bytes_total.fetch_add(bytes, Ordering::Relaxed);
|
||||
counter!("rustfs_system_network_internode_recv_bytes_total").increment(bytes);
|
||||
}
|
||||
|
||||
pub fn record_recv_bytes_for_operation(&self, operation: &'static str, bytes: usize) {
|
||||
self.record_recv_bytes(bytes);
|
||||
|
||||
let bytes = bytes as u64;
|
||||
if bytes == 0 {
|
||||
return;
|
||||
}
|
||||
counter!(INTERNODE_OPERATION_RECV_BYTES_TOTAL, OPERATION_LABEL => operation).increment(bytes);
|
||||
}
|
||||
|
||||
pub fn record_outgoing_request(&self) {
|
||||
self.outgoing_requests_total.fetch_add(1, Ordering::Relaxed);
|
||||
counter!("rustfs_system_network_internode_requests_outgoing_total").increment(1);
|
||||
}
|
||||
|
||||
pub fn record_outgoing_request_for_operation(&self, operation: &'static str) {
|
||||
self.record_outgoing_request();
|
||||
counter!(INTERNODE_OPERATION_REQUESTS_OUTGOING_TOTAL, OPERATION_LABEL => operation).increment(1);
|
||||
}
|
||||
|
||||
pub fn record_incoming_request(&self) {
|
||||
self.incoming_requests_total.fetch_add(1, Ordering::Relaxed);
|
||||
counter!("rustfs_system_network_internode_requests_incoming_total").increment(1);
|
||||
}
|
||||
|
||||
pub fn record_incoming_request_for_operation(&self, operation: &'static str) {
|
||||
self.record_incoming_request();
|
||||
counter!(INTERNODE_OPERATION_REQUESTS_INCOMING_TOTAL, OPERATION_LABEL => operation).increment(1);
|
||||
}
|
||||
|
||||
pub fn record_error(&self) {
|
||||
self.errors_total.fetch_add(1, Ordering::Relaxed);
|
||||
counter!("rustfs_system_network_internode_errors_total").increment(1);
|
||||
}
|
||||
|
||||
pub fn record_error_for_operation(&self, operation: &'static str) {
|
||||
self.record_error();
|
||||
counter!(INTERNODE_OPERATION_ERRORS_TOTAL, OPERATION_LABEL => operation).increment(1);
|
||||
}
|
||||
|
||||
pub fn record_dial_result(&self, duration: Duration, success: bool) {
|
||||
let elapsed_nanos = duration.as_nanos().min(u128::from(u64::MAX)) as u64;
|
||||
self.dial_total_time_nanos.fetch_add(elapsed_nanos, Ordering::Relaxed);
|
||||
let samples = self.dial_samples_total.fetch_add(1, Ordering::Relaxed) + 1;
|
||||
let total = self.dial_total_time_nanos.load(Ordering::Relaxed);
|
||||
gauge!("rustfs_system_network_internode_dial_avg_time_nanos").set(total as f64 / samples as f64);
|
||||
|
||||
if !success {
|
||||
self.dial_errors_total.fetch_add(1, Ordering::Relaxed);
|
||||
counter!("rustfs_system_network_internode_dial_errors_total").increment(1);
|
||||
}
|
||||
|
||||
let now_ms = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_millis()
|
||||
.min(u128::from(u64::MAX)) as u64;
|
||||
self.last_dial_unix_millis.store(now_ms, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub fn snapshot(&self) -> InternodeMetricsSnapshot {
|
||||
let dial_samples_total = self.dial_samples_total.load(Ordering::Relaxed);
|
||||
let dial_total_time_nanos = self.dial_total_time_nanos.load(Ordering::Relaxed);
|
||||
let dial_avg_time_nanos = dial_total_time_nanos.checked_div(dial_samples_total).unwrap_or(0);
|
||||
|
||||
InternodeMetricsSnapshot {
|
||||
sent_bytes_total: self.sent_bytes_total.load(Ordering::Relaxed),
|
||||
recv_bytes_total: self.recv_bytes_total.load(Ordering::Relaxed),
|
||||
outgoing_requests_total: self.outgoing_requests_total.load(Ordering::Relaxed),
|
||||
incoming_requests_total: self.incoming_requests_total.load(Ordering::Relaxed),
|
||||
errors_total: self.errors_total.load(Ordering::Relaxed),
|
||||
dial_errors_total: self.dial_errors_total.load(Ordering::Relaxed),
|
||||
dial_avg_time_nanos,
|
||||
last_dial_unix_millis: self.last_dial_unix_millis.load(Ordering::Relaxed),
|
||||
}
|
||||
}
|
||||
|
||||
#[doc(hidden)]
|
||||
pub fn reset_for_test(&self) {
|
||||
self.sent_bytes_total.store(0, Ordering::Relaxed);
|
||||
self.recv_bytes_total.store(0, Ordering::Relaxed);
|
||||
self.outgoing_requests_total.store(0, Ordering::Relaxed);
|
||||
self.incoming_requests_total.store(0, Ordering::Relaxed);
|
||||
self.errors_total.store(0, Ordering::Relaxed);
|
||||
self.dial_errors_total.store(0, Ordering::Relaxed);
|
||||
self.dial_total_time_nanos.store(0, Ordering::Relaxed);
|
||||
self.dial_samples_total.store(0, Ordering::Relaxed);
|
||||
self.last_dial_unix_millis.store(0, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn global_internode_metrics() -> &'static Arc<InternodeMetrics> {
|
||||
static GLOBAL_INTERNODE_METRICS: LazyLock<Arc<InternodeMetrics>> = LazyLock::new(|| Arc::new(InternodeMetrics::default()));
|
||||
&GLOBAL_INTERNODE_METRICS
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn snapshot_reports_recorded_values() {
|
||||
let metrics = global_internode_metrics();
|
||||
metrics.reset_for_test();
|
||||
|
||||
metrics.record_sent_bytes(64);
|
||||
metrics.record_recv_bytes(32);
|
||||
metrics.record_outgoing_request();
|
||||
metrics.record_incoming_request();
|
||||
metrics.record_error();
|
||||
metrics.record_dial_result(Duration::from_millis(9), true);
|
||||
metrics.record_dial_result(Duration::from_millis(3), false);
|
||||
|
||||
let snapshot = metrics.snapshot();
|
||||
assert_eq!(snapshot.sent_bytes_total, 64);
|
||||
assert_eq!(snapshot.recv_bytes_total, 32);
|
||||
assert_eq!(snapshot.outgoing_requests_total, 1);
|
||||
assert_eq!(snapshot.incoming_requests_total, 1);
|
||||
assert_eq!(snapshot.errors_total, 1);
|
||||
assert_eq!(snapshot.dial_errors_total, 1);
|
||||
assert_eq!(snapshot.dial_avg_time_nanos, 6_000_000);
|
||||
assert!(snapshot.last_dial_unix_millis > 0);
|
||||
|
||||
metrics.reset_for_test();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn operation_metrics_also_update_aggregate_snapshot() {
|
||||
let metrics = InternodeMetrics::default();
|
||||
|
||||
metrics.record_sent_bytes_for_operation(INTERNODE_OPERATION_READ_FILE_STREAM, 128);
|
||||
metrics.record_recv_bytes_for_operation(INTERNODE_OPERATION_PUT_FILE_STREAM, 256);
|
||||
metrics.record_outgoing_request_for_operation(INTERNODE_OPERATION_GRPC_WRITE_ALL);
|
||||
metrics.record_incoming_request_for_operation(INTERNODE_OPERATION_GRPC_READ_ALL);
|
||||
metrics.record_error_for_operation(INTERNODE_OPERATION_WALK_DIR);
|
||||
|
||||
let snapshot = metrics.snapshot();
|
||||
assert_eq!(snapshot.sent_bytes_total, 128);
|
||||
assert_eq!(snapshot.recv_bytes_total, 256);
|
||||
assert_eq!(snapshot.outgoing_requests_total, 1);
|
||||
assert_eq!(snapshot.incoming_requests_total, 1);
|
||||
assert_eq!(snapshot.errors_total, 1);
|
||||
}
|
||||
}
|
||||
@@ -13,12 +13,9 @@
|
||||
// limitations under the License.
|
||||
|
||||
pub mod bucket_stats;
|
||||
pub mod capacity_scope;
|
||||
// pub mod error;
|
||||
pub mod data_usage;
|
||||
pub mod globals;
|
||||
pub mod heal_channel;
|
||||
pub mod internode_metrics;
|
||||
pub mod last_minute;
|
||||
pub mod metrics;
|
||||
mod readiness;
|
||||
|
||||
@@ -14,7 +14,6 @@
|
||||
|
||||
use crate::last_minute::{AccElem, LastMinuteLatency};
|
||||
use chrono::{DateTime, Utc};
|
||||
use rustfs_madmin::metrics::{ScannerMetrics as M_ScannerMetrics, TimedAction};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
@@ -351,6 +350,32 @@ pub struct CurrentCycle {
|
||||
pub started: DateTime<Utc>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct ScannerTimedAction {
|
||||
pub count: u64,
|
||||
pub acc_time: u64,
|
||||
pub bytes: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
|
||||
pub struct ScannerLastMinute {
|
||||
pub actions: HashMap<String, ScannerTimedAction>,
|
||||
pub ilm: HashMap<String, ScannerTimedAction>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
|
||||
pub struct ScannerMetricsReport {
|
||||
pub collected_at: DateTime<Utc>,
|
||||
pub current_cycle: u64,
|
||||
pub current_started: DateTime<Utc>,
|
||||
pub cycles_completed_at: Vec<DateTime<Utc>>,
|
||||
pub ongoing_buckets: usize,
|
||||
pub life_time_ops: HashMap<String, u64>,
|
||||
pub life_time_ilm: HashMap<String, u64>,
|
||||
pub last_minute: ScannerLastMinute,
|
||||
pub active_paths: Vec<String>,
|
||||
}
|
||||
|
||||
impl CurrentCycle {
|
||||
pub fn unmarshal(&mut self, buf: &[u8]) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
*self = rmp_serde::from_slice(buf)?;
|
||||
@@ -572,8 +597,8 @@ impl Metrics {
|
||||
}
|
||||
|
||||
/// Build a full metrics report snapshot.
|
||||
pub async fn report(&self) -> M_ScannerMetrics {
|
||||
let mut m = M_ScannerMetrics::default();
|
||||
pub async fn report(&self) -> ScannerMetricsReport {
|
||||
let mut m = ScannerMetricsReport::default();
|
||||
|
||||
if let Some(cycle) = self.get_cycle().await {
|
||||
m.current_cycle = cycle.current;
|
||||
@@ -606,7 +631,7 @@ impl Metrics {
|
||||
{
|
||||
m.last_minute.actions.insert(
|
||||
metric.as_str().to_string(),
|
||||
TimedAction {
|
||||
ScannerTimedAction {
|
||||
count: last_min.n,
|
||||
acc_time: last_min.total,
|
||||
bytes: last_min.size,
|
||||
@@ -633,7 +658,7 @@ impl Metrics {
|
||||
{
|
||||
m.last_minute.ilm.insert(
|
||||
action.as_str().to_string(),
|
||||
TimedAction {
|
||||
ScannerTimedAction {
|
||||
count: last_min.n,
|
||||
acc_time: last_min.total,
|
||||
bytes: last_min.size,
|
||||
|
||||
Reference in New Issue
Block a user