// Copyright 2024 RustFS Team // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. use std::sync::atomic::{AtomicU64, Ordering}; #[cfg(target_os = "linux")] use std::fs; #[cfg(any( target_os = "windows", target_os = "macos", target_os = "freebsd", target_os = "openbsd", target_os = "netbsd", target_os = "dragonfly" ))] use std::process::Command; static READ_LOCKS_HELD: AtomicU64 = AtomicU64::new(0); static WRITE_LOCKS_HELD: AtomicU64 = AtomicU64::new(0); /// Monotonic count of expired lock guards forcibly reclaimed by a contender (#899 observability). static LOCKS_RECLAIMED: AtomicU64 = AtomicU64::new(0); /// Monotonic count of guard heartbeats that observed a refresh-quorum loss (#899 observability). static LOCK_REFRESH_QUORUM_LOST: AtomicU64 = AtomicU64::new(0); #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] pub struct ProcessLockSnapshot { pub read_locks_held: u64, pub write_locks_held: u64, } /// Monotonic lock lifecycle counters (distinct from the held gauges above). #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] pub struct ProcessLockEventSnapshot { pub locks_reclaimed: u64, pub lock_refresh_quorum_lost: u64, } #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] pub struct ProcessPlatformSnapshot { pub io_rchar_bytes: Option, pub io_read_bytes: Option, pub io_wchar_bytes: Option, pub io_write_bytes: Option, pub syscall_read_total: Option, pub syscall_write_total: Option, pub virtual_memory_max_bytes: Option, } #[inline(always)] pub fn record_read_lock_held_acquire() { READ_LOCKS_HELD.fetch_add(1, Ordering::Relaxed); } #[inline(always)] pub fn record_write_lock_held_acquire() { WRITE_LOCKS_HELD.fetch_add(1, Ordering::Relaxed); } #[inline(always)] pub fn record_read_lock_held_release() { let _ = READ_LOCKS_HELD.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |value| Some(value.saturating_sub(1))); } #[inline(always)] pub fn record_write_lock_held_release() { let _ = WRITE_LOCKS_HELD.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |value| Some(value.saturating_sub(1))); } #[inline(always)] pub fn record_lock_reclaimed() { LOCKS_RECLAIMED.fetch_add(1, Ordering::Relaxed); } #[inline(always)] pub fn record_lock_refresh_quorum_lost() { LOCK_REFRESH_QUORUM_LOST.fetch_add(1, Ordering::Relaxed); } #[inline(always)] pub fn snapshot_process_lock_counts() -> ProcessLockSnapshot { ProcessLockSnapshot { read_locks_held: READ_LOCKS_HELD.load(Ordering::Relaxed), write_locks_held: WRITE_LOCKS_HELD.load(Ordering::Relaxed), } } #[inline(always)] pub fn snapshot_process_lock_events() -> ProcessLockEventSnapshot { ProcessLockEventSnapshot { locks_reclaimed: LOCKS_RECLAIMED.load(Ordering::Relaxed), lock_refresh_quorum_lost: LOCK_REFRESH_QUORUM_LOST.load(Ordering::Relaxed), } } #[inline] pub fn snapshot_process_platform_stats() -> ProcessPlatformSnapshot { platform::snapshot() } #[cfg(any( target_os = "windows", target_os = "macos", target_os = "freebsd", target_os = "openbsd", target_os = "netbsd", target_os = "dragonfly" ))] fn run_command(command: &str, args: &[&str]) -> Option { let output = Command::new(command).args(args).output().ok()?; if !output.status.success() { return None; } let stdout = String::from_utf8_lossy(&output.stdout).trim().to_string(); if stdout.is_empty() { None } else { Some(stdout) } } #[cfg(any(target_os = "windows", test))] fn parse_kv_u64(content: &str, key: &str) -> Option { for line in content.lines() { let Some((k, v)) = line.split_once(':') else { continue; }; if k.trim().eq_ignore_ascii_case(key) { return v.trim().parse::().ok(); } } None } #[cfg(target_os = "linux")] mod platform { use super::*; pub(super) fn snapshot() -> ProcessPlatformSnapshot { let io = fs::read_to_string("/proc/self/io").ok(); let status = fs::read_to_string("/proc/self/status").ok(); let io_stats = io.as_deref().map(parse_proc_self_io).unwrap_or_default(); ProcessPlatformSnapshot { io_rchar_bytes: io_stats.rchar_bytes, io_read_bytes: io_stats.read_bytes, io_wchar_bytes: io_stats.wchar_bytes, io_write_bytes: io_stats.write_bytes, syscall_read_total: io_stats.syscall_read_total, syscall_write_total: io_stats.syscall_write_total, virtual_memory_max_bytes: status.as_deref().and_then(parse_vm_peak_bytes), } } #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] struct ProcSelfIoStats { rchar_bytes: Option, read_bytes: Option, syscall_read_total: Option, syscall_write_total: Option, wchar_bytes: Option, write_bytes: Option, } fn parse_proc_self_io(content: &str) -> ProcSelfIoStats { let mut stats = ProcSelfIoStats::default(); for line in content.lines() { let Some((key, value)) = line.split_once(':') else { continue; }; let Ok(value) = value.trim().parse::() else { continue; }; match key.trim() { "rchar" => stats.rchar_bytes = Some(value), "read_bytes" => stats.read_bytes = Some(value), "wchar" => stats.wchar_bytes = Some(value), "write_bytes" => stats.write_bytes = Some(value), "syscr" => stats.syscall_read_total = Some(value), "syscw" => stats.syscall_write_total = Some(value), _ => {} } } stats } fn parse_vm_peak_bytes(content: &str) -> Option { for line in content.lines() { let Some(rest) = line.strip_prefix("VmPeak:") else { continue; }; let mut parts = rest.split_whitespace(); let value = parts.next()?.parse::().ok()?; let unit = parts.next().unwrap_or_default(); return Some(match unit { "kB" | "KB" | "kb" => value.saturating_mul(1024), _ => value, }); } None } #[cfg(test)] mod tests { use super::*; #[test] fn parse_vm_peak() { let status = "Name:\trustfs\nVmPeak:\t 2048 kB\nVmRSS:\t 1024 kB\n"; assert_eq!(parse_vm_peak_bytes(status), Some(2048 * 1024)); } #[test] fn parse_proc_self_io_extracts_expected_fields() { let stats = parse_proc_self_io( "rchar: 11\nwchar: 22\nsyscr: 33\nsyscw: 44\nread_bytes: 55\nwrite_bytes: 66\ncancelled_write_bytes: 77\n", ); assert_eq!(stats.rchar_bytes, Some(11)); assert_eq!(stats.wchar_bytes, Some(22)); assert_eq!(stats.syscall_read_total, Some(33)); assert_eq!(stats.syscall_write_total, Some(44)); assert_eq!(stats.read_bytes, Some(55)); assert_eq!(stats.write_bytes, Some(66)); } } } #[cfg(target_os = "windows")] mod platform { use super::*; pub(super) fn snapshot() -> ProcessPlatformSnapshot { let pid = std::process::id(); let script = format!( "Get-CimInstance Win32_Process -Filter \"ProcessId = {pid}\" | \ Format-List -Property ReadOperationCount,WriteOperationCount,PeakVirtualSize" ); let output = run_command("powershell", &["-NoProfile", "-Command", &script]); ProcessPlatformSnapshot { syscall_read_total: output.as_deref().and_then(|v| parse_kv_u64(v, "ReadOperationCount")), syscall_write_total: output.as_deref().and_then(|v| parse_kv_u64(v, "WriteOperationCount")), virtual_memory_max_bytes: output.as_deref().and_then(|v| parse_kv_u64(v, "PeakVirtualSize")), ..Default::default() } } } #[cfg(target_os = "macos")] mod platform { use super::*; pub(super) fn snapshot() -> ProcessPlatformSnapshot { let pid = std::process::id().to_string(); let output = run_command("ps", &["-o", "inblock=", "-o", "oublock=", "-o", "vsz=", "-p", &pid]); parse_ps_stats(output.as_deref()) } fn parse_ps_stats(output: Option<&str>) -> ProcessPlatformSnapshot { let Some(output) = output else { return ProcessPlatformSnapshot::default(); }; let Some(line) = output.lines().find(|line| !line.trim().is_empty()) else { return ProcessPlatformSnapshot::default(); }; let mut parts = line.split_whitespace(); let inblock = parts.next().and_then(|v| v.parse::().ok()); let oublock = parts.next().and_then(|v| v.parse::().ok()); let vsz_kb = parts.next().and_then(|v| v.parse::().ok()); ProcessPlatformSnapshot { syscall_read_total: inblock, syscall_write_total: oublock, virtual_memory_max_bytes: vsz_kb.map(|v| v.saturating_mul(1024)), ..Default::default() } } #[cfg(test)] mod tests { use super::*; #[test] fn parse_ps_line() { let snapshot = parse_ps_stats(Some("12 34 5678")); assert_eq!(snapshot.syscall_read_total, Some(12)); assert_eq!(snapshot.syscall_write_total, Some(34)); assert_eq!(snapshot.virtual_memory_max_bytes, Some(5678 * 1024)); } } } #[cfg(any(target_os = "freebsd", target_os = "openbsd", target_os = "netbsd", target_os = "dragonfly"))] mod platform { use super::*; pub(super) fn snapshot() -> ProcessPlatformSnapshot { let pid = std::process::id().to_string(); let output = run_command("ps", &["-o", "inblk=", "-o", "oublk=", "-o", "vsz=", "-p", &pid]); parse_ps_stats(output.as_deref()) } fn parse_ps_stats(output: Option<&str>) -> ProcessPlatformSnapshot { let Some(output) = output else { return ProcessPlatformSnapshot::default(); }; let Some(line) = output.lines().find(|line| !line.trim().is_empty()) else { return ProcessPlatformSnapshot::default(); }; let mut parts = line.split_whitespace(); let inblock = parts.next().and_then(|v| v.parse::().ok()); let oublock = parts.next().and_then(|v| v.parse::().ok()); let vsz_kb = parts.next().and_then(|v| v.parse::().ok()); ProcessPlatformSnapshot { syscall_read_total: inblock, syscall_write_total: oublock, virtual_memory_max_bytes: vsz_kb.map(|v| v.saturating_mul(1024)), ..Default::default() } } } #[cfg(not(any( target_os = "linux", target_os = "windows", target_os = "macos", target_os = "freebsd", target_os = "openbsd", target_os = "netbsd", target_os = "dragonfly" )))] mod platform { use super::*; pub(super) fn snapshot() -> ProcessPlatformSnapshot { ProcessPlatformSnapshot::default() } } #[cfg(test)] pub fn reset_process_lock_counts() { READ_LOCKS_HELD.store(0, Ordering::Relaxed); WRITE_LOCKS_HELD.store(0, Ordering::Relaxed); } #[cfg(test)] mod tests { use super::*; #[test] fn held_lock_counters_round_trip() { reset_process_lock_counts(); record_read_lock_held_acquire(); record_read_lock_held_acquire(); record_write_lock_held_acquire(); assert_eq!( snapshot_process_lock_counts(), ProcessLockSnapshot { read_locks_held: 2, write_locks_held: 1, } ); record_read_lock_held_release(); record_write_lock_held_release(); record_write_lock_held_release(); assert_eq!( snapshot_process_lock_counts(), ProcessLockSnapshot { read_locks_held: 1, write_locks_held: 0, } ); } #[test] fn parse_kv_u64_round_trip() { let content = "syscr: 123\nsyscw: 456\n"; assert_eq!(parse_kv_u64(content, "syscr"), Some(123)); assert_eq!(parse_kv_u64(content, "syscw"), Some(456)); assert_eq!(parse_kv_u64(content, "missing"), None); } }