diff --git a/Cargo.lock b/Cargo.lock index eb2496278..c89af78a1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9175,10 +9175,12 @@ version = "1.0.0-beta.12" dependencies = [ "chrono", "hotpath", + "jiff", "metrics", "rmp-serde", "s3s", "serde", + "serde_json", "tokio", "tonic", "tracing", @@ -9294,6 +9296,7 @@ dependencies = [ "hyper-rustls", "hyper-util", "insta", + "jiff", "lazy_static", "libc", "md-5 0.11.0", @@ -9719,10 +9722,10 @@ dependencies = [ name = "rustfs-madmin" version = "1.0.0-beta.12" dependencies = [ - "chrono", "hotpath", "humantime", "hyper", + "jiff", "rmp-serde", "serde", "serde_json", diff --git a/crates/common/Cargo.toml b/crates/common/Cargo.toml index df98511d6..5c9094d61 100644 --- a/crates/common/Cargo.toml +++ b/crates/common/Cargo.toml @@ -39,11 +39,15 @@ tokio = { workspace = true, features = ["fs", "rt-multi-thread"] } tonic = { workspace = true, features = ["gzip", "deflate"] } uuid = { workspace = true, features = ["v4", "fast-rng", "macro-diagnostics"] } chrono = { workspace = true, features = ["serde"] } +jiff = { workspace = true, features = ["serde"] } metrics = { workspace = true } serde = { workspace = true, features = ["derive"] } rmp-serde = { workspace = true } s3s = { workspace = true, features = ["minio"] } tracing = { workspace = true } +[dev-dependencies] +serde_json = { workspace = true } + [lib] doctest = false diff --git a/crates/common/src/metrics.rs b/crates/common/src/metrics.rs index 68f0bfde5..cb524b371 100644 --- a/crates/common/src/metrics.rs +++ b/crates/common/src/metrics.rs @@ -15,6 +15,7 @@ use crate::heal_channel::HealScanMode; use crate::last_minute::{AccElem, LastMinuteLatency}; use chrono::{DateTime, Utc}; +use jiff::Timestamp; use serde::{Deserialize, Serialize}; use std::{ collections::{BTreeSet, HashMap}, @@ -669,7 +670,7 @@ impl LockedLastMinuteLatency { #[derive(Clone, Debug)] struct CurrentPathState { path: String, - updated_at: DateTime, + updated_at: Timestamp, } struct CurrentPathTracker { @@ -678,10 +679,10 @@ struct CurrentPathTracker { impl CurrentPathTracker { fn new(initial_path: String) -> Self { - Self::new_at(initial_path, Utc::now()) + Self::new_at(initial_path, Timestamp::now()) } - fn new_at(initial_path: String, updated_at: DateTime) -> Self { + fn new_at(initial_path: String, updated_at: Timestamp) -> Self { Self { state: Arc::new(RwLock::new(CurrentPathState { path: initial_path, @@ -693,7 +694,7 @@ impl CurrentPathTracker { async fn update_path(&self, path: String) { let mut state = self.state.write().await; state.path = path; - state.updated_at = Utc::now(); + state.updated_at = Timestamp::now(); } async fn get_state(&self) -> CurrentPathState { @@ -701,6 +702,36 @@ impl CurrentPathTracker { } } +fn chrono_to_jiff_timestamp(dt: DateTime) -> Timestamp { + let seconds = dt.timestamp(); + let nanoseconds = match i32::try_from(dt.timestamp_subsec_nanos()) { + Ok(nanoseconds) => nanoseconds, + Err(_) => { + return if seconds < 0 { Timestamp::MIN } else { Timestamp::MAX }; + } + }; + + match Timestamp::new(seconds, nanoseconds) { + Ok(timestamp) => timestamp, + Err(_) => { + if seconds < 0 { + Timestamp::MIN + } else { + Timestamp::MAX + } + } + } +} + +fn timestamp_elapsed_seconds_since(now: Timestamp, earlier: Timestamp) -> u64 { + let duration = now.duration_since(earlier); + if duration.is_negative() { + return 0; + } + + u64::try_from(duration.as_secs()).map_or(u64::MAX, |seconds| seconds) +} + #[derive(Clone, Copy, Debug, Default)] struct ScannerDiskBucketScanState { concurrency_limit: u64, @@ -1166,12 +1197,12 @@ pub struct ScannerLastMinute { #[derive(Clone, Debug, Default, Serialize, Deserialize)] pub struct ScannerMetricsReport { - pub collected_at: DateTime, + pub collected_at: Timestamp, pub current_cycle: u64, #[serde(default)] pub current_cycle_active: bool, - pub current_started: DateTime, - pub cycles_completed_at: Vec>, + pub current_started: Timestamp, + pub cycles_completed_at: Vec, pub ongoing_buckets: usize, #[serde(default)] pub active_scan_paths: usize, @@ -2988,8 +3019,8 @@ impl Metrics { let cycle = self.cycle_info.read().await; let has_cycle = if let Some(cycle) = cycle.as_ref() { m.current_cycle = cycle.current; - m.cycles_completed_at = cycle.cycle_completed.clone(); - m.current_started = cycle.started; + m.cycles_completed_at = cycle.cycle_completed.iter().copied().map(chrono_to_jiff_timestamp).collect(); + m.current_started = chrono_to_jiff_timestamp(cycle.started); true } else { false @@ -3024,15 +3055,15 @@ impl Metrics { }; if !has_cycle && let Some(init_time) = crate::get_global_init_time().await { - m.current_started = init_time; + m.current_started = chrono_to_jiff_timestamp(init_time); } - m.collected_at = Utc::now(); + m.collected_at = Timestamp::now(); let current_path_snapshots = self.current_path_snapshots().await; m.active_scan_paths = current_path_snapshots.len(); m.oldest_active_path_age_seconds = current_path_snapshots .iter() - .map(|(_, state)| m.collected_at.signed_duration_since(state.updated_at).num_seconds().max(0) as u64) + .map(|(_, state)| timestamp_elapsed_seconds_since(m.collected_at, state.updated_at)) .max() .unwrap_or_default(); m.active_paths = current_path_snapshots @@ -3308,6 +3339,22 @@ impl Drop for CloseDiskGuard { mod tests { use super::*; + #[test] + fn scanner_metrics_report_timestamps_serialize_as_rfc3339_utc() { + let report = ScannerMetricsReport { + collected_at: Timestamp::constant(1_700_000_000, 123_456_000), + current_started: Timestamp::constant(1_699_999_940, 0), + cycles_completed_at: vec![Timestamp::constant(1_700_000_060, 987_654_000)], + ..Default::default() + }; + + let value = serde_json::to_value(&report).expect("scanner metrics report should serialize"); + + assert_eq!(value["collected_at"].as_str(), Some("2023-11-14T22:13:20.123456Z")); + assert_eq!(value["current_started"].as_str(), Some("2023-11-14T22:12:20Z")); + assert_eq!(value["cycles_completed_at"][0].as_str(), Some("2023-11-14T22:14:20.987654Z")); + } + #[tokio::test] async fn close_disk_guard_runs_cleanup_when_an_early_return_drops_it() { let (closed_tx, closed_rx) = tokio::sync::oneshot::channel(); @@ -3366,7 +3413,7 @@ mod tests { #[tokio::test] async fn report_counts_active_scan_paths() { let metrics = Metrics::new(); - let updated_at = Utc::now() - chrono::Duration::seconds(12); + let updated_at = Timestamp::now() - jiff::SignedDuration::from_secs(12); metrics.current_paths.write().await.insert( "disk-a".to_string(), Arc::new(CurrentPathTracker::new_at("bucket-a".to_string(), updated_at)), @@ -3388,7 +3435,7 @@ mod tests { let metrics = Metrics::new(); let tracker = Arc::new(CurrentPathTracker::new_at( "bucket-a".to_string(), - Utc::now() - chrono::Duration::hours(1), + Timestamp::now() - jiff::SignedDuration::from_secs(60 * 60), )); metrics .current_paths @@ -4161,7 +4208,7 @@ mod tests { let report = metrics.report().await; *crate::globals::GLOBAL_INIT_TIME.write().await = previous_init_time; - assert_eq!(report.current_started, cycle_started); + assert_eq!(report.current_started, chrono_to_jiff_timestamp(cycle_started)); } #[tokio::test] @@ -4584,7 +4631,7 @@ mod tests { let active = metrics.report().await; assert!(active.current_cycle_active); assert_eq!(active.current_cycle, 12); - assert_eq!(active.current_started, cycle_started); + assert_eq!(active.current_started, chrono_to_jiff_timestamp(cycle_started)); let idle_cycle = CurrentCycle { current: 0, diff --git a/crates/ecstore/Cargo.toml b/crates/ecstore/Cargo.toml index 84ad18647..83499c027 100644 --- a/crates/ecstore/Cargo.toml +++ b/crates/ecstore/Cargo.toml @@ -149,6 +149,7 @@ async-trait.workspace = true bytes = { workspace = true, features = ["serde"] } byteorder = { workspace = true } chrono = { workspace = true, features = ["serde"] } +jiff = { workspace = true, features = ["serde"] } glob = { workspace = true } thiserror.workspace = true flatbuffers.workspace = true diff --git a/crates/ecstore/src/bucket/target/bucket_target.rs b/crates/ecstore/src/bucket/target/bucket_target.rs index cc96b7c5b..d55d86a2f 100644 --- a/crates/ecstore/src/bucket/target/bucket_target.rs +++ b/crates/ecstore/src/bucket/target/bucket_target.rs @@ -13,6 +13,7 @@ // limitations under the License. use crate::error::{Error, Result}; +use jiff::Timestamp; use rmp_serde::Serializer as rmpSerializer; use serde::{Deserialize, Serialize}; use std::{ @@ -32,7 +33,7 @@ pub struct Credentials { #[serde(rename = "secretKey")] pub secret_key: String, pub session_token: Option, - pub expiration: Option>, + pub expiration: Option, } impl Credentials { @@ -408,7 +409,11 @@ mod tests { assert_eq!(credentials.access_key, "test-access-key"); assert_eq!(credentials.secret_key, "test-secret-key"); assert_eq!(credentials.session_token, Some("test-session-token".to_string())); - assert!(credentials.expiration.is_some()); + assert_eq!( + serde_json::to_value(credentials.expiration.expect("expiration should parse")) + .expect("expiration should serialize to JSON"), + serde_json::json!("2024-12-31T23:59:59Z") + ); // Verify latency statistics assert_eq!(target.latency.curr, Duration::from_millis(100)); @@ -562,12 +567,15 @@ mod tests { .and_then(|credentials| credentials.session_token.as_deref()), Some("legacy-session-token") ); - assert!( + assert_eq!( target .credentials .as_ref() .and_then(|credentials| credentials.expiration) - .is_some() + .map(serde_json::to_value) + .transpose() + .expect("expiration should serialize to JSON"), + Some(serde_json::json!("2024-12-31T23:59:59Z")) ); } @@ -609,7 +617,11 @@ mod tests { credentials.session_token, Some("AQoEXAMPLEH4aoAH0gNCAPyJxz4BlCFFxWNE1OPTgk5TthT".to_string()) ); - assert!(credentials.expiration.is_some()); + assert_eq!( + serde_json::to_value(credentials.expiration.expect("expiration should parse")) + .expect("expiration should serialize to JSON"), + serde_json::json!("2024-12-31T23:59:59Z") + ); } #[test] diff --git a/crates/ecstore/src/services/metrics_realtime.rs b/crates/ecstore/src/services/metrics_realtime.rs index f99e018c3..e4fc919e6 100644 --- a/crates/ecstore/src/services/metrics_realtime.rs +++ b/crates/ecstore/src/services/metrics_realtime.rs @@ -15,7 +15,9 @@ use crate::diagnostics::admin_server_info::get_local_server_property; use crate::runtime::sources as runtime_sources; use crate::storage_api_contracts::admin::StorageAdminApi; +#[cfg(test)] use chrono::Utc; +use jiff::Timestamp; use rustfs_common::{heal_channel::DriveState, metrics::global_metrics}; use rustfs_io_metrics::internode_metrics::global_internode_metrics; use rustfs_madmin::metrics::{ @@ -67,6 +69,18 @@ impl MetricType { } } +fn unix_millis_to_jiff_timestamp(millis: u64, fallback: Timestamp) -> Timestamp { + let millis = match i64::try_from(millis) { + Ok(millis) => millis, + Err(_) => return fallback, + }; + + match Timestamp::from_millisecond(millis) { + Ok(timestamp) => timestamp, + Err(_) => fallback, + } +} + fn to_madmin_scanner_metrics(metrics: rustfs_common::metrics::ScannerMetricsReport) -> MadminScannerMetrics { MadminScannerMetrics { collected_at: metrics.collected_at, @@ -386,7 +400,7 @@ pub async fn collect_local_metrics(types: MetricType, opts: &CollectMetricsOpts) if types.contains(&MetricType::DISK) { debug!("start get disk metrics"); let mut aggr = DiskMetric { - collected_at: Utc::now(), + collected_at: Timestamp::now(), ..Default::default() }; for (name, disk) in collect_local_disks_metrics(&opts.disks).await.into_iter() { @@ -412,7 +426,7 @@ pub async fn collect_local_metrics(types: MetricType, opts: &CollectMetricsOpts) if types.contains(&MetricType::NET) { let snapshot = global_internode_metrics().snapshot(); real_time_metrics.aggregated.net = Some(NetMetrics { - collected_at: Utc::now(), + collected_at: Timestamp::now(), interface_name: "internode".to_string(), net_stats: NetDevLine { name: "internode".to_string(), @@ -428,10 +442,9 @@ pub async fn collect_local_metrics(types: MetricType, opts: &CollectMetricsOpts) // if types.contains(&MetricType::CPU) {} if types.contains(&MetricType::RPC) { - let collected_at = Utc::now(); + let collected_at = Timestamp::now(); let snapshot = global_internode_metrics().snapshot(); - let last_connect_time = - chrono::DateTime::::from_timestamp_millis(snapshot.last_dial_unix_millis as i64).unwrap_or(collected_at); + let last_connect_time = unix_millis_to_jiff_timestamp(snapshot.last_dial_unix_millis, collected_at); real_time_metrics.aggregated.rpc = Some(RPCMetrics { collected_at, @@ -543,6 +556,10 @@ mod test { use serial_test::serial; use std::time::Duration; + fn chrono_to_jiff_timestamp(timestamp: chrono::DateTime) -> jiff::Timestamp { + jiff::Timestamp::try_from(std::time::SystemTime::from(timestamp)).expect("test timestamp should fit in jiff") + } + #[test] fn tes_types() { let t = MetricType::ALL; @@ -591,7 +608,7 @@ mod test { let current_started = Utc::now() - chrono::Duration::seconds(5); let scanner = to_madmin_scanner_metrics(rustfs_common::metrics::ScannerMetricsReport { current_cycle_active: true, - current_started, + current_started: chrono_to_jiff_timestamp(current_started), last_cycle_partial_source: "usage".to_string(), last_cycle_partial_source_code: 1, partial_cycles_by_source: vec![rustfs_common::metrics::ScannerSourceCycleSnapshot { @@ -602,7 +619,7 @@ mod test { }); assert_eq!(scanner.current_cycle_active, Some(true)); - assert_eq!(scanner.current_started, current_started); + assert_eq!(scanner.current_started, chrono_to_jiff_timestamp(current_started)); assert_eq!(scanner.last_cycle_partial_source, "usage"); assert_eq!(scanner.last_cycle_partial_source_code, 1); let usage = scanner @@ -643,7 +660,7 @@ mod test { aggregated.merge(decoded); let scanner = aggregated.aggregated.scanner.expect("scanner metrics"); assert_eq!(scanner.current_cycle_active, Some(true)); - assert_eq!(scanner.current_started, cycle_started); + assert_eq!(scanner.current_started, chrono_to_jiff_timestamp(cycle_started)); } #[test] diff --git a/crates/madmin/Cargo.toml b/crates/madmin/Cargo.toml index 2d704d2d4..6a1fe0ff5 100644 --- a/crates/madmin/Cargo.toml +++ b/crates/madmin/Cargo.toml @@ -36,9 +36,9 @@ hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"] [dependencies] hotpath.workspace = true -chrono = { workspace = true, features = ["serde"] } humantime.workspace = true hyper = { workspace = true, features = ["http2", "http1", "server"] } +jiff = { workspace = true, features = ["serde"] } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true, features = ["raw_value"] } sysinfo.workspace = true diff --git a/crates/madmin/src/metrics.rs b/crates/madmin/src/metrics.rs index 05940d2a3..49e1e5ff2 100644 --- a/crates/madmin/src/metrics.rs +++ b/crates/madmin/src/metrics.rs @@ -14,7 +14,7 @@ use std::collections::HashMap; -use chrono::{DateTime, Utc}; +use jiff::Timestamp; use serde::{Deserialize, Serialize}; use crate::health::MemInfo; @@ -78,7 +78,7 @@ pub struct DiskIOStats { #[derive(Clone, Debug, Default, Serialize, Deserialize)] pub struct DiskMetric { #[serde(rename = "collected")] - pub collected_at: DateTime, + pub collected_at: Timestamp, #[serde(rename = "n_disks")] pub n_disks: usize, #[serde(rename = "offline")] @@ -542,15 +542,15 @@ impl ScannerLifecycleTransitionSnapshot { #[derive(Clone, Debug, Default, Serialize, Deserialize)] pub struct ScannerMetrics { #[serde(rename = "collected")] - pub collected_at: DateTime, + pub collected_at: Timestamp, #[serde(rename = "current_cycle")] pub current_cycle: u64, #[serde(rename = "current_cycle_active", default, skip_serializing_if = "Option::is_none")] pub current_cycle_active: Option, #[serde(rename = "current_started")] - pub current_started: DateTime, + pub current_started: Timestamp, #[serde(rename = "cycle_complete_times")] - pub cycles_completed_at: Vec>, + pub cycles_completed_at: Vec, #[serde(rename = "ongoing_buckets")] pub ongoing_buckets: usize, #[serde(rename = "active_scan_paths", default)] @@ -1011,7 +1011,7 @@ impl Metrics { #[derive(Clone, Debug, Default, Serialize, Deserialize)] pub struct RPCMetrics { #[serde(rename = "collectedAt")] - pub collected_at: DateTime, + pub collected_at: Timestamp, pub connected: i32, @@ -1041,7 +1041,7 @@ pub struct RPCMetrics { pub out_queue: i32, #[serde(rename = "lastPongTime")] - pub last_pong_time: DateTime, + pub last_pong_time: Timestamp, #[serde(rename = "lastPingMS")] pub last_ping_ms: f64, @@ -1050,7 +1050,7 @@ pub struct RPCMetrics { pub max_ping_dur_ms: f64, // Maximum across all merged entries. #[serde(rename = "lastConnectTime")] - pub last_connect_time: DateTime, + pub last_connect_time: Timestamp, #[serde(rename = "byDestination", skip_serializing_if = "Option::is_none")] pub by_destination: Option>, @@ -1125,7 +1125,7 @@ pub struct CPUMetrics {} #[derive(Clone, Debug, Default, Serialize, Deserialize)] pub struct NetMetrics { #[serde(rename = "collected")] - pub collected_at: DateTime, + pub collected_at: Timestamp, #[serde(rename = "interfaceName")] pub interface_name: String, #[serde(rename = "netstats")] @@ -1214,7 +1214,7 @@ pub struct NetDevLine { #[derive(Clone, Debug, Default, Serialize, Deserialize)] pub struct MemMetrics { #[serde(rename = "collected")] - pub collected_at: DateTime, + pub collected_at: Timestamp, #[serde(rename = "memInfo")] pub info: MemInfo, } @@ -1222,13 +1222,13 @@ pub struct MemMetrics { #[derive(Clone, Debug, Default, Serialize, Deserialize)] pub struct SiteResyncMetrics { #[serde(rename = "collected")] - pub collected_at: DateTime, + pub collected_at: Timestamp, #[serde(rename = "resyncStatus", skip_serializing_if = "Option::is_none")] pub resync_status: Option, #[serde(rename = "startTime")] - pub start_time: DateTime, + pub start_time: Timestamp, #[serde(rename = "lastUpdate")] - pub last_update: DateTime, + pub last_update: Timestamp, #[serde(rename = "numBuckets")] pub num_buckets: i64, #[serde(rename = "resyncID")] @@ -1262,7 +1262,7 @@ impl SiteResyncMetrics { #[derive(Clone, Debug, Default, Serialize, Deserialize)] pub struct BatchJobMetrics { #[serde(rename = "collected")] - pub collected_at: DateTime, + pub collected_at: Timestamp, #[serde(rename = "Jobs")] pub jobs: HashMap, } @@ -1290,9 +1290,9 @@ pub struct JobMetric { #[serde(rename = "jobType")] pub job_type: String, #[serde(rename = "startTime")] - pub start_time: DateTime, + pub start_time: Timestamp, #[serde(rename = "lastUpdate")] - pub last_update: DateTime, + pub last_update: Timestamp, #[serde(rename = "retryAttempts")] pub retry_attempts: i32, pub complete: bool, @@ -1385,7 +1385,7 @@ impl RealtimeMetrics { #[derive(Clone, Debug, Default, Serialize, Deserialize)] pub struct OsMetrics { #[serde(rename = "collected")] - pub collected_at: DateTime, + pub collected_at: Timestamp, #[serde(rename = "life_time_ops")] pub life_time_ops: HashMap, #[serde(rename = "last_minute")] @@ -1417,6 +1417,119 @@ pub struct Operations { #[cfg(test)] mod tests { use super::*; + use jiff::SignedDuration; + + fn fixed_timestamp() -> Timestamp { + Timestamp::constant(1_700_000_000, 123_456_000) + } + + #[test] + fn admin_metrics_timestamps_serialize_as_rfc3339_utc() { + let timestamp = fixed_timestamp(); + + let disk = serde_json::to_value(DiskMetric { + collected_at: timestamp, + ..Default::default() + }) + .expect("disk metrics should serialize"); + assert_eq!(disk["collected"], "2023-11-14T22:13:20.123456Z"); + let disk: DiskMetric = serde_json::from_value(disk).expect("disk metrics should deserialize"); + assert_eq!(disk.collected_at, timestamp); + + let scanner = serde_json::to_value(ScannerMetrics { + collected_at: timestamp, + current_started: timestamp, + cycles_completed_at: vec![timestamp], + ..Default::default() + }) + .expect("scanner metrics should serialize"); + assert_eq!(scanner["collected"], "2023-11-14T22:13:20.123456Z"); + assert_eq!(scanner["current_started"], "2023-11-14T22:13:20.123456Z"); + assert_eq!(scanner["cycle_complete_times"][0], "2023-11-14T22:13:20.123456Z"); + let scanner: ScannerMetrics = serde_json::from_value(scanner).expect("scanner metrics should deserialize"); + assert_eq!(scanner.collected_at, timestamp); + assert_eq!(scanner.current_started, timestamp); + assert_eq!(scanner.cycles_completed_at, vec![timestamp]); + + let rpc = serde_json::to_value(RPCMetrics { + collected_at: timestamp, + last_pong_time: timestamp, + last_connect_time: timestamp, + ..Default::default() + }) + .expect("rpc metrics should serialize"); + assert_eq!(rpc["collectedAt"], "2023-11-14T22:13:20.123456Z"); + assert_eq!(rpc["lastPongTime"], "2023-11-14T22:13:20.123456Z"); + assert_eq!(rpc["lastConnectTime"], "2023-11-14T22:13:20.123456Z"); + let rpc: RPCMetrics = serde_json::from_value(rpc).expect("rpc metrics should deserialize"); + assert_eq!(rpc.collected_at, timestamp); + assert_eq!(rpc.last_pong_time, timestamp); + assert_eq!(rpc.last_connect_time, timestamp); + + let batch = serde_json::to_value(BatchJobMetrics { + collected_at: timestamp, + jobs: HashMap::from([( + "job-a".to_string(), + JobMetric { + job_id: "job-a".to_string(), + start_time: timestamp, + last_update: timestamp, + ..Default::default() + }, + )]), + }) + .expect("batch metrics should serialize"); + assert_eq!(batch["collected"], "2023-11-14T22:13:20.123456Z"); + assert_eq!(batch["Jobs"]["job-a"]["startTime"], "2023-11-14T22:13:20.123456Z"); + assert_eq!(batch["Jobs"]["job-a"]["lastUpdate"], "2023-11-14T22:13:20.123456Z"); + let batch: BatchJobMetrics = serde_json::from_value(batch).expect("batch metrics should deserialize"); + assert_eq!(batch.collected_at, timestamp); + let job = batch.jobs.get("job-a").expect("job should deserialize"); + assert_eq!(job.start_time, timestamp); + assert_eq!(job.last_update, timestamp); + + let net = serde_json::to_value(NetMetrics { + collected_at: timestamp, + ..Default::default() + }) + .expect("net metrics should serialize"); + assert_eq!(net["collected"], "2023-11-14T22:13:20.123456Z"); + let net: NetMetrics = serde_json::from_value(net).expect("net metrics should deserialize"); + assert_eq!(net.collected_at, timestamp); + + let mem = serde_json::to_value(MemMetrics { + collected_at: timestamp, + ..Default::default() + }) + .expect("mem metrics should serialize"); + assert_eq!(mem["collected"], "2023-11-14T22:13:20.123456Z"); + let mem: MemMetrics = serde_json::from_value(mem).expect("mem metrics should deserialize"); + assert_eq!(mem.collected_at, timestamp); + + let site_resync = serde_json::to_value(SiteResyncMetrics { + collected_at: timestamp, + start_time: timestamp, + last_update: timestamp, + ..Default::default() + }) + .expect("site resync metrics should serialize"); + assert_eq!(site_resync["collected"], "2023-11-14T22:13:20.123456Z"); + assert_eq!(site_resync["startTime"], "2023-11-14T22:13:20.123456Z"); + assert_eq!(site_resync["lastUpdate"], "2023-11-14T22:13:20.123456Z"); + let site_resync: SiteResyncMetrics = serde_json::from_value(site_resync).expect("site resync metrics should deserialize"); + assert_eq!(site_resync.collected_at, timestamp); + assert_eq!(site_resync.start_time, timestamp); + assert_eq!(site_resync.last_update, timestamp); + + let os = serde_json::to_value(OsMetrics { + collected_at: timestamp, + ..Default::default() + }) + .expect("os metrics should serialize"); + assert_eq!(os["collected"], "2023-11-14T22:13:20.123456Z"); + let os: OsMetrics = serde_json::from_value(os).expect("os metrics should deserialize"); + assert_eq!(os.collected_at, timestamp); + } #[test] fn scanner_metrics_serializes_cycle_active_presence() { @@ -1439,9 +1552,9 @@ mod tests { #[test] fn scanner_metrics_merge_prefers_an_active_first_cycle() { - let collected_at = Utc::now(); - let idle_started = collected_at - chrono::Duration::hours(1); - let active_started = collected_at - chrono::Duration::seconds(5); + let collected_at = Timestamp::now(); + let idle_started = collected_at - SignedDuration::from_hours(1); + let active_started = collected_at - SignedDuration::from_secs(5); let mut scanner = ScannerMetrics { collected_at, current_cycle: 0, @@ -1451,7 +1564,7 @@ mod tests { }; scanner.merge(&ScannerMetrics { - collected_at: collected_at + chrono::Duration::seconds(1), + collected_at: collected_at + SignedDuration::from_secs(1), current_cycle: 0, current_cycle_active: Some(true), current_started: active_started, @@ -1481,14 +1594,14 @@ mod tests { #[test] fn scanner_metrics_merge_preserves_legacy_nonzero_active_signal() { - let collected_at = Utc::now(); + let collected_at = Timestamp::now(); let mut scanner = ScannerMetrics { collected_at, ..Default::default() }; scanner.merge(&ScannerMetrics { - collected_at: collected_at + chrono::Duration::seconds(1), + collected_at: collected_at + SignedDuration::from_secs(1), current_cycle: 7, ..Default::default() }); @@ -1517,7 +1630,7 @@ mod tests { #[test] fn scanner_metrics_merge_preserves_explicit_inactive_nonzero_cycle() { - let collected_at = Utc::now(); + let collected_at = Timestamp::now(); let mut scanner = ScannerMetrics::default(); scanner.merge(&ScannerMetrics { @@ -1533,18 +1646,18 @@ mod tests { #[test] fn scanner_metrics_merge_cycle_active_is_order_independent() { - let collected_at = Utc::now(); + let collected_at = Timestamp::now(); let legacy_active = ScannerMetrics { collected_at, current_cycle: 7, - current_started: collected_at - chrono::Duration::seconds(10), + current_started: collected_at - SignedDuration::from_secs(10), ..Default::default() }; let explicit_idle = ScannerMetrics { collected_at, current_cycle: 0, current_cycle_active: Some(false), - current_started: collected_at - chrono::Duration::hours(1), + current_started: collected_at - SignedDuration::from_hours(1), ..Default::default() }; @@ -1575,21 +1688,21 @@ mod tests { #[test] fn scanner_metrics_merge_cycle_authority_is_order_independent() { - let collected_at = Utc::now(); - let completion = collected_at - chrono::Duration::minutes(1); + let collected_at = Timestamp::now(); + let completion = collected_at - SignedDuration::from_mins(1); let earlier_active = ScannerMetrics { collected_at, current_cycle: 7, current_cycle_active: Some(true), - current_started: collected_at - chrono::Duration::seconds(10), + current_started: collected_at - SignedDuration::from_secs(10), cycles_completed_at: vec![completion], ..Default::default() }; let later_active = ScannerMetrics { - collected_at: collected_at + chrono::Duration::seconds(1), + collected_at: collected_at + SignedDuration::from_secs(1), current_cycle: 7, current_cycle_active: Some(true), - current_started: collected_at - chrono::Duration::seconds(5), + current_started: collected_at - SignedDuration::from_secs(5), cycles_completed_at: vec![completion], ..Default::default() }; @@ -1607,13 +1720,13 @@ mod tests { let stale_idle = ScannerMetrics { collected_at, current_cycle_active: Some(false), - current_started: collected_at - chrono::Duration::hours(1), + current_started: collected_at - SignedDuration::from_hours(1), ..Default::default() }; let completed_idle = ScannerMetrics { - collected_at: collected_at + chrono::Duration::seconds(1), + collected_at: collected_at + SignedDuration::from_secs(1), current_cycle_active: Some(false), - current_started: collected_at - chrono::Duration::seconds(5), + current_started: collected_at - SignedDuration::from_secs(5), cycles_completed_at: vec![completion], ..Default::default() }; @@ -1631,29 +1744,29 @@ mod tests { #[test] fn scanner_metrics_merge_cycle_authority_is_associative() { - let collected_at = Utc::now(); - let older_completion = collected_at - chrono::Duration::minutes(3); - let last_completion = collected_at - chrono::Duration::minutes(1); + let collected_at = Timestamp::now(); + let older_completion = collected_at - SignedDuration::from_mins(3); + let last_completion = collected_at - SignedDuration::from_mins(1); let cycle_seven = ScannerMetrics { collected_at, current_cycle: 7, current_cycle_active: Some(true), - current_started: collected_at - chrono::Duration::seconds(10), + current_started: collected_at - SignedDuration::from_secs(10), cycles_completed_at: vec![older_completion, last_completion], ..Default::default() }; let cycle_eight = ScannerMetrics { - collected_at: collected_at + chrono::Duration::seconds(1), + collected_at: collected_at + SignedDuration::from_secs(1), current_cycle: 8, current_cycle_active: Some(true), - current_started: collected_at - chrono::Duration::seconds(5), + current_started: collected_at - SignedDuration::from_secs(5), cycles_completed_at: vec![last_completion], ..Default::default() }; let newer_idle = ScannerMetrics { - collected_at: collected_at + chrono::Duration::hours(1), + collected_at: collected_at + SignedDuration::from_hours(1), current_cycle_active: Some(false), - current_started: collected_at - chrono::Duration::hours(1), + current_started: collected_at - SignedDuration::from_hours(1), ..Default::default() }; @@ -1694,7 +1807,7 @@ mod tests { #[test] fn scanner_metrics_merge_aggregates_partial_cycles_by_source() { - let collected_at = Utc::now(); + let collected_at = Timestamp::now(); let mut scanner = ScannerMetrics { collected_at, last_cycle_partial_source: "usage".to_string(), @@ -1717,7 +1830,7 @@ mod tests { }; scanner.merge(&ScannerMetrics { - collected_at: collected_at + chrono::Duration::seconds(1), + collected_at: collected_at + SignedDuration::from_secs(1), last_cycle_partial_source: "lifecycle".to_string(), last_cycle_partial_source_code: 2, pacing_pressure: ScannerPacingPressureSnapshot { @@ -1769,7 +1882,7 @@ mod tests { #[test] fn scanner_metrics_merge_preserves_pause_pressure_without_duration() { - let collected_at = Utc::now(); + let collected_at = Timestamp::now(); let mut scanner = ScannerMetrics { collected_at, pacing_pressure: ScannerPacingPressureSnapshot { @@ -1780,7 +1893,7 @@ mod tests { }; scanner.merge(&ScannerMetrics { - collected_at: collected_at + chrono::Duration::seconds(1), + collected_at: collected_at + SignedDuration::from_secs(1), pacing_pressure: ScannerPacingPressureSnapshot::default(), ..Default::default() }); @@ -1805,7 +1918,7 @@ mod tests { #[test] fn scanner_metrics_merge_aggregates_lifecycle_transition_status() { - let collected_at = Utc::now(); + let collected_at = Timestamp::now(); let mut scanner = ScannerMetrics { collected_at, current_cycle_lifecycle_expiry_actions: 2, @@ -1843,7 +1956,7 @@ mod tests { }; scanner.merge(&ScannerMetrics { - collected_at: collected_at + chrono::Duration::seconds(1), + collected_at: collected_at + SignedDuration::from_secs(1), current_cycle_lifecycle_expiry_actions: 11, current_cycle_lifecycle_transition_actions: 13, last_cycle_lifecycle_expiry_actions: 17, @@ -1909,7 +2022,7 @@ mod tests { #[test] fn scanner_metrics_merge_aggregates_maintenance_control_status() { - let collected_at = Utc::now(); + let collected_at = Timestamp::now(); let mut scanner = ScannerMetrics { collected_at, maintenance_control: ScannerMaintenanceControlSnapshot { @@ -1930,7 +2043,7 @@ mod tests { }; scanner.merge(&ScannerMetrics { - collected_at: collected_at + chrono::Duration::seconds(1), + collected_at: collected_at + SignedDuration::from_secs(1), maintenance_control: ScannerMaintenanceControlSnapshot { primary_control: "blocked_source".to_string(), sources: vec![ @@ -1990,7 +2103,7 @@ mod tests { #[test] fn scanner_metrics_merge_aggregates_replication_repair_status() { - let collected_at = Utc::now(); + let collected_at = Timestamp::now(); let mut scanner = ScannerMetrics { collected_at, replication_repair: vec![ScannerReplicationRepairSnapshot { @@ -2005,7 +2118,7 @@ mod tests { }; scanner.merge(&ScannerMetrics { - collected_at: collected_at + chrono::Duration::seconds(1), + collected_at: collected_at + SignedDuration::from_secs(1), replication_repair: vec![ ScannerReplicationRepairSnapshot { source: "bucket_replication".to_string(), @@ -2066,7 +2179,7 @@ mod tests { #[test] fn scanner_metrics_merge_preserves_replication_repair_metadata_from_newer_nodes() { - let collected_at = Utc::now(); + let collected_at = Timestamp::now(); let mut scanner = ScannerMetrics { collected_at, replication_repair: vec![ScannerReplicationRepairSnapshot { @@ -2091,7 +2204,7 @@ mod tests { }; scanner.merge(&ScannerMetrics { - collected_at: collected_at + chrono::Duration::seconds(1), + collected_at: collected_at + SignedDuration::from_secs(1), replication_repair: vec![ScannerReplicationRepairSnapshot { source: "bucket_replication".to_string(), kind: "object".to_string(), @@ -2149,7 +2262,7 @@ mod tests { #[test] fn scanner_metrics_merge_preserves_distributed_status_fields() { - let collected_at = Utc::now(); + let collected_at = Timestamp::now(); let mut scanner = ScannerMetrics { collected_at, active_scan_paths: 1, @@ -2223,7 +2336,7 @@ mod tests { }; scanner.merge(&ScannerMetrics { - collected_at: collected_at + chrono::Duration::seconds(1), + collected_at: collected_at + SignedDuration::from_secs(1), active_scan_paths: 2, oldest_active_path_age_seconds: 45, active_paths: vec!["node-b/disk-b/bucket-b".to_string()], diff --git a/crates/madmin/src/trace.rs b/crates/madmin/src/trace.rs index 2f5c72aa7..1c63d81e5 100644 --- a/crates/madmin/src/trace.rs +++ b/crates/madmin/src/trace.rs @@ -14,7 +14,7 @@ use std::{collections::HashMap, time::Duration}; -use chrono::{DateTime, Utc}; +use jiff::Timestamp; use serde::{Deserialize, Serialize}; use crate::heal_commands::HealResultItem; @@ -85,7 +85,7 @@ pub struct TraceInfo { #[serde(rename = "funcname")] func_name: String, #[serde(rename = "time")] - time: DateTime, + time: Timestamp, #[serde(rename = "path")] path: String, #[serde(rename = "dur")] @@ -154,7 +154,7 @@ pub struct TraceCallStats { #[derive(Debug, Default, Clone, Serialize, Deserialize)] pub struct TraceRequestInfo { - time: DateTime, + time: Timestamp, proto: String, method: String, #[serde(skip_serializing_if = "Option::is_none")] @@ -170,7 +170,7 @@ pub struct TraceRequestInfo { #[derive(Debug, Default, Clone, Serialize, Deserialize)] pub struct TraceResponseInfo { - time: DateTime, + time: Timestamp, #[serde(skip_serializing_if = "Option::is_none")] headers: Option>, #[serde(skip_serializing_if = "Option::is_none")] @@ -178,3 +178,38 @@ pub struct TraceResponseInfo { #[serde(skip_serializing_if = "Option::is_none")] status_code: Option, } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn trace_timestamps_serialize_as_rfc3339_utc() { + let timestamp = Timestamp::constant(1_700_000_000, 123_456_000); + let trace = TraceInfo { + time: timestamp, + http: Some(TraceHTTPStats { + req_info: TraceRequestInfo { + time: timestamp, + ..Default::default() + }, + resp_info: TraceResponseInfo { + time: timestamp, + ..Default::default() + }, + ..Default::default() + }), + ..Default::default() + }; + + let value = serde_json::to_value(trace).expect("trace should serialize"); + assert_eq!(value["time"], "2023-11-14T22:13:20.123456Z"); + assert_eq!(value["http"]["req_info"]["time"], "2023-11-14T22:13:20.123456Z"); + assert_eq!(value["http"]["resp_info"]["time"], "2023-11-14T22:13:20.123456Z"); + let trace: TraceInfo = serde_json::from_value(value).expect("trace should deserialize"); + assert_eq!(trace.time, timestamp); + let http = trace.http.expect("http trace should deserialize"); + assert_eq!(http.req_info.time, timestamp); + assert_eq!(http.resp_info.time, timestamp); + } +} diff --git a/crates/obs/src/metrics/stats_collector.rs b/crates/obs/src/metrics/stats_collector.rs index 951447c79..9e9ea26f8 100644 --- a/crates/obs/src/metrics/stats_collector.rs +++ b/crates/obs/src/metrics/stats_collector.rs @@ -38,7 +38,7 @@ use crate::metrics::{ obs_replication_site_stats_snapshot, obs_resolve_object_store_handle, }; use crate::node_identity::current_local_node_identity; -use chrono::Utc; +use jiff::Timestamp; use rustfs_common::heal_channel::HealScanMode; use rustfs_common::metrics::{ScannerBucketDriveResultSnapshot, ScannerMetricsReport, ScannerSourceWorkSnapshot, global_metrics}; use rustfs_io_metrics::internode_metrics::global_internode_metrics; @@ -320,18 +320,23 @@ async fn obs_site_replication_stats() -> ReplicationStats { } } -fn current_scanner_cycle_age_seconds( - current_cycle_active: bool, - current_started: chrono::DateTime, - now: chrono::DateTime, -) -> u64 { +fn current_scanner_cycle_age_seconds(current_cycle_active: bool, current_started: Timestamp, now: Timestamp) -> u64 { if !current_cycle_active { 0 } else { - now.signed_duration_since(current_started).num_seconds().max(0) as u64 + timestamp_elapsed_seconds_since(now, current_started) } } +fn timestamp_elapsed_seconds_since(now: Timestamp, earlier: Timestamp) -> u64 { + let duration = now.duration_since(earlier); + if duration.is_negative() { + return 0; + } + + u64::try_from(duration.as_secs()).map_or(u64::MAX, |seconds| seconds) +} + fn scanner_scan_mode_code(scan_mode: &str) -> u64 { match scan_mode { mode if mode == HealScanMode::Normal.as_str() => HealScanMode::Normal as u8 as u64, @@ -1377,7 +1382,7 @@ pub async fn collect_scanner_metric_stats() -> Option { pub(crate) async fn collect_scanner_runtime_metric_stats() -> Option { let (metrics, runtime_details) = global_metrics().report_with_runtime_details().await; - let now = Utc::now(); + let now = Timestamp::now(); let bucket_scans_finished = metrics.life_time_ops.get("scan_bucket_drive").copied().unwrap_or_default(); let bucket_scans_started = scanner_bucket_scans_started(&metrics.life_time_ops, bucket_scans_finished); let bucket_scans_failed = metrics @@ -1395,7 +1400,7 @@ pub(crate) async fn collect_scanner_runtime_metric_stats() -> Option, - expiration: Option>, + expiration: Option, } impl From for TargetCredentials { @@ -1050,8 +1051,9 @@ impl Operation for ReplicationMrfHandler { #[cfg(test)] mod tests { use super::{ - REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS, RemoteTargetRequest, SUPPORTED_REMOTE_TARGET_API, - build_mrf_response, extract_query_params, unique_replication_peers, validate_remote_target_tls_settings, + REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS, RemoteTargetCredentialsRequest, RemoteTargetRequest, + SUPPORTED_REMOTE_TARGET_API, build_mrf_response, extract_query_params, unique_replication_peers, + validate_remote_target_tls_settings, }; use crate::admin::storage_api::bucket::target::BucketTarget; use crate::admin::storage_api::replication::{BucketStats, DurableMrfBacklog, MrfOpKind, MrfReplicateEntry}; @@ -1350,6 +1352,23 @@ mod tests { assert!(err.to_string().contains("credentials.secretKey is required")); } + #[test] + fn remote_target_credentials_expiration_json_remains_rfc3339() { + let credentials: RemoteTargetCredentialsRequest = serde_json::from_value(serde_json::json!({ + "accessKey": "access", + "secretKey": "secret", + "expiration": "2026-01-01T00:00:00Z" + })) + .expect("credentials expiration should deserialize from RFC3339 JSON"); + let credentials = crate::admin::storage_api::bucket::target::Credentials::from(credentials); + let expiration = credentials.expiration.expect("expiration should be preserved"); + + assert_eq!( + serde_json::to_value(expiration).expect("expiration should serialize to JSON"), + serde_json::json!("2026-01-01T00:00:00Z") + ); + } + #[test] fn remote_target_request_rejects_unimplemented_fields() { for (field, value) in [ diff --git a/rustfs/src/storage/rpc/node_service/heal.rs b/rustfs/src/storage/rpc/node_service/heal.rs index 34ec0991b..8a475c097 100644 --- a/rustfs/src/storage/rpc/node_service/heal.rs +++ b/rustfs/src/storage/rpc/node_service/heal.rs @@ -14,6 +14,7 @@ use crate::startup_background::{heal_enabled_from_env, scanner_enabled_from_env}; use crate::storage::storage_api::runtime_sources_consumer::EndpointServerPools; +use jiff::Timestamp; use rmp_serde::Deserializer; use rustfs_common::heal_channel::HealScanMode; use rustfs_heal::HealOperationsSnapshot; @@ -28,6 +29,31 @@ const NODE_HEAL_STATUS_VERSION: u8 = 1; const NODE_HEAL_STATUS_MAX_SIZE: usize = 64 * 1024; const HEAL_TOPOLOGY_FINGERPRINT_DOMAIN: &[u8] = b"rustfs-heal-topology-v1\0"; +fn chrono_to_jiff_timestamp(timestamp: chrono::DateTime) -> Timestamp { + let seconds = timestamp.timestamp(); + let nanoseconds = match i32::try_from(timestamp.timestamp_subsec_nanos()) { + Ok(nanoseconds) => nanoseconds, + Err(_) => { + return if seconds < 0 { Timestamp::MIN } else { Timestamp::MAX }; + } + }; + + match Timestamp::new(seconds, nanoseconds) { + Ok(timestamp) => timestamp, + Err(_) => { + if seconds < 0 { + Timestamp::MIN + } else { + Timestamp::MAX + } + } + } +} + +fn jiff_to_chrono_datetime(timestamp: Timestamp) -> chrono::DateTime { + chrono::DateTime::::from(std::time::SystemTime::from(timestamp)) +} + #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) struct HealControlCoordinator { pub grid_host: String, @@ -156,7 +182,7 @@ pub(crate) struct NodeHealProgress { #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] struct NodeHealInfo { - bitrot_start_time: Option>, + bitrot_start_time: Option, bitrot_start_cycle: u64, current_scan_mode: HealScanMode, } @@ -164,7 +190,7 @@ struct NodeHealInfo { impl From for NodeHealInfo { fn from(info: BackgroundHealInfo) -> Self { Self { - bitrot_start_time: info.bitrot_start_time, + bitrot_start_time: info.bitrot_start_time.map(chrono_to_jiff_timestamp), bitrot_start_cycle: info.bitrot_start_cycle, current_scan_mode: info.current_scan_mode, } @@ -174,7 +200,7 @@ impl From for NodeHealInfo { impl From for BackgroundHealInfo { fn from(info: NodeHealInfo) -> Self { Self { - bitrot_start_time: info.bitrot_start_time, + bitrot_start_time: info.bitrot_start_time.map(jiff_to_chrono_datetime), bitrot_start_cycle: info.bitrot_start_cycle, current_scan_mode: info.current_scan_mode, } @@ -213,7 +239,7 @@ impl NodeHealStatusSnapshot { pub(crate) fn info(&self) -> BackgroundHealInfo { BackgroundHealInfo { - bitrot_start_time: self.info.bitrot_start_time, + bitrot_start_time: self.info.bitrot_start_time.map(jiff_to_chrono_datetime), bitrot_start_cycle: self.info.bitrot_start_cycle, current_scan_mode: self.info.current_scan_mode, } @@ -270,6 +296,7 @@ mod tests { Endpoint, ecstore_layout::{EndpointServerPools, Endpoints, PoolEndpoints}, }; + use chrono::SecondsFormat; use rustfs_heal::HealOperationsSnapshot; use rustfs_scanner::scanner::BackgroundHealInfo; @@ -461,6 +488,54 @@ mod tests { assert_eq!(decoded.operations.queue_length, 2); } + #[test] + fn node_heal_status_timestamp_json_remains_rfc3339() { + let fixture = serde_json::json!({ + "version": 1, + "servicesEnabled": true, + "initialized": true, + "info": {"bitrotStartTime": "2023-11-14T22:13:20.123456Z", "bitrotStartCycle": 9, "currentScanMode": 1}, + "operations": { + "queueLength": 2, "activeTasks": 1, "retryingTasks": 0, + "queuedByPriority": {"low": 0, "normal": 2, "high": 0, "urgent": 0}, + "activeByPriority": {"low": 0, "normal": 0, "high": 1, "urgent": 0}, + "retryingByPriority": {"low": 0, "normal": 0, "high": 0, "urgent": 0}, + "queuedBySource": {"scanner": 2, "admin": 0, "autoHeal": 0, "internal": 0, "readRepair": 0}, + "activeBySource": {"scanner": 0, "admin": 1, "autoHeal": 0, "internal": 0, "readRepair": 0}, + "retryingBySource": {"scanner": 0, "admin": 0, "autoHeal": 0, "internal": 0, "readRepair": 0} + }, + "progress": null + }); + let encoded = rmp_serde::to_vec_named(&fixture).expect("fixture should encode"); + let decoded = decode_node_heal_status(&encoded).expect("timestamp fixture should decode"); + let info = decoded.info(); + + assert_eq!( + info.bitrot_start_time + .expect("bitrot start time should be preserved") + .to_rfc3339_opts(SecondsFormat::Micros, true), + "2023-11-14T22:13:20.123456Z" + ); + + let started_at = chrono::DateTime::parse_from_rfc3339("2023-11-14T22:13:20.123456Z") + .expect("fixture timestamp should parse") + .with_timezone(&chrono::Utc); + let snapshot = NodeHealStatusSnapshot::for_test( + true, + true, + BackgroundHealInfo { + bitrot_start_time: Some(started_at), + bitrot_start_cycle: 9, + current_scan_mode: rustfs_common::heal_channel::HealScanMode::Deep, + }, + HealOperationsSnapshot::default(), + None, + ); + let encoded = encode_node_heal_status(&snapshot).expect("snapshot should encode"); + let encoded_json: serde_json::Value = rmp_serde::from_slice(&encoded).expect("encoded snapshot should decode as JSON"); + assert_eq!(encoded_json["info"]["bitrotStartTime"], serde_json::json!("2023-11-14T22:13:20.123456Z")); + } + #[test] fn node_heal_status_rejects_nested_unknown_trailing_truncated_and_oversized_data() { let mut fixture = serde_json::json!({