fix(ecstore): log peer offline/online transitions for console reporting (#4367)

A node could show OFFLINE in the console with zero logs explaining why
(rustfs/backlog#888, reported in rustfs#4304): the consecutive-failure
threshold crossing in handle_server_info_failure marked the peer offline
silently, the per-call warnings only existed on the observing node and
never named the transition, and the PeerRestClient offline flag plus its
background recovery monitor ran without any start/success log.

Logging-only change, no behavior change:

- handle_server_info_failure / handle_storage_info_failure: WARN
  event="peer_marked_offline" exactly once at the threshold crossing
  (later failures while already offline stay DEBUG), and DEBUG
  event="peer_probe_failure" while returning cached state below the
  threshold.
- update_server_info_cache / update_storage_info_cache: INFO
  event="peer_recovered_online" when a probe succeeds after the peer had
  been reported offline.
- PeerRestClient::mark_offline_and_spawn_recovery: WARN
  event="peer_connection_marked_offline" when the offline flag is first
  set (guarded by the recovery_running CAS so repeated failures do not
  spam), and INFO event="peer_connection_recovered" with the attempt
  count when connectivity is restored.

An "offline then back" episode now leaves a complete, correlatable
trace: N probe failures -> peer_marked_offline -> recovery monitor ->
peer_recovered_online.

Verification:
- cargo test -p rustfs-ecstore --lib (notification + peer_rest suites)
- make pre-commit

Ref: rustfs/backlog#888, rustfs#4304

Co-authored-by: heihutu <heihutu@gmail.com>

crates/ecstore/src/cluster/rpc/peer_rest_client.rs

@@ -53,7 +53,7 @@ use tokio::{net::TcpStream, time::Duration};
 use tonic::Request;
 use tonic::service::interceptor::InterceptedService;
 use tonic::transport::Channel;
-use tracing::{debug, warn};
+use tracing::{debug, info, warn};
 
 pub const PEER_RESTSIGNAL: &str = "signal";
 pub const PEER_RESTSUB_SYS: &str = "sub-sys";
@@ -184,15 +184,29 @@ impl PeerRestClient {
         let offline = Arc::clone(&self.offline);
         let recovery_running = Arc::clone(&self.recovery_running);
         let span = Self::recovery_monitor_span(&grid_host);
+        // The offline flag and its recovery are the silent half of
+        // rustfs/backlog#888: log the monitor's start and its success so an
+        // "offline then back" episode leaves a trace on the observing node.
+        warn!(
+            event = "peer_connection_marked_offline",
+            grid_host = %self.grid_host,
+            "peer RPC connection marked offline after a network-like failure; starting background recovery monitor"
+        );
         super::spawn_background_monitor(span, async move {
             let mut delay = get_drive_active_check_interval();
             let connect_timeout = get_drive_active_check_timeout();
 
-            for _ in 0..PEER_REST_RECOVERY_MAX_ATTEMPTS {
+            for attempt in 1..=PEER_REST_RECOVERY_MAX_ATTEMPTS {
                 tokio::time::sleep(delay).await;
                 if Self::perform_connectivity_check(&grid_host, connect_timeout).await.is_ok() {
                     offline.store(false, Ordering::Release);
                     recovery_running.store(false, Ordering::Release);
+                    info!(
+                        event = "peer_connection_recovered",
+                        grid_host = %grid_host,
+                        attempts = attempt,
+                        "peer connectivity restored by background recovery monitor"
+                    );
                     return;
                 }