fix(storage): harden rebalance decommission state (#3515)

crates/ecstore/src/rpc/peer_rest_client.rs

@@ -52,7 +52,7 @@ use tokio::{net::TcpStream, time::Duration};
 use tonic::Request;
 use tonic::service::interceptor::InterceptedService;
 use tonic::transport::Channel;
-use tracing::{Instrument, warn};
+use tracing::{Instrument, debug, warn};
 
 pub const PEER_RESTSIGNAL: &str = "signal";
 pub const PEER_RESTSUB_SYS: &str = "sub-sys";
@@ -916,11 +916,13 @@ impl PeerRestClient {
         .await
     }
 
-    pub async fn stop_rebalance(&self) -> Result<()> {
+    pub async fn stop_rebalance(&self, expected_rebalance_id: Option<&str>) -> Result<()> {
         self.finalize_result(
             async {
                 let mut client = self.get_client().await?;
-                let request = Request::new(StopRebalanceRequest {});
+                let request = Request::new(StopRebalanceRequest {
+                    expected_rebalance_id: expected_rebalance_id.unwrap_or_default().to_string(),
+                });
 
                 let response = client.stop_rebalance(request).await?.into_inner();
                 if !response.success {
@@ -945,7 +947,17 @@ impl PeerRestClient {
 
                 let response = client.load_rebalance_meta(request).await?.into_inner();
 
-                warn!("load_rebalance_meta response {:?}, grid_host: {:?}", response, &self.grid_host);
+                debug!(
+                    event = "peer_rebalance_meta",
+                    component = "ecstore",
+                    subsystem = "peer_rest_client",
+                    action = "load_rebalance_meta",
+                    result = "response_received",
+                    peer = %self.grid_host,
+                    success = response.success,
+                    start_rebalance = start_rebalance,
+                    "peer rebalance metadata response"
+                );
                 if !response.success {
                     if let Some(msg) = response.error_info {
                         return Err(Error::other(msg));

crates/ecstore/src/rpc/peer_s3_client.rs

@@ -346,7 +346,8 @@ impl S3PeerSys {
 
 #[derive(Debug)]
 pub struct LocalPeerS3Client {
-    // pub local_disks: Vec<DiskStore>,
+    #[cfg(test)]
+    local_disks: Option<Vec<DiskStore>>,
     // pub node: Node,
     pub pools: Option<Vec<usize>>,
 }
@@ -354,13 +355,29 @@ pub struct LocalPeerS3Client {
 impl LocalPeerS3Client {
     pub fn new(_node: Option<Node>, pools: Option<Vec<usize>>) -> Self {
         Self {
-            // local_disks,
+            #[cfg(test)]
+            local_disks: None,
             // node,
             pools,
         }
     }
 
+    #[cfg(test)]
+    fn new_with_local_disks(_node: Option<Node>, pools: Option<Vec<usize>>, local_disks: Vec<DiskStore>) -> Self {
+        Self {
+            local_disks: Some(local_disks),
+            pools,
+        }
+    }
+
     async fn local_disks_for_pools(&self) -> Vec<DiskStore> {
+        #[cfg(test)]
+        let local_disks = if let Some(local_disks) = self.local_disks.as_ref() {
+            local_disks.clone()
+        } else {
+            all_local_disk().await
+        };
+        #[cfg(not(test))]
         let local_disks = all_local_disk().await;
         let Some(pools) = self.pools.as_ref() else {
             return local_disks;
@@ -1323,7 +1340,7 @@ mod tests {
 
             assert_eq!(walk_err, DiskError::Timeout);
 
-            let info = LocalPeerS3Client::new(None, Some(vec![0]))
+            let info = LocalPeerS3Client::new_with_local_disks(None, Some(vec![0]), disks.clone())
                 .get_bucket_info(bucket, &BucketOptions::default())
                 .await
                 .expect("bucket info should still succeed after prior walk timeout");
@@ -1347,7 +1364,7 @@ mod tests {
             .await
             .expect("bucket should be created on one disk");
 
-        let err = LocalPeerS3Client::new(None, Some(vec![0]))
+        let err = LocalPeerS3Client::new_with_local_disks(None, Some(vec![0]), disks.clone())
             .get_bucket_info("partial-bucket", &BucketOptions::default())
             .await
             .expect_err("partial bucket should not satisfy local write quorum");
@@ -1375,19 +1392,19 @@ mod tests {
             .await
             .expect("bucket should be created on pool 0 disk 1");
 
-        let pool0_info = LocalPeerS3Client::new(None, Some(vec![0]))
+        let pool0_info = LocalPeerS3Client::new_with_local_disks(None, Some(vec![0]), disks.clone())
             .get_bucket_info(bucket, &BucketOptions::default())
             .await
             .expect("pool 0 peer should see bucket on pool 0 disks");
         assert_eq!(pool0_info.name, bucket);
 
-        let pool1_err = LocalPeerS3Client::new(None, Some(vec![1]))
+        let pool1_err = LocalPeerS3Client::new_with_local_disks(None, Some(vec![1]), disks.clone())
             .get_bucket_info(bucket, &BucketOptions::default())
             .await
             .expect_err("pool 1 peer should not count pool 0 disks");
         assert_eq!(pool1_err, Error::VolumeNotFound);
 
-        let pool1_buckets = LocalPeerS3Client::new(None, Some(vec![1]))
+        let pool1_buckets = LocalPeerS3Client::new_with_local_disks(None, Some(vec![1]), disks.clone())
             .list_bucket(&BucketOptions::default())
             .await
             .expect("pool 1 local listing should succeed against its own disks");