fix(heal): correct progress accounting (#6382)

* fix(heal): correct progress accounting

* fix(heal): atomically persist page progress

* fix(heal): preserve terminal progress counters

* fix(heal): make resume handoff crash safe

* fix(heal): preserve resumable bucket checkpoints

* fix(heal): satisfy checkpoint outcome lint

* fix(heal): preserve progress status across nodes

* fix(heal): stabilize progress generations

* style: restore rebalance formatting

* test(heal): cover cross-set baseline generation

---------

Signed-off-by: houseme <housemecn@gmail.com>
Co-authored-by: overtrue <anzhengchao@gmail.com>
Co-authored-by: houseme <housemecn@gmail.com>
Co-authored-by: heihutu <heihutu@gmail.com>

crates/protos/src/generated/proto_gen/node_service.rs

@@ -1215,7 +1215,10 @@ pub struct ScannerActivityResponse {
     pub dirty_usage_pending: bool,
 }
 #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
-pub struct BackgroundHealStatusRequest {}
+pub struct BackgroundHealStatusRequest {
+    #[prost(uint32, tag = "1")]
+    pub protocol_version: u32,
+}
 #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct BackgroundHealStatusResponse {
     #[prost(bool, tag = "1")]

crates/protos/src/lib.rs

@@ -170,6 +170,7 @@ pub fn internode_rpc_max_message_size() -> usize {
 pub const HEAL_CONTROL_RPC_MAX_MESSAGE_SIZE: usize = heal_control::RESULT_MAX_SIZE + 1024;
 pub const HEAL_CONTROL_PROTOCOL_VERSION: u32 = 3;
 pub const DYNAMIC_CONFIG_PROTOCOL_VERSION: u32 = 1;
+pub const BACKGROUND_HEAL_STATUS_PROTOCOL_VERSION: u32 = 2;
 pub const HEAL_CONTROL_CAPABILITY_PROBE_PREFIX: &[u8] = b"rustfs-heal-control-capability-v3\0";
 pub const REMOTE_VERSION_STATE_CAPABILITY_PROBE_PREFIX: &[u8] = b"rustfs-tier-remote-version-state-capability-v1\0";
 pub const CROSS_POOL_FENCE_CAPABILITY_PROBE_PREFIX: &[u8] = b"rustfs-cross-pool-fence-capability-v1\0";
@@ -2346,10 +2347,28 @@ pub async fn evict_failed_connection_with_log_level(addr: &str, log_level: Conne
 #[cfg(test)]
 mod tests {
     use super::*;
+    use prost::Message as _;
     use std::sync::Mutex;
 
     static INTERNODE_RPC_MSGPACK_ONLY_ENV_LOCK: Mutex<()> = Mutex::new(());
 
+    #[derive(Clone, PartialEq, prost::Message)]
+    struct BackgroundHealStatusRequestV1 {}
+
+    #[test]
+    fn background_heal_status_request_remains_rolling_upgrade_compatible() {
+        let current = proto_gen::node_service::BackgroundHealStatusRequest {
+            protocol_version: BACKGROUND_HEAL_STATUS_PROTOCOL_VERSION,
+        };
+        let encoded = current.encode_to_vec();
+        BackgroundHealStatusRequestV1::decode(encoded.as_slice()).expect("v1 server should ignore the version field");
+
+        let encoded = BackgroundHealStatusRequestV1 {}.encode_to_vec();
+        let decoded = proto_gen::node_service::BackgroundHealStatusRequest::decode(encoded.as_slice())
+            .expect("v2 server should accept a v1 request");
+        assert_eq!(decoded.protocol_version, 0);
+    }
+
     #[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)]
     struct CompatPayloadField {
         message: &'static str,

crates/protos/src/node.proto

@@ -846,7 +846,9 @@ message ScannerActivityResponse {
   bool dirty_usage_pending = 9;
 }
 
-message BackgroundHealStatusRequest {}
+message BackgroundHealStatusRequest {
+  uint32 protocol_version = 1;
+}
 
 message BackgroundHealStatusResponse {
   bool success = 1;