From 650a3e5734cf89c947847f19838db2d14b831b9c Mon Sep 17 00:00:00 2001
From: Zhengchao An <anzhengchao@gmail.com>
Date: Mon, 6 Jul 2026 21:20:41 +0800
Subject: [PATCH] fix(replication): classify resync verification HEAD errors
 instead of miscounting (backlog#862) (#4312)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The resync result verification HEADed the target after replicating and counted
the outcome with inverted error handling:

- for a delete marker, ANY HEAD error (timeout, 5xx, auth, malformed) was
  counted as replicated (success);
- for a versioned object against an AWS-style target, HEAD was sent with the
  RustFS UUID versionId, which AWS rejects with 400, so a well-replicated
  object was counted as failed.

Classify the error before counting:

- delete marker: only a definitive 404/NoSuchKey or 405/MethodNotAllowed
  confirms the marker propagated (`is_retryable_delete_replication_head_error`
  == false); any retryable/ambiguous error now counts as failed;
- versioned object with a version-id-format rejection: re-verify via
  `head_object_fallback` (versionId-less HEAD) before deciding — present ->
  replicated, absent/error -> failed;
- all other errors: failed, as before.

Reuses the existing, unit-tested classifier helpers. Verified against the
existing resyncer suite (24 tests).

Refs backlog#799 (B13).
---
 .../replication/replication_resyncer.rs       | 60 ++++++++++++++++---
 1 file changed, 51 insertions(+), 9 deletions(-)

diff --git a/crates/ecstore/src/bucket/replication/replication_resyncer.rs b/crates/ecstore/src/bucket/replication/replication_resyncer.rs
index 7e4bcfc41..f3fc517e4 100644
--- a/crates/ecstore/src/bucket/replication/replication_resyncer.rs
+++ b/crates/ecstore/src/bucket/replication/replication_resyncer.rs
@@ -708,17 +708,59 @@ impl ReplicationResyncer {
                         roi.version_id.map(|v| v.to_string()),
                     )
                     .await;
-                    let (size, err) = if let Err(err) = head_result {
-                        if roi.delete_marker {
+                    let (size, err) = match head_result {
+                        Ok(_) => {
                             st.replicated_count += 1;
-                        } else {
-                            st.failed_count += 1;
+                            st.replicated_size += roi.size;
+                            (roi.size, None)
+                        }
+                        Err(err) if roi.delete_marker => {
+                            // Verifying a replicated delete marker: only a
+                            // definitive 404/NoSuchKey or 405/MethodNotAllowed
+                            // confirms the marker propagated. Any other
+                            // (retryable/ambiguous) HEAD error leaves the outcome
+                            // unverified, so it must count as failed — not as a
+                            // blanket success (backlog#862 / #799 B13).
+                            let retryable = {
+                                let (is_not_found, code) = err
+                                    .as_service_error()
+                                    .map(|se| (se.is_not_found(), se.code()))
+                                    .unwrap_or((false, None));
+                                is_retryable_delete_replication_head_error(is_not_found, code)
+                            };
+                            if retryable {
+                                st.failed_count += 1;
+                                (0, Some(err))
+                            } else {
+                                st.replicated_count += 1;
+                                (0, None)
+                            }
+                        }
+                        Err(err) if is_version_id_format_mismatch(&err) => {
+                            // AWS-style target rejects the RustFS UUID versionId
+                            // (400). Re-verify without the versionId before
+                            // concluding the object failed to replicate, instead
+                            // of counting a well-replicated object as failed.
+                            match head_object_fallback(&bucket_name, target_client.as_ref(), &roi.name).await {
+                                Ok(Some(_)) => {
+                                    st.replicated_count += 1;
+                                    st.replicated_size += roi.size;
+                                    (roi.size, None)
+                                }
+                                Ok(None) => {
+                                    st.failed_count += 1;
+                                    (0, Some(err))
+                                }
+                                Err(e2) => {
+                                    st.failed_count += 1;
+                                    (0, Some(e2))
+                                }
+                            }
+                        }
+                        Err(err) => {
+                            st.failed_count += 1;
+                            (0, Some(err))
                         }
-                        (0, Some(err))
-                    } else {
-                        st.replicated_count += 1;
-                        st.replicated_size += roi.size;
-                        (roi.size, None)
                     };
 
                     if err.is_some() {