fix(storage): harden scanner and recovery edge cases (#5521)

* fix(ecstore): handle benign listing and GET disconnects Co-Authored-By: heihutu <heihutu@gmail.com> * fix(scanner): scope cache locks by set Co-Authored-By: heihutu <heihutu@gmail.com> * test(kms): stabilize Vault transport retry coverage Co-Authored-By: heihutu <heihutu@gmail.com> * fix(scanner): fence scoped cache locks by protocol Co-Authored-By: heihutu <heihutu@gmail.com> * test(ecstore): stabilize topology DNS fallback coverage Co-Authored-By: heihutu <heihutu@gmail.com> * fix(kms): remove stale local export test import Co-Authored-By: heihutu <heihutu@gmail.com> --------- Co-authored-by: heihutu <heihutu@gmail.com>
2026-08-13 08:36:54 +00:00 · 2026-08-01 11:25:13 +08:00
parent 1524ed891f
commit b965bd6eef
13 changed files with 504 additions and 135 deletions
@@ -26,16 +26,16 @@ use std::sync::{Arc, Mutex};
 use tokio::io::{AsyncReadExt, AsyncWriteExt};
 use tokio::net::{TcpListener, TcpStream};

-/// One canned HTTP response.
-pub(crate) struct ScriptedResponse {
-    status: u16,
-    body: String,
+/// One scripted connection outcome.
+pub(crate) enum ScriptedResponse {
+    Http { status: u16, body: String },
+    Close,
 }

 impl ScriptedResponse {
    /// A 200 response carrying `data` inside the standard Vault envelope.
    pub(crate) fn ok(data: serde_json::Value) -> Self {
-        Self {
+        Self::Http {
            status: 200,
            body: serde_json::json!({
                "request_id": "scripted",
@@ -50,11 +50,16 @@ impl ScriptedResponse {

    /// An error response in Vault's `{"errors": [...]}` format.
    pub(crate) fn error(status: u16, message: &str) -> Self {
-        Self {
+        Self::Http {
            status,
            body: serde_json::json!({ "errors": [message] }).to_string(),
        }
    }
+
+    /// Close the connection after consuming a request without sending an HTTP response.
+    pub(crate) fn close() -> Self {
+        Self::Close
+    }
 }

 /// A scripted stand-in Vault listening on a loopback port.
@@ -88,14 +93,14 @@ impl ScriptedVault {
                let response = responses
                    .next()
                    .unwrap_or_else(|| ScriptedResponse::error(599, "scripted vault: script exhausted"));
-                let payload = format!(
-                    "HTTP/1.1 {} Scripted\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}",
-                    response.status,
-                    response.body.len(),
-                    response.body
-                );
-                let _ = stream.write_all(payload.as_bytes()).await;
-                let _ = stream.shutdown().await;
+                if let ScriptedResponse::Http { status, body } = response {
+                    let payload = format!(
+                        "HTTP/1.1 {status} Scripted\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
+                        body.len(),
+                    );
+                    let _ = stream.write_all(payload.as_bytes()).await;
+                    let _ = stream.shutdown().await;
+                }
            }
        });

@@ -1512,6 +1512,8 @@ mod tests {
    use crate::backends::scripted_vault::{ScriptedResponse, ScriptedVault};
    use crate::config::{VaultAuthMethod, VaultConfig};

+    const SCRIPTED_RETRY_ATTEMPTS: u32 = 3;
+
    /// Vault + KMS config pair pointing at a scripted loopback Vault.
    fn scripted_configs(address: &str) -> (VaultConfig, KmsConfig) {
        let vault_config = VaultConfig {
@@ -1527,7 +1529,7 @@ mod tests {
        };
        let kms_config = KmsConfig {
            timeout: Duration::from_secs(5),
-            retry_attempts: 3,
+            retry_attempts: SCRIPTED_RETRY_ATTEMPTS,
            ..KmsConfig::default()
        };
        (vault_config, kms_config)
@@ -1596,6 +1598,31 @@ mod tests {
        );
    }

+    #[tokio::test]
+    async fn wired_read_retries_closed_connections_within_budget() {
+        let (vault, client) = scripted_client(vec![ScriptedResponse::close(), ScriptedResponse::close()]).await;
+
+        let error = client
+            .get_key_data("wired-key")
+            .await
+            .expect_err("closed connections must exhaust the retry budget");
+        assert!(matches!(error, KmsError::BackendError { .. }), "got {error:?}");
+
+        let requests = vault.requests();
+        let expected_requests = usize::try_from(SCRIPTED_RETRY_ATTEMPTS).expect("retry attempts must fit usize");
+        assert_eq!(
+            requests.len(),
+            expected_requests,
+            "all budgeted retry attempts must reach Vault: {requests:?}"
+        );
+        assert!(
+            requests
+                .iter()
+                .all(|line| line == "GET /v1/secret/data/rustfs/kms/keys/wired-key"),
+            "all attempts must hit the same read endpoint: {requests:?}"
+        );
+    }
+
    #[tokio::test]
    async fn wired_read_does_not_retry_permission_errors() {
        let (vault, client) = scripted_client(vec![ScriptedResponse::error(403, "permission denied")]).await;