feat(ecstore): implement decommission and rebalance (#2281)

Co-authored-by: weisd <im@weisd.in> Co-authored-by: houseme <housemecn@gmail.com>
2026-08-24 21:26:28 +00:00 · 2026-03-26 11:44:02 +08:00
parent 59c437d901
commit a236b0d01d
20 changed files with 7812 additions and 1304 deletions
@@ -18,6 +18,7 @@ use std::io::{self, Read, Seek, SeekFrom};

 const S2_INDEX_HEADER: &[u8] = b"s2idx\x00";
 const S2_INDEX_TRAILER: &[u8] = b"\x00xdi2s";
+const LEGACY_INDEX_HEADER_PADDING: &[u8] = &[0, 0, 0];
 const MAX_INDEX_ENTRIES: usize = 1 << 16;
 const MIN_INDEX_DIST: i64 = 1 << 20;
 // const MIN_INDEX_DIST: i64 = 0;
@@ -76,10 +77,14 @@ impl Index {
    }

    fn alloc_infos(&mut self, n: usize) {
-        if n > MAX_INDEX_ENTRIES {
-            panic!("n > MAX_INDEX_ENTRIES");
-        }
-        self.info = Vec::with_capacity(n);
+        debug_assert!(n <= MAX_INDEX_ENTRIES, "n > MAX_INDEX_ENTRIES");
+        self.info = vec![
+            IndexInfo {
+                compressed_offset: 0,
+                uncompressed_offset: 0,
+            };
+            n
+        ];
    }

    pub fn add(&mut self, compressed_offset: i64, uncompressed_offset: i64) -> io::Result<()> {
@@ -217,9 +222,8 @@ impl Index {
        self.reduce();
        let init_size = b.len();

-        // Add skippable header
-        b.extend_from_slice(&[0x50, 0x2A, 0x4D, 0x18]); // ChunkTypeIndex
-        b.extend_from_slice(&[0, 0, 0]); // Placeholder for chunk length
+        // Add skippable header (1-byte marker + 24-bit length placeholder)
+        b.extend_from_slice(&[0x50, 0x2A, 0x4D, 0x18]); // length is written back into bytes 1..=3

        // Add header
        b.extend_from_slice(S2_INDEX_HEADER);
@@ -295,7 +299,7 @@ impl Index {
            return Err(io::Error::new(io::ErrorKind::UnexpectedEof, "buffer too small"));
        }

-        if b[0] != 0x50 || b[1] != 0x2A || b[2] != 0x4D || b[3] != 0x18 {
+        if b[0] != 0x50 {
            return Err(io::Error::other("invalid chunk type"));
        }

@@ -306,6 +310,10 @@ impl Index {
            return Err(io::Error::new(io::ErrorKind::UnexpectedEof, "buffer too small"));
        }

+        if b.starts_with(LEGACY_INDEX_HEADER_PADDING) {
+            b = &b[LEGACY_INDEX_HEADER_PADDING.len()..];
+        }
+
        if !b.starts_with(S2_INDEX_HEADER) {
            return Err(io::Error::other("invalid header"));
        }
@@ -687,4 +695,72 @@ mod tests {

        Ok(())
    }
+
+    #[test]
+    fn test_index_into_vec_round_trip_via_load() -> io::Result<()> {
+        let mut source = Index::new();
+        source.add(100, 1_000)?;
+        source.add(300, 1_000 + MIN_INDEX_DIST)?;
+
+        let encoded = source.clone().into_vec();
+
+        let mut decoded = Index::new();
+        let rest = decoded.load(encoded.as_ref())?;
+
+        assert!(rest.is_empty());
+        assert_eq!(decoded.total_uncompressed, source.total_uncompressed);
+        assert_eq!(decoded.total_compressed, source.total_compressed);
+        assert_eq!(decoded.info.len(), source.info.len());
+        assert_eq!(decoded.info[0].compressed_offset, source.info[0].compressed_offset);
+        assert_eq!(decoded.info[0].uncompressed_offset, source.info[0].uncompressed_offset);
+        assert_eq!(decoded.info[1].uncompressed_offset, source.info[1].uncompressed_offset);
+        assert!(decoded.info[1].compressed_offset > decoded.info[0].compressed_offset);
+
+        Ok(())
+    }
+
+    #[test]
+    fn test_index_load_rejects_invalid_chunk_type_marker() -> io::Result<()> {
+        let mut source = Index::new();
+        source.add(100, 1_000)?;
+        source.add(300, 1_000 + MIN_INDEX_DIST)?;
+        let mut encoded = source.into_vec().to_vec();
+
+        encoded[0] = 0x51;
+
+        let mut decoded = Index::new();
+        let err = decoded
+            .load(encoded.as_slice())
+            .expect_err("invalid marker should be rejected");
+        assert_eq!(err.kind(), io::ErrorKind::Other);
+        assert_eq!(err.to_string(), "invalid chunk type");
+
+        Ok(())
+    }
+
+    #[test]
+    fn test_index_load_accepts_legacy_zero_padded_header() -> io::Result<()> {
+        let mut source = Index::new();
+        source.add(100, 1_000)?;
+        source.add(300, 1_000 + MIN_INDEX_DIST)?;
+
+        let mut encoded = source.clone().into_vec().to_vec();
+        let chunk_len = (encoded[1] as usize) | ((encoded[2] as usize) << 8) | ((encoded[3] as usize) << 16);
+        let legacy_chunk_len = chunk_len + LEGACY_INDEX_HEADER_PADDING.len();
+
+        encoded[1] = legacy_chunk_len as u8;
+        encoded[2] = (legacy_chunk_len >> 8) as u8;
+        encoded[3] = (legacy_chunk_len >> 16) as u8;
+        encoded.splice(4..4, LEGACY_INDEX_HEADER_PADDING.iter().copied());
+
+        let mut decoded = Index::new();
+        let rest = decoded.load(encoded.as_slice())?;
+
+        assert!(rest.is_empty());
+        assert_eq!(decoded.total_uncompressed, source.total_uncompressed);
+        assert_eq!(decoded.total_compressed, source.total_compressed);
+        assert_eq!(decoded.info.len(), source.info.len());
+
+        Ok(())
+    }
 }
@@ -22,6 +22,7 @@ use rustfs_common::internode_metrics::global_internode_metrics;
 use rustfs_utils::get_env_opt_str;
 use std::io::IoSlice;
 use std::io::{self, Error};
+use std::net::IpAddr;
 use std::ops::Not as _;
 use std::pin::Pin;
 use std::sync::LazyLock;
@@ -91,25 +92,48 @@ fn load_optional_mtls_identity_from_tls_path() -> Option<Identity> {
    }
 }

-fn get_http_client() -> Client {
-    // Reuse the HTTP connection pool in the global `reqwest::Client` instance
-    // TODO: interact with load balancing?
-    static CLIENT: LazyLock<Client> = LazyLock::new(|| {
-        let mut builder = Client::builder()
-            .connect_timeout(std::time::Duration::from_secs(5))
-            .tcp_keepalive(std::time::Duration::from_secs(10))
-            .http2_keep_alive_interval(std::time::Duration::from_secs(5))
-            .http2_keep_alive_timeout(std::time::Duration::from_secs(3))
-            .http2_keep_alive_while_idle(true);
+fn build_http_client(disable_proxy: bool) -> Client {
+    let mut builder = Client::builder()
+        .connect_timeout(std::time::Duration::from_secs(5))
+        .tcp_keepalive(std::time::Duration::from_secs(10))
+        .http2_keep_alive_interval(std::time::Duration::from_secs(5))
+        .http2_keep_alive_timeout(std::time::Duration::from_secs(3))
+        .http2_keep_alive_while_idle(true);

-        // HTTPS root trust + optional mTLS identity from RUSTFS_TLS_PATH
-        builder = load_ca_roots_from_tls_path(builder);
-        if let Some(id) = load_optional_mtls_identity_from_tls_path() {
-            builder = builder.identity(id);
-        }
+    if disable_proxy {
+        builder = builder.no_proxy();
+    }
+
+    builder = load_ca_roots_from_tls_path(builder);
+    if let Some(id) = load_optional_mtls_identity_from_tls_path() {
+        builder = builder.identity(id);
+    }
+
+    builder.build().expect("Failed to create global HTTP client")
+}
+
+fn should_bypass_proxy_for_url(url: &str) -> bool {
+    let Some(host) = reqwest::Url::parse(url)
+        .ok()
+        .and_then(|url| url.host_str().map(str::to_owned))
+    else {
+        return false;
+    };
+    let host = host.trim_matches(['[', ']']);
+
+    host.eq_ignore_ascii_case("localhost") || host.parse::<IpAddr>().is_ok_and(|addr| addr.is_loopback())
+}
+
+fn get_http_client(url: &str) -> Client {
+    // Reuse HTTP connection pools while keeping loopback traffic away from
+    // system proxies so local RPC/tests do not leak to proxy listeners.
+    static CLIENT: LazyLock<Client> = LazyLock::new(|| build_http_client(false));
+    static LOCAL_CLIENT: LazyLock<Client> = LazyLock::new(|| build_http_client(true));
+
+    if should_bypass_proxy_for_url(url) {
+        return LOCAL_CLIENT.clone();
+    }

-        builder.build().expect("Failed to create global HTTP client")
-    });
    CLIENT.clone()
 }

@@ -138,7 +162,7 @@ impl HttpReader {
        _read_buf_size: usize,
    ) -> io::Result<Self> {
        let track_internode_metrics = is_internode_rpc_url(&url);
-        let client = get_http_client();
+        let client = get_http_client(&url);
        let mut request: RequestBuilder = client.request(method.clone(), url.clone()).headers(headers.clone());
        if let Some(body) = body {
            request = request.body(body);
@@ -302,7 +326,7 @@ impl HttpWriter {
            //     "[HttpWriter::spawn] sending HTTP request: url={url_clone}, method={method_clone:?}, headers={headers_clone:?}"
            // );

-            let client = get_http_client();
+            let client = get_http_client(&url_clone);
            let request = client
                .request(method_clone, url_clone.clone())
                .headers(headers_clone.clone())
@@ -664,4 +688,14 @@ mod tests {

        handle.abort();
    }
+
+    #[test]
+    fn loopback_urls_bypass_proxy_selection() {
+        assert!(should_bypass_proxy_for_url("http://127.0.0.1:9000/stream"));
+        assert!(should_bypass_proxy_for_url("http://localhost:9000/stream"));
+        assert!(should_bypass_proxy_for_url("http://[::1]:9000/stream"));
+        assert!(!should_bypass_proxy_for_url("http://192.168.1.10:9000/stream"));
+        assert!(!should_bypass_proxy_for_url("http://example.com/stream"));
+        assert!(!should_bypass_proxy_for_url("not-a-url"));
+    }
 }