feat: enhance console separation with enterprise-grade security, monitoring, and advanced tower-http integration (#513)

* Initial plan

* feat: implement console service separation from endpoint

Co-authored-by: houseme <4829346+houseme@users.noreply.github.com>

* feat: add console separation documentation and tests

Co-authored-by: houseme <4829346+houseme@users.noreply.github.com>

* feat: enhance console separation with configurable CORS and improved Docker support

Co-authored-by: houseme <4829346+houseme@users.noreply.github.com>

* feat: implement enhanced console separation with security hardening and monitoring

Co-authored-by: houseme <4829346+houseme@users.noreply.github.com>

* refactor: implement console TLS following endpoint logic and improve configuration

Co-authored-by: houseme <4829346+houseme@users.noreply.github.com>

* add tower-http feature "timeout｜limit"

* add dependencies crates `axum-server`

* refactor: reconstruct console server with enhanced tower-http features and environment variables

Co-authored-by: houseme <4829346+houseme@users.noreply.github.com>

* upgrade dep

* improve code for dns and console port `:9001`

* improve code

* fix

* docs: comprehensive improvement of console separation documentation and Docker deployment standards

Co-authored-by: houseme <4829346+houseme@users.noreply.github.com>

* fmt

* add logs

* improve code for Config handler

* remove logs

* fix

---------

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: houseme <4829346+houseme@users.noreply.github.com>
Co-authored-by: houseme <housemecn@gmail.com>

crates/ahm/Cargo.toml

@@ -45,4 +45,4 @@ tracing-subscriber = { workspace = true }
 walkdir = "2.5.0"
 tempfile = { workspace = true }
 criterion = { workspace = true, features = ["html_reports"] }
-sysinfo = "0.30.8"
+sysinfo = { workspace = true }

crates/config/src/constants/app.rs

@@ -124,7 +124,7 @@ pub const DEFAULT_LOG_FILENAME: &str = "rustfs";
 /// This is the default log filename for OBS.
 /// It is used to store the logs of the application.
 /// Default value: rustfs.log
-pub const DEFAULT_OBS_LOG_FILENAME: &str = concat!(DEFAULT_LOG_FILENAME, ".");
+pub const DEFAULT_OBS_LOG_FILENAME: &str = concat!(DEFAULT_LOG_FILENAME, "");
 
 /// Default sink file log file for rustfs
 /// This is the default sink file log file for rustfs.
@@ -160,6 +160,16 @@ pub const DEFAULT_LOG_ROTATION_TIME: &str = "day";
 /// Environment variable: RUSTFS_OBS_LOG_KEEP_FILES
 pub const DEFAULT_LOG_KEEP_FILES: u16 = 30;
 
+/// This is the external address for rustfs to access endpoint (used in Docker deployments).
+/// This should match the mapped host port when using Docker port mapping.
+/// Example: ":9020" when mapping host port 9020 to container port 9000.
+/// Default value: DEFAULT_ADDRESS
+/// Environment variable: RUSTFS_EXTERNAL_ADDRESS
+/// Command line argument: --external-address
+/// Example: RUSTFS_EXTERNAL_ADDRESS=":9020"
+/// Example: --external-address ":9020"
+pub const ENV_EXTERNAL_ADDRESS: &str = "RUSTFS_EXTERNAL_ADDRESS";
+
 #[cfg(test)]
 mod tests {
     use super::*;

crates/config/src/constants/console.rs

@@ -0,0 +1,81 @@
+//  Copyright 2024 RustFS Team
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+/// CORS allowed origins for the endpoint service
+/// Comma-separated list of origins or "*" for all origins
+pub const ENV_CORS_ALLOWED_ORIGINS: &str = "RUSTFS_CORS_ALLOWED_ORIGINS";
+
+/// Default CORS allowed origins for the endpoint service
+/// Comes from the console service default
+/// See DEFAULT_CONSOLE_CORS_ALLOWED_ORIGINS
+pub const DEFAULT_CORS_ALLOWED_ORIGINS: &str = DEFAULT_CONSOLE_CORS_ALLOWED_ORIGINS;
+
+/// CORS allowed origins for the console service
+/// Comma-separated list of origins or "*" for all origins
+pub const ENV_CONSOLE_CORS_ALLOWED_ORIGINS: &str = "RUSTFS_CONSOLE_CORS_ALLOWED_ORIGINS";
+
+/// Default CORS allowed origins for the console service
+pub const DEFAULT_CONSOLE_CORS_ALLOWED_ORIGINS: &str = "*";
+
+/// Enable or disable the console service
+pub const ENV_CONSOLE_ENABLE: &str = "RUSTFS_CONSOLE_ENABLE";
+
+/// Address for the console service to bind to
+pub const ENV_CONSOLE_ADDRESS: &str = "RUSTFS_CONSOLE_ADDRESS";
+
+/// RUSTFS_CONSOLE_RATE_LIMIT_ENABLE
+/// Enable or disable rate limiting for the console service
+pub const ENV_CONSOLE_RATE_LIMIT_ENABLE: &str = "RUSTFS_CONSOLE_RATE_LIMIT_ENABLE";
+
+/// Default console rate limit enable
+/// This is the default value for enabling rate limiting on the console server.
+/// Rate limiting helps protect against abuse and DoS attacks on the management interface.
+/// Default value: false
+/// Environment variable: RUSTFS_CONSOLE_RATE_LIMIT_ENABLE
+/// Command line argument: --console-rate-limit-enable
+/// Example: RUSTFS_CONSOLE_RATE_LIMIT_ENABLE=true
+/// Example: --console-rate-limit-enable true
+pub const DEFAULT_CONSOLE_RATE_LIMIT_ENABLE: bool = false;
+
+/// Set the rate limit requests per minute for the console service
+/// Limits the number of requests per minute per client IP when rate limiting is enabled
+/// Default: 100 requests per minute
+pub const ENV_CONSOLE_RATE_LIMIT_RPM: &str = "RUSTFS_CONSOLE_RATE_LIMIT_RPM";
+
+/// Default console rate limit requests per minute
+/// This is the default rate limit for console requests when rate limiting is enabled.
+/// Limits the number of requests per minute per client IP to prevent abuse.
+/// Default value: 100 requests per minute
+/// Environment variable: RUSTFS_CONSOLE_RATE_LIMIT_RPM
+/// Command line argument: --console-rate-limit-rpm
+/// Example: RUSTFS_CONSOLE_RATE_LIMIT_RPM=100
+/// Example: --console-rate-limit-rpm 100
+pub const DEFAULT_CONSOLE_RATE_LIMIT_RPM: u32 = 100;
+
+/// Set the console authentication timeout in seconds
+/// Specifies how long a console authentication session remains valid
+/// Default: 3600 seconds (1 hour)
+/// Minimum: 300 seconds (5 minutes)
+/// Maximum: 86400 seconds (24 hours)
+pub const ENV_CONSOLE_AUTH_TIMEOUT: &str = "RUSTFS_CONSOLE_AUTH_TIMEOUT";
+
+/// Default console authentication timeout in seconds
+/// This is the default timeout for console authentication sessions.
+/// After this timeout, users need to re-authenticate to access the console.
+/// Default value: 3600 seconds (1 hour)
+/// Environment variable: RUSTFS_CONSOLE_AUTH_TIMEOUT
+/// Command line argument: --console-auth-timeout
+/// Example: RUSTFS_CONSOLE_AUTH_TIMEOUT=3600
+/// Example: --console-auth-timeout 3600
+pub const DEFAULT_CONSOLE_AUTH_TIMEOUT: u64 = 3600;

crates/config/src/constants/mod.rs

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-pub mod app;
-pub mod env;
-pub mod tls;
+pub(crate) mod app;
+pub(crate) mod console;
+pub(crate) mod env;
+pub(crate) mod tls;

crates/config/src/lib.rs

@@ -17,6 +17,8 @@ pub mod constants;
 #[cfg(feature = "constants")]
 pub use constants::app::*;
 #[cfg(feature = "constants")]
+pub use constants::console::*;
+#[cfg(feature = "constants")]
 pub use constants::env::*;
 #[cfg(feature = "constants")]
 pub use constants::tls::*;

crates/ecstore/src/endpoints.rs

@@ -169,7 +169,7 @@ impl AsMut<Vec<Endpoints>> for PoolEndpointList {
 impl PoolEndpointList {
     /// creates a list of endpoints per pool, resolves their relevant
     /// hostnames and discovers those are local or remote.
-    fn create_pool_endpoints(server_addr: &str, disks_layout: &DisksLayout) -> Result<Self> {
+    async fn create_pool_endpoints(server_addr: &str, disks_layout: &DisksLayout) -> Result<Self> {
         if disks_layout.is_empty_layout() {
             return Err(Error::other("invalid number of endpoints"));
         }
@@ -244,7 +244,7 @@ impl PoolEndpointList {
                 let host_ip_set = if let Some(set) = host_ip_cache.get(&host) {
                     set
                 } else {
-                    let ips = match get_host_ip(host.clone()) {
+                    let ips = match get_host_ip(host.clone()).await {
                         Ok(ips) => ips,
                         Err(e) => {
                             error!("host {} not found, error:{}", host, e);
@@ -466,19 +466,22 @@ impl EndpointServerPools {
         }
         None
     }
-    pub fn from_volumes(server_addr: &str, endpoints: Vec<String>) -> Result<(EndpointServerPools, SetupType)> {
+    pub async fn from_volumes(server_addr: &str, endpoints: Vec<String>) -> Result<(EndpointServerPools, SetupType)> {
         let layouts = DisksLayout::from_volumes(endpoints.as_slice())?;
 
-        Self::create_server_endpoints(server_addr, &layouts)
+        Self::create_server_endpoints(server_addr, &layouts).await
     }
     /// validates and creates new endpoints from input args, supports
     /// both ellipses and without ellipses transparently.
-    pub fn create_server_endpoints(server_addr: &str, disks_layout: &DisksLayout) -> Result<(EndpointServerPools, SetupType)> {
+    pub async fn create_server_endpoints(
+        server_addr: &str,
+        disks_layout: &DisksLayout,
+    ) -> Result<(EndpointServerPools, SetupType)> {
         if disks_layout.pools.is_empty() {
             return Err(Error::other("Invalid arguments specified"));
         }
 
-        let pool_eps = PoolEndpointList::create_pool_endpoints(server_addr, disks_layout)?;
+        let pool_eps = PoolEndpointList::create_pool_endpoints(server_addr, disks_layout).await?;
 
         let mut ret: EndpointServerPools = Vec::with_capacity(pool_eps.as_ref().len()).into();
         for (i, eps) in pool_eps.inner.into_iter().enumerate() {
@@ -753,8 +756,8 @@ mod test {
         }
     }
 
-    #[test]
-    fn test_create_pool_endpoints() {
+    #[tokio::test]
+    async fn test_create_pool_endpoints() {
         #[derive(Default)]
         struct TestCase<'a> {
             num: usize,
@@ -1276,7 +1279,7 @@ mod test {
 
             match (
                 test_case.expected_err,
-                PoolEndpointList::create_pool_endpoints(test_case.server_addr, &disks_layout),
+                PoolEndpointList::create_pool_endpoints(test_case.server_addr, &disks_layout).await,
             ) {
                 (None, Err(err)) => panic!("Test {}: error: expected = <nil>, got = {}", test_case.num, err),
                 (Some(err), Ok(_)) => panic!("Test {}: error: expected = {}, got = <nil>", test_case.num, err),
@@ -1343,8 +1346,8 @@ mod test {
         (urls, local_flags)
     }
 
-    #[test]
-    fn test_create_server_endpoints() {
+    #[tokio::test]
+    async fn test_create_server_endpoints() {
         let test_cases = [
             // Invalid input.
             ("", vec![], false),
@@ -1379,7 +1382,7 @@ mod test {
                 }
             };
 
-            let ret = EndpointServerPools::create_server_endpoints(test_case.0, &disks_layout);
+            let ret = EndpointServerPools::create_server_endpoints(test_case.0, &disks_layout).await;
 
             if let Err(err) = ret {
                 if test_case.2 {

crates/ecstore/src/global.rs

@@ -37,26 +37,27 @@ pub const DISK_FILL_FRACTION: f64 = 0.99;
 pub const DISK_RESERVE_FRACTION: f64 = 0.15;
 
 lazy_static! {
-static ref GLOBAL_RUSTFS_PORT: OnceLock<u16> = OnceLock::new();
-pub static ref GLOBAL_OBJECT_API: OnceLock<Arc<ECStore>> = OnceLock::new();
-pub static ref GLOBAL_LOCAL_DISK: Arc<RwLock<Vec<Option<DiskStore>>>> = Arc::new(RwLock::new(Vec::new()));
-pub static ref GLOBAL_IsErasure: RwLock<bool> = RwLock::new(false);
-pub static ref GLOBAL_IsDistErasure: RwLock<bool> = RwLock::new(false);
-pub static ref GLOBAL_IsErasureSD: RwLock<bool> = RwLock::new(false);
-pub static ref GLOBAL_LOCAL_DISK_MAP: Arc<RwLock<HashMap<String, Option<DiskStore>>>> = Arc::new(RwLock::new(HashMap::new()));
-pub static ref GLOBAL_LOCAL_DISK_SET_DRIVES: Arc<RwLock<TypeLocalDiskSetDrives>> = Arc::new(RwLock::new(Vec::new()));
-pub static ref GLOBAL_Endpoints: OnceLock<EndpointServerPools> = OnceLock::new();
-pub static ref GLOBAL_RootDiskThreshold: RwLock<u64> = RwLock::new(0);
-pub static ref GLOBAL_TierConfigMgr: Arc<RwLock<TierConfigMgr>> = TierConfigMgr::new();
-pub static ref GLOBAL_LifecycleSys: Arc<LifecycleSys> = LifecycleSys::new();
-pub static ref GLOBAL_EventNotifier: Arc<RwLock<EventNotifier>> = EventNotifier::new();
-//pub static ref GLOBAL_RemoteTargetTransport
-static ref globalDeploymentIDPtr: OnceLock<Uuid> = OnceLock::new();
-pub static ref GLOBAL_BOOT_TIME: OnceCell<SystemTime> = OnceCell::new();
-pub static ref GLOBAL_LocalNodeName: String = "127.0.0.1:9000".to_string();
-pub static ref GLOBAL_LocalNodeNameHex: String = rustfs_utils::crypto::hex(GLOBAL_LocalNodeName.as_bytes());
-pub static ref GLOBAL_NodeNamesHex: HashMap<String, ()> = HashMap::new();
-pub static ref GLOBAL_REGION: OnceLock<String> = OnceLock::new();
+    static ref GLOBAL_RUSTFS_PORT: OnceLock<u16> = OnceLock::new();
+    static ref GLOBAL_RUSTFS_EXTERNAL_PORT: OnceLock<u16> = OnceLock::new();
+    pub static ref GLOBAL_OBJECT_API: OnceLock<Arc<ECStore>> = OnceLock::new();
+    pub static ref GLOBAL_LOCAL_DISK: Arc<RwLock<Vec<Option<DiskStore>>>> = Arc::new(RwLock::new(Vec::new()));
+    pub static ref GLOBAL_IsErasure: RwLock<bool> = RwLock::new(false);
+    pub static ref GLOBAL_IsDistErasure: RwLock<bool> = RwLock::new(false);
+    pub static ref GLOBAL_IsErasureSD: RwLock<bool> = RwLock::new(false);
+    pub static ref GLOBAL_LOCAL_DISK_MAP: Arc<RwLock<HashMap<String, Option<DiskStore>>>> = Arc::new(RwLock::new(HashMap::new()));
+    pub static ref GLOBAL_LOCAL_DISK_SET_DRIVES: Arc<RwLock<TypeLocalDiskSetDrives>> = Arc::new(RwLock::new(Vec::new()));
+    pub static ref GLOBAL_Endpoints: OnceLock<EndpointServerPools> = OnceLock::new();
+    pub static ref GLOBAL_RootDiskThreshold: RwLock<u64> = RwLock::new(0);
+    pub static ref GLOBAL_TierConfigMgr: Arc<RwLock<TierConfigMgr>> = TierConfigMgr::new();
+    pub static ref GLOBAL_LifecycleSys: Arc<LifecycleSys> = LifecycleSys::new();
+    pub static ref GLOBAL_EventNotifier: Arc<RwLock<EventNotifier>> = EventNotifier::new();
+    //pub static ref GLOBAL_RemoteTargetTransport
+    static ref globalDeploymentIDPtr: OnceLock<Uuid> = OnceLock::new();
+    pub static ref GLOBAL_BOOT_TIME: OnceCell<SystemTime> = OnceCell::new();
+    pub static ref GLOBAL_LocalNodeName: String = "127.0.0.1:9000".to_string();
+    pub static ref GLOBAL_LocalNodeNameHex: String = rustfs_utils::crypto::hex(GLOBAL_LocalNodeName.as_bytes());
+    pub static ref GLOBAL_NodeNamesHex: HashMap<String, ()> = HashMap::new();
+    pub static ref GLOBAL_REGION: OnceLock<String> = OnceLock::new();
 }
 
 // Global cancellation token for background services (data scanner and auto heal)
@@ -108,6 +109,22 @@ pub fn set_global_rustfs_port(value: u16) {
     GLOBAL_RUSTFS_PORT.set(value).expect("set_global_rustfs_port fail");
 }
 
+/// Get the global rustfs external port
+pub fn global_rustfs_external_port() -> u16 {
+    if let Some(p) = GLOBAL_RUSTFS_EXTERNAL_PORT.get() {
+        *p
+    } else {
+        rustfs_config::DEFAULT_PORT
+    }
+}
+
+/// Set the global rustfs external port
+pub fn set_global_rustfs_external_port(value: u16) {
+    GLOBAL_RUSTFS_EXTERNAL_PORT
+        .set(value)
+        .expect("set_global_rustfs_external_port fail");
+}
+
 /// Get the global rustfs port
 pub fn set_global_deployment_id(id: Uuid) {
     globalDeploymentIDPtr.set(id).unwrap();

crates/lock/Cargo.toml

@@ -42,8 +42,8 @@ url.workspace = true
 uuid.workspace = true
 thiserror.workspace = true
 once_cell.workspace = true
-parking_lot = "0.12"
-smallvec = "1.11"
-smartstring = "1.0"
-crossbeam-queue = "0.3"
-heapless = "0.8"
+parking_lot.workspace = true
+smallvec.workspace = true
+smartstring.workspace = true
+crossbeam-queue = { workspace = true }
+heapless = { workspace = true }

crates/mcp/src/server.rs

@@ -616,6 +616,7 @@ impl ServerHandler for RustfsMcpServer {
             server_info: Implementation {
                 name: "rustfs-mcp-server".into(),
                 version: env!("CARGO_PKG_VERSION").into(),
+                ..Default::default()
             },
         }
     }

crates/targets/src/target/mqtt.rs

@@ -438,7 +438,7 @@ fn is_fatal_mqtt_error(err: &ConnectionError) -> bool {
                 rumqttc::StateError::InvalidState          // The internal state machine is in invalid state
                 | rumqttc::StateError::WrongPacket         // Agreement Violation: Unexpected Data Packet Received
                 | rumqttc::StateError::Unsolicited(_)      // Agreement Violation: Unsolicited ACK Received
-                | rumqttc::StateError::OutgoingPacketTooLarge { .. } // Try to send too large packets
+                | rumqttc::StateError::CollisionTimeout    // Agreement Violation (if this stage occurs)
                 | rumqttc::StateError::EmptySubscription   // Agreement violation (if this stage occurs)
                 => true,
 

crates/utils/Cargo.toml

@@ -81,7 +81,7 @@ workspace = true
 default = ["ip"]  # features that are enabled by default
 ip = ["dep:local-ip-address"]  # ip characteristics and their dependencies
 tls = ["dep:rustls", "dep:rustls-pemfile", "dep:rustls-pki-types"]  # tls characteristics and their dependencies
-net = ["ip", "dep:url", "dep:netif", "dep:futures", "dep:transform-stream", "dep:bytes", "dep:s3s", "dep:hyper", "dep:hyper-util", "dep:hickory-resolver", "dep:hickory-proto", "dep:moka", "dep:thiserror"]  # network features with DNS resolver
+net = ["ip", "dep:url", "dep:netif", "dep:futures", "dep:transform-stream", "dep:bytes", "dep:s3s", "dep:hyper", "dep:hyper-util", "dep:hickory-resolver", "dep:hickory-proto", "dep:moka", "dep:thiserror", "dep:tokio"]  # network features with DNS resolver
 io = ["dep:tokio"]
 path = []
 notify = ["dep:hyper", "dep:s3s"]  # file system notification features

crates/utils/src/dns_resolver.rs

@@ -127,6 +127,7 @@ impl LayeredDnsResolver {
     /// Validate domain format according to RFC standards
     #[instrument(skip_all, fields(domain = %domain))]
     fn validate_domain_format(domain: &str) -> Result<(), DnsError> {
+        info!("Validating domain format start");
         // Check FQDN length
         if domain.len() > MAX_FQDN_LENGTH {
             return Err(DnsError::InvalidFormat {
@@ -157,7 +158,7 @@ impl LayeredDnsResolver {
                 });
             }
         }
-
+        info!("DNS resolver validated successfully");
         Ok(())
     }
 
@@ -209,7 +210,6 @@ impl LayeredDnsResolver {
                 let ips: Vec<IpAddr> = lookup.iter().collect();
                 if !ips.is_empty() {
                     info!("System DNS resolution successful for domain: {} -> {} IPs", domain, ips.len());
-                    debug!("System DNS resolved IPs: {:?}", ips);
                     Ok(ips)
                 } else {
                     warn!("System DNS returned empty result for domain: {}", domain);
@@ -242,7 +242,6 @@ impl LayeredDnsResolver {
                 let ips: Vec<IpAddr> = lookup.iter().collect();
                 if !ips.is_empty() {
                     info!("Public DNS resolution successful for domain: {} -> {} IPs", domain, ips.len());
-                    debug!("Public DNS resolved IPs: {:?}", ips);
                     Ok(ips)
                 } else {
                     warn!("Public DNS returned empty result for domain: {}", domain);
@@ -270,6 +269,7 @@ impl LayeredDnsResolver {
     /// 3. Public DNS (hickory-resolver with TLS-enabled Cloudflare DNS fallback)
     #[instrument(skip_all, fields(domain = %domain))]
     pub async fn resolve(&self, domain: &str) -> Result<Vec<IpAddr>, DnsError> {
+        info!("Starting DNS resolution process for domain: {} start", domain);
         // Validate domain format first
         Self::validate_domain_format(domain)?;
 
@@ -305,7 +305,7 @@ impl LayeredDnsResolver {
             }
             Err(public_err) => {
                 error!(
-                    "All DNS resolution attempts failed for domain: {}. System DNS: failed, Public DNS: {}",
+                    "All DNS resolution attempts failed for domain:` {}`. System DNS: failed, Public DNS: {}",
                     domain, public_err
                 );
                 Err(DnsError::AllAttemptsFailed {
@@ -345,6 +345,7 @@ pub fn get_global_dns_resolver() -> Option<&'static LayeredDnsResolver> {
 /// Resolve domain using the global DNS resolver with comprehensive tracing
 #[instrument(skip_all, fields(domain = %domain))]
 pub async fn resolve_domain(domain: &str) -> Result<Vec<IpAddr>, DnsError> {
+    info!("resolving domain for: {}", domain);
     match get_global_dns_resolver() {
         Some(resolver) => resolver.resolve(domain).await,
         None => Err(DnsError::InitializationFailed {

crates/utils/src/net.rs

@@ -15,6 +15,7 @@
 use bytes::Bytes;
 use futures::pin_mut;
 use futures::{Stream, StreamExt};
+use std::io::Error;
 use std::net::Ipv6Addr;
 use std::sync::{LazyLock, Mutex};
 use std::{
@@ -23,6 +24,7 @@ use std::{
     net::{IpAddr, SocketAddr, TcpListener, ToSocketAddrs},
     time::{Duration, Instant},
 };
+use tracing::{error, info};
 use transform_stream::AsyncTryStream;
 use url::{Host, Url};
 
@@ -61,7 +63,7 @@ pub fn is_socket_addr(addr: &str) -> bool {
 pub fn check_local_server_addr(server_addr: &str) -> std::io::Result<SocketAddr> {
     let addr: Vec<SocketAddr> = match server_addr.to_socket_addrs() {
         Ok(addr) => addr.collect(),
-        Err(err) => return Err(std::io::Error::other(err)),
+        Err(err) => return Err(Error::other(err)),
     };
 
     // 0.0.0.0 is a wildcard address and refers to local network
@@ -82,7 +84,7 @@ pub fn check_local_server_addr(server_addr: &str) -> std::io::Result<SocketAddr>
         }
     }
 
-    Err(std::io::Error::other("host in server address should be this server"))
+    Err(Error::other("host in server address should be this server"))
 }
 
 /// checks if the given parameter correspond to one of
@@ -93,7 +95,7 @@ pub fn is_local_host(host: Host<&str>, port: u16, local_port: u16) -> std::io::R
         Host::Domain(domain) => {
             let ips = match (domain, 0).to_socket_addrs().map(|v| v.map(|v| v.ip()).collect::<Vec<_>>()) {
                 Ok(ips) => ips,
-                Err(err) => return Err(std::io::Error::other(err)),
+                Err(err) => return Err(Error::other(err)),
             };
 
             ips.iter().any(|ip| local_set.contains(ip))
@@ -113,49 +115,20 @@ pub fn is_local_host(host: Host<&str>, port: u16, local_port: u16) -> std::io::R
 ///
 /// This is the async version of `get_host_ip()` that provides enhanced DNS resolution
 /// with Kubernetes support when the "net" feature is enabled.
-pub async fn get_host_ip_async(host: Host<&str>) -> std::io::Result<HashSet<IpAddr>> {
+pub async fn get_host_ip(host: Host<&str>) -> std::io::Result<HashSet<IpAddr>> {
     match host {
         Host::Domain(domain) => {
-            #[cfg(feature = "net")]
-            {
-                use crate::dns_resolver::resolve_domain;
-                match resolve_domain(domain).await {
-                    Ok(ips) => Ok(ips.into_iter().collect()),
-                    Err(e) => Err(std::io::Error::other(format!("DNS resolution failed: {}", e))),
+            match crate::dns_resolver::resolve_domain(domain).await {
+                Ok(ips) => {
+                    info!("Resolved domain {domain} using custom DNS resolver: {ips:?}");
+                    return Ok(ips.into_iter().collect());
+                }
+                Err(err) => {
+                    error!(
+                        "Failed to resolve domain {domain} using custom DNS resolver, falling back to system resolver,err: {err}"
+                    );
                 }
             }
-            #[cfg(not(feature = "net"))]
-            {
-                // Fallback to standard resolution when DNS resolver is not available
-                match (domain, 0)
-                    .to_socket_addrs()
-                    .map(|v| v.map(|v| v.ip()).collect::<HashSet<_>>())
-                {
-                    Ok(ips) => Ok(ips),
-                    Err(err) => Err(std::io::Error::other(err)),
-                }
-            }
-        }
-        Host::Ipv4(ip) => {
-            let mut set = HashSet::with_capacity(1);
-            set.insert(IpAddr::V4(ip));
-            Ok(set)
-        }
-        Host::Ipv6(ip) => {
-            let mut set = HashSet::with_capacity(1);
-            set.insert(IpAddr::V6(ip));
-            Ok(set)
-        }
-    }
-}
-
-/// returns IP address of given host using standard resolution.
-///
-/// **Note**: This function uses standard library DNS resolution with caching.
-/// For enhanced DNS resolution with Kubernetes support, use `get_host_ip_async()`.
-pub fn get_host_ip(host: Host<&str>) -> std::io::Result<HashSet<IpAddr>> {
-    match host {
-        Host::Domain(domain) => {
             // Check cache first
             if let Ok(mut cache) = DNS_CACHE.lock() {
                 if let Some(entry) = cache.get(domain) {
@@ -167,7 +140,9 @@ pub fn get_host_ip(host: Host<&str>) -> std::io::Result<HashSet<IpAddr>> {
                 }
             }
 
-            // Perform DNS resolution
+            info!("Cache miss for domain {domain}, querying system resolver.");
+
+            // Fallback to standard resolution when DNS resolver is not available
             match (domain, 0)
                 .to_socket_addrs()
                 .map(|v| v.map(|v| v.ip()).collect::<HashSet<_>>())
@@ -181,21 +156,17 @@ pub fn get_host_ip(host: Host<&str>) -> std::io::Result<HashSet<IpAddr>> {
                             cache.retain(|_, v| !v.is_expired(DNS_CACHE_TTL));
                         }
                     }
+                    info!("System query for domain {domain}: {:?}", ips);
                     Ok(ips)
                 }
-                Err(err) => Err(std::io::Error::other(err)),
+                Err(err) => {
+                    error!("Failed to resolve domain {domain} using system resolver, err: {err}");
+                    Err(Error::other(err))
+                }
             }
         }
-        Host::Ipv4(ip) => {
-            let mut set = HashSet::with_capacity(1);
-            set.insert(IpAddr::V4(ip));
-            Ok(set)
-        }
-        Host::Ipv6(ip) => {
-            let mut set = HashSet::with_capacity(1);
-            set.insert(IpAddr::V6(ip));
-            Ok(set)
-        }
+        Host::Ipv4(ip) => Ok([IpAddr::V4(ip)].into_iter().collect()),
+        Host::Ipv6(ip) => Ok([IpAddr::V6(ip)].into_iter().collect()),
     }
 }
 
@@ -207,7 +178,7 @@ pub fn get_available_port() -> u16 {
 pub fn must_get_local_ips() -> std::io::Result<Vec<IpAddr>> {
     match netif::up() {
         Ok(up) => Ok(up.map(|x| x.address().to_owned()).collect()),
-        Err(err) => Err(std::io::Error::other(format!("Unable to get IP addresses of this host: {err}"))),
+        Err(err) => Err(Error::other(format!("Unable to get IP addresses of this host: {err}"))),
     }
 }
 
@@ -215,7 +186,7 @@ pub fn get_default_location(_u: Url, _region_override: &str) -> String {
     todo!();
 }
 
-pub fn get_endpoint_url(endpoint: &str, secure: bool) -> Result<Url, std::io::Error> {
+pub fn get_endpoint_url(endpoint: &str, secure: bool) -> Result<Url, Error> {
     let mut scheme = "https";
     if !secure {
         scheme = "http";
@@ -223,7 +194,7 @@ pub fn get_endpoint_url(endpoint: &str, secure: bool) -> Result<Url, std::io::Er
 
     let endpoint_url_str = format!("{scheme}://{endpoint}");
     let Ok(endpoint_url) = Url::parse(&endpoint_url_str) else {
-        return Err(std::io::Error::other("url parse error."));
+        return Err(Error::other("url parse error."));
     };
 
     //is_valid_endpoint_url(endpoint_url)?;
@@ -258,7 +229,7 @@ impl Display for XHost {
 }
 
 impl TryFrom<String> for XHost {
-    type Error = std::io::Error;
+    type Error = Error;
 
     fn try_from(value: String) -> Result<Self, Self::Error> {
         if let Some(addr) = value.to_socket_addrs()?.next() {
@@ -268,7 +239,7 @@ impl TryFrom<String> for XHost {
                 is_port_set: addr.port() > 0,
             })
         } else {
-            Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "value invalid"))
+            Err(Error::new(std::io::ErrorKind::InvalidData, "value invalid"))
         }
     }
 }
@@ -278,7 +249,7 @@ pub fn parse_and_resolve_address(addr_str: &str) -> std::io::Result<SocketAddr>
         let port_str = port;
         let port: u16 = port_str
             .parse()
-            .map_err(|e| std::io::Error::other(format!("Invalid port format: {addr_str}, err:{e:?}")))?;
+            .map_err(|e| Error::other(format!("Invalid port format: {addr_str}, err:{e:?}")))?;
         let final_port = if port == 0 {
             get_available_port() // assume get_available_port is available here
         } else {
@@ -318,9 +289,9 @@ where
 
 #[cfg(test)]
 mod test {
-    use std::net::{Ipv4Addr, Ipv6Addr};
-
     use super::*;
+    use crate::init_global_dns_resolver;
+    use std::net::{Ipv4Addr, Ipv6Addr};
 
     #[test]
     fn test_is_socket_addr() {
@@ -424,23 +395,29 @@ mod test {
         assert!(is_local_host(invalid_host, 0, 0).is_err());
     }
 
-    #[test]
-    fn test_get_host_ip() {
+    #[tokio::test]
+    async fn test_get_host_ip() {
+        match init_global_dns_resolver().await {
+            Ok(_) => {}
+            Err(e) => {
+                error!("Failed to initialize global DNS resolver: {e}");
+            }
+        }
         // Test IPv4 address
         let ipv4_host = Host::Ipv4(Ipv4Addr::new(192, 168, 1, 1));
-        let ipv4_result = get_host_ip(ipv4_host).unwrap();
+        let ipv4_result = get_host_ip(ipv4_host).await.unwrap();
         assert_eq!(ipv4_result.len(), 1);
         assert!(ipv4_result.contains(&IpAddr::V4(Ipv4Addr::new(192, 168, 1, 1))));
 
         // Test IPv6 address
         let ipv6_host = Host::Ipv6(Ipv6Addr::new(0x2001, 0xdb8, 0, 0, 0, 0, 0, 1));
-        let ipv6_result = get_host_ip(ipv6_host).unwrap();
+        let ipv6_result = get_host_ip(ipv6_host).await.unwrap();
         assert_eq!(ipv6_result.len(), 1);
         assert!(ipv6_result.contains(&IpAddr::V6(Ipv6Addr::new(0x2001, 0xdb8, 0, 0, 0, 0, 0, 1))));
 
         // Test localhost domain
         let localhost_host = Host::Domain("localhost");
-        let localhost_result = get_host_ip(localhost_host).unwrap();
+        let localhost_result = get_host_ip(localhost_host).await.unwrap();
         assert!(!localhost_result.is_empty());
         // Should contain at least loopback address
         assert!(
@@ -450,7 +427,16 @@ mod test {
 
         // Test invalid domain
         let invalid_host = Host::Domain("invalid.nonexistent.domain.example");
-        assert!(get_host_ip(invalid_host).is_err());
+        match get_host_ip(invalid_host.clone()).await {
+            Ok(ips) => {
+                // Depending on DNS resolver behavior, it might return empty set or error
+                assert!(ips.is_empty(), "Expected empty IP set for invalid domain, got: {:?}", ips);
+            }
+            Err(_) => {
+                error!("Expected error for invalid domain");
+            } // Expected error
+        }
+        assert!(get_host_ip(invalid_host).await.is_err());
     }
 
     #[test]