From c4836916b0395f7a653780f2689f2f4f66172113 Mon Sep 17 00:00:00 2001 From: JaminMartin Date: Wed, 10 Dec 2025 13:13:19 +1300 Subject: [PATCH 1/4] feat: testing wan consul discovery --- src/rpc/consul.rs | 56 ++++++++++++++++++++++++++-------------------- src/util/config.rs | 2 ++ 2 files changed, 34 insertions(+), 24 deletions(-) diff --git a/src/rpc/consul.rs b/src/rpc/consul.rs index 760e9fcb..a9e0ec93 100644 --- a/src/rpc/consul.rs +++ b/src/rpc/consul.rs @@ -128,36 +128,44 @@ impl ConsulDiscovery { // ---- READING FROM CONSUL CATALOG ---- pub async fn get_consul_nodes(&self) -> Result, ConsulError> { - let url = format!( - "{}/v1/catalog/service/{}", - self.config.consul_http_addr, self.config.service_name - ); - - let http = self.client.get(&url).send().await?; - let entries: Vec = http.json().await?; - let mut ret = vec![]; - for ent in entries { - let ip = ent.address.parse::().ok(); - let pubkey = ent - .meta - .get(&format!("{}-pubkey", META_PREFIX)) - .and_then(|k| hex::decode(k).ok()) - .and_then(|k| NodeID::from_slice(&k[..])); - if let (Some(ip), Some(pubkey)) = (ip, pubkey) { - ret.push((pubkey, SocketAddr::new(ip, ent.service_port))); - } else { - warn!( - "Could not process node spec from Consul: {:?} (invalid IP address or node ID/pubkey)", - ent - ); + + let dcs_to_query: Vec> = match &self.config.datacenters { + Some(dcs) if !dcs.is_empty() => dcs.iter().map(|dc| Some(dc.as_str())).collect(), + _ => vec![None], + }; + + for dc in dcs_to_query { + let url = match dc { + Some(datacenter) => format!( + "{}/v1/catalog/service/{}?dc={}", + self.config.consul_http_addr, self.config.service_name, datacenter + ), + None => format!( + "{}/v1/catalog/service/{}", + self.config.consul_http_addr, self.config.service_name + ), + }; + + let http = self.client.get(&url).send().await?; + let entries: Vec = http.json().await?; + + for ent in entries { + let ip = ent.address.parse::().ok(); + let pubkey = ent + .meta + .get(&format!("{}-pubkey", META_PREFIX)) + .and_then(|k| hex::decode(k).ok()) + .and_then(|k| NodeID::from_slice(&k[..])); + if let (Some(ip), Some(pubkey)) = (ip, pubkey) { + ret.push((pubkey, SocketAddr::new(ip, ent.service_port))); + } } } - debug!("Got nodes from Consul: {:?}", ret); + debug!("Got {} nodes from Consul", ret.len()); Ok(ret) } - // ---- PUBLISHING TO CONSUL CATALOG ---- pub async fn publish_consul_service( diff --git a/src/util/config.rs b/src/util/config.rs index a4521eb5..29047f8c 100644 --- a/src/util/config.rs +++ b/src/util/config.rs @@ -257,6 +257,8 @@ pub struct ConsulDiscoveryConfig { /// Additional service metadata to add #[serde(default)] pub meta: Option>, + #[serde(default)] + pub datacenters: Option>, } #[derive(Deserialize, Debug, Clone)] From 2f7a649870071614d2b73dd5f81b0a1bfb95f55d Mon Sep 17 00:00:00 2001 From: JaminMartin Date: Fri, 19 Dec 2025 11:26:24 +1300 Subject: [PATCH 2/4] doc: added documentation for WAN service discovery --- doc/book/reference-manual/configuration.md | 16 +++++++++++++--- src/rpc/consul.rs | 12 +++++++++++- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/doc/book/reference-manual/configuration.md b/doc/book/reference-manual/configuration.md index d26d9253..fcf731e6 100644 --- a/doc/book/reference-manual/configuration.md +++ b/doc/book/reference-manual/configuration.md @@ -61,8 +61,9 @@ client_key = "/etc/consul/consul-key.crt" tls_skip_verify = false tags = [ "dns-enabled" ] meta = { dns-acl = "allow trusted" } - - +# If your consul cluster is in a WAN configuration, you can provide the datacenter names to allow garage to do discovery across a WAN federation. +# This is not required for non-WAN consul instances. +# datacenters = ["dc1", "dc2", "dc3"] [kubernetes_discovery] namespace = "garage" service_name = "garage-daemon" @@ -127,12 +128,14 @@ The `[consul_discovery]` section: [`client_cert`](#consul_client_cert_and_key), [`client_key`](#consul_client_cert_and_key), [`consul_http_addr`](#consul_http_addr), +[`datacenters`](#consul_datacenters) [`meta`](#consul_tags_and_meta), [`service_name`](#consul_service_name), [`tags`](#consul_tags_and_meta), [`tls_skip_verify`](#consul_tls_skip_verify), [`token`](#consul_token). + The `[kubernetes_discovery]` section: [`namespace`](#kube_namespace), [`service_name`](#kube_service_name), @@ -726,8 +729,15 @@ node_prefix "" { } ``` -#### `tags` and `meta` {#consul_tags_and_meta} +### `consul datacenters` {#consul_datacenters} + +Optional list of datacenters that allow garage to do service discovery when consul is configured in WAN federation. +e.g datacenters = ["dc1", "dc2", "dc3"] +In a WAN configuration the consul services API only responds with local `LAN` services. +This queries the consul server API by datacenter directly, allowing for garage to discover nodes across consul WAN. + +#### `tags` and `meta` {#consul_tags_and_meta} Additional list of tags and map of service meta to add during service registration. ### The `[kubernetes_discovery]` section diff --git a/src/rpc/consul.rs b/src/rpc/consul.rs index a9e0ec93..54f75e74 100644 --- a/src/rpc/consul.rs +++ b/src/rpc/consul.rs @@ -126,7 +126,17 @@ impl ConsulDiscovery { } // ---- READING FROM CONSUL CATALOG ---- - + /// Query Consul for Garage nodes registered under the configured service name. + /// + /// This method supports querying multiple Consul datacenters for WAN or + /// multi-datacenter deployments. If `config.datacenters` is set and non-empty, + /// each listed datacenter is queried and the results are aggregated. Otherwise, + /// only the local datacenter is queried. `config.datacenters` does not need to be set + /// when all the datacenters are on the same LAN, in this case service discovery works normally + /// + /// # Returns + /// A list of `(NodeID, SocketAddr)` pairs corresponding to all valid discovered + /// nodes across the queried datacenters. pub async fn get_consul_nodes(&self) -> Result, ConsulError> { let mut ret = vec![]; From dc8d93698b7e5bb8515aa2945a457da9c4f97a5e Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Tue, 6 Jan 2026 14:32:07 +0100 Subject: [PATCH 3/4] small documentation fixes and simplify config struct --- doc/book/reference-manual/configuration.md | 27 ++++++++++++++-------- src/rpc/consul.rs | 2 +- src/util/config.rs | 2 +- 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/doc/book/reference-manual/configuration.md b/doc/book/reference-manual/configuration.md index fcf731e6..9b2d57b6 100644 --- a/doc/book/reference-manual/configuration.md +++ b/doc/book/reference-manual/configuration.md @@ -51,19 +51,21 @@ allow_punycode = false [consul_discovery] api = "catalog" -consul_http_addr = "http://127.0.0.1:8500" +consul_http_addr = "https://127.0.0.1:8500" +tls_skip_verify = false service_name = "garage-daemon" + ca_cert = "/etc/consul/consul-ca.crt" client_cert = "/etc/consul/consul-client.crt" client_key = "/etc/consul/consul-key.crt" + # for `agent` API mode, unset client_cert and client_key, and optionally enable `token` # token = "abcdef-01234-56789" -tls_skip_verify = false + tags = [ "dns-enabled" ] meta = { dns-acl = "allow trusted" } -# If your consul cluster is in a WAN configuration, you can provide the datacenter names to allow garage to do discovery across a WAN federation. -# This is not required for non-WAN consul instances. -# datacenters = ["dc1", "dc2", "dc3"] +datacenters = ["dc1", "dc2", "dc3"] + [kubernetes_discovery] namespace = "garage" service_name = "garage-daemon" @@ -730,14 +732,19 @@ node_prefix "" { ``` -### `consul datacenters` {#consul_datacenters} +#### `datacenters` {#consul_datacenters} -Optional list of datacenters that allow garage to do service discovery when consul is configured in WAN federation. -e.g datacenters = ["dc1", "dc2", "dc3"] -In a WAN configuration the consul services API only responds with local `LAN` services. -This queries the consul server API by datacenter directly, allowing for garage to discover nodes across consul WAN. +Optional list of datacenters that allow garage to do service discovery when Consul is configured in WAN federation. + +Example: `datacenters = ["dc1", "dc2", "dc3"]` + +In a WAN configuration, by default the Consul services API only responds with +local LAN services. When a list of datacenters is specified using this option, +Garage will query the consul server API by datacenter directly, allowing for +Garage to discover nodes across the Consul WAN. #### `tags` and `meta` {#consul_tags_and_meta} + Additional list of tags and map of service meta to add during service registration. ### The `[kubernetes_discovery]` section diff --git a/src/rpc/consul.rs b/src/rpc/consul.rs index 54f75e74..f177ef95 100644 --- a/src/rpc/consul.rs +++ b/src/rpc/consul.rs @@ -141,7 +141,7 @@ impl ConsulDiscovery { let mut ret = vec![]; let dcs_to_query: Vec> = match &self.config.datacenters { - Some(dcs) if !dcs.is_empty() => dcs.iter().map(|dc| Some(dc.as_str())).collect(), + dcs if !dcs.is_empty() => dcs.iter().map(|dc| Some(dc.as_str())).collect(), _ => vec![None], }; diff --git a/src/util/config.rs b/src/util/config.rs index 29047f8c..83f07d03 100644 --- a/src/util/config.rs +++ b/src/util/config.rs @@ -258,7 +258,7 @@ pub struct ConsulDiscoveryConfig { #[serde(default)] pub meta: Option>, #[serde(default)] - pub datacenters: Option>, + pub datacenters: Vec, } #[derive(Deserialize, Debug, Clone)] From cf22e7b71df4546fb509a247a418e6b2f333c247 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Tue, 6 Jan 2026 14:35:51 +0100 Subject: [PATCH 4/4] reintroduce warning when invalid node id is present in consul --- src/rpc/consul.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/rpc/consul.rs b/src/rpc/consul.rs index f177ef95..f16a323e 100644 --- a/src/rpc/consul.rs +++ b/src/rpc/consul.rs @@ -169,6 +169,11 @@ impl ConsulDiscovery { .and_then(|k| NodeID::from_slice(&k[..])); if let (Some(ip), Some(pubkey)) = (ip, pubkey) { ret.push((pubkey, SocketAddr::new(ip, ent.service_port))); + } else { + warn!( + "Could not process node spec from Consul: {:?} (invalid IP address or node ID/pubkey)", + ent + ); } } }