Merge pull request 'Adding consul discovery for WAN federated consul servers' (#1252) from jamin/garage:feature/consul_wan_discovery_v2 into main-v2

Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1252

doc/book/reference-manual/configuration.md

@@ -51,17 +51,20 @@ allow_punycode = false
 
 [consul_discovery]
 api = "catalog"
-consul_http_addr = "http://127.0.0.1:8500"
+consul_http_addr = "https://127.0.0.1:8500"
+tls_skip_verify = false
 service_name = "garage-daemon"
+
 ca_cert = "/etc/consul/consul-ca.crt"
 client_cert = "/etc/consul/consul-client.crt"
 client_key = "/etc/consul/consul-key.crt"
+
 # for `agent` API mode, unset client_cert and client_key, and optionally enable `token`
 # token = "abcdef-01234-56789"
-tls_skip_verify = false
+
 tags = [ "dns-enabled" ]
 meta = { dns-acl = "allow trusted" }
-
+datacenters = ["dc1", "dc2", "dc3"]
 
 [kubernetes_discovery]
 namespace = "garage"
@@ -127,12 +130,14 @@ The `[consul_discovery]` section:
 [`client_cert`](#consul_client_cert_and_key),
 [`client_key`](#consul_client_cert_and_key),
 [`consul_http_addr`](#consul_http_addr),
+[`datacenters`](#consul_datacenters)
 [`meta`](#consul_tags_and_meta),
 [`service_name`](#consul_service_name),
 [`tags`](#consul_tags_and_meta),
 [`tls_skip_verify`](#consul_tls_skip_verify),
 [`token`](#consul_token).
 
+
 The `[kubernetes_discovery]` section:
 [`namespace`](#kube_namespace),
 [`service_name`](#kube_service_name),
@@ -726,6 +731,18 @@ node_prefix "" {
 }
 ```
 
+
+#### `datacenters` {#consul_datacenters}
+
+Optional list of datacenters that allow garage to do service discovery when Consul is configured in WAN federation.
+
+Example: `datacenters = ["dc1", "dc2", "dc3"]`
+
+In a WAN configuration, by default the Consul services API only responds with
+local LAN services.  When a list of datacenters is specified using this option,
+Garage will query the consul server API by datacenter directly, allowing for
+Garage to discover nodes across the Consul WAN.
+
 #### `tags` and `meta` {#consul_tags_and_meta}
 
 Additional list of tags and map of service meta to add during service registration.

src/rpc/consul.rs

@@ -126,38 +126,61 @@ impl ConsulDiscovery {
 	}
 
 	// ---- READING FROM CONSUL CATALOG ----
-
+	/// Query Consul for Garage nodes registered under the configured service name.
+	///
+	/// This method supports querying multiple Consul datacenters for WAN or
+	/// multi-datacenter deployments. If `config.datacenters` is set and non-empty,
+	/// each listed datacenter is queried and the results are aggregated. Otherwise,
+	/// only the local datacenter is queried. `config.datacenters` does not need to be set
+	/// when all the datacenters are on the same LAN, in this case service discovery works normally
+	///
+	/// # Returns
+	/// A list of `(NodeID, SocketAddr)` pairs corresponding to all valid discovered
+	/// nodes across the queried datacenters.
 	pub async fn get_consul_nodes(&self) -> Result<Vec<(NodeID, SocketAddr)>, ConsulError> {
-		let url = format!(
-			"{}/v1/catalog/service/{}",
-			self.config.consul_http_addr, self.config.service_name
-		);
-
-		let http = self.client.get(&url).send().await?;
-		let entries: Vec<ConsulQueryEntry> = http.json().await?;
-
 		let mut ret = vec![];
-		for ent in entries {
-			let ip = ent.address.parse::<IpAddr>().ok();
-			let pubkey = ent
-				.meta
-				.get(&format!("{}-pubkey", META_PREFIX))
-				.and_then(|k| hex::decode(k).ok())
-				.and_then(|k| NodeID::from_slice(&k[..]));
-			if let (Some(ip), Some(pubkey)) = (ip, pubkey) {
-				ret.push((pubkey, SocketAddr::new(ip, ent.service_port)));
-			} else {
-				warn!(
-					"Could not process node spec from Consul: {:?} (invalid IP address or node ID/pubkey)",
-					ent
-				);
+
+		let dcs_to_query: Vec<Option<&str>> = match &self.config.datacenters {
+			dcs if !dcs.is_empty() => dcs.iter().map(|dc| Some(dc.as_str())).collect(),
+			_ => vec![None],
+		};
+
+		for dc in dcs_to_query {
+			let url = match dc {
+				Some(datacenter) => format!(
+					"{}/v1/catalog/service/{}?dc={}",
+					self.config.consul_http_addr, self.config.service_name, datacenter
+				),
+				None => format!(
+					"{}/v1/catalog/service/{}",
+					self.config.consul_http_addr, self.config.service_name
+				),
+			};
+
+			let http = self.client.get(&url).send().await?;
+			let entries: Vec<ConsulQueryEntry> = http.json().await?;
+
+			for ent in entries {
+				let ip = ent.address.parse::<IpAddr>().ok();
+				let pubkey = ent
+					.meta
+					.get(&format!("{}-pubkey", META_PREFIX))
+					.and_then(|k| hex::decode(k).ok())
+					.and_then(|k| NodeID::from_slice(&k[..]));
+				if let (Some(ip), Some(pubkey)) = (ip, pubkey) {
+					ret.push((pubkey, SocketAddr::new(ip, ent.service_port)));
+				} else {
+					warn!(
+						"Could not process node spec from Consul: {:?} (invalid IP address or node ID/pubkey)",
+						ent
+					);
+				}
 			}
 		}
-		debug!("Got nodes from Consul: {:?}", ret);
 
+		debug!("Got {} nodes from Consul", ret.len());
 		Ok(ret)
 	}
-
 	// ---- PUBLISHING TO CONSUL CATALOG ----
 
 	pub async fn publish_consul_service(

src/util/config.rs

@@ -257,6 +257,8 @@ pub struct ConsulDiscoveryConfig {
 	/// Additional service metadata to add
 	#[serde(default)]
 	pub meta: Option<std::collections::HashMap<String, String>>,
+	#[serde(default)]
+	pub datacenters: Vec<String>,
 }
 
 #[derive(Deserialize, Debug, Clone)]