diff --git a/doc/book/reference-manual/configuration.md b/doc/book/reference-manual/configuration.md index d26d9253..9b2d57b6 100644 --- a/doc/book/reference-manual/configuration.md +++ b/doc/book/reference-manual/configuration.md @@ -51,17 +51,20 @@ allow_punycode = false [consul_discovery] api = "catalog" -consul_http_addr = "http://127.0.0.1:8500" +consul_http_addr = "https://127.0.0.1:8500" +tls_skip_verify = false service_name = "garage-daemon" + ca_cert = "/etc/consul/consul-ca.crt" client_cert = "/etc/consul/consul-client.crt" client_key = "/etc/consul/consul-key.crt" + # for `agent` API mode, unset client_cert and client_key, and optionally enable `token` # token = "abcdef-01234-56789" -tls_skip_verify = false + tags = [ "dns-enabled" ] meta = { dns-acl = "allow trusted" } - +datacenters = ["dc1", "dc2", "dc3"] [kubernetes_discovery] namespace = "garage" @@ -127,12 +130,14 @@ The `[consul_discovery]` section: [`client_cert`](#consul_client_cert_and_key), [`client_key`](#consul_client_cert_and_key), [`consul_http_addr`](#consul_http_addr), +[`datacenters`](#consul_datacenters) [`meta`](#consul_tags_and_meta), [`service_name`](#consul_service_name), [`tags`](#consul_tags_and_meta), [`tls_skip_verify`](#consul_tls_skip_verify), [`token`](#consul_token). + The `[kubernetes_discovery]` section: [`namespace`](#kube_namespace), [`service_name`](#kube_service_name), @@ -726,6 +731,18 @@ node_prefix "" { } ``` + +#### `datacenters` {#consul_datacenters} + +Optional list of datacenters that allow garage to do service discovery when Consul is configured in WAN federation. + +Example: `datacenters = ["dc1", "dc2", "dc3"]` + +In a WAN configuration, by default the Consul services API only responds with +local LAN services. When a list of datacenters is specified using this option, +Garage will query the consul server API by datacenter directly, allowing for +Garage to discover nodes across the Consul WAN. + #### `tags` and `meta` {#consul_tags_and_meta} Additional list of tags and map of service meta to add during service registration. diff --git a/src/rpc/consul.rs b/src/rpc/consul.rs index 760e9fcb..f16a323e 100644 --- a/src/rpc/consul.rs +++ b/src/rpc/consul.rs @@ -126,38 +126,61 @@ impl ConsulDiscovery { } // ---- READING FROM CONSUL CATALOG ---- - + /// Query Consul for Garage nodes registered under the configured service name. + /// + /// This method supports querying multiple Consul datacenters for WAN or + /// multi-datacenter deployments. If `config.datacenters` is set and non-empty, + /// each listed datacenter is queried and the results are aggregated. Otherwise, + /// only the local datacenter is queried. `config.datacenters` does not need to be set + /// when all the datacenters are on the same LAN, in this case service discovery works normally + /// + /// # Returns + /// A list of `(NodeID, SocketAddr)` pairs corresponding to all valid discovered + /// nodes across the queried datacenters. pub async fn get_consul_nodes(&self) -> Result, ConsulError> { - let url = format!( - "{}/v1/catalog/service/{}", - self.config.consul_http_addr, self.config.service_name - ); - - let http = self.client.get(&url).send().await?; - let entries: Vec = http.json().await?; - let mut ret = vec![]; - for ent in entries { - let ip = ent.address.parse::().ok(); - let pubkey = ent - .meta - .get(&format!("{}-pubkey", META_PREFIX)) - .and_then(|k| hex::decode(k).ok()) - .and_then(|k| NodeID::from_slice(&k[..])); - if let (Some(ip), Some(pubkey)) = (ip, pubkey) { - ret.push((pubkey, SocketAddr::new(ip, ent.service_port))); - } else { - warn!( - "Could not process node spec from Consul: {:?} (invalid IP address or node ID/pubkey)", - ent - ); + + let dcs_to_query: Vec> = match &self.config.datacenters { + dcs if !dcs.is_empty() => dcs.iter().map(|dc| Some(dc.as_str())).collect(), + _ => vec![None], + }; + + for dc in dcs_to_query { + let url = match dc { + Some(datacenter) => format!( + "{}/v1/catalog/service/{}?dc={}", + self.config.consul_http_addr, self.config.service_name, datacenter + ), + None => format!( + "{}/v1/catalog/service/{}", + self.config.consul_http_addr, self.config.service_name + ), + }; + + let http = self.client.get(&url).send().await?; + let entries: Vec = http.json().await?; + + for ent in entries { + let ip = ent.address.parse::().ok(); + let pubkey = ent + .meta + .get(&format!("{}-pubkey", META_PREFIX)) + .and_then(|k| hex::decode(k).ok()) + .and_then(|k| NodeID::from_slice(&k[..])); + if let (Some(ip), Some(pubkey)) = (ip, pubkey) { + ret.push((pubkey, SocketAddr::new(ip, ent.service_port))); + } else { + warn!( + "Could not process node spec from Consul: {:?} (invalid IP address or node ID/pubkey)", + ent + ); + } } } - debug!("Got nodes from Consul: {:?}", ret); + debug!("Got {} nodes from Consul", ret.len()); Ok(ret) } - // ---- PUBLISHING TO CONSUL CATALOG ---- pub async fn publish_consul_service( diff --git a/src/util/config.rs b/src/util/config.rs index a4521eb5..83f07d03 100644 --- a/src/util/config.rs +++ b/src/util/config.rs @@ -257,6 +257,8 @@ pub struct ConsulDiscoveryConfig { /// Additional service metadata to add #[serde(default)] pub meta: Option>, + #[serde(default)] + pub datacenters: Vec, } #[derive(Deserialize, Debug, Clone)]