deregister from consul on shutdown (fix #1062) (#1507)

Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1507

src/garage/server.rs

@@ -152,6 +152,14 @@ pub async fn run_server(
 		}
 	}
 
+	// Deregister from Consul (if enabled) in the background, in parallel with the
+	// rest of the shutdown sequence, so that it doesn't add to shutdown latency.
+	#[cfg(feature = "consul-discovery")]
+	let deregister_consul_task = tokio::spawn({
+		let system = garage.system.clone();
+		async move { system.deregister_from_discovery().await }
+	});
+
 	// Remove RPC handlers for system to break reference cycles
 	info!("Deregistering RPC handlers for shutdown...");
 	garage.system.netapp.drop_all_handlers();
@@ -168,6 +176,12 @@ pub async fn run_server(
 	// Await for all background tasks to end
 	await_background_done.await?;
 
+	// Await for Consul deregistration to end, if it hasn't already
+	#[cfg(feature = "consul-discovery")]
+	if let Err(e) = deregister_consul_task.await {
+		error!("Error while joining Consul deregistration task: {}", e);
+	}
+
 	info!("Cleaning up...");
 
 	Ok(())

src/rpc/consul.rs

@@ -183,6 +183,36 @@ impl ConsulDiscovery {
 	}
 	// ---- PUBLISHING TO CONSUL CATALOG ----
 
+	#[cfg(feature = "consul-discovery")]
+	pub async fn deregister_consul_service(&self, node_id: NodeID) -> Result<(), ConsulError> {
+		let node = format!("garage:{}", hex::encode(&node_id[..8]));
+		let url = format!(
+			"{}/v1/{}",
+			self.config.consul_http_addr,
+			(match &self.config.api {
+				ConsulDiscoveryAPI::Catalog => format!("catalog/deregister"),
+				ConsulDiscoveryAPI::Agent => format!("agent/service/deregister/{}", node),
+			})
+		);
+
+		let req = self.client.put(&url);
+
+		let http = if matches!(&self.config.api, ConsulDiscoveryAPI::Catalog) {
+			let deregister_request = serde_json::json!({
+				"Node": node,
+				"ServiceID": node,
+			});
+			let req = req.json(&deregister_request);
+			req.send().await?
+		} else {
+			req.send().await?
+		};
+		http.error_for_status()?;
+
+		debug!("Deregistered service {} from Consul", node);
+		Ok(())
+	}
+
 	pub async fn publish_consul_service(
 		&self,
 		node_id: NodeID,

src/rpc/system.rs

@@ -358,7 +358,7 @@ impl System {
 		);
 	}
 
-	pub fn cleanup(&self) {
+	pub fn cleanup(self: &Arc<Self>) {
 		// Break reference cycle
 		self.metrics.store(None);
 	}
@@ -650,6 +650,15 @@ impl System {
 		}
 	}
 
+	#[cfg(feature = "consul-discovery")]
+	pub async fn deregister_from_discovery(self: &Arc<Self>) {
+		if let Some(c) = &self.consul_discovery {
+			if let Err(e) = c.deregister_consul_service(self.netapp.id).await {
+				error!("Error while deregistering from Consul: {}", e);
+			}
+		}
+	}
+
 	async fn discovery_loop(self: &Arc<Self>, mut stop_signal: watch::Receiver<bool>) {
 		while !*stop_signal.borrow() {
 			let peers_up = self