do not assume Garage boots in 2sec during tests

Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/521

Cargo.lock

@@ -1309,6 +1309,7 @@ dependencies = [
  "opentelemetry",
  "rand",
  "rmp-serde",
+ "rustc_version",
  "serde",
  "serde_json",
  "sha2 0.10.6",

Cargo.nix

@@ -32,7 +32,7 @@ args@{
   ignoreLockHash,
 }:
 let
-  nixifiedLockHash = "9941a539c374cfd67691e633af6c6a5da9e8badbb7b14159b3ff24a7cf5ddf8a";
+  nixifiedLockHash = "0ee5ebe74e19a2d2d3c41b80ed443c93f079f696681902dda4ca659be282d985";
   workspaceSrc = if args.workspaceSrc == null then ./. else args.workspaceSrc;
   currentLockHash = builtins.hashFile "sha256" (workspaceSrc + /Cargo.lock);
   lockHashIgnored = if ignoreLockHash
@@ -1861,6 +1861,9 @@ in
     devDependencies = {
       mktemp = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".mktemp."0.5.0" { inherit profileName; }).out;
     };
+    buildDependencies = {
+      rustc_version = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".rustc_version."0.4.0" { profileName = "__noProfile"; }).out;
+    };
   });
   
   "unknown".garage_web."0.8.1" = overridableMkRustCrate (profileName: rec {

doc/book/cookbook/_index.md

@@ -6,7 +6,7 @@ sort_by = "weight"
 +++
 
 A cookbook, when you cook, is a collection of recipes.
-Similarly, Garage's cookbook contains a collection of recipes that are known to works well!
+Similarly, Garage's cookbook contains a collection of recipes that are known to work well!
 This chapter could also be referred as "Tutorials" or "Best practices".
 
 - **[Multi-node deployment](@/documentation/cookbook/real-world.md):** This page will walk you through all of the necessary
@@ -16,6 +16,10 @@ This chapter could also be referred as "Tutorials" or "Best practices".
   source in case a binary is not provided for your architecture, or if you want to
   hack with us!
 
+- **[Binary packages](@/documentation/cookbook/binary-packages.md):** This page
+  lists the different platforms that provide ready-built software packages for
+  Garage.
+
 - **[Integration with Systemd](@/documentation/cookbook/systemd.md):** This page explains how to run Garage
   as a Systemd service (instead of as a Docker container).
 
@@ -26,6 +30,10 @@ This chapter could also be referred as "Tutorials" or "Best practices".
 
 - **[Configuring a reverse-proxy](@/documentation/cookbook/reverse-proxy.md):** This page explains how to configure a reverse-proxy to add TLS support to your S3 api endpoint.
 
+- **[Deploying on Kubernetes](@/documentation/cookbook/kubernetes.md):** This page explains how to deploy Garage on Kubernetes using our Helm chart.
+
+- **[Deploying with Ansible](@/documentation/cookbook/ansible.md):** This page lists available Ansible roles developed by the community to deploy Garage.
+
 - **[Monitoring Garage](@/documentation/cookbook/monitoring.md)** This page
   explains the Prometheus metrics available for monitoring the Garage
   cluster/nodes.

doc/book/cookbook/ansible.md

@@ -0,0 +1,51 @@
++++
+title = "Deploying with Ansible"
+weight = 35
++++
+
+While Ansible is not officially supported to deploy Garage, several community members
+have published Ansible roles.  We list them and compare them below.
+
+## Comparison of Ansible roles
+
+| Feature                            | [ansible-role-garage](#zorun-ansible-role-garage) | [garage-docker-ansible-deploy](#moan0s-garage-docker-ansible-deploy) |
+|------------------------------------|---------------------------------------------|---------------------------------------------------------------|
+| **Runtime**                        | Systemd                                     | Docker                                                        |
+| **Target OS**                      | Any Linux                                   | Any Linux                                                     |
+| **Architecture**                   | amd64, arm64, i686                          | amd64, arm64                                                  |
+| **Additional software**            | None                                        | Traefik                                                       |
+| **Automatic node connection**      | ❌                                          | ✅                                                            |
+| **Layout management**              | ❌                                          | ✅                                                            |
+| **Manage buckets & keys**          | ❌                                          | ✅ (basic)                                                    |
+| **Allow custom Garage config**     | ✅                                          | ❌                                                            |
+| **Facilitate Garage upgrades**     | ✅                                          | ❌                                                            |
+| **Multiple instances on one host** | ✅                                          | ✅                                                            |
+
+
+## zorun/ansible-role-garage
+
+[Source code](https://github.com/zorun/ansible-role-garage), [Ansible galaxy](https://galaxy.ansible.com/zorun/garage)
+
+This role is voluntarily simple: it relies on the official Garage static
+binaries and only requires Systemd.  As such, it should work on any
+Linux-based OS.
+
+To make things more flexible, the user has to provide a Garage
+configuration template.  This allows to customize Garage configuration in
+any way.
+
+Some more features might be added, such as a way to automatically connect
+nodes to each other or to define a layout.
+
+## moan0s/garage-docker-ansible-deploy
+
+[Source code](https://github.com/moan0s/garage-docker-ansible-deploy), [Blog post](https://hyteck.de/post/garage/)
+
+This role is based on the Docker image for Garage, and comes with
+"batteries included": it will additionally install Docker and Traefik. In
+addition, it is "opinionated" in the sense that it expects a particular
+deployment structure (one instance per disk, one gateway per host,
+structured DNS names, etc).
+
+As a result, this role makes it easier to start with Garage on Ansible,
+but is less flexible.

doc/book/cookbook/binary-packages.md

@@ -0,0 +1,28 @@
++++
+title = "Binary packages"
+weight = 11
++++
+
+Garage is also available in binary packages on:
+
+## Alpine Linux
+
+```bash
+apk install garage
+```
+
+## Arch Linux
+
+Garage is available in the [AUR](https://aur.archlinux.org/packages/garage).
+
+## FreeBSD
+
+```bash
+pkg install garage
+```
+
+## NixOS
+
+```bash
+nix-shell -p garage
+```

src/api/admin/cluster.rs

@@ -20,6 +20,7 @@ pub async fn handle_get_cluster_status(garage: &Arc<Garage>) -> Result<Response<
 		node: hex::encode(garage.system.id),
 		garage_version: garage_util::version::garage_version(),
 		garage_features: garage_util::version::garage_features(),
+		rust_version: garage_util::version::rust_version(),
 		db_engine: garage.db.engine(),
 		known_nodes: garage
 			.system
@@ -106,6 +107,7 @@ struct GetClusterStatusResponse {
 	node: String,
 	garage_version: &'static str,
 	garage_features: Option<&'static [&'static str]>,
+	rust_version: &'static str,
 	db_engine: String,
 	known_nodes: HashMap<String, KnownNodeResp>,
 	layout: GetClusterLayoutResponse,

src/api/generic_server.rs

@@ -19,6 +19,7 @@ use opentelemetry::{
 };
 
 use garage_util::error::Error as GarageError;
+use garage_util::forwarded_headers;
 use garage_util::metrics::{gen_trace_id, RecordDuration};
 
 pub(crate) trait ApiEndpoint: Send + Sync + 'static {
@@ -126,15 +127,9 @@ impl<A: ApiHandler> ApiServer<A> {
 	) -> Result<Response<Body>, GarageError> {
 		let uri = req.uri().clone();
 
-		let has_forwarded_for_header = req.headers().contains_key("x-forwarded-for");
+		if let Ok(forwarded_for_ip_addr) =
-		if has_forwarded_for_header {
+			forwarded_headers::handle_forwarded_for_headers(&req.headers())
-			let forwarded_for_ip_addr = &req
+		{
-				.headers()
-				.get("x-forwarded-for")
-				.expect("Could not parse X-Forwarded-For header")
-				.to_str()
-				.unwrap_or_default();
-
 			info!(
 				"{} (via {}) {} {}",
 				forwarded_for_ip_addr,

src/block/manager.rs

@@ -185,6 +185,9 @@ impl BlockManager {
 		vars.register_ro(&self.scrub_persister, "scrub-last-completed", |p| {
 			p.get_with(|x| msec_to_rfc3339(x.time_last_complete_scrub))
 		});
+		vars.register_ro(&self.scrub_persister, "scrub-next-run", |p| {
+			p.get_with(|x| msec_to_rfc3339(x.time_next_run_scrub))
+		});
 		vars.register_ro(&self.scrub_persister, "scrub-corruptions_detected", |p| {
 			p.get_with(|x| x.corruptions_detected)
 		});

src/block/repair.rs

@@ -4,6 +4,7 @@ use std::sync::Arc;
 use std::time::Duration;
 
 use async_trait::async_trait;
+use rand::Rng;
 use serde::{Deserialize, Serialize};
 use tokio::fs;
 use tokio::select;
@@ -19,8 +20,8 @@ use garage_util::tranquilizer::Tranquilizer;
 
 use crate::manager::*;
 
-// Full scrub every 30 days
+// Full scrub every 25 days with a random element of 10 days mixed in below
-const SCRUB_INTERVAL: Duration = Duration::from_secs(3600 * 24 * 30);
+const SCRUB_INTERVAL: Duration = Duration::from_secs(3600 * 24 * 25);
 // Scrub tranquility is initially set to 4, but can be changed in the CLI
 // and the updated version is persisted over Garage restarts
 const INITIAL_SCRUB_TRANQUILITY: u32 = 4;
@@ -175,13 +176,30 @@ pub struct ScrubWorker {
 pub struct ScrubWorkerPersisted {
 	pub tranquility: u32,
 	pub(crate) time_last_complete_scrub: u64,
+	pub(crate) time_next_run_scrub: u64,
 	pub(crate) corruptions_detected: u64,
 }
+
+fn randomize_next_scrub_run_time() -> u64 {
+	// Take SCRUB_INTERVAL and mix in a random interval of 10 days to attempt to
+	// balance scrub load across different cluster nodes.
+
+	let next_run_timestamp = now_msec()
+		+ SCRUB_INTERVAL
+			.saturating_add(Duration::from_secs(
+				rand::thread_rng().gen_range(0..3600 * 24 * 10),
+			))
+			.as_millis() as u64;
+
+	next_run_timestamp
+}
+
 impl garage_util::migrate::InitialFormat for ScrubWorkerPersisted {}
 impl Default for ScrubWorkerPersisted {
 	fn default() -> Self {
 		ScrubWorkerPersisted {
 			time_last_complete_scrub: 0,
+			time_next_run_scrub: randomize_next_scrub_run_time(),
 			tranquility: INITIAL_SCRUB_TRANQUILITY,
 			corruptions_detected: 0,
 		}
@@ -279,12 +297,13 @@ impl Worker for ScrubWorker {
 	}
 
 	fn status(&self) -> WorkerStatus {
-		let (corruptions_detected, tranquility, time_last_complete_scrub) =
+		let (corruptions_detected, tranquility, time_last_complete_scrub, time_next_run_scrub) =
 			self.persister.get_with(|p| {
 				(
 					p.corruptions_detected,
 					p.tranquility,
 					p.time_last_complete_scrub,
+					p.time_next_run_scrub,
 				)
 			});
 
@@ -302,10 +321,16 @@ impl Worker for ScrubWorker {
 				s.freeform = vec![format!("Scrub paused, resumes at {}", msec_to_rfc3339(*rt))];
 			}
 			ScrubWorkerState::Finished => {
-				s.freeform = vec![format!(
+				s.freeform = vec![
-					"Last scrub completed at {}",
+					format!(
-					msec_to_rfc3339(time_last_complete_scrub)
+						"Last scrub completed at {}",
-				)];
+						msec_to_rfc3339(time_last_complete_scrub),
+					),
+					format!(
+						"Next scrub scheduled for {}",
+						msec_to_rfc3339(time_next_run_scrub)
+					),
+				];
 			}
 		}
 		s
@@ -334,8 +359,10 @@ impl Worker for ScrubWorker {
 						.tranquilizer
 						.tranquilize_worker(self.persister.get_with(|p| p.tranquility)))
 				} else {
-					self.persister
+					self.persister.set_with(|p| {
-						.set_with(|p| p.time_last_complete_scrub = now_msec())?;
+						p.time_last_complete_scrub = now_msec();
+						p.time_next_run_scrub = randomize_next_scrub_run_time();
+					})?;
 					self.work = ScrubWorkerState::Finished;
 					self.tranquilizer.clear();
 					Ok(WorkerState::Idle)
@@ -350,8 +377,7 @@ impl Worker for ScrubWorker {
 			ScrubWorkerState::Running(_) => return WorkerState::Busy,
 			ScrubWorkerState::Paused(_, resume_time) => (*resume_time, ScrubWorkerCommand::Resume),
 			ScrubWorkerState::Finished => (
-				self.persister.get_with(|p| p.time_last_complete_scrub)
+				self.persister.get_with(|p| p.time_next_run_scrub),
-					+ SCRUB_INTERVAL.as_millis() as u64,
 				ScrubWorkerCommand::Start,
 			),
 		};

src/garage/admin.rs

@@ -820,11 +820,12 @@ impl AdminRpcHandler {
 		let mut ret = String::new();
 		writeln!(
 			&mut ret,
-			"\nGarage version: {} [features: {}]",
+			"\nGarage version: {} [features: {}]\nRust compiler version: {}",
 			garage_util::version::garage_version(),
 			garage_util::version::garage_features()
 				.map(|list| list.join(", "))
 				.unwrap_or_else(|| "(unknown)".into()),
+			garage_util::version::rust_version(),
 		)
 		.unwrap();
 

src/garage/tests/common/garage.rs

@@ -111,14 +111,27 @@ api_bind_addr = "127.0.0.1:{admin_port}"
 	}
 
 	fn setup(&mut self) {
+		self.wait_for_boot();
+		self.setup_layout();
+		self.key = self.new_key("garage_test");
+	}
+
+	fn wait_for_boot(&mut self) {
 		use std::{thread, time::Duration};
 
-		// Wait for node to be ready
+		// 60 * 2 seconds = 120 seconds = 2min
-		thread::sleep(Duration::from_secs(2));
+		for _ in 0..60 {
-
+			let termination = self
-		self.setup_layout();
+				.command()
-
+				.args(["status"])
-		self.key = self.new_key("garage_test");
+				.quiet()
+				.status()
+				.expect("Unable to run command");
+			if termination.success() {
+				break;
+			}
+			thread::sleep(Duration::from_secs(2));
+		}
 	}
 
 	fn setup_layout(&self) {

src/model/garage.rs

@@ -136,9 +136,16 @@ impl Garage {
 					env_builder.flag(heed::flags::Flags::MdbNoSync);
 					env_builder.flag(heed::flags::Flags::MdbNoMetaSync);
 				}
-				let db = env_builder
+				let db = match env_builder.open(&db_path) {
-					.open(&db_path)
+					Err(heed::Error::Io(e)) if e.kind() == std::io::ErrorKind::OutOfMemory => {
-					.ok_or_message("Unable to open LMDB DB")?;
+						return Err(Error::Message(
+							"OutOfMemory error while trying to open LMDB database. This can happen \
+							if your operating system is not allowing you to use sufficient virtual \
+							memory address space. Please check that no limit is set (ulimit -v). \
+							On 32-bit machines, you should probably switch to another database engine.".into()))
+					}
+					x => x.ok_or_message("Unable to open LMDB DB")?,
+				};
 				db::lmdb_adapter::LmdbDb::init(db)
 			}
 			#[cfg(not(feature = "lmdb"))]

src/rpc/system_metrics.rs

@@ -31,10 +31,10 @@ impl SystemMetrics {
 				.u64_value_observer("garage_build_info", move |observer| {
 					observer.observe(
 						1,
-						&[KeyValue::new(
+						&[
-							"version",
+							KeyValue::new("rustversion", garage_util::version::rust_version()),
-							garage_util::version::garage_version(),
+							KeyValue::new("version", garage_util::version::garage_version()),
-						)],
+						],
 					)
 				})
 				.with_description("Garage build info")

src/util/Cargo.toml

@@ -47,6 +47,9 @@ hyper = "0.14"
 
 opentelemetry = { version = "0.17", features = [ "rt-tokio", "metrics", "trace" ] }
 
+[build-dependencies]
+rustc_version = "0.4.0"
+
 [dev-dependencies]
 mktemp = "0.5"
 

src/util/build.rs

@@ -0,0 +1,8 @@
+use rustc_version::version;
+
+fn main() {
+	// Acquire the version of Rust used to compile, this is added as a label to
+	// the garage_build_info metric.
+	let v = version().unwrap();
+	println!("cargo:rustc-env=RUSTC_VERSION={v}");
+}

src/util/forwarded_headers.rs

@@ -0,0 +1,63 @@
+use http::{HeaderMap, HeaderValue};
+use std::net::IpAddr;
+use std::str::FromStr;
+
+use crate::error::{Error, OkOrMessage};
+
+pub fn handle_forwarded_for_headers(headers: &HeaderMap<HeaderValue>) -> Result<String, Error> {
+	let forwarded_for_header = headers
+		.get("x-forwarded-for")
+		.ok_or_message("X-Forwarded-For header not provided")?;
+
+	let forwarded_for_ip_str = forwarded_for_header
+		.to_str()
+		.ok_or_message("Error parsing X-Forwarded-For header")?;
+
+	let client_ip = IpAddr::from_str(&forwarded_for_ip_str)
+		.ok_or_message("Valid IP address not found in X-Forwarded-For header")?;
+
+	Ok(client_ip.to_string())
+}
+
+#[cfg(test)]
+mod test {
+	use super::*;
+
+	#[test]
+	fn test_handle_forwarded_for_headers_ipv4_client() {
+		let mut test_headers = HeaderMap::new();
+		test_headers.insert("X-Forwarded-For", "192.0.2.100".parse().unwrap());
+
+		if let Ok(forwarded_ip) = handle_forwarded_for_headers(&test_headers) {
+			assert_eq!(forwarded_ip, "192.0.2.100");
+		}
+	}
+
+	#[test]
+	fn test_handle_forwarded_for_headers_ipv6_client() {
+		let mut test_headers = HeaderMap::new();
+		test_headers.insert("X-Forwarded-For", "2001:db8::f00d:cafe".parse().unwrap());
+
+		if let Ok(forwarded_ip) = handle_forwarded_for_headers(&test_headers) {
+			assert_eq!(forwarded_ip, "2001:db8::f00d:cafe");
+		}
+	}
+
+	#[test]
+	fn test_handle_forwarded_for_headers_invalid_ip() {
+		let mut test_headers = HeaderMap::new();
+		test_headers.insert("X-Forwarded-For", "www.example.com".parse().unwrap());
+
+		let result = handle_forwarded_for_headers(&test_headers);
+		assert!(result.is_err());
+	}
+
+	#[test]
+	fn test_handle_forwarded_for_headers_missing() {
+		let mut test_headers = HeaderMap::new();
+		test_headers.insert("Host", "www.deuxfleurs.fr".parse().unwrap());
+
+		let result = handle_forwarded_for_headers(&test_headers);
+		assert!(result.is_err());
+	}
+}

src/util/lib.rs

@@ -11,6 +11,7 @@ pub mod data;
 pub mod encode;
 pub mod error;
 pub mod formater;
+pub mod forwarded_headers;
 pub mod metrics;
 pub mod migrate;
 pub mod persister;

src/util/version.rs

@@ -26,3 +26,7 @@ pub fn init_version(version: &'static str) {
 pub fn init_features(features: &'static [&'static str]) {
 	FEATURES.store(Some(Arc::new(features)));
 }
+
+pub fn rust_version() -> &'static str {
+	env!("RUSTC_VERSION")
+}

src/web/web_server.rs

@@ -29,6 +29,7 @@ use garage_model::garage::Garage;
 
 use garage_table::*;
 use garage_util::error::Error as GarageError;
+use garage_util::forwarded_headers;
 use garage_util::metrics::{gen_trace_id, RecordDuration};
 
 struct WebMetrics {
@@ -104,7 +105,19 @@ impl WebServer {
 		req: Request<Body>,
 		addr: SocketAddr,
 	) -> Result<Response<Body>, Infallible> {
-		info!("{} {} {}", addr, req.method(), req.uri());
+		if let Ok(forwarded_for_ip_addr) =
+			forwarded_headers::handle_forwarded_for_headers(&req.headers())
+		{
+			info!(
+				"{} (via {}) {} {}",
+				forwarded_for_ip_addr,
+				addr,
+				req.method(),
+				req.uri()
+			);
+		} else {
+			info!("{} {} {}", addr, req.method(), req.uri());
+		}
 
 		// Lots of instrumentation
 		let tracer = opentelemetry::global::tracer("garage");