force flag "no read ahead" on LMDB

Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/846
Reviewed-by: Quentin <quentin@dufour.io>

Cargo.lock

@@ -1448,6 +1448,7 @@ dependencies = [
  "heed",
  "hexdump",
  "mktemp",
+ "opentelemetry",
  "r2d2",
  "r2d2_sqlite",
  "rusqlite",
@@ -1527,6 +1528,7 @@ dependencies = [
  "garage_util",
  "gethostname",
  "hex",
+ "ipnet",
  "itertools 0.12.1",
  "k8s-openapi",
  "kube",

Cargo.nix

@@ -34,7 +34,7 @@ args@{
   ignoreLockHash,
 }:
 let
-  nixifiedLockHash = "1ccd5eb25a83962821e0e9da4ce6df31717b2b97a5b3a0c80c9e0e0759710143";
+  nixifiedLockHash = "5826893fb6082581d96992c846c892c623f3b1b17cefba3b6b8ca10c9f45c589";
   workspaceSrc = if args.workspaceSrc == null then ./. else args.workspaceSrc;
   currentLockHash = builtins.hashFile "sha256" (workspaceSrc + /Cargo.lock);
   lockHashIgnored = if ignoreLockHash
@@ -2104,6 +2104,7 @@ in
       err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out;
       ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/lmdb" then "heed" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".heed."0.11.0" { inherit profileName; }).out;
       hexdump = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hexdump."0.1.1" { inherit profileName; }).out;
+      opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out;
       ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "r2d2" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".r2d2."0.8.10" { inherit profileName; }).out;
       ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "r2d2_sqlite" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".r2d2_sqlite."0.24.0" { inherit profileName; }).out;
       ${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "rusqlite" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rusqlite."0.31.0" { inherit profileName; }).out;
@@ -2219,6 +2220,7 @@ in
       garage_util = (rustPackages."unknown".garage_util."1.0.0" { inherit profileName; }).out;
       gethostname = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".gethostname."0.4.3" { inherit profileName; }).out;
       hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
+      ipnet = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".ipnet."2.9.0" { inherit profileName; }).out;
       itertools = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".itertools."0.12.1" { inherit profileName; }).out;
       ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/k8s-openapi" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "k8s_openapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".k8s-openapi."0.21.0" { inherit profileName; }).out;
       ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "kube" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".kube."0.88.1" { inherit profileName; }).out;
@@ -3016,8 +3018,8 @@ in
     registry = "registry+https://github.com/rust-lang/crates.io-index";
     src = fetchCratesIo { inherit name version; sha256 = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3"; };
     features = builtins.concatLists [
-      (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "default")
-      (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "std")
+      [ "default" ]
+      [ "std" ]
     ];
   });
   

Cargo.toml

@@ -55,6 +55,7 @@ hexdump = "0.1"
 hmac = "0.12"
 idna = "0.5"
 itertools = "0.12"
+ipnet = "2.9.0"
 lazy_static = "1.4"
 md-5 = "0.10"
 mktemp = "0.5"

doc/book/cookbook/real-world.md

@@ -152,6 +152,8 @@ Check the following for your configuration files:
 - Make sure `rpc_public_addr` contains the public IP address of the node you are configuring.
   This parameter is optional but recommended: if your nodes have trouble communicating with
   one another, consider adding it.
+  Alternatively, you can also set `rpc_public_addr_subnet`, which can filter
+  the addresses announced to other peers to a specific subnet.
 
 - Make sure `rpc_secret` is the same value on all nodes. It should be a 32-bytes hex-encoded secret key.
   You can generate such a key with `openssl rand -hex 32`.

doc/book/design/related-work.md

@@ -67,7 +67,7 @@ Pithos has been abandonned and should probably not used yet, in the following we
 Pithos was relying as a S3 proxy in front of Cassandra (and was working with Scylla DB too).
 From its designers' mouth, storing data in Cassandra has shown its limitations justifying the project abandonment.
 They built a closed-source version 2 that does not store blobs in the database (only metadata) but did not communicate further on it.
-We considered there v2's design but concluded that it does not fit both our *Self-contained & lightweight* and *Simple* properties. It makes the development, the deployment and the operations more complicated while reducing the flexibility.
+We considered their v2's design but concluded that it does not fit both our *Self-contained & lightweight* and *Simple* properties. It makes the development, the deployment and the operations more complicated while reducing the flexibility.
 
 **[Riak CS](https://docs.riak.com/riak/cs/2.1.1/index.html):**
 *Not written yet*

doc/book/reference-manual/configuration.md

@@ -31,6 +31,9 @@ rpc_secret = "4425f5c26c5e11581d3223904324dcb5b5d5dfb14e5e7f35e38c595424f5f1e6"
 rpc_bind_addr = "[::]:3901"
 rpc_bind_outgoing = false
 rpc_public_addr = "[fc00:1::1]:3901"
+# or set rpc_public_adr_subnet to filter down autodiscovery to a subnet:
+# rpc_public_addr_subnet = "2001:0db8:f00:b00:/64"
+
 
 allow_world_readable_secrets = false
 
@@ -105,6 +108,7 @@ Top-level configuration options:
 [`rpc_bind_addr`](#rpc_bind_addr),
 [`rpc_bind_outgoing`](#rpc_bind_outgoing),
 [`rpc_public_addr`](#rpc_public_addr),
+[`rpc_public_addr_subnet`](#rpc_public_addr_subnet)
 [`rpc_secret`/`rpc_secret_file`](#rpc_secret).
 
 The `[consul_discovery]` section:
@@ -295,7 +299,7 @@ Since `v0.8.0`, Garage can use alternative storage backends as follows:
 
 | DB engine | `db_engine` value | Database path |
 | --------- | ----------------- | ------------- |
-| [LMDB](https://www.lmdb.tech) (since `v0.8.0`, default since `v0.9.0`) | `"lmdb"` | `<metadata_dir>/db.lmdb/` |
+| [LMDB](https://www.symas.com/lmdb) (since `v0.8.0`, default since `v0.9.0`) | `"lmdb"` | `<metadata_dir>/db.lmdb/` |
 | [Sqlite](https://sqlite.org) (since `v0.8.0`) | `"sqlite"` | `<metadata_dir>/db.sqlite` |
 | [Sled](https://sled.rs) (old default, removed since `v1.0`) | `"sled"` | `<metadata_dir>/db/` |
 
@@ -414,7 +418,7 @@ month, with a random delay to avoid all nodes running at the same time.  When
 it scrubs the data directory, Garage will read all of the data files stored on
 disk to check their integrity, and will rebuild any data files that it finds
 corrupted, using the remaining valid copies stored on other nodes.
-See [this page](@/documentation/operations/durability-repair.md#scrub) for details.
+See [this page](@/documentation/operations/durability-repairs.md#scrub) for details.
 
 Set the `disable_scrub` configuration value to `true` if you don't need Garage
 to scrub the data directory, for instance if you are already scrubbing at the
@@ -543,6 +547,14 @@ RPC calls.  **This parameter is optional but recommended.** In case you have
 a NAT that binds the RPC port to a port that is different on your public IP,
 this field might help making it work.
 
+#### `rpc_public_addr_subnet` {#rpc_public_addr_subnet}
+In case `rpc_public_addr` is not set, but autodiscovery is used, this allows
+filtering the list of automatically discovered IPs to a specific subnet.
+
+For example, if nodes should pick *their* IP inside a specific subnet, but you
+don't want to explicitly write the IP down (as it's dynamic, or you want to
+share configs across nodes), you can use this option.
+
 #### `bootstrap_peers` {#bootstrap_peers}
 
 A list of peer identifiers on which to contact other Garage peers of this cluster.

doc/book/working-documents/migration-1.md

@@ -19,15 +19,15 @@ The following are **breaking changes** in Garage v1.0 that require your attentio
 - The Sled metadata db engine has been **removed**. If your cluster was still
   using Sled, you will need to **use a Garage v0.9.x binary** to convert the
   database using the `garage convert-db` subcommand. See
-  [here](@/documentation/reference-manual/configuration/#db_engine) for the
+  [here](@/documentation/reference-manual/configuration.md#db_engine) for the
   details of the procedure.
 
 The following syntax changes have been made to the configuration file:
 
 - The `replication_mode` parameter has been split into two parameters:
-  [`replication_factor`](@/documentation/reference-manual/configuration/#replication_factor)
+  [`replication_factor`](@/documentation/reference-manual/configuration.md#replication_factor)
   and
-  [`consistency_mode`](@/documentation/reference-manual/configuration/#consistency_mode).
+  [`consistency_mode`](@/documentation/reference-manual/configuration.md#consistency_mode).
   The old syntax using `replication_mode` is still supported for legacy
   reasons and can still be used.
 

flake.nix

@@ -76,6 +76,7 @@
             # import the full shell using `nix develop .#full`
             full = shellWithPackages (with pkgs; [
               rustfmt
+              rust-analyzer
               clang
               mold
               # ---- extra packages for dev tasks ----

script/helm/garage/templates/workload.yaml

@@ -11,6 +11,7 @@ spec:
   {{- if eq .Values.deployment.kind "StatefulSet" }}
   replicas: {{ .Values.deployment.replicaCount }}
   serviceName: {{ include "garage.fullname" . }}
+  podManagementPolicy: {{ .Values.deployment.podManagementPolicy }}
   {{- end }}
   template:
     metadata:
@@ -63,6 +64,10 @@ spec:
               name: web-api
             - containerPort: 3903
               name: admin
+          {{- with .Values.environment }}
+          env:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
           volumeMounts:
             - name: meta
               mountPath: /mnt/meta

script/helm/garage/values.yaml

@@ -96,6 +96,8 @@ deployment:
   kind: StatefulSet
   # Number of StatefulSet replicas/garage nodes to start
   replicaCount: 3
+  # If using statefulset, allow Parallel or OrderedReady (default)
+  podManagementPolicy: OrderedReady
 
 image:
   repository: dxflrs/amd64_garage
@@ -214,6 +216,8 @@ tolerations: []
 
 affinity: {}
 
+environment: {}
+
 monitoring:
   metrics:
     # If true, a service for monitoring is created with a prometheus.io/scrape annotation

shell.nix

@@ -11,6 +11,7 @@ in
 {
   # --- Dev shell inherited from flake.nix ---
   devShell = devShells.default;
+  devShellFull = devShells.full;
 
   # --- Continuous integration shell ---
   # The shell used for all CI jobs (along with devShell)

src/db/Cargo.toml

@@ -15,6 +15,7 @@ path = "lib.rs"
 err-derive.workspace = true
 hexdump.workspace = true
 tracing.workspace = true
+opentelemetry.workspace = true
 
 heed = { workspace = true, optional = true }
 rusqlite = { workspace = true, optional = true, features = ["backup"] }

src/db/lib.rs

@@ -3,6 +3,9 @@ extern crate tracing;
 
 #[cfg(feature = "lmdb")]
 pub mod lmdb_adapter;
+#[cfg(feature = "lmdb")]
+pub mod metric_proxy;
+
 #[cfg(feature = "sqlite")]
 pub mod sqlite_adapter;
 

src/db/lmdb_adapter.rs

@@ -39,12 +39,15 @@ pub struct LmdbDb {
 }
 
 impl LmdbDb {
-	pub fn init(db: Env) -> Db {
-		let s = Self {
+	pub fn to_wrap(db: Env) -> Self {
+		Self {
 			db,
 			trees: RwLock::new((Vec::new(), HashMap::new())),
-		};
-		Db(Arc::new(s))
+		}
+	}
+
+	pub fn init(db: Env) -> Db {
+		Db(Arc::new(Self::to_wrap(db)))
 	}
 
 	fn get_tree(&self, i: usize) -> Result<Database> {
@@ -226,7 +229,7 @@ impl IDb for LmdbDb {
 
 // ----
 
-struct LmdbTx<'a> {
+pub(crate) struct LmdbTx<'a> {
 	trees: &'a [Database],
 	tx: RwTxn<'a, 'a>,
 }

src/db/metric_proxy.rs

@@ -0,0 +1,217 @@
+use std::path::PathBuf;
+use std::sync::Arc;
+use std::time::Instant;
+
+use crate::lmdb_adapter::LmdbDb;
+use crate::{
+	Bound, Db, IDb, ITx, ITxFn, OnCommit, Result, TxFnResult, TxOpResult, TxResult, TxValueIter,
+	Value, ValueIter,
+};
+use opentelemetry::{
+	global,
+	metrics::{Unit, ValueRecorder},
+	KeyValue,
+};
+
+pub struct MetricDbProxy {
+	//@FIXME Replace with a template
+	db: LmdbDb,
+	op: ValueRecorder<f64>,
+}
+
+impl MetricDbProxy {
+	pub fn init(db: LmdbDb) -> Db {
+		let meter = global::meter("garage/db");
+		let s = Self {
+			db,
+			op: meter
+				.f64_value_recorder("db.op")
+				.with_description("Duration and amount of operations on the local metadata engine")
+				.with_unit(Unit::new("ms"))
+				.init(),
+		};
+		Db(Arc::new(s))
+	}
+
+	fn instrument<T>(
+		&self,
+		fx: impl FnOnce() -> T,
+		op: &'static str,
+		cat: &'static str,
+		tx: &'static str,
+	) -> T {
+		let metric_tags = [
+			KeyValue::new("op", op),
+			KeyValue::new("cat", cat),
+			KeyValue::new("tx", tx),
+		];
+
+		let request_start = Instant::now();
+		let res = fx();
+		let delay_nanos = Instant::now()
+			.saturating_duration_since(request_start)
+			.as_nanos();
+		let delay_millis: f64 = delay_nanos as f64 / 1_000_000f64;
+		self.op.record(delay_millis, &metric_tags);
+
+		res
+	}
+}
+
+impl IDb for MetricDbProxy {
+	fn engine(&self) -> String {
+		format!("Metric Proxy on {}", self.db.engine())
+	}
+
+	fn open_tree(&self, name: &str) -> Result<usize> {
+		self.instrument(|| self.db.open_tree(name), "open_tree", "control", "no")
+	}
+
+	fn list_trees(&self) -> Result<Vec<String>> {
+		self.instrument(|| self.db.list_trees(), "list_trees", "control", "no")
+	}
+
+	fn snapshot(&self, to: &PathBuf) -> Result<()> {
+		self.instrument(|| self.db.snapshot(to), "snapshot", "control", "no")
+	}
+
+	// ---
+
+	fn get(&self, tree: usize, key: &[u8]) -> Result<Option<Value>> {
+		self.instrument(|| self.db.get(tree, key), "get", "data", "no")
+	}
+
+	fn len(&self, tree: usize) -> Result<usize> {
+		self.instrument(|| self.db.len(tree), "len", "data", "no")
+	}
+
+	fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<Option<Value>> {
+		self.instrument(|| self.db.insert(tree, key, value), "insert", "data", "no")
+	}
+
+	fn remove(&self, tree: usize, key: &[u8]) -> Result<Option<Value>> {
+		self.instrument(|| self.db.remove(tree, key), "remove", "data", "no")
+	}
+
+	fn clear(&self, tree: usize) -> Result<()> {
+		self.instrument(|| self.db.clear(tree), "clear", "data", "no")
+	}
+
+	fn iter(&self, tree: usize) -> Result<ValueIter<'_>> {
+		self.instrument(|| self.db.iter(tree), "iter", "data", "no")
+	}
+
+	fn iter_rev(&self, tree: usize) -> Result<ValueIter<'_>> {
+		self.instrument(|| self.db.iter_rev(tree), "iter_rev", "data", "no")
+	}
+
+	fn range<'r>(
+		&self,
+		tree: usize,
+		low: Bound<&'r [u8]>,
+		high: Bound<&'r [u8]>,
+	) -> Result<ValueIter<'_>> {
+		self.instrument(|| self.db.range(tree, low, high), "range", "data", "no")
+	}
+
+	fn range_rev<'r>(
+		&self,
+		tree: usize,
+		low: Bound<&'r [u8]>,
+		high: Bound<&'r [u8]>,
+	) -> Result<ValueIter<'_>> {
+		self.instrument(
+			|| self.db.range_rev(tree, low, high),
+			"range_rev",
+			"data",
+			"no",
+		)
+	}
+
+	// ----
+
+	fn transaction(&self, f: &dyn ITxFn) -> TxResult<OnCommit, ()> {
+		self.instrument(
+			|| self.db.transaction(&MetricITxFnProxy { f, metrics: self }),
+			"transaction",
+			"control",
+			"yes",
+		)
+	}
+}
+
+struct MetricITxFnProxy<'a> {
+	f: &'a dyn ITxFn,
+	metrics: &'a MetricDbProxy,
+}
+impl<'a> ITxFn for MetricITxFnProxy<'a> {
+	fn try_on(&self, tx: &mut dyn ITx) -> TxFnResult {
+		self.f.try_on(&mut MetricTxProxy {
+			tx,
+			metrics: self.metrics,
+		})
+	}
+}
+
+struct MetricTxProxy<'a> {
+	tx: &'a mut dyn ITx,
+	metrics: &'a MetricDbProxy,
+}
+impl<'a> ITx for MetricTxProxy<'a> {
+	fn get(&self, tree: usize, key: &[u8]) -> TxOpResult<Option<Value>> {
+		self.metrics
+			.instrument(|| self.tx.get(tree, key), "get", "data", "yes")
+	}
+
+	fn len(&self, tree: usize) -> TxOpResult<usize> {
+		self.metrics
+			.instrument(|| self.tx.len(tree), "len", "data", "yes")
+	}
+
+	fn insert(&mut self, tree: usize, key: &[u8], value: &[u8]) -> TxOpResult<Option<Value>> {
+		self.metrics
+			.instrument(|| self.tx.insert(tree, key, value), "insert", "data", "yes")
+	}
+
+	fn remove(&mut self, tree: usize, key: &[u8]) -> TxOpResult<Option<Value>> {
+		self.metrics
+			.instrument(|| self.tx.remove(tree, key), "remove", "data", "yes")
+	}
+
+	fn clear(&mut self, tree: usize) -> TxOpResult<()> {
+		self.metrics
+			.instrument(|| self.tx.clear(tree), "clear", "data", "yes")
+	}
+
+	fn iter(&self, tree: usize) -> TxOpResult<TxValueIter<'_>> {
+		self.metrics
+			.instrument(|| self.tx.iter(tree), "iter", "data", "yes")
+	}
+	fn iter_rev(&self, tree: usize) -> TxOpResult<TxValueIter<'_>> {
+		self.metrics
+			.instrument(|| self.tx.iter_rev(tree), "iter_rev", "data", "yes")
+	}
+	fn range<'r>(
+		&self,
+		tree: usize,
+		low: Bound<&'r [u8]>,
+		high: Bound<&'r [u8]>,
+	) -> TxOpResult<TxValueIter<'_>> {
+		self.metrics
+			.instrument(|| self.tx.range(tree, low, high), "range", "data", "yes")
+	}
+
+	fn range_rev<'r>(
+		&self,
+		tree: usize,
+		low: Bound<&'r [u8]>,
+		high: Bound<&'r [u8]>,
+	) -> TxOpResult<TxValueIter<'_>> {
+		self.metrics.instrument(
+			|| self.tx.range_rev(tree, low, high),
+			"range_rev",
+			"data",
+			"yes",
+		)
+	}
+}

src/db/open.rs

@@ -10,6 +10,7 @@ use crate::{Db, Error, Result};
 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
 pub enum Engine {
 	Lmdb,
+	LmdbWithMetrics,
 	Sqlite,
 }
 
@@ -18,6 +19,7 @@ impl Engine {
 	pub fn as_str(&self) -> &'static str {
 		match self {
 			Self::Lmdb => "lmdb",
+			Self::LmdbWithMetrics => "lmdb-with-metrics",
 			Self::Sqlite => "sqlite",
 		}
 	}
@@ -35,6 +37,7 @@ impl std::str::FromStr for Engine {
 	fn from_str(text: &str) -> Result<Engine> {
 		match text {
 			"lmdb" | "heed" => Ok(Self::Lmdb),
+			"lmdb-with-metrics" | "heed-with-metrics" => Ok(Self::LmdbWithMetrics),
 			"sqlite" | "sqlite3" | "rusqlite" => Ok(Self::Sqlite),
 			"sled" => Err(Error("Sled is no longer supported as a database engine. Converting your old metadata db can be done using an older Garage binary (e.g. v0.9.4).".into())),
 			kind => Err(Error(
@@ -74,7 +77,7 @@ pub fn open_db(path: &PathBuf, engine: Engine, opt: &OpenOpt) -> Result<Db> {
 
 		// ---- LMDB DB ----
 		#[cfg(feature = "lmdb")]
-		Engine::Lmdb => {
+		Engine::Lmdb | Engine::LmdbWithMetrics => {
 			info!("Opening LMDB database at: {}", path.display());
 			if let Err(e) = std::fs::create_dir_all(&path) {
 				return Err(Error(
@@ -92,6 +95,7 @@ pub fn open_db(path: &PathBuf, engine: Engine, opt: &OpenOpt) -> Result<Db> {
 			env_builder.map_size(map_size);
 			env_builder.max_readers(2048);
 			unsafe {
+				env_builder.flag(crate::lmdb_adapter::heed::flags::Flags::MdbNoRdAhead);
 				env_builder.flag(crate::lmdb_adapter::heed::flags::Flags::MdbNoMetaSync);
 				if !opt.fsync {
 					env_builder.flag(heed::flags::Flags::MdbNoSync);
@@ -109,7 +113,13 @@ pub fn open_db(path: &PathBuf, engine: Engine, opt: &OpenOpt) -> Result<Db> {
 					))
 				}
 				Err(e) => Err(Error(format!("Cannot open LMDB database: {}", e).into())),
-				Ok(db) => Ok(crate::lmdb_adapter::LmdbDb::init(db)),
+				Ok(db) => match engine {
+					Engine::LmdbWithMetrics => {
+						let to_wrap = crate::lmdb_adapter::LmdbDb::to_wrap(db);
+						Ok(crate::metric_proxy::MetricDbProxy::init(to_wrap))
+					}
+					_ => Ok(crate::lmdb_adapter::LmdbDb::init(db)),
+				},
 			}
 		}
 

src/garage/cli/structs.rs

@@ -48,7 +48,7 @@ pub enum Command {
 	#[structopt(name = "worker", version = garage_version())]
 	Worker(WorkerOperation),
 
-	/// Low-level debug operations on data blocks
+	/// Low-level node-local debug operations on data blocks
 	#[structopt(name = "block", version = garage_version())]
 	Block(BlockOperation),
 

src/model/garage.rs

@@ -121,7 +121,7 @@ impl Garage {
 			db::Engine::Sqlite => {
 				db_path.push("db.sqlite");
 			}
-			db::Engine::Lmdb => {
+			db::Engine::Lmdb | db::Engine::LmdbWithMetrics => {
 				db_path.push("db.lmdb");
 			}
 		}
@@ -141,7 +141,7 @@ impl Garage {
 		)?)
 		.ok()
 		.and_then(|x| NetworkKey::from_slice(&x))
-		.ok_or_message("Invalid RPC secret key")?;
+		.ok_or_message("Invalid RPC secret key: expected 32 bits of entropy, please check the documentation for requirements")?;
 
 		let (replication_factor, consistency_mode) = parse_replication_mode(&config)?;
 

src/rpc/Cargo.toml

@@ -24,6 +24,7 @@ bytes.workspace = true
 bytesize.workspace = true
 gethostname.workspace = true
 hex.workspace = true
+ipnet.workspace = true
 tracing.workspace = true
 rand.workspace = true
 itertools.workspace = true

src/rpc/system.rs

@@ -844,12 +844,20 @@ impl NodeStatus {
 	}
 }
 
-fn get_default_ip() -> Option<IpAddr> {
+/// Obtain the list of currently available IP addresses on all non-loopback
+/// interfaces, optionally filtering them to be inside a given IpNet.
+fn get_default_ip(filter_ipnet: Option<ipnet::IpNet>) -> Option<IpAddr> {
 	pnet_datalink::interfaces()
-		.iter()
-		.find(|e| e.is_up() && !e.is_loopback() && !e.ips.is_empty())
-		.and_then(|e| e.ips.first())
-		.map(|a| a.ip())
+		.into_iter()
+		// filter down and loopback interfaces
+		.filter(|i| i.is_up() && !i.is_loopback())
+		// get all IPs
+		.flat_map(|e| e.ips)
+		// optionally, filter to be inside filter_ipnet
+		.find(|ipn| {
+			filter_ipnet.is_some_and(|ipnet| ipnet.contains(&ipn.ip())) || filter_ipnet.is_none()
+		})
+		.map(|ipn| ipn.ip())
 }
 
 fn get_rpc_public_addr(config: &Config) -> Option<SocketAddr> {
@@ -877,7 +885,28 @@ fn get_rpc_public_addr(config: &Config) -> Option<SocketAddr> {
 			}
 		}
 		None => {
-			let addr = get_default_ip().map(|ip| SocketAddr::new(ip, config.rpc_bind_addr.port()));
+			// `No rpc_public_addr` specified, try to discover one, optionally filtering by `rpc_public_addr_subnet`.
+			let filter_subnet: Option<ipnet::IpNet> = config
+				.rpc_public_addr_subnet
+				.as_ref()
+				.and_then(|filter_subnet_str| match filter_subnet_str.parse::<ipnet::IpNet>() {
+					Ok(filter_subnet) => {
+						let filter_subnet_trunc = filter_subnet.trunc();
+						if filter_subnet_trunc != filter_subnet {
+							warn!("`rpc_public_addr_subnet` changed after applying netmask, continuing with {}", filter_subnet.trunc());
+						}
+						Some(filter_subnet_trunc)
+					}
+					Err(e) => {
+						panic!(
+							"Cannot parse rpc_public_addr_subnet {} from config file: {}. Bailing out.",
+							filter_subnet_str, e
+						);
+					}
+				});
+
+			let addr = get_default_ip(filter_subnet)
+				.map(|ip| SocketAddr::new(ip, config.rpc_bind_addr.port()));
 			if let Some(a) = addr {
 				warn!("Using autodetected rpc_public_addr: {}. Consider specifying it explicitly in configuration file if possible.", a);
 			}

src/util/background/worker.rs

@@ -1,11 +1,13 @@
 use std::collections::HashMap;
 use std::sync::Arc;
 use std::time::Duration;
+use std::time::Instant;
 
 use async_trait::async_trait;
 use futures::future::*;
 use futures::stream::FuturesUnordered;
 use futures::StreamExt;
+use opentelemetry::{global, metrics::ValueRecorder, KeyValue};
 use serde::{Deserialize, Serialize};
 use tokio::select;
 use tokio::sync::{mpsc, watch};
@@ -62,6 +64,7 @@ pub(crate) struct WorkerProcessor {
 	stop_signal: watch::Receiver<bool>,
 	worker_chan: mpsc::UnboundedReceiver<Box<dyn Worker>>,
 	worker_info: Arc<std::sync::Mutex<HashMap<usize, WorkerInfo>>>,
+	metrics: ValueRecorder<f64>,
 }
 
 impl WorkerProcessor {
@@ -70,10 +73,15 @@ impl WorkerProcessor {
 		stop_signal: watch::Receiver<bool>,
 		worker_info: Arc<std::sync::Mutex<HashMap<usize, WorkerInfo>>>,
 	) -> Self {
+		let meter = global::meter("garage/util");
 		Self {
 			stop_signal,
 			worker_chan,
 			worker_info,
+			metrics: meter
+				.f64_value_recorder("util.worker_step")
+				.with_description("Duration and amount of worker steps executed")
+				.init(),
 		}
 	}
 
@@ -103,6 +111,7 @@ impl WorkerProcessor {
 								errors: 0,
 								consecutive_errors: 0,
 								last_error: None,
+								metrics: self.metrics.clone(),
 							};
 						workers.push(async move {
 							worker.step().await;
@@ -183,10 +192,13 @@ struct WorkerHandler {
 	errors: usize,
 	consecutive_errors: usize,
 	last_error: Option<(String, u64)>,
+	metrics: ValueRecorder<f64>,
 }
 
 impl WorkerHandler {
 	async fn step(&mut self) {
+		let request_start = Instant::now();
+		//@FIXME we also want to track errors in metrics but I don't know how yet.
 		match self.state {
 			WorkerState::Busy => match self.worker.work(&mut self.stop_signal).await {
 				Ok(s) => {
@@ -229,5 +241,17 @@ impl WorkerHandler {
 			}
 			WorkerState::Done => unreachable!(),
 		}
+
+		// metrics
+		let metric_tags = [
+			KeyValue::new("state", self.state.to_string()),
+			KeyValue::new("name", self.worker.name()),
+			KeyValue::new("id", format!("{}", self.task_id)),
+		];
+
+		let delay_secs = Instant::now()
+			.saturating_duration_since(request_start)
+			.as_secs_f64();
+		self.metrics.record(delay_secs, &metric_tags);
 	}
 }

src/util/config.rs

@@ -85,6 +85,10 @@ pub struct Config {
 	/// Public IP address of this node
 	pub rpc_public_addr: Option<String>,
 
+	/// In case `rpc_public_addr` was not set, this can filter
+	/// the addresses announced to other peers to a specific subnet.
+	pub rpc_public_addr_subnet: Option<String>,
+
 	/// Timeout for Netapp's ping messagess
 	pub rpc_ping_timeout_msec: Option<u64>,
 	/// Timeout for Netapp RPC calls