doc: quick-start: rework flag description (#1496)

Add a link to the manual install procedure, use a
quote to make the manual install path more visible
and mention the --default-access-key option.

Closes: #1446

Rendered output:

![image](/attachments/84bbe650-daff-40d1-b4bd-629abece27b0)

Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1496

Cargo.lock

@@ -1399,6 +1399,30 @@ dependencies = [
 name = "format_table"
 version = "0.1.1"
 
+[[package]]
+name = "fundu"
+version = "2.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2ce12752fc64f35be3d53e0a57017cd30970f0cffd73f62c791837d8845badbd"
+dependencies = [
+ "fundu-core",
+]
+
+[[package]]
+name = "fundu-core"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e463452e2d8b7600d38dcea1ed819773a57f0d710691bfc78db3961bd3f4c3ba"
+
+[[package]]
+name = "fundu-systemd"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b8641ae8f534f853a552beb3a9f12cc72fd1343260964af82cf09fda66e3d94e"
+dependencies = [
+ "fundu",
+]
+
 [[package]]
 name = "futures"
 version = "0.3.32"
@@ -1502,6 +1526,7 @@ dependencies = [
  "chrono",
  "crc-fast",
  "format_table",
+ "fundu-systemd",
  "futures",
  "garage_api_admin",
  "garage_api_common",
@@ -1528,7 +1553,6 @@ dependencies = [
  "opentelemetry",
  "opentelemetry-otlp",
  "opentelemetry-prometheus",
- "parse_duration",
  "serde_json",
  "sha1 0.10.6",
  "sha2 0.10.9",
@@ -1749,7 +1773,6 @@ dependencies = [
  "garage_util",
  "hex",
  "http 1.4.0",
- "parse_duration",
  "rand 0.9.4",
  "serde",
  "serde_bytes",
@@ -1849,6 +1872,8 @@ dependencies = [
  "blake2",
  "bytesize",
  "chrono",
+ "fundu",
+ "fundu-systemd",
  "futures",
  "garage_db",
  "garage_net",
@@ -1881,6 +1906,7 @@ dependencies = [
  "garage_model",
  "garage_table",
  "garage_util",
+ "html-escape",
  "http 1.4.0",
  "http-body-util",
  "hyper 1.9.0",
@@ -2194,6 +2220,15 @@ dependencies = [
  "windows-link",
 ]
 
+[[package]]
+name = "html-escape"
+version = "0.2.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6d1ad449764d627e22bfd7cd5e8868264fc9236e07c752972b4080cd351cb476"
+dependencies = [
+ "utf8-width",
+]
+
 [[package]]
 name = "http"
 version = "0.2.12"
@@ -3213,41 +3248,6 @@ dependencies = [
  "windows-sys 0.61.2",
 ]
 
-[[package]]
-name = "num"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b8536030f9fea7127f841b45bb6243b27255787fb4eb83958aa1ef9d2fdc0c36"
-dependencies = [
- "num-bigint",
- "num-complex",
- "num-integer",
- "num-iter",
- "num-rational",
- "num-traits",
-]
-
-[[package]]
-name = "num-bigint"
-version = "0.2.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304"
-dependencies = [
- "autocfg",
- "num-integer",
- "num-traits",
-]
-
-[[package]]
-name = "num-complex"
-version = "0.2.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b6b19411a9719e753aff12e5187b74d60d3dc449ec3f4dc21e3989c3f554bc95"
-dependencies = [
- "autocfg",
- "num-traits",
-]
-
 [[package]]
 name = "num-conv"
 version = "0.2.1"
@@ -3263,29 +3263,6 @@ dependencies = [
  "num-traits",
 ]
 
-[[package]]
-name = "num-iter"
-version = "0.1.45"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf"
-dependencies = [
- "autocfg",
- "num-integer",
- "num-traits",
-]
-
-[[package]]
-name = "num-rational"
-version = "0.2.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5c000134b5dbf44adc5cb772486d335293351644b801551abe8f75c84cfa4aef"
-dependencies = [
- "autocfg",
- "num-bigint",
- "num-integer",
- "num-traits",
-]
-
 [[package]]
 name = "num-traits"
 version = "0.2.19"
@@ -3454,17 +3431,6 @@ dependencies = [
  "windows-link",
 ]
 
-[[package]]
-name = "parse_duration"
-version = "2.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7037e5e93e0172a5a96874380bf73bc6ecef022e26fa25f2be26864d6b3ba95d"
-dependencies = [
- "lazy_static",
- "num",
- "regex",
-]
-
 [[package]]
 name = "password-hash"
 version = "0.5.0"
@@ -5328,6 +5294,12 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "utf8-width"
+version = "0.1.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1292c0d970b54115d14f2492fe0170adf21d68a1de108eebc51c1df4f346a091"
+
 [[package]]
 name = "utf8_iter"
 version = "1.0.4"

Cargo.toml

@@ -53,10 +53,13 @@ cfg-if = "1.0"
 chrono = { version = "0.4", features = ["serde"] }
 crc-fast = "1.9"
 crypto-common = "0.1"
+fundu = "2.0"
+fundu-systemd = "0.3"
 gethostname = "1.1"
 git-version = "0.3"
 hex = "0.4"
 hexdump = "0.1"
+html-escape = "0.2.13"
 hmac = "0.12"
 itertools = "0.14"
 ipnet = "2.11"
@@ -67,7 +70,6 @@ mktemp = "0.5"
 nix = { version = "0.31", default-features = false, features = ["fs"] }
 nom = "8.0"
 parking_lot = "0.12"
-parse_duration = "2.1"
 paste = "1.0"
 pin-project = "1.1"
 pnet_datalink = "0.35"

Dockerfile

@@ -4,6 +4,4 @@ ENV RUST_BACKTRACE=1
 ENV RUST_LOG=garage=info
 
 COPY result/bin/garage /
-
+CMD [ "/garage", "server"]
-ENTRYPOINT ["/garage"]
-CMD ["server"]

GOVERNANCE.md

@@ -0,0 +1,45 @@
+# Governance of Gararge
+
+This documents how the Garage project operates. It reflects the state of the project as of July 2026 and is not optimal. The team is interested to improve it in the future.
+
+## Team organization
+
+* **Contributors**: anyone can contribute by proposing changes in issues and pull requests.
+
+* **Maintainers**: they are responsible for reviewing, merging pull requests, publishing releases and triaging issues.
+  The current maintainers are:
+   * Alex (handle `lx`)
+   * Trinity (handle `trinity-1686a`)
+   * Quentin (handle `quentin`)
+   * Maximilien (handle `halfa`), who is in particular responsible for coordinating effort on the Kubernetes integration / Helm chart.
+ 
+  They are added to a white-list of the branch protection rule of the repository to enable them to merge pull requests.
+  To become a maintainer, you need to be a long-term contributor and earn the personal trust of Alex.
+  There is no set process for leaving the maintainer role.
+
+* **Lead developer**: Alex (handle `lx`) is the lead developer and is responsible of ensuring the
+correctness of Garage and stability between version upgrades. He may transfer this role to someone else as he sees fit.
+
+## Communication channels
+
+The team coordinates in the following channels:
+* The issue tracker and pull requests of the official repository.
+* The `#garage:deuxfleurs.fr` matrix channel (in English), open to anyone.
+  On this channel, users may ask for support and discussions about development also happen.
+* The `#garage-dev:deuxfleurs.fr` matrix channel (in French), not advertised to contributors but de facto accessible to anyone.
+  Discussions about development and project coordination happen there.
+
+The moderators for those discussion channels are the Garage maintainers.
+
+## Decision procedures
+
+Decisions are taken by lazy consensus, with the lead developer settling discussions when a consensus cannot be reached.
+
+## Governance changes
+
+There is no set process for changing the governance of garage.
+
+## See also
+
+* [Project goals](https://garagehq.deuxfleurs.fr/documentation/design/goals/)
+* [Contributing instructions](https://git.deuxfleurs.fr/Deuxfleurs/garage/src/branch/main-v2/CONTRIBUTING.md)

doc/book/connect/apps/index.md

@@ -547,7 +547,7 @@ ejabberdctl module_install mod_s3_upload
 Create the required key and bucket with:
 
 ```bash
-garage key new --name ejabberd
+garage key create ejabberd
 garage bucket create objects.xmpp-server.fr
 garage bucket allow objects.xmpp-server.fr --read --write --key ejabberd
 garage bucket website --allow objects.xmpp-server.fr
@@ -678,7 +678,7 @@ For more information on deployment you can check the [ente documentation](https:
 This is the usual Garage setup:
 
 ```bash
-garage key new --name pleroma-key
+garage key create pleroma-key
 garage bucket create pleroma
 garage bucket allow pleroma --read --write --owner --key pleroma-key
 ```
@@ -759,7 +759,7 @@ This feature requires `pict-rs >= 4.0.0`.
 This is the usual Garage setup:
 
 ```bash
-garage key new --name pictrs-key
+garage key create pictrs-key
 garage bucket create pictrs-data
 garage bucket allow pictrs-data --read --write --key pictrs-key
 ```

doc/book/connect/backup.md

@@ -22,7 +22,7 @@ Note that `git-annex` requires to be compiled with Haskell package version
 `aws-0.24` to work with Garage.
 
 ```bash
-garage key new --name my-key
+garage key create my-key
 garage bucket create my-git-annex
 garage bucket allow my-git-annex --read --write --key my-key
 ```

doc/book/connect/cli.md

@@ -268,7 +268,7 @@ duck --delete garage:/my-files/an-object.txt
 
 ## WinSCP (libs3) {#winscp}
 
-*You can find instructions on how to use the GUI in french [in our wiki](https://guide.deuxfleurs.fr/prise_en_main/winscp/).*
+*You can find instructions on how to use the GUI in french [in our wiki](https://guide.deuxfleurs.fr/services/winscp/).*
 
 How to use `winscp.com`, the CLI interface of WinSCP:
 

doc/book/connect/observability.md

@@ -27,7 +27,7 @@ which support storing metrics in an object store:
 This can be configured with Garage with the following:
 
 ```bash
-garage key new --name vector-system-logs
+garage key create vector-system-logs
 garage bucket create system-logs
 garage bucket allow system-logs --read --write --key vector-system-logs
 ```

doc/book/cookbook/exposing-websites.md

@@ -25,6 +25,13 @@ garage bucket website --allow my-website
 
 Now it will be **publicly** exposed on the web endpoint (by default listening on port 3902).
 
+> The bucket needs to have a *global alias* to be exposed as a website. If the
+> bucket was created with `garage bucket create` it will have an alias;
+> if created via the S3 API [you will have to manually add the alias
+> ](https://git.deuxfleurs.fr/Deuxfleurs/garage/issues/742) first.
+> Creating globally aliased buckets from the S3 API is [currently under
+> discussion](https://git.deuxfleurs.fr/Deuxfleurs/garage/issues/649).
+
 ## How exposed websites work
 
 Our website serving logic is as follow:

doc/book/cookbook/real-world.md

@@ -213,12 +213,7 @@ If your configuration file is at `/etc/garage.toml`, the `garage` binary should
 You can also use an alias as follows to use the Garage binary inside your docker container:
 
 ```bash
-# garage 3.x, we have an entrypoint and you can use
-alias garage="docker exec -ti <container name>"
-
-# For garage 2.x, you need to specify the absolute path to binary
 alias garage="docker exec -ti <container name> /garage"
-
 ```
 
 You can test your `garage` CLI utility by running a simple command such as:

doc/book/operations/durability-repairs.md

@@ -91,7 +91,7 @@ is definitely lost, then there is no other choice than to declare your S3 object
 as unrecoverable, and to delete them properly from the data store. This can be done
 using the `garage block purge` command.
 
-## Rebalancing data directories
+## Rebalancing data directories {#rebalance}
 
 In [multi-HDD setups](@/documentation/operations/multi-hdd.md), to ensure that
 data blocks are well balanced between storage locations, you may run a

doc/book/operations/multi-hdd.md

@@ -68,10 +68,11 @@ To rebalance data, two strategies can be used:
   secondary directory. This might never end up rebalancing everything if there
   are data blocks that are only read and never written.
 
-- Active rebalancing: an operator of a Garage node can explicitly launch a repair
+- Active rebalancing: an operator of a Garage node can [explicitly launch a
-  procedure that rebalances the data directories, moving all blocks to their
+  repair procedure](@/documentation/operations/durability-repairs.md#rebalance)
-  primary location. Once done, all secondary locations for all hash slices are
+  that rebalances the data directories, moving all blocks to their primary
-  removed so that they won't be checked anymore when looking for a data block.
+  location. Once done, all secondary locations for all hash slices are removed
+  so that they won't be checked anymore when looking for a data block.
 
 ## Read-only storage locations
 

doc/book/quick-start/_index.md

@@ -133,12 +133,17 @@ Use the following command to launch the Garage server:
 garage server --single-node --default-bucket
 ```
 
-The `--single-node` flag instructs Garage to automatically configure a single-node cluster without data replication.
+- the `--single-node` flag instructs Garage to automatically configure a
-The `--default-bucket` flag instructs Garage to create a default access key and a default bucket using the environment variables we defined above.
+  single-node cluster without data replication;
-Both flags are optional and can be omitted, in which case you will have to follow manual configuration steps described below.
+- the `--default-bucket` flag instructs Garage to create a default access key
+  and a default bucket using the environment variables we defined above (it
+  implies `--default-access-key`).
 
-**For older versions of Garage (before v2.3.0):** automatic configuration using `--single-node` and `--default-bucket` is not available,
+> You can refer to the [manual configuration
-you must follow the manual configuration steps.
+> steps](#manual-configuration) if:
+>
+> - you decide to no use these optional flags;
+> - you are running an **older version of Garage (before v2.3.0)**.
 
 Alternatively, if you cannot or do not wish to run the Garage binary directly,
 you may use Docker to run Garage in a container using the following command:
@@ -178,11 +183,8 @@ garage status
 
 If you are running Garage in a Docker container, you can use the following command instead:
 
-NOTE: Garage 3.x uses docker `ENTRYPOINT` and it's easier to use,
-while garage 2.x does not and you need to specify path `/garage`
-
 ```bash
-docker exec garage-container status
+docker exec garage-container /garage status
 ```
 
 This should show something like this:
@@ -295,7 +297,7 @@ An exhaustive list is maintained in the ["Integrations" > "Browsing tools" secti
 
 
 
-## Manual configuration
+## Manual configuration {#manual-configuration}
 
 This section provides instructions that are equivalent to using the
 `--single-node` and `--default-bucket` flags for automatic configuration.  If
@@ -323,7 +325,7 @@ If you are running Garage in a Docker container, you can set the following alias
 to provide a fake `garage`command that uses the Garage binary inside your container:
 
 ```bash
-alias garage="docker exec -ti <container name>"
+alias garage="docker exec -ti <container name> /garage"
 ```
 
 You can test that your `garage` CLI is configured correctly by running a basic command such as `garage status`.

doc/book/reference-manual/admin-api.md

@@ -182,15 +182,15 @@ content-type: text/plain; version=0.0.4
 content-length: 12145
 date: Tue, 08 Aug 2023 07:25:05 GMT
 
-# HELP garage_api_admin_error_count Number of API calls to the various Admin API endpoints that resulted in errors
+# HELP api_admin_error_counter Number of API calls to the various Admin API endpoints that resulted in errors
-# TYPE garage_api_admin_error_count counter
+# TYPE api_admin_error_counter counter
-garage_api_admin_error_count{api_endpoint="CheckWebsiteEnabled",status_code="400"} 1
+api_admin_error_counter{api_endpoint="CheckWebsiteEnabled",status_code="400"} 1
-garage_api_admin_error_count{api_endpoint="CheckWebsiteEnabled",status_code="404"} 3
+api_admin_error_counter{api_endpoint="CheckWebsiteEnabled",status_code="404"} 3
-# HELP garage_api_admin_request_count Number of API calls to the various Admin API endpoints
+# HELP api_admin_request_counter Number of API calls to the various Admin API endpoints
-# TYPE garage_api_admin_request_count counter
+# TYPE api_admin_request_counter counter
-garage_api_admin_request_count{api_endpoint="CheckWebsiteEnabled"} 7
+api_admin_request_counter{api_endpoint="CheckWebsiteEnabled"} 7
-garage_api_admin_request_count{api_endpoint="Health"} 3
+api_admin_request_counter{api_endpoint="Health"} 3
-# HELP garage_api_admin_request_duration Duration of API calls to the various Admin API endpoints
+# HELP api_admin_request_duration Duration of API calls to the various Admin API endpoints
 ...
 ```
 

doc/book/reference-manual/configuration.md

@@ -448,7 +448,7 @@ If this value is set, Garage will automatically take a snapshot of the metadata
 DB file at a regular interval and save it in the metadata directory,
 or in [`metadata_snapshots_dir`](#metadata_snapshots_dir) if it is set.
 This parameter can take any duration string that can be parsed by
-the [`parse_duration`](https://docs.rs/parse_duration/latest/parse_duration/#syntax) crate.
+the [`fundu_systemd`](https://docs.rs/fundu-systemd) crate.
 
 Snapshots can allow to recover from situations where the metadata DB file is
 corrupted, for instance after an unclean shutdown.  See [this

doc/book/reference-manual/monitoring.md

@@ -40,146 +40,146 @@ garage_local_disk_total{volume="metadata"} 763063566336
 
 ### Cluster health status metrics
 
-#### `garage_cluster_healthy` (gauge)
+#### `cluster_healthy` (gauge)
 
 Whether all storage nodes are connected (0 or 1)
 
 ```
-garage_cluster_healthy 0
+cluster_healthy 0
 ```
 
-#### `garage_cluster_available` (gauge)
+#### `cluster_available` (gauge)
 
 Whether all requests can be served, even if some storage nodes are disconnected
 
 ```
-garage_cluster_available 1
+cluster_available 1
 ```
 
-#### `garage_cluster_connected_nodes` (gauge)
+#### `cluster_connected_nodes` (gauge)
 
 Number of nodes currently connected
 
 ```
-garage_cluster_connected_nodes 3
+cluster_connected_nodes 3
 ```
 
-#### `garage_cluster_known_nodes` (gauge)
+#### `cluster_known_nodes` (gauge)
 
 Number of nodes already seen once in the cluster
 
 ```
-garage_cluster_known_nodes 3
+cluster_known_nodes 3
 ```
 
-#### `garage_cluster_layout_node_connected` (gauge)
+#### `cluster_layout_node_connected` (gauge)
 
 Connection status for individual nodes of the cluster layout
 
 ```
-garage_cluster_layout_node_connected{id="62b218d848e86a64",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 1
+cluster_layout_node_connected{id="62b218d848e86a64",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 1
-garage_cluster_layout_node_connected{id="a11c7cf18af29737",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 0
+cluster_layout_node_connected{id="a11c7cf18af29737",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 0
-garage_cluster_layout_node_connected{id="a235ac7695e0c54d",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 1
+cluster_layout_node_connected{id="a235ac7695e0c54d",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 1
-garage_cluster_layout_node_connected{id="b10c110e4e854e5a",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 1
+cluster_layout_node_connected{id="b10c110e4e854e5a",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 1
 ```
 
-#### `garage_cluster_layout_node_disconnected_time` (gauge)
+#### `cluster_layout_node_disconnected_time` (gauge)
 
 Time (in seconds) since last connection to individual nodes of the cluster layout
 
 ```
-garage_cluster_layout_node_disconnected_time{id="62b218d848e86a64",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 0
+cluster_layout_node_disconnected_time{id="62b218d848e86a64",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 0
-garage_cluster_layout_node_disconnected_time{id="a235ac7695e0c54d",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 0
+cluster_layout_node_disconnected_time{id="a235ac7695e0c54d",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 0
-garage_cluster_layout_node_disconnected_time{id="b10c110e4e854e5a",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 0
+cluster_layout_node_disconnected_time{id="b10c110e4e854e5a",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 0
 ```
 
-#### `garage_cluster_storage_nodes` (gauge)
+#### `cluster_storage_nodes` (gauge)
 
 Number of storage nodes declared in the current layout
 
 ```
-garage_cluster_storage_nodes 4
+cluster_storage_nodes 4
 ```
 
-#### `garage_cluster_storage_nodes_ok` (gauge)
+#### `cluster_storage_nodes_ok` (gauge)
 
 Number of storage nodes currently connected
 
 ```
-garage_cluster_storage_nodes_ok 3
+cluster_storage_nodes_ok 3
 ```
 
-#### `garage_cluster_partitions` (gauge)
+#### `cluster_partitions` (gauge)
 
 Number of partitions in the layout (this is always 256)
 
 ```
-garage_cluster_partitions 256
+cluster_partitions 256
 ```
 
-#### `garage_cluster_partitions_all_ok` (gauge)
+#### `cluster_partitions_all_ok` (gauge)
 
 Number of partitions for which all storage nodes are connected
 
 ```
-garage_cluster_partitions_all_ok 64
+cluster_partitions_all_ok 64
 ```
 
-#### `garage_cluster_partitions_quorum` (gauge)
+#### `cluster_partitions_quorum` (gauge)
 
 Number of partitions for which we have a quorum of connected nodes and all requests can be served
 
 ```
-garage_cluster_partitions_quorum 256
+cluster_partitions_quorum 256
 ```
 
 ### Metrics of the API endpoints
 
-#### `garage_api_admin_request_count` (counter)
+#### `api_admin_request_counter` (counter)
 
 Counts the number of requests to a given endpoint of the administration API. Example:
 
 ```
-garage_api_admin_request_count{api_endpoint="Metrics"} 127041
+api_admin_request_counter{api_endpoint="Metrics"} 127041
 ```
 
-#### `garage_api_admin_request_duration` (histogram)
+#### `api_admin_request_duration` (histogram)
 
 Evaluates the duration of API calls to the various administration API endpoint. Example:
 
 ```
-garage_api_admin_request_duration_bucket{api_endpoint="Metrics",le="0.5"} 127041
+api_admin_request_duration_bucket{api_endpoint="Metrics",le="0.5"} 127041
-garage_api_admin_request_duration_sum{api_endpoint="Metrics"} 605.250344830999
+api_admin_request_duration_sum{api_endpoint="Metrics"} 605.250344830999
-garage_api_admin_request_duration_count{api_endpoint="Metrics"} 127041
+api_admin_request_duration_count{api_endpoint="Metrics"} 127041
 ```
 
-#### `garage_api_s3_request_count` (counter)
+#### `api_s3_request_counter` (counter)
 
 Counts the number of requests to a given endpoint of the S3 API. Example:
 
 ```
-garage_api_s3_request_count{api_endpoint="CreateMultipartUpload"} 1
+api_s3_request_counter{api_endpoint="CreateMultipartUpload"} 1
 ```
 
-#### `garage_api_s3_error_count` (counter)
+#### `api_s3_error_counter` (counter)
 
 Counts the number of requests to a given endpoint of the S3 API that returned an error. Example:
 
 ```
-garage_api_s3_error_count{api_endpoint="GetObject",status_code="404"} 39
+api_s3_error_counter{api_endpoint="GetObject",status_code="404"} 39
 ```
 
-#### `garage_api_s3_request_duration` (histogram)
+#### `api_s3_request_duration` (histogram)
 
 Evaluates the duration of API calls to the various S3 API endpoints. Example:
 
 ```
-garage_api_s3_request_duration_bucket{api_endpoint="CreateMultipartUpload",le="0.5"} 1
+api_s3_request_duration_bucket{api_endpoint="CreateMultipartUpload",le="0.5"} 1
-garage_api_s3_request_duration_sum{api_endpoint="CreateMultipartUpload"} 0.046340762
+api_s3_request_duration_sum{api_endpoint="CreateMultipartUpload"} 0.046340762
-garage_api_s3_request_duration_count{api_endpoint="CreateMultipartUpload"} 1
+api_s3_request_duration_count{api_endpoint="CreateMultipartUpload"} 1
 ```
 
-#### `garage_api_k2v_request_count` (counter), `garage_api_k2v_error_count` (counter), `garage_api_k2v_error_duration` (histogram)
+#### `api_k2v_request_counter` (counter), `api_k2v_error_counter` (counter), `api_k2v_error_duration` (histogram)
 
 Same as for S3, for the K2V API.
 
@@ -187,45 +187,45 @@ Same as for S3, for the K2V API.
 ### Metrics of the Web endpoint
 
 
-#### `garage_web_request_count` (counter)
+#### `web_request_counter` (counter)
 
 Number of requests to the web endpoint
 
 ```
-garage_web_request_count{method="GET"} 80
+web_request_counter{method="GET"} 80
 ```
 
-#### `garage_web_request_duration` (histogram)
+#### `web_request_duration` (histogram)
 
 Duration of requests to the web endpoint
 
 ```
-garage_web_request_duration_bucket{method="GET",le="0.5"} 80
+web_request_duration_bucket{method="GET",le="0.5"} 80
-garage_web_request_duration_sum{method="GET"} 1.0528433229999998
+web_request_duration_sum{method="GET"} 1.0528433229999998
-garage_web_request_duration_count{method="GET"} 80
+web_request_duration_count{method="GET"} 80
 ```
 
-#### `garage_web_error_count` (counter)
+#### `web_error_counter` (counter)
 
 Number of requests to the web endpoint resulting in errors
 
 ```
-garage_web_error_count{method="GET",status_code="404 Not Found"} 64
+web_error_counter{method="GET",status_code="404 Not Found"} 64
 ```
 
 
 ### Metrics of the data block manager
 
-#### `garage_block_bytes_read`, `garage_block_bytes_written` (counter)
+#### `block_bytes_read`, `block_bytes_written` (counter)
 
 Number of bytes read/written to/from disk in the data storage directory.
 
 ```
-garage_block_bytes_read 120586322022
+block_bytes_read 120586322022
-garage_block_bytes_written 3386618077
+block_bytes_written 3386618077
 ```
 
-#### `garage_block_ram_buffer_free_kb` (gauge)
+#### `block_ram_buffer_free_kb` (gauge)
 
 Kibibytes available for buffering blocks that have to be sent to remote nodes.
 When clients send too much data to this node and a storage node is not receiving
@@ -233,168 +233,170 @@ data fast enough due to slower network conditions, this will decrease down to
 zero and backpressure will be applied.
 
 ```
-garage_block_ram_buffer_free_kb 219829
+block_ram_buffer_free_kb 219829
 ```
 
-#### `garage_block_compression_level` (counter)
+#### `block_compression_level` (counter)
 
 Exposes the block compression level configured for the Garage node.
 
 ```
-garage_block_compression_level 3
+block_compression_level 3
 ```
 
-#### `garage_block_read_duration`, `garage_block_write_duration` (histograms)
+#### `block_read_duration`, `block_write_duration` (histograms)
 
 Evaluates the duration of the reading/writing of individual data blocks in the data storage directory.
 
 ```
-garage_block_read_duration_bucket{le="0.5"} 169229
+block_read_duration_bucket{le="0.5"} 169229
-garage_block_read_duration_sum 2761.6902550310056
+block_read_duration_sum 2761.6902550310056
-garage_block_read_duration_count 169240
+block_read_duration_count 169240
-garage_block_write_duration_bucket{le="0.5"} 3559
+block_write_duration_bucket{le="0.5"} 3559
-garage_block_write_duration_sum 195.59170078500006
+block_write_duration_sum 195.59170078500006
-garage_block_write_duration_count 3571
+block_write_duration_count 3571
 ```
 
-#### `garage_block_delete_count` (counter)
+#### `block_delete_counter` (counter)
 
 Counts the number of data blocks that have been deleted from storage.
 
 ```
-garage_block_delete_count 122
+block_delete_counter 122
 ```
 
-#### `garage_block_resync_count` (counter), `garage_block_resync_duration` (histogram)
+#### `block_resync_counter` (counter), `block_resync_duration` (histogram)
 
 Counts the number of resync operations the node has executed, and evaluates their duration.
 
 ```
-garage_block_resync_count 308897
+block_resync_counter 308897
-garage_block_resync_duration_bucket{le="0.5"} 308892
+block_resync_duration_bucket{le="0.5"} 308892
-garage_block_resync_duration_sum 139.64204196100016
+block_resync_duration_sum 139.64204196100016
-garage_block_resync_duration_count 308897
+block_resync_duration_count 308897
 ```
 
-#### `garage_block_resync_queue_length` (gauge)
+#### `block_resync_queue_length` (gauge)
 
 The number of block hashes currently queued for a resync.
 This is normal to be nonzero for long periods of time.
 
 ```
-garage_block_resync_queue_length 0
+block_resync_queue_length 0
 ```
 
-#### `garage_block_resync_errored_blocks` (gauge)
+#### `block_resync_errored_blocks` (gauge)
 
 The number of block hashes that we were unable to resync last time we tried.
 **THIS SHOULD BE ZERO, OR FALL BACK TO ZERO RAPIDLY, IN A HEALTHY CLUSTER.**
 Persistent nonzero values indicate that some data is likely to be lost.
 
 ```
-garage_block_resync_errored_blocks 0
+block_resync_errored_blocks 0
 ```
 
 
 ### Metrics related to RPCs (remote procedure calls) between nodes
 
-#### `garage_rpc_netapp_request_count` (counter)
+#### `rpc_netapp_request_counter` (counter)
 
 Number of RPC requests emitted
 
 ```
-garage_rpc_request_count{from="<this node>",rpc_endpoint="garage_block/manager.rs/Rpc",to="<remote node>"} 176
+rpc_request_counter{from="<this node>",rpc_endpoint="garage_block/manager.rs/Rpc",to="<remote node>"} 176
 ```
 
-#### `garage_rpc_netapp_error_count` (counter)
+#### `rpc_netapp_error_counter` (counter)
 
 Number of communication errors (errors in the Netapp library, generally due to disconnected nodes)
 
 ```
-garage_rpc_netapp_error_count{from="<this node>",rpc_endpoint="garage_block/manager.rs/Rpc",to="<remote node>"} 354
+rpc_netapp_error_counter{from="<this node>",rpc_endpoint="garage_block/manager.rs/Rpc",to="<remote node>"} 354
 ```
 
-#### `garage_rpc_timeout_count` (counter)
+#### `rpc_timeout_counter` (counter)
 
 Number of RPC timeouts, should be close to zero in a healthy cluster.
 
 ```
-garage_rpc_timeout_count{from="<this node>",rpc_endpoint="garage_rpc/membership.rs/SystemRpc",to="<remote node>"} 1
+rpc_timeout_counter{from="<this node>",rpc_endpoint="garage_rpc/membership.rs/SystemRpc",to="<remote node>"} 1
 ```
 
-#### `garage_rpc_duration` (histogram)
+#### `rpc_duration` (histogram)
 
 The duration of internal RPC calls between Garage nodes.
 
 ```
-garage_rpc_duration_bucket{from="<this node>",rpc_endpoint="garage_block/manager.rs/Rpc",to="<remote node>",le="0.5"} 166
+rpc_duration_bucket{from="<this node>",rpc_endpoint="garage_block/manager.rs/Rpc",to="<remote node>",le="0.5"} 166
-garage_rpc_duration_sum{from="<this node>",rpc_endpoint="garage_block/manager.rs/Rpc",to="<remote node>"} 35.172253716
+rpc_duration_sum{from="<this node>",rpc_endpoint="garage_block/manager.rs/Rpc",to="<remote node>"} 35.172253716
-garage_rpc_duration_count{from="<this node>",rpc_endpoint="garage_block/manager.rs/Rpc",to="<remote node>"} 174
+rpc_duration_count{from="<this node>",rpc_endpoint="garage_block/manager.rs/Rpc",to="<remote node>"} 174
 ```
 
 
 ### Metrics of the metadata table manager
 
-#### `garage_table_gc_todo_queue_length` (gauge)
+#### `table_gc_todo_queue_length` (gauge)
 
 Table garbage collector TODO queue length
 
 ```
-garage_table_gc_todo_queue_length{table_name="block_ref"} 0
+table_gc_todo_queue_length{table_name="block_ref"} 0
 ```
 
-#### `garage_table_get_request_count` (counter), `garage_table_get_request_duration` (histogram)
+#### `table_get_request_counter` (counter), `table_get_request_duration` (histogram)
 
 Number of get/get_range requests internally made on each table, and their duration.
 
 ```
-garage_table_get_request_count{table_name="bucket_alias"} 315
+table_get_request_counter{table_name="bucket_alias"} 315
-garage_table_get_request_duration_bucket{table_name="bucket_alias",le="0.5"} 315
+table_get_request_duration_bucket{table_name="bucket_alias",le="0.5"} 315
-garage_table_get_request_duration_sum{table_name="bucket_alias"} 0.048509778000000024
+table_get_request_duration_sum{table_name="bucket_alias"} 0.048509778000000024
-garage_table_get_request_duration_count{table_name="bucket_alias"} 315
+table_get_request_duration_count{table_name="bucket_alias"} 315
 ```
 
 
-#### `garage_table_put_request_count` (counter), `garage_table_put_request_duration` (histogram)
+#### `table_put_request_counter` (counter), `table_put_request_duration` (histogram)
 
 Number of insert/insert_many requests internally made on this table, and their duration
 
 ```
-garage_table_put_request_count{table_name="block_ref"} 677
+table_put_request_counter{table_name="block_ref"} 677
-garage_table_put_request_duration_bucket{table_name="block_ref",le="0.5"} 677
+table_put_request_duration_bucket{table_name="block_ref",le="0.5"} 677
-garage_table_put_request_duration_sum{table_name="block_ref"} 61.617528636
+table_put_request_duration_sum{table_name="block_ref"} 61.617528636
-garage_table_put_request_duration_count{table_name="block_ref"} 677
+table_put_request_duration_count{table_name="block_ref"} 677
 ```
 
-#### `garage_table_internal_delete_count` (counter)
+#### `table_internal_delete_counter` (counter)
 
 Number of value deletions in the tree (due to GC or repartitioning)
 
 ```
-garage_table_internal_delete_count{table_name="block_ref"} 2296
+table_internal_delete_counter{table_name="block_ref"} 2296
 ```
 
-#### `garage_table_internal_update_count` (counter)
+#### `table_internal_update_counter` (counter)
 
 Number of value updates where the value actually changes (includes creation of new key and update of existing key)
 
 ```
-garage_table_internal_update_count{table_name="block_ref"} 5996
+table_internal_update_counter{table_name="block_ref"} 5996
 ```
 
-#### `garage_table_merkle_updater_todo_queue_length` (gauge)
+#### `table_merkle_updater_todo_queue_length` (gauge)
 
 Merkle tree updater TODO queue length (should fall to zero rapidly)
 
 ```
-garage_table_merkle_updater_todo_queue_length{table_name="block_ref"} 0
+table_merkle_updater_todo_queue_length{table_name="block_ref"} 0
 ```
 
-#### `garage_table_sync_items_received`, `garage_table_sync_items_sent` (counters)
+#### `table_sync_items_received`, `table_sync_items_sent` (counters)
 
 Number of data items sent to/received from other nodes during resync procedures
 
 ```
-garage_table_sync_items_received{from="<remote node>",table_name="bucket_v2"} 3
+table_sync_items_received{from="<remote node>",table_name="bucket_v2"} 3
-garage_table_sync_items_sent{table_name="block_ref",to="<remote node>"} 2
+table_sync_items_sent{table_name="block_ref",to="<remote node>"} 2
 ```
+
+

doc/drafts/k2v-spec.md

@@ -166,6 +166,25 @@ that map to zeroes.  Note that we need to filter out values from nodes that are
 no longer part of the cluster layout, as when nodes are removed they won't
 necessarily have had the time to set their counters to zero.
 
+### Consistency guarantees
+
+K2V provides the following consistency guarantees:
+
+**Read after Write**. After a write has been acknowledged (the request returned
+successfully), a subsequent read is guaranteed to contain the value that was
+written.
+
+**Monotonic Reads**. Two sequential reads will return values in an order that is
+consistent with the order in which they are written (e.g. by concurrent writes).
+For example, consider a scenario where a value is set initially set to 0 and a
+request writing 1 is performed. Doing two subsequent reads concurrently with the
+write is guaranteed to return either `0`, `0` or `0`,`1` or `1`,`1`, but not
+`1`,`0`.
+
+It is also possible to perform non-monotonic reads (allowing this last
+behavior), which are slightly faster than monotonic reads. This is done by
+passing a dedicated flag to read operations (see the endpoints documentation).
+
 ## Important details
 
 **THIS SECTION CONTAINS A FEW WARNINGS ON THE K2V API WHICH ARE IMPORTANT
@@ -210,6 +229,12 @@ Query parameters:
 |------------|---------------|----------------------------------|
 | `sort_key` | **mandatory** | The sort key of the item to read |
 
+Headers:
+
+| name                          | default value | meaning                                  |
+|-------------------------------|---------------|------------------------------------------|
+| `X-Garage-Non-Monotonic-Read` | `false`       | Whether to allow for non-monotonic reads |
+
 Returns the item with specified partition key and sort key. Values can be
 returned in either of two ways:
 
@@ -325,6 +350,12 @@ Query parameters:
 
 The timeout can be set to any number of seconds, with a maximum of 600 seconds (10 minutes).
 
+Headers:
+
+| name                          | default value | meaning                                  |
+|-------------------------------|---------------|------------------------------------------|
+| `X-Garage-Non-Monotonic-Read` | `false`       | Whether to allow for non-monotonic reads |
+
 
 **InsertItem: `PUT /<bucket>/<partition key>?sort_key=<sort_key>`**
 
@@ -521,6 +552,14 @@ HTTP/1.1 204 NO CONTENT
 
 Batch read of triplets in a bucket.
 
+Headers:
+
+| name                          | default value | meaning                                  |
+|-------------------------------|---------------|------------------------------------------|
+| `X-Garage-Non-Monotonic-Read` | `false`       | Whether to allow for non-monotonic reads |
+
+Body:
+
 The request body is a JSON list of searches, that each specify a range of
 items to get (to get single items, set `singleItem` to `true`). A search is a
 JSON struct with the following fields:
@@ -711,6 +750,14 @@ HTTP/1.1 200 OK
 
 Polls a range of items for changes.
 
+Headers:
+
+| name                          | default value | meaning                                  |
+|-------------------------------|---------------|------------------------------------------|
+| `X-Garage-Non-Monotonic-Read` | `false`       | Whether to allow for non-monotonic reads |
+
+Body:
+
 The query body is a JSON object consisting of the following fields:
 
 | name            | default value | meaning                                                                                |

script/telemetry/grafana-garage-dashboard-elasticsearch.json

@@ -161,7 +161,7 @@
           },
           "metrics": [
             {
-              "field": "garage_api_request_count",
+              "field": "api_request_counter",
               "hide": true,
               "id": "1",
               "type": "sum"
@@ -284,7 +284,7 @@
           "hide": false,
           "metrics": [
             {
-              "field": "garage_api_request_duration",
+              "field": "api_request_duration",
               "id": "1",
               "type": "avg"
             }
@@ -412,7 +412,7 @@
           },
           "metrics": [
             {
-              "field": "garage_api_error_count",
+              "field": "api_error_counter",
               "hide": true,
               "id": "1",
               "type": "sum"
@@ -540,7 +540,7 @@
           },
           "metrics": [
             {
-              "field": "garage_web_request_count",
+              "field": "web_request_counter",
               "hide": true,
               "id": "1",
               "type": "sum"
@@ -666,7 +666,7 @@
           "hide": false,
           "metrics": [
             {
-              "field": "garage_web_request_duration",
+              "field": "web_request_duration",
               "id": "1",
               "type": "avg"
             }
@@ -794,7 +794,7 @@
           },
           "metrics": [
             {
-              "field": "garage_web_error_count",
+              "field": "web_error_counter",
               "hide": true,
               "id": "1",
               "type": "sum"
@@ -918,7 +918,7 @@
           "hide": false,
           "metrics": [
             {
-              "field": "garage_table_get_request_count",
+              "field": "table_get_request_counter",
               "hide": true,
               "id": "1",
               "type": "sum"
@@ -1042,7 +1042,7 @@
           "hide": false,
           "metrics": [
             {
-              "field": "garage_table_put_request_count",
+              "field": "table_put_request_counter",
               "hide": true,
               "id": "1",
               "type": "sum"
@@ -1154,7 +1154,7 @@
           "hide": false,
           "metrics": [
             {
-              "field": "garage_block_bytes_read",
+              "field": "block_bytes_read",
               "hide": true,
               "id": "1",
               "type": "sum"
@@ -1270,7 +1270,7 @@
           },
           "metrics": [
             {
-              "field": "garage_block_bytes_written",
+              "field": "block_bytes_written",
               "hide": true,
               "id": "1",
               "type": "sum"
@@ -1386,7 +1386,7 @@
           },
           "metrics": [
             {
-              "field": "garage_block_resync_count",
+              "field": "block_resync_counter",
               "hide": true,
               "id": "1",
               "type": "sum"
@@ -1500,7 +1500,7 @@
           "hide": false,
           "metrics": [
             {
-              "field": "garage_block_resync_queue_length",
+              "field": "block_resync_queue_length",
               "id": "1",
               "type": "avg"
             }
@@ -1610,7 +1610,7 @@
           },
           "metrics": [
             {
-              "field": "garage_table_merkle_updater_todo_queue_length",
+              "field": "table_merkle_updater_todo_queue_length",
               "id": "1",
               "type": "avg"
             }
@@ -1724,7 +1724,7 @@
           },
           "metrics": [
             {
-              "field": "garage_table_gc_todo_queue_length",
+              "field": "table_gc_todo_queue_length",
               "id": "1",
               "type": "avg"
             }
@@ -1824,7 +1824,7 @@
           },
           "metrics": [
             {
-              "field": "garage_block_resync_error_count",
+              "field": "block_resync_error_counter",
               "hide": true,
               "id": "1",
               "settings": {},
@@ -1938,7 +1938,7 @@
           },
           "metrics": [
             {
-              "field": "garage_block_resync_errored_blocks",
+              "field": "block_resync_errored_blocks",
               "hide": false,
               "id": "1",
               "type": "sum"
@@ -2041,7 +2041,7 @@
           },
           "metrics": [
             {
-              "field": "garage_block_corruption_count",
+              "field": "block_corruption_counter",
               "hide": true,
               "id": "1",
               "type": "sum"
@@ -2165,7 +2165,7 @@
           },
           "metrics": [
             {
-              "field": "garage_rpc_netapp_error_count",
+              "field": "rpc_netapp_error_counter",
               "hide": true,
               "id": "1",
               "type": "sum"
@@ -2292,7 +2292,7 @@
           },
           "metrics": [
             {
-              "field": "garage_rpc_request_count",
+              "field": "rpc_request_counter",
               "hide": true,
               "id": "1",
               "type": "sum"
@@ -2418,7 +2418,7 @@
           },
           "metrics": [
             {
-              "field": "garage_rpc_duration",
+              "field": "rpc_duration",
               "id": "1",
               "type": "avg"
             }
@@ -2521,7 +2521,7 @@
           },
           "metrics": [
             {
-              "field": "garage_admin_http_requests_total",
+              "field": "admin_http_requests_total",
               "hide": true,
               "id": "1",
               "type": "sum"
@@ -2654,7 +2654,7 @@
           },
           "metrics": [
             {
-              "field": "garage_rpc_garage_error_count",
+              "field": "rpc_garage_error_counter",
               "hide": true,
               "id": "1",
               "type": "sum"
@@ -2765,7 +2765,7 @@
           },
           "metrics": [
             {
-              "field": "garage_rpc_duration",
+              "field": "rpc_duration",
               "id": "1",
               "type": "avg"
             }
@@ -2995,4 +2995,4 @@
   "uid": "ODT8K4B7e",
   "version": 7,
   "weekStart": ""
-}
+}

script/telemetry/grafana-garage-dashboard-prometheus.json

@@ -143,7 +143,7 @@
             "uid": "${DS_DS_PROMETHEUS}"
           },
           "exemplar": true,
-          "expr": "sum(rate(garage_block_bytes_read{job=\"garage\"}[$__rate_interval])    )",
+          "expr": "sum(rate(block_bytes_read{job=\"garage\"}[$__rate_interval])    )",
           "hide": false,
           "interval": "",
           "legendFormat": "Disk bytes read",
@@ -155,7 +155,7 @@
             "uid": "${DS_DS_PROMETHEUS}"
           },
           "exemplar": true,
-          "expr": "-sum(rate(garage_block_bytes_written{job=\"garage\"}[$__rate_interval])    )",
+          "expr": "-sum(rate(block_bytes_written{job=\"garage\"}[$__rate_interval])    )",
           "hide": false,
           "interval": "",
           "legendFormat": "Disk bytes written",
@@ -250,7 +250,7 @@
           },
           "editorMode": "code",
           "exemplar": true,
-          "expr": "sum by (api_endpoint) (rate(garage_api_s3_request_count {job=\"garage\"}[$__rate_interval]))",
+          "expr": "sum by (api_endpoint) (rate(api_s3_request_counter {job=\"garage\"}[$__rate_interval]))",
           "hide": false,
           "interval": "",
           "legendFormat": "{{api_endpoint}}",
@@ -345,7 +345,7 @@
             "uid": "${DS_DS_PROMETHEUS}"
           },
           "exemplar": true,
-          "expr": "sum(rate(garage_web_request_count {job=\"garage\"}[$__rate_interval]))",
+          "expr": "sum(rate(web_request_counter {job=\"garage\"}[$__rate_interval]))",
           "hide": false,
           "interval": "",
           "legendFormat": "Web request rate",
@@ -439,7 +439,7 @@
             "uid": "${DS_DS_PROMETHEUS}"
           },
           "exemplar": true,
-          "expr": "sum by (rpc_endpoint) (rate(garage_rpc_request_count {job=\"garage\"}[$__rate_interval]))",
+          "expr": "sum by (rpc_endpoint) (rate(rpc_request_counter {job=\"garage\"}[$__rate_interval]))",
           "hide": false,
           "interval": "",
           "legendFormat": "{{rpc_endpoint}}",
@@ -534,7 +534,7 @@
           },
           "editorMode": "code",
           "exemplar": true,
-          "expr": "sum by (api_endpoint, status_code) (rate(garage_api_s3_error_count {job=\"garage\"}[$__rate_interval]))",
+          "expr": "sum by (api_endpoint, status_code) (rate(api_s3_error_counter {job=\"garage\"}[$__rate_interval]))",
           "hide": false,
           "interval": "",
           "legendFormat": "{{api_endpoint}} {{status_code}}",
@@ -629,7 +629,7 @@
             "uid": "${DS_DS_PROMETHEUS}"
           },
           "exemplar": true,
-          "expr": "sum by(status_code) (rate(garage_web_error_count {job=\"garage\"}[$__rate_interval]))",
+          "expr": "sum by(status_code) (rate(web_error_counter {job=\"garage\"}[$__rate_interval]))",
           "hide": false,
           "interval": "",
           "legendFormat": "{{status_code}}",
@@ -722,7 +722,7 @@
             "uid": "${DS_DS_PROMETHEUS}"
           },
           "exemplar": true,
-          "expr": "garage_block_resync_queue_length{job=\"garage\"}",
+          "expr": "block_resync_queue_length{job=\"garage\"}",
           "interval": "",
           "legendFormat": "{{instance}}",
           "refId": "A"
@@ -814,7 +814,7 @@
             "uid": "${DS_DS_PROMETHEUS}"
           },
           "exemplar": true,
-          "expr": "sum by(table_name) (garage_table_gc_todo_queue_length{job=\"garage\"})",
+          "expr": "sum by(table_name) (table_gc_todo_queue_length{job=\"garage\"})",
           "interval": "",
           "legendFormat": "{{ table_name}}",
           "refId": "A"
@@ -906,7 +906,7 @@
             "uid": "${DS_DS_PROMETHEUS}"
           },
           "exemplar": true,
-          "expr": "sum by(table_name) (garage_table_merkle_updater_todo_queue_length{job=\"garage\"})",
+          "expr": "sum by(table_name) (table_merkle_updater_todo_queue_length{job=\"garage\"})",
           "interval": "",
           "legendFormat": "{{ table_name}}",
           "refId": "A"
@@ -998,7 +998,7 @@
             "uid": "${DS_DS_PROMETHEUS}"
           },
           "exemplar": true,
-          "expr": "garage_block_resync_errored_blocks{job=\"garage\"}",
+          "expr": "block_resync_errored_blocks{job=\"garage\"}",
           "interval": "",
           "legendFormat": "{{instance}}",
           "refId": "A"
@@ -1025,4 +1025,4 @@
   "uid": "ys3pnpZ4k",
   "version": 26,
   "weekStart": ""
-}
+}

src/api/admin/bucket.rs

@@ -690,45 +690,36 @@ async fn bucket_info_results(
 		.map(|x| x.filtered_values(&garage.system.cluster_layout()))
 		.unwrap_or_default();
 
-	let mut relevant_keys = HashMap::new();
+	let state = bucket.state.as_option().unwrap();
-	for (k, _) in bucket
+
-		.state
+	let keys1 = state
-		.as_option()
-		.unwrap()
 		.authorized_keys
 		.items()
 		.iter()
-	{
+		.filter(|(_, p)| p.is_any())
-		if let Some(key) = garage
+		.map(|(k, _)| k);
-			.key_table
+	let keys2 = state
-			.get(&EmptyKey, k)
-			.await?
-			.filter(|k| !k.is_deleted())
-		{
-			if !key.state.is_deleted() {
-				relevant_keys.insert(k.clone(), key);
-			}
-		}
-	}
-	for ((k, _), _, _) in bucket
-		.state
-		.as_option()
-		.unwrap()
 		.local_aliases
 		.items()
 		.iter()
-	{
+		.filter(|(_, _, p)| *p)
-		if relevant_keys.contains_key(k) {
+		.map(|((k, _), _, _)| k);
+
+	let mut relevant_keys = HashMap::new();
+	for key_id in keys1.chain(keys2) {
+		if relevant_keys.contains_key(key_id) {
 			continue;
 		}
-		if let Some(key) = garage.key_table.get(&EmptyKey, k).await? {
+		if let Some(key) = garage.key_table.get(&EmptyKey, key_id).await? {
-			if !key.state.is_deleted() {
+			relevant_keys.insert(key_id.clone(), key);
-				relevant_keys.insert(k.clone(), key);
+		} else {
-			}
+			warn!(
+				"Bucket {:?} references non-existent key {}",
+				bucket.id, key_id
+			);
 		}
 	}
-
+	relevant_keys.retain(|_, k| !k.is_deleted());
-	let state = bucket.state.as_option().unwrap();
 
 	let quotas = state.quotas.get();
 	let res = GetBucketInfoResponse {
@@ -768,29 +759,30 @@ async fn bucket_info_results(
 		}),
 		keys: relevant_keys
 			.into_values()
-			.filter_map(|key| {
+			.map(|key| {
-				let p = key.state.as_option().unwrap();
+				let st = key.state.as_option().unwrap();
-				let permissions = p
+				let permissions = st
 					.authorized_buckets
 					.get(&bucket.id)
-					.filter(|p| p.is_any())
 					.map(|p| ApiBucketKeyPerm {
 						read: p.allow_read,
 						write: p.allow_write,
 						owner: p.allow_owner,
-					})?;
+					})
-				Some(GetBucketInfoKey {
+					.unwrap_or_default();
+				let bucket_local_aliases = st
+					.local_aliases
+					.items()
+					.iter()
+					.filter(|(_, _, b)| b.into_inner() == Some(bucket.id))
+					.map(|(n, _, _)| n.to_string())
+					.collect::<Vec<_>>();
+				GetBucketInfoKey {
 					access_key_id: key.key_id,
-					name: p.name.get().to_string(),
+					name: st.name.get().to_string(),
 					permissions,
-					bucket_local_aliases: p
+					bucket_local_aliases,
-						.local_aliases
+				}
-						.items()
-						.iter()
-						.filter(|(_, _, b)| b.into_inner() == Some(bucket.id))
-						.map(|(n, _, _)| n.to_string())
-						.collect::<Vec<_>>(),
-				})
 			})
 			.collect::<Vec<_>>(),
 		objects: *counters.get(OBJECTS).unwrap_or(&0),

src/api/admin/key.rs

@@ -188,30 +188,34 @@ async fn key_info_results(
 	key: Key,
 	show_secret: bool,
 ) -> Result<GetKeyInfoResponse, Error> {
-	let mut relevant_buckets = HashMap::new();
-
 	let key_state = key.state.as_option().unwrap();
 
-	for id in key_state
+	let buckets1 = key_state
 		.authorized_buckets
 		.items()
 		.iter()
-		.map(|(id, _)| id)
+		.filter(|(_, p)| p.is_any())
-		.chain(
+		.map(|(id, _)| id);
-			key_state
+	let buckets2 = key_state
-				.local_aliases
+		.local_aliases
-				.items()
+		.items()
-				.iter()
+		.iter()
-				.filter_map(|(_, _, v)| v.inner()),
+		.filter_map(|(_, _, v)| v.inner());
-		) {
+
-		if !relevant_buckets.contains_key(id) {
+	let mut relevant_buckets = HashMap::new();
-			if let Some(b) = garage.bucket_table.get(&EmptyKey, id).await? {
+	for bucket_id in buckets1.chain(buckets2) {
-				if b.state.as_option().is_some() {
+		if !relevant_buckets.contains_key(bucket_id) {
-					relevant_buckets.insert(*id, b);
+			if let Some(b) = garage.bucket_table.get(&EmptyKey, bucket_id).await? {
-				}
+				relevant_buckets.insert(*bucket_id, b);
+			} else {
+				warn!(
+					"Key {} references non-existent bucket {:?}",
+					key.key_id, bucket_id
+				);
 			}
 		}
 	}
+	relevant_buckets.retain(|_, b| !b.is_deleted());
 
 	let res = GetKeyInfoResponse {
 		name: key_state.name.get().clone(),
@@ -233,7 +237,7 @@ async fn key_info_results(
 		},
 		buckets: relevant_buckets
 			.into_values()
-			.filter_map(|bucket| {
+			.map(|bucket| {
 				let state = bucket.state.as_option().unwrap();
 				let permissions = key_state
 					.authorized_buckets
@@ -243,8 +247,9 @@ async fn key_info_results(
 						read: p.allow_read,
 						write: p.allow_write,
 						owner: p.allow_owner,
-					})?;
+					})
-				Some(KeyInfoBucketResponse {
+					.unwrap_or_default();
+				KeyInfoBucketResponse {
 					id: hex::encode(bucket.id),
 					global_aliases: state
 						.aliases
@@ -261,7 +266,7 @@ async fn key_info_results(
 						.map(|((_, n), _, _)| n.to_string())
 						.collect::<Vec<_>>(),
 					permissions,
-				})
+				}
 			})
 			.collect::<Vec<_>>(),
 	};

src/api/admin/repair.rs

@@ -360,14 +360,7 @@ impl Worker for BlockRcRepair {
 		_must_exit: &mut watch::Receiver<bool>,
 	) -> Result<WorkerState, GarageError> {
 		for _i in 0..RC_REPAIR_ITER_COUNT {
-			let next1 = self
+			let next1 = self.block_manager.rc.get_first_hash_from(self.cursor)?;
-				.block_manager
-				.rc
-				.rc_table
-				.range(self.cursor.as_slice()..)?
-				.next()
-				.transpose()?
-				.map(|(k, _)| Hash::try_from(k.as_slice()).unwrap());
 			let next2 = self
 				.block_ref_table
 				.data

src/api/common/generic_server.rs

@@ -84,21 +84,21 @@ impl<A: ApiHandler> ApiServer<A> {
 			region,
 			api_handler,
 			request_counter: meter
-				.u64_counter(format!("garage_api.{}.request_count", A::API_NAME))
+				.u64_counter(format!("api.{}.request_counter", A::API_NAME))
 				.with_description(format!(
 					"Number of API calls to the various {} API endpoints",
 					A::API_NAME_DISPLAY
 				))
 				.init(),
 			error_counter: meter
-				.u64_counter(format!("garage_api.{}.error_count", A::API_NAME))
+				.u64_counter(format!("api.{}.error_counter", A::API_NAME))
 				.with_description(format!(
 					"Number of API calls to the various {} API endpoints that resulted in errors",
 					A::API_NAME_DISPLAY
 				))
 				.init(),
 			request_duration: meter
-				.f64_value_recorder(format!("garage_api.{}.request_duration", A::API_NAME))
+				.f64_value_recorder(format!("api.{}.request_duration", A::API_NAME))
 				.with_description(format!(
 					"Duration of API calls to the various {} API endpoints",
 					A::API_NAME_DISPLAY

src/api/k2v/batch.rs

@@ -5,12 +5,13 @@ use serde::{Deserialize, Serialize};
 use garage_table::{EnumerationOrder, TableSchema};
 
 use garage_model::k2v::item_table::*;
+use garage_model::k2v::rpc::K2VMonotonicRead;
 
 use garage_api_common::helpers::*;
 
 use crate::api_server::{ReqBody, ResBody};
 use crate::error::*;
-use crate::item::parse_causality_token;
+use crate::item::{is_monotonic_read, parse_causality_token};
 use crate::range::read_range;
 
 pub async fn handle_insert_batch(
@@ -47,12 +48,13 @@ pub async fn handle_read_batch(
 	ctx: ReqCtx,
 	req: Request<ReqBody>,
 ) -> Result<Response<ResBody>, Error> {
+	let monotonic_read = is_monotonic_read(&req)?;
 	let queries = req.into_body().json::<Vec<ReadBatchQuery>>().await?;
 
 	let resp_results = futures::future::join_all(
 		queries
 			.into_iter()
-			.map(|q| handle_read_batch_query(&ctx, q)),
+			.map(|q| handle_read_batch_query(&ctx, q, monotonic_read)),
 	)
 	.await;
 
@@ -67,6 +69,7 @@ pub async fn handle_read_batch(
 async fn handle_read_batch_query(
 	ctx: &ReqCtx,
 	query: ReadBatchQuery,
+	monotonic_read: K2VMonotonicRead,
 ) -> Result<ReadBatchResponse, Error> {
 	let ReqCtx {
 		garage, bucket_id, ..
@@ -90,12 +93,13 @@ async fn handle_read_batch_query(
 			.start
 			.as_ref()
 			.ok_or_bad_request("start should be specified if single_item is set")?;
-		let item = garage
+		let item = match monotonic_read {
-			.k2v
+			K2VMonotonicRead::Monotonic => {
-			.item_table
+				garage.k2v.item_table.get_monotonic(&partition, sk).await?
-			.get(&partition, sk)
+			}
-			.await?
+			K2VMonotonicRead::NonMonotonic => garage.k2v.item_table.get(&partition, sk).await?,
-			.filter(|e| K2VItemTable::matches_filter(e, &filter));
+		}
+		.filter(|e| K2VItemTable::matches_filter(e, &filter));
 		match item {
 			Some(i) => (vec![ReadBatchResponseItem::from(i)], false, None),
 			None => (vec![], false, None),
@@ -110,6 +114,7 @@ async fn handle_read_batch_query(
 			query.limit,
 			Some(filter),
 			EnumerationOrder::from_reverse(query.reverse),
+			monotonic_read,
 		)
 		.await?;
 
@@ -218,6 +223,7 @@ async fn handle_delete_batch_query(
 			None,
 			Some(filter),
 			EnumerationOrder::Forward,
+			K2VMonotonicRead::NonMonotonic,
 		)
 		.await?;
 		assert!(!more);
@@ -260,6 +266,7 @@ pub(crate) async fn handle_poll_range(
 	let ReqCtx {
 		garage, bucket_id, ..
 	} = ctx;
+	let monotonic_read = is_monotonic_read(&req)?;
 	use garage_model::k2v::sub::PollRange;
 
 	let query = req.into_body().json::<PollRangeQuery>().await?;
@@ -281,6 +288,7 @@ pub(crate) async fn handle_poll_range(
 			},
 			query.seen_marker,
 			timeout_msec,
+			monotonic_read,
 		)
 		.await
 		.map_err(pass_helper_error)?;

src/api/k2v/error.rs

@@ -44,6 +44,10 @@ pub enum Error {
 	#[error("Invalid causality token")]
 	InvalidCausalityToken,
 
+	/// Invalid parameter for x-garage-non-monotonic-read
+	#[error("Invalid X-Garage-Non-Monotonic-Read value: {0}")]
+	InvalidNonMonotonicRead(String),
+
 	/// The client asked for an invalid return format (invalid Accept header)
 	#[error("Not acceptable: {0}")]
 	NotAcceptable(String),
@@ -85,6 +89,7 @@ impl Error {
 			Error::InvalidBase64(_) => "InvalidBase64",
 			Error::InvalidUtf8Str(_) => "InvalidUtf8String",
 			Error::InvalidCausalityToken => "CausalityToken",
+			Error::InvalidNonMonotonicRead(_) => "InvalidNonMonotonicRead",
 			Error::InvalidDigest(_) => "InvalidDigest",
 		}
 	}
@@ -101,7 +106,8 @@ impl ApiError for Error {
 			| Error::InvalidBase64(_)
 			| Error::InvalidUtf8Str(_)
 			| Error::InvalidDigest(_)
-			| Error::InvalidCausalityToken => StatusCode::BAD_REQUEST,
+			| Error::InvalidCausalityToken
+			| Error::InvalidNonMonotonicRead(_) => StatusCode::BAD_REQUEST,
 		}
 	}
 

src/api/k2v/index.rs

@@ -4,6 +4,7 @@ use serde::Serialize;
 use garage_table::util::*;
 
 use garage_model::k2v::item_table::{BYTES, CONFLICTS, ENTRIES, VALUES};
+use garage_model::k2v::rpc::K2VMonotonicRead;
 
 use garage_api_common::helpers::*;
 
@@ -40,6 +41,7 @@ pub async fn handle_read_index(
 		limit,
 		Some((DeletedFilter::NotDeleted, node_id_vec)),
 		EnumerationOrder::from_reverse(reverse),
+		K2VMonotonicRead::NonMonotonic,
 	)
 	.await?;
 

src/api/k2v/item.rs

@@ -5,6 +5,7 @@ use hyper::{Request, Response, StatusCode};
 
 use garage_model::k2v::causality::*;
 use garage_model::k2v::item_table::*;
+use garage_model::k2v::rpc::K2VMonotonicRead;
 
 use garage_api_common::helpers::*;
 
@@ -12,6 +13,7 @@ use crate::api_server::{ReqBody, ResBody};
 use crate::error::*;
 
 pub const X_GARAGE_CAUSALITY_TOKEN: &str = "X-Garage-Causality-Token";
+pub const X_GARAGE_NON_MONOTONIC_READ: &str = "X-Garage-Non-Monotonic-Read";
 
 pub enum ReturnFormat {
 	Json,
@@ -23,6 +25,21 @@ pub(crate) fn parse_causality_token(s: &str) -> Result<CausalContext, Error> {
 	CausalContext::parse(s).ok_or(Error::InvalidCausalityToken)
 }
 
+pub(crate) fn is_monotonic_read(req: &Request<ReqBody>) -> Result<K2VMonotonicRead, Error> {
+	let v_opt = req
+		.headers()
+		.get(X_GARAGE_NON_MONOTONIC_READ)
+		.map(|s| s.to_str())
+		.transpose()?;
+
+	match v_opt {
+		Some("true") => Ok(K2VMonotonicRead::NonMonotonic),
+		// Reads are monotonic by default
+		Some("false") | None => Ok(K2VMonotonicRead::Monotonic),
+		Some(s) => Err(Error::InvalidNonMonotonicRead(s.to_string())),
+	}
+}
+
 impl ReturnFormat {
 	pub fn from(req: &Request<ReqBody>) -> Result<Self, Error> {
 		let accept = match req.headers().get(header::ACCEPT) {
@@ -108,21 +125,26 @@ pub async fn handle_read_item(
 	let ReqCtx {
 		garage, bucket_id, ..
 	} = &ctx;
-
+	let monotonic_read = is_monotonic_read(req)?;
 	let format = ReturnFormat::from(req)?;
+	let partition_key = K2VItemPartition {
+		bucket_id: *bucket_id,
+		partition_key: partition_key.to_string(),
+	};
 
-	let item = garage
+	let item = match monotonic_read {
-		.k2v
+		K2VMonotonicRead::Monotonic => {
-		.item_table
+			garage
-		.get(
+				.k2v
-			&K2VItemPartition {
+				.item_table
-				bucket_id: *bucket_id,
+				.get_monotonic(&partition_key, sort_key)
-				partition_key: partition_key.to_string(),
+				.await?
-			},
+		}
-			sort_key,
+		K2VMonotonicRead::NonMonotonic => {
-		)
+			garage.k2v.item_table.get(&partition_key, sort_key).await?
-		.await?
+		}
-		.ok_or(Error::NoSuchKey)?;
+	}
+	.ok_or(Error::NoSuchKey)?;
 
 	format.make_response(&item)
 }
@@ -214,6 +236,7 @@ pub async fn handle_poll_item(
 	let ReqCtx {
 		garage, bucket_id, ..
 	} = &ctx;
+	let monotonic_read = is_monotonic_read(req)?;
 	let format = ReturnFormat::from(req)?;
 
 	let causal_context =
@@ -230,6 +253,7 @@ pub async fn handle_poll_item(
 			sort_key,
 			causal_context,
 			timeout_msec,
+			monotonic_read,
 		)
 		.await?;
 

src/api/k2v/range.rs

@@ -4,6 +4,8 @@
 
 use std::sync::Arc;
 
+use garage_model::k2v::rpc::K2VMonotonicRead;
+
 use garage_table::replication::TableShardedReplication;
 use garage_table::*;
 
@@ -23,6 +25,7 @@ pub(crate) async fn read_range<F>(
 	limit: Option<u64>,
 	filter: Option<F::Filter>,
 	enumeration_order: EnumerationOrder,
+	monotonic_read: K2VMonotonicRead,
 ) -> Result<(Vec<F::E>, bool, Option<String>), Error>
 where
 	F: TableSchema<S = String> + 'static,
@@ -53,15 +56,30 @@ where
 			1000,
 			limit.map(|x| x as usize).unwrap_or(usize::MAX - 10) - entries.len() + 2,
 		);
-		let get_ret = table
+		let get_ret = match monotonic_read {
-			.get_range(
+			K2VMonotonicRead::Monotonic => {
-				partition_key,
+				table
-				start.clone(),
+					.get_range_monotonic(
-				filter.clone(),
+						partition_key,
-				n_get,
+						start.clone(),
-				enumeration_order,
+						filter.clone(),
-			)
+						n_get,
-			.await?;
+						enumeration_order,
+					)
+					.await?
+			}
+			K2VMonotonicRead::NonMonotonic => {
+				table
+					.get_range(
+						partition_key,
+						start.clone(),
+						filter.clone(),
+						n_get,
+						enumeration_order,
+					)
+					.await?
+			}
+		};
 
 		let get_ret_len = get_ret.len();
 

src/api/s3/delete.rs

@@ -83,8 +83,21 @@ pub async fn handle_delete_objects(
 				}
 				ret_deleted.push(s3_xml::Deleted {
 					key: s3_xml::Value(obj.key.clone()),
-					version_id: s3_xml::Value(hex::encode(deleted_version)),
+					version_id: Some(s3_xml::Value(hex::encode(deleted_version))),
-					delete_marker_version_id: s3_xml::Value(hex::encode(delete_marker_version)),
+					delete_marker_version_id: Some(s3_xml::Value(hex::encode(
+						delete_marker_version,
+					))),
+				});
+			}
+			Err(Error::NoSuchKey) => {
+				if cmd.quiet {
+					continue;
+				}
+				// Deleting a non-existent key is a success in S3
+				ret_deleted.push(s3_xml::Deleted {
+					key: s3_xml::Value(obj.key.clone()),
+					version_id: None,
+					delete_marker_version_id: None,
 				});
 			}
 			Err(e) => {

src/api/s3/post_object.rs

@@ -83,10 +83,12 @@ pub async fn handle_post_object(
 	};
 
 	// Current part is file. Do some checks before handling to PutObject code
-	let key = params
+	let key = std::str::from_utf8(
-		.get("key")
+		params
-		.ok_or_bad_request("No key was provided")?
+			.get("key")
-		.to_str()?;
+			.ok_or_bad_request("No key was provided")?
+			.as_bytes(),
+	)?;
 	let policy = params
 		.get("policy")
 		.ok_or_bad_request("No policy was provided")?

src/api/s3/put.rs

@@ -679,7 +679,10 @@ pub(crate) fn extract_metadata_headers(
 	];
 	for name in standard_header.iter() {
 		if let Some(value) = headers.get(name) {
-			ret.push((name.to_string(), value.to_str()?.to_string()));
+			ret.push((
+				name.to_string(),
+				std::str::from_utf8(value.as_bytes())?.to_string(),
+			));
 		}
 	}
 

src/api/s3/router.rs

@@ -315,11 +315,7 @@ impl Endpoint {
 		bucket: Option<String>,
 	) -> Result<(Self, Option<String>), Error> {
 		let uri = req.uri();
-		let path = uri.path().strip_prefix('/');
+		let path = uri.path().trim_start_matches('/');
-		if path.is_none() {
-			return Err(Error::bad_request("URI path must start with a '/'"));
-		}
-		let path = path.unwrap();
 		let query = uri.query();
 		if bucket.is_none() && path.is_empty() {
 			if *req.method() == Method::OPTIONS {
@@ -333,7 +329,7 @@ impl Endpoint {
 			(bucket, path)
 		} else {
 			path.split_once('/')
-				.map(|(b, p)| (b.to_owned(), p))
+				.map(|(b, p)| (b.to_owned(), p.trim_start_matches('/')))
 				.unwrap_or_else(|| (path.to_owned(), ""))
 		};
 
@@ -847,40 +843,6 @@ mod tests {
 			"&+?%é/something"
 		);
 
-		// A double-slash in the URL means the key begins with '/'.
-		// path-style: HEAD /bucket// → key "/"
-		assert_eq!(
-			parse("HEAD", "/my_bucket//", None, None)
-				.0
-				.get_key()
-				.unwrap(),
-			"/"
-		);
-		// virtual-hosted-style: HEAD // → key "/"
-		assert_eq!(
-			parse("HEAD", "//", Some("my_bucket".to_owned()), None)
-				.0
-				.get_key()
-				.unwrap(),
-			"/"
-		);
-		// same for GET: path-style GET /bucket// → key "/"
-		assert_eq!(
-			parse("GET", "/my_bucket//", None, None)
-				.0
-				.get_key()
-				.unwrap(),
-			"/"
-		);
-		// virtual-hosted-style: GET // → key "/"
-		assert_eq!(
-			parse("GET", "//", Some("my_bucket".to_owned()), None)
-				.0
-				.get_key()
-				.unwrap(),
-			"/"
-		);
-
 		/*
 		 * this case is failing. We should verify how clients encode space in url
 		assert_eq!(
@@ -971,7 +933,6 @@ mod tests {
 			GET "/{Key+}?torrent" => GetObjectTorrent
 			GET "/?publicAccessBlock" => GetPublicAccessBlock
 			HEAD "/" => HeadBucket
-			HEAD "//" => HeadObject
 			HEAD "/my-image.jpg" => HeadObject
 			HEAD "/my-image.jpg?versionId=3HL4kqCxf3vjVBH40Nrjfkd" => HeadObject
 			HEAD "/Key+?partNumber=3&versionId=VersionId" => HeadObject
@@ -988,7 +949,6 @@ mod tests {
 			GET "/?uploads&delimiter=/&prefix=photos/2006/" => ListMultipartUploads
 			GET "/?uploads&delimiter=D&encoding-type=EncodingType&key-marker=KeyMarker&max-uploads=1&prefix=Prefix&upload-id-marker=UploadIdMarker" => ListMultipartUploads
 			GET "/" => ListObjects
-			GET "//" => GetObject
 			GET "/?prefix=N&marker=Need&max-keys=40" => ListObjects
 			GET "/?delimiter=/" => ListObjects
 			GET "/?prefix=photos/2006/&delimiter=/" => ListObjects

src/api/s3/xml.rs

@@ -44,10 +44,13 @@ pub struct LocationConstraint {
 pub struct Deleted {
 	#[serde(rename = "Key")]
 	pub key: Value,
-	#[serde(rename = "VersionId")]
+	#[serde(rename = "VersionId", skip_serializing_if = "Option::is_none")]
-	pub version_id: Value,
+	pub version_id: Option<Value>,
-	#[serde(rename = "DeleteMarkerVersionId")]
+	#[serde(
-	pub delete_marker_version_id: Value,
+		rename = "DeleteMarkerVersionId",
+		skip_serializing_if = "Option::is_none"
+	)]
+	pub delete_marker_version_id: Option<Value>,
 }
 
 #[derive(Debug, Serialize, PartialEq, Eq)]
@@ -497,13 +500,13 @@ mod tests {
 			deleted: vec![
 				Deleted {
 					key: Value("a/plop".to_string()),
-					version_id: Value("qsdfjklm".to_string()),
+					version_id: Some(Value("qsdfjklm".to_string())),
-					delete_marker_version_id: Value("wxcvbn".to_string()),
+					delete_marker_version_id: Some(Value("wxcvbn".to_string())),
 				},
 				Deleted {
 					key: Value("b/plip".to_string()),
-					version_id: Value("1234".to_string()),
+					version_id: Some(Value("1234".to_string())),
-					delete_marker_version_id: Value("4321".to_string()),
+					delete_marker_version_id: Some(Value("4321".to_string())),
 				},
 			],
 			errors: vec![

src/block/manager.rs

@@ -144,7 +144,7 @@ impl BlockManager {
 
 		// Open metadata tables
 		let rc = db
-			.open_tree("block_local_rc")
+			.open_typed_tree("block_local_rc")
 			.expect("Unable to open block_local_rc tree");
 		let rc = BlockRc::new(rc);
 
@@ -158,9 +158,9 @@ impl BlockManager {
 
 		let metrics = BlockManagerMetrics::new(
 			config.compression_level,
-			rc.rc_table.clone(),
+			rc.rc_table.untyped().clone(),
-			resync.queue.clone(),
+			resync.queue.untyped().clone(),
-			resync.errors.clone(),
+			resync.errors.untyped().clone(),
 			buffer_kb_semaphore.clone(),
 		);
 
@@ -449,9 +449,8 @@ impl BlockManager {
 		let mut blocks = Vec::with_capacity(self.resync.errors.approximate_len()?);
 		for ent in self.resync.errors.iter()? {
 			let (hash, cnt) = ent?;
-			let cnt = ErrorCounter::decode(&cnt);
 			blocks.push(BlockResyncErrorInfo {
-				hash: Hash::try_from(&hash).unwrap(),
+				hash,
 				refcount: 0,
 				error_count: cnt.errors,
 				last_try: cnt.last_try,

src/block/metrics.rs

@@ -41,7 +41,7 @@ impl BlockManagerMetrics {
 		let meter = global::meter("garage_model/block");
 		Self {
 			_compression_level: meter
-				.u64_value_observer("garage_block.compression_level", move |observer| {
+				.u64_value_observer("block.compression_level", move |observer| {
 					match compression_level {
 						Some(v) => observer.observe(v as u64, &[]),
 						None => observer.observe(0_u64, &[]),
@@ -50,7 +50,7 @@ impl BlockManagerMetrics {
 				.with_description("Garage compression level for node")
 				.init(),
 			_rc_size: meter
-				.u64_value_observer("garage_block.rc_size", move |observer| {
+				.u64_value_observer("block.rc_size", move |observer| {
 					if let Ok(value) = rc_tree.approximate_len() {
 						observer.observe(value as u64, &[]);
 					}
@@ -58,7 +58,7 @@ impl BlockManagerMetrics {
 				.with_description("Number of blocks known to the reference counter")
 				.init(),
 			_resync_queue_len: meter
-				.u64_value_observer("garage_block.resync_queue_length", move |observer| {
+				.u64_value_observer("block.resync_queue_length", move |observer| {
 					if let Ok(value) = resync_queue.approximate_len() {
 						observer.observe(value as u64, &[]);
 					}
@@ -68,7 +68,7 @@ impl BlockManagerMetrics {
 				)
 				.init(),
 			_resync_errored_blocks: meter
-				.u64_value_observer("garage_block.resync_errored_blocks", move |observer| {
+				.u64_value_observer("block.resync_errored_blocks", move |observer| {
 					if let Ok(value) = resync_errors.approximate_len() {
 						observer.observe(value as u64, &[]);
 					}
@@ -77,7 +77,7 @@ impl BlockManagerMetrics {
 				.init(),
 
 			_buffer_free_kb: meter
-				.u64_value_observer("garage_block.ram_buffer_free_kb", move |observer| {
+				.u64_value_observer("block.ram_buffer_free_kb", move |observer| {
 					observer.observe(buffer_semaphore.available_permits() as u64, &[]);
 				})
 				.with_description(
@@ -86,63 +86,63 @@ impl BlockManagerMetrics {
 				.init(),
 
 			resync_counter: meter
-				.u64_counter("garage_block.resync_count")
+				.u64_counter("block.resync_counter")
 				.with_description("Number of calls to resync_block")
 				.init()
 				.bind(&[]),
 			resync_error_counter: meter
-				.u64_counter("garage_block.resync_error_count")
+				.u64_counter("block.resync_error_counter")
 				.with_description("Number of calls to resync_block that returned an error")
 				.init()
 				.bind(&[]),
 			resync_duration: meter
-				.f64_value_recorder("garage_block.resync_duration")
+				.f64_value_recorder("block.resync_duration")
 				.with_description("Duration of resync_block operations")
 				.init()
 				.bind(&[]),
 			resync_send_counter: meter
-				.u64_counter("garage_block.resync_send_count")
+				.u64_counter("block.resync_send_counter")
 				.with_description("Number of blocks sent to another node in resync operations")
 				.init(),
 			resync_recv_counter: meter
-				.u64_counter("garage_block.resync_recv_count")
+				.u64_counter("block.resync_recv_counter")
 				.with_description("Number of blocks received from other nodes in resync operations")
 				.init()
 				.bind(&[]),
 
 			bytes_read: meter
-				.u64_counter("garage_block.bytes_read")
+				.u64_counter("block.bytes_read")
 				.with_description("Number of bytes read from disk")
 				.init()
 				.bind(&[]),
 			block_read_duration: meter
-				.f64_value_recorder("garage_block.read_duration")
+				.f64_value_recorder("block.read_duration")
 				.with_description("Duration of block read operations")
 				.init()
 				.bind(&[]),
 			block_read_semaphore_timeouts: meter
-				.u64_counter("garage_block.read_semaphore_timeouts")
+				.u64_counter("block.read_semaphore_timeouts")
 				.with_description("Number of block reads that failed due to semaphore acquire timeout")
 				.init()
 				.bind(&[]),
 			bytes_written: meter
-				.u64_counter("garage_block.bytes_written")
+				.u64_counter("block.bytes_written")
 				.with_description("Number of bytes written to disk")
 				.init()
 				.bind(&[]),
 			block_write_duration: meter
-				.f64_value_recorder("garage_block.write_duration")
+				.f64_value_recorder("block.write_duration")
 				.with_description("Duration of block write operations")
 				.init()
 				.bind(&[]),
 			delete_counter: meter
-				.u64_counter("garage_block.delete_count")
+				.u64_counter("block.delete_counter")
 				.with_description("Number of blocks deleted")
 				.init()
 				.bind(&[]),
 
 			corruption_counter: meter
-				.u64_counter("garage_block.corruption_count")
+				.u64_counter("block.corruption_counter")
 				.with_description("Data corruptions detected on block reads")
 				.init()
 				.bind(&[]),

src/block/rc.rs

@@ -1,4 +1,5 @@
 use std::convert::TryInto;
+use std::num::NonZeroU64;
 
 use arc_swap::ArcSwapOption;
 
@@ -14,12 +15,12 @@ pub type CalculateRefcount =
 	Box<dyn Fn(&db::Transaction, &Hash) -> db::TxResult<usize, Error> + Send + Sync>;
 
 pub struct BlockRc {
-	pub rc_table: db::Tree,
+	pub(crate) rc_table: db::TypedTree<Hash, RcEntry>,
 	pub(crate) recalc_rc: ArcSwapOption<Vec<CalculateRefcount>>,
 }
 
 impl BlockRc {
-	pub(crate) fn new(rc: db::Tree) -> Self {
+	pub(crate) fn new(rc: db::TypedTree<Hash, RcEntry>) -> Self {
 		Self {
 			rc_table: rc,
 			recalc_rc: ArcSwapOption::new(None),
@@ -33,11 +34,8 @@ impl BlockRc {
 		tx: &mut db::Transaction,
 		hash: &Hash,
 	) -> db::TxOpResult<bool> {
-		let old_rc = RcEntry::parse_opt(tx.get(&self.rc_table, hash)?);
+		let old_rc = RcState(self.rc_table.tx_get(tx, hash)?);
-		match old_rc.increment().serialize() {
+		self.rc_table.tx_insert(tx, hash, &old_rc.increment())?;
-			Some(x) => tx.insert(&self.rc_table, hash, x)?,
-			None => unreachable!(),
-		}
 		Ok(old_rc.is_zero())
 	}
 
@@ -48,17 +46,27 @@ impl BlockRc {
 		tx: &mut db::Transaction,
 		hash: &Hash,
 	) -> db::TxOpResult<bool> {
-		let new_rc = RcEntry::parse_opt(tx.get(&self.rc_table, hash)?).decrement();
+		let new_rc = RcState(self.rc_table.tx_get(tx, hash)?).decrement();
-		match new_rc.serialize() {
+		match &new_rc.0 {
-			Some(x) => tx.insert(&self.rc_table, hash, x)?,
+			None => self.rc_table.tx_remove(tx, hash)?,
-			None => tx.remove(&self.rc_table, hash)?,
+			Some(rc) => self.rc_table.tx_insert(tx, hash, rc)?,
 		}
-		Ok(matches!(new_rc, RcEntry::Deletable { .. }))
+		Ok(matches!(new_rc.0, Some(RcEntry::Deletable { .. })))
 	}
 
-	/// Read a block's reference count
+	/// Read a block's reference counting state
-	pub(crate) fn get_block_rc(&self, hash: &Hash) -> Result<RcEntry, Error> {
+	pub(crate) fn get_block_rc(&self, hash: &Hash) -> Result<RcState, Error> {
-		Ok(RcEntry::parse_opt(self.rc_table.get(hash.as_ref())?))
+		Ok(RcState(self.rc_table.get(hash)?))
+	}
+
+	/// Return the first hash stored in the RC table at or after `cursor`
+	pub fn get_first_hash_from(&self, cursor: Hash) -> Result<Option<Hash>, Error> {
+		Ok(self
+			.rc_table
+			.range(cursor..)?
+			.next()
+			.transpose()?
+			.map(|(k, _)| k))
 	}
 
 	/// Delete an entry in the RC table if it is deletable and the
@@ -66,12 +74,11 @@ impl BlockRc {
 	pub(crate) fn clear_deleted_block_rc(&self, hash: &Hash) -> Result<(), Error> {
 		let now = now_msec();
 		self.rc_table.db().transaction(|tx| {
-			let rcval = RcEntry::parse_opt(tx.get(&self.rc_table, hash)?);
+			let rcval = self.rc_table.tx_get(tx, hash)?;
-			match rcval {
+			if let Some(RcEntry::Deletable { at_time }) = rcval {
-				RcEntry::Deletable { at_time } if now > at_time => {
+				if now > at_time {
-					tx.remove(&self.rc_table, hash)?;
+					self.rc_table.tx_remove(tx, hash)?;
 				}
-				_ => (),
 			}
 			Ok(())
 		})?;
@@ -91,28 +98,25 @@ impl BlockRc {
 					for f in recalc_fns.iter() {
 						cnt += f(tx, hash)?;
 					}
-					let old_rc = RcEntry::parse_opt(tx.get(&self.rc_table, hash)?);
+					let old_count = RcState(self.rc_table.tx_get(tx, hash)?).as_u64();
 					trace!(
 						"Block RC for {:?}: stored={}, calculated={}",
 						hash,
-						old_rc.as_u64(),
+						old_count,
 						cnt
 					);
-					if cnt as u64 != old_rc.as_u64() {
+					if cnt as u64 != old_count {
 						warn!(
 							"Fixing inconsistent block RC for {:?}: was {}, should be {}",
-							hash,
+							hash, old_count, cnt
-							old_rc.as_u64(),
-							cnt
 						);
-						let new_rc = if cnt > 0 {
+						let new_rc = match NonZeroU64::new(cnt as u64) {
-							RcEntry::Present { count: cnt as u64 }
+							Some(count) => RcEntry::Present { count },
-						} else {
+							None => RcEntry::Deletable {
-							RcEntry::Deletable {
 								at_time: now_msec() + BLOCK_GC_DELAY.as_millis() as u64,
-							}
+							},
 						};
-						tx.insert(&self.rc_table, hash, new_rc.serialize().unwrap())?;
+						self.rc_table.tx_insert(tx, hash, &new_rc)?;
 						Ok((cnt, true))
 					} else {
 						Ok((cnt, false))
@@ -131,13 +135,47 @@ impl BlockRc {
 	}
 }
 
-/// Describes the state of the reference counter for a block
+impl db::DbBytes for RcEntry {
+	fn encode(&self) -> Vec<u8> {
+		match self {
+			RcEntry::Present { count } => u64::to_be_bytes(count.get()).to_vec(),
+			RcEntry::Deletable { at_time } => {
+				[u64::to_be_bytes(0), u64::to_be_bytes(*at_time)].concat()
+			}
+		}
+	}
+
+	fn decode(bytes: &[u8]) -> std::result::Result<Self, db::DecodeError> {
+		if bytes.len() == 8 {
+			let count = NonZeroU64::new(u64::from_be_bytes(bytes.try_into().unwrap()))
+				.ok_or(db::DecodeError("invalid RC entry: zero count".into()))?;
+			Ok(RcEntry::Present { count })
+		} else if bytes.len() == 16 {
+			Ok(RcEntry::Deletable {
+				at_time: u64::from_be_bytes(bytes[8..16].try_into().unwrap()),
+			})
+		} else {
+			Err(db::DecodeError(
+				format!(
+					"invalid RC entry: expected 8 or 16 bytes, got {}",
+					bytes.len()
+				)
+				.into(),
+			))
+		}
+	}
+}
+
+/// A block's entry in the RC table.
+///
+/// A block with zero references and no pending deletion has no entry
+/// in the RC table at all: see [`RcState`].
 #[derive(Clone, Copy, Debug)]
 pub(crate) enum RcEntry {
-	/// Present: the block has `count` references, with `count` > 0.
+	/// Present: the block has `count` references.
 	///
 	/// This is stored as `u64::to_be_bytes(count)`
-	Present { count: u64 },
+	Present { count: NonZeroU64 },
 
 	/// Deletable: the block has zero references, and can be deleted
 	/// once time (returned by `now_msec`) is larger than `at_time`
@@ -147,72 +185,39 @@ pub(crate) enum RcEntry {
 	/// (this allows for the data format to be backwards compatible with
 	/// previous Garage versions that didn't have this intermediate state)
 	Deletable { at_time: u64 },
-
-	/// Absent: the block has zero references, and can be deleted
-	/// immediately
-	Absent,
 }
 
-impl RcEntry {
+/// Describes the state of the reference counter for a block: the block's
-	fn parse(bytes: &[u8]) -> Self {
+/// entry in the RC table, or `None` if it has none, meaning the block has
-		if bytes.len() == 8 {
+/// zero references and can be deleted immediately.
-			RcEntry::Present {
+#[derive(Clone, Copy, Debug)]
-				count: u64::from_be_bytes(bytes.try_into().unwrap()),
+pub(crate) struct RcState(Option<RcEntry>);
-			}
-		} else if bytes.len() == 16 {
-			RcEntry::Deletable {
-				at_time: u64::from_be_bytes(bytes[8..16].try_into().unwrap()),
-			}
-		} else {
-			panic!("Invalid RC entry: {:?}, database is corrupted. This is an error Garage is currently unable to recover from. Sorry, and also please report a bug.",
-				bytes
-			)
-		}
-	}
 
-	fn parse_opt<V: AsRef<[u8]>>(bytes: Option<V>) -> Self {
+impl RcState {
-		bytes
+	/// The new RC table entry after a reference is taken on the block
-			.map(|b| Self::parse(b.as_ref()))
+	fn increment(&self) -> RcEntry {
-			.unwrap_or(Self::Absent)
+		let count = match self.0 {
-	}
+			Some(RcEntry::Present { count }) => count.saturating_add(1),
-
+			_ => NonZeroU64::new(1).unwrap(),
-	fn serialize(self) -> Option<Vec<u8>> {
-		match self {
-			RcEntry::Present { count } => Some(u64::to_be_bytes(count).to_vec()),
-			RcEntry::Deletable { at_time } => {
-				Some([u64::to_be_bytes(0), u64::to_be_bytes(at_time)].concat())
-			}
-			RcEntry::Absent => None,
-		}
-	}
-
-	fn increment(self) -> Self {
-		let old_count = match self {
-			RcEntry::Present { count } => count,
-			_ => 0,
 		};
-		RcEntry::Present {
+		RcEntry::Present { count }
-			count: old_count + 1,
-		}
 	}
 
-	fn decrement(self) -> Self {
+	/// The new state after a reference to the block is dropped
-		match self {
+	fn decrement(&self) -> Self {
-			RcEntry::Present { count } => {
+		RcState(match self.0 {
-				if count > 1 {
+			Some(RcEntry::Present { count }) => Some(match NonZeroU64::new(count.get() - 1) {
-					RcEntry::Present { count: count - 1 }
+				Some(count) => RcEntry::Present { count },
-				} else {
+				None => RcEntry::Deletable {
-					RcEntry::Deletable {
+					at_time: now_msec() + BLOCK_GC_DELAY.as_millis() as u64,
-						at_time: now_msec() + BLOCK_GC_DELAY.as_millis() as u64,
+				},
-					}
+			}),
-				}
+			unchanged => unchanged,
-			}
+		})
-			del => del,
-		}
 	}
 
 	pub(crate) fn is_zero(&self) -> bool {
-		matches!(self, RcEntry::Deletable { .. } | RcEntry::Absent)
+		matches!(self.0, None | Some(RcEntry::Deletable { .. }))
 	}
 
 	pub(crate) fn is_nonzero(&self) -> bool {
@@ -220,10 +225,10 @@ impl RcEntry {
 	}
 
 	pub(crate) fn is_deletable(&self) -> bool {
-		match self {
+		match self.0 {
-			RcEntry::Present { .. } => false,
+			Some(RcEntry::Present { .. }) => false,
-			RcEntry::Deletable { at_time } => now_msec() > *at_time,
+			Some(RcEntry::Deletable { at_time }) => now_msec() > at_time,
-			RcEntry::Absent => true,
+			None => true,
 		}
 	}
 
@@ -232,8 +237,8 @@ impl RcEntry {
 	}
 
 	pub(crate) fn as_u64(&self) -> u64 {
-		match self {
+		match self.0 {
-			RcEntry::Present { count } => *count,
+			Some(RcEntry::Present { count }) => count.get(),
 			_ => 0,
 		}
 	}

src/block/repair.rs

@@ -102,16 +102,15 @@ impl Worker for RepairWorker {
 				let mut batch_of_hashes = vec![];
 				let start_bound = match self.next_start.as_ref() {
 					None => Bound::Unbounded,
-					Some(x) => Bound::Excluded(x.as_slice()),
+					Some(x) => Bound::Excluded(*x),
 				};
 				for entry in self
 					.manager
 					.rc
 					.rc_table
-					.range::<&[u8], _>((start_bound, Bound::Unbounded))?
+					.range((start_bound, Bound::Unbounded))?
 				{
 					let (hash, _) = entry?;
-					let hash = Hash::try_from(&hash[..]).unwrap();
 					batch_of_hashes.push(hash);
 					if batch_of_hashes.len() >= 1000 {
 						break;

src/block/resync.rs

@@ -44,15 +44,58 @@ pub(crate) const MAX_RESYNC_WORKERS: usize = 8;
 const INITIAL_RESYNC_TRANQUILITY: u32 = 2;
 
 pub struct BlockResyncManager {
-	pub(crate) queue: db::Tree,
+	pub(crate) queue: db::TypedTree<ResyncQueueKey, Hash>,
 	pub(crate) notify: Arc<Notify>,
-	pub(crate) errors: db::Tree,
+	pub(crate) errors: db::TypedTree<Hash, ErrorCounter>,
 
 	busy_set: BusySet,
 
 	persister: PersisterShared<ResyncPersistedConfig>,
 }
 
+/// Key of the resync queue tree: blocks are resynced in order of increasing
+/// `when` (msec timestamp of the next try), with the block hash as tie-breaker.
+///
+// CAREFUL: this type implements `DbOrdKey`, so its byte encoding must be
+// order-preserving.
+// The derived `Ord` compares fields in declaration order, which must match
+// the order in which `encode()` writes them; and `when` must remain an
+// *unsigned* integer, as the big-endian encoding is only order-preserving
+// for unsigned types.
+#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub(crate) struct ResyncQueueKey {
+	pub(crate) when: u64,
+	pub(crate) hash: Hash,
+}
+
+impl db::DbBytes for ResyncQueueKey {
+	fn encode(&self) -> Vec<u8> {
+		let mut v = Vec::with_capacity(40);
+		v.extend_from_slice(&u64::to_be_bytes(self.when));
+		v.extend_from_slice(self.hash.as_slice());
+		v
+	}
+
+	fn decode(bytes: &[u8]) -> std::result::Result<Self, db::DecodeError> {
+		if bytes.len() != 40 {
+			return Err(db::DecodeError(
+				format!(
+					"invalid resync queue key: expected 40 bytes, got {}",
+					bytes.len()
+				)
+				.into(),
+			));
+		}
+		Ok(ResyncQueueKey {
+			when: u64::from_be_bytes(bytes[..8].try_into().unwrap()),
+			hash: Hash::try_from(&bytes[8..])
+				.ok_or_else(|| db::DecodeError("invalid resync queue key: bad hash".into()))?,
+		})
+	}
+}
+
+impl db::DbOrdKey for ResyncQueueKey {}
+
 #[derive(Serialize, Deserialize, Clone, Copy)]
 struct ResyncPersistedConfig {
 	n_workers: usize,
@@ -74,22 +117,21 @@ enum ResyncIterResult {
 	IdleFor(Duration),
 }
 
-type BusySet = Arc<Mutex<HashSet<Vec<u8>>>>;
+type BusySet = Arc<Mutex<HashSet<ResyncQueueKey>>>;
 
 struct BusyBlock {
-	time_bytes: Vec<u8>,
+	key: ResyncQueueKey,
-	hash_bytes: Vec<u8>,
 	busy_set: BusySet,
 }
 
 impl BlockResyncManager {
 	pub(crate) fn new(db: &db::Db, system: &System) -> Self {
 		let queue = db
-			.open_tree("block_local_resync_queue")
+			.open_typed_tree("block_local_resync_queue")
 			.expect("Unable to open block_local_resync_queue tree");
 
 		let errors = db
-			.open_tree("block_local_resync_errors")
+			.open_typed_tree("block_local_resync_errors")
 			.expect("Unable to open block_local_resync_errors tree");
 
 		let persister = PersisterShared::new(&system.metadata_dir, "resync_cfg");
@@ -116,11 +158,10 @@ impl BlockResyncManager {
 	/// Clear the error counter for a block and put it in queue immediately
 	pub fn clear_backoff(&self, hash: &Hash) -> Result<(), Error> {
 		let now = now_msec();
-		if let Some(ec) = self.errors.get(hash)? {
+		if let Some(mut ec) = self.errors.get(hash)? {
-			let mut ec = ErrorCounter::decode(&ec);
 			if ec.errors > 0 {
 				ec.last_try = now - ec.delay_msec();
-				self.errors.insert(hash, ec.encode())?;
+				self.errors.insert(hash, &ec)?;
 				self.put_to_resync_at(hash, now)?;
 				return Ok(());
 			}
@@ -251,23 +292,21 @@ impl BlockResyncManager {
 
 	pub(crate) fn put_to_resync_at(&self, hash: &Hash, when: u64) -> db::Result<()> {
 		trace!("Put resync_queue: {} {:?}", when, hash);
-		let mut key = u64::to_be_bytes(when).to_vec();
+		let qkey = ResyncQueueKey { when, hash: *hash };
-		key.extend(hash.as_ref());
+		self.queue.insert(&qkey, hash)?;
-		self.queue.insert(key, hash.as_ref())?;
 		self.notify.notify_waiters();
 		Ok(())
 	}
 
 	async fn resync_iter(&self, manager: &BlockManager) -> Result<ResyncIterResult, db::Error> {
 		if let Some(block) = self.get_block_to_resync()? {
-			let time_msec = u64::from_be_bytes(block.time_bytes[0..8].try_into().unwrap());
+			let time_msec = block.key.when;
 			let now = now_msec();
 
 			if now >= time_msec {
-				let hash = Hash::try_from(&block.hash_bytes[..]).unwrap();
+				let hash = block.key.hash;
 
-				if let Some(ec) = self.errors.get(hash.as_slice())? {
+				if let Some(ec) = self.errors.get(&hash)? {
-					let ec = ErrorCounter::decode(&ec);
 					if now < ec.next_try() {
 						// if next retry after an error is not yet,
 						// don't do resync and return early, but still
@@ -277,7 +316,7 @@ impl BlockResyncManager {
 						// is not removing the one we added just above
 						// (we want to do the remove after the insert to ensure
 						// that the item is not lost if we crash in-between)
-						self.queue.remove(&block.time_bytes)?;
+						self.queue.remove(&block.key)?;
 						return Ok(ResyncIterResult::BusyDidNothing);
 					}
 				}
@@ -307,21 +346,21 @@ impl BlockResyncManager {
 					manager.metrics.resync_error_counter.add(1);
 					error!("Error when resyncing {:?}: {}", hash, e);
 
-					let err_counter = match self.errors.get(hash.as_slice())? {
+					let err_counter = match self.errors.get(&hash)? {
-						Some(ec) => ErrorCounter::decode(&ec).add1(now + 1),
+						Some(ec) => ec.add1(now + 1),
 						None => ErrorCounter::new(now + 1),
 					};
 
-					self.errors.insert(hash.as_slice(), err_counter.encode())?;
+					self.errors.insert(&hash, &err_counter)?;
 
 					self.put_to_resync_at(&hash, err_counter.next_try())?;
 					// err_counter.next_try() >= now + 1 > now,
 					// the entry we remove from the queue is not
 					// the entry we inserted with put_to_resync_at
-					self.queue.remove(&block.time_bytes)?;
+					self.queue.remove(&block.key)?;
 				} else {
-					self.errors.remove(hash.as_slice())?;
+					self.errors.remove(&hash)?;
-					self.queue.remove(&block.time_bytes)?;
+					self.queue.remove(&block.key)?;
 				}
 
 				Ok(ResyncIterResult::BusyDidSomething)
@@ -344,12 +383,11 @@ impl BlockResyncManager {
 	fn get_block_to_resync(&self) -> Result<Option<BusyBlock>, db::Error> {
 		let mut busy = self.busy_set.lock().unwrap();
 		for it in self.queue.iter()? {
-			let (time_bytes, hash_bytes) = it?;
+			let (key, _) = it?;
-			if !busy.contains(&time_bytes) {
+			if !busy.contains(&key) {
-				busy.insert(time_bytes.clone());
+				busy.insert(key);
 				return Ok(Some(BusyBlock {
-					time_bytes,
+					key,
-					hash_bytes,
 					busy_set: self.busy_set.clone(),
 				}));
 			}
@@ -506,7 +544,7 @@ impl BlockResyncManager {
 impl Drop for BusyBlock {
 	fn drop(&mut self) {
 		let mut busy = self.busy_set.lock().unwrap();
-		busy.remove(&self.time_bytes);
+		busy.remove(&self.key);
 	}
 }
 
@@ -516,6 +554,7 @@ pub(crate) struct ResyncWorker {
 	tranquilizer: Tranquilizer,
 	next_delay: Duration,
 	persister: PersisterShared<ResyncPersistedConfig>,
+	had_decode_error: bool,
 }
 
 impl ResyncWorker {
@@ -527,6 +566,7 @@ impl ResyncWorker {
 			tranquilizer: Tranquilizer::new(30),
 			next_delay: Duration::from_secs(10),
 			persister,
+			had_decode_error: false,
 		}
 	}
 }
@@ -574,6 +614,15 @@ impl Worker for ResyncWorker {
 				self.next_delay = delay;
 				Ok(WorkerState::Idle)
 			}
+			Err(db::Error::Decode(e)) => {
+				// We give it one second chance in the very unlikely case that the bytes would somehow
+				// have been corrupted during read and that a new read would lead to a correct decoding.
+				if self.had_decode_error {
+					panic!("An error has happened when decoding something stored in the local k/v store: {}.", e);
+				}
+				self.had_decode_error = true;
+				Ok(WorkerState::Busy)
+			}
 			Err(e) => {
 				// The errors that we have here are only db errors
 				// We don't really know how to handle them so just ¯\_(ツ)_/¯
@@ -581,6 +630,7 @@ impl Worker for ResyncWorker {
 				// if it does there is not much we can do -- TODO should we just panic?)
 				// Here we just give the error to the worker manager,
 				// it will print it to the logs and increment a counter
+				self.had_decode_error = false;
 				Err(e.into())
 			}
 		}
@@ -602,6 +652,7 @@ impl Worker for ResyncWorker {
 
 /// Counts the number of errors when resyncing a block,
 /// and the time of the last try.
+///
 /// Used to implement exponential backoff.
 #[derive(Clone, Copy, Debug)]
 pub(crate) struct ErrorCounter {
@@ -609,6 +660,31 @@ pub(crate) struct ErrorCounter {
 	pub(crate) last_try: u64,
 }
 
+impl db::DbBytes for ErrorCounter {
+	fn encode(&self) -> Vec<u8> {
+		let mut v = Vec::with_capacity(16);
+		v.extend_from_slice(&u64::to_be_bytes(self.errors));
+		v.extend_from_slice(&u64::to_be_bytes(self.last_try));
+		v
+	}
+
+	fn decode(bytes: &[u8]) -> std::result::Result<Self, db::DecodeError> {
+		if bytes.len() != 16 {
+			return Err(db::DecodeError(
+				format!(
+					"invalid error counter: expected 16 bytes, got {}",
+					bytes.len()
+				)
+				.into(),
+			));
+		}
+		Ok(Self {
+			errors: u64::from_be_bytes(bytes[..8].try_into().unwrap()),
+			last_try: u64::from_be_bytes(bytes[8..].try_into().unwrap()),
+		})
+	}
+}
+
 impl ErrorCounter {
 	fn new(now: u64) -> Self {
 		Self {
@@ -617,21 +693,6 @@ impl ErrorCounter {
 		}
 	}
 
-	pub(crate) fn decode(data: &[u8]) -> Self {
-		Self {
-			errors: u64::from_be_bytes(data[0..8].try_into().unwrap()),
-			last_try: u64::from_be_bytes(data[8..16].try_into().unwrap()),
-		}
-	}
-
-	fn encode(&self) -> Vec<u8> {
-		[
-			u64::to_be_bytes(self.errors),
-			u64::to_be_bytes(self.last_try),
-		]
-		.concat()
-	}
-
 	fn add1(self, now: u64) -> Self {
 		Self {
 			errors: self.errors + 1,

src/db/fjall_adapter.rs

@@ -12,7 +12,7 @@ use fjall::{
 
 use crate::{
 	open::{Engine, OpenOpt},
-	Db, Error, IDb, ITx, ITxFn, OnCommit, Result, TxError, TxFnResult, TxOpError, TxOpResult,
+	Db, DbError, DbResult, Error, IDb, ITx, ITxFn, OnCommit, TxError, TxFnResult, TxOpError,
 	TxResult, TxValueIter, Value, ValueIter,
 };
 
@@ -20,10 +20,10 @@ pub use fjall;
 
 // --
 
-pub(crate) fn open_db(path: &Path, opt: &OpenOpt) -> Result<Db> {
+pub(crate) fn open_db(path: &Path, opt: &OpenOpt) -> DbResult<Db> {
 	info!("Opening Fjall database at: {}", path.display());
 	if opt.fsync {
-		return Err(Error(
+		return Err(DbError(
 			"metadata_fsync is not supported with the Fjall database engine".into(),
 		));
 	}
@@ -37,21 +37,27 @@ pub(crate) fn open_db(path: &Path, opt: &OpenOpt) -> Result<Db> {
 
 // -- err
 
-impl From<fjall::Error> for Error {
+impl From<fjall::Error> for DbError {
-	fn from(e: fjall::Error) -> Error {
+	fn from(e: fjall::Error) -> DbError {
-		Error(format!("fjall: {}", e).into())
+		DbError(format!("fjall: {}", e).into())
 	}
 }
 
-impl From<fjall::LsmError> for Error {
+impl From<fjall::LsmError> for DbError {
-	fn from(e: fjall::LsmError) -> Error {
+	fn from(e: fjall::LsmError) -> DbError {
-		Error(format!("fjall lsm_tree: {}", e).into())
+		DbError(format!("fjall lsm_tree: {}", e).into())
+	}
+}
+
+impl From<fjall::Error> for Error {
+	fn from(e: fjall::Error) -> Error {
+		Error::Db(DbError::from(e))
 	}
 }
 
 impl From<fjall::Error> for TxOpError {
 	fn from(e: fjall::Error) -> TxOpError {
-		TxOpError(e.into())
+		DbError::from(e).into()
 	}
 }
 
@@ -76,11 +82,11 @@ impl FjallDb {
 	fn get_tree(
 		&self,
 		i: usize,
-	) -> Result<MappedRwLockReadGuard<'_, TransactionalPartitionHandle>> {
+	) -> DbResult<MappedRwLockReadGuard<'_, TransactionalPartitionHandle>> {
 		RwLockReadGuard::try_map(self.trees.read(), |trees: &Vec<_>| {
 			trees.get(i).map(|tup| &tup.1)
 		})
-		.map_err(|_| Error("invalid tree id".into()))
+		.map_err(|_| DbError("invalid tree id".into()))
 	}
 }
 
@@ -89,7 +95,7 @@ impl IDb for FjallDb {
 		"Fjall (EXPERIMENTAL!)".into()
 	}
 
-	fn open_tree(&self, name: &str) -> Result<usize> {
+	fn open_tree(&self, name: &str) -> DbResult<usize> {
 		let mut trees = self.trees.write();
 		let safe_name = encode_name(name)?;
 		if let Some(i) = trees.iter().position(|(name, _)| *name == safe_name) {
@@ -104,15 +110,15 @@ impl IDb for FjallDb {
 		}
 	}
 
-	fn list_trees(&self) -> Result<Vec<String>> {
+	fn list_trees(&self) -> DbResult<Vec<String>> {
 		self.keyspace
 			.list_partitions()
 			.iter()
 			.map(|n| decode_name(n))
-			.collect::<Result<Vec<_>>>()
+			.collect::<DbResult<Vec<_>>>()
 	}
 
-	fn snapshot(&self, base_path: &Path) -> Result<()> {
+	fn snapshot(&self, base_path: &Path) -> DbResult<()> {
 		std::fs::create_dir_all(base_path)?;
 		let path = Engine::Fjall.db_path(base_path);
 
@@ -138,7 +144,7 @@ impl IDb for FjallDb {
 
 	// ----
 
-	fn get(&self, tree_idx: usize, key: &[u8]) -> Result<Option<Value>> {
+	fn get(&self, tree_idx: usize, key: &[u8]) -> DbResult<Option<Value>> {
 		let tree = self.get_tree(tree_idx)?;
 		let tx = self.keyspace.read_tx();
 		let val = tx.get(&tree, key)?;
@@ -148,17 +154,17 @@ impl IDb for FjallDb {
 		}
 	}
 
-	fn approximate_len(&self, tree_idx: usize) -> Result<usize> {
+	fn approximate_len(&self, tree_idx: usize) -> DbResult<usize> {
 		let tree = self.get_tree(tree_idx)?;
 		Ok(tree.approximate_len())
 	}
-	fn is_empty(&self, tree_idx: usize) -> Result<bool> {
+	fn is_empty(&self, tree_idx: usize) -> DbResult<bool> {
 		let tree = self.get_tree(tree_idx)?;
 		let tx = self.keyspace.read_tx();
 		Ok(tx.is_empty(&tree)?)
 	}
 
-	fn insert(&self, tree_idx: usize, key: &[u8], value: &[u8]) -> Result<()> {
+	fn insert(&self, tree_idx: usize, key: &[u8], value: &[u8]) -> DbResult<()> {
 		let tree = self.get_tree(tree_idx)?;
 		let mut tx = self.keyspace.write_tx();
 		tx.insert(&tree, key, value);
@@ -166,7 +172,7 @@ impl IDb for FjallDb {
 		Ok(())
 	}
 
-	fn remove(&self, tree_idx: usize, key: &[u8]) -> Result<()> {
+	fn remove(&self, tree_idx: usize, key: &[u8]) -> DbResult<()> {
 		let tree = self.get_tree(tree_idx)?;
 		let mut tx = self.keyspace.write_tx();
 		tx.remove(&tree, key);
@@ -174,11 +180,11 @@ impl IDb for FjallDb {
 		Ok(())
 	}
 
-	fn clear(&self, tree_idx: usize) -> Result<()> {
+	fn clear(&self, tree_idx: usize) -> DbResult<()> {
 		let mut trees = self.trees.write();
 
 		if tree_idx >= trees.len() {
-			return Err(Error("invalid tree id".into()));
+			return Err(DbError("invalid tree id".into()));
 		}
 		let (name, tree) = trees.remove(tree_idx);
 
@@ -191,13 +197,13 @@ impl IDb for FjallDb {
 		Ok(())
 	}
 
-	fn iter(&self, tree_idx: usize) -> Result<ValueIter<'_>> {
+	fn iter(&self, tree_idx: usize) -> DbResult<ValueIter<'_>> {
 		let tree = self.get_tree(tree_idx)?;
 		let tx = self.keyspace.read_tx();
 		Ok(Box::new(tx.iter(&tree).map(iterator_remap)))
 	}
 
-	fn iter_rev(&self, tree_idx: usize) -> Result<ValueIter<'_>> {
+	fn iter_rev(&self, tree_idx: usize) -> DbResult<ValueIter<'_>> {
 		let tree = self.get_tree(tree_idx)?;
 		let tx = self.keyspace.read_tx();
 		Ok(Box::new(tx.iter(&tree).rev().map(iterator_remap)))
@@ -208,7 +214,7 @@ impl IDb for FjallDb {
 		tree_idx: usize,
 		low: Bound<&'r [u8]>,
 		high: Bound<&'r [u8]>,
-	) -> Result<ValueIter<'_>> {
+	) -> DbResult<ValueIter<'_>> {
 		let tree = self.get_tree(tree_idx)?;
 		let tx = self.keyspace.read_tx();
 		Ok(Box::new(
@@ -221,7 +227,7 @@ impl IDb for FjallDb {
 		tree_idx: usize,
 		low: Bound<&'r [u8]>,
 		high: Bound<&'r [u8]>,
-	) -> Result<ValueIter<'_>> {
+	) -> DbResult<ValueIter<'_>> {
 		let tree = self.get_tree(tree_idx)?;
 		let tx = self.keyspace.read_tx();
 		Ok(Box::new(
@@ -252,9 +258,9 @@ impl IDb for FjallDb {
 			}
 			TxFnResult::DbErr => {
 				tx.tx.rollback();
-				Err(TxError::Db(Error(
+				Err(TxError::Db(
-					"(this message will be discarded)".into(),
+					DbError("(this message will be discarded)".into()).into(),
-				)))
+				))
 			}
 		}
 	}
@@ -268,47 +274,47 @@ struct FjallTx<'a> {
 }
 
 impl<'a> FjallTx<'a> {
-	fn get_tree(&self, i: usize) -> TxOpResult<&TransactionalPartitionHandle> {
+	fn get_tree(&self, i: usize) -> DbResult<&TransactionalPartitionHandle> {
 		self.trees.get(i).map(|tup| &tup.1).ok_or_else(|| {
-			TxOpError(Error(
+			DbError(
 				"invalid tree id (it might have been opened after the transaction started)".into(),
-			))
+			)
 		})
 	}
 }
 
 impl<'a> ITx for FjallTx<'a> {
-	fn get(&self, tree_idx: usize, key: &[u8]) -> TxOpResult<Option<Value>> {
+	fn get(&self, tree_idx: usize, key: &[u8]) -> DbResult<Option<Value>> {
 		let tree = self.get_tree(tree_idx)?;
 		match self.tx.get(tree, key)? {
 			Some(v) => Ok(Some(v.to_vec())),
 			None => Ok(None),
 		}
 	}
-	fn len(&self, tree_idx: usize) -> TxOpResult<usize> {
+	fn len(&self, tree_idx: usize) -> DbResult<usize> {
 		let tree = self.get_tree(tree_idx)?;
 		Ok(self.tx.len(tree)?)
 	}
 
-	fn insert(&mut self, tree_idx: usize, key: &[u8], value: &[u8]) -> TxOpResult<()> {
+	fn insert(&mut self, tree_idx: usize, key: &[u8], value: &[u8]) -> DbResult<()> {
 		let tree = self.get_tree(tree_idx)?.clone();
 		self.tx.insert(&tree, key, value);
 		Ok(())
 	}
-	fn remove(&mut self, tree_idx: usize, key: &[u8]) -> TxOpResult<()> {
+	fn remove(&mut self, tree_idx: usize, key: &[u8]) -> DbResult<()> {
 		let tree = self.get_tree(tree_idx)?.clone();
 		self.tx.remove(&tree, key);
 		Ok(())
 	}
-	fn clear(&mut self, _tree_idx: usize) -> TxOpResult<()> {
+	fn clear(&mut self, _tree_idx: usize) -> DbResult<()> {
 		unimplemented!("LSM tree clearing in cross-partition transaction is not supported")
 	}
 
-	fn iter(&self, tree_idx: usize) -> TxOpResult<TxValueIter<'_>> {
+	fn iter(&self, tree_idx: usize) -> DbResult<TxValueIter<'_>> {
 		let tree = self.get_tree(tree_idx)?.clone();
 		Ok(Box::new(self.tx.iter(&tree).map(iterator_remap_tx)))
 	}
-	fn iter_rev(&self, tree_idx: usize) -> TxOpResult<TxValueIter<'_>> {
+	fn iter_rev(&self, tree_idx: usize) -> DbResult<TxValueIter<'_>> {
 		let tree = self.get_tree(tree_idx)?.clone();
 		Ok(Box::new(self.tx.iter(&tree).rev().map(iterator_remap_tx)))
 	}
@@ -318,7 +324,7 @@ impl<'a> ITx for FjallTx<'a> {
 		tree_idx: usize,
 		low: Bound<&'r [u8]>,
 		high: Bound<&'r [u8]>,
-	) -> TxOpResult<TxValueIter<'_>> {
+	) -> DbResult<TxValueIter<'_>> {
 		let tree = self.get_tree(tree_idx)?;
 		let low = clone_bound(low);
 		let high = clone_bound(high);
@@ -333,7 +339,7 @@ impl<'a> ITx for FjallTx<'a> {
 		tree_idx: usize,
 		low: Bound<&'r [u8]>,
 		high: Bound<&'r [u8]>,
-	) -> TxOpResult<TxValueIter<'_>> {
+	) -> DbResult<TxValueIter<'_>> {
 		let tree = self.get_tree(tree_idx)?;
 		let low = clone_bound(low);
 		let high = clone_bound(high);
@@ -348,14 +354,14 @@ impl<'a> ITx for FjallTx<'a> {
 
 // -- maps fjall's (k, v) to ours
 
-fn iterator_remap(r: fjall::Result<(fjall::Slice, fjall::Slice)>) -> Result<(Value, Value)> {
+fn iterator_remap(r: fjall::Result<(fjall::Slice, fjall::Slice)>) -> DbResult<(Value, Value)> {
 	r.map(|(k, v)| (k.to_vec(), v.to_vec()))
-		.map_err(|e| e.into())
+		.map_err(DbError::from)
 }
 
-fn iterator_remap_tx(r: fjall::Result<(fjall::Slice, fjall::Slice)>) -> TxOpResult<(Value, Value)> {
+fn iterator_remap_tx(r: fjall::Result<(fjall::Slice, fjall::Slice)>) -> DbResult<(Value, Value)> {
 	r.map(|(k, v)| (k.to_vec(), v.to_vec()))
-		.map_err(|e| e.into())
+		.map_err(DbError::from)
 }
 
 // -- utils to deal with Garage's tightness on Bound lifetimes
@@ -378,7 +384,7 @@ fn clone_bound(bound: Bound<&[u8]>) -> ByteVecBound {
 
 // -- utils to encode table names --
 
-fn encode_name(s: &str) -> Result<String> {
+fn encode_name(s: &str) -> DbResult<String> {
 	let base = 'A' as u32;
 
 	let mut ret = String::with_capacity(s.len() + 10);
@@ -392,7 +398,7 @@ fn encode_name(s: &str) -> Result<String> {
 			ret.push(char::from_u32(base + c_hi).unwrap());
 			ret.push(char::from_u32(base + c_lo).unwrap());
 		} else {
-			return Err(Error(
+			return Err(DbError(
 				format!("table name {} could not be safely encoded", s).into(),
 			));
 		}
@@ -400,10 +406,10 @@ fn encode_name(s: &str) -> Result<String> {
 	Ok(ret)
 }
 
-fn decode_name(s: &str) -> Result<String> {
+fn decode_name(s: &str) -> DbResult<String> {
 	use std::convert::TryFrom;
 
-	let errfn = || Error(format!("encoded table name {} is invalid", s).into());
+	let errfn = || DbError(format!("encoded table name {} is invalid", s).into());
 	let c_map = |c: char| {
 		let c = c as u32;
 		let base = 'A' as u32;

src/db/lib.rs

@@ -9,6 +9,7 @@ pub mod lmdb_adapter;
 pub mod sqlite_adapter;
 
 pub mod open;
+pub mod typed;
 
 #[cfg(test)]
 pub mod test;
@@ -23,6 +24,7 @@ use std::sync::Arc;
 use thiserror::Error;
 
 pub use open::*;
+pub use typed::{DbBytes, DbOrdKey, TypedIter, TypedTree, TypedTxIter};
 
 pub(crate) type OnCommit = Vec<Box<dyn FnOnce()>>;
 
@@ -38,21 +40,34 @@ pub struct Transaction<'a> {
 pub struct Tree(Arc<dyn IDb>, usize);
 
 pub type Value = Vec<u8>;
-pub type ValueIter<'a> = Box<dyn std::iter::Iterator<Item = Result<(Value, Value)>> + 'a>;
+pub type ValueIter<'a> = Box<dyn std::iter::Iterator<Item = DbResult<(Value, Value)>> + 'a>;
-pub type TxValueIter<'a> = Box<dyn std::iter::Iterator<Item = TxOpResult<(Value, Value)>> + 'a>;
+pub type TxValueIter<'a> = Box<dyn std::iter::Iterator<Item = DbResult<(Value, Value)>> + 'a>;
 
 // ----
 
 #[derive(Debug, Error)]
-#[error("{0}")]
+#[error("database error: {0}")]
-pub struct Error(pub Cow<'static, str>);
+pub struct DbError(pub Cow<'static, str>);
 
-impl From<std::io::Error> for Error {
+#[derive(Debug, Error)]
-	fn from(e: std::io::Error) -> Error {
+#[error("decode error: {0}")]
-		Error(format!("IO: {}", e).into())
+pub struct DecodeError(pub Cow<'static, str>);
+
+#[derive(Debug, Error)]
+pub enum Error {
+	#[error(transparent)]
+	Db(#[from] DbError),
+	#[error(transparent)]
+	Decode(#[from] DecodeError),
+}
+
+impl From<std::io::Error> for DbError {
+	fn from(e: std::io::Error) -> DbError {
+		DbError(format!("IO: {}", e).into())
 	}
 }
 
+pub type DbResult<T> = std::result::Result<T, DbError>;
 pub type Result<T> = std::result::Result<T, Error>;
 
 #[derive(Debug, Error)]
@@ -60,6 +75,18 @@ pub type Result<T> = std::result::Result<T, Error>;
 pub struct TxOpError(pub(crate) Error);
 pub type TxOpResult<T> = std::result::Result<T, TxOpError>;
 
+impl From<DbError> for TxOpError {
+	fn from(e: DbError) -> TxOpError {
+		TxOpError(e.into())
+	}
+}
+
+impl From<DecodeError> for TxOpError {
+	fn from(e: DecodeError) -> TxOpError {
+		TxOpError(e.into())
+	}
+}
+
 #[derive(Debug)]
 pub enum TxError<E> {
 	Abort(E),
@@ -73,6 +100,12 @@ impl<E> From<TxOpError> for TxError<E> {
 	}
 }
 
+impl<E> From<DbError> for TxError<E> {
+	fn from(e: DbError) -> TxError<E> {
+		TxError::Db(e.into())
+	}
+}
+
 pub fn unabort<R, E>(res: TxResult<R, E>) -> TxOpResult<std::result::Result<R, E>> {
 	match res {
 		Ok(v) => Ok(Ok(v)),
@@ -88,12 +121,12 @@ impl Db {
 		self.0.engine()
 	}
 
-	pub fn open_tree<S: AsRef<str>>(&self, name: S) -> Result<Tree> {
+	pub fn open_tree<S: AsRef<str>>(&self, name: S) -> DbResult<Tree> {
 		let tree_id = self.0.open_tree(name.as_ref())?;
 		Ok(Tree(self.0.clone(), tree_id))
 	}
 
-	pub fn list_trees(&self) -> Result<Vec<String>> {
+	pub fn list_trees(&self) -> DbResult<Vec<String>> {
 		self.0.list_trees()
 	}
 
@@ -147,27 +180,28 @@ impl Db {
 		}
 	}
 
-	pub fn snapshot(&self, path: &Path) -> Result<()> {
+	pub fn snapshot(&self, path: &Path) -> DbResult<()> {
 		self.0.snapshot(path)
 	}
 
 	pub fn import(&self, other: &Db) -> Result<()> {
 		let existing_trees = self.list_trees()?;
 		if !existing_trees.is_empty() {
-			return Err(Error(
+			return Err(DbError(
 				format!(
 					"destination database already contains data: {:?}",
 					existing_trees
 				)
 				.into(),
-			));
+			)
+			.into());
 		}
 
 		let tree_names = other.list_trees()?;
 		for name in tree_names {
 			let tree = self.open_tree(&name)?;
 			if !tree.is_empty()? {
-				return Err(Error(format!("tree {} already contains data", name).into()));
+				return Err(DbError(format!("tree {} already contains data", name).into()).into());
 			}
 
 			let ex_tree = other.open_tree(&name)?;
@@ -186,7 +220,7 @@ impl Db {
 			});
 			let total = match tx_res {
 				Err(TxError::Db(e)) => return Err(e),
-				Err(TxError::Abort(e)) => return Err(e),
+				Err(TxError::Abort(e)) => return Err(e.into()),
 				Ok(x) => x,
 			};
 
@@ -204,24 +238,24 @@ impl Tree {
 	}
 
 	#[inline]
-	pub fn get<T: AsRef<[u8]>>(&self, key: T) -> Result<Option<Value>> {
+	pub fn get<T: AsRef<[u8]>>(&self, key: T) -> DbResult<Option<Value>> {
 		self.0.get(self.1, key.as_ref())
 	}
 	#[inline]
-	pub fn approximate_len(&self) -> Result<usize> {
+	pub fn approximate_len(&self) -> DbResult<usize> {
 		self.0.approximate_len(self.1)
 	}
 	#[inline]
-	pub fn is_empty(&self) -> Result<bool> {
+	pub fn is_empty(&self) -> DbResult<bool> {
 		self.0.is_empty(self.1)
 	}
 
 	#[inline]
-	pub fn first(&self) -> Result<Option<(Value, Value)>> {
+	pub fn first(&self) -> DbResult<Option<(Value, Value)>> {
 		self.iter()?.next().transpose()
 	}
 	#[inline]
-	pub fn get_gt<T: AsRef<[u8]>>(&self, from: T) -> Result<Option<(Value, Value)>> {
+	pub fn get_gt<T: AsRef<[u8]>>(&self, from: T) -> DbResult<Option<(Value, Value)>> {
 		if from.as_ref().is_empty() {
 			self.iter()?.next().transpose()
 		} else {
@@ -233,31 +267,31 @@ impl Tree {
 
 	/// Returns the old value if there was one
 	#[inline]
-	pub fn insert<T: AsRef<[u8]>, U: AsRef<[u8]>>(&self, key: T, value: U) -> Result<()> {
+	pub fn insert<T: AsRef<[u8]>, U: AsRef<[u8]>>(&self, key: T, value: U) -> DbResult<()> {
 		self.0.insert(self.1, key.as_ref(), value.as_ref())
 	}
 	/// Returns the old value if there was one
 	#[inline]
-	pub fn remove<T: AsRef<[u8]>>(&self, key: T) -> Result<()> {
+	pub fn remove<T: AsRef<[u8]>>(&self, key: T) -> DbResult<()> {
 		self.0.remove(self.1, key.as_ref())
 	}
 	/// Clears all values from the tree
 	#[inline]
-	pub fn clear(&self) -> Result<()> {
+	pub fn clear(&self) -> DbResult<()> {
 		self.0.clear(self.1)
 	}
 
 	#[inline]
-	pub fn iter(&self) -> Result<ValueIter<'_>> {
+	pub fn iter(&self) -> DbResult<ValueIter<'_>> {
 		self.0.iter(self.1)
 	}
 	#[inline]
-	pub fn iter_rev(&self) -> Result<ValueIter<'_>> {
+	pub fn iter_rev(&self) -> DbResult<ValueIter<'_>> {
 		self.0.iter_rev(self.1)
 	}
 
 	#[inline]
-	pub fn range<K, R>(&self, range: R) -> Result<ValueIter<'_>>
+	pub fn range<K, R>(&self, range: R) -> DbResult<ValueIter<'_>>
 	where
 		K: AsRef<[u8]>,
 		R: RangeBounds<K>,
@@ -267,7 +301,7 @@ impl Tree {
 		self.0.range(self.1, get_bound(sb), get_bound(eb))
 	}
 	#[inline]
-	pub fn range_rev<K, R>(&self, range: R) -> Result<ValueIter<'_>>
+	pub fn range_rev<K, R>(&self, range: R) -> DbResult<ValueIter<'_>>
 	where
 		K: AsRef<[u8]>,
 		R: RangeBounds<K>,
@@ -282,11 +316,11 @@ impl Tree {
 impl<'a> Transaction<'a> {
 	#[inline]
 	pub fn get<T: AsRef<[u8]>>(&self, tree: &Tree, key: T) -> TxOpResult<Option<Value>> {
-		self.tx.get(tree.1, key.as_ref())
+		self.tx.get(tree.1, key.as_ref()).map_err(Into::into)
 	}
 	#[inline]
 	pub fn len(&self, tree: &Tree) -> TxOpResult<usize> {
-		self.tx.len(tree.1)
+		self.tx.len(tree.1).map_err(Into::into)
 	}
 
 	/// Returns the old value if there was one
@@ -297,26 +331,28 @@ impl<'a> Transaction<'a> {
 		key: T,
 		value: U,
 	) -> TxOpResult<()> {
-		self.tx.insert(tree.1, key.as_ref(), value.as_ref())
+		self.tx
+			.insert(tree.1, key.as_ref(), value.as_ref())
+			.map_err(Into::into)
 	}
 	/// Returns the old value if there was one
 	#[inline]
 	pub fn remove<T: AsRef<[u8]>>(&mut self, tree: &Tree, key: T) -> TxOpResult<()> {
-		self.tx.remove(tree.1, key.as_ref())
+		self.tx.remove(tree.1, key.as_ref()).map_err(Into::into)
 	}
 	/// Clears all values in a tree
 	#[inline]
 	pub fn clear(&mut self, tree: &Tree) -> TxOpResult<()> {
-		self.tx.clear(tree.1)
+		self.tx.clear(tree.1).map_err(Into::into)
 	}
 
 	#[inline]
 	pub fn iter(&self, tree: &Tree) -> TxOpResult<TxValueIter<'_>> {
-		self.tx.iter(tree.1)
+		self.tx.iter(tree.1).map_err(Into::into)
 	}
 	#[inline]
 	pub fn iter_rev(&self, tree: &Tree) -> TxOpResult<TxValueIter<'_>> {
-		self.tx.iter_rev(tree.1)
+		self.tx.iter_rev(tree.1).map_err(Into::into)
 	}
 
 	#[inline]
@@ -327,7 +363,9 @@ impl<'a> Transaction<'a> {
 	{
 		let sb = range.start_bound();
 		let eb = range.end_bound();
-		self.tx.range(tree.1, get_bound(sb), get_bound(eb))
+		self.tx
+			.range(tree.1, get_bound(sb), get_bound(eb))
+			.map_err(Into::into)
 	}
 	#[inline]
 	pub fn range_rev<K, R>(&self, tree: &Tree, range: R) -> TxOpResult<TxValueIter<'_>>
@@ -337,7 +375,9 @@ impl<'a> Transaction<'a> {
 	{
 		let sb = range.start_bound();
 		let eb = range.end_bound();
-		self.tx.range_rev(tree.1, get_bound(sb), get_bound(eb))
+		self.tx
+			.range_rev(tree.1, get_bound(sb), get_bound(eb))
+			.map_err(Into::into)
 	}
 
 	#[inline]
@@ -350,60 +390,60 @@ impl<'a> Transaction<'a> {
 
 pub(crate) trait IDb: Send + Sync {
 	fn engine(&self) -> String;
-	fn open_tree(&self, name: &str) -> Result<usize>;
+	fn open_tree(&self, name: &str) -> DbResult<usize>;
-	fn list_trees(&self) -> Result<Vec<String>>;
+	fn list_trees(&self) -> DbResult<Vec<String>>;
-	fn snapshot(&self, path: &Path) -> Result<()>;
+	fn snapshot(&self, path: &Path) -> DbResult<()>;
 
-	fn get(&self, tree: usize, key: &[u8]) -> Result<Option<Value>>;
+	fn get(&self, tree: usize, key: &[u8]) -> DbResult<Option<Value>>;
-	fn approximate_len(&self, tree: usize) -> Result<usize>;
+	fn approximate_len(&self, tree: usize) -> DbResult<usize>;
-	fn is_empty(&self, tree: usize) -> Result<bool>;
+	fn is_empty(&self, tree: usize) -> DbResult<bool>;
 
-	fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()>;
+	fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> DbResult<()>;
-	fn remove(&self, tree: usize, key: &[u8]) -> Result<()>;
+	fn remove(&self, tree: usize, key: &[u8]) -> DbResult<()>;
-	fn clear(&self, tree: usize) -> Result<()>;
+	fn clear(&self, tree: usize) -> DbResult<()>;
 
-	fn iter(&self, tree: usize) -> Result<ValueIter<'_>>;
+	fn iter(&self, tree: usize) -> DbResult<ValueIter<'_>>;
-	fn iter_rev(&self, tree: usize) -> Result<ValueIter<'_>>;
+	fn iter_rev(&self, tree: usize) -> DbResult<ValueIter<'_>>;
 
 	fn range<'r>(
 		&self,
 		tree: usize,
 		low: Bound<&'r [u8]>,
 		high: Bound<&'r [u8]>,
-	) -> Result<ValueIter<'_>>;
+	) -> DbResult<ValueIter<'_>>;
 	fn range_rev<'r>(
 		&self,
 		tree: usize,
 		low: Bound<&'r [u8]>,
 		high: Bound<&'r [u8]>,
-	) -> Result<ValueIter<'_>>;
+	) -> DbResult<ValueIter<'_>>;
 
 	fn transaction(&self, f: &dyn ITxFn) -> TxResult<OnCommit, ()>;
 }
 
 pub(crate) trait ITx {
-	fn get(&self, tree: usize, key: &[u8]) -> TxOpResult<Option<Value>>;
+	fn get(&self, tree: usize, key: &[u8]) -> DbResult<Option<Value>>;
-	fn len(&self, tree: usize) -> TxOpResult<usize>;
+	fn len(&self, tree: usize) -> DbResult<usize>;
 
-	fn insert(&mut self, tree: usize, key: &[u8], value: &[u8]) -> TxOpResult<()>;
+	fn insert(&mut self, tree: usize, key: &[u8], value: &[u8]) -> DbResult<()>;
-	fn remove(&mut self, tree: usize, key: &[u8]) -> TxOpResult<()>;
+	fn remove(&mut self, tree: usize, key: &[u8]) -> DbResult<()>;
-	fn clear(&mut self, tree: usize) -> TxOpResult<()>;
+	fn clear(&mut self, tree: usize) -> DbResult<()>;
 
-	fn iter(&self, tree: usize) -> TxOpResult<TxValueIter<'_>>;
+	fn iter(&self, tree: usize) -> DbResult<TxValueIter<'_>>;
-	fn iter_rev(&self, tree: usize) -> TxOpResult<TxValueIter<'_>>;
+	fn iter_rev(&self, tree: usize) -> DbResult<TxValueIter<'_>>;
 
 	fn range<'r>(
 		&self,
 		tree: usize,
 		low: Bound<&'r [u8]>,
 		high: Bound<&'r [u8]>,
-	) -> TxOpResult<TxValueIter<'_>>;
+	) -> DbResult<TxValueIter<'_>>;
 	fn range_rev<'r>(
 		&self,
 		tree: usize,
 		low: Bound<&'r [u8]>,
 		high: Bound<&'r [u8]>,
-	) -> TxOpResult<TxValueIter<'_>>;
+	) -> DbResult<TxValueIter<'_>>;
 }
 
 pub(crate) trait ITxFn {

src/db/lmdb_adapter.rs

@@ -14,7 +14,7 @@ type Database = heed::Database<Bytes, Bytes>;
 
 use crate::{
 	open::{Engine, OpenOpt},
-	Db, Error, IDb, ITx, ITxFn, OnCommit, Result, TxError, TxFnResult, TxOpError, TxOpResult,
+	Db, DbError, DbResult, Error, IDb, ITx, ITxFn, OnCommit, TxError, TxFnResult, TxOpError,
 	TxResult, TxValueIter, Value, ValueIter,
 };
 
@@ -22,10 +22,10 @@ pub use heed;
 
 // ---- top-level open function
 
-pub(crate) fn open_db(path: &PathBuf, opt: &OpenOpt) -> Result<Db> {
+pub(crate) fn open_db(path: &PathBuf, opt: &OpenOpt) -> DbResult<Db> {
 	info!("Opening LMDB database at: {}", path.display());
 	if let Err(e) = std::fs::create_dir_all(path) {
-		return Err(Error(
+		return Err(DbError(
 			format!("Unable to create LMDB data directory: {}", e).into(),
 		));
 	}
@@ -48,7 +48,7 @@ pub(crate) fn open_db(path: &PathBuf, opt: &OpenOpt) -> Result<Db> {
 		env_builder.open(path)
 	};
 	match open_res {
-		Err(heed::Error::Io(e)) if e.kind() == std::io::ErrorKind::OutOfMemory => Err(Error(
+		Err(heed::Error::Io(e)) if e.kind() == std::io::ErrorKind::OutOfMemory => Err(DbError(
 			"OutOfMemory error while trying to open LMDB database. This can happen \
                 if your operating system is not allowing you to use sufficient virtual \
                 memory address space. Please check that no limit is set (ulimit -v). \
@@ -56,22 +56,28 @@ pub(crate) fn open_db(path: &PathBuf, opt: &OpenOpt) -> Result<Db> {
                 On 32-bit machines, you should probably switch to another database engine."
 				.into(),
 		)),
-		Err(e) => Err(Error(format!("Cannot open LMDB database: {}", e).into())),
+		Err(e) => Err(DbError(format!("Cannot open LMDB database: {}", e).into())),
 		Ok(db) => Ok(LmdbDb::init(db)),
 	}
 }
 
 // -- err
 
+impl From<heed::Error> for DbError {
+	fn from(e: heed::Error) -> DbError {
+		DbError(format!("LMDB: {}", e).into())
+	}
+}
+
 impl From<heed::Error> for Error {
 	fn from(e: heed::Error) -> Error {
-		Error(format!("LMDB: {}", e).into())
+		DbError::from(e).into()
 	}
 }
 
 impl From<heed::Error> for TxOpError {
 	fn from(e: heed::Error) -> TxOpError {
-		TxOpError(e.into())
+		DbError::from(e).into()
 	}
 }
 
@@ -91,14 +97,14 @@ impl LmdbDb {
 		Db(Arc::new(s))
 	}
 
-	fn get_tree(&self, i: usize) -> Result<Database> {
+	fn get_tree(&self, i: usize) -> DbResult<Database> {
 		self.trees
 			.read()
 			.unwrap()
 			.0
 			.get(i)
 			.cloned()
-			.ok_or_else(|| Error("invalid tree id".into()))
+			.ok_or_else(|| DbError("invalid tree id".into()))
 	}
 }
 
@@ -107,7 +113,7 @@ impl IDb for LmdbDb {
 		"LMDB (using Heed crate)".into()
 	}
 
-	fn open_tree(&self, name: &str) -> Result<usize> {
+	fn open_tree(&self, name: &str) -> DbResult<usize> {
 		let mut trees = self.trees.write().unwrap();
 		if let Some(i) = trees.1.get(name) {
 			Ok(*i)
@@ -122,7 +128,7 @@ impl IDb for LmdbDb {
 		}
 	}
 
-	fn list_trees(&self) -> Result<Vec<String>> {
+	fn list_trees(&self) -> DbResult<Vec<String>> {
 		let rtxn = self.db.read_txn()?;
 		let tree0 = match self
 			.db
@@ -153,7 +159,7 @@ impl IDb for LmdbDb {
 		Ok(ret2)
 	}
 
-	fn snapshot(&self, base_path: &Path) -> Result<()> {
+	fn snapshot(&self, base_path: &Path) -> DbResult<()> {
 		std::fs::create_dir_all(base_path)?;
 		let path = Engine::Lmdb.db_path(base_path);
 		self.db
@@ -163,7 +169,7 @@ impl IDb for LmdbDb {
 
 	// ----
 
-	fn get(&self, tree: usize, key: &[u8]) -> Result<Option<Value>> {
+	fn get(&self, tree: usize, key: &[u8]) -> DbResult<Option<Value>> {
 		let tree = self.get_tree(tree)?;
 
 		let tx = self.db.read_txn()?;
@@ -174,18 +180,18 @@ impl IDb for LmdbDb {
 		}
 	}
 
-	fn approximate_len(&self, tree: usize) -> Result<usize> {
+	fn approximate_len(&self, tree: usize) -> DbResult<usize> {
 		let tree = self.get_tree(tree)?;
 		let tx = self.db.read_txn()?;
 		Ok(tree.len(&tx)?.try_into().unwrap())
 	}
-	fn is_empty(&self, tree: usize) -> Result<bool> {
+	fn is_empty(&self, tree: usize) -> DbResult<bool> {
 		let tree = self.get_tree(tree)?;
 		let tx = self.db.read_txn()?;
 		Ok(tree.is_empty(&tx)?)
 	}
 
-	fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()> {
+	fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> DbResult<()> {
 		let tree = self.get_tree(tree)?;
 		let mut tx = self.db.write_txn()?;
 		tree.put(&mut tx, key, value)?;
@@ -193,7 +199,7 @@ impl IDb for LmdbDb {
 		Ok(())
 	}
 
-	fn remove(&self, tree: usize, key: &[u8]) -> Result<()> {
+	fn remove(&self, tree: usize, key: &[u8]) -> DbResult<()> {
 		let tree = self.get_tree(tree)?;
 		let mut tx = self.db.write_txn()?;
 		tree.delete(&mut tx, key)?;
@@ -201,7 +207,7 @@ impl IDb for LmdbDb {
 		Ok(())
 	}
 
-	fn clear(&self, tree: usize) -> Result<()> {
+	fn clear(&self, tree: usize) -> DbResult<()> {
 		let tree = self.get_tree(tree)?;
 		let mut tx = self.db.write_txn()?;
 		tree.clear(&mut tx)?;
@@ -209,14 +215,14 @@ impl IDb for LmdbDb {
 		Ok(())
 	}
 
-	fn iter(&self, tree: usize) -> Result<ValueIter<'_>> {
+	fn iter(&self, tree: usize) -> DbResult<ValueIter<'_>> {
 		let tree = self.get_tree(tree)?;
 		let tx = self.db.read_txn()?;
 		// Safety: the cloture does not store its argument anywhere,
 		unsafe { TxAndIterator::make(tx, |tx| Ok(tree.iter(tx)?)) }
 	}
 
-	fn iter_rev(&self, tree: usize) -> Result<ValueIter<'_>> {
+	fn iter_rev(&self, tree: usize) -> DbResult<ValueIter<'_>> {
 		let tree = self.get_tree(tree)?;
 		let tx = self.db.read_txn()?;
 		// Safety: the cloture does not store its argument anywhere,
@@ -228,7 +234,7 @@ impl IDb for LmdbDb {
 		tree: usize,
 		low: Bound<&'r [u8]>,
 		high: Bound<&'r [u8]>,
-	) -> Result<ValueIter<'_>> {
+	) -> DbResult<ValueIter<'_>> {
 		let tree = self.get_tree(tree)?;
 		let tx = self.db.read_txn()?;
 		// Safety: the cloture does not store its argument anywhere,
@@ -239,7 +245,7 @@ impl IDb for LmdbDb {
 		tree: usize,
 		low: Bound<&'r [u8]>,
 		high: Bound<&'r [u8]>,
-	) -> Result<ValueIter<'_>> {
+	) -> DbResult<ValueIter<'_>> {
 		let tree = self.get_tree(tree)?;
 		let tx = self.db.read_txn()?;
 		// Safety: the cloture does not store its argument anywhere,
@@ -271,9 +277,9 @@ impl IDb for LmdbDb {
 			}
 			TxFnResult::DbErr => {
 				tx.tx.abort();
-				Err(TxError::Db(Error(
+				Err(TxError::Db(
-					"(this message will be discarded)".into(),
+					DbError("(this message will be discarded)".into()).into(),
-				)))
+				))
 			}
 		}
 	}
@@ -287,49 +293,49 @@ struct LmdbTx<'a> {
 }
 
 impl<'a> LmdbTx<'a> {
-	fn get_tree(&self, i: usize) -> TxOpResult<&Database> {
+	fn get_tree(&self, i: usize) -> DbResult<&Database> {
 		self.trees.get(i).ok_or_else(|| {
-			TxOpError(Error(
+			DbError(
 				"invalid tree id (it might have been opened after the transaction started)".into(),
-			))
+			)
 		})
 	}
 }
 
 impl<'a> ITx for LmdbTx<'a> {
-	fn get(&self, tree: usize, key: &[u8]) -> TxOpResult<Option<Value>> {
+	fn get(&self, tree: usize, key: &[u8]) -> DbResult<Option<Value>> {
 		let tree = self.get_tree(tree)?;
 		match tree.get(&self.tx, key)? {
 			Some(v) => Ok(Some(v.to_vec())),
 			None => Ok(None),
 		}
 	}
-	fn len(&self, tree: usize) -> TxOpResult<usize> {
+	fn len(&self, tree: usize) -> DbResult<usize> {
 		let tree = self.get_tree(tree)?;
 		Ok(tree.len(&self.tx)? as usize)
 	}
 
-	fn insert(&mut self, tree: usize, key: &[u8], value: &[u8]) -> TxOpResult<()> {
+	fn insert(&mut self, tree: usize, key: &[u8], value: &[u8]) -> DbResult<()> {
 		let tree = *self.get_tree(tree)?;
 		tree.put(&mut self.tx, key, value)?;
 		Ok(())
 	}
-	fn remove(&mut self, tree: usize, key: &[u8]) -> TxOpResult<()> {
+	fn remove(&mut self, tree: usize, key: &[u8]) -> DbResult<()> {
 		let tree = *self.get_tree(tree)?;
 		tree.delete(&mut self.tx, key)?;
 		Ok(())
 	}
-	fn clear(&mut self, tree: usize) -> TxOpResult<()> {
+	fn clear(&mut self, tree: usize) -> DbResult<()> {
 		let tree = *self.get_tree(tree)?;
 		tree.clear(&mut self.tx)?;
 		Ok(())
 	}
 
-	fn iter(&self, tree: usize) -> TxOpResult<TxValueIter<'_>> {
+	fn iter(&self, tree: usize) -> DbResult<TxValueIter<'_>> {
 		let tree = *self.get_tree(tree)?;
 		Ok(Box::new(tree.iter(&self.tx)?.map(tx_iter_item)))
 	}
-	fn iter_rev(&self, tree: usize) -> TxOpResult<TxValueIter<'_>> {
+	fn iter_rev(&self, tree: usize) -> DbResult<TxValueIter<'_>> {
 		let tree = *self.get_tree(tree)?;
 		Ok(Box::new(tree.rev_iter(&self.tx)?.map(tx_iter_item)))
 	}
@@ -339,7 +345,7 @@ impl<'a> ITx for LmdbTx<'a> {
 		tree: usize,
 		low: Bound<&'r [u8]>,
 		high: Bound<&'r [u8]>,
-	) -> TxOpResult<TxValueIter<'_>> {
+	) -> DbResult<TxValueIter<'_>> {
 		let tree = *self.get_tree(tree)?;
 		Ok(Box::new(
 			tree.range(&self.tx, &(low, high))?.map(tx_iter_item),
@@ -350,7 +356,7 @@ impl<'a> ITx for LmdbTx<'a> {
 		tree: usize,
 		low: Bound<&'r [u8]>,
 		high: Bound<&'r [u8]>,
-	) -> TxOpResult<TxValueIter<'_>> {
+	) -> DbResult<TxValueIter<'_>> {
 		let tree = *self.get_tree(tree)?;
 		Ok(Box::new(
 			tree.rev_range(&self.tx, &(low, high))?.map(tx_iter_item),
@@ -386,9 +392,9 @@ where
 	}
 
 	/// Safety: iterfun must not store its argument anywhere but in its result.
-	unsafe fn make<F>(tx: RoTxn<'a, WithTls>, iterfun: F) -> Result<ValueIter<'a>>
+	unsafe fn make<F>(tx: RoTxn<'a, WithTls>, iterfun: F) -> DbResult<ValueIter<'a>>
 	where
-		F: FnOnce(&'a RoTxn<'a>) -> Result<I>,
+		F: FnOnce(&'a RoTxn<'a>) -> DbResult<I>,
 	{
 		let res = TxAndIterator {
 			tx,
@@ -436,13 +442,13 @@ impl<'a, I> Iterator for TxAndIteratorPin<'a, I>
 where
 	I: Iterator<Item = IteratorItem<'a>> + 'a,
 {
-	type Item = Result<(Value, Value)>;
+	type Item = DbResult<(Value, Value)>;
 
 	fn next(&mut self) -> Option<Self::Item> {
 		let mut_ref = Pin::as_mut(&mut self.0);
 		let next = mut_ref.iter().as_mut()?.next()?;
 		let res = match next {
-			Err(e) => Err(e.into()),
+			Err(e) => Err(DbError::from(e)),
 			Ok((k, v)) => Ok((k.to_vec(), v.to_vec())),
 		};
 		Some(res)
@@ -453,9 +459,9 @@ where
 
 fn tx_iter_item<'a>(
 	item: std::result::Result<(&'a [u8], &'a [u8]), heed::Error>,
-) -> TxOpResult<(Vec<u8>, Vec<u8>)> {
+) -> DbResult<(Vec<u8>, Vec<u8>)> {
 	item.map(|(k, v)| (k.to_vec(), v.to_vec()))
-		.map_err(|e| TxOpError(Error::from(e)))
+		.map_err(DbError::from)
 }
 
 // ---- utility ----

src/db/open.rs

@@ -1,6 +1,6 @@
 use std::path::{Path, PathBuf};
 
-use crate::{Db, Error, Result};
+use crate::{Db, DbError, Error, Result};
 
 /// List of supported database engine types
 ///
@@ -49,14 +49,14 @@ impl std::str::FromStr for Engine {
 			"lmdb" | "heed" => Ok(Self::Lmdb),
 			"sqlite" | "sqlite3" | "rusqlite" => Ok(Self::Sqlite),
             "fjall" => Ok(Self::Fjall),
-			"sled" => Err(Error("Sled is no longer supported as a database engine. Converting your old metadata db can be done using an older Garage binary (e.g. v0.9.4).".into())),
+			"sled" => Err(DbError("Sled is no longer supported as a database engine. Converting your old metadata db can be done using an older Garage binary (e.g. v0.9.4).".into()).into()),
-			kind => Err(Error(
+			kind => Err(DbError(
 				format!(
 					"Invalid DB engine: {} (options are: lmdb, sqlite, fjall)",
 					kind
 				)
 				.into(),
-			)),
+			).into()),
 		}
 	}
 }
@@ -72,22 +72,23 @@ pub fn open_db(path: &PathBuf, engine: Engine, opt: &OpenOpt) -> Result<Db> {
 	match engine {
 		// ---- Sqlite DB ----
 		#[cfg(feature = "sqlite")]
-		Engine::Sqlite => crate::sqlite_adapter::open_db(path, opt),
+		Engine::Sqlite => Ok(crate::sqlite_adapter::open_db(path, opt)?),
 
 		// ---- LMDB DB ----
 		#[cfg(feature = "lmdb")]
-		Engine::Lmdb => crate::lmdb_adapter::open_db(path, opt),
+		Engine::Lmdb => Ok(crate::lmdb_adapter::open_db(path, opt)?),
 
 		// ---- Fjall DB ----
 		#[cfg(feature = "fjall")]
-		Engine::Fjall => crate::fjall_adapter::open_db(path, opt),
+		Engine::Fjall => Ok(crate::fjall_adapter::open_db(path, opt)?),
 
 		// Pattern is unreachable when all supported DB engines are compiled into binary. The allow
 		// attribute is added so that we won't have to change this match in case stop building
 		// support for one or more engines by default.
 		#[allow(unreachable_patterns)]
-		engine => Err(Error(
+		engine => Err(DbError(
 			format!("DB engine support not available in this build: {}", engine).into(),
-		)),
+		)
+		.into()),
 	}
 }

src/db/sqlite_adapter.rs

@@ -12,7 +12,7 @@ use rusqlite::{params, Rows, Statement, Transaction};
 
 use crate::{
 	open::{Engine, OpenOpt},
-	Db, Error, IDb, ITx, ITxFn, OnCommit, Result, TxError, TxFnResult, TxOpError, TxOpResult,
+	Db, DbError, DbResult, Error, IDb, ITx, ITxFn, OnCommit, TxError, TxFnResult, TxOpError,
 	TxResult, TxValueIter, Value, ValueIter,
 };
 
@@ -20,7 +20,7 @@ pub use rusqlite;
 
 // ---- top-level open function
 
-pub(crate) fn open_db(path: &PathBuf, opt: &OpenOpt) -> Result<Db> {
+pub(crate) fn open_db(path: &PathBuf, opt: &OpenOpt) -> DbResult<Db> {
 	info!("Opening Sqlite database at: {}", path.display());
 	let manager = r2d2_sqlite::SqliteConnectionManager::file(path);
 	SqliteDb::open(manager, opt.fsync)
@@ -32,21 +32,33 @@ type Connection = r2d2::PooledConnection<SqliteConnectionManager>;
 
 // --- err
 
+impl From<rusqlite::Error> for DbError {
+	fn from(e: rusqlite::Error) -> DbError {
+		DbError(format!("Sqlite: {}", e).into())
+	}
+}
+
 impl From<rusqlite::Error> for Error {
 	fn from(e: rusqlite::Error) -> Error {
-		Error(format!("Sqlite: {}", e).into())
+		DbError::from(e).into()
+	}
+}
+
+impl From<r2d2::Error> for DbError {
+	fn from(e: r2d2::Error) -> DbError {
+		DbError(format!("Sqlite: {}", e).into())
 	}
 }
 
 impl From<r2d2::Error> for Error {
 	fn from(e: r2d2::Error) -> Error {
-		Error(format!("Sqlite: {}", e).into())
+		DbError::from(e).into()
 	}
 }
 
 impl From<rusqlite::Error> for TxOpError {
 	fn from(e: rusqlite::Error) -> TxOpError {
-		TxOpError(e.into())
+		DbError::from(e).into()
 	}
 }
 
@@ -62,7 +74,7 @@ pub struct SqliteDb {
 }
 
 impl SqliteDb {
-	pub fn open(manager: SqliteConnectionManager, sync_mode: bool) -> Result<Db> {
+	pub fn open(manager: SqliteConnectionManager, sync_mode: bool) -> DbResult<Db> {
 		let manager = manager.with_init(move |db| {
 			db.pragma_update(None, "journal_mode", "WAL")?;
 			if sync_mode {
@@ -82,16 +94,16 @@ impl SqliteDb {
 }
 
 impl SqliteDb {
-	fn get_tree(&self, i: usize) -> Result<Arc<str>> {
+	fn get_tree(&self, i: usize) -> DbResult<Arc<str>> {
 		self.trees
 			.read()
 			.unwrap()
 			.get(i)
 			.cloned()
-			.ok_or_else(|| Error("invalid tree id".into()))
+			.ok_or_else(|| DbError("invalid tree id".into()))
 	}
 
-	fn internal_get(&self, db: &Connection, tree: &str, key: &[u8]) -> Result<Option<Value>> {
+	fn internal_get(&self, db: &Connection, tree: &str, key: &[u8]) -> DbResult<Option<Value>> {
 		let mut stmt = db.prepare(&format!("SELECT v FROM {} WHERE k = ?1", tree))?;
 		let mut res_iter = stmt.query([key])?;
 		match res_iter.next()? {
@@ -106,7 +118,7 @@ impl IDb for SqliteDb {
 		format!("sqlite3 v{} (using rusqlite crate)", rusqlite::version())
 	}
 
-	fn open_tree(&self, name: &str) -> Result<usize> {
+	fn open_tree(&self, name: &str) -> DbResult<usize> {
 		let name = format!("tree_{}", name.replace(':', "_COLON_"));
 		let mut trees = self.trees.write().unwrap();
 
@@ -133,7 +145,7 @@ impl IDb for SqliteDb {
 		}
 	}
 
-	fn list_trees(&self) -> Result<Vec<String>> {
+	fn list_trees(&self) -> DbResult<Vec<String>> {
 		let mut trees = vec![];
 
 		let db = self.db.get()?;
@@ -150,13 +162,13 @@ impl IDb for SqliteDb {
 		Ok(trees)
 	}
 
-	fn snapshot(&self, base_path: &Path) -> Result<()> {
+	fn snapshot(&self, base_path: &Path) -> DbResult<()> {
 		std::fs::create_dir_all(base_path)?;
 		let path = Engine::Sqlite
 			.db_path(base_path)
 			.into_os_string()
 			.into_string()
-			.map_err(|_| Error("invalid sqlite path string".into()))?;
+			.map_err(|_| DbError("invalid sqlite path string".into()))?;
 
 		info!("Start sqlite VACUUM INTO `{}`", path);
 		self.db.get()?.execute("VACUUM INTO ?1", params![path])?;
@@ -167,12 +179,12 @@ impl IDb for SqliteDb {
 
 	// ----
 
-	fn get(&self, tree: usize, key: &[u8]) -> Result<Option<Value>> {
+	fn get(&self, tree: usize, key: &[u8]) -> DbResult<Option<Value>> {
 		let tree = self.get_tree(tree)?;
 		self.internal_get(&self.db.get()?, &tree, key)
 	}
 
-	fn approximate_len(&self, tree: usize) -> Result<usize> {
+	fn approximate_len(&self, tree: usize) -> DbResult<usize> {
 		let tree = self.get_tree(tree)?;
 		let db = self.db.get()?;
 
@@ -184,11 +196,11 @@ impl IDb for SqliteDb {
 		}
 	}
 
-	fn is_empty(&self, tree: usize) -> Result<bool> {
+	fn is_empty(&self, tree: usize) -> DbResult<bool> {
 		Ok(self.approximate_len(tree)? == 0)
 	}
 
-	fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()> {
+	fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> DbResult<()> {
 		let tree = self.get_tree(tree)?;
 		let db = self.db.get()?;
 		let lock = self.write_lock.lock();
@@ -206,7 +218,7 @@ impl IDb for SqliteDb {
 		Ok(())
 	}
 
-	fn remove(&self, tree: usize, key: &[u8]) -> Result<()> {
+	fn remove(&self, tree: usize, key: &[u8]) -> DbResult<()> {
 		let tree = self.get_tree(tree)?;
 		let db = self.db.get()?;
 		let lock = self.write_lock.lock();
@@ -217,7 +229,7 @@ impl IDb for SqliteDb {
 		Ok(())
 	}
 
-	fn clear(&self, tree: usize) -> Result<()> {
+	fn clear(&self, tree: usize) -> DbResult<()> {
 		let tree = self.get_tree(tree)?;
 		let db = self.db.get()?;
 		let lock = self.write_lock.lock();
@@ -228,13 +240,13 @@ impl IDb for SqliteDb {
 		Ok(())
 	}
 
-	fn iter(&self, tree: usize) -> Result<ValueIter<'_>> {
+	fn iter(&self, tree: usize) -> DbResult<ValueIter<'_>> {
 		let tree = self.get_tree(tree)?;
 		let sql = format!("SELECT k, v FROM {} ORDER BY k ASC", tree);
 		DbValueIterator::make(self.db.get()?, &sql, [])
 	}
 
-	fn iter_rev(&self, tree: usize) -> Result<ValueIter<'_>> {
+	fn iter_rev(&self, tree: usize) -> DbResult<ValueIter<'_>> {
 		let tree = self.get_tree(tree)?;
 		let sql = format!("SELECT k, v FROM {} ORDER BY k DESC", tree);
 		DbValueIterator::make(self.db.get()?, &sql, [])
@@ -245,7 +257,7 @@ impl IDb for SqliteDb {
 		tree: usize,
 		low: Bound<&'r [u8]>,
 		high: Bound<&'r [u8]>,
-	) -> Result<ValueIter<'_>> {
+	) -> DbResult<ValueIter<'_>> {
 		let tree = self.get_tree(tree)?;
 
 		let (bounds_sql, params) = bounds_sql(low, high);
@@ -263,7 +275,7 @@ impl IDb for SqliteDb {
 		tree: usize,
 		low: Bound<&'r [u8]>,
 		high: Bound<&'r [u8]>,
-	) -> Result<ValueIter<'_>> {
+	) -> DbResult<ValueIter<'_>> {
 		let tree = self.get_tree(tree)?;
 
 		let (bounds_sql, params) = bounds_sql(low, high);
@@ -300,9 +312,9 @@ impl IDb for SqliteDb {
 			}
 			TxFnResult::DbErr => {
 				tx.tx.rollback().map_err(Error::from).map_err(TxError::Db)?;
-				Err(TxError::Db(Error(
+				Err(TxError::Db(
-					"(this message will be discarded)".into(),
+					DbError("(this message will be discarded)".into()).into(),
-				)))
+				))
 			}
 		};
 
@@ -320,15 +332,15 @@ struct SqliteTx<'a> {
 }
 
 impl<'a> SqliteTx<'a> {
-	fn get_tree(&self, i: usize) -> TxOpResult<&'_ str> {
+	fn get_tree(&self, i: usize) -> DbResult<&'_ str> {
 		self.trees.get(i).map(Arc::as_ref).ok_or_else(|| {
-			TxOpError(Error(
+			DbError(
 				"invalid tree id (it might have been opened after the transaction started)".into(),
-			))
+			)
 		})
 	}
 
-	fn internal_get(&self, tree: &str, key: &[u8]) -> TxOpResult<Option<Value>> {
+	fn internal_get(&self, tree: &str, key: &[u8]) -> DbResult<Option<Value>> {
 		let mut stmt = self
 			.tx
 			.prepare(&format!("SELECT v FROM {} WHERE k = ?1", tree))?;
@@ -341,11 +353,11 @@ impl<'a> SqliteTx<'a> {
 }
 
 impl<'a> ITx for SqliteTx<'a> {
-	fn get(&self, tree: usize, key: &[u8]) -> TxOpResult<Option<Value>> {
+	fn get(&self, tree: usize, key: &[u8]) -> DbResult<Option<Value>> {
 		let tree = self.get_tree(tree)?;
 		self.internal_get(tree, key)
 	}
-	fn len(&self, tree: usize) -> TxOpResult<usize> {
+	fn len(&self, tree: usize) -> DbResult<usize> {
 		let tree = self.get_tree(tree)?;
 		let mut stmt = self.tx.prepare(&format!("SELECT COUNT(*) FROM {}", tree))?;
 		let mut res_iter = stmt.query([])?;
@@ -355,30 +367,30 @@ impl<'a> ITx for SqliteTx<'a> {
 		}
 	}
 
-	fn insert(&mut self, tree: usize, key: &[u8], value: &[u8]) -> TxOpResult<()> {
+	fn insert(&mut self, tree: usize, key: &[u8], value: &[u8]) -> DbResult<()> {
 		let tree = self.get_tree(tree)?;
 		let sql = format!("INSERT OR REPLACE INTO {} (k, v) VALUES (?1, ?2)", tree);
 		self.tx.execute(&sql, params![key, value])?;
 		Ok(())
 	}
-	fn remove(&mut self, tree: usize, key: &[u8]) -> TxOpResult<()> {
+	fn remove(&mut self, tree: usize, key: &[u8]) -> DbResult<()> {
 		let tree = self.get_tree(tree)?;
 		self.tx
 			.execute(&format!("DELETE FROM {} WHERE k = ?1", tree), params![key])?;
 		Ok(())
 	}
-	fn clear(&mut self, tree: usize) -> TxOpResult<()> {
+	fn clear(&mut self, tree: usize) -> DbResult<()> {
 		let tree = self.get_tree(tree)?;
 		self.tx.execute(&format!("DELETE FROM {}", tree), [])?;
 		Ok(())
 	}
 
-	fn iter(&self, tree: usize) -> TxOpResult<TxValueIter<'_>> {
+	fn iter(&self, tree: usize) -> DbResult<TxValueIter<'_>> {
 		let tree = self.get_tree(tree)?;
 		let sql = format!("SELECT k, v FROM {} ORDER BY k ASC", tree);
 		TxValueIterator::make(self, &sql, [])
 	}
-	fn iter_rev(&self, tree: usize) -> TxOpResult<TxValueIter<'_>> {
+	fn iter_rev(&self, tree: usize) -> DbResult<TxValueIter<'_>> {
 		let tree = self.get_tree(tree)?;
 		let sql = format!("SELECT k, v FROM {} ORDER BY k DESC", tree);
 		TxValueIterator::make(self, &sql, [])
@@ -389,7 +401,7 @@ impl<'a> ITx for SqliteTx<'a> {
 		tree: usize,
 		low: Bound<&'r [u8]>,
 		high: Bound<&'r [u8]>,
-	) -> TxOpResult<TxValueIter<'_>> {
+	) -> DbResult<TxValueIter<'_>> {
 		let tree = self.get_tree(tree)?;
 
 		let (bounds_sql, params) = bounds_sql(low, high);
@@ -407,7 +419,7 @@ impl<'a> ITx for SqliteTx<'a> {
 		tree: usize,
 		low: Bound<&'r [u8]>,
 		high: Bound<&'r [u8]>,
-	) -> TxOpResult<TxValueIter<'_>> {
+	) -> DbResult<TxValueIter<'_>> {
 		let tree = self.get_tree(tree)?;
 
 		let (bounds_sql, params) = bounds_sql(low, high);
@@ -439,7 +451,7 @@ impl DbValueIterator {
 		db: Connection,
 		sql: &str,
 		args: P,
-	) -> Result<ValueIter<'res>> {
+	) -> DbResult<ValueIter<'res>> {
 		let res = DbValueIterator {
 			db,
 			stmt: None,
@@ -484,7 +496,7 @@ impl Drop for DbValueIterator {
 struct DbValueIteratorPin(Pin<Box<DbValueIterator>>);
 
 impl Iterator for DbValueIteratorPin {
-	type Item = Result<(Value, Value)>;
+	type Item = DbResult<(Value, Value)>;
 
 	fn next(&mut self) -> Option<Self::Item> {
 		let mut_ref = Pin::as_mut(&mut self.0);
@@ -509,7 +521,7 @@ impl<'a> TxValueIterator<'a> {
 		tx: &'a SqliteTx<'a>,
 		sql: &str,
 		args: P,
-	) -> TxOpResult<TxValueIter<'a>> {
+	) -> DbResult<TxValueIter<'a>> {
 		let stmt = tx.tx.prepare(sql)?;
 		let res = TxValueIterator {
 			stmt,
@@ -543,7 +555,7 @@ impl<'a> Drop for TxValueIterator<'a> {
 struct TxValueIteratorPin<'a>(Pin<Box<TxValueIterator<'a>>>);
 
 impl<'a> Iterator for TxValueIteratorPin<'a> {
-	type Item = TxOpResult<(Value, Value)>;
+	type Item = DbResult<(Value, Value)>;
 
 	fn next(&mut self) -> Option<Self::Item> {
 		let mut_ref = Pin::as_mut(&mut self.0);

src/db/typed.rs

@@ -0,0 +1,241 @@
+use std::marker::PhantomData;
+use std::ops::{Bound, RangeBounds};
+
+// Todo: some parts of this code, notably around ranges are never used but are here to prepare
+// the migration of the parts of the codebase that still use untyped trees. At one point, this
+// migration should be done or these functions deleted.
+
+use super::{
+	DbResult, DecodeError, Error, Result, Transaction, Tree, TxOpError, TxOpResult, TxValueIter,
+	ValueIter,
+};
+
+pub use super::Db;
+
+pub trait DbBytes: Sized {
+	fn encode(&self) -> Vec<u8>;
+	fn decode(bytes: &[u8]) -> std::result::Result<Self, DecodeError>;
+}
+
+/// Subtrait of [`DbBytes`] for types used as tree keys with operations where order matters
+/// (`get_gt`, range, etc...).
+///
+/// Implementors must guarantee that the byte encoding is order-preserving:
+/// for any `a, b: Self`, `a.cmp(&b) == a.encode().cmp(&b.encode())`.
+pub trait DbOrdKey: DbBytes + Ord {}
+
+#[derive(Clone)]
+pub struct TypedTree<K, V> {
+	inner: Tree,
+	_phantom: PhantomData<[(K, V)]>,
+}
+
+impl<K: DbBytes, V: DbBytes> TypedTree<K, V> {
+	pub fn new(tree: Tree) -> Self {
+		Self {
+			inner: tree,
+			_phantom: PhantomData,
+		}
+	}
+
+	pub fn db(&self) -> Db {
+		self.inner.db()
+	}
+
+	pub fn untyped(&self) -> &Tree {
+		&self.inner
+	}
+
+	pub fn get(&self, key: &K) -> Result<Option<V>> {
+		self.inner
+			.get(key.encode())?
+			.map(|v| V::decode(&v).map_err(Error::from))
+			.transpose()
+	}
+
+	pub fn approximate_len(&self) -> DbResult<usize> {
+		self.inner.approximate_len()
+	}
+
+	pub fn is_empty(&self) -> DbResult<bool> {
+		self.inner.is_empty()
+	}
+
+	pub fn insert(&self, key: &K, value: &V) -> DbResult<()> {
+		self.inner.insert(key.encode(), value.encode())
+	}
+
+	pub fn remove(&self, key: &K) -> DbResult<()> {
+		self.inner.remove(key.encode())
+	}
+
+	pub fn clear(&self) -> DbResult<()> {
+		self.inner.clear()
+	}
+
+	pub fn tx_get(&self, tx: &Transaction<'_>, key: &K) -> TxOpResult<Option<V>> {
+		tx.get(&self.inner, key.encode())?
+			.map(|v| V::decode(&v).map_err(TxOpError::from))
+			.transpose()
+	}
+
+	pub fn tx_insert(&self, tx: &mut Transaction<'_>, key: &K, value: &V) -> TxOpResult<()> {
+		tx.insert(&self.inner, key.encode(), value.encode())
+	}
+
+	pub fn tx_remove(&self, tx: &mut Transaction<'_>, key: &K) -> TxOpResult<()> {
+		tx.remove(&self.inner, key.encode())
+	}
+
+	pub fn tx_clear(&self, tx: &mut Transaction<'_>) -> TxOpResult<()> {
+		tx.clear(&self.inner)
+	}
+}
+
+impl<K: DbOrdKey, V: DbBytes> TypedTree<K, V> {
+	pub fn first(&self) -> Result<Option<(K, V)>> {
+		self.iter()?.next().transpose()
+	}
+
+	pub fn get_gt(&self, from: &K) -> Result<Option<(K, V)>> {
+		self.inner
+			.get_gt(from.encode())?
+			.map(|(k, v)| {
+				Ok((
+					K::decode(&k).map_err(Error::from)?,
+					V::decode(&v).map_err(Error::from)?,
+				))
+			})
+			.transpose()
+	}
+
+	pub fn iter(&self) -> Result<TypedIter<'_, K, V>> {
+		Ok(TypedIter::new(self.inner.iter()?))
+	}
+
+	pub fn iter_rev(&self) -> Result<TypedIter<'_, K, V>> {
+		Ok(TypedIter::new(self.inner.iter_rev()?))
+	}
+
+	pub fn range<R: RangeBounds<K>>(&self, range: R) -> Result<TypedIter<'_, K, V>> {
+		Ok(TypedIter::new(self.inner.range(encode_range(range))?))
+	}
+
+	pub fn range_rev<R: RangeBounds<K>>(&self, range: R) -> Result<TypedIter<'_, K, V>> {
+		Ok(TypedIter::new(self.inner.range_rev(encode_range(range))?))
+	}
+
+	pub fn tx_iter<'t>(&self, tx: &'t Transaction<'_>) -> TxOpResult<TypedTxIter<'t, K, V>> {
+		Ok(TypedTxIter::new(tx.iter(&self.inner)?))
+	}
+
+	pub fn tx_iter_rev<'t>(&self, tx: &'t Transaction<'_>) -> TxOpResult<TypedTxIter<'t, K, V>> {
+		Ok(TypedTxIter::new(tx.iter_rev(&self.inner)?))
+	}
+
+	pub fn tx_range<'t, R: RangeBounds<K>>(
+		&self,
+		tx: &'t Transaction<'_>,
+		range: R,
+	) -> TxOpResult<TypedTxIter<'t, K, V>> {
+		Ok(TypedTxIter::new(
+			tx.range(&self.inner, encode_range(range))?,
+		))
+	}
+
+	pub fn tx_range_rev<'t, R: RangeBounds<K>>(
+		&self,
+		tx: &'t Transaction<'_>,
+		range: R,
+	) -> TxOpResult<TypedTxIter<'t, K, V>> {
+		Ok(TypedTxIter::new(
+			tx.range_rev(&self.inner, encode_range(range))?,
+		))
+	}
+}
+
+impl<K: DbBytes, V: DbBytes> From<Tree> for TypedTree<K, V> {
+	fn from(tree: Tree) -> Self {
+		Self::new(tree)
+	}
+}
+
+impl Db {
+	pub fn open_typed_tree<K: DbBytes, V: DbBytes, S: AsRef<str>>(
+		&self,
+		name: S,
+	) -> DbResult<TypedTree<K, V>> {
+		Ok(TypedTree::new(self.open_tree(name)?))
+	}
+}
+
+pub struct TypedIter<'a, K, V> {
+	inner: ValueIter<'a>,
+	_phantom: PhantomData<(K, V)>,
+}
+
+impl<'a, K, V> TypedIter<'a, K, V> {
+	fn new(inner: ValueIter<'a>) -> Self {
+		Self {
+			inner,
+			_phantom: PhantomData,
+		}
+	}
+}
+
+impl<K: DbOrdKey, V: DbBytes> Iterator for TypedIter<'_, K, V> {
+	type Item = Result<(K, V)>;
+
+	fn next(&mut self) -> Option<Self::Item> {
+		self.inner.next().map(|res| {
+			let (k, v) = res?;
+			Ok((
+				K::decode(&k).map_err(Error::from)?,
+				V::decode(&v).map_err(Error::from)?,
+			))
+		})
+	}
+}
+
+pub struct TypedTxIter<'a, K, V> {
+	inner: TxValueIter<'a>,
+	_phantom: PhantomData<(K, V)>,
+}
+
+impl<'a, K, V> TypedTxIter<'a, K, V> {
+	fn new(inner: TxValueIter<'a>) -> Self {
+		Self {
+			inner,
+			_phantom: PhantomData,
+		}
+	}
+}
+
+impl<K: DbOrdKey, V: DbBytes> Iterator for TypedTxIter<'_, K, V> {
+	type Item = TxOpResult<(K, V)>;
+
+	fn next(&mut self) -> Option<Self::Item> {
+		self.inner.next().map(|res| {
+			let (k, v) = res?;
+			Ok((
+				K::decode(&k).map_err(TxOpError::from)?,
+				V::decode(&v).map_err(TxOpError::from)?,
+			))
+		})
+	}
+}
+
+fn encode_range<K: DbOrdKey, R: RangeBounds<K>>(range: R) -> (Bound<Vec<u8>>, Bound<Vec<u8>>) {
+	(
+		encode_bound(range.start_bound()),
+		encode_bound(range.end_bound()),
+	)
+}
+
+fn encode_bound<K: DbOrdKey>(bound: Bound<&K>) -> Bound<Vec<u8>> {
+	match bound {
+		Bound::Included(k) => Bound::Included(k.encode()),
+		Bound::Excluded(k) => Bound::Excluded(k.encode()),
+		Bound::Unbounded => Bound::Unbounded,
+	}
+}

src/garage/Cargo.toml

@@ -39,8 +39,8 @@ backtrace.workspace = true
 bytes.workspace = true
 bytesize.workspace = true
 chrono.workspace = true
+fundu-systemd.workspace = true
 timeago.workspace = true
-parse_duration.workspace = true
 hex.workspace = true
 tracing.workspace = true
 tracing-subscriber.workspace = true

src/garage/cli/local/convert_db.rs

@@ -49,7 +49,7 @@ pub struct OpenLmdbOpt {
 
 pub(crate) fn do_conversion(args: ConvertDbOpt) -> Result<()> {
 	if args.input_engine == args.output_engine {
-		return Err(Error("input and output database engine must differ".into()));
+		return Err(DbError("input and output database engine must differ".into()).into());
 	}
 
 	let opt = OpenOpt {

src/garage/cli/remote/bucket.rs

@@ -393,7 +393,7 @@ impl Cli {
 		&self,
 		opt: CleanupIncompleteUploadsOpt,
 	) -> Result<(), Error> {
-		let older_than = parse_duration::parse::parse(&opt.older_than)
+		let older_than = garage_util::time::parse_duration(&opt.older_than)
 			.ok_or_message("Invalid duration passed for --older-than parameter")?;
 
 		for b in opt.buckets.iter() {

src/garage/cli/remote/key.rs

@@ -226,7 +226,7 @@ impl Cli {
 
 	pub async fn cmd_import_key(&self, opt: KeyImportOpt) -> Result<(), Error> {
 		if !opt.yes {
-			return Err(Error::Message("This command is intended to re-import keys that were previously generated by Garage. If you want to create a new key, use `garage key new` instead. Add the --yes flag if you really want to re-import a key.".to_string()));
+			return Err(Error::Message("This command is intended to re-import keys that were previously generated by Garage. If you want to create a new key, use `garage key create` instead. Add the --yes flag if you really want to re-import a key.".to_string()));
 		}
 
 		let new_key = self

src/garage/cli/remote/mod.rs

@@ -169,7 +169,7 @@ pub fn table_list_abbr<T: IntoIterator<Item = S>, S: AsRef<str>>(values: T) -> S
 pub fn parse_expires_in(expires_in: &Option<String>) -> Result<Option<DateTime<Utc>>, Error> {
 	expires_in
 		.as_ref()
-		.map(|x| parse_duration::parse::parse(x).map(|dur| Utc::now() + dur))
+		.map(|x| garage_util::time::parse_duration(x).map(|dur| Utc::now() + dur))
 		.transpose()
 		.ok_or_message("Invalid duration passed for --expires-in parameter")
 }

src/garage/cli/structs.rs

@@ -494,7 +494,7 @@ pub struct KeyNewOpt {
 	#[structopt(default_value = "Unnamed key")]
 	pub name: String,
 	/// Set an expiration time for the access key
-	/// (see `docs.rs/parse_duration` for date format)
+	/// (see `docs.rs/fundu-systemd` for date format)
 	#[structopt(long = "expires-in")]
 	pub expires_in: Option<String>,
 }
@@ -505,7 +505,7 @@ pub struct KeySetOpt {
 	pub key_pattern: String,
 
 	/// Set an expiration time for the access key
-	/// (see `docs.rs/parse_duration` for date format)
+	/// (see `docs.rs/fundu-systemd` for date format)
 	#[structopt(long = "expires-in")]
 	pub expires_in: Option<String>,
 	/// Set the access key to never expire
@@ -616,7 +616,7 @@ pub enum AdminTokenOperation {
 pub struct AdminTokenCreateOp {
 	/// Set a name for the token
 	pub name: Option<String>,
-	/// Set an expiration time for the token (see `docs.rs/parse_duration` for date
+	/// Set an expiration time for the token (see `docs.rs/fundu-systemd` for date
 	/// format)
 	#[structopt(long = "expires-in")]
 	pub expires_in: Option<String>,
@@ -638,7 +638,7 @@ pub struct AdminTokenSetOp {
 	/// Name or prefix of the ID of the token to modify
 	pub api_token: String,
 
-	/// Set an expiration time for the token (see `docs.rs/parse_duration` for date
+	/// Set an expiration time for the token (see `docs.rs/fundu-systemd` for date
 	/// format)
 	#[structopt(long = "expires-in")]
 	pub expires_in: Option<String>,

src/garage/secrets.rs

@@ -137,14 +137,18 @@ fn read_secret_file(file_path: &PathBuf, allow_world_readable: bool) -> Result<S
 		#[cfg(unix)]
 		{
 			use std::os::unix::fs::MetadataExt;
-			let metadata = std::fs::metadata(file_path)?;
+			let metadata = std::fs::metadata(file_path).map_err(|e| {
+				format!("Failed to read secret file {}: {}", file_path.display(), e)
+			})?;
+
 			if metadata.mode() & 0o077 != 0 {
 				return Err(format!("File {} is world-readable! (mode: 0{:o}, expected 0600)\nRefusing to start until this is fixed, or environment variable GARAGE_ALLOW_WORLD_READABLE_SECRETS is set to true.", file_path.display(), metadata.mode()).into());
 			}
 		}
 	}
 
-	let secret_buf = std::fs::read_to_string(file_path)?;
+	let secret_buf = std::fs::read_to_string(file_path)
+		.map_err(|e| format!("Failed to read secret file {}: {}", file_path.display(), e))?;
 
 	// trim_end: allows for use case such as `echo "$(openssl rand -hex 32)" > somefile`.
 	//           also editors sometimes add a trailing newline

src/garage/tests/common/garage.rs

@@ -194,15 +194,6 @@ api_bind_addr = "127.0.0.1:{admin_port}"
 			.expect("Could not build garage endpoint URI")
 	}
 
-	pub fn admin_uri(&self, path: &str) -> http::Uri {
-		format!(
-			"http://127.0.0.1:{admin_port}/{path}",
-			admin_port = self.admin_port,
-		)
-		.parse()
-		.expect("Could not build garage endpoint URI")
-	}
-
 	pub fn key(&self, maybe_name: Option<&str>) -> Key {
 		let mut key = Key::default();
 

src/garage/tests/lib.rs

@@ -4,9 +4,6 @@ mod common;
 mod admin;
 mod bucket;
 
-#[cfg(feature = "metrics")]
-mod metrics;
-
 mod s3;
 
 #[cfg(feature = "k2v")]

src/garage/tests/metrics.rs

@@ -1,49 +0,0 @@
-use bytes::Bytes;
-use http::{Request, StatusCode};
-use http_body_util::{BodyExt, Full};
-
-use crate::common;
-
-#[tokio::test]
-async fn check_metrics_name() {
-	let ctx = common::context();
-
-	let req_url = ctx.garage.admin_uri("metrics");
-	let client = ctx.custom_request.client();
-	let get_metrics_req = Request::builder()
-		.method("GET")
-		.uri(req_url)
-		.body(Full::new(Bytes::new()))
-		.unwrap();
-
-	let response = client
-		.request(get_metrics_req)
-		.await
-		.expect("failed to build 'get metrics' request");
-
-	assert_eq!(response.status(), StatusCode::OK);
-	let body = BodyExt::collect(response.into_body())
-		.await
-		.expect("failed to collect bytes from body stream")
-		.to_bytes();
-	let body = String::from_utf8_lossy(&body);
-
-	//dbg!(&body);
-	let invalid_metrics_name = body
-		.lines()
-		.filter(isnot_comment_line) // skip the comment lines
-		.filter(hasnt_prefix_garage)
-		.collect::<Vec<_>>();
-
-	if !invalid_metrics_name.is_empty() {
-		panic!("metrics name should all start with 'garage_' prefix.\nDoc: https://prometheus.io/docs/practices/naming/#metric-names\n\nInvalid:\n{:#?}", invalid_metrics_name);
-	}
-}
-
-fn isnot_comment_line(line: &&str) -> bool {
-	!line.starts_with("#")
-}
-
-fn hasnt_prefix_garage(line: &&str) -> bool {
-	!line.starts_with("garage_")
-}

src/garage/tests/s3/mod.rs

@@ -2,6 +2,7 @@ mod cors;
 mod list;
 mod multipart;
 mod objects;
+mod postobject;
 mod presigned;
 mod signature_encoding;
 mod simple;

src/garage/tests/s3/objects.rs

@@ -554,7 +554,7 @@ async fn test_deleteobject() {
 
 	assert!(l.contents.is_none());
 
-	// Deleting a non-existing object shouldn't be a problem
+	// Deleting a non-existing object shouldn't be a problem...
 	ctx.client
 		.delete_object()
 		.bucket(&bucket)
@@ -562,4 +562,27 @@ async fn test_deleteobject() {
 		.send()
 		.await
 		.unwrap();
+
+	// ...and bulk-deleting a non-existing object shouldn't be either
+	let r = ctx
+		.client
+		.delete_objects()
+		.bucket(&bucket)
+		.delete(
+			Delete::builder()
+				.objects(
+					ObjectIdentifier::builder()
+						.key("does-not-exist")
+						.build()
+						.unwrap(),
+				)
+				.build()
+				.unwrap(),
+		)
+		.send()
+		.await
+		.unwrap();
+
+	assert_eq!(r.deleted.unwrap().len(), 1); // reported as deleted...
+	assert!(r.errors.unwrap_or_default().is_empty()); // ...not as an error
 }

src/garage/tests/s3/postobject.rs

@@ -0,0 +1,166 @@
+use base64::prelude::*;
+use bytes::Bytes;
+use chrono::{Duration, Utc};
+use hmac::Mac;
+use http_body_util::Full;
+use hyper::body::Incoming;
+use hyper::{header, Method, Request, Response, StatusCode};
+
+use garage_api_common::signature;
+
+use crate::common;
+
+const UTF8_KEY: &str = "uploads/test/копия файла.jpg";
+const UTF8_FILENAME: &str = "café-日本語.txt";
+const UTF8_CONTENT_DISPOSITION: &str = "attachment; filename=\"копия файла.jpg\"";
+const REGION: &str = "garage-integ-test";
+const BOUNDARY: &str = "boundary-garage-integ-test";
+
+async fn send_post_object(
+	ctx: &common::Context,
+	bucket: &str,
+	key_field: &str,
+	filename: &str,
+	extra_fields: &[(&str, &str)],
+	file_body: &str,
+) -> Response<Incoming> {
+	let now = Utc::now();
+	let scope = signature::compute_scope(&now, REGION, "s3");
+	let credential = format!("{}/{}", ctx.key.id, scope);
+	let date = now.format(signature::LONG_DATETIME).to_string();
+	let expiration = (now + Duration::hours(1)).to_rfc3339();
+
+	let mut conditions = vec![
+		serde_json::json!({ "bucket": bucket }),
+		serde_json::json!(["starts-with", "$key", ""]),
+		serde_json::json!({ "x-amz-algorithm": "AWS4-HMAC-SHA256" }),
+		serde_json::json!({ "x-amz-credential": &credential }),
+		serde_json::json!({ "x-amz-date": &date }),
+	];
+	for (name, value) in extra_fields {
+		conditions.push(serde_json::json!(["eq", format!("${}", name), value]));
+	}
+	let policy = serde_json::json!({
+		"expiration": expiration,
+		"conditions": conditions,
+	})
+	.to_string();
+
+	let policy_b64 = BASE64_STANDARD.encode(policy.as_bytes());
+
+	let mut signer = signature::signing_hmac(&now, &ctx.key.secret, REGION, "s3").unwrap();
+	signer.update(policy_b64.as_bytes());
+	let x_amz_signature = hex::encode(signer.finalize().into_bytes());
+
+	let mut fields = vec![
+		("key".to_string(), key_field.to_string()),
+		("x-amz-algorithm".into(), "AWS4-HMAC-SHA256".into()),
+		("x-amz-credential".into(), credential),
+		("x-amz-date".into(), date),
+		("policy".into(), policy_b64),
+		("x-amz-signature".into(), x_amz_signature),
+	];
+	for (name, value) in extra_fields {
+		fields.push((name.to_string(), value.to_string()));
+	}
+
+	let mut body = String::new();
+	for (name, value) in &fields {
+		body.push_str(&format!(
+			"--{BOUNDARY}\r\nContent-Disposition: form-data; name=\"{name}\"\r\n\r\n{value}\r\n"
+		));
+	}
+
+	body.push_str(&format!(
+		"--{BOUNDARY}\r\nContent-Disposition: form-data; name=\"file\"; filename=\"{filename}\"\r\nContent-Type: text/plain\r\n\r\n{file_body}\r\n--{BOUNDARY}--\r\n"
+	));
+
+	let req = Request::builder()
+		.method(Method::POST)
+		.uri(format!("{}{}", ctx.garage.s3_uri(), bucket))
+		.header(header::HOST, "s3.garage")
+		.header(
+			header::CONTENT_TYPE,
+			format!("multipart/form-data; boundary={BOUNDARY}"),
+		)
+		.body(Full::new(Bytes::from(body.into_bytes())))
+		.unwrap();
+
+	ctx.custom_request.client().request(req).await.unwrap()
+}
+
+#[tokio::test]
+async fn test_post_object_utf8_key() {
+	let ctx = common::context();
+	let bucket = ctx.create_bucket("post-object-utf8-key");
+
+	let res = send_post_object(&ctx, &bucket, UTF8_KEY, "копия файла.jpg", &[], "hello").await;
+	assert_eq!(res.status(), StatusCode::NO_CONTENT);
+
+	let obj = ctx
+		.client
+		.get_object()
+		.bucket(&bucket)
+		.key(UTF8_KEY)
+		.send()
+		.await
+		.unwrap();
+	assert_bytes_eq!(obj.body, b"hello");
+}
+
+#[tokio::test]
+async fn test_post_object_utf8_filename_substitution() {
+	let ctx = common::context();
+	let bucket = ctx.create_bucket("post-object-utf8-filename");
+
+	let res = send_post_object(
+		&ctx,
+		&bucket,
+		"uploads/${filename}",
+		UTF8_FILENAME,
+		&[],
+		"bonjour",
+	)
+	.await;
+	assert_eq!(res.status(), StatusCode::NO_CONTENT);
+
+	let obj = ctx
+		.client
+		.get_object()
+		.bucket(&bucket)
+		.key(format!("uploads/{}", UTF8_FILENAME))
+		.send()
+		.await
+		.unwrap();
+	assert_bytes_eq!(obj.body, b"bonjour");
+}
+
+#[tokio::test]
+async fn test_post_object_utf8_content_disposition_metadata() {
+	let ctx = common::context();
+	let bucket = ctx.create_bucket("post-object-utf8-contdisp");
+
+	let res = send_post_object(
+		&ctx,
+		&bucket,
+		"ascii-key.jpg",
+		"file.jpg",
+		&[("content-disposition", UTF8_CONTENT_DISPOSITION)],
+		"data",
+	)
+	.await;
+	assert_eq!(res.status(), StatusCode::NO_CONTENT);
+
+	let obj = ctx
+		.client
+		.get_object()
+		.bucket(&bucket)
+		.key("ascii-key.jpg")
+		.send()
+		.await
+		.unwrap();
+	assert_eq!(
+		obj.content_disposition.as_deref(),
+		Some(UTF8_CONTENT_DISPOSITION)
+	);
+}

src/k2v-client/Cargo.toml

@@ -24,7 +24,7 @@ percent-encoding.workspace = true
 hyper = { workspace = true, default-features = false, features = ["http1", "http2"] }
 hyper-util.workspace = true
 hyper-rustls.workspace = true
-serde.workspace = true
+serde = { workspace = true, default-features = false, features = ["derive", "std"] }
 serde_json.workspace = true
 thiserror.workspace = true
 tokio.workspace = true

src/k2v-client/lib.rs

@@ -26,6 +26,10 @@ mod error;
 
 pub use error::Error;
 
+// Re-export these crates: values from these are passed to K2vClient::new_with_client
+pub use hyper_rustls;
+pub use hyper_util;
+
 pub type Body = FullBody<Bytes>;
 
 const DEFAULT_TIMEOUT: Duration = Duration::from_secs(5);
@@ -46,6 +50,7 @@ const PATH_ENCODE_SET: AsciiSet = NON_ALPHANUMERIC
 	.remove(b'.')
 	.remove(b'~');
 
+#[derive(Debug, Clone)]
 pub struct K2vClientConfig {
 	pub endpoint: String,
 	pub region: String,
@@ -56,6 +61,7 @@ pub struct K2vClientConfig {
 }
 
 /// Client used to query a K2V server.
+#[derive(Debug, Clone)]
 pub struct K2vClient {
 	config: K2vClientConfig,
 	user_agent: HeaderValue,

src/model/Cargo.toml

@@ -30,7 +30,6 @@ thiserror.workspace = true
 hex.workspace = true
 http.workspace = true
 base64.workspace = true
-parse_duration.workspace = true
 tracing.workspace = true
 rand.workspace = true
 zstd.workspace = true

src/model/garage.rs

@@ -304,7 +304,7 @@ impl Garage {
 		self.k2v.spawn_workers(bg);
 
 		if let Some(itv) = self.config.metadata_auto_snapshot_interval.as_deref() {
-			let interval = parse_duration::parse(itv)
+			let interval = garage_util::time::parse_duration(itv)
 				.ok_or_message("Invalid `metadata_auto_snapshot_interval`")?;
 			if interval < std::time::Duration::from_secs(600) {
 				return Err(Error::Message(

src/model/index_counter.rs

@@ -163,6 +163,7 @@ impl<T: CountedItem> TableSchema for CounterTable<T> {
 
 pub struct IndexCounter<T: CountedItem> {
 	this_node: Uuid,
+	// TODO: migrate to TypedTree
 	local_counter: db::Tree,
 	pub table: Arc<Table<CounterTable<T>, TableShardedReplication>>,
 }

src/model/k2v/rpc.rs

@@ -5,7 +5,7 @@
 //! node does not process the entry directly, as this would
 //! mean the vector clock gets much larger than needed).
 
-use std::collections::{BTreeMap, HashMap};
+use std::collections::{BTreeMap, BTreeSet, HashMap};
 use std::convert::TryInto;
 use std::sync::{Arc, Mutex, MutexGuard};
 use std::time::{Duration, Instant};
@@ -38,6 +38,12 @@ const POLL_RANGE_EXTRA_DELAY: Duration = Duration::from_millis(200);
 
 const TIMESTAMP_KEY: &[u8] = b"timestamp";
 
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub enum K2VMonotonicRead {
+	Monotonic,
+	NonMonotonic,
+}
+
 /// RPC messages for K2V
 #[derive(Debug, Serialize, Deserialize)]
 enum K2VRpc {
@@ -78,6 +84,7 @@ pub struct K2VRpcHandler {
 	// Using a mutex on the local_timestamp_tree is not strictly necessary,
 	// but it helps to not try to do several inserts at the same time,
 	// which would create transaction conflicts and force many useless retries.
+	// TODO: migrate to TypedTree
 	local_timestamp_tree: Mutex<db::Tree>,
 
 	endpoint: Arc<Endpoint<K2VRpc, Self>>,
@@ -210,6 +217,7 @@ impl K2VRpcHandler {
 		sort_key: String,
 		causal_context: CausalContext,
 		timeout_msec: u64,
+		monotonic_read: K2VMonotonicRead,
 	) -> Result<Option<K2VItem>, Error> {
 		let poll_key = PollKey {
 			partition: K2VItemPartition {
@@ -244,17 +252,35 @@ impl K2VRpcHandler {
 		};
 
 		let mut resp: Option<K2VItem> = None;
-		for v in resps {
+		let mut not_all_same = false;
-			match v {
+		{
-				K2VRpc::PollItemResponse(Some(x)) => {
+			let mut vals_nb = 0;
-					if let Some(y) = &mut resp {
+			let resps_nb = resps.len();
-						y.merge(&x);
+			for v in resps {
-					} else {
+				match v {
-						resp = Some(x);
+					K2VRpc::PollItemResponse(Some(x)) => {
+						vals_nb += 1;
+						if let Some(y) = &mut resp {
+							if *y != x {
+								not_all_same = true;
+								y.merge(&x);
+							}
+						} else {
+							resp = Some(x);
+						}
 					}
+					K2VRpc::PollItemResponse(None) => (),
+					v => return Err(Error::unexpected_rpc_message(v)),
 				}
-				K2VRpc::PollItemResponse(None) => (),
+			}
-				v => return Err(Error::unexpected_rpc_message(v)),
+			if vals_nb < resps_nb {
+				not_all_same = true;
+			}
+		}
+
+		if let Some(v) = &resp {
+			if monotonic_read == K2VMonotonicRead::Monotonic && not_all_same {
+				self.item_table.repair_on_read(&nodes, &[v]).await?;
 			}
 		}
 
@@ -266,6 +292,7 @@ impl K2VRpcHandler {
 		range: PollRange,
 		seen_str: Option<String>,
 		timeout_msec: u64,
+		monotonic_read: K2VMonotonicRead,
 	) -> Result<Option<(BTreeMap<String, K2VItem>, String)>, HelperError> {
 		let has_seen_marker = seen_str.is_some();
 
@@ -343,24 +370,53 @@ impl K2VRpcHandler {
 
 		// Take all returned items into account to produce the response.
 		let mut new_items = BTreeMap::<String, K2VItem>::new();
-		for v in resps {
+		let mut to_repair = BTreeSet::new();
-			if let K2VRpc::PollRangeResponse(node, items) = v {
+		{
-				seen.mark_seen_node_items(node, items.iter());
+			let mut all_items: BTreeMap<_, Vec<_>> = BTreeMap::new();
+			let resps_nb = resps.len();
+			for v in resps {
+				if let K2VRpc::PollRangeResponse(node, items) = v {
+					seen.mark_seen_node_items(node, items.iter());
+					for item in items.into_iter() {
+						all_items
+							.entry(item.sort_key.clone())
+							.or_default()
+							.push(item);
+					}
+				} else {
+					return Err(Error::unexpected_rpc_message(v).into());
+				}
+			}
+			for (item_key, items) in all_items {
+				// Only some nodes store this item; we must propagate it during repair
+				if items.len() < resps_nb {
+					to_repair.insert(item_key.clone());
+				}
+				// Merge all items for this key together
 				for item in items.into_iter() {
 					match new_items.get_mut(&item.sort_key) {
 						Some(ent) => {
-							ent.merge(&item);
+							if *ent != item {
+								ent.merge(&item);
+								to_repair.insert(item.sort_key.clone());
+							}
 						}
 						None => {
 							new_items.insert(item.sort_key.clone(), item);
 						}
 					}
 				}
-			} else {
-				return Err(Error::unexpected_rpc_message(v).into());
 			}
 		}
 
+		if monotonic_read == K2VMonotonicRead::Monotonic && !to_repair.is_empty() {
+			let to_repair: Vec<_> = to_repair
+				.into_iter()
+				.map(|k| new_items.get(&k).unwrap())
+				.collect();
+			self.item_table.repair_on_read(&nodes, &to_repair).await?;
+		}
+
 		if new_items.is_empty() && has_seen_marker {
 			Ok(None)
 		} else {

src/rpc/metrics.rs

@@ -14,23 +14,23 @@ impl RpcMetrics {
 		let meter = global::meter("garage_rpc");
 		RpcMetrics {
 			rpc_counter: meter
-				.u64_counter("garage_rpc.request_count")
+				.u64_counter("rpc.request_counter")
 				.with_description("Number of RPC requests emitted")
 				.init(),
 			rpc_timeout_counter: meter
-				.u64_counter("garage_rpc.timeout_count")
+				.u64_counter("rpc.timeout_counter")
 				.with_description("Number of RPC timeouts")
 				.init(),
 			rpc_netapp_error_counter: meter
-				.u64_counter("garage_rpc.netapp_error_count")
+				.u64_counter("rpc.netapp_error_counter")
 				.with_description("Number of communication errors (errors in the Netapp library)")
 				.init(),
 			rpc_garage_error_counter: meter
-				.u64_counter("garage_rpc.garage_error_count")
+				.u64_counter("rpc.garage_error_counter")
 				.with_description("Number of RPC errors (errors happening when handling the RPC)")
 				.init(),
 			rpc_duration: meter
-				.f64_value_recorder("garage_rpc.duration")
+				.f64_value_recorder("rpc.duration")
 				.with_description("Duration of RPCs")
 				.init(),
 		}

src/rpc/system.rs

@@ -3,7 +3,7 @@ use std::collections::{HashMap, HashSet};
 use std::io::{Read, Write};
 use std::net::{IpAddr, SocketAddr};
 use std::path::{Path, PathBuf};
-use std::sync::{Arc, RwLock, RwLockReadGuard};
+use std::sync::{Arc, Mutex, RwLock, RwLockReadGuard};
 use std::time::{Duration, Instant};
 
 use arc_swap::ArcSwapOption;
@@ -24,6 +24,7 @@ use garage_util::config::KubernetesDiscoveryConfig;
 use garage_util::config::{Config, DataDirEnum};
 use garage_util::data::*;
 use garage_util::error::*;
+use garage_util::migrate::Migrate;
 use garage_util::persister::Persister;
 use garage_util::time::*;
 
@@ -89,6 +90,9 @@ pub struct System {
 	pub id: Uuid,
 
 	persist_peer_list: Persister<PeerList>,
+	// Cache of the bytes last read from / written to the peer list file,
+	// to avoid waking up the disk with a read on every save_peer_list() call.
+	cached_peer_list_bytes: Mutex<Option<Vec<u8>>>,
 
 	pub(crate) local_status: RwLock<NodeStatus>,
 	node_status: RwLock<HashMap<Uuid, (u64, NodeStatus)>>,
@@ -311,6 +315,7 @@ impl System {
 		let sys = Arc::new(System {
 			id: netapp.id.into(),
 			persist_peer_list,
+			cached_peer_list_bytes: Mutex::new(None),
 			local_status: RwLock::new(local_status),
 			node_status: RwLock::new(HashMap::new()),
 			netapp: netapp.clone(),
@@ -779,10 +784,23 @@ impl System {
 			peer_list.extend(prev_peer_list.0);
 		}
 
-		// Save new peer list to file
+		// Sort for a deterministic encoding, so that we can detect below
-		self.persist_peer_list
+		// whether the peer list actually changed since last time.
-			.save_async(&PeerList(peer_list))
+		peer_list.sort_by_key(|(id, _)| *id);
-			.await
+
+		let new_peer_list = PeerList(peer_list);
+		let new_peer_list_bytes = new_peer_list.encode()?;
+
+		// Skip writing to disk if nothing changed since the last save,
+		// but still cache the bytes so we don't have to read the file again.
+		let cached_peer_list_bytes = self.cached_peer_list_bytes.lock().unwrap().clone();
+		if cached_peer_list_bytes.as_deref() == Some(&new_peer_list_bytes[..]) {
+			return Ok(());
+		}
+
+		self.persist_peer_list.save_async(&new_peer_list).await?;
+		*self.cached_peer_list_bytes.lock().unwrap() = Some(new_peer_list_bytes);
+		Ok(())
 	}
 }
 

src/rpc/system_metrics.rs

@@ -110,7 +110,7 @@ impl SystemMetrics {
 			_cluster_healthy: {
 				let get_health = get_health.clone();
 				meter
-					.u64_value_observer("garage_cluster_healthy", move |observer| {
+					.u64_value_observer("cluster_healthy", move |observer| {
 						let h = get_health();
 						if h.status == ClusterHealthStatus::Healthy {
 							observer.observe(1, &[]);
@@ -123,7 +123,7 @@ impl SystemMetrics {
 			},
 			_cluster_available: {
 				let get_health = get_health.clone();
-				meter.u64_value_observer("garage_cluster_available", move |observer| {
+				meter.u64_value_observer("cluster_available", move |observer| {
 					let h = get_health();
 					if h.status != ClusterHealthStatus::Unavailable {
 						observer.observe(1, &[]);
@@ -137,7 +137,7 @@ impl SystemMetrics {
 			_known_nodes: {
 				let get_health = get_health.clone();
 				meter
-					.u64_value_observer("garage_cluster_known_nodes", move |observer| {
+					.u64_value_observer("cluster_known_nodes", move |observer| {
 						let h = get_health();
 						observer.observe(h.known_nodes as u64, &[]);
 					})
@@ -147,7 +147,7 @@ impl SystemMetrics {
 			_connected_nodes: {
 				let get_health = get_health.clone();
 				meter
-					.u64_value_observer("garage_cluster_connected_nodes", move |observer| {
+					.u64_value_observer("cluster_connected_nodes", move |observer| {
 						let h = get_health();
 						observer.observe(h.connected_nodes as u64, &[]);
 					})
@@ -157,7 +157,7 @@ impl SystemMetrics {
 			_storage_nodes: {
 				let get_health = get_health.clone();
 				meter
-					.u64_value_observer("garage_cluster_storage_nodes", move |observer| {
+					.u64_value_observer("cluster_storage_nodes", move |observer| {
 						let h = get_health();
 						observer.observe(h.storage_nodes as u64, &[]);
 					})
@@ -167,7 +167,7 @@ impl SystemMetrics {
 			_storage_nodes_ok: {
 				let get_health = get_health.clone();
 				meter
-					.u64_value_observer("garage_cluster_storage_nodes_ok", move |observer| {
+					.u64_value_observer("cluster_storage_nodes_ok", move |observer| {
 						let h = get_health();
 						observer.observe(h.storage_nodes_ok as u64, &[]);
 					})
@@ -177,7 +177,7 @@ impl SystemMetrics {
 			_partitions: {
 				let get_health = get_health.clone();
 				meter
-					.u64_value_observer("garage_cluster_partitions", move |observer| {
+					.u64_value_observer("cluster_partitions", move |observer| {
 						let h = get_health();
 						observer.observe(h.partitions as u64, &[]);
 					})
@@ -187,7 +187,7 @@ impl SystemMetrics {
 			_partitions_quorum: {
 				let get_health = get_health.clone();
 				meter
-					.u64_value_observer("garage_cluster_partitions_quorum", move |observer| {
+					.u64_value_observer("cluster_partitions_quorum", move |observer| {
 						let h = get_health();
 						observer.observe(h.partitions_quorum as u64, &[]);
 					})
@@ -199,7 +199,7 @@ impl SystemMetrics {
 			_partitions_all_ok: {
 				let get_health = get_health.clone();
 				meter
-					.u64_value_observer("garage_cluster_partitions_all_ok", move |observer| {
+					.u64_value_observer("cluster_partitions_all_ok", move |observer| {
 						let h = get_health();
 						observer.observe(h.partitions_all_ok as u64, &[]);
 					})
@@ -213,7 +213,7 @@ impl SystemMetrics {
 			_layout_node_connected: {
 				let system = system.clone();
 				meter
-					.u64_value_observer("garage_cluster_layout_node_connected", move |observer| {
+					.u64_value_observer("cluster_layout_node_connected", move |observer| {
 						let layout = system.cluster_layout();
 						let nodes = system.get_known_nodes();
 						for id in layout.all_nodes().unwrap_or_default().iter() {
@@ -260,47 +260,44 @@ impl SystemMetrics {
 			_layout_node_disconnected_time: {
 				let system = system.clone();
 				meter
-					.u64_value_observer(
+					.u64_value_observer("cluster_layout_node_disconnected_time", move |observer| {
-						"garage_cluster_layout_node_disconnected_time",
+						let layout = system.cluster_layout();
-						move |observer| {
+						let nodes = system.get_known_nodes();
-							let layout = system.cluster_layout();
+						for id in layout.all_nodes().unwrap_or_default().iter() {
-							let nodes = system.get_known_nodes();
+							let mut kv = vec![KeyValue::new("id", format!("{:?}", id))];
-							for id in layout.all_nodes().unwrap_or_default().iter() {
+							if let Some(role) = layout
-								let mut kv = vec![KeyValue::new("id", format!("{:?}", id))];
+								.current()
-								if let Some(role) = layout
+								.ok()
-									.current()
+								.and_then(|l| l.roles.get(id))
-									.ok()
+								.and_then(|r| r.0.as_ref())
-									.and_then(|l| l.roles.get(id))
+							{
-									.and_then(|r| r.0.as_ref())
+								kv.push(KeyValue::new("role_zone", role.zone.clone()));
-								{
+								match role.capacity {
-									kv.push(KeyValue::new("role_zone", role.zone.clone()));
+									Some(cap) => {
-									match role.capacity {
+										kv.push(KeyValue::new("role_capacity", cap as i64));
-										Some(cap) => {
+										kv.push(KeyValue::new("role_gateway", 0));
-											kv.push(KeyValue::new("role_capacity", cap as i64));
-											kv.push(KeyValue::new("role_gateway", 0));
-										}
-										None => {
-											kv.push(KeyValue::new("role_gateway", 1));
-										}
 									}
-								}
+									None => {
-
+										kv.push(KeyValue::new("role_gateway", 1));
-								if let Some(node) = nodes.iter().find(|n| n.id == *id) {
-									// TODO: see comment above
-									// kv.push(KeyValue::new("address", node.addr.to_string()));
-									// kv.push(KeyValue::new(
-									//	"hostname",
-									//	node.status.hostname.clone(),
-									// ));
-									if node.is_up {
-										observer.observe(0, &kv);
-									} else if let Some(secs) = node.last_seen_secs_ago {
-										observer.observe(secs, &kv);
 									}
 								}
 							}
-						},
+
-					)
+							if let Some(node) = nodes.iter().find(|n| n.id == *id) {
+								// TODO: see comment above
+								// kv.push(KeyValue::new("address", node.addr.to_string()));
+								// kv.push(KeyValue::new(
+								//	"hostname",
+								//	node.status.hostname.clone(),
+								// ));
+								if node.is_up {
+									observer.observe(0, &kv);
+								} else if let Some(secs) = node.last_seen_secs_ago {
+									observer.observe(secs, &kv);
+								}
+							}
+						}
+					})
 					.with_description(
 						"Time (in seconds) since last connection to nodes in the cluster layout",
 					)

src/table/data.rs

@@ -26,15 +26,20 @@ pub struct TableData<F: TableSchema, R: TableReplication> {
 	pub instance: F,
 	pub replication: R,
 
+	// TODO: migrate to TypedTree
 	pub store: db::Tree,
 
+	// TODO: migrate to TypedTree
 	pub(crate) merkle_tree: db::Tree,
+	// TODO: migrate to TypedTree
 	pub(crate) merkle_todo: db::Tree,
 	pub(crate) merkle_todo_notify: Notify,
 
+	// TODO: migrate to TypedTree
 	pub(crate) insert_queue: db::Tree,
 	pub(crate) insert_queue_notify: Arc<Notify>,
 
+	// TODO: migrate to TypedTree
 	pub(crate) gc_todo: db::Tree,
 
 	pub(crate) metrics: TableMetrics,

src/table/metrics.rs

@@ -34,7 +34,7 @@ impl TableMetrics {
 		TableMetrics {
 			_table_size: meter
 				.u64_value_observer(
-					"garage_table.size",
+					"table.size",
 					move |observer| {
 						if let Ok(value) = store.approximate_len() {
 							observer.observe(
@@ -48,7 +48,7 @@ impl TableMetrics {
 				.init(),
 			_merkle_tree_size: meter
 				.u64_value_observer(
-					"garage_table.merkle_tree_size",
+					"table.merkle_tree_size",
 					move |observer| {
 						if let Ok(value) = merkle_tree.approximate_len() {
 							observer.observe(
@@ -62,7 +62,7 @@ impl TableMetrics {
 				.init(),
 			_merkle_todo_len: meter
 				.u64_value_observer(
-					"garage_table.merkle_updater_todo_queue_length",
+					"table.merkle_updater_todo_queue_length",
 					move |observer| {
 						if let Ok(v) = merkle_todo.approximate_len() {
 							observer.observe(
@@ -76,7 +76,7 @@ impl TableMetrics {
 				.init(),
 			_insert_queue_len: meter
 				.u64_value_observer(
-					"garage_table.insert_queue_length",
+					"table.insert_queue_length",
 					move |observer| {
 						if let Ok(v) = insert_queue.approximate_len() {
 							observer.observe(
@@ -90,7 +90,7 @@ impl TableMetrics {
 				.init(),
 			_gc_todo_len: meter
 				.u64_value_observer(
-					"garage_table.gc_todo_queue_length",
+					"table.gc_todo_queue_length",
 					move |observer| {
                         if let Ok(value) = gc_todo.approximate_len() {
                             observer.observe(
@@ -104,43 +104,43 @@ impl TableMetrics {
 				.init(),
 
 			get_request_counter: meter
-				.u64_counter("garage_table.get_request_count")
+				.u64_counter("table.get_request_counter")
 				.with_description("Number of get/get_range requests internally made on this table")
 				.init()
 				.bind(&[KeyValue::new("table_name", table_name)]),
 			get_request_duration: meter
-				.f64_value_recorder("garage_table.get_request_duration")
+				.f64_value_recorder("table.get_request_duration")
 				.with_description("Duration of get/get_range requests internally made on this table, in seconds")
 				.init()
 				.bind(&[KeyValue::new("table_name", table_name)]),
 			put_request_counter: meter
-				.u64_counter("garage_table.put_request_count")
+				.u64_counter("table.put_request_counter")
 				.with_description("Number of insert/insert_many requests internally made on this table")
 				.init()
 				.bind(&[KeyValue::new("table_name", table_name)]),
 			put_request_duration: meter
-				.f64_value_recorder("garage_table.put_request_duration")
+				.f64_value_recorder("table.put_request_duration")
 				.with_description("Duration of insert/insert_many requests internally made on this table, in seconds")
 				.init()
 				.bind(&[KeyValue::new("table_name", table_name)]),
 
 			internal_update_counter: meter
-				.u64_counter("garage_table.internal_update_count")
+				.u64_counter("table.internal_update_counter")
 				.with_description("Number of value updates where the value actually changes (includes creation of new key and update of existing key)")
 				.init()
 				.bind(&[KeyValue::new("table_name", table_name)]),
 			internal_delete_counter: meter
-				.u64_counter("garage_table.internal_delete_count")
+				.u64_counter("table.internal_delete_counter")
 				.with_description("Number of value deletions in the tree (due to GC or repartitioning)")
 				.init()
 				.bind(&[KeyValue::new("table_name", table_name)]),
 
 			sync_items_sent: meter
-				.u64_counter("garage_table.sync_items_sent")
+				.u64_counter("table.sync_items_sent")
 				.with_description("Number of data items sent to other nodes during resync procedures")
 				.init(),
 			sync_items_received: meter
-				.u64_counter("garage_table.sync_items_received")
+				.u64_counter("table.sync_items_received")
 				.with_description("Number of data items received from other nodes during resync procedures")
 				.init(),
 		}

src/table/table.rs

@@ -293,7 +293,26 @@ impl<F: TableSchema, R: TableReplication> Table<F, R> {
 		let span = tracer.start(format!("{} get", F::TABLE_NAME));
 
 		let res = self
-			.get_internal(partition_key, sort_key)
+			.get_internal(partition_key, sort_key, false)
+			.bound_record_duration(&self.data.metrics.get_request_duration)
+			.with_context(Context::current_with_span(span))
+			.await?;
+
+		self.data.metrics.get_request_counter.add(1);
+
+		Ok(res)
+	}
+
+	pub async fn get_monotonic(
+		self: &Arc<Self>,
+		partition_key: &F::P,
+		sort_key: &F::S,
+	) -> Result<Option<F::E>, Error> {
+		let tracer = opentelemetry::global::tracer("garage_table");
+		let span = tracer.start(format!("{} get_monotonic", F::TABLE_NAME));
+
+		let res = self
+			.get_internal(partition_key, sort_key, true)
 			.bound_record_duration(&self.data.metrics.get_request_duration)
 			.with_context(Context::current_with_span(span))
 			.await?;
@@ -307,6 +326,7 @@ impl<F: TableSchema, R: TableReplication> Table<F, R> {
 		self: &Arc<Self>,
 		partition_key: &F::P,
 		sort_key: &F::S,
+		monotonic_read: bool,
 	) -> Result<Option<F::E>, Error> {
 		let hash = partition_key.hash();
 		let who = self.data.replication.read_nodes(&hash)?;
@@ -326,34 +346,37 @@ impl<F: TableSchema, R: TableReplication> Table<F, R> {
 
 		let mut ret = None;
 		let mut not_all_same = false;
-		for resp in resps {
+		{
-			if let TableRpc::ReadEntryResponse(value) = resp {
+			let mut vals_nb = 0;
-				if let Some(v_bytes) = value {
+			for resp in &resps {
-					let v = self.data.decode_entry(v_bytes.as_slice())?;
+				if let TableRpc::ReadEntryResponse(value) = resp {
-					ret = match ret {
+					if let Some(v_bytes) = value {
-						None => Some(v),
+						vals_nb += 1;
-						Some(mut x) => {
+						let v = self.data.decode_entry(v_bytes.as_slice())?;
-							if x != v {
+						ret = match ret {
-								not_all_same = true;
+							None => Some(v),
-								x.merge(&v);
+							Some(mut x) => {
+								if x != v {
+									not_all_same = true;
+									x.merge(&v);
+								}
+								Some(x)
 							}
-							Some(x)
 						}
 					}
+				} else {
+					return Err(Error::Message("Invalid return value to read".to_string()));
 				}
-			} else {
+			}
-				return Err(Error::Message("Invalid return value to read".to_string()));
+			// Only some nodes store this value; we must propagate it during repair
+			if vals_nb < resps.len() {
+				not_all_same = true;
 			}
 		}
+
 		if let Some(ret_entry) = &ret {
-			if not_all_same {
+			if monotonic_read && not_all_same {
-				let self2 = self.clone();
+				self.repair_on_read(&who, &[ret_entry]).await?;
-				let ent2 = ret_entry.clone();
-				tokio::spawn(async move {
-					if let Err(e) = self2.repair_on_read(&who[..], ent2).await {
-						warn!("Error doing repair on read: {}", e);
-					}
-				});
 			}
 		}
 
@@ -378,6 +401,36 @@ impl<F: TableSchema, R: TableReplication> Table<F, R> {
 				filter,
 				limit,
 				enumeration_order,
+				false,
+			)
+			.bound_record_duration(&self.data.metrics.get_request_duration)
+			.with_context(Context::current_with_span(span))
+			.await?;
+
+		self.data.metrics.get_request_counter.add(1);
+
+		Ok(res)
+	}
+
+	pub async fn get_range_monotonic(
+		self: &Arc<Self>,
+		partition_key: &F::P,
+		begin_sort_key: Option<F::S>,
+		filter: Option<F::Filter>,
+		limit: usize,
+		enumeration_order: EnumerationOrder,
+	) -> Result<Vec<F::E>, Error> {
+		let tracer = opentelemetry::global::tracer("garage_table");
+		let span = tracer.start(format!("{} get_range_monotonic", F::TABLE_NAME));
+
+		let res = self
+			.get_range_internal(
+				partition_key,
+				begin_sort_key,
+				filter,
+				limit,
+				enumeration_order,
+				true,
 			)
 			.bound_record_duration(&self.data.metrics.get_request_duration)
 			.with_context(Context::current_with_span(span))
@@ -395,6 +448,7 @@ impl<F: TableSchema, R: TableReplication> Table<F, R> {
 		filter: Option<F::Filter>,
 		limit: usize,
 		enumeration_order: EnumerationOrder,
+		monotonic_read: bool,
 	) -> Result<Vec<F::E>, Error> {
 		let hash = partition_key.hash();
 		let who = self.data.replication.read_nodes(&hash)?;
@@ -421,11 +475,26 @@ impl<F: TableSchema, R: TableReplication> Table<F, R> {
 
 		let mut ret: BTreeMap<Vec<u8>, F::E> = BTreeMap::new();
 		let mut to_repair = BTreeSet::new();
-		for resp in resps {
+		{
-			if let TableRpc::Update(entries) = resp {
+			let mut all_entries: BTreeMap<Vec<u8>, Vec<F::E>> = BTreeMap::new();
-				for entry_bytes in entries.iter() {
+			for resp in &resps {
-					let entry = self.data.decode_entry(entry_bytes.as_slice())?;
+				if let TableRpc::Update(entries) = resp {
-					let entry_key = self.data.tree_key(entry.partition_key(), entry.sort_key());
+					for entry_bytes in entries.iter() {
+						let entry = self.data.decode_entry(entry_bytes.as_slice())?;
+						let entry_key = self.data.tree_key(entry.partition_key(), entry.sort_key());
+						all_entries.entry(entry_key).or_default().push(entry);
+					}
+				} else {
+					return Err(Error::unexpected_rpc_message(resp));
+				}
+			}
+			for (entry_key, entries) in all_entries {
+				// Only some nodes store this entry; we must propagate it during repair
+				if entries.len() < resps.len() {
+					to_repair.insert(entry_key.clone());
+				}
+				// Merge all entries for this key together
+				for entry in entries {
 					match ret.get_mut(&entry_key) {
 						Some(e) => {
 							if *e != entry {
@@ -434,28 +503,19 @@ impl<F: TableSchema, R: TableReplication> Table<F, R> {
 							}
 						}
 						None => {
-							ret.insert(entry_key, entry);
+							ret.insert(entry_key.clone(), entry);
 						}
 					}
 				}
-			} else {
-				return Err(Error::unexpected_rpc_message(resp));
 			}
 		}
 
-		if !to_repair.is_empty() {
+		if monotonic_read && !to_repair.is_empty() {
-			let self2 = self.clone();
+			let to_repair: Vec<_> = to_repair
-			let to_repair = to_repair
 				.into_iter()
-				.map(|k| ret.get(&k).unwrap().clone())
+				.map(|k| ret.get(&k).unwrap())
-				.collect::<Vec<_>>();
+				.collect();
-			tokio::spawn(async move {
+			self.repair_on_read(&who, &to_repair).await?;
-				for v in to_repair {
-					if let Err(e) = self2.repair_on_read(&who[..], v).await {
-						warn!("Error doing repair on read: {}", e);
-					}
-				}
-			});
 		}
 
 		// At this point, the `ret` btreemap might contain more than `limit`
@@ -493,14 +553,17 @@ impl<F: TableSchema, R: TableReplication> Table<F, R> {
 
 	// =============== UTILITY FUNCTION FOR CLIENT OPERATIONS ===============
 
-	async fn repair_on_read(&self, who: &[Uuid], what: F::E) -> Result<(), Error> {
+	pub async fn repair_on_read(&self, who: &[Uuid], what: &[&F::E]) -> Result<(), Error> {
-		let what_enc = Arc::new(ByteBuf::from(what.encode()?));
+		let what_enc = what
+			.iter()
+			.map(|v| Ok(Arc::new(ByteBuf::from(v.encode()?))))
+			.collect::<Result<Vec<_>, Error>>()?;
 		self.system
 			.rpc_helper()
 			.try_call_many(
 				&self.endpoint,
 				who,
-				TableRpc::<F>::Update(vec![what_enc]),
+				TableRpc::<F>::Update(what_enc),
 				RequestStrategy::with_priority(PRIO_NORMAL).with_quorum(who.len()),
 			)
 			.await?;

src/util/Cargo.toml

@@ -22,6 +22,8 @@ arc-swap.workspace = true
 async-trait.workspace = true
 blake2.workspace = true
 bytesize.workspace = true
+fundu.workspace = true
+fundu-systemd.workspace = true
 thiserror.workspace = true
 hexdump.workspace = true
 xxhash-rust.workspace = true

src/util/config.rs

@@ -282,7 +282,13 @@ pub fn default_block_max_concurrent_writes_per_request() -> usize {
 }
 /// Read and parse configuration
 pub fn read_config(config_file: PathBuf) -> Result<Config, Error> {
-	let config = std::fs::read_to_string(config_file)?;
+	let config = std::fs::read_to_string(&config_file).map_err(|e| {
+		format!(
+			"Failed to read config file {}: {}",
+			config_file.display(),
+			e
+		)
+	})?;
 
 	Ok(toml::from_str(&config)?)
 }

src/util/data.rs

@@ -155,6 +155,21 @@ pub fn gen_uuid() -> Uuid {
 	rand::rng().random::<[u8; 32]>().into()
 }
 
+impl garage_db::DbBytes for FixedBytes32 {
+	fn encode(&self) -> Vec<u8> {
+		self.0.into()
+	}
+	fn decode(bytes: &[u8]) -> std::result::Result<Self, garage_db::DecodeError> {
+		Self::try_from(bytes).ok_or_else(|| {
+			garage_db::DecodeError(
+				format!("invalid hash: expected 32 bytes, got {}", bytes.len()).into(),
+			)
+		})
+	}
+}
+
+impl garage_db::DbOrdKey for FixedBytes32 {}
+
 #[cfg(test)]
 mod test {
 	use super::*;

src/util/error.rs

@@ -79,6 +79,18 @@ impl Error {
 	}
 }
 
+impl From<garage_db::DbError> for Error {
+	fn from(e: garage_db::DbError) -> Error {
+		Error::Db(e.into())
+	}
+}
+
+impl From<garage_db::DecodeError> for Error {
+	fn from(e: garage_db::DecodeError) -> Error {
+		Error::Db(e.into())
+	}
+}
+
 impl From<garage_db::TxError<Error>> for Error {
 	fn from(e: garage_db::TxError<Error>) -> Error {
 		match e {

src/util/time.rs

@@ -1,6 +1,9 @@
 //! Module containing helper functions to manipulate time
 use chrono::{SecondsFormat, TimeZone, Utc};
-use std::time::{SystemTime, UNIX_EPOCH};
+use std::convert::TryInto;
+use std::time::{Duration, SystemTime, UNIX_EPOCH};
+
+use crate::error::Error;
 
 /// Returns milliseconds since UNIX Epoch
 pub fn now_msec() -> u64 {
@@ -28,3 +31,13 @@ pub fn msec_to_rfc3339(msecs: u64) -> String {
 	let timestamp = Utc.timestamp_opt(secs, nanos).unwrap();
 	timestamp.to_rfc3339_opts(SecondsFormat::Millis, true)
 }
+
+/// Parse a systemd-style duration using fundu
+pub fn parse_duration(s: &str) -> Result<Duration, Error> {
+	fundu_systemd::parse(s, Some(fundu::TimeUnit::Second), None)
+		.map_err(|err| Error::Message(err.to_string()))
+		.and_then(|dur| {
+			dur.try_into()
+				.map_err(|err: fundu::TryFromDurationError| Error::Message(err.to_string()))
+		})
+}

src/web/Cargo.toml

@@ -20,6 +20,7 @@ garage_model.workspace = true
 garage_util.workspace = true
 garage_table.workspace = true
 
+html-escape.workspace = true
 thiserror.workspace = true
 tracing.workspace = true
 percent-encoding.workspace = true

src/web/web_server.rs

@@ -54,15 +54,15 @@ impl WebMetrics {
 		let meter = global::meter("garage/web");
 		Self {
 			request_counter: meter
-				.u64_counter("garage_web.request_count")
+				.u64_counter("web.request_counter")
 				.with_description("Number of requests to the web endpoint")
 				.init(),
 			error_counter: meter
-				.u64_counter("garage_web.error_count")
+				.u64_counter("web.error_counter")
 				.with_description("Number of requests to the web endpoint resulting in errors")
 				.init(),
 			request_duration: meter
-				.f64_value_recorder("garage_web.request_duration")
+				.f64_value_recorder("web.request_duration")
 				.with_description("Duration of requests to the web endpoint")
 				.init(),
 		}
@@ -483,7 +483,7 @@ fn error_to_res(e: Error) -> Response<BoxBody<Error>> {
 <li>Message: {s3_message}.</li>
 </ul>",
 			s3_code = err.aws_code(),
-			s3_message = err,
+			s3_message = html_escape::encode_text(&err.to_string()),
 		));
 	}
 	let mut http_error = Response::new(string_body(body_str));