fix: improve routing of keys starting with "/" (fix #1178) (#1465)

Path-style URLs of the form /bucket//key address an object whose key begins with "/". Two greedy uses of `trim_start_matches('/')` were collapsing these leading slashes away:

- `uri.path().trim_start_matches('/')` stripped all leading slashes from the raw path before any further parsing.
- `p.trim_start_matches('/')` stripped leading slashes from the remainder after `split_once('/')` had already consumed the bucket/key separator

The combined effect wath that `HEAD /bucket//` and `GET /bucket//` produced an empty key, which the router treated as bucket-level operations (HeadBucket -> 200 OK, and ListObjectsV2) instead of an object-level op (HeadObject/GetObject -> 404 NoSuchKey).

The fix is simple: Replace the first `trim_start_matches` with `strip_prefix` (to strip exactly one separator slash) and remove the second one entirely. Path-style and vhost-style requests are now consistent: a double slash in the URL correctly addresses a key whose name begins with "/".

Regression tests added for `HEAD //` and `GET //` requests in both request styles.

Fixes: #1464

---

Disclaimer: I'm not fluent in Rust and I did use an LLM to explain the code to me. All code was written by me.

I'm not sure whether the large `test_cases!` block in the `test_aws_doc_examples` function is the right place for my tests (it certainly was a convenient one).

Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1465

Dockerfile

@@ -4,4 +4,6 @@ ENV RUST_BACKTRACE=1
 ENV RUST_LOG=garage=info
 
 COPY result/bin/garage /
-CMD [ "/garage", "server"]
+
+ENTRYPOINT ["/garage"]
+CMD ["server"]

doc/book/cookbook/real-world.md

@@ -213,7 +213,12 @@ If your configuration file is at `/etc/garage.toml`, the `garage` binary should
 You can also use an alias as follows to use the Garage binary inside your docker container:
 
 ```bash
+# garage 3.x, we have an entrypoint and you can use
+alias garage="docker exec -ti <container name>"
+
+# For garage 2.x, you need to specify the absolute path to binary
 alias garage="docker exec -ti <container name> /garage"
+
 ```
 
 You can test your `garage` CLI utility by running a simple command such as:

doc/book/quick-start/_index.md

@@ -178,8 +178,11 @@ garage status
 
 If you are running Garage in a Docker container, you can use the following command instead:
 
+NOTE: Garage 3.x uses docker `ENTRYPOINT` and it's easier to use,
+while garage 2.x does not and you need to specify path `/garage`
+
 ```bash
-docker exec garage-container /garage status
+docker exec garage-container status
 ```
 
 This should show something like this:
@@ -320,7 +323,7 @@ If you are running Garage in a Docker container, you can set the following alias
 to provide a fake `garage`command that uses the Garage binary inside your container:
 
 ```bash
-alias garage="docker exec -ti <container name> /garage"
+alias garage="docker exec -ti <container name>"
 ```
 
 You can test that your `garage` CLI is configured correctly by running a basic command such as `garage status`.

doc/book/reference-manual/admin-api.md

@@ -182,15 +182,15 @@ content-type: text/plain; version=0.0.4
 content-length: 12145
 date: Tue, 08 Aug 2023 07:25:05 GMT
 
-# HELP api_admin_error_counter Number of API calls to the various Admin API endpoints that resulted in errors
+# HELP garage_api_admin_error_count Number of API calls to the various Admin API endpoints that resulted in errors
-# TYPE api_admin_error_counter counter
+# TYPE garage_api_admin_error_count counter
-api_admin_error_counter{api_endpoint="CheckWebsiteEnabled",status_code="400"} 1
+garage_api_admin_error_count{api_endpoint="CheckWebsiteEnabled",status_code="400"} 1
-api_admin_error_counter{api_endpoint="CheckWebsiteEnabled",status_code="404"} 3
+garage_api_admin_error_count{api_endpoint="CheckWebsiteEnabled",status_code="404"} 3
-# HELP api_admin_request_counter Number of API calls to the various Admin API endpoints
+# HELP garage_api_admin_request_count Number of API calls to the various Admin API endpoints
-# TYPE api_admin_request_counter counter
+# TYPE garage_api_admin_request_count counter
-api_admin_request_counter{api_endpoint="CheckWebsiteEnabled"} 7
+garage_api_admin_request_count{api_endpoint="CheckWebsiteEnabled"} 7
-api_admin_request_counter{api_endpoint="Health"} 3
+garage_api_admin_request_count{api_endpoint="Health"} 3
-# HELP api_admin_request_duration Duration of API calls to the various Admin API endpoints
+# HELP garage_api_admin_request_duration Duration of API calls to the various Admin API endpoints
 ...
 ```
 

doc/book/reference-manual/monitoring.md

@@ -40,146 +40,146 @@ garage_local_disk_total{volume="metadata"} 763063566336
 
 ### Cluster health status metrics
 
-#### `cluster_healthy` (gauge)
+#### `garage_cluster_healthy` (gauge)
 
 Whether all storage nodes are connected (0 or 1)
 
 ```
-cluster_healthy 0
+garage_cluster_healthy 0
 ```
 
-#### `cluster_available` (gauge)
+#### `garage_cluster_available` (gauge)
 
 Whether all requests can be served, even if some storage nodes are disconnected
 
 ```
-cluster_available 1
+garage_cluster_available 1
 ```
 
-#### `cluster_connected_nodes` (gauge)
+#### `garage_cluster_connected_nodes` (gauge)
 
 Number of nodes currently connected
 
 ```
-cluster_connected_nodes 3
+garage_cluster_connected_nodes 3
 ```
 
-#### `cluster_known_nodes` (gauge)
+#### `garage_cluster_known_nodes` (gauge)
 
 Number of nodes already seen once in the cluster
 
 ```
-cluster_known_nodes 3
+garage_cluster_known_nodes 3
 ```
 
-#### `cluster_layout_node_connected` (gauge)
+#### `garage_cluster_layout_node_connected` (gauge)
 
 Connection status for individual nodes of the cluster layout
 
 ```
-cluster_layout_node_connected{id="62b218d848e86a64",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 1
+garage_cluster_layout_node_connected{id="62b218d848e86a64",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 1
-cluster_layout_node_connected{id="a11c7cf18af29737",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 0
+garage_cluster_layout_node_connected{id="a11c7cf18af29737",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 0
-cluster_layout_node_connected{id="a235ac7695e0c54d",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 1
+garage_cluster_layout_node_connected{id="a235ac7695e0c54d",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 1
-cluster_layout_node_connected{id="b10c110e4e854e5a",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 1
+garage_cluster_layout_node_connected{id="b10c110e4e854e5a",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 1
 ```
 
-#### `cluster_layout_node_disconnected_time` (gauge)
+#### `garage_cluster_layout_node_disconnected_time` (gauge)
 
 Time (in seconds) since last connection to individual nodes of the cluster layout
 
 ```
-cluster_layout_node_disconnected_time{id="62b218d848e86a64",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 0
+garage_cluster_layout_node_disconnected_time{id="62b218d848e86a64",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 0
-cluster_layout_node_disconnected_time{id="a235ac7695e0c54d",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 0
+garage_cluster_layout_node_disconnected_time{id="a235ac7695e0c54d",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 0
-cluster_layout_node_disconnected_time{id="b10c110e4e854e5a",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 0
+garage_cluster_layout_node_disconnected_time{id="b10c110e4e854e5a",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 0
 ```
 
-#### `cluster_storage_nodes` (gauge)
+#### `garage_cluster_storage_nodes` (gauge)
 
 Number of storage nodes declared in the current layout
 
 ```
-cluster_storage_nodes 4
+garage_cluster_storage_nodes 4
 ```
 
-#### `cluster_storage_nodes_ok` (gauge)
+#### `garage_cluster_storage_nodes_ok` (gauge)
 
 Number of storage nodes currently connected
 
 ```
-cluster_storage_nodes_ok 3
+garage_cluster_storage_nodes_ok 3
 ```
 
-#### `cluster_partitions` (gauge)
+#### `garage_cluster_partitions` (gauge)
 
 Number of partitions in the layout (this is always 256)
 
 ```
-cluster_partitions 256
+garage_cluster_partitions 256
 ```
 
-#### `cluster_partitions_all_ok` (gauge)
+#### `garage_cluster_partitions_all_ok` (gauge)
 
 Number of partitions for which all storage nodes are connected
 
 ```
-cluster_partitions_all_ok 64
+garage_cluster_partitions_all_ok 64
 ```
 
-#### `cluster_partitions_quorum` (gauge)
+#### `garage_cluster_partitions_quorum` (gauge)
 
 Number of partitions for which we have a quorum of connected nodes and all requests can be served
 
 ```
-cluster_partitions_quorum 256
+garage_cluster_partitions_quorum 256
 ```
 
 ### Metrics of the API endpoints
 
-#### `api_admin_request_counter` (counter)
+#### `garage_api_admin_request_count` (counter)
 
 Counts the number of requests to a given endpoint of the administration API. Example:
 
 ```
-api_admin_request_counter{api_endpoint="Metrics"} 127041
+garage_api_admin_request_count{api_endpoint="Metrics"} 127041
 ```
 
-#### `api_admin_request_duration` (histogram)
+#### `garage_api_admin_request_duration` (histogram)
 
 Evaluates the duration of API calls to the various administration API endpoint. Example:
 
 ```
-api_admin_request_duration_bucket{api_endpoint="Metrics",le="0.5"} 127041
+garage_api_admin_request_duration_bucket{api_endpoint="Metrics",le="0.5"} 127041
-api_admin_request_duration_sum{api_endpoint="Metrics"} 605.250344830999
+garage_api_admin_request_duration_sum{api_endpoint="Metrics"} 605.250344830999
-api_admin_request_duration_count{api_endpoint="Metrics"} 127041
+garage_api_admin_request_duration_count{api_endpoint="Metrics"} 127041
 ```
 
-#### `api_s3_request_counter` (counter)
+#### `garage_api_s3_request_count` (counter)
 
 Counts the number of requests to a given endpoint of the S3 API. Example:
 
 ```
-api_s3_request_counter{api_endpoint="CreateMultipartUpload"} 1
+garage_api_s3_request_count{api_endpoint="CreateMultipartUpload"} 1
 ```
 
-#### `api_s3_error_counter` (counter)
+#### `garage_api_s3_error_count` (counter)
 
 Counts the number of requests to a given endpoint of the S3 API that returned an error. Example:
 
 ```
-api_s3_error_counter{api_endpoint="GetObject",status_code="404"} 39
+garage_api_s3_error_count{api_endpoint="GetObject",status_code="404"} 39
 ```
 
-#### `api_s3_request_duration` (histogram)
+#### `garage_api_s3_request_duration` (histogram)
 
 Evaluates the duration of API calls to the various S3 API endpoints. Example:
 
 ```
-api_s3_request_duration_bucket{api_endpoint="CreateMultipartUpload",le="0.5"} 1
+garage_api_s3_request_duration_bucket{api_endpoint="CreateMultipartUpload",le="0.5"} 1
-api_s3_request_duration_sum{api_endpoint="CreateMultipartUpload"} 0.046340762
+garage_api_s3_request_duration_sum{api_endpoint="CreateMultipartUpload"} 0.046340762
-api_s3_request_duration_count{api_endpoint="CreateMultipartUpload"} 1
+garage_api_s3_request_duration_count{api_endpoint="CreateMultipartUpload"} 1
 ```
 
-#### `api_k2v_request_counter` (counter), `api_k2v_error_counter` (counter), `api_k2v_error_duration` (histogram)
+#### `garage_api_k2v_request_count` (counter), `garage_api_k2v_error_count` (counter), `garage_api_k2v_error_duration` (histogram)
 
 Same as for S3, for the K2V API.
 
@@ -187,45 +187,45 @@ Same as for S3, for the K2V API.
 ### Metrics of the Web endpoint
 
 
-#### `web_request_counter` (counter)
+#### `garage_web_request_count` (counter)
 
 Number of requests to the web endpoint
 
 ```
-web_request_counter{method="GET"} 80
+garage_web_request_count{method="GET"} 80
 ```
 
-#### `web_request_duration` (histogram)
+#### `garage_web_request_duration` (histogram)
 
 Duration of requests to the web endpoint
 
 ```
-web_request_duration_bucket{method="GET",le="0.5"} 80
+garage_web_request_duration_bucket{method="GET",le="0.5"} 80
-web_request_duration_sum{method="GET"} 1.0528433229999998
+garage_web_request_duration_sum{method="GET"} 1.0528433229999998
-web_request_duration_count{method="GET"} 80
+garage_web_request_duration_count{method="GET"} 80
 ```
 
-#### `web_error_counter` (counter)
+#### `garage_web_error_count` (counter)
 
 Number of requests to the web endpoint resulting in errors
 
 ```
-web_error_counter{method="GET",status_code="404 Not Found"} 64
+garage_web_error_count{method="GET",status_code="404 Not Found"} 64
 ```
 
 
 ### Metrics of the data block manager
 
-#### `block_bytes_read`, `block_bytes_written` (counter)
+#### `garage_block_bytes_read`, `garage_block_bytes_written` (counter)
 
 Number of bytes read/written to/from disk in the data storage directory.
 
 ```
-block_bytes_read 120586322022
+garage_block_bytes_read 120586322022
-block_bytes_written 3386618077
+garage_block_bytes_written 3386618077
 ```
 
-#### `block_ram_buffer_free_kb` (gauge)
+#### `garage_block_ram_buffer_free_kb` (gauge)
 
 Kibibytes available for buffering blocks that have to be sent to remote nodes.
 When clients send too much data to this node and a storage node is not receiving
@@ -233,170 +233,168 @@ data fast enough due to slower network conditions, this will decrease down to
 zero and backpressure will be applied.
 
 ```
-block_ram_buffer_free_kb 219829
+garage_block_ram_buffer_free_kb 219829
 ```
 
-#### `block_compression_level` (counter)
+#### `garage_block_compression_level` (counter)
 
 Exposes the block compression level configured for the Garage node.
 
 ```
-block_compression_level 3
+garage_block_compression_level 3
 ```
 
-#### `block_read_duration`, `block_write_duration` (histograms)
+#### `garage_block_read_duration`, `garage_block_write_duration` (histograms)
 
 Evaluates the duration of the reading/writing of individual data blocks in the data storage directory.
 
 ```
-block_read_duration_bucket{le="0.5"} 169229
+garage_block_read_duration_bucket{le="0.5"} 169229
-block_read_duration_sum 2761.6902550310056
+garage_block_read_duration_sum 2761.6902550310056
-block_read_duration_count 169240
+garage_block_read_duration_count 169240
-block_write_duration_bucket{le="0.5"} 3559
+garage_block_write_duration_bucket{le="0.5"} 3559
-block_write_duration_sum 195.59170078500006
+garage_block_write_duration_sum 195.59170078500006
-block_write_duration_count 3571
+garage_block_write_duration_count 3571
 ```
 
-#### `block_delete_counter` (counter)
+#### `garage_block_delete_count` (counter)
 
 Counts the number of data blocks that have been deleted from storage.
 
 ```
-block_delete_counter 122
+garage_block_delete_count 122
 ```
 
-#### `block_resync_counter` (counter), `block_resync_duration` (histogram)
+#### `garage_block_resync_count` (counter), `garage_block_resync_duration` (histogram)
 
 Counts the number of resync operations the node has executed, and evaluates their duration.
 
 ```
-block_resync_counter 308897
+garage_block_resync_count 308897
-block_resync_duration_bucket{le="0.5"} 308892
+garage_block_resync_duration_bucket{le="0.5"} 308892
-block_resync_duration_sum 139.64204196100016
+garage_block_resync_duration_sum 139.64204196100016
-block_resync_duration_count 308897
+garage_block_resync_duration_count 308897
 ```
 
-#### `block_resync_queue_length` (gauge)
+#### `garage_block_resync_queue_length` (gauge)
 
 The number of block hashes currently queued for a resync.
 This is normal to be nonzero for long periods of time.
 
 ```
-block_resync_queue_length 0
+garage_block_resync_queue_length 0
 ```
 
-#### `block_resync_errored_blocks` (gauge)
+#### `garage_block_resync_errored_blocks` (gauge)
 
 The number of block hashes that we were unable to resync last time we tried.
 **THIS SHOULD BE ZERO, OR FALL BACK TO ZERO RAPIDLY, IN A HEALTHY CLUSTER.**
 Persistent nonzero values indicate that some data is likely to be lost.
 
 ```
-block_resync_errored_blocks 0
+garage_block_resync_errored_blocks 0
 ```
 
 
 ### Metrics related to RPCs (remote procedure calls) between nodes
 
-#### `rpc_netapp_request_counter` (counter)
+#### `garage_rpc_netapp_request_count` (counter)
 
 Number of RPC requests emitted
 
 ```
-rpc_request_counter{from="<this node>",rpc_endpoint="garage_block/manager.rs/Rpc",to="<remote node>"} 176
+garage_rpc_request_count{from="<this node>",rpc_endpoint="garage_block/manager.rs/Rpc",to="<remote node>"} 176
 ```
 
-#### `rpc_netapp_error_counter` (counter)
+#### `garage_rpc_netapp_error_count` (counter)
 
 Number of communication errors (errors in the Netapp library, generally due to disconnected nodes)
 
 ```
-rpc_netapp_error_counter{from="<this node>",rpc_endpoint="garage_block/manager.rs/Rpc",to="<remote node>"} 354
+garage_rpc_netapp_error_count{from="<this node>",rpc_endpoint="garage_block/manager.rs/Rpc",to="<remote node>"} 354
 ```
 
-#### `rpc_timeout_counter` (counter)
+#### `garage_rpc_timeout_count` (counter)
 
 Number of RPC timeouts, should be close to zero in a healthy cluster.
 
 ```
-rpc_timeout_counter{from="<this node>",rpc_endpoint="garage_rpc/membership.rs/SystemRpc",to="<remote node>"} 1
+garage_rpc_timeout_count{from="<this node>",rpc_endpoint="garage_rpc/membership.rs/SystemRpc",to="<remote node>"} 1
 ```
 
-#### `rpc_duration` (histogram)
+#### `garage_rpc_duration` (histogram)
 
 The duration of internal RPC calls between Garage nodes.
 
 ```
-rpc_duration_bucket{from="<this node>",rpc_endpoint="garage_block/manager.rs/Rpc",to="<remote node>",le="0.5"} 166
+garage_rpc_duration_bucket{from="<this node>",rpc_endpoint="garage_block/manager.rs/Rpc",to="<remote node>",le="0.5"} 166
-rpc_duration_sum{from="<this node>",rpc_endpoint="garage_block/manager.rs/Rpc",to="<remote node>"} 35.172253716
+garage_rpc_duration_sum{from="<this node>",rpc_endpoint="garage_block/manager.rs/Rpc",to="<remote node>"} 35.172253716
-rpc_duration_count{from="<this node>",rpc_endpoint="garage_block/manager.rs/Rpc",to="<remote node>"} 174
+garage_rpc_duration_count{from="<this node>",rpc_endpoint="garage_block/manager.rs/Rpc",to="<remote node>"} 174
 ```
 
 
 ### Metrics of the metadata table manager
 
-#### `table_gc_todo_queue_length` (gauge)
+#### `garage_table_gc_todo_queue_length` (gauge)
 
 Table garbage collector TODO queue length
 
 ```
-table_gc_todo_queue_length{table_name="block_ref"} 0
+garage_table_gc_todo_queue_length{table_name="block_ref"} 0
 ```
 
-#### `table_get_request_counter` (counter), `table_get_request_duration` (histogram)
+#### `garage_table_get_request_count` (counter), `garage_table_get_request_duration` (histogram)
 
 Number of get/get_range requests internally made on each table, and their duration.
 
 ```
-table_get_request_counter{table_name="bucket_alias"} 315
+garage_table_get_request_count{table_name="bucket_alias"} 315
-table_get_request_duration_bucket{table_name="bucket_alias",le="0.5"} 315
+garage_table_get_request_duration_bucket{table_name="bucket_alias",le="0.5"} 315
-table_get_request_duration_sum{table_name="bucket_alias"} 0.048509778000000024
+garage_table_get_request_duration_sum{table_name="bucket_alias"} 0.048509778000000024
-table_get_request_duration_count{table_name="bucket_alias"} 315
+garage_table_get_request_duration_count{table_name="bucket_alias"} 315
 ```
 
 
-#### `table_put_request_counter` (counter), `table_put_request_duration` (histogram)
+#### `garage_table_put_request_count` (counter), `garage_table_put_request_duration` (histogram)
 
 Number of insert/insert_many requests internally made on this table, and their duration
 
 ```
-table_put_request_counter{table_name="block_ref"} 677
+garage_table_put_request_count{table_name="block_ref"} 677
-table_put_request_duration_bucket{table_name="block_ref",le="0.5"} 677
+garage_table_put_request_duration_bucket{table_name="block_ref",le="0.5"} 677
-table_put_request_duration_sum{table_name="block_ref"} 61.617528636
+garage_table_put_request_duration_sum{table_name="block_ref"} 61.617528636
-table_put_request_duration_count{table_name="block_ref"} 677
+garage_table_put_request_duration_count{table_name="block_ref"} 677
 ```
 
-#### `table_internal_delete_counter` (counter)
+#### `garage_table_internal_delete_count` (counter)
 
 Number of value deletions in the tree (due to GC or repartitioning)
 
 ```
-table_internal_delete_counter{table_name="block_ref"} 2296
+garage_table_internal_delete_count{table_name="block_ref"} 2296
 ```
 
-#### `table_internal_update_counter` (counter)
+#### `garage_table_internal_update_count` (counter)
 
 Number of value updates where the value actually changes (includes creation of new key and update of existing key)
 
 ```
-table_internal_update_counter{table_name="block_ref"} 5996
+garage_table_internal_update_count{table_name="block_ref"} 5996
 ```
 
-#### `table_merkle_updater_todo_queue_length` (gauge)
+#### `garage_table_merkle_updater_todo_queue_length` (gauge)
 
 Merkle tree updater TODO queue length (should fall to zero rapidly)
 
 ```
-table_merkle_updater_todo_queue_length{table_name="block_ref"} 0
+garage_table_merkle_updater_todo_queue_length{table_name="block_ref"} 0
 ```
 
-#### `table_sync_items_received`, `table_sync_items_sent` (counters)
+#### `garage_table_sync_items_received`, `garage_table_sync_items_sent` (counters)
 
 Number of data items sent to/received from other nodes during resync procedures
 
 ```
-table_sync_items_received{from="<remote node>",table_name="bucket_v2"} 3
+garage_table_sync_items_received{from="<remote node>",table_name="bucket_v2"} 3
-table_sync_items_sent{table_name="block_ref",to="<remote node>"} 2
+garage_table_sync_items_sent{table_name="block_ref",to="<remote node>"} 2
 ```
-
-

doc/talks/2026-06-03-adr/README.md

@@ -1,6 +0,0 @@
-Compile with:
-
-```
-typst compile talk.typ --root ../..
-```
-

doc/talks/2026-06-03-adr/talk.typ

@@ -1,378 +0,0 @@
-#import "@preview/slydst:0.1.5": *
-
-// some display rules
-
-#set par(spacing: 2em)
-#set list(spacing: 1em)
-
-#show link: set text(font: "DejaVu Sans Mono", size: 9pt)
-
-// some functions to customize styles
-
-#let vhcenter(content) = [
-  #v(1fr)
-  #align(center)[#content]
-  #v(1fr)
-]
-
-#let imgcenter(..args) = vhcenter(image(..args))
-
-#let mytable(..args) = {
-  show table.cell: set text(size: 9pt)
-  set table(stroke: 0.5pt + black)
-  grid(
-    columns: (1cm, 1fr, 1cm),
-    [], table(..args), []
-  )
-}
-
-// actual slides
-
-#show: slides.with(
-  //title: "Garage",
-  authors: ("Alex Auvolat",),
-  date: "2026-06-03",
-  layout: "large",
-  //ratio: 16/9,
-  ratio: 4/3,
-  title-color: rgb("#ff9329"),
-)
-
-#title-slide[
-  #align(center)[
-    #image("../../sticker/Garage.png", width: 20%)
-    #v(1em)
-    *An introduction to Garage*\
-    Alex Auvolat, Deuxfleurs
-
-    #v(1em)
-    #link("https://garagehq.deuxfleurs.fr/")\
-    Matrix channel: `#garage:deuxfleurs.fr`
-  ]
-]
-
-== A non-profit initiative
-
-#grid(
-  columns: (2fr, 8fr),
-  [#v(2em)],[],
-  [
-    #image("../assets/logos/deuxfleurs.svg", width: 50%)
-  ],
-  [
-    *Part of a degrowth initiative*\
-    Garage has been created at Deuxfleurs, where we experiment running Internet services without datacenter on commodity and refurbished hardware.
-  ],
-  [#v(2em)],[],
-  [
-    #image("../assets/community.png", width: 50%)
-  ],
-  [
-    *Developed by a community*\
-    #text(size: 0.8em)[Some recent contributors: Arthur C, Charles H, dongdigua, Etienne L, Jonah A, Julien K, Lapineige, MagicRR, Milas B, Niklas M, RockWolf, Schwitzd, trinity-1686a, Xavier S, babykart, Baptiste J, eddster2309, James O'C, Joker9944, Maximilien R, Renjaya RZ, Yureka...]
-  ],
-  [#v(3em)],[],
-  [
-    #image("../assets/logos/AGPLv3_Logo.png", width: 50%)
-  ],
-  [
-    *Owned by nobody*\
-    AGPL + no Contributor License Agreement = Garage ownership spreads among dozens of contributors.
-  ]
-)
-
-== Our initial objective at Deuxfleurs
-
-#v(4em)
-
-#align(center)[
-  #text(weight: "bold")[
-    Promote self-hosting and small-scale hosting\
-    as an alternative to large cloud providers
-  ]
-]
-
-#v(2em)
-
-Why is it hard?
-
-#v(2em)
-
-#align(center)[
-  #underline[Resilience]\
-  #text(size: 0.8em)[we want good uptime/availability with low supervision]
-]
-
-== Our very low-tech infrastructure
-//== Building a resilient system with cheap stuff
-//
-#v(4em)
-
-#[
-  #set list(spacing: 2em)
-  - Commodity hardware (e.g. old desktop PCs)\
-    #text(size: 0.8em)[(can die at any time)]
-  - Regular Internet (e.g. FTTB, FTTH) and power grid connections\
-    #text(size: 0.8em)[(can be unavailable randomly)]
-  - *Geographical redundancy* (multi-site replication)
-]
-
-#pagebreak()
-#imgcenter("../assets/neptune.jpg", width: 100%)
-
-#pagebreak()
-#imgcenter("../assets/atuin.jpg", width: 100%)
-
-#pagebreak()
-#imgcenter("../assets/inframap_jdll2023.pdf", width: 100%)
-
-== Object storage: a crucial component
-
-#vhcenter[
-  #grid(
-    columns: (3fr, 3fr, 3fr),
-    [#image("../assets/logos/Amazon-S3.jpg", height: 6em)],
-    [#image("../assets/logos/minio.png", height: 5em)],
-    [#image("../../logo/garage_hires_crop.png", height: 6em)]
-  )
-]
-
-S3: a de-facto standard, many compatible applications
-
-MinIO: not suited for geo-distributed deployments, becoming closed source
-
-*Garage is a self-hosted drop-in replacement for the Amazon S3 object store*
-
-#v(2em)
-
-== Principle 1: geo-distributed data model
-
-#imgcenter("../assets/map.png", width: 90%)
-
-Garage stores replicas on different zones when possible
-
-== Zone-aware cluster configuration
-
-#imgcenter("../assets/screenshots/garage_status_0.9_prod_zonehl.png", width: 100%)
-
-Trust model: full trust between zones
-
-#v(5em)
-
-== Principle 2: based on CRDTs
-
-#v(1cm)
-
-#underline[Internally, Garage uses only CRDTs] (conflict-free replicated data types)
-
-Why not Raft, Paxos, ...? Issues of consensus algorithms:
-
-- *Software complexity*
-- *Performance issues:*
-  - The leader is a *bottleneck* for all requests
-  - *Sensitive to higher latency* between nodes
-  - *Takes time to reconverge* when disrupted (e.g. node going down)
-
-
-== The data model of object storage
-
-#[
-  #set list(spacing: 1em)
-
-  Object storage is basically a *key-value store*:
-  
-  #mytable(
-    columns: (2fr, 5fr),
-    align: left,
-    [*Key: file path + name*], [*Value: file data + metadata*],
-    [`index.html`], text(size: 8pt)[
-      `Content-Type: text/html; charset=utf-8`\
-      `Content-Length: 24929`\
-      `<binary blob>`
-    ],
-    [`img/logo.svg`], text(size: 8pt)[
-      `Content-Type: text/svg+xml`\
-      `Content-Length: 13429`\
-      `<binary blob>`
-    ],
-    [`download/index.html`], text(size: 8pt)[
-      `Content-Type: text/html; charset=utf-8`\
-      `Content-Length: 26563`\
-      `<binary blob>`
-    ]
-  )
-  
-*Consistency model:*
-
-- Not ACID (not required by S3 spec) / not linearizable
-- *Read-after-write consistency*\
-  #text(size: 0.8em)[(stronger than eventual consistency)]
-]
-
-
-== Performance evaluation
-
-#imgcenter("../assets/perf/endpoint_latency_0.7_0.8_minio.png", width: 100%)
-#pagebreak()
-
-#imgcenter("../assets/perf/ttfb.png", width: 100%)
-#pagebreak()
-
-#imgcenter("../assets/perf/io-0.7-0.8-minio.png", width: 100%)
-
-
-== Garage in the wild
-
-#imgcenter("../assets/cluster_kind.png", width: 100%)
-
-== Size of known deployments
-
-#imgcenter("../assets/cluster_size.png", width: 100%)
-
-_"Petabyte storage setup for a video site. Nginx as CDN in-front using garage-s3-website feature. Each storage node has ~64TB storage with raid10, no replication within garage. 25gbit nic. haproxy to loadbalance across 5 nodes. mostly reads with very few writes."_
-
-_"We currently manage 7 Garage nodes, 28TB total storage, 6M blocks for 3M objects and 4TB of object data. We have been running Garage in production for 2.5 years."_
-
-= Deploying Garage
-
-== Chosing a replication factor
-
-#vhcenter[
-  #mytable(
-    columns: (0.7fr, 1fr, 1.3fr),
-    inset: 0.8em,
-    align: center + horizon,
-    table.header[*Replication factor*][*Pro*][*Cons*],
-    [*1*], [easy single-node setup\ full space efficiency], [no metadata redundancy\ *vunlerable to hardware crash or data corruption*\ no high-availability],
-    [*2*], [redundancy\ limited storage overhead], [limited high-availability\ (read-only when one node is unavailable)],
-    [*3*], [high-availability setup\ best data resilience], [big storage overhead],
-    [*4, 5, ...*], [possible if needed], [...],
-
-  )
-
-  #v(0.5cm)
-  *Important note:* metadata replication == data replication\
-  Choose well, this cannot be changed easily!
-]
-
-== Setting up data and metadata storage
-
-#vhcenter[
-  #mytable(
-    columns: (0.7fr, 1fr, 1fr),
-    inset: 0.8em,
-    align: center + horizon,
-    table.header[][*Metadata storage*][*Data storage*],
-    [*Content*],[access keys, buckets\ index of objects],[raw data blocks],
-    [*Size*],[\< 10\% of data\ rarely over 100GB],[replication × dataset size\ *no erasure-coding*],
-    [*Constraints*],[latency sensitive\ write-intensive under load],[big\ many files],
-    [*Ideal hardware*],[entreprise-grade SSD],[HDD],
-    [*Recommended redundancy*],[RAID1],[none, use disks directly\ *avoid RAID if possible*],
-    [*Recommended filesystem*],[ZFS, Btrfs],[XFS on invidual disks],
-    [*Tunables in Garage*],[database engine\
-                            automatic snapshots],[block size\ compression],
-  )
-]
-
-== Picking a metadata engine
-
-
-#vhcenter[
-  All files-to-block mappings are stored in the metadata engine, including bucket and object metadata. Files below 3KB are stored directly in the metadata engine.
-  #v(0.5cm)
-
-  #mytable(
-    columns: (0.7fr, 1fr, 1.3fr),
-    inset: 0.8em,
-    align: center + horizon,
-    table.header[*Metadata engine*][*Characteristics*][*Use case*],
-    [*SQlite*],[safer],[single node deployment\ small clusters\ clusters with infrequent access],
-    [*LMDB*],[faster\ sometimes has inexplicable corruptions],[larger clusters with metadata redundancy],
-    [*Fjall*],[experimental\ best of both worlds?],[help us test it!],
-  )
-
-  #v(0.5cm)
-  Metadata engine can be set node per-node, and changed later with a migration tool
-]
-
-== Avoiding common issues as soon as possible
-
-#vhcenter[
-  #mytable(
-    columns: (1fr, 1.4fr),
-    inset: 0.8em,
-    align: center + horizon,
-    table.header[*Risk*][*How to avoid*],
-    [*Metadata corruption*\ (esp. with LMDB)],[Configure automatic snapshots with\ `metadata_auto_snapshot_interval`\ Use replication factor 2 or 3],
-    [*Data not well balanced between nodes*],[Avoid clusters with too many nodes\ Target: \#nodes ≤ 10 × replication_factor],
-    [*Performance issues with many objects in one single bucket*],[Spread your data over multiple buckets],
-    [*Performance issues with big objects*],[Increase `block_size` configuration parameter\ Target: object size ≤ 1000 × `block_size`,\ `block_size` ≤ 100MB],
-    [*Performance issues with many small objects*],[Have enough RAM to fit the entire metadata DB],
-  )
-]
-
-== Other things to consider during set-up
-
-#vhcenter[
-  #mytable(
-    columns: (1fr, 1.2fr),
-    inset: 0.8em,
-    align: center + horizon,
-    [*Tools for cluster deployment*],[Ansible + systemd\ NixOS\ Kubernetes or Nomad with Docker],
-    [*Initial cluster setup*],[Manual layout configuration\ Read the documentation!],
-    [*TLS support on public endpoints*],[Add an external reverse-proxy (Nginx, ...)],
-    [*S3 anonymous access*],[Not implemented, use website endpoint],
-    [*Monitoring*],[Prometheus + Grafana for Garage metrics\ External tool to monitor HDD health],
-  )
-]
-
-== Monitoring with Prometheus + Grafana
-
-#imgcenter("../2026-01-31-fosdem/assets/garage-stats.png", width: 83%)
-
-== Common issues and their solutions
-
-#vhcenter[
-  #mytable(
-    columns: (1fr, 1.5fr),
-    inset: 0.8em,
-    align: center + horizon,
-    table.header[*Problem*][*Solution*],
-    table.cell(rowspan: 2)[*S3 access authorization issues*],[Correctly set the `region` parameter in your S3 client\ default = `garage`, not `us-east-1`],[Check your reverse proxy configuration],
-    [*Debugging other API issues*],[Set `RUST_LOG=garage=debug` to investigate],
-    [*Resync queue fills up*],[`garage worker set -a resync-worker-count 8`\ `garage worker set -a resync-tranquility 0`],
-    [*LMDB database too big*],[Stop garage and compact with `mdb_copy -c`],
-    [*Data recovery with dead/unavailable nodes*],[Consistency mode `degraded` allows to read data from an unhealthy cluster. *Do not use it for regular operation.*],
-    [*Other issues*],[Ask us on matrix `#garage:deuxfleurs.fr` or open an issue on `git.deuxfleurs.fr`\
-      Provide the output of `garage status`, `garage stats` and relevant metrics and logs],
-
-  )
-]
-
-== Future developments
-
-#imgcenter("../assets/survey_requested_features.png", width: 80%)
-
-#pagebreak()
-#imgcenter("../2026-01-31-fosdem/assets/Garage Web Admin - Dashboard@2x.png", width: 100%)
-
-#pagebreak()
-#imgcenter("../2026-01-31-fosdem/assets/Garage Web Admin - Bucket details page@2x.png", width: 100%)
-
-== Where to find us
-
-#align(center)[
-  #v(1fr)
-
-  #image("../../logo/garage_hires.png", width: 25%)
-  #link("https://garagehq.deuxfleurs.fr/")\
-  #link("mailto:garagehq@deuxfleurs.fr")\
-  `#garage:deuxfleurs.fr` on Matrix
-
-  #v(1fr)
-  #grid(columns: (6%,3%,13%),
-    image("../assets/logos/rust_logo.png"),
-    [],
-    image("../assets/logos/AGPLv3_Logo.png"),
-  )
-]

fuzz/Cargo.toml

@@ -14,7 +14,7 @@ libfuzzer-sys = { workspace = true }
 garage_db.workspace = true
 garage_table.workspace = true
 garage_util.workspace = true
-garage_model = { workspace = true, default-features = false, features = ["arbitrary"] }
+garage_model = { workspace = true, default-features = false, features = ["arbitrary", "k2v"] }
 
 [[bin]]
 name = "version_crdt"
@@ -64,3 +64,10 @@ path = "fuzz_targets/bucket_alias_crdt.rs"
 test = false
 doc = false
 bench = false
+
+[[bin]]
+name = "k2v_item_crdt"
+path = "fuzz_targets/k2v_item_crdt.rs"
+test = false
+doc = false
+bench = false

fuzz/fuzz_targets/admin_api_token_crdt.rs

@@ -2,13 +2,14 @@
 
 use garage_fuzz::check_crdt_laws;
 use garage_model::admin_token_table::{AdminApiToken, AdminApiTokenParams, AdminApiTokenScope};
+use garage_model::permission::ExpirationTime;
 use garage_util::crdt;
 use libfuzzer_sys::fuzz_target;
 
 type Input = (
 	bool,
 	crdt::Lww<String>,
-	crdt::Lww<Option<u64>>,
+	crdt::Lww<crdt::MergingOption<ExpirationTime>>,
 	crdt::Lww<AdminApiTokenScope>,
 );
 

fuzz/fuzz_targets/k2v_item_crdt.rs

@@ -0,0 +1,36 @@
+#![no_main]
+
+use std::collections::BTreeMap;
+
+use garage_fuzz::check_crdt_laws;
+use garage_model::k2v::item_table::{DvvsEntry, DvvsValue, K2VItem};
+use libfuzzer_sys::fuzz_target;
+
+// Timestamps are encoded as `(ts << 32) | shift` so that items built with different
+// shifts (0, 1, 2) have disjoint timestamp spaces that still interleave in the sorted merge.
+fn make(raw: BTreeMap<u64, (u32, BTreeMap<u32, DvvsValue>)>, shift: u32) -> K2VItem {
+	let shift = shift as u64;
+	let items = raw
+		.into_iter()
+		.map(|(node, (t_discard, values))| {
+			let entry = DvvsEntry::from_raw(
+				(t_discard as u64) << 32 | shift,
+				values
+					.into_iter()
+					.map(|(ts, v)| ((ts as u64) << 32 | shift, v))
+					.collect(),
+			);
+			(node, entry)
+		})
+		.collect();
+	K2VItem::with_raw_items(items)
+}
+
+fuzz_target!(|inputs: (
+	BTreeMap<u64, (u32, BTreeMap<u32, DvvsValue>)>,
+	BTreeMap<u64, (u32, BTreeMap<u32, DvvsValue>)>,
+	BTreeMap<u64, (u32, BTreeMap<u32, DvvsValue>)>,
+)| {
+	let (a, b, c) = inputs;
+	check_crdt_laws(make(a, 0), make(b, 1), make(c, 2));
+});

fuzz/fuzz_targets/key_crdt.rs

@@ -2,7 +2,7 @@
 
 use garage_fuzz::check_crdt_laws;
 use garage_model::key_table::{Key, KeyParams};
-use garage_model::permission::BucketKeyPerm;
+use garage_model::permission::{BucketKeyPerm, ExpirationTime};
 use garage_util::crdt;
 use garage_util::data::Uuid;
 use libfuzzer_sys::fuzz_target;
@@ -10,10 +10,10 @@ use libfuzzer_sys::fuzz_target;
 type Input = (
 	bool,
 	crdt::Lww<String>,
-	crdt::Lww<Option<u64>>,
+	crdt::Lww<crdt::MergingOption<ExpirationTime>>,
 	crdt::Lww<bool>,
 	crdt::Map<Uuid, BucketKeyPerm>,
-	crdt::LwwMap<String, Option<Uuid>>,
+	crdt::LwwMap<String, crdt::CancelingOption<Uuid>>,
 );
 
 fn make(input: Input) -> Key {

script/telemetry/grafana-garage-dashboard-elasticsearch.json

@@ -161,7 +161,7 @@
           },
           "metrics": [
             {
-              "field": "api_request_counter",
+              "field": "garage_api_request_count",
               "hide": true,
               "id": "1",
               "type": "sum"
@@ -284,7 +284,7 @@
           "hide": false,
           "metrics": [
             {
-              "field": "api_request_duration",
+              "field": "garage_api_request_duration",
               "id": "1",
               "type": "avg"
             }
@@ -412,7 +412,7 @@
           },
           "metrics": [
             {
-              "field": "api_error_counter",
+              "field": "garage_api_error_count",
               "hide": true,
               "id": "1",
               "type": "sum"
@@ -540,7 +540,7 @@
           },
           "metrics": [
             {
-              "field": "web_request_counter",
+              "field": "garage_web_request_count",
               "hide": true,
               "id": "1",
               "type": "sum"
@@ -666,7 +666,7 @@
           "hide": false,
           "metrics": [
             {
-              "field": "web_request_duration",
+              "field": "garage_web_request_duration",
               "id": "1",
               "type": "avg"
             }
@@ -794,7 +794,7 @@
           },
           "metrics": [
             {
-              "field": "web_error_counter",
+              "field": "garage_web_error_count",
               "hide": true,
               "id": "1",
               "type": "sum"
@@ -918,7 +918,7 @@
           "hide": false,
           "metrics": [
             {
-              "field": "table_get_request_counter",
+              "field": "garage_table_get_request_count",
               "hide": true,
               "id": "1",
               "type": "sum"
@@ -1042,7 +1042,7 @@
           "hide": false,
           "metrics": [
             {
-              "field": "table_put_request_counter",
+              "field": "garage_table_put_request_count",
               "hide": true,
               "id": "1",
               "type": "sum"
@@ -1154,7 +1154,7 @@
           "hide": false,
           "metrics": [
             {
-              "field": "block_bytes_read",
+              "field": "garage_block_bytes_read",
               "hide": true,
               "id": "1",
               "type": "sum"
@@ -1270,7 +1270,7 @@
           },
           "metrics": [
             {
-              "field": "block_bytes_written",
+              "field": "garage_block_bytes_written",
               "hide": true,
               "id": "1",
               "type": "sum"
@@ -1386,7 +1386,7 @@
           },
           "metrics": [
             {
-              "field": "block_resync_counter",
+              "field": "garage_block_resync_count",
               "hide": true,
               "id": "1",
               "type": "sum"
@@ -1500,7 +1500,7 @@
           "hide": false,
           "metrics": [
             {
-              "field": "block_resync_queue_length",
+              "field": "garage_block_resync_queue_length",
               "id": "1",
               "type": "avg"
             }
@@ -1610,7 +1610,7 @@
           },
           "metrics": [
             {
-              "field": "table_merkle_updater_todo_queue_length",
+              "field": "garage_table_merkle_updater_todo_queue_length",
               "id": "1",
               "type": "avg"
             }
@@ -1724,7 +1724,7 @@
           },
           "metrics": [
             {
-              "field": "table_gc_todo_queue_length",
+              "field": "garage_table_gc_todo_queue_length",
               "id": "1",
               "type": "avg"
             }
@@ -1824,7 +1824,7 @@
           },
           "metrics": [
             {
-              "field": "block_resync_error_counter",
+              "field": "garage_block_resync_error_count",
               "hide": true,
               "id": "1",
               "settings": {},
@@ -1938,7 +1938,7 @@
           },
           "metrics": [
             {
-              "field": "block_resync_errored_blocks",
+              "field": "garage_block_resync_errored_blocks",
               "hide": false,
               "id": "1",
               "type": "sum"
@@ -2041,7 +2041,7 @@
           },
           "metrics": [
             {
-              "field": "block_corruption_counter",
+              "field": "garage_block_corruption_count",
               "hide": true,
               "id": "1",
               "type": "sum"
@@ -2165,7 +2165,7 @@
           },
           "metrics": [
             {
-              "field": "rpc_netapp_error_counter",
+              "field": "garage_rpc_netapp_error_count",
               "hide": true,
               "id": "1",
               "type": "sum"
@@ -2292,7 +2292,7 @@
           },
           "metrics": [
             {
-              "field": "rpc_request_counter",
+              "field": "garage_rpc_request_count",
               "hide": true,
               "id": "1",
               "type": "sum"
@@ -2418,7 +2418,7 @@
           },
           "metrics": [
             {
-              "field": "rpc_duration",
+              "field": "garage_rpc_duration",
               "id": "1",
               "type": "avg"
             }
@@ -2521,7 +2521,7 @@
           },
           "metrics": [
             {
-              "field": "admin_http_requests_total",
+              "field": "garage_admin_http_requests_total",
               "hide": true,
               "id": "1",
               "type": "sum"
@@ -2654,7 +2654,7 @@
           },
           "metrics": [
             {
-              "field": "rpc_garage_error_counter",
+              "field": "garage_rpc_garage_error_count",
               "hide": true,
               "id": "1",
               "type": "sum"
@@ -2765,7 +2765,7 @@
           },
           "metrics": [
             {
-              "field": "rpc_duration",
+              "field": "garage_rpc_duration",
               "id": "1",
               "type": "avg"
             }

script/telemetry/grafana-garage-dashboard-prometheus.json

@@ -143,7 +143,7 @@
             "uid": "${DS_DS_PROMETHEUS}"
           },
           "exemplar": true,
-          "expr": "sum(rate(block_bytes_read{job=\"garage\"}[$__rate_interval])    )",
+          "expr": "sum(rate(garage_block_bytes_read{job=\"garage\"}[$__rate_interval])    )",
           "hide": false,
           "interval": "",
           "legendFormat": "Disk bytes read",
@@ -155,7 +155,7 @@
             "uid": "${DS_DS_PROMETHEUS}"
           },
           "exemplar": true,
-          "expr": "-sum(rate(block_bytes_written{job=\"garage\"}[$__rate_interval])    )",
+          "expr": "-sum(rate(garage_block_bytes_written{job=\"garage\"}[$__rate_interval])    )",
           "hide": false,
           "interval": "",
           "legendFormat": "Disk bytes written",
@@ -250,7 +250,7 @@
           },
           "editorMode": "code",
           "exemplar": true,
-          "expr": "sum by (api_endpoint) (rate(api_s3_request_counter {job=\"garage\"}[$__rate_interval]))",
+          "expr": "sum by (api_endpoint) (rate(garage_api_s3_request_count {job=\"garage\"}[$__rate_interval]))",
           "hide": false,
           "interval": "",
           "legendFormat": "{{api_endpoint}}",
@@ -345,7 +345,7 @@
             "uid": "${DS_DS_PROMETHEUS}"
           },
           "exemplar": true,
-          "expr": "sum(rate(web_request_counter {job=\"garage\"}[$__rate_interval]))",
+          "expr": "sum(rate(garage_web_request_count {job=\"garage\"}[$__rate_interval]))",
           "hide": false,
           "interval": "",
           "legendFormat": "Web request rate",
@@ -439,7 +439,7 @@
             "uid": "${DS_DS_PROMETHEUS}"
           },
           "exemplar": true,
-          "expr": "sum by (rpc_endpoint) (rate(rpc_request_counter {job=\"garage\"}[$__rate_interval]))",
+          "expr": "sum by (rpc_endpoint) (rate(garage_rpc_request_count {job=\"garage\"}[$__rate_interval]))",
           "hide": false,
           "interval": "",
           "legendFormat": "{{rpc_endpoint}}",
@@ -534,7 +534,7 @@
           },
           "editorMode": "code",
           "exemplar": true,
-          "expr": "sum by (api_endpoint, status_code) (rate(api_s3_error_counter {job=\"garage\"}[$__rate_interval]))",
+          "expr": "sum by (api_endpoint, status_code) (rate(garage_api_s3_error_count {job=\"garage\"}[$__rate_interval]))",
           "hide": false,
           "interval": "",
           "legendFormat": "{{api_endpoint}} {{status_code}}",
@@ -629,7 +629,7 @@
             "uid": "${DS_DS_PROMETHEUS}"
           },
           "exemplar": true,
-          "expr": "sum by(status_code) (rate(web_error_counter {job=\"garage\"}[$__rate_interval]))",
+          "expr": "sum by(status_code) (rate(garage_web_error_count {job=\"garage\"}[$__rate_interval]))",
           "hide": false,
           "interval": "",
           "legendFormat": "{{status_code}}",
@@ -722,7 +722,7 @@
             "uid": "${DS_DS_PROMETHEUS}"
           },
           "exemplar": true,
-          "expr": "block_resync_queue_length{job=\"garage\"}",
+          "expr": "garage_block_resync_queue_length{job=\"garage\"}",
           "interval": "",
           "legendFormat": "{{instance}}",
           "refId": "A"
@@ -814,7 +814,7 @@
             "uid": "${DS_DS_PROMETHEUS}"
           },
           "exemplar": true,
-          "expr": "sum by(table_name) (table_gc_todo_queue_length{job=\"garage\"})",
+          "expr": "sum by(table_name) (garage_table_gc_todo_queue_length{job=\"garage\"})",
           "interval": "",
           "legendFormat": "{{ table_name}}",
           "refId": "A"
@@ -906,7 +906,7 @@
             "uid": "${DS_DS_PROMETHEUS}"
           },
           "exemplar": true,
-          "expr": "sum by(table_name) (table_merkle_updater_todo_queue_length{job=\"garage\"})",
+          "expr": "sum by(table_name) (garage_table_merkle_updater_todo_queue_length{job=\"garage\"})",
           "interval": "",
           "legendFormat": "{{ table_name}}",
           "refId": "A"
@@ -998,7 +998,7 @@
             "uid": "${DS_DS_PROMETHEUS}"
           },
           "exemplar": true,
-          "expr": "block_resync_errored_blocks{job=\"garage\"}",
+          "expr": "garage_block_resync_errored_blocks{job=\"garage\"}",
           "interval": "",
           "legendFormat": "{{instance}}",
           "refId": "A"

src/api/admin/admin_token.rs

@@ -7,6 +7,7 @@ use garage_util::time::now_msec;
 
 use garage_model::admin_token_table::*;
 use garage_model::garage::Garage;
+use garage_model::permission::ExpirationTime;
 
 use crate::api::*;
 use crate::error::*;
@@ -244,8 +245,8 @@ fn admin_token_info_results(token: &AdminApiToken, now: u64) -> GetAdminTokenInf
 				.expect("invalid timestamp stored in db"),
 		),
 		name: params.name.get().to_string(),
-		expiration: params.expiration.get().map(|x| {
+		expiration: params.expiration.get().inner().map(|x| {
-			DateTime::from_timestamp_millis(x as i64).expect("invalid timestamp stored in db")
+			DateTime::from_timestamp_millis(x.0 as i64).expect("invalid timestamp stored in db")
 		}),
 		expired: params.is_expired(now),
 		scope: params.scope.get().0.clone(),
@@ -279,10 +280,10 @@ fn apply_token_updates(
 	if let Some(expiration) = updates.expiration {
 		params
 			.expiration
-			.update(Some(expiration.timestamp_millis() as u64));
+			.update(Some(ExpirationTime(expiration.timestamp_millis() as u64)).into());
 	}
 	if updates.never_expires {
-		params.expiration.update(None);
+		params.expiration.update(None.into());
 	}
 	if let Some(scope) = updates.scope {
 		params.scope.update(AdminApiTokenScope(scope));

src/api/admin/bucket.rs

@@ -90,7 +90,7 @@ impl RequestHandler for GetBucketInfoRequest {
 				.bucket_alias_table
 				.get(&EmptyKey, &ga)
 				.await?
-				.and_then(|x| *x.state.get())
+				.and_then(|x| x.state.get().into_inner())
 				.ok_or_else(|| HelperError::NoSuchBucket(ga.to_string()))?,
 			(None, None, Some(search)) => {
 				let helper = garage.bucket_helper();
@@ -168,7 +168,7 @@ impl RequestHandler for CreateBucketRequest {
 			}
 
 			if let Some(alias) = garage.bucket_alias_table.get(&EmptyKey, ga).await? {
-				if alias.state.get().is_some() {
+				if alias.state.get().inner().is_some() {
 					return Err(CommonError::BucketAlreadyExists.into());
 				}
 			}
@@ -297,7 +297,7 @@ impl RequestHandler for UpdateBucketRequest {
 				let redirect_all = state
 					.website_config
 					.get()
-					.as_ref()
+					.inner()
 					.and_then(|wc| wc.redirect_all.clone());
 
 				let routing_rules = if let Some(rr) = wa.routing_rules {
@@ -311,26 +311,29 @@ impl RequestHandler for UpdateBucketRequest {
 					state
 						.website_config
 						.get()
-						.as_ref()
+						.inner()
 						.map(|wc| wc.routing_rules.clone())
 						.unwrap_or_default()
 				};
 
-				state.website_config.update(Some(WebsiteConfig {
+				state.website_config.update(
-					index_document: wa.index_document.ok_or_bad_request(
+					Some(WebsiteConfig {
-						"Please specify indexDocument when enabling website access.",
+						index_document: wa.index_document.ok_or_bad_request(
-					)?,
+							"Please specify indexDocument when enabling website access.",
-					error_document: wa.error_document,
+						)?,
-					redirect_all,
+						error_document: wa.error_document,
-					routing_rules,
+						redirect_all,
-				}));
+						routing_rules,
+					})
+					.into(),
+				);
 			} else {
 				if wa.index_document.is_some() || wa.error_document.is_some() {
 					return Err(Error::bad_request(
                         "Cannot specify indexDocument or errorDocument when disabling website access.",
                     ));
 				}
-				state.website_config.update(None);
+				state.website_config.update(None.into());
 			}
 		}
 
@@ -353,7 +356,7 @@ impl RequestHandler for UpdateBucketRequest {
 				Some(cc.into_garage_cors_config()?)
 			};
 
-			state.cors_config.update(cors_config);
+			state.cors_config.update(cors_config.into());
 		}
 
 		if let Some(lr) = self.body.lifecycle_rules {
@@ -370,7 +373,7 @@ impl RequestHandler for UpdateBucketRequest {
 				)
 			};
 
-			state.lifecycle_config.update(lifecycle_config);
+			state.lifecycle_config.update(lifecycle_config.into());
 		}
 
 		garage.bucket_table.insert(&bucket).await?;
@@ -739,8 +742,8 @@ async fn bucket_info_results(
 			.filter(|(_, _, a)| *a)
 			.map(|(n, _, _)| n.to_string())
 			.collect::<Vec<_>>(),
-		website_access: state.website_config.get().is_some(),
+		website_access: state.website_config.get().inner().is_some(),
-		website_config: state.website_config.get().clone().map(|wsc| {
+		website_config: state.website_config.get().inner().cloned().map(|wsc| {
 			GetBucketInfoWebsiteResponse {
 				index_document: wsc.index_document,
 				error_document: wsc.error_document,
@@ -752,13 +755,13 @@ async fn bucket_info_results(
 				),
 			}
 		}),
-		cors_rules: state.cors_config.get().as_ref().map(|rules| {
+		cors_rules: state.cors_config.get().inner().map(|rules| {
 			rules
 				.iter()
 				.map(xml::cors::CorsRule::from_garage_cors_rule)
 				.collect::<Vec<_>>()
 		}),
-		lifecycle_rules: state.lifecycle_config.get().as_ref().map(|lc| {
+		lifecycle_rules: state.lifecycle_config.get().inner().map(|lc| {
 			lc.iter()
 				.map(xml::lifecycle::LifecycleRule::from_garage_lifecycle_rule)
 				.collect::<Vec<_>>()
@@ -784,7 +787,7 @@ async fn bucket_info_results(
 						.local_aliases
 						.items()
 						.iter()
-						.filter(|(_, _, b)| *b == Some(bucket.id))
+						.filter(|(_, _, b)| b.into_inner() == Some(bucket.id))
 						.map(|(n, _, _)| n.to_string())
 						.collect::<Vec<_>>(),
 				})

src/api/admin/key.rs

@@ -8,6 +8,7 @@ use garage_util::time::now_msec;
 
 use garage_model::garage::Garage;
 use garage_model::key_table::*;
+use garage_model::permission::ExpirationTime;
 
 use crate::api::*;
 use crate::error::*;
@@ -40,8 +41,8 @@ impl RequestHandler for ListKeysRequest {
 						DateTime::from_timestamp_millis(x as i64)
 							.expect("invalid timestamp stored in db")
 					}),
-					expiration: p.expiration.get().map(|x| {
+					expiration: p.expiration.get().inner().map(|x| {
-						DateTime::from_timestamp_millis(x as i64)
+						DateTime::from_timestamp_millis(x.0 as i64)
 							.expect("invalid timestamp stored in db")
 					}),
 					expired: p.is_expired(now),
@@ -201,7 +202,7 @@ async fn key_info_results(
 				.local_aliases
 				.items()
 				.iter()
-				.filter_map(|(_, _, v)| v.as_ref()),
+				.filter_map(|(_, _, v)| v.inner()),
 		) {
 		if !relevant_buckets.contains_key(id) {
 			if let Some(b) = garage.bucket_table.get(&EmptyKey, id).await? {
@@ -217,8 +218,8 @@ async fn key_info_results(
 		created: key_state.created.map(|x| {
 			DateTime::from_timestamp_millis(x as i64).expect("invalid timestamp stored in db")
 		}),
-		expiration: key_state.expiration.get().map(|x| {
+		expiration: key_state.expiration.get().inner().map(|x| {
-			DateTime::from_timestamp_millis(x as i64).expect("invalid timestamp stored in db")
+			DateTime::from_timestamp_millis(x.0 as i64).expect("invalid timestamp stored in db")
 		}),
 		expired: key_state.is_expired(now_msec()),
 		access_key_id: key.key_id.clone(),
@@ -283,10 +284,10 @@ fn apply_key_updates(key: &mut Key, updates: UpdateKeyRequestBody) -> Result<(),
 	if let Some(expiration) = updates.expiration {
 		key_state
 			.expiration
-			.update(Some(expiration.timestamp_millis() as u64));
+			.update(Some(ExpirationTime(expiration.timestamp_millis() as u64)).into());
 	}
 	if updates.never_expires {
-		key_state.expiration.update(None);
+		key_state.expiration.update(None.into());
 	}
 	if let Some(allow) = updates.allow {
 		if allow.create_bucket {

src/api/admin/special.rs

@@ -164,7 +164,7 @@ async fn check_domain(garage: &Arc<Garage>, domain: &str) -> Result<bool, Error>
 	}
 
 	let bucket_state = bucket.state.as_option().unwrap();
-	let bucket_website_config = bucket_state.website_config.get();
+	let bucket_website_config = bucket_state.website_config.get().inner();
 
 	match bucket_website_config {
 		Some(_v) => Ok(true),

src/api/common/cors.rs

@@ -19,7 +19,7 @@ pub fn find_matching_cors_rule<'a, B>(
 	bucket_params: &'a BucketParams,
 	req: &'a Request<B>,
 ) -> Result<Option<(&'a GarageCorsRule, &'a str)>, CommonError> {
-	if let Some(cors_config) = bucket_params.cors_config.get() {
+	if let Some(cors_config) = bucket_params.cors_config.get().inner() {
 		if let Some(origin) = req.headers().get("Origin") {
 			let origin = origin.to_str()?;
 			let request_headers = match req.headers().get(ACCESS_CONTROL_REQUEST_HEADERS) {
@@ -158,7 +158,7 @@ pub fn handle_options_for_bucket<B>(
 		None => vec![],
 	};
 
-	if let Some(cors_config) = bucket_params.cors_config.get() {
+	if let Some(cors_config) = bucket_params.cors_config.get().inner() {
 		let matching_rule = cors_config
 			.iter()
 			.find(|rule| cors_rule_matches(rule, origin, request_method, request_headers.iter()));
@@ -192,14 +192,17 @@ mod tests {
 
 	fn bucket_params_with_rule(allow_origins: Vec<&str>) -> BucketParams {
 		let mut bucket_params = BucketParams::default();
-		bucket_params.cors_config.update(Some(vec![GarageCorsRule {
+		bucket_params.cors_config.update(
-			id: Some("cors-test".into()),
+			Some(vec![GarageCorsRule {
-			max_age_seconds: None,
+				id: Some("cors-test".into()),
-			allow_origins: allow_origins.into_iter().map(str::to_string).collect(),
+				max_age_seconds: None,
-			allow_methods: vec!["GET".into(), "PUT".into()],
+				allow_origins: allow_origins.into_iter().map(str::to_string).collect(),
-			allow_headers: vec!["*".into()],
+				allow_methods: vec!["GET".into(), "PUT".into()],
-			expose_headers: vec![],
+				allow_headers: vec!["*".into()],
-		}]));
+				expose_headers: vec![],
+			}])
+			.into(),
+		);
 		bucket_params
 	}
 

src/api/common/generic_server.rs

@@ -84,21 +84,21 @@ impl<A: ApiHandler> ApiServer<A> {
 			region,
 			api_handler,
 			request_counter: meter
-				.u64_counter(format!("api.{}.request_counter", A::API_NAME))
+				.u64_counter(format!("garage_api.{}.request_count", A::API_NAME))
 				.with_description(format!(
 					"Number of API calls to the various {} API endpoints",
 					A::API_NAME_DISPLAY
 				))
 				.init(),
 			error_counter: meter
-				.u64_counter(format!("api.{}.error_counter", A::API_NAME))
+				.u64_counter(format!("garage_api.{}.error_count", A::API_NAME))
 				.with_description(format!(
 					"Number of API calls to the various {} API endpoints that resulted in errors",
 					A::API_NAME_DISPLAY
 				))
 				.init(),
 			request_duration: meter
-				.f64_value_recorder(format!("api.{}.request_duration", A::API_NAME))
+				.f64_value_recorder(format!("garage_api.{}.request_duration", A::API_NAME))
 				.with_description(format!(
 					"Duration of API calls to the various {} API endpoints",
 					A::API_NAME_DISPLAY

src/api/s3/bucket.rs

@@ -122,7 +122,7 @@ pub async fn handle_list_buckets(
 			for (alias, _, _active) in bucket.aliases().iter().filter(|(_, _, active)| *active) {
 				let alias_opt = garage.bucket_alias_table.get(&EmptyKey, alias).await?;
 				if let Some(alias_ent) = alias_opt {
-					if *alias_ent.state.get() == Some(*bucket_id) {
+					if alias_ent.state.get().inner() == Some(bucket_id) {
 						aliases.insert(alias_ent.name().to_string(), *bucket_id);
 					}
 				}
@@ -134,7 +134,7 @@ pub async fn handle_list_buckets(
 	}
 
 	for (alias, _, id_opt) in key_p.local_aliases.items() {
-		if let Some(id) = id_opt {
+		if let Some(id) = id_opt.inner() {
 			aliases.insert(alias.clone(), *id);
 		}
 	}
@@ -256,7 +256,10 @@ pub async fn handle_delete_bucket(ctx: ReqCtx) -> Result<Response<ResBody>, Erro
 
 	let key_params = api_key.params().unwrap();
 
-	let is_local_alias = matches!(key_params.local_aliases.get(bucket_name), Some(Some(_)));
+	let is_local_alias = matches!(
+		key_params.local_aliases.get(bucket_name).map(|x| x.inner()),
+		Some(Some(_))
+	);
 
 	// If the bucket has no other aliases, this is a true deletion.
 	// Otherwise, it is just an alias removal.

src/api/s3/cors.rs

@@ -13,7 +13,7 @@ use crate::xml::to_xml_with_header;
 
 pub async fn handle_get_cors(ctx: ReqCtx) -> Result<Response<ResBody>, Error> {
 	let ReqCtx { bucket_params, .. } = ctx;
-	if let Some(cors) = bucket_params.cors_config.get() {
+	if let Some(cors) = bucket_params.cors_config.get().inner() {
 		let wc = CorsConfiguration {
 			xmlns: (),
 			cors_rules: cors
@@ -38,7 +38,7 @@ pub async fn handle_delete_cors(ctx: ReqCtx) -> Result<Response<ResBody>, Error>
 		mut bucket_params,
 		..
 	} = ctx;
-	bucket_params.cors_config.update(None);
+	bucket_params.cors_config.update(None.into());
 	garage
 		.bucket_table
 		.insert(&Bucket::present(bucket_id, bucket_params))
@@ -67,7 +67,7 @@ pub async fn handle_put_cors(
 
 	bucket_params
 		.cors_config
-		.update(Some(conf.into_garage_cors_config()?));
+		.update(Some(conf.into_garage_cors_config()?).into());
 	garage
 		.bucket_table
 		.insert(&Bucket::present(bucket_id, bucket_params))

src/api/s3/lifecycle.rs

@@ -14,7 +14,7 @@ use garage_model::bucket_table::Bucket;
 pub async fn handle_get_lifecycle(ctx: ReqCtx) -> Result<Response<ResBody>, Error> {
 	let ReqCtx { bucket_params, .. } = ctx;
 
-	if let Some(lifecycle) = bucket_params.lifecycle_config.get() {
+	if let Some(lifecycle) = bucket_params.lifecycle_config.get().inner() {
 		let wc = LifecycleConfiguration::from_garage_lifecycle_config(lifecycle);
 		let xml = to_xml_with_header(&wc)?;
 		Ok(Response::builder()
@@ -33,7 +33,7 @@ pub async fn handle_delete_lifecycle(ctx: ReqCtx) -> Result<Response<ResBody>, E
 		mut bucket_params,
 		..
 	} = ctx;
-	bucket_params.lifecycle_config.update(None);
+	bucket_params.lifecycle_config.update(None.into());
 	garage
 		.bucket_table
 		.insert(&Bucket::present(bucket_id, bucket_params))
@@ -62,7 +62,7 @@ pub async fn handle_put_lifecycle(
 		.validate_into_garage_lifecycle_config()
 		.ok_or_bad_request("Invalid lifecycle configuration")?;
 
-	bucket_params.lifecycle_config.update(Some(config));
+	bucket_params.lifecycle_config.update(Some(config).into());
 	garage
 		.bucket_table
 		.insert(&Bucket::present(bucket_id, bucket_params))

src/api/s3/router.rs

@@ -315,7 +315,11 @@ impl Endpoint {
 		bucket: Option<String>,
 	) -> Result<(Self, Option<String>), Error> {
 		let uri = req.uri();
-		let path = uri.path().trim_start_matches('/');
+		let path = uri.path().strip_prefix('/');
+		if path.is_none() {
+			return Err(Error::bad_request("URI path must start with a '/'"));
+		}
+		let path = path.unwrap();
 		let query = uri.query();
 		if bucket.is_none() && path.is_empty() {
 			if *req.method() == Method::OPTIONS {
@@ -329,7 +333,7 @@ impl Endpoint {
 			(bucket, path)
 		} else {
 			path.split_once('/')
-				.map(|(b, p)| (b.to_owned(), p.trim_start_matches('/')))
+				.map(|(b, p)| (b.to_owned(), p))
 				.unwrap_or_else(|| (path.to_owned(), ""))
 		};
 
@@ -843,6 +847,40 @@ mod tests {
 			"&+?%é/something"
 		);
 
+		// A double-slash in the URL means the key begins with '/'.
+		// path-style: HEAD /bucket// → key "/"
+		assert_eq!(
+			parse("HEAD", "/my_bucket//", None, None)
+				.0
+				.get_key()
+				.unwrap(),
+			"/"
+		);
+		// virtual-hosted-style: HEAD // → key "/"
+		assert_eq!(
+			parse("HEAD", "//", Some("my_bucket".to_owned()), None)
+				.0
+				.get_key()
+				.unwrap(),
+			"/"
+		);
+		// same for GET: path-style GET /bucket// → key "/"
+		assert_eq!(
+			parse("GET", "/my_bucket//", None, None)
+				.0
+				.get_key()
+				.unwrap(),
+			"/"
+		);
+		// virtual-hosted-style: GET // → key "/"
+		assert_eq!(
+			parse("GET", "//", Some("my_bucket".to_owned()), None)
+				.0
+				.get_key()
+				.unwrap(),
+			"/"
+		);
+
 		/*
 		 * this case is failing. We should verify how clients encode space in url
 		assert_eq!(
@@ -933,6 +971,7 @@ mod tests {
 			GET "/{Key+}?torrent" => GetObjectTorrent
 			GET "/?publicAccessBlock" => GetPublicAccessBlock
 			HEAD "/" => HeadBucket
+			HEAD "//" => HeadObject
 			HEAD "/my-image.jpg" => HeadObject
 			HEAD "/my-image.jpg?versionId=3HL4kqCxf3vjVBH40Nrjfkd" => HeadObject
 			HEAD "/Key+?partNumber=3&versionId=VersionId" => HeadObject
@@ -949,6 +988,7 @@ mod tests {
 			GET "/?uploads&delimiter=/&prefix=photos/2006/" => ListMultipartUploads
 			GET "/?uploads&delimiter=D&encoding-type=EncodingType&key-marker=KeyMarker&max-uploads=1&prefix=Prefix&upload-id-marker=UploadIdMarker" => ListMultipartUploads
 			GET "/" => ListObjects
+			GET "//" => GetObject
 			GET "/?prefix=N&marker=Need&max-keys=40" => ListObjects
 			GET "/?delimiter=/" => ListObjects
 			GET "/?prefix=photos/2006/&delimiter=/" => ListObjects

src/api/s3/website.rs

@@ -16,7 +16,7 @@ pub const X_AMZ_WEBSITE_REDIRECT_LOCATION: HeaderName =
 
 pub async fn handle_get_website(ctx: ReqCtx) -> Result<Response<ResBody>, Error> {
 	let ReqCtx { bucket_params, .. } = ctx;
-	if let Some(website) = bucket_params.website_config.get() {
+	if let Some(website) = bucket_params.website_config.get().inner() {
 		let wc = WebsiteConfiguration {
 			xmlns: (),
 			error_document: website.error_document.as_ref().map(|v| Key {
@@ -54,7 +54,7 @@ pub async fn handle_delete_website(ctx: ReqCtx) -> Result<Response<ResBody>, Err
 		mut bucket_params,
 		..
 	} = ctx;
-	bucket_params.website_config.update(None);
+	bucket_params.website_config.update(None.into());
 	garage
 		.bucket_table
 		.insert(&Bucket::present(bucket_id, bucket_params))
@@ -83,7 +83,7 @@ pub async fn handle_put_website(
 
 	bucket_params
 		.website_config
-		.update(Some(conf.into_garage_website_config()?));
+		.update(Some(conf.into_garage_website_config()?).into());
 	garage
 		.bucket_table
 		.insert(&Bucket::present(bucket_id, bucket_params))

src/block/metrics.rs

@@ -41,7 +41,7 @@ impl BlockManagerMetrics {
 		let meter = global::meter("garage_model/block");
 		Self {
 			_compression_level: meter
-				.u64_value_observer("block.compression_level", move |observer| {
+				.u64_value_observer("garage_block.compression_level", move |observer| {
 					match compression_level {
 						Some(v) => observer.observe(v as u64, &[]),
 						None => observer.observe(0_u64, &[]),
@@ -50,7 +50,7 @@ impl BlockManagerMetrics {
 				.with_description("Garage compression level for node")
 				.init(),
 			_rc_size: meter
-				.u64_value_observer("block.rc_size", move |observer| {
+				.u64_value_observer("garage_block.rc_size", move |observer| {
 					if let Ok(value) = rc_tree.approximate_len() {
 						observer.observe(value as u64, &[]);
 					}
@@ -58,7 +58,7 @@ impl BlockManagerMetrics {
 				.with_description("Number of blocks known to the reference counter")
 				.init(),
 			_resync_queue_len: meter
-				.u64_value_observer("block.resync_queue_length", move |observer| {
+				.u64_value_observer("garage_block.resync_queue_length", move |observer| {
 					if let Ok(value) = resync_queue.approximate_len() {
 						observer.observe(value as u64, &[]);
 					}
@@ -68,7 +68,7 @@ impl BlockManagerMetrics {
 				)
 				.init(),
 			_resync_errored_blocks: meter
-				.u64_value_observer("block.resync_errored_blocks", move |observer| {
+				.u64_value_observer("garage_block.resync_errored_blocks", move |observer| {
 					if let Ok(value) = resync_errors.approximate_len() {
 						observer.observe(value as u64, &[]);
 					}
@@ -77,7 +77,7 @@ impl BlockManagerMetrics {
 				.init(),
 
 			_buffer_free_kb: meter
-				.u64_value_observer("block.ram_buffer_free_kb", move |observer| {
+				.u64_value_observer("garage_block.ram_buffer_free_kb", move |observer| {
 					observer.observe(buffer_semaphore.available_permits() as u64, &[]);
 				})
 				.with_description(
@@ -86,63 +86,63 @@ impl BlockManagerMetrics {
 				.init(),
 
 			resync_counter: meter
-				.u64_counter("block.resync_counter")
+				.u64_counter("garage_block.resync_count")
 				.with_description("Number of calls to resync_block")
 				.init()
 				.bind(&[]),
 			resync_error_counter: meter
-				.u64_counter("block.resync_error_counter")
+				.u64_counter("garage_block.resync_error_count")
 				.with_description("Number of calls to resync_block that returned an error")
 				.init()
 				.bind(&[]),
 			resync_duration: meter
-				.f64_value_recorder("block.resync_duration")
+				.f64_value_recorder("garage_block.resync_duration")
 				.with_description("Duration of resync_block operations")
 				.init()
 				.bind(&[]),
 			resync_send_counter: meter
-				.u64_counter("block.resync_send_counter")
+				.u64_counter("garage_block.resync_send_count")
 				.with_description("Number of blocks sent to another node in resync operations")
 				.init(),
 			resync_recv_counter: meter
-				.u64_counter("block.resync_recv_counter")
+				.u64_counter("garage_block.resync_recv_count")
 				.with_description("Number of blocks received from other nodes in resync operations")
 				.init()
 				.bind(&[]),
 
 			bytes_read: meter
-				.u64_counter("block.bytes_read")
+				.u64_counter("garage_block.bytes_read")
 				.with_description("Number of bytes read from disk")
 				.init()
 				.bind(&[]),
 			block_read_duration: meter
-				.f64_value_recorder("block.read_duration")
+				.f64_value_recorder("garage_block.read_duration")
 				.with_description("Duration of block read operations")
 				.init()
 				.bind(&[]),
 			block_read_semaphore_timeouts: meter
-				.u64_counter("block.read_semaphore_timeouts")
+				.u64_counter("garage_block.read_semaphore_timeouts")
 				.with_description("Number of block reads that failed due to semaphore acquire timeout")
 				.init()
 				.bind(&[]),
 			bytes_written: meter
-				.u64_counter("block.bytes_written")
+				.u64_counter("garage_block.bytes_written")
 				.with_description("Number of bytes written to disk")
 				.init()
 				.bind(&[]),
 			block_write_duration: meter
-				.f64_value_recorder("block.write_duration")
+				.f64_value_recorder("garage_block.write_duration")
 				.with_description("Duration of block write operations")
 				.init()
 				.bind(&[]),
 			delete_counter: meter
-				.u64_counter("block.delete_counter")
+				.u64_counter("garage_block.delete_count")
 				.with_description("Number of blocks deleted")
 				.init()
 				.bind(&[]),
 
 			corruption_counter: meter
-				.u64_counter("block.corruption_counter")
+				.u64_counter("garage_block.corruption_count")
 				.with_description("Data corruptions detected on block reads")
 				.init()
 				.bind(&[]),

src/garage/tests/common/garage.rs

@@ -194,6 +194,15 @@ api_bind_addr = "127.0.0.1:{admin_port}"
 			.expect("Could not build garage endpoint URI")
 	}
 
+	pub fn admin_uri(&self, path: &str) -> http::Uri {
+		format!(
+			"http://127.0.0.1:{admin_port}/{path}",
+			admin_port = self.admin_port,
+		)
+		.parse()
+		.expect("Could not build garage endpoint URI")
+	}
+
 	pub fn key(&self, maybe_name: Option<&str>) -> Key {
 		let mut key = Key::default();
 

src/garage/tests/lib.rs

@@ -4,6 +4,9 @@ mod common;
 mod admin;
 mod bucket;
 
+#[cfg(feature = "metrics")]
+mod metrics;
+
 mod s3;
 
 #[cfg(feature = "k2v")]

src/garage/tests/metrics.rs

@@ -0,0 +1,49 @@
+use bytes::Bytes;
+use http::{Request, StatusCode};
+use http_body_util::{BodyExt, Full};
+
+use crate::common;
+
+#[tokio::test]
+async fn check_metrics_name() {
+	let ctx = common::context();
+
+	let req_url = ctx.garage.admin_uri("metrics");
+	let client = ctx.custom_request.client();
+	let get_metrics_req = Request::builder()
+		.method("GET")
+		.uri(req_url)
+		.body(Full::new(Bytes::new()))
+		.unwrap();
+
+	let response = client
+		.request(get_metrics_req)
+		.await
+		.expect("failed to build 'get metrics' request");
+
+	assert_eq!(response.status(), StatusCode::OK);
+	let body = BodyExt::collect(response.into_body())
+		.await
+		.expect("failed to collect bytes from body stream")
+		.to_bytes();
+	let body = String::from_utf8_lossy(&body);
+
+	//dbg!(&body);
+	let invalid_metrics_name = body
+		.lines()
+		.filter(isnot_comment_line) // skip the comment lines
+		.filter(hasnt_prefix_garage)
+		.collect::<Vec<_>>();
+
+	if !invalid_metrics_name.is_empty() {
+		panic!("metrics name should all start with 'garage_' prefix.\nDoc: https://prometheus.io/docs/practices/naming/#metric-names\n\nInvalid:\n{:#?}", invalid_metrics_name);
+	}
+}
+
+fn isnot_comment_line(line: &&str) -> bool {
+	!line.starts_with("#")
+}
+
+fn hasnt_prefix_garage(line: &&str) -> bool {
+	!line.starts_with("garage_")
+}

src/model/admin_token_table.rs

@@ -8,6 +8,7 @@ use garage_table::{EmptyKey, Entry, TableSchema};
 pub use crate::key_table::KeyFilter;
 
 mod v2 {
+	use crate::permission::ExpirationTime;
 	use garage_util::crdt;
 	use serde::{Deserialize, Serialize};
 
@@ -35,7 +36,7 @@ mod v2 {
 		pub name: crdt::Lww<String>,
 
 		/// The optional time of expiration of the token
-		pub expiration: crdt::Lww<Option<u64>>,
+		pub expiration: crdt::Lww<crdt::MergingOption<ExpirationTime>>,
 
 		/// The scope of the token, i.e. list of authorized admin API calls
 		pub scope: crdt::Lww<AdminApiTokenScope>,
@@ -106,7 +107,7 @@ impl AdminApiToken {
 				created: now_msec(),
 				token_hash: hashed_token,
 				name: crdt::Lww::new(name.to_string()),
-				expiration: crdt::Lww::new(None),
+				expiration: crdt::Lww::new(None.into()),
 				scope: crdt::Lww::new(AdminApiTokenScope(vec!["*".to_string()])),
 			}),
 		};
@@ -147,9 +148,9 @@ impl AdminApiToken {
 
 impl AdminApiTokenParams {
 	pub fn is_expired(&self, ts_now: u64) -> bool {
-		match *self.expiration.get() {
+		match self.expiration.get().inner() {
 			None => false,
-			Some(exp) => ts_now >= exp,
+			Some(exp) => ts_now >= exp.0,
 		}
 	}
 

src/model/bucket_alias_table.rs

@@ -13,7 +13,7 @@ mod v08 {
 	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
 	pub struct BucketAlias {
 		pub(super) name: String,
-		pub state: crdt::Lww<Option<Uuid>>,
+		pub state: crdt::Lww<crdt::CancelingOption<Uuid>>,
 	}
 
 	impl garage_util::migrate::InitialFormat for BucketAlias {}
@@ -25,12 +25,12 @@ impl BucketAlias {
 	pub fn new(name: String, ts: u64, bucket_id: Option<Uuid>) -> Self {
 		BucketAlias {
 			name,
-			state: crdt::Lww::raw(ts, bucket_id),
+			state: crdt::Lww::raw(ts, CancelingOption(bucket_id)),
 		}
 	}
 
 	pub fn is_deleted(&self) -> bool {
-		self.state.get().is_none()
+		self.state.get().inner().is_none()
 	}
 	pub fn name(&self) -> &str {
 		&self.name

src/model/bucket_table.rs

@@ -45,12 +45,12 @@ mod v08 {
 		/// Whether this bucket is allowed for website access
 		/// (under all of its global alias names),
 		/// and if so, the website configuration XML document
-		pub website_config: crdt::Lww<Option<WebsiteConfig>>,
+		pub website_config: crdt::Lww<crdt::CancelingOption<WebsiteConfig>>,
 		/// CORS rules
-		pub cors_config: crdt::Lww<Option<Vec<CorsRule>>>,
+		pub cors_config: crdt::Lww<crdt::CancelingOption<Vec<CorsRule>>>,
 		/// Lifecycle configuration
 		#[serde(default)]
-		pub lifecycle_config: crdt::Lww<Option<Vec<LifecycleRule>>>,
+		pub lifecycle_config: crdt::Lww<crdt::CancelingOption<Vec<LifecycleRule>>>,
 		/// Bucket quotas
 		#[serde(default)]
 		pub quotas: crdt::Lww<BucketQuotas>,
@@ -164,11 +164,11 @@ mod v2 {
 		/// Whether this bucket is allowed for website access
 		/// (under all of its global alias names),
 		/// and if so, the website configuration XML document
-		pub website_config: crdt::Lww<Option<WebsiteConfig>>,
+		pub website_config: crdt::Lww<crdt::CancelingOption<WebsiteConfig>>,
 		/// CORS rules
-		pub cors_config: crdt::Lww<Option<Vec<CorsRule>>>,
+		pub cors_config: crdt::Lww<crdt::CancelingOption<Vec<CorsRule>>>,
 		/// Lifecycle configuration
-		pub lifecycle_config: crdt::Lww<Option<Vec<LifecycleRule>>>,
+		pub lifecycle_config: crdt::Lww<crdt::CancelingOption<Vec<LifecycleRule>>>,
 		/// Bucket quotas
 		pub quotas: crdt::Lww<BucketQuotas>,
 	}
@@ -259,9 +259,9 @@ impl BucketParams {
 			authorized_keys: crdt::Map::new(),
 			aliases: crdt::LwwMap::new(),
 			local_aliases: crdt::LwwMap::new(),
-			website_config: crdt::Lww::new(None),
+			website_config: crdt::Lww::new(None.into()),
-			cors_config: crdt::Lww::new(None),
+			cors_config: crdt::Lww::new(None.into()),
-			lifecycle_config: crdt::Lww::new(None),
+			lifecycle_config: crdt::Lww::new(None.into()),
 			quotas: crdt::Lww::new(BucketQuotas::default()),
 		}
 	}

src/model/helper/bucket.rs

@@ -52,7 +52,7 @@ impl<'a> BucketHelper<'a> {
 					.0
 					.bucket_alias_table
 					.get_local(&EmptyKey, bucket_name)?
-					.and_then(|x| *x.state.get());
+					.and_then(|x| x.state.get().into_inner());
 				match alias {
 					Some(id) => id,
 					None => return Ok(None),
@@ -91,15 +91,18 @@ impl<'a> BucketHelper<'a> {
 			.as_option()
 			.ok_or_message("Key should not be deleted at this point")?;
 
-		let bucket_opt =
+		let bucket_opt = if let Some(bucket_id) = api_key_params
-			if let Some(Some(bucket_id)) = api_key_params.local_aliases.get(bucket_name) {
+			.local_aliases
-				self.0
+			.get(bucket_name)
-					.bucket_table
+			.and_then(|x| x.inner())
-					.get_local(&EmptyKey, bucket_id)?
+		{
-					.filter(|x| !x.state.is_deleted())
+			self.0
-			} else {
+				.bucket_table
-				self.resolve_global_bucket_fast(bucket_name)?
+				.get_local(&EmptyKey, bucket_id)?
-			};
+				.filter(|x| !x.state.is_deleted())
+		} else {
+			self.resolve_global_bucket_fast(bucket_name)?
+		};
 		bucket_opt.ok_or_else(|| Error::NoSuchBucket(bucket_name.to_string()))
 	}
 
@@ -125,7 +128,7 @@ impl<'a> BucketHelper<'a> {
 					.bucket_alias_table
 					.get(&EmptyKey, bucket_name)
 					.await?
-					.and_then(|x| *x.state.get());
+					.and_then(|x| x.state.get().into_inner());
 				match alias {
 					Some(id) => id,
 					None => return Ok(None),
@@ -163,8 +166,7 @@ impl<'a> BucketHelper<'a> {
 			.ok_or_else(|| GarageError::Message(format!("access key {} has been deleted", key_id)))?
 			.local_aliases
 			.get(bucket_name)
-			.copied()
+			.and_then(|x| x.inner().copied());
-			.flatten();
 
 		if let Some(bucket_id) = local_alias {
 			Ok(self

src/model/helper/locked.rs

@@ -74,8 +74,8 @@ impl<'a> LockedHelper<'a> {
 		let alias = self.0.bucket_alias_table.get(&EmptyKey, alias_name).await?;
 
 		if let Some(existing_alias) = alias.as_ref() {
-			if let Some(p_bucket) = existing_alias.state.get() {
+			if let Some(p_bucket) = existing_alias.state.get().into_inner() {
-				if *p_bucket != bucket_id {
+				if p_bucket != bucket_id {
 					return Err(Error::BadRequest(format!(
 						"Alias {} already exists and points to different bucket: {:?}",
 						alias_name, p_bucket
@@ -98,7 +98,7 @@ impl<'a> LockedHelper<'a> {
 		let alias = match alias {
 			None => BucketAlias::new(alias_name.clone(), alias_ts, Some(bucket_id)),
 			Some(mut a) => {
-				a.state = Lww::raw(alias_ts, Some(bucket_id));
+				a.state = Lww::raw(alias_ts, Some(bucket_id).into());
 				a
 			}
 		};
@@ -128,7 +128,13 @@ impl<'a> LockedHelper<'a> {
 			.bucket_alias_table
 			.get(&EmptyKey, alias_name)
 			.await?
-			.filter(|a| a.state.get().map(|x| x == bucket_id).unwrap_or(false))
+			.filter(|a| {
+				a.state
+					.get()
+					.into_inner()
+					.map(|x| x == bucket_id)
+					.unwrap_or(false)
+			})
 			.ok_or_message(format!(
 				"Internal error: alias not found or does not point to bucket {:?}",
 				bucket_id
@@ -157,7 +163,7 @@ impl<'a> LockedHelper<'a> {
 		// ---- timestamp-ensured causality barrier ----
 		// writes are now done and all writes use timestamp alias_ts
 
-		alias.state = Lww::raw(alias_ts, None);
+		alias.state = Lww::raw(alias_ts, None.into());
 		self.0.bucket_alias_table.insert(&alias).await?;
 
 		bucket_state.aliases = LwwMap::raw_item(alias_name.clone(), alias_ts, false);
@@ -199,8 +205,8 @@ impl<'a> LockedHelper<'a> {
 		// ---- timestamp-ensured causality barrier ----
 		// writes are now done and all writes use timestamp alias_ts
 
-		if alias.state.get() == &Some(bucket_id) {
+		if alias.state.get().inner() == Some(&bucket_id) {
-			alias.state = Lww::raw(alias_ts, None);
+			alias.state = Lww::raw(alias_ts, None.into());
 			self.0.bucket_alias_table.insert(&alias).await?;
 		}
 
@@ -237,7 +243,11 @@ impl<'a> LockedHelper<'a> {
 
 		let key_param = key.state.as_option_mut().unwrap();
 
-		if let Some(Some(existing_alias)) = key_param.local_aliases.get(alias_name) {
+		if let Some(Some(existing_alias)) = key_param
+			.local_aliases
+			.get(alias_name)
+			.map(CancelingOption::inner)
+		{
 			if *existing_alias != bucket_id {
 				return Err(Error::BadRequest(format!("Alias {} already exists in namespace of key {} and points to different bucket: {:?}", alias_name,  key.key_id, existing_alias)));
 			}
@@ -261,7 +271,8 @@ impl<'a> LockedHelper<'a> {
 		// ---- timestamp-ensured causality barrier ----
 		// writes are now done and all writes use timestamp alias_ts
 
-		key_param.local_aliases = LwwMap::raw_item(alias_name.clone(), alias_ts, Some(bucket_id));
+		key_param.local_aliases =
+			LwwMap::raw_item(alias_name.clone(), alias_ts, Some(bucket_id).into());
 		self.0.key_table.insert(&key).await?;
 
 		bucket_p.local_aliases = LwwMap::raw_item(bucket_p_local_alias_key, alias_ts, true);
@@ -288,7 +299,12 @@ impl<'a> LockedHelper<'a> {
 		let key_p = key.state.as_option().unwrap();
 		let bucket_p = bucket.state.as_option_mut().unwrap();
 
-		if key_p.local_aliases.get(alias_name).cloned().flatten() != Some(bucket_id) {
+		if key_p
+			.local_aliases
+			.get(alias_name)
+			.and_then(CancelingOption::inner)
+			!= Some(&bucket_id)
+		{
 			return Err(GarageError::Message(format!(
 				"Bucket {:?} does not have alias {} in namespace of key {}",
 				bucket_id, alias_name, key_id
@@ -325,7 +341,7 @@ impl<'a> LockedHelper<'a> {
 		// writes are now done and all writes use timestamp alias_ts
 
 		key.state.as_option_mut().unwrap().local_aliases =
-			LwwMap::raw_item(alias_name.clone(), alias_ts, None);
+			LwwMap::raw_item(alias_name.clone(), alias_ts, None.into());
 		self.0.key_table.insert(&key).await?;
 
 		bucket_p.local_aliases = LwwMap::raw_item(bucket_p_local_alias_key, alias_ts, false);
@@ -367,7 +383,7 @@ impl<'a> LockedHelper<'a> {
 		// writes are now done and all writes use timestamp alias_ts
 
 		if let Some(kp) = key.state.as_option_mut() {
-			kp.local_aliases = LwwMap::raw_item(alias_name.clone(), alias_ts, None);
+			kp.local_aliases = LwwMap::raw_item(alias_name.clone(), alias_ts, None.into());
 			self.0.key_table.insert(&key).await?;
 		}
 
@@ -444,8 +460,8 @@ impl<'a> LockedHelper<'a> {
 
 		// 1. Delete local aliases
 		for (alias, _, to) in state.local_aliases.items().iter() {
-			if let Some(bucket_id) = to {
+			if let Some(bucket_id) = to.into_inner() {
-				self.purge_local_bucket_alias(*bucket_id, &key.key_id, alias)
+				self.purge_local_bucket_alias(bucket_id, &key.key_id, alias)
 					.await?;
 			}
 		}
@@ -501,7 +517,7 @@ impl<'a> LockedHelper<'a> {
 						.data
 						.decode_entry(&(item?.1))
 						.map_err(db::TxError::Abort)?;
-					if let Some(id) = alias.state.get() {
+					if let Some(id) = alias.state.get().inner() {
 						if all_buckets.contains(id) {
 							// keep aliases
 							global_aliases.insert(alias.name().to_string(), *id);
@@ -512,7 +528,7 @@ impl<'a> LockedHelper<'a> {
 								alias.name(),
 								id
 							);
-							alias.state.update(None);
+							alias.state.update(None.into());
 							delete_global.push(alias);
 						}
 					}
@@ -544,7 +560,7 @@ impl<'a> LockedHelper<'a> {
 					};
 					let mut has_changes = false;
 					for (name, _, to) in p.local_aliases.items().to_vec() {
-						if let Some(id) = to {
+						if let Some(id) = to.into_inner() {
 							if all_buckets.contains(&id) {
 								local_aliases.insert((key.key_id.clone(), name), id);
 							} else {
@@ -552,7 +568,7 @@ impl<'a> LockedHelper<'a> {
 									"local alias: remove ({}, {}) -> {:?} (bucket is deleted)",
 									key.key_id, name, id
 								);
-								p.local_aliases.update_in_place(name, None);
+								p.local_aliases.update_in_place(name, None.into());
 								has_changes = true;
 							}
 						}

src/model/k2v/item_table.rs

@@ -45,6 +45,7 @@ mod v08 {
 	}
 
 	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
+	#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))]
 	pub enum DvvsValue {
 		Value(#[serde(with = "serde_bytes")] Vec<u8>),
 		Deleted,
@@ -131,9 +132,26 @@ impl K2VItem {
 			ent.discard();
 		}
 	}
+
+	pub fn with_raw_items(items: BTreeMap<K2VNodeId, DvvsEntry>) -> Self {
+		let mut item = K2VItem {
+			partition: K2VItemPartition {
+				bucket_id: [0u8; 32].into(),
+				partition_key: String::new(),
+			},
+			sort_key: String::new(),
+			items,
+		};
+		item.discard();
+		item
+	}
 }
 
 impl DvvsEntry {
+	pub fn from_raw(t_discard: u64, values: Vec<(u64, DvvsValue)>) -> Self {
+		DvvsEntry { t_discard, values }
+	}
+
 	fn max_time(&self) -> u64 {
 		self.values
 			.iter()
@@ -162,15 +180,26 @@ impl Crdt for K2VItem {
 
 impl Crdt for DvvsEntry {
 	fn merge(&mut self, other: &Self) {
-		self.t_discard = std::cmp::max(self.t_discard, other.t_discard);
+		let mut slf = std::mem::take(&mut self.values).into_iter().peekable();
-		self.discard();
+		let mut otr = other.values.iter().peekable();
-
+		while let (Some((slf_t, _)), Some((otr_t, _))) = (slf.peek(), otr.peek()) {
-		let t_max = self.max_time();
+			match slf_t.cmp(otr_t) {
-		for (vt, vv) in other.values.iter() {
+				std::cmp::Ordering::Less => {
-			if *vt > t_max {
+					self.values.push(slf.next().unwrap());
-				self.values.push((*vt, vv.clone()));
+				}
+				std::cmp::Ordering::Equal => {
+					self.values.push(slf.next().unwrap());
+					otr.next();
+				}
+				std::cmp::Ordering::Greater => {
+					self.values.push(otr.next().unwrap().clone());
+				}
 			}
 		}
+		self.values.extend(slf);
+		self.values.extend(otr.cloned());
+		self.t_discard = std::cmp::max(self.t_discard, other.t_discard);
+		self.discard();
 	}
 }
 

src/model/key_table.rs

@@ -43,7 +43,7 @@ mod v08 {
 
 		/// A key can have a local view of buckets names it is
 		/// the only one to see, this is the namespace for these aliases
-		pub local_aliases: crdt::LwwMap<String, Option<Uuid>>,
+		pub local_aliases: crdt::LwwMap<String, crdt::CancelingOption<Uuid>>,
 	}
 
 	impl garage_util::migrate::InitialFormat for Key {}
@@ -51,6 +51,7 @@ mod v08 {
 
 mod v2 {
 	use crate::permission::BucketKeyPerm;
+	use crate::permission::ExpirationTime;
 	use garage_util::crdt;
 	use garage_util::data::Uuid;
 	use serde::{Deserialize, Serialize};
@@ -79,7 +80,7 @@ mod v2 {
 		/// Name for the key
 		pub name: crdt::Lww<String>,
 		/// The optional time of expiration of the key
-		pub expiration: crdt::Lww<Option<u64>>,
+		pub expiration: crdt::Lww<crdt::MergingOption<ExpirationTime>>,
 
 		/// Flag to allow users having this key to create buckets
 		pub allow_create_bucket: crdt::Lww<bool>,
@@ -91,7 +92,7 @@ mod v2 {
 
 		/// A key can have a local view of buckets names it is
 		/// the only one to see, this is the namespace for these aliases
-		pub local_aliases: crdt::LwwMap<String, Option<Uuid>>,
+		pub local_aliases: crdt::LwwMap<String, crdt::CancelingOption<Uuid>>,
 	}
 
 	impl garage_util::migrate::Migrate for Key {
@@ -106,7 +107,7 @@ mod v2 {
 					created: None,
 					secret_key: x.secret_key,
 					name: x.name,
-					expiration: crdt::Lww::raw(0, None),
+					expiration: crdt::Lww::raw(0, None.into()),
 					allow_create_bucket: x.allow_create_bucket,
 					authorized_buckets: x.authorized_buckets,
 					local_aliases: x.local_aliases,
@@ -124,7 +125,7 @@ impl KeyParams {
 			created: Some(now_msec()),
 			secret_key: secret_key.to_string(),
 			name: crdt::Lww::new(name.to_string()),
-			expiration: crdt::Lww::new(None),
+			expiration: crdt::Lww::new(None.into()),
 			allow_create_bucket: crdt::Lww::new(false),
 			authorized_buckets: crdt::Map::new(),
 			local_aliases: crdt::LwwMap::new(),
@@ -229,9 +230,9 @@ impl Key {
 
 impl KeyParams {
 	pub fn is_expired(&self, ts_now: u64) -> bool {
-		match *self.expiration.get() {
+		match self.expiration.get().inner() {
 			None => false,
-			Some(exp) => ts_now >= exp,
+			Some(exp) => ts_now >= exp.0,
 		}
 	}
 }

src/model/permission.rs

@@ -63,3 +63,15 @@ impl Crdt for BucketKeyPerm {
 		}
 	}
 }
+
+/// Expiration date for a key or token
+#[derive(PartialOrd, Ord, PartialEq, Eq, Clone, Copy, Debug, Serialize, Deserialize)]
+#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))]
+#[serde(transparent)]
+pub struct ExpirationTime(pub u64);
+
+impl Crdt for ExpirationTime {
+	fn merge(&mut self, other: &Self) {
+		self.0 = std::cmp::min(self.0, other.0);
+	}
+}

src/model/s3/lifecycle_worker.rs

@@ -271,7 +271,7 @@ async fn process_object(
 	let lifecycle_policy: &[LifecycleRule] = bucket
 		.state
 		.as_option()
-		.and_then(|s| s.lifecycle_config.get().as_deref())
+		.and_then(|s| s.lifecycle_config.get().inner().map(|x| &x[..]))
 		.unwrap_or_default();
 
 	if lifecycle_policy.iter().all(|x| !x.enabled) {

src/net/netapp.rs

@@ -43,6 +43,7 @@ pub(crate) const NETAPP_VERSION_TAG: u64 = 0x6772676e65740010; // grgnet 0x0010
 /// Time a connection must be idle before the first keepalive probe is sent.
 const TCP_KEEPALIVE_TIME: Duration = Duration::from_secs(30);
 /// Interval between keepalive probes after the first.
+#[cfg(not(target_os = "openbsd"))]
 const TCP_KEEPALIVE_INTERVAL: Duration = Duration::from_secs(10);
 
 /// Timeout for outgoing TCP connection attempts.
@@ -52,9 +53,15 @@ const CONNECT_TIMEOUT: Duration = Duration::from_secs(10);
 
 fn set_keepalive(stream: &TcpStream) -> Result<(), std::io::Error> {
 	let sock_ref = socket2::SockRef::from(stream);
+	// OpenBSD does not support with_interval method
+	#[cfg(not(target_os = "openbsd"))]
 	let keepalive = socket2::TcpKeepalive::new()
 		.with_time(TCP_KEEPALIVE_TIME)
 		.with_interval(TCP_KEEPALIVE_INTERVAL);
+
+	#[cfg(target_os = "openbsd")]
+	let keepalive = socket2::TcpKeepalive::new().with_time(TCP_KEEPALIVE_TIME);
+
 	sock_ref.set_tcp_keepalive(&keepalive)
 }
 

src/rpc/metrics.rs

@@ -14,23 +14,23 @@ impl RpcMetrics {
 		let meter = global::meter("garage_rpc");
 		RpcMetrics {
 			rpc_counter: meter
-				.u64_counter("rpc.request_counter")
+				.u64_counter("garage_rpc.request_count")
 				.with_description("Number of RPC requests emitted")
 				.init(),
 			rpc_timeout_counter: meter
-				.u64_counter("rpc.timeout_counter")
+				.u64_counter("garage_rpc.timeout_count")
 				.with_description("Number of RPC timeouts")
 				.init(),
 			rpc_netapp_error_counter: meter
-				.u64_counter("rpc.netapp_error_counter")
+				.u64_counter("garage_rpc.netapp_error_count")
 				.with_description("Number of communication errors (errors in the Netapp library)")
 				.init(),
 			rpc_garage_error_counter: meter
-				.u64_counter("rpc.garage_error_counter")
+				.u64_counter("garage_rpc.garage_error_count")
 				.with_description("Number of RPC errors (errors happening when handling the RPC)")
 				.init(),
 			rpc_duration: meter
-				.f64_value_recorder("rpc.duration")
+				.f64_value_recorder("garage_rpc.duration")
 				.with_description("Duration of RPCs")
 				.init(),
 		}

src/rpc/system_metrics.rs

@@ -110,7 +110,7 @@ impl SystemMetrics {
 			_cluster_healthy: {
 				let get_health = get_health.clone();
 				meter
-					.u64_value_observer("cluster_healthy", move |observer| {
+					.u64_value_observer("garage_cluster_healthy", move |observer| {
 						let h = get_health();
 						if h.status == ClusterHealthStatus::Healthy {
 							observer.observe(1, &[]);
@@ -123,7 +123,7 @@ impl SystemMetrics {
 			},
 			_cluster_available: {
 				let get_health = get_health.clone();
-				meter.u64_value_observer("cluster_available", move |observer| {
+				meter.u64_value_observer("garage_cluster_available", move |observer| {
 					let h = get_health();
 					if h.status != ClusterHealthStatus::Unavailable {
 						observer.observe(1, &[]);
@@ -137,7 +137,7 @@ impl SystemMetrics {
 			_known_nodes: {
 				let get_health = get_health.clone();
 				meter
-					.u64_value_observer("cluster_known_nodes", move |observer| {
+					.u64_value_observer("garage_cluster_known_nodes", move |observer| {
 						let h = get_health();
 						observer.observe(h.known_nodes as u64, &[]);
 					})
@@ -147,7 +147,7 @@ impl SystemMetrics {
 			_connected_nodes: {
 				let get_health = get_health.clone();
 				meter
-					.u64_value_observer("cluster_connected_nodes", move |observer| {
+					.u64_value_observer("garage_cluster_connected_nodes", move |observer| {
 						let h = get_health();
 						observer.observe(h.connected_nodes as u64, &[]);
 					})
@@ -157,7 +157,7 @@ impl SystemMetrics {
 			_storage_nodes: {
 				let get_health = get_health.clone();
 				meter
-					.u64_value_observer("cluster_storage_nodes", move |observer| {
+					.u64_value_observer("garage_cluster_storage_nodes", move |observer| {
 						let h = get_health();
 						observer.observe(h.storage_nodes as u64, &[]);
 					})
@@ -167,7 +167,7 @@ impl SystemMetrics {
 			_storage_nodes_ok: {
 				let get_health = get_health.clone();
 				meter
-					.u64_value_observer("cluster_storage_nodes_ok", move |observer| {
+					.u64_value_observer("garage_cluster_storage_nodes_ok", move |observer| {
 						let h = get_health();
 						observer.observe(h.storage_nodes_ok as u64, &[]);
 					})
@@ -177,7 +177,7 @@ impl SystemMetrics {
 			_partitions: {
 				let get_health = get_health.clone();
 				meter
-					.u64_value_observer("cluster_partitions", move |observer| {
+					.u64_value_observer("garage_cluster_partitions", move |observer| {
 						let h = get_health();
 						observer.observe(h.partitions as u64, &[]);
 					})
@@ -187,7 +187,7 @@ impl SystemMetrics {
 			_partitions_quorum: {
 				let get_health = get_health.clone();
 				meter
-					.u64_value_observer("cluster_partitions_quorum", move |observer| {
+					.u64_value_observer("garage_cluster_partitions_quorum", move |observer| {
 						let h = get_health();
 						observer.observe(h.partitions_quorum as u64, &[]);
 					})
@@ -199,7 +199,7 @@ impl SystemMetrics {
 			_partitions_all_ok: {
 				let get_health = get_health.clone();
 				meter
-					.u64_value_observer("cluster_partitions_all_ok", move |observer| {
+					.u64_value_observer("garage_cluster_partitions_all_ok", move |observer| {
 						let h = get_health();
 						observer.observe(h.partitions_all_ok as u64, &[]);
 					})
@@ -213,7 +213,7 @@ impl SystemMetrics {
 			_layout_node_connected: {
 				let system = system.clone();
 				meter
-					.u64_value_observer("cluster_layout_node_connected", move |observer| {
+					.u64_value_observer("garage_cluster_layout_node_connected", move |observer| {
 						let layout = system.cluster_layout();
 						let nodes = system.get_known_nodes();
 						for id in layout.all_nodes().unwrap_or_default().iter() {
@@ -260,44 +260,47 @@ impl SystemMetrics {
 			_layout_node_disconnected_time: {
 				let system = system.clone();
 				meter
-					.u64_value_observer("cluster_layout_node_disconnected_time", move |observer| {
+					.u64_value_observer(
-						let layout = system.cluster_layout();
+						"garage_cluster_layout_node_disconnected_time",
-						let nodes = system.get_known_nodes();
+						move |observer| {
-						for id in layout.all_nodes().unwrap_or_default().iter() {
+							let layout = system.cluster_layout();
-							let mut kv = vec![KeyValue::new("id", format!("{:?}", id))];
+							let nodes = system.get_known_nodes();
-							if let Some(role) = layout
+							for id in layout.all_nodes().unwrap_or_default().iter() {
-								.current()
+								let mut kv = vec![KeyValue::new("id", format!("{:?}", id))];
-								.ok()
+								if let Some(role) = layout
-								.and_then(|l| l.roles.get(id))
+									.current()
-								.and_then(|r| r.0.as_ref())
+									.ok()
-							{
+									.and_then(|l| l.roles.get(id))
-								kv.push(KeyValue::new("role_zone", role.zone.clone()));
+									.and_then(|r| r.0.as_ref())
-								match role.capacity {
+								{
-									Some(cap) => {
+									kv.push(KeyValue::new("role_zone", role.zone.clone()));
-										kv.push(KeyValue::new("role_capacity", cap as i64));
+									match role.capacity {
-										kv.push(KeyValue::new("role_gateway", 0));
+										Some(cap) => {
-									}
+											kv.push(KeyValue::new("role_capacity", cap as i64));
-									None => {
+											kv.push(KeyValue::new("role_gateway", 0));
-										kv.push(KeyValue::new("role_gateway", 1));
+										}
+										None => {
+											kv.push(KeyValue::new("role_gateway", 1));
+										}
 									}
 								}
-							}
 
-							if let Some(node) = nodes.iter().find(|n| n.id == *id) {
+								if let Some(node) = nodes.iter().find(|n| n.id == *id) {
-								// TODO: see comment above
+									// TODO: see comment above
-								// kv.push(KeyValue::new("address", node.addr.to_string()));
+									// kv.push(KeyValue::new("address", node.addr.to_string()));
-								// kv.push(KeyValue::new(
+									// kv.push(KeyValue::new(
-								//	"hostname",
+									//	"hostname",
-								//	node.status.hostname.clone(),
+									//	node.status.hostname.clone(),
-								// ));
+									// ));
-								if node.is_up {
+									if node.is_up {
-									observer.observe(0, &kv);
+										observer.observe(0, &kv);
-								} else if let Some(secs) = node.last_seen_secs_ago {
+									} else if let Some(secs) = node.last_seen_secs_ago {
-									observer.observe(secs, &kv);
+										observer.observe(secs, &kv);
+									}
 								}
 							}
-						}
+						},
-					})
+					)
 					.with_description(
 						"Time (in seconds) since last connection to nodes in the cluster layout",
 					)

src/table/metrics.rs

@@ -34,7 +34,7 @@ impl TableMetrics {
 		TableMetrics {
 			_table_size: meter
 				.u64_value_observer(
-					"table.size",
+					"garage_table.size",
 					move |observer| {
 						if let Ok(value) = store.approximate_len() {
 							observer.observe(
@@ -48,7 +48,7 @@ impl TableMetrics {
 				.init(),
 			_merkle_tree_size: meter
 				.u64_value_observer(
-					"table.merkle_tree_size",
+					"garage_table.merkle_tree_size",
 					move |observer| {
 						if let Ok(value) = merkle_tree.approximate_len() {
 							observer.observe(
@@ -62,7 +62,7 @@ impl TableMetrics {
 				.init(),
 			_merkle_todo_len: meter
 				.u64_value_observer(
-					"table.merkle_updater_todo_queue_length",
+					"garage_table.merkle_updater_todo_queue_length",
 					move |observer| {
 						if let Ok(v) = merkle_todo.approximate_len() {
 							observer.observe(
@@ -76,7 +76,7 @@ impl TableMetrics {
 				.init(),
 			_insert_queue_len: meter
 				.u64_value_observer(
-					"table.insert_queue_length",
+					"garage_table.insert_queue_length",
 					move |observer| {
 						if let Ok(v) = insert_queue.approximate_len() {
 							observer.observe(
@@ -90,7 +90,7 @@ impl TableMetrics {
 				.init(),
 			_gc_todo_len: meter
 				.u64_value_observer(
-					"table.gc_todo_queue_length",
+					"garage_table.gc_todo_queue_length",
 					move |observer| {
                         if let Ok(value) = gc_todo.approximate_len() {
                             observer.observe(
@@ -104,43 +104,43 @@ impl TableMetrics {
 				.init(),
 
 			get_request_counter: meter
-				.u64_counter("table.get_request_counter")
+				.u64_counter("garage_table.get_request_count")
 				.with_description("Number of get/get_range requests internally made on this table")
 				.init()
 				.bind(&[KeyValue::new("table_name", table_name)]),
 			get_request_duration: meter
-				.f64_value_recorder("table.get_request_duration")
+				.f64_value_recorder("garage_table.get_request_duration")
 				.with_description("Duration of get/get_range requests internally made on this table, in seconds")
 				.init()
 				.bind(&[KeyValue::new("table_name", table_name)]),
 			put_request_counter: meter
-				.u64_counter("table.put_request_counter")
+				.u64_counter("garage_table.put_request_count")
 				.with_description("Number of insert/insert_many requests internally made on this table")
 				.init()
 				.bind(&[KeyValue::new("table_name", table_name)]),
 			put_request_duration: meter
-				.f64_value_recorder("table.put_request_duration")
+				.f64_value_recorder("garage_table.put_request_duration")
 				.with_description("Duration of insert/insert_many requests internally made on this table, in seconds")
 				.init()
 				.bind(&[KeyValue::new("table_name", table_name)]),
 
 			internal_update_counter: meter
-				.u64_counter("table.internal_update_counter")
+				.u64_counter("garage_table.internal_update_count")
 				.with_description("Number of value updates where the value actually changes (includes creation of new key and update of existing key)")
 				.init()
 				.bind(&[KeyValue::new("table_name", table_name)]),
 			internal_delete_counter: meter
-				.u64_counter("table.internal_delete_counter")
+				.u64_counter("garage_table.internal_delete_count")
 				.with_description("Number of value deletions in the tree (due to GC or repartitioning)")
 				.init()
 				.bind(&[KeyValue::new("table_name", table_name)]),
 
 			sync_items_sent: meter
-				.u64_counter("table.sync_items_sent")
+				.u64_counter("garage_table.sync_items_sent")
 				.with_description("Number of data items sent to other nodes during resync procedures")
 				.init(),
 			sync_items_received: meter
-				.u64_counter("table.sync_items_received")
+				.u64_counter("garage_table.sync_items_received")
 				.with_description("Number of data items received from other nodes during resync procedures")
 				.init(),
 		}

src/util/crdt/crdt.rs

@@ -26,28 +26,6 @@ pub trait Crdt {
 	fn merge(&mut self, other: &Self);
 }
 
-/// `Option<T>` implements Crdt for any type T, even if T doesn't implement CRDT itself: when
-/// different values are detected, they are always merged to None.  This can be used for value
-/// types which shoulnd't be merged, instead of trying to merge things when we know we don't want
-/// to merge them (which is what the `AutoCrdt` trait is used for most of the time). This cases
-/// arises very often, for example with a Lww or a `LwwMap`: the value type has to be a CRDT so that
-/// we have a rule for what to do when timestamps aren't enough to disambiguate (in a distributed
-/// system, anything can happen!), and with `AutoCrdt` the rule is to make an arbitrary (but
-/// deterministic) choice between the two.  When using an `Option<T>` instead with this impl, ambiguity
-/// cases are explicitly stored as None, which allows us to detect the ambiguity and handle it in
-/// the way we want. (this can only work if we are happy with losing the value when an ambiguity
-/// arises)
-impl<T> Crdt for Option<T>
-where
-	T: Eq,
-{
-	fn merge(&mut self, other: &Self) {
-		if self != other {
-			*self = None;
-		}
-	}
-}
-
 /// All types that implement `Ord` (a total order) can also implement a trivial CRDT
 /// defined by the merge rule: `a ⊔ b = max(a, b)`. Implement this trait for your type
 /// to enable this behavior.

src/util/crdt/mod.rs

@@ -16,6 +16,7 @@ mod deletable;
 mod lww;
 mod lww_map;
 mod map;
+mod option;
 
 pub use self::bool::*;
 pub use crdt::*;
@@ -23,3 +24,4 @@ pub use deletable::*;
 pub use lww::*;
 pub use lww_map::*;
 pub use map::*;
+pub use option::*;

src/util/crdt/option.rs

@@ -0,0 +1,97 @@
+use serde::{Deserialize, Serialize};
+
+use crate::crdt::Crdt;
+
+#[derive(Serialize, Deserialize, Clone, Default, Copy, PartialEq, Eq, Debug)]
+#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))]
+#[serde(transparent)]
+pub struct CancelingOption<T>(pub Option<T>);
+
+/// `CancelingOption<T>` implements Crdt for any type T, even if T doesn't implement CRDT itself: when
+/// different values are detected, they are always merged to None.  This can be used for value
+/// types which shoulnd't be merged, instead of trying to merge things when we know we don't want
+/// to merge them (which is what the `AutoCrdt` trait is used for most of the time). This cases
+/// arises very often, for example with a Lww or a `LwwMap`: the value type has to be a CRDT so that
+/// we have a rule for what to do when timestamps aren't enough to disambiguate (in a distributed
+/// system, anything can happen!), and with `AutoCrdt` the rule is to make an arbitrary (but
+/// deterministic) choice between the two.  When using an `CancelingOption<T>` instead with this impl, ambiguity
+/// cases are explicitly stored as None, which allows us to detect the ambiguity and handle it in
+/// the way we want. (this can only work if we are happy with losing the value when an ambiguity
+/// arises)
+impl<T> Crdt for CancelingOption<T>
+where
+	T: Eq + Clone,
+{
+	fn merge(&mut self, other: &Self) {
+		match (self.0.as_ref(), other.0.as_ref()) {
+			(Some(a), Some(b)) if a != b => {
+				self.0 = None;
+			}
+			(None, Some(b)) => {
+				self.0 = Some(b.clone());
+			}
+			_ => {}
+		}
+	}
+}
+
+impl<T> CancelingOption<T> {
+	pub fn inner(&self) -> Option<&T> {
+		self.0.as_ref()
+	}
+
+	pub fn into_inner(self) -> Option<T> {
+		self.0
+	}
+
+	pub fn map<U>(self, f: impl FnOnce(T) -> U) -> CancelingOption<U> {
+		CancelingOption(self.0.map(f))
+	}
+}
+
+impl<T> From<Option<T>> for CancelingOption<T> {
+	fn from(x: Option<T>) -> Self {
+		Self(x)
+	}
+}
+
+#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, Copy, Default)]
+#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))]
+#[serde(transparent)]
+pub struct MergingOption<T>(pub Option<T>);
+
+/// `MergingOption<T>` implements `Crdt` when `T` implements `Crdt`:
+/// None is a bottom value, and different Some values get merged according
+/// to their Crdt operator.
+impl<T> Crdt for MergingOption<T>
+where
+	T: Crdt + Clone,
+{
+	fn merge(&mut self, other: &Self) {
+		if let (Some(a), Some(b)) = (self.0.as_mut(), other.0.as_ref()) {
+			a.merge(b);
+		} else {
+			self.0 = self.0.take().or_else(|| other.0.clone());
+		}
+	}
+}
+
+impl<T> MergingOption<T> {
+	pub fn inner(&self) -> Option<&T> {
+		self.0.as_ref()
+	}
+
+	pub fn into_inner(self) -> Option<T> {
+		self.0
+	}
+
+	pub fn map<U>(self, f: impl FnOnce(T) -> U) -> MergingOption<U> {
+		MergingOption(self.0.map(f))
+	}
+}
+
+impl<T> From<Option<T>> for MergingOption<T> {
+	fn from(x: Option<T>) -> Self {
+		Self(x)
+	}
+}

src/web/web_server.rs

@@ -54,15 +54,15 @@ impl WebMetrics {
 		let meter = global::meter("garage/web");
 		Self {
 			request_counter: meter
-				.u64_counter("web.request_counter")
+				.u64_counter("garage_web.request_count")
 				.with_description("Number of requests to the web endpoint")
 				.init(),
 			error_counter: meter
-				.u64_counter("web.error_counter")
+				.u64_counter("garage_web.error_count")
 				.with_description("Number of requests to the web endpoint resulting in errors")
 				.init(),
 			request_duration: meter
-				.f64_value_recorder("web.request_duration")
+				.f64_value_recorder("garage_web.request_duration")
 				.with_description("Duration of requests to the web endpoint")
 				.init(),
 		}
@@ -241,7 +241,7 @@ impl WebServer {
 			.bucket_alias_table
 			.get(&EmptyKey, &bucket_name.to_string())
 			.await?
-			.and_then(|x| x.state.take())
+			.and_then(|x| x.state.get().into_inner())
 			.ok_or(Error::NotFound)?;
 
 		// Check bucket isn't deleted and has website access enabled
@@ -256,7 +256,7 @@ impl WebServer {
 		let website_config = bucket_params
 			.website_config
 			.get()
-			.as_ref()
+			.inner()
 			.ok_or(Error::NotFound)?;
 
 		// Get path