fix: improve routing of keys starting with "/" (fix #1178) (#1465)

Path-style URLs of the form /bucket//key address an object whose key begins with "/". Two greedy uses of `trim_start_matches('/')` were collapsing these leading slashes away:

- `uri.path().trim_start_matches('/')` stripped all leading slashes from the raw path before any further parsing.
- `p.trim_start_matches('/')` stripped leading slashes from the remainder after `split_once('/')` had already consumed the bucket/key separator

The combined effect wath that `HEAD /bucket//` and `GET /bucket//` produced an empty key, which the router treated as bucket-level operations (HeadBucket -> 200 OK, and ListObjectsV2) instead of an object-level op (HeadObject/GetObject -> 404 NoSuchKey).

The fix is simple: Replace the first `trim_start_matches` with `strip_prefix` (to strip exactly one separator slash) and remove the second one entirely. Path-style and vhost-style requests are now consistent: a double slash in the URL correctly addresses a key whose name begins with "/".

Regression tests added for `HEAD //` and `GET //` requests in both request styles.

Fixes: #1464

---

Disclaimer: I'm not fluent in Rust and I did use an LLM to explain the code to me. All code was written by me.

I'm not sure whether the large `test_cases!` block in the `test_aws_doc_examples` function is the right place for my tests (it certainly was a convenient one).

Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1465

Dockerfile

@@ -4,4 +4,6 @@ ENV RUST_BACKTRACE=1
 ENV RUST_LOG=garage=info
 
 COPY result/bin/garage /
-CMD [ "/garage", "server"]
+
+ENTRYPOINT ["/garage"]
+CMD ["server"]

doc/book/cookbook/real-world.md

@@ -213,7 +213,12 @@ If your configuration file is at `/etc/garage.toml`, the `garage` binary should
 You can also use an alias as follows to use the Garage binary inside your docker container:
 
 ```bash
+# garage 3.x, we have an entrypoint and you can use
+alias garage="docker exec -ti <container name>"
+
+# For garage 2.x, you need to specify the absolute path to binary
 alias garage="docker exec -ti <container name> /garage"
+
 ```
 
 You can test your `garage` CLI utility by running a simple command such as:

doc/book/quick-start/_index.md

@@ -178,8 +178,11 @@ garage status
 
 If you are running Garage in a Docker container, you can use the following command instead:
 
+NOTE: Garage 3.x uses docker `ENTRYPOINT` and it's easier to use,
+while garage 2.x does not and you need to specify path `/garage`
+
 ```bash
-docker exec garage-container /garage status
+docker exec garage-container status
 ```
 
 This should show something like this:
@@ -320,7 +323,7 @@ If you are running Garage in a Docker container, you can set the following alias
 to provide a fake `garage`command that uses the Garage binary inside your container:
 
 ```bash
-alias garage="docker exec -ti <container name> /garage"
+alias garage="docker exec -ti <container name>"
 ```
 
 You can test that your `garage` CLI is configured correctly by running a basic command such as `garage status`.

doc/book/reference-manual/admin-api.md

@@ -182,15 +182,15 @@ content-type: text/plain; version=0.0.4
 content-length: 12145
 date: Tue, 08 Aug 2023 07:25:05 GMT
 
-# HELP api_admin_error_counter Number of API calls to the various Admin API endpoints that resulted in errors
+# HELP garage_api_admin_error_count Number of API calls to the various Admin API endpoints that resulted in errors
-# TYPE api_admin_error_counter counter
+# TYPE garage_api_admin_error_count counter
-api_admin_error_counter{api_endpoint="CheckWebsiteEnabled",status_code="400"} 1
+garage_api_admin_error_count{api_endpoint="CheckWebsiteEnabled",status_code="400"} 1
-api_admin_error_counter{api_endpoint="CheckWebsiteEnabled",status_code="404"} 3
+garage_api_admin_error_count{api_endpoint="CheckWebsiteEnabled",status_code="404"} 3
-# HELP api_admin_request_counter Number of API calls to the various Admin API endpoints
+# HELP garage_api_admin_request_count Number of API calls to the various Admin API endpoints
-# TYPE api_admin_request_counter counter
+# TYPE garage_api_admin_request_count counter
-api_admin_request_counter{api_endpoint="CheckWebsiteEnabled"} 7
+garage_api_admin_request_count{api_endpoint="CheckWebsiteEnabled"} 7
-api_admin_request_counter{api_endpoint="Health"} 3
+garage_api_admin_request_count{api_endpoint="Health"} 3
-# HELP api_admin_request_duration Duration of API calls to the various Admin API endpoints
+# HELP garage_api_admin_request_duration Duration of API calls to the various Admin API endpoints
 ...
 ```
 

doc/book/reference-manual/monitoring.md

@@ -40,146 +40,146 @@ garage_local_disk_total{volume="metadata"} 763063566336
 
 ### Cluster health status metrics
 
-#### `cluster_healthy` (gauge)
+#### `garage_cluster_healthy` (gauge)
 
 Whether all storage nodes are connected (0 or 1)
 
 ```
-cluster_healthy 0
+garage_cluster_healthy 0
 ```
 
-#### `cluster_available` (gauge)
+#### `garage_cluster_available` (gauge)
 
 Whether all requests can be served, even if some storage nodes are disconnected
 
 ```
-cluster_available 1
+garage_cluster_available 1
 ```
 
-#### `cluster_connected_nodes` (gauge)
+#### `garage_cluster_connected_nodes` (gauge)
 
 Number of nodes currently connected
 
 ```
-cluster_connected_nodes 3
+garage_cluster_connected_nodes 3
 ```
 
-#### `cluster_known_nodes` (gauge)
+#### `garage_cluster_known_nodes` (gauge)
 
 Number of nodes already seen once in the cluster
 
 ```
-cluster_known_nodes 3
+garage_cluster_known_nodes 3
 ```
 
-#### `cluster_layout_node_connected` (gauge)
+#### `garage_cluster_layout_node_connected` (gauge)
 
 Connection status for individual nodes of the cluster layout
 
 ```
-cluster_layout_node_connected{id="62b218d848e86a64",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 1
+garage_cluster_layout_node_connected{id="62b218d848e86a64",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 1
-cluster_layout_node_connected{id="a11c7cf18af29737",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 0
+garage_cluster_layout_node_connected{id="a11c7cf18af29737",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 0
-cluster_layout_node_connected{id="a235ac7695e0c54d",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 1
+garage_cluster_layout_node_connected{id="a235ac7695e0c54d",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 1
-cluster_layout_node_connected{id="b10c110e4e854e5a",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 1
+garage_cluster_layout_node_connected{id="b10c110e4e854e5a",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 1
 ```
 
-#### `cluster_layout_node_disconnected_time` (gauge)
+#### `garage_cluster_layout_node_disconnected_time` (gauge)
 
 Time (in seconds) since last connection to individual nodes of the cluster layout
 
 ```
-cluster_layout_node_disconnected_time{id="62b218d848e86a64",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 0
+garage_cluster_layout_node_disconnected_time{id="62b218d848e86a64",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 0
-cluster_layout_node_disconnected_time{id="a235ac7695e0c54d",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 0
+garage_cluster_layout_node_disconnected_time{id="a235ac7695e0c54d",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 0
-cluster_layout_node_disconnected_time{id="b10c110e4e854e5a",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 0
+garage_cluster_layout_node_disconnected_time{id="b10c110e4e854e5a",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 0
 ```
 
-#### `cluster_storage_nodes` (gauge)
+#### `garage_cluster_storage_nodes` (gauge)
 
 Number of storage nodes declared in the current layout
 
 ```
-cluster_storage_nodes 4
+garage_cluster_storage_nodes 4
 ```
 
-#### `cluster_storage_nodes_ok` (gauge)
+#### `garage_cluster_storage_nodes_ok` (gauge)
 
 Number of storage nodes currently connected
 
 ```
-cluster_storage_nodes_ok 3
+garage_cluster_storage_nodes_ok 3
 ```
 
-#### `cluster_partitions` (gauge)
+#### `garage_cluster_partitions` (gauge)
 
 Number of partitions in the layout (this is always 256)
 
 ```
-cluster_partitions 256
+garage_cluster_partitions 256
 ```
 
-#### `cluster_partitions_all_ok` (gauge)
+#### `garage_cluster_partitions_all_ok` (gauge)
 
 Number of partitions for which all storage nodes are connected
 
 ```
-cluster_partitions_all_ok 64
+garage_cluster_partitions_all_ok 64
 ```
 
-#### `cluster_partitions_quorum` (gauge)
+#### `garage_cluster_partitions_quorum` (gauge)
 
 Number of partitions for which we have a quorum of connected nodes and all requests can be served
 
 ```
-cluster_partitions_quorum 256
+garage_cluster_partitions_quorum 256
 ```
 
 ### Metrics of the API endpoints
 
-#### `api_admin_request_counter` (counter)
+#### `garage_api_admin_request_count` (counter)
 
 Counts the number of requests to a given endpoint of the administration API. Example:
 
 ```
-api_admin_request_counter{api_endpoint="Metrics"} 127041
+garage_api_admin_request_count{api_endpoint="Metrics"} 127041
 ```
 
-#### `api_admin_request_duration` (histogram)
+#### `garage_api_admin_request_duration` (histogram)
 
 Evaluates the duration of API calls to the various administration API endpoint. Example:
 
 ```
-api_admin_request_duration_bucket{api_endpoint="Metrics",le="0.5"} 127041
+garage_api_admin_request_duration_bucket{api_endpoint="Metrics",le="0.5"} 127041
-api_admin_request_duration_sum{api_endpoint="Metrics"} 605.250344830999
+garage_api_admin_request_duration_sum{api_endpoint="Metrics"} 605.250344830999
-api_admin_request_duration_count{api_endpoint="Metrics"} 127041
+garage_api_admin_request_duration_count{api_endpoint="Metrics"} 127041
 ```
 
-#### `api_s3_request_counter` (counter)
+#### `garage_api_s3_request_count` (counter)
 
 Counts the number of requests to a given endpoint of the S3 API. Example:
 
 ```
-api_s3_request_counter{api_endpoint="CreateMultipartUpload"} 1
+garage_api_s3_request_count{api_endpoint="CreateMultipartUpload"} 1
 ```
 
-#### `api_s3_error_counter` (counter)
+#### `garage_api_s3_error_count` (counter)
 
 Counts the number of requests to a given endpoint of the S3 API that returned an error. Example:
 
 ```
-api_s3_error_counter{api_endpoint="GetObject",status_code="404"} 39
+garage_api_s3_error_count{api_endpoint="GetObject",status_code="404"} 39
 ```
 
-#### `api_s3_request_duration` (histogram)
+#### `garage_api_s3_request_duration` (histogram)
 
 Evaluates the duration of API calls to the various S3 API endpoints. Example:
 
 ```
-api_s3_request_duration_bucket{api_endpoint="CreateMultipartUpload",le="0.5"} 1
+garage_api_s3_request_duration_bucket{api_endpoint="CreateMultipartUpload",le="0.5"} 1
-api_s3_request_duration_sum{api_endpoint="CreateMultipartUpload"} 0.046340762
+garage_api_s3_request_duration_sum{api_endpoint="CreateMultipartUpload"} 0.046340762
-api_s3_request_duration_count{api_endpoint="CreateMultipartUpload"} 1
+garage_api_s3_request_duration_count{api_endpoint="CreateMultipartUpload"} 1
 ```
 
-#### `api_k2v_request_counter` (counter), `api_k2v_error_counter` (counter), `api_k2v_error_duration` (histogram)
+#### `garage_api_k2v_request_count` (counter), `garage_api_k2v_error_count` (counter), `garage_api_k2v_error_duration` (histogram)
 
 Same as for S3, for the K2V API.
 
@@ -187,45 +187,45 @@ Same as for S3, for the K2V API.
 ### Metrics of the Web endpoint
 
 
-#### `web_request_counter` (counter)
+#### `garage_web_request_count` (counter)
 
 Number of requests to the web endpoint
 
 ```
-web_request_counter{method="GET"} 80
+garage_web_request_count{method="GET"} 80
 ```
 
-#### `web_request_duration` (histogram)
+#### `garage_web_request_duration` (histogram)
 
 Duration of requests to the web endpoint
 
 ```
-web_request_duration_bucket{method="GET",le="0.5"} 80
+garage_web_request_duration_bucket{method="GET",le="0.5"} 80
-web_request_duration_sum{method="GET"} 1.0528433229999998
+garage_web_request_duration_sum{method="GET"} 1.0528433229999998
-web_request_duration_count{method="GET"} 80
+garage_web_request_duration_count{method="GET"} 80
 ```
 
-#### `web_error_counter` (counter)
+#### `garage_web_error_count` (counter)
 
 Number of requests to the web endpoint resulting in errors
 
 ```
-web_error_counter{method="GET",status_code="404 Not Found"} 64
+garage_web_error_count{method="GET",status_code="404 Not Found"} 64
 ```
 
 
 ### Metrics of the data block manager
 
-#### `block_bytes_read`, `block_bytes_written` (counter)
+#### `garage_block_bytes_read`, `garage_block_bytes_written` (counter)
 
 Number of bytes read/written to/from disk in the data storage directory.
 
 ```
-block_bytes_read 120586322022
+garage_block_bytes_read 120586322022
-block_bytes_written 3386618077
+garage_block_bytes_written 3386618077
 ```
 
-#### `block_ram_buffer_free_kb` (gauge)
+#### `garage_block_ram_buffer_free_kb` (gauge)
 
 Kibibytes available for buffering blocks that have to be sent to remote nodes.
 When clients send too much data to this node and a storage node is not receiving
@@ -233,170 +233,168 @@ data fast enough due to slower network conditions, this will decrease down to
 zero and backpressure will be applied.
 
 ```
-block_ram_buffer_free_kb 219829
+garage_block_ram_buffer_free_kb 219829
 ```
 
-#### `block_compression_level` (counter)
+#### `garage_block_compression_level` (counter)
 
 Exposes the block compression level configured for the Garage node.
 
 ```
-block_compression_level 3
+garage_block_compression_level 3
 ```
 
-#### `block_read_duration`, `block_write_duration` (histograms)
+#### `garage_block_read_duration`, `garage_block_write_duration` (histograms)
 
 Evaluates the duration of the reading/writing of individual data blocks in the data storage directory.
 
 ```
-block_read_duration_bucket{le="0.5"} 169229
+garage_block_read_duration_bucket{le="0.5"} 169229
-block_read_duration_sum 2761.6902550310056
+garage_block_read_duration_sum 2761.6902550310056
-block_read_duration_count 169240
+garage_block_read_duration_count 169240
-block_write_duration_bucket{le="0.5"} 3559
+garage_block_write_duration_bucket{le="0.5"} 3559
-block_write_duration_sum 195.59170078500006
+garage_block_write_duration_sum 195.59170078500006
-block_write_duration_count 3571
+garage_block_write_duration_count 3571
 ```
 
-#### `block_delete_counter` (counter)
+#### `garage_block_delete_count` (counter)
 
 Counts the number of data blocks that have been deleted from storage.
 
 ```
-block_delete_counter 122
+garage_block_delete_count 122
 ```
 
-#### `block_resync_counter` (counter), `block_resync_duration` (histogram)
+#### `garage_block_resync_count` (counter), `garage_block_resync_duration` (histogram)
 
 Counts the number of resync operations the node has executed, and evaluates their duration.
 
 ```
-block_resync_counter 308897
+garage_block_resync_count 308897
-block_resync_duration_bucket{le="0.5"} 308892
+garage_block_resync_duration_bucket{le="0.5"} 308892
-block_resync_duration_sum 139.64204196100016
+garage_block_resync_duration_sum 139.64204196100016
-block_resync_duration_count 308897
+garage_block_resync_duration_count 308897
 ```
 
-#### `block_resync_queue_length` (gauge)
+#### `garage_block_resync_queue_length` (gauge)
 
 The number of block hashes currently queued for a resync.
 This is normal to be nonzero for long periods of time.
 
 ```
-block_resync_queue_length 0
+garage_block_resync_queue_length 0
 ```
 
-#### `block_resync_errored_blocks` (gauge)
+#### `garage_block_resync_errored_blocks` (gauge)
 
 The number of block hashes that we were unable to resync last time we tried.
 **THIS SHOULD BE ZERO, OR FALL BACK TO ZERO RAPIDLY, IN A HEALTHY CLUSTER.**
 Persistent nonzero values indicate that some data is likely to be lost.
 
 ```
-block_resync_errored_blocks 0
+garage_block_resync_errored_blocks 0
 ```
 
 
 ### Metrics related to RPCs (remote procedure calls) between nodes
 
-#### `rpc_netapp_request_counter` (counter)
+#### `garage_rpc_netapp_request_count` (counter)
 
 Number of RPC requests emitted
 
 ```
-rpc_request_counter{from="<this node>",rpc_endpoint="garage_block/manager.rs/Rpc",to="<remote node>"} 176
+garage_rpc_request_count{from="<this node>",rpc_endpoint="garage_block/manager.rs/Rpc",to="<remote node>"} 176
 ```
 
-#### `rpc_netapp_error_counter` (counter)
+#### `garage_rpc_netapp_error_count` (counter)
 
 Number of communication errors (errors in the Netapp library, generally due to disconnected nodes)
 
 ```
-rpc_netapp_error_counter{from="<this node>",rpc_endpoint="garage_block/manager.rs/Rpc",to="<remote node>"} 354
+garage_rpc_netapp_error_count{from="<this node>",rpc_endpoint="garage_block/manager.rs/Rpc",to="<remote node>"} 354
 ```
 
-#### `rpc_timeout_counter` (counter)
+#### `garage_rpc_timeout_count` (counter)
 
 Number of RPC timeouts, should be close to zero in a healthy cluster.
 
 ```
-rpc_timeout_counter{from="<this node>",rpc_endpoint="garage_rpc/membership.rs/SystemRpc",to="<remote node>"} 1
+garage_rpc_timeout_count{from="<this node>",rpc_endpoint="garage_rpc/membership.rs/SystemRpc",to="<remote node>"} 1
 ```
 
-#### `rpc_duration` (histogram)
+#### `garage_rpc_duration` (histogram)
 
 The duration of internal RPC calls between Garage nodes.
 
 ```
-rpc_duration_bucket{from="<this node>",rpc_endpoint="garage_block/manager.rs/Rpc",to="<remote node>",le="0.5"} 166
+garage_rpc_duration_bucket{from="<this node>",rpc_endpoint="garage_block/manager.rs/Rpc",to="<remote node>",le="0.5"} 166
-rpc_duration_sum{from="<this node>",rpc_endpoint="garage_block/manager.rs/Rpc",to="<remote node>"} 35.172253716
+garage_rpc_duration_sum{from="<this node>",rpc_endpoint="garage_block/manager.rs/Rpc",to="<remote node>"} 35.172253716
-rpc_duration_count{from="<this node>",rpc_endpoint="garage_block/manager.rs/Rpc",to="<remote node>"} 174
+garage_rpc_duration_count{from="<this node>",rpc_endpoint="garage_block/manager.rs/Rpc",to="<remote node>"} 174
 ```
 
 
 ### Metrics of the metadata table manager
 
-#### `table_gc_todo_queue_length` (gauge)
+#### `garage_table_gc_todo_queue_length` (gauge)
 
 Table garbage collector TODO queue length
 
 ```
-table_gc_todo_queue_length{table_name="block_ref"} 0
+garage_table_gc_todo_queue_length{table_name="block_ref"} 0
 ```
 
-#### `table_get_request_counter` (counter), `table_get_request_duration` (histogram)
+#### `garage_table_get_request_count` (counter), `garage_table_get_request_duration` (histogram)
 
 Number of get/get_range requests internally made on each table, and their duration.
 
 ```
-table_get_request_counter{table_name="bucket_alias"} 315
+garage_table_get_request_count{table_name="bucket_alias"} 315
-table_get_request_duration_bucket{table_name="bucket_alias",le="0.5"} 315
+garage_table_get_request_duration_bucket{table_name="bucket_alias",le="0.5"} 315
-table_get_request_duration_sum{table_name="bucket_alias"} 0.048509778000000024
+garage_table_get_request_duration_sum{table_name="bucket_alias"} 0.048509778000000024
-table_get_request_duration_count{table_name="bucket_alias"} 315
+garage_table_get_request_duration_count{table_name="bucket_alias"} 315
 ```
 
 
-#### `table_put_request_counter` (counter), `table_put_request_duration` (histogram)
+#### `garage_table_put_request_count` (counter), `garage_table_put_request_duration` (histogram)
 
 Number of insert/insert_many requests internally made on this table, and their duration
 
 ```
-table_put_request_counter{table_name="block_ref"} 677
+garage_table_put_request_count{table_name="block_ref"} 677
-table_put_request_duration_bucket{table_name="block_ref",le="0.5"} 677
+garage_table_put_request_duration_bucket{table_name="block_ref",le="0.5"} 677
-table_put_request_duration_sum{table_name="block_ref"} 61.617528636
+garage_table_put_request_duration_sum{table_name="block_ref"} 61.617528636
-table_put_request_duration_count{table_name="block_ref"} 677
+garage_table_put_request_duration_count{table_name="block_ref"} 677
 ```
 
-#### `table_internal_delete_counter` (counter)
+#### `garage_table_internal_delete_count` (counter)
 
 Number of value deletions in the tree (due to GC or repartitioning)
 
 ```
-table_internal_delete_counter{table_name="block_ref"} 2296
+garage_table_internal_delete_count{table_name="block_ref"} 2296
 ```
 
-#### `table_internal_update_counter` (counter)
+#### `garage_table_internal_update_count` (counter)
 
 Number of value updates where the value actually changes (includes creation of new key and update of existing key)
 
 ```
-table_internal_update_counter{table_name="block_ref"} 5996
+garage_table_internal_update_count{table_name="block_ref"} 5996
 ```
 
-#### `table_merkle_updater_todo_queue_length` (gauge)
+#### `garage_table_merkle_updater_todo_queue_length` (gauge)
 
 Merkle tree updater TODO queue length (should fall to zero rapidly)
 
 ```
-table_merkle_updater_todo_queue_length{table_name="block_ref"} 0
+garage_table_merkle_updater_todo_queue_length{table_name="block_ref"} 0
 ```
 
-#### `table_sync_items_received`, `table_sync_items_sent` (counters)
+#### `garage_table_sync_items_received`, `garage_table_sync_items_sent` (counters)
 
 Number of data items sent to/received from other nodes during resync procedures
 
 ```
-table_sync_items_received{from="<remote node>",table_name="bucket_v2"} 3
+garage_table_sync_items_received{from="<remote node>",table_name="bucket_v2"} 3
-table_sync_items_sent{table_name="block_ref",to="<remote node>"} 2
+garage_table_sync_items_sent{table_name="block_ref",to="<remote node>"} 2
 ```
-
-

script/telemetry/grafana-garage-dashboard-elasticsearch.json

@@ -161,7 +161,7 @@
           },
           "metrics": [
             {
-              "field": "api_request_counter",
+              "field": "garage_api_request_count",
               "hide": true,
               "id": "1",
               "type": "sum"
@@ -284,7 +284,7 @@
           "hide": false,
           "metrics": [
             {
-              "field": "api_request_duration",
+              "field": "garage_api_request_duration",
               "id": "1",
               "type": "avg"
             }
@@ -412,7 +412,7 @@
           },
           "metrics": [
             {
-              "field": "api_error_counter",
+              "field": "garage_api_error_count",
               "hide": true,
               "id": "1",
               "type": "sum"
@@ -540,7 +540,7 @@
           },
           "metrics": [
             {
-              "field": "web_request_counter",
+              "field": "garage_web_request_count",
               "hide": true,
               "id": "1",
               "type": "sum"
@@ -666,7 +666,7 @@
           "hide": false,
           "metrics": [
             {
-              "field": "web_request_duration",
+              "field": "garage_web_request_duration",
               "id": "1",
               "type": "avg"
             }
@@ -794,7 +794,7 @@
           },
           "metrics": [
             {
-              "field": "web_error_counter",
+              "field": "garage_web_error_count",
               "hide": true,
               "id": "1",
               "type": "sum"
@@ -918,7 +918,7 @@
           "hide": false,
           "metrics": [
             {
-              "field": "table_get_request_counter",
+              "field": "garage_table_get_request_count",
               "hide": true,
               "id": "1",
               "type": "sum"
@@ -1042,7 +1042,7 @@
           "hide": false,
           "metrics": [
             {
-              "field": "table_put_request_counter",
+              "field": "garage_table_put_request_count",
               "hide": true,
               "id": "1",
               "type": "sum"
@@ -1154,7 +1154,7 @@
           "hide": false,
           "metrics": [
             {
-              "field": "block_bytes_read",
+              "field": "garage_block_bytes_read",
               "hide": true,
               "id": "1",
               "type": "sum"
@@ -1270,7 +1270,7 @@
           },
           "metrics": [
             {
-              "field": "block_bytes_written",
+              "field": "garage_block_bytes_written",
               "hide": true,
               "id": "1",
               "type": "sum"
@@ -1386,7 +1386,7 @@
           },
           "metrics": [
             {
-              "field": "block_resync_counter",
+              "field": "garage_block_resync_count",
               "hide": true,
               "id": "1",
               "type": "sum"
@@ -1500,7 +1500,7 @@
           "hide": false,
           "metrics": [
             {
-              "field": "block_resync_queue_length",
+              "field": "garage_block_resync_queue_length",
               "id": "1",
               "type": "avg"
             }
@@ -1610,7 +1610,7 @@
           },
           "metrics": [
             {
-              "field": "table_merkle_updater_todo_queue_length",
+              "field": "garage_table_merkle_updater_todo_queue_length",
               "id": "1",
               "type": "avg"
             }
@@ -1724,7 +1724,7 @@
           },
           "metrics": [
             {
-              "field": "table_gc_todo_queue_length",
+              "field": "garage_table_gc_todo_queue_length",
               "id": "1",
               "type": "avg"
             }
@@ -1824,7 +1824,7 @@
           },
           "metrics": [
             {
-              "field": "block_resync_error_counter",
+              "field": "garage_block_resync_error_count",
               "hide": true,
               "id": "1",
               "settings": {},
@@ -1938,7 +1938,7 @@
           },
           "metrics": [
             {
-              "field": "block_resync_errored_blocks",
+              "field": "garage_block_resync_errored_blocks",
               "hide": false,
               "id": "1",
               "type": "sum"
@@ -2041,7 +2041,7 @@
           },
           "metrics": [
             {
-              "field": "block_corruption_counter",
+              "field": "garage_block_corruption_count",
               "hide": true,
               "id": "1",
               "type": "sum"
@@ -2165,7 +2165,7 @@
           },
           "metrics": [
             {
-              "field": "rpc_netapp_error_counter",
+              "field": "garage_rpc_netapp_error_count",
               "hide": true,
               "id": "1",
               "type": "sum"
@@ -2292,7 +2292,7 @@
           },
           "metrics": [
             {
-              "field": "rpc_request_counter",
+              "field": "garage_rpc_request_count",
               "hide": true,
               "id": "1",
               "type": "sum"
@@ -2418,7 +2418,7 @@
           },
           "metrics": [
             {
-              "field": "rpc_duration",
+              "field": "garage_rpc_duration",
               "id": "1",
               "type": "avg"
             }
@@ -2521,7 +2521,7 @@
           },
           "metrics": [
             {
-              "field": "admin_http_requests_total",
+              "field": "garage_admin_http_requests_total",
               "hide": true,
               "id": "1",
               "type": "sum"
@@ -2654,7 +2654,7 @@
           },
           "metrics": [
             {
-              "field": "rpc_garage_error_counter",
+              "field": "garage_rpc_garage_error_count",
               "hide": true,
               "id": "1",
               "type": "sum"
@@ -2765,7 +2765,7 @@
           },
           "metrics": [
             {
-              "field": "rpc_duration",
+              "field": "garage_rpc_duration",
               "id": "1",
               "type": "avg"
             }
@@ -2995,4 +2995,4 @@
   "uid": "ODT8K4B7e",
   "version": 7,
   "weekStart": ""
-}
+}

script/telemetry/grafana-garage-dashboard-prometheus.json

@@ -143,7 +143,7 @@
             "uid": "${DS_DS_PROMETHEUS}"
           },
           "exemplar": true,
-          "expr": "sum(rate(block_bytes_read{job=\"garage\"}[$__rate_interval])    )",
+          "expr": "sum(rate(garage_block_bytes_read{job=\"garage\"}[$__rate_interval])    )",
           "hide": false,
           "interval": "",
           "legendFormat": "Disk bytes read",
@@ -155,7 +155,7 @@
             "uid": "${DS_DS_PROMETHEUS}"
           },
           "exemplar": true,
-          "expr": "-sum(rate(block_bytes_written{job=\"garage\"}[$__rate_interval])    )",
+          "expr": "-sum(rate(garage_block_bytes_written{job=\"garage\"}[$__rate_interval])    )",
           "hide": false,
           "interval": "",
           "legendFormat": "Disk bytes written",
@@ -250,7 +250,7 @@
           },
           "editorMode": "code",
           "exemplar": true,
-          "expr": "sum by (api_endpoint) (rate(api_s3_request_counter {job=\"garage\"}[$__rate_interval]))",
+          "expr": "sum by (api_endpoint) (rate(garage_api_s3_request_count {job=\"garage\"}[$__rate_interval]))",
           "hide": false,
           "interval": "",
           "legendFormat": "{{api_endpoint}}",
@@ -345,7 +345,7 @@
             "uid": "${DS_DS_PROMETHEUS}"
           },
           "exemplar": true,
-          "expr": "sum(rate(web_request_counter {job=\"garage\"}[$__rate_interval]))",
+          "expr": "sum(rate(garage_web_request_count {job=\"garage\"}[$__rate_interval]))",
           "hide": false,
           "interval": "",
           "legendFormat": "Web request rate",
@@ -439,7 +439,7 @@
             "uid": "${DS_DS_PROMETHEUS}"
           },
           "exemplar": true,
-          "expr": "sum by (rpc_endpoint) (rate(rpc_request_counter {job=\"garage\"}[$__rate_interval]))",
+          "expr": "sum by (rpc_endpoint) (rate(garage_rpc_request_count {job=\"garage\"}[$__rate_interval]))",
           "hide": false,
           "interval": "",
           "legendFormat": "{{rpc_endpoint}}",
@@ -534,7 +534,7 @@
           },
           "editorMode": "code",
           "exemplar": true,
-          "expr": "sum by (api_endpoint, status_code) (rate(api_s3_error_counter {job=\"garage\"}[$__rate_interval]))",
+          "expr": "sum by (api_endpoint, status_code) (rate(garage_api_s3_error_count {job=\"garage\"}[$__rate_interval]))",
           "hide": false,
           "interval": "",
           "legendFormat": "{{api_endpoint}} {{status_code}}",
@@ -629,7 +629,7 @@
             "uid": "${DS_DS_PROMETHEUS}"
           },
           "exemplar": true,
-          "expr": "sum by(status_code) (rate(web_error_counter {job=\"garage\"}[$__rate_interval]))",
+          "expr": "sum by(status_code) (rate(garage_web_error_count {job=\"garage\"}[$__rate_interval]))",
           "hide": false,
           "interval": "",
           "legendFormat": "{{status_code}}",
@@ -722,7 +722,7 @@
             "uid": "${DS_DS_PROMETHEUS}"
           },
           "exemplar": true,
-          "expr": "block_resync_queue_length{job=\"garage\"}",
+          "expr": "garage_block_resync_queue_length{job=\"garage\"}",
           "interval": "",
           "legendFormat": "{{instance}}",
           "refId": "A"
@@ -814,7 +814,7 @@
             "uid": "${DS_DS_PROMETHEUS}"
           },
           "exemplar": true,
-          "expr": "sum by(table_name) (table_gc_todo_queue_length{job=\"garage\"})",
+          "expr": "sum by(table_name) (garage_table_gc_todo_queue_length{job=\"garage\"})",
           "interval": "",
           "legendFormat": "{{ table_name}}",
           "refId": "A"
@@ -906,7 +906,7 @@
             "uid": "${DS_DS_PROMETHEUS}"
           },
           "exemplar": true,
-          "expr": "sum by(table_name) (table_merkle_updater_todo_queue_length{job=\"garage\"})",
+          "expr": "sum by(table_name) (garage_table_merkle_updater_todo_queue_length{job=\"garage\"})",
           "interval": "",
           "legendFormat": "{{ table_name}}",
           "refId": "A"
@@ -998,7 +998,7 @@
             "uid": "${DS_DS_PROMETHEUS}"
           },
           "exemplar": true,
-          "expr": "block_resync_errored_blocks{job=\"garage\"}",
+          "expr": "garage_block_resync_errored_blocks{job=\"garage\"}",
           "interval": "",
           "legendFormat": "{{instance}}",
           "refId": "A"
@@ -1025,4 +1025,4 @@
   "uid": "ys3pnpZ4k",
   "version": 26,
   "weekStart": ""
-}
+}

src/api/common/generic_server.rs

@@ -84,21 +84,21 @@ impl<A: ApiHandler> ApiServer<A> {
 			region,
 			api_handler,
 			request_counter: meter
-				.u64_counter(format!("api.{}.request_counter", A::API_NAME))
+				.u64_counter(format!("garage_api.{}.request_count", A::API_NAME))
 				.with_description(format!(
 					"Number of API calls to the various {} API endpoints",
 					A::API_NAME_DISPLAY
 				))
 				.init(),
 			error_counter: meter
-				.u64_counter(format!("api.{}.error_counter", A::API_NAME))
+				.u64_counter(format!("garage_api.{}.error_count", A::API_NAME))
 				.with_description(format!(
 					"Number of API calls to the various {} API endpoints that resulted in errors",
 					A::API_NAME_DISPLAY
 				))
 				.init(),
 			request_duration: meter
-				.f64_value_recorder(format!("api.{}.request_duration", A::API_NAME))
+				.f64_value_recorder(format!("garage_api.{}.request_duration", A::API_NAME))
 				.with_description(format!(
 					"Duration of API calls to the various {} API endpoints",
 					A::API_NAME_DISPLAY

src/api/s3/router.rs

@@ -315,7 +315,11 @@ impl Endpoint {
 		bucket: Option<String>,
 	) -> Result<(Self, Option<String>), Error> {
 		let uri = req.uri();
-		let path = uri.path().trim_start_matches('/');
+		let path = uri.path().strip_prefix('/');
+		if path.is_none() {
+			return Err(Error::bad_request("URI path must start with a '/'"));
+		}
+		let path = path.unwrap();
 		let query = uri.query();
 		if bucket.is_none() && path.is_empty() {
 			if *req.method() == Method::OPTIONS {
@@ -329,7 +333,7 @@ impl Endpoint {
 			(bucket, path)
 		} else {
 			path.split_once('/')
-				.map(|(b, p)| (b.to_owned(), p.trim_start_matches('/')))
+				.map(|(b, p)| (b.to_owned(), p))
 				.unwrap_or_else(|| (path.to_owned(), ""))
 		};
 
@@ -843,6 +847,40 @@ mod tests {
 			"&+?%é/something"
 		);
 
+		// A double-slash in the URL means the key begins with '/'.
+		// path-style: HEAD /bucket// → key "/"
+		assert_eq!(
+			parse("HEAD", "/my_bucket//", None, None)
+				.0
+				.get_key()
+				.unwrap(),
+			"/"
+		);
+		// virtual-hosted-style: HEAD // → key "/"
+		assert_eq!(
+			parse("HEAD", "//", Some("my_bucket".to_owned()), None)
+				.0
+				.get_key()
+				.unwrap(),
+			"/"
+		);
+		// same for GET: path-style GET /bucket// → key "/"
+		assert_eq!(
+			parse("GET", "/my_bucket//", None, None)
+				.0
+				.get_key()
+				.unwrap(),
+			"/"
+		);
+		// virtual-hosted-style: GET // → key "/"
+		assert_eq!(
+			parse("GET", "//", Some("my_bucket".to_owned()), None)
+				.0
+				.get_key()
+				.unwrap(),
+			"/"
+		);
+
 		/*
 		 * this case is failing. We should verify how clients encode space in url
 		assert_eq!(
@@ -933,6 +971,7 @@ mod tests {
 			GET "/{Key+}?torrent" => GetObjectTorrent
 			GET "/?publicAccessBlock" => GetPublicAccessBlock
 			HEAD "/" => HeadBucket
+			HEAD "//" => HeadObject
 			HEAD "/my-image.jpg" => HeadObject
 			HEAD "/my-image.jpg?versionId=3HL4kqCxf3vjVBH40Nrjfkd" => HeadObject
 			HEAD "/Key+?partNumber=3&versionId=VersionId" => HeadObject
@@ -949,6 +988,7 @@ mod tests {
 			GET "/?uploads&delimiter=/&prefix=photos/2006/" => ListMultipartUploads
 			GET "/?uploads&delimiter=D&encoding-type=EncodingType&key-marker=KeyMarker&max-uploads=1&prefix=Prefix&upload-id-marker=UploadIdMarker" => ListMultipartUploads
 			GET "/" => ListObjects
+			GET "//" => GetObject
 			GET "/?prefix=N&marker=Need&max-keys=40" => ListObjects
 			GET "/?delimiter=/" => ListObjects
 			GET "/?prefix=photos/2006/&delimiter=/" => ListObjects

src/block/metrics.rs

@@ -41,7 +41,7 @@ impl BlockManagerMetrics {
 		let meter = global::meter("garage_model/block");
 		Self {
 			_compression_level: meter
-				.u64_value_observer("block.compression_level", move |observer| {
+				.u64_value_observer("garage_block.compression_level", move |observer| {
 					match compression_level {
 						Some(v) => observer.observe(v as u64, &[]),
 						None => observer.observe(0_u64, &[]),
@@ -50,7 +50,7 @@ impl BlockManagerMetrics {
 				.with_description("Garage compression level for node")
 				.init(),
 			_rc_size: meter
-				.u64_value_observer("block.rc_size", move |observer| {
+				.u64_value_observer("garage_block.rc_size", move |observer| {
 					if let Ok(value) = rc_tree.approximate_len() {
 						observer.observe(value as u64, &[]);
 					}
@@ -58,7 +58,7 @@ impl BlockManagerMetrics {
 				.with_description("Number of blocks known to the reference counter")
 				.init(),
 			_resync_queue_len: meter
-				.u64_value_observer("block.resync_queue_length", move |observer| {
+				.u64_value_observer("garage_block.resync_queue_length", move |observer| {
 					if let Ok(value) = resync_queue.approximate_len() {
 						observer.observe(value as u64, &[]);
 					}
@@ -68,7 +68,7 @@ impl BlockManagerMetrics {
 				)
 				.init(),
 			_resync_errored_blocks: meter
-				.u64_value_observer("block.resync_errored_blocks", move |observer| {
+				.u64_value_observer("garage_block.resync_errored_blocks", move |observer| {
 					if let Ok(value) = resync_errors.approximate_len() {
 						observer.observe(value as u64, &[]);
 					}
@@ -77,7 +77,7 @@ impl BlockManagerMetrics {
 				.init(),
 
 			_buffer_free_kb: meter
-				.u64_value_observer("block.ram_buffer_free_kb", move |observer| {
+				.u64_value_observer("garage_block.ram_buffer_free_kb", move |observer| {
 					observer.observe(buffer_semaphore.available_permits() as u64, &[]);
 				})
 				.with_description(
@@ -86,63 +86,63 @@ impl BlockManagerMetrics {
 				.init(),
 
 			resync_counter: meter
-				.u64_counter("block.resync_counter")
+				.u64_counter("garage_block.resync_count")
 				.with_description("Number of calls to resync_block")
 				.init()
 				.bind(&[]),
 			resync_error_counter: meter
-				.u64_counter("block.resync_error_counter")
+				.u64_counter("garage_block.resync_error_count")
 				.with_description("Number of calls to resync_block that returned an error")
 				.init()
 				.bind(&[]),
 			resync_duration: meter
-				.f64_value_recorder("block.resync_duration")
+				.f64_value_recorder("garage_block.resync_duration")
 				.with_description("Duration of resync_block operations")
 				.init()
 				.bind(&[]),
 			resync_send_counter: meter
-				.u64_counter("block.resync_send_counter")
+				.u64_counter("garage_block.resync_send_count")
 				.with_description("Number of blocks sent to another node in resync operations")
 				.init(),
 			resync_recv_counter: meter
-				.u64_counter("block.resync_recv_counter")
+				.u64_counter("garage_block.resync_recv_count")
 				.with_description("Number of blocks received from other nodes in resync operations")
 				.init()
 				.bind(&[]),
 
 			bytes_read: meter
-				.u64_counter("block.bytes_read")
+				.u64_counter("garage_block.bytes_read")
 				.with_description("Number of bytes read from disk")
 				.init()
 				.bind(&[]),
 			block_read_duration: meter
-				.f64_value_recorder("block.read_duration")
+				.f64_value_recorder("garage_block.read_duration")
 				.with_description("Duration of block read operations")
 				.init()
 				.bind(&[]),
 			block_read_semaphore_timeouts: meter
-				.u64_counter("block.read_semaphore_timeouts")
+				.u64_counter("garage_block.read_semaphore_timeouts")
 				.with_description("Number of block reads that failed due to semaphore acquire timeout")
 				.init()
 				.bind(&[]),
 			bytes_written: meter
-				.u64_counter("block.bytes_written")
+				.u64_counter("garage_block.bytes_written")
 				.with_description("Number of bytes written to disk")
 				.init()
 				.bind(&[]),
 			block_write_duration: meter
-				.f64_value_recorder("block.write_duration")
+				.f64_value_recorder("garage_block.write_duration")
 				.with_description("Duration of block write operations")
 				.init()
 				.bind(&[]),
 			delete_counter: meter
-				.u64_counter("block.delete_counter")
+				.u64_counter("garage_block.delete_count")
 				.with_description("Number of blocks deleted")
 				.init()
 				.bind(&[]),
 
 			corruption_counter: meter
-				.u64_counter("block.corruption_counter")
+				.u64_counter("garage_block.corruption_count")
 				.with_description("Data corruptions detected on block reads")
 				.init()
 				.bind(&[]),

src/garage/tests/common/garage.rs

@@ -194,6 +194,15 @@ api_bind_addr = "127.0.0.1:{admin_port}"
 			.expect("Could not build garage endpoint URI")
 	}
 
+	pub fn admin_uri(&self, path: &str) -> http::Uri {
+		format!(
+			"http://127.0.0.1:{admin_port}/{path}",
+			admin_port = self.admin_port,
+		)
+		.parse()
+		.expect("Could not build garage endpoint URI")
+	}
+
 	pub fn key(&self, maybe_name: Option<&str>) -> Key {
 		let mut key = Key::default();
 

src/garage/tests/lib.rs

@@ -4,6 +4,9 @@ mod common;
 mod admin;
 mod bucket;
 
+#[cfg(feature = "metrics")]
+mod metrics;
+
 mod s3;
 
 #[cfg(feature = "k2v")]

src/garage/tests/metrics.rs

@@ -0,0 +1,49 @@
+use bytes::Bytes;
+use http::{Request, StatusCode};
+use http_body_util::{BodyExt, Full};
+
+use crate::common;
+
+#[tokio::test]
+async fn check_metrics_name() {
+	let ctx = common::context();
+
+	let req_url = ctx.garage.admin_uri("metrics");
+	let client = ctx.custom_request.client();
+	let get_metrics_req = Request::builder()
+		.method("GET")
+		.uri(req_url)
+		.body(Full::new(Bytes::new()))
+		.unwrap();
+
+	let response = client
+		.request(get_metrics_req)
+		.await
+		.expect("failed to build 'get metrics' request");
+
+	assert_eq!(response.status(), StatusCode::OK);
+	let body = BodyExt::collect(response.into_body())
+		.await
+		.expect("failed to collect bytes from body stream")
+		.to_bytes();
+	let body = String::from_utf8_lossy(&body);
+
+	//dbg!(&body);
+	let invalid_metrics_name = body
+		.lines()
+		.filter(isnot_comment_line) // skip the comment lines
+		.filter(hasnt_prefix_garage)
+		.collect::<Vec<_>>();
+
+	if !invalid_metrics_name.is_empty() {
+		panic!("metrics name should all start with 'garage_' prefix.\nDoc: https://prometheus.io/docs/practices/naming/#metric-names\n\nInvalid:\n{:#?}", invalid_metrics_name);
+	}
+}
+
+fn isnot_comment_line(line: &&str) -> bool {
+	!line.starts_with("#")
+}
+
+fn hasnt_prefix_garage(line: &&str) -> bool {
+	!line.starts_with("garage_")
+}

src/rpc/metrics.rs

@@ -14,23 +14,23 @@ impl RpcMetrics {
 		let meter = global::meter("garage_rpc");
 		RpcMetrics {
 			rpc_counter: meter
-				.u64_counter("rpc.request_counter")
+				.u64_counter("garage_rpc.request_count")
 				.with_description("Number of RPC requests emitted")
 				.init(),
 			rpc_timeout_counter: meter
-				.u64_counter("rpc.timeout_counter")
+				.u64_counter("garage_rpc.timeout_count")
 				.with_description("Number of RPC timeouts")
 				.init(),
 			rpc_netapp_error_counter: meter
-				.u64_counter("rpc.netapp_error_counter")
+				.u64_counter("garage_rpc.netapp_error_count")
 				.with_description("Number of communication errors (errors in the Netapp library)")
 				.init(),
 			rpc_garage_error_counter: meter
-				.u64_counter("rpc.garage_error_counter")
+				.u64_counter("garage_rpc.garage_error_count")
 				.with_description("Number of RPC errors (errors happening when handling the RPC)")
 				.init(),
 			rpc_duration: meter
-				.f64_value_recorder("rpc.duration")
+				.f64_value_recorder("garage_rpc.duration")
 				.with_description("Duration of RPCs")
 				.init(),
 		}

src/rpc/system_metrics.rs

@@ -110,7 +110,7 @@ impl SystemMetrics {
 			_cluster_healthy: {
 				let get_health = get_health.clone();
 				meter
-					.u64_value_observer("cluster_healthy", move |observer| {
+					.u64_value_observer("garage_cluster_healthy", move |observer| {
 						let h = get_health();
 						if h.status == ClusterHealthStatus::Healthy {
 							observer.observe(1, &[]);
@@ -123,7 +123,7 @@ impl SystemMetrics {
 			},
 			_cluster_available: {
 				let get_health = get_health.clone();
-				meter.u64_value_observer("cluster_available", move |observer| {
+				meter.u64_value_observer("garage_cluster_available", move |observer| {
 					let h = get_health();
 					if h.status != ClusterHealthStatus::Unavailable {
 						observer.observe(1, &[]);
@@ -137,7 +137,7 @@ impl SystemMetrics {
 			_known_nodes: {
 				let get_health = get_health.clone();
 				meter
-					.u64_value_observer("cluster_known_nodes", move |observer| {
+					.u64_value_observer("garage_cluster_known_nodes", move |observer| {
 						let h = get_health();
 						observer.observe(h.known_nodes as u64, &[]);
 					})
@@ -147,7 +147,7 @@ impl SystemMetrics {
 			_connected_nodes: {
 				let get_health = get_health.clone();
 				meter
-					.u64_value_observer("cluster_connected_nodes", move |observer| {
+					.u64_value_observer("garage_cluster_connected_nodes", move |observer| {
 						let h = get_health();
 						observer.observe(h.connected_nodes as u64, &[]);
 					})
@@ -157,7 +157,7 @@ impl SystemMetrics {
 			_storage_nodes: {
 				let get_health = get_health.clone();
 				meter
-					.u64_value_observer("cluster_storage_nodes", move |observer| {
+					.u64_value_observer("garage_cluster_storage_nodes", move |observer| {
 						let h = get_health();
 						observer.observe(h.storage_nodes as u64, &[]);
 					})
@@ -167,7 +167,7 @@ impl SystemMetrics {
 			_storage_nodes_ok: {
 				let get_health = get_health.clone();
 				meter
-					.u64_value_observer("cluster_storage_nodes_ok", move |observer| {
+					.u64_value_observer("garage_cluster_storage_nodes_ok", move |observer| {
 						let h = get_health();
 						observer.observe(h.storage_nodes_ok as u64, &[]);
 					})
@@ -177,7 +177,7 @@ impl SystemMetrics {
 			_partitions: {
 				let get_health = get_health.clone();
 				meter
-					.u64_value_observer("cluster_partitions", move |observer| {
+					.u64_value_observer("garage_cluster_partitions", move |observer| {
 						let h = get_health();
 						observer.observe(h.partitions as u64, &[]);
 					})
@@ -187,7 +187,7 @@ impl SystemMetrics {
 			_partitions_quorum: {
 				let get_health = get_health.clone();
 				meter
-					.u64_value_observer("cluster_partitions_quorum", move |observer| {
+					.u64_value_observer("garage_cluster_partitions_quorum", move |observer| {
 						let h = get_health();
 						observer.observe(h.partitions_quorum as u64, &[]);
 					})
@@ -199,7 +199,7 @@ impl SystemMetrics {
 			_partitions_all_ok: {
 				let get_health = get_health.clone();
 				meter
-					.u64_value_observer("cluster_partitions_all_ok", move |observer| {
+					.u64_value_observer("garage_cluster_partitions_all_ok", move |observer| {
 						let h = get_health();
 						observer.observe(h.partitions_all_ok as u64, &[]);
 					})
@@ -213,7 +213,7 @@ impl SystemMetrics {
 			_layout_node_connected: {
 				let system = system.clone();
 				meter
-					.u64_value_observer("cluster_layout_node_connected", move |observer| {
+					.u64_value_observer("garage_cluster_layout_node_connected", move |observer| {
 						let layout = system.cluster_layout();
 						let nodes = system.get_known_nodes();
 						for id in layout.all_nodes().unwrap_or_default().iter() {
@@ -260,44 +260,47 @@ impl SystemMetrics {
 			_layout_node_disconnected_time: {
 				let system = system.clone();
 				meter
-					.u64_value_observer("cluster_layout_node_disconnected_time", move |observer| {
+					.u64_value_observer(
-						let layout = system.cluster_layout();
+						"garage_cluster_layout_node_disconnected_time",
-						let nodes = system.get_known_nodes();
+						move |observer| {
-						for id in layout.all_nodes().unwrap_or_default().iter() {
+							let layout = system.cluster_layout();
-							let mut kv = vec![KeyValue::new("id", format!("{:?}", id))];
+							let nodes = system.get_known_nodes();
-							if let Some(role) = layout
+							for id in layout.all_nodes().unwrap_or_default().iter() {
-								.current()
+								let mut kv = vec![KeyValue::new("id", format!("{:?}", id))];
-								.ok()
+								if let Some(role) = layout
-								.and_then(|l| l.roles.get(id))
+									.current()
-								.and_then(|r| r.0.as_ref())
+									.ok()
-							{
+									.and_then(|l| l.roles.get(id))
-								kv.push(KeyValue::new("role_zone", role.zone.clone()));
+									.and_then(|r| r.0.as_ref())
-								match role.capacity {
+								{
-									Some(cap) => {
+									kv.push(KeyValue::new("role_zone", role.zone.clone()));
-										kv.push(KeyValue::new("role_capacity", cap as i64));
+									match role.capacity {
-										kv.push(KeyValue::new("role_gateway", 0));
+										Some(cap) => {
-									}
+											kv.push(KeyValue::new("role_capacity", cap as i64));
-									None => {
+											kv.push(KeyValue::new("role_gateway", 0));
-										kv.push(KeyValue::new("role_gateway", 1));
+										}
+										None => {
+											kv.push(KeyValue::new("role_gateway", 1));
+										}
 									}
 								}
-							}
 
-							if let Some(node) = nodes.iter().find(|n| n.id == *id) {
+								if let Some(node) = nodes.iter().find(|n| n.id == *id) {
-								// TODO: see comment above
+									// TODO: see comment above
-								// kv.push(KeyValue::new("address", node.addr.to_string()));
+									// kv.push(KeyValue::new("address", node.addr.to_string()));
-								// kv.push(KeyValue::new(
+									// kv.push(KeyValue::new(
-								//	"hostname",
+									//	"hostname",
-								//	node.status.hostname.clone(),
+									//	node.status.hostname.clone(),
-								// ));
+									// ));
-								if node.is_up {
+									if node.is_up {
-									observer.observe(0, &kv);
+										observer.observe(0, &kv);
-								} else if let Some(secs) = node.last_seen_secs_ago {
+									} else if let Some(secs) = node.last_seen_secs_ago {
-									observer.observe(secs, &kv);
+										observer.observe(secs, &kv);
+									}
 								}
 							}
-						}
+						},
-					})
+					)
 					.with_description(
 						"Time (in seconds) since last connection to nodes in the cluster layout",
 					)

src/table/metrics.rs

@@ -34,7 +34,7 @@ impl TableMetrics {
 		TableMetrics {
 			_table_size: meter
 				.u64_value_observer(
-					"table.size",
+					"garage_table.size",
 					move |observer| {
 						if let Ok(value) = store.approximate_len() {
 							observer.observe(
@@ -48,7 +48,7 @@ impl TableMetrics {
 				.init(),
 			_merkle_tree_size: meter
 				.u64_value_observer(
-					"table.merkle_tree_size",
+					"garage_table.merkle_tree_size",
 					move |observer| {
 						if let Ok(value) = merkle_tree.approximate_len() {
 							observer.observe(
@@ -62,7 +62,7 @@ impl TableMetrics {
 				.init(),
 			_merkle_todo_len: meter
 				.u64_value_observer(
-					"table.merkle_updater_todo_queue_length",
+					"garage_table.merkle_updater_todo_queue_length",
 					move |observer| {
 						if let Ok(v) = merkle_todo.approximate_len() {
 							observer.observe(
@@ -76,7 +76,7 @@ impl TableMetrics {
 				.init(),
 			_insert_queue_len: meter
 				.u64_value_observer(
-					"table.insert_queue_length",
+					"garage_table.insert_queue_length",
 					move |observer| {
 						if let Ok(v) = insert_queue.approximate_len() {
 							observer.observe(
@@ -90,7 +90,7 @@ impl TableMetrics {
 				.init(),
 			_gc_todo_len: meter
 				.u64_value_observer(
-					"table.gc_todo_queue_length",
+					"garage_table.gc_todo_queue_length",
 					move |observer| {
                         if let Ok(value) = gc_todo.approximate_len() {
                             observer.observe(
@@ -104,43 +104,43 @@ impl TableMetrics {
 				.init(),
 
 			get_request_counter: meter
-				.u64_counter("table.get_request_counter")
+				.u64_counter("garage_table.get_request_count")
 				.with_description("Number of get/get_range requests internally made on this table")
 				.init()
 				.bind(&[KeyValue::new("table_name", table_name)]),
 			get_request_duration: meter
-				.f64_value_recorder("table.get_request_duration")
+				.f64_value_recorder("garage_table.get_request_duration")
 				.with_description("Duration of get/get_range requests internally made on this table, in seconds")
 				.init()
 				.bind(&[KeyValue::new("table_name", table_name)]),
 			put_request_counter: meter
-				.u64_counter("table.put_request_counter")
+				.u64_counter("garage_table.put_request_count")
 				.with_description("Number of insert/insert_many requests internally made on this table")
 				.init()
 				.bind(&[KeyValue::new("table_name", table_name)]),
 			put_request_duration: meter
-				.f64_value_recorder("table.put_request_duration")
+				.f64_value_recorder("garage_table.put_request_duration")
 				.with_description("Duration of insert/insert_many requests internally made on this table, in seconds")
 				.init()
 				.bind(&[KeyValue::new("table_name", table_name)]),
 
 			internal_update_counter: meter
-				.u64_counter("table.internal_update_counter")
+				.u64_counter("garage_table.internal_update_count")
 				.with_description("Number of value updates where the value actually changes (includes creation of new key and update of existing key)")
 				.init()
 				.bind(&[KeyValue::new("table_name", table_name)]),
 			internal_delete_counter: meter
-				.u64_counter("table.internal_delete_counter")
+				.u64_counter("garage_table.internal_delete_count")
 				.with_description("Number of value deletions in the tree (due to GC or repartitioning)")
 				.init()
 				.bind(&[KeyValue::new("table_name", table_name)]),
 
 			sync_items_sent: meter
-				.u64_counter("table.sync_items_sent")
+				.u64_counter("garage_table.sync_items_sent")
 				.with_description("Number of data items sent to other nodes during resync procedures")
 				.init(),
 			sync_items_received: meter
-				.u64_counter("table.sync_items_received")
+				.u64_counter("garage_table.sync_items_received")
 				.with_description("Number of data items received from other nodes during resync procedures")
 				.init(),
 		}

src/web/web_server.rs

@@ -54,15 +54,15 @@ impl WebMetrics {
 		let meter = global::meter("garage/web");
 		Self {
 			request_counter: meter
-				.u64_counter("web.request_counter")
+				.u64_counter("garage_web.request_count")
 				.with_description("Number of requests to the web endpoint")
 				.init(),
 			error_counter: meter
-				.u64_counter("web.error_counter")
+				.u64_counter("garage_web.error_count")
 				.with_description("Number of requests to the web endpoint resulting in errors")
 				.init(),
 			request_duration: meter
-				.f64_value_recorder("web.request_duration")
+				.f64_value_recorder("garage_web.request_duration")
 				.with_description("Duration of requests to the web endpoint")
 				.init(),
 		}