From fad82751b90902bf66664e3914695cf9818cae6d Mon Sep 17 00:00:00 2001 From: Gwen Lg Date: Fri, 13 Feb 2026 20:02:32 +0100 Subject: [PATCH] chore: add `garage_` prefix for metrics who didn't have it update: - monitoring doc - grafana dashboard elasticsearch.json --- doc/book/reference-manual/admin-api.md | 18 +- doc/book/reference-manual/monitoring.md | 214 +++++++++--------- ...rafana-garage-dashboard-elasticsearch.json | 46 ++-- .../grafana-garage-dashboard-prometheus.json | 24 +- src/api/common/generic_server.rs | 6 +- src/block/metrics.rs | 34 +-- src/rpc/metrics.rs | 10 +- src/rpc/system_metrics.rs | 89 ++++---- src/table/metrics.rs | 26 +-- src/web/web_server.rs | 6 +- 10 files changed, 237 insertions(+), 236 deletions(-) diff --git a/doc/book/reference-manual/admin-api.md b/doc/book/reference-manual/admin-api.md index e96fcaff..fb860d7e 100644 --- a/doc/book/reference-manual/admin-api.md +++ b/doc/book/reference-manual/admin-api.md @@ -182,15 +182,15 @@ content-type: text/plain; version=0.0.4 content-length: 12145 date: Tue, 08 Aug 2023 07:25:05 GMT -# HELP api_admin_error_counter Number of API calls to the various Admin API endpoints that resulted in errors -# TYPE api_admin_error_counter counter -api_admin_error_counter{api_endpoint="CheckWebsiteEnabled",status_code="400"} 1 -api_admin_error_counter{api_endpoint="CheckWebsiteEnabled",status_code="404"} 3 -# HELP api_admin_request_counter Number of API calls to the various Admin API endpoints -# TYPE api_admin_request_counter counter -api_admin_request_counter{api_endpoint="CheckWebsiteEnabled"} 7 -api_admin_request_counter{api_endpoint="Health"} 3 -# HELP api_admin_request_duration Duration of API calls to the various Admin API endpoints +# HELP garage_api_admin_error_counter Number of API calls to the various Admin API endpoints that resulted in errors +# TYPE garage_api_admin_error_counter counter +garage_api_admin_error_counter{api_endpoint="CheckWebsiteEnabled",status_code="400"} 1 +garage_api_admin_error_counter{api_endpoint="CheckWebsiteEnabled",status_code="404"} 3 +# HELP garage_api_admin_request_counter Number of API calls to the various Admin API endpoints +# TYPE garage_api_admin_request_counter counter +garage_api_admin_request_counter{api_endpoint="CheckWebsiteEnabled"} 7 +garage_api_admin_request_counter{api_endpoint="Health"} 3 +# HELP garage_api_admin_request_duration Duration of API calls to the various Admin API endpoints ... ``` diff --git a/doc/book/reference-manual/monitoring.md b/doc/book/reference-manual/monitoring.md index 4696791e..6e3fea05 100644 --- a/doc/book/reference-manual/monitoring.md +++ b/doc/book/reference-manual/monitoring.md @@ -40,146 +40,146 @@ garage_local_disk_total{volume="metadata"} 763063566336 ### Cluster health status metrics -#### `cluster_healthy` (gauge) +#### `garage_cluster_healthy` (gauge) Whether all storage nodes are connected (0 or 1) ``` -cluster_healthy 0 +garage_cluster_healthy 0 ``` -#### `cluster_available` (gauge) +#### `garage_cluster_available` (gauge) Whether all requests can be served, even if some storage nodes are disconnected ``` -cluster_available 1 +garage_cluster_available 1 ``` -#### `cluster_connected_nodes` (gauge) +#### `garage_cluster_connected_nodes` (gauge) Number of nodes currently connected ``` -cluster_connected_nodes 3 +garage_cluster_connected_nodes 3 ``` -#### `cluster_known_nodes` (gauge) +#### `garage_cluster_known_nodes` (gauge) Number of nodes already seen once in the cluster ``` -cluster_known_nodes 3 +garage_cluster_known_nodes 3 ``` -#### `cluster_layout_node_connected` (gauge) +#### `garage_cluster_layout_node_connected` (gauge) Connection status for individual nodes of the cluster layout ``` -cluster_layout_node_connected{id="62b218d848e86a64",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 1 -cluster_layout_node_connected{id="a11c7cf18af29737",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 0 -cluster_layout_node_connected{id="a235ac7695e0c54d",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 1 -cluster_layout_node_connected{id="b10c110e4e854e5a",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 1 +garage_cluster_layout_node_connected{id="62b218d848e86a64",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 1 +garage_cluster_layout_node_connected{id="a11c7cf18af29737",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 0 +garage_cluster_layout_node_connected{id="a235ac7695e0c54d",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 1 +garage_cluster_layout_node_connected{id="b10c110e4e854e5a",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 1 ``` -#### `cluster_layout_node_disconnected_time` (gauge) +#### `garage_cluster_layout_node_disconnected_time` (gauge) Time (in seconds) since last connection to individual nodes of the cluster layout ``` -cluster_layout_node_disconnected_time{id="62b218d848e86a64",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 0 -cluster_layout_node_disconnected_time{id="a235ac7695e0c54d",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 0 -cluster_layout_node_disconnected_time{id="b10c110e4e854e5a",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 0 +garage_cluster_layout_node_disconnected_time{id="62b218d848e86a64",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 0 +garage_cluster_layout_node_disconnected_time{id="a235ac7695e0c54d",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 0 +garage_cluster_layout_node_disconnected_time{id="b10c110e4e854e5a",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 0 ``` -#### `cluster_storage_nodes` (gauge) +#### `garage_cluster_storage_nodes` (gauge) Number of storage nodes declared in the current layout ``` -cluster_storage_nodes 4 +garage_cluster_storage_nodes 4 ``` -#### `cluster_storage_nodes_ok` (gauge) +#### `garage_cluster_storage_nodes_ok` (gauge) Number of storage nodes currently connected ``` -cluster_storage_nodes_ok 3 +garage_cluster_storage_nodes_ok 3 ``` -#### `cluster_partitions` (gauge) +#### `garage_cluster_partitions` (gauge) Number of partitions in the layout (this is always 256) ``` -cluster_partitions 256 +garage_cluster_partitions 256 ``` -#### `cluster_partitions_all_ok` (gauge) +#### `garage_cluster_partitions_all_ok` (gauge) Number of partitions for which all storage nodes are connected ``` -cluster_partitions_all_ok 64 +garage_cluster_partitions_all_ok 64 ``` -#### `cluster_partitions_quorum` (gauge) +#### `garage_cluster_partitions_quorum` (gauge) Number of partitions for which we have a quorum of connected nodes and all requests can be served ``` -cluster_partitions_quorum 256 +garage_cluster_partitions_quorum 256 ``` ### Metrics of the API endpoints -#### `api_admin_request_counter` (counter) +#### `garage_api_admin_request_counter` (counter) Counts the number of requests to a given endpoint of the administration API. Example: ``` -api_admin_request_counter{api_endpoint="Metrics"} 127041 +garage_api_admin_request_counter{api_endpoint="Metrics"} 127041 ``` -#### `api_admin_request_duration` (histogram) +#### `garage_api_admin_request_duration` (histogram) Evaluates the duration of API calls to the various administration API endpoint. Example: ``` -api_admin_request_duration_bucket{api_endpoint="Metrics",le="0.5"} 127041 -api_admin_request_duration_sum{api_endpoint="Metrics"} 605.250344830999 -api_admin_request_duration_count{api_endpoint="Metrics"} 127041 +garage_api_admin_request_duration_bucket{api_endpoint="Metrics",le="0.5"} 127041 +garage_api_admin_request_duration_sum{api_endpoint="Metrics"} 605.250344830999 +garage_api_admin_request_duration_count{api_endpoint="Metrics"} 127041 ``` -#### `api_s3_request_counter` (counter) +#### `garage_api_s3_request_counter` (counter) Counts the number of requests to a given endpoint of the S3 API. Example: ``` -api_s3_request_counter{api_endpoint="CreateMultipartUpload"} 1 +garage_api_s3_request_counter{api_endpoint="CreateMultipartUpload"} 1 ``` -#### `api_s3_error_counter` (counter) +#### `garage_api_s3_error_counter` (counter) Counts the number of requests to a given endpoint of the S3 API that returned an error. Example: ``` -api_s3_error_counter{api_endpoint="GetObject",status_code="404"} 39 +garage_api_s3_error_counter{api_endpoint="GetObject",status_code="404"} 39 ``` -#### `api_s3_request_duration` (histogram) +#### `garage_api_s3_request_duration` (histogram) Evaluates the duration of API calls to the various S3 API endpoints. Example: ``` -api_s3_request_duration_bucket{api_endpoint="CreateMultipartUpload",le="0.5"} 1 -api_s3_request_duration_sum{api_endpoint="CreateMultipartUpload"} 0.046340762 -api_s3_request_duration_count{api_endpoint="CreateMultipartUpload"} 1 +garage_api_s3_request_duration_bucket{api_endpoint="CreateMultipartUpload",le="0.5"} 1 +garage_api_s3_request_duration_sum{api_endpoint="CreateMultipartUpload"} 0.046340762 +garage_api_s3_request_duration_count{api_endpoint="CreateMultipartUpload"} 1 ``` -#### `api_k2v_request_counter` (counter), `api_k2v_error_counter` (counter), `api_k2v_error_duration` (histogram) +#### `garage_api_k2v_request_counter` (counter), `garage_api_k2v_error_counter` (counter), `garage_api_k2v_error_duration` (histogram) Same as for S3, for the K2V API. @@ -187,45 +187,45 @@ Same as for S3, for the K2V API. ### Metrics of the Web endpoint -#### `web_request_counter` (counter) +#### `garage_web_request_counter` (counter) Number of requests to the web endpoint ``` -web_request_counter{method="GET"} 80 +garage_web_request_counter{method="GET"} 80 ``` -#### `web_request_duration` (histogram) +#### `garage_web_request_duration` (histogram) Duration of requests to the web endpoint ``` -web_request_duration_bucket{method="GET",le="0.5"} 80 -web_request_duration_sum{method="GET"} 1.0528433229999998 -web_request_duration_count{method="GET"} 80 +garage_web_request_duration_bucket{method="GET",le="0.5"} 80 +garage_web_request_duration_sum{method="GET"} 1.0528433229999998 +garage_web_request_duration_count{method="GET"} 80 ``` -#### `web_error_counter` (counter) +#### `garage_web_error_counter` (counter) Number of requests to the web endpoint resulting in errors ``` -web_error_counter{method="GET",status_code="404 Not Found"} 64 +garage_web_error_counter{method="GET",status_code="404 Not Found"} 64 ``` ### Metrics of the data block manager -#### `block_bytes_read`, `block_bytes_written` (counter) +#### `garage_block_bytes_read`, `garage_block_bytes_written` (counter) Number of bytes read/written to/from disk in the data storage directory. ``` -block_bytes_read 120586322022 -block_bytes_written 3386618077 +garage_block_bytes_read 120586322022 +garage_block_bytes_written 3386618077 ``` -#### `block_ram_buffer_free_kb` (gauge) +#### `garage_block_ram_buffer_free_kb` (gauge) Kibibytes available for buffering blocks that have to be sent to remote nodes. When clients send too much data to this node and a storage node is not receiving @@ -233,170 +233,168 @@ data fast enough due to slower network conditions, this will decrease down to zero and backpressure will be applied. ``` -block_ram_buffer_free_kb 219829 +garage_block_ram_buffer_free_kb 219829 ``` -#### `block_compression_level` (counter) +#### `garage_block_compression_level` (counter) Exposes the block compression level configured for the Garage node. ``` -block_compression_level 3 +garage_block_compression_level 3 ``` -#### `block_read_duration`, `block_write_duration` (histograms) +#### `garage_block_read_duration`, `garage_block_write_duration` (histograms) Evaluates the duration of the reading/writing of individual data blocks in the data storage directory. ``` -block_read_duration_bucket{le="0.5"} 169229 -block_read_duration_sum 2761.6902550310056 -block_read_duration_count 169240 -block_write_duration_bucket{le="0.5"} 3559 -block_write_duration_sum 195.59170078500006 -block_write_duration_count 3571 +garage_block_read_duration_bucket{le="0.5"} 169229 +garage_block_read_duration_sum 2761.6902550310056 +garage_block_read_duration_count 169240 +garage_block_write_duration_bucket{le="0.5"} 3559 +garage_block_write_duration_sum 195.59170078500006 +garage_block_write_duration_count 3571 ``` -#### `block_delete_counter` (counter) +#### `garage_block_delete_counter` (counter) Counts the number of data blocks that have been deleted from storage. ``` -block_delete_counter 122 +garage_block_delete_counter 122 ``` -#### `block_resync_counter` (counter), `block_resync_duration` (histogram) +#### `garage_block_resync_counter` (counter), `garage_block_resync_duration` (histogram) Counts the number of resync operations the node has executed, and evaluates their duration. ``` -block_resync_counter 308897 -block_resync_duration_bucket{le="0.5"} 308892 -block_resync_duration_sum 139.64204196100016 -block_resync_duration_count 308897 +garage_block_resync_counter 308897 +garage_block_resync_duration_bucket{le="0.5"} 308892 +garage_block_resync_duration_sum 139.64204196100016 +garage_block_resync_duration_count 308897 ``` -#### `block_resync_queue_length` (gauge) +#### `garage_block_resync_queue_length` (gauge) The number of block hashes currently queued for a resync. This is normal to be nonzero for long periods of time. ``` -block_resync_queue_length 0 +garage_block_resync_queue_length 0 ``` -#### `block_resync_errored_blocks` (gauge) +#### `garage_block_resync_errored_blocks` (gauge) The number of block hashes that we were unable to resync last time we tried. **THIS SHOULD BE ZERO, OR FALL BACK TO ZERO RAPIDLY, IN A HEALTHY CLUSTER.** Persistent nonzero values indicate that some data is likely to be lost. ``` -block_resync_errored_blocks 0 +garage_block_resync_errored_blocks 0 ``` ### Metrics related to RPCs (remote procedure calls) between nodes -#### `rpc_netapp_request_counter` (counter) +#### `garage_rpc_netapp_request_counter` (counter) Number of RPC requests emitted ``` -rpc_request_counter{from="",rpc_endpoint="garage_block/manager.rs/Rpc",to=""} 176 +garage_rpc_request_counter{from="",rpc_endpoint="garage_block/manager.rs/Rpc",to=""} 176 ``` -#### `rpc_netapp_error_counter` (counter) +#### `garage_rpc_netapp_error_counter` (counter) Number of communication errors (errors in the Netapp library, generally due to disconnected nodes) ``` -rpc_netapp_error_counter{from="",rpc_endpoint="garage_block/manager.rs/Rpc",to=""} 354 +garage_rpc_netapp_error_counter{from="",rpc_endpoint="garage_block/manager.rs/Rpc",to=""} 354 ``` -#### `rpc_timeout_counter` (counter) +#### `garage_rpc_timeout_counter` (counter) Number of RPC timeouts, should be close to zero in a healthy cluster. ``` -rpc_timeout_counter{from="",rpc_endpoint="garage_rpc/membership.rs/SystemRpc",to=""} 1 +garage_rpc_timeout_counter{from="",rpc_endpoint="garage_rpc/membership.rs/SystemRpc",to=""} 1 ``` -#### `rpc_duration` (histogram) +#### `garage_rpc_duration` (histogram) The duration of internal RPC calls between Garage nodes. ``` -rpc_duration_bucket{from="",rpc_endpoint="garage_block/manager.rs/Rpc",to="",le="0.5"} 166 -rpc_duration_sum{from="",rpc_endpoint="garage_block/manager.rs/Rpc",to=""} 35.172253716 -rpc_duration_count{from="",rpc_endpoint="garage_block/manager.rs/Rpc",to=""} 174 +garage_rpc_duration_bucket{from="",rpc_endpoint="garage_block/manager.rs/Rpc",to="",le="0.5"} 166 +garage_rpc_duration_sum{from="",rpc_endpoint="garage_block/manager.rs/Rpc",to=""} 35.172253716 +garage_rpc_duration_count{from="",rpc_endpoint="garage_block/manager.rs/Rpc",to=""} 174 ``` ### Metrics of the metadata table manager -#### `table_gc_todo_queue_length` (gauge) +#### `garage_table_gc_todo_queue_length` (gauge) Table garbage collector TODO queue length ``` -table_gc_todo_queue_length{table_name="block_ref"} 0 +garage_table_gc_todo_queue_length{table_name="block_ref"} 0 ``` -#### `table_get_request_counter` (counter), `table_get_request_duration` (histogram) +#### `garage_table_get_request_counter` (counter), `garage_table_get_request_duration` (histogram) Number of get/get_range requests internally made on each table, and their duration. ``` -table_get_request_counter{table_name="bucket_alias"} 315 -table_get_request_duration_bucket{table_name="bucket_alias",le="0.5"} 315 -table_get_request_duration_sum{table_name="bucket_alias"} 0.048509778000000024 -table_get_request_duration_count{table_name="bucket_alias"} 315 +garage_table_get_request_counter{table_name="bucket_alias"} 315 +garage_table_get_request_duration_bucket{table_name="bucket_alias",le="0.5"} 315 +garage_table_get_request_duration_sum{table_name="bucket_alias"} 0.048509778000000024 +garage_table_get_request_duration_count{table_name="bucket_alias"} 315 ``` -#### `table_put_request_counter` (counter), `table_put_request_duration` (histogram) +#### `garage_table_put_request_counter` (counter), `garage_table_put_request_duration` (histogram) Number of insert/insert_many requests internally made on this table, and their duration ``` -table_put_request_counter{table_name="block_ref"} 677 -table_put_request_duration_bucket{table_name="block_ref",le="0.5"} 677 -table_put_request_duration_sum{table_name="block_ref"} 61.617528636 -table_put_request_duration_count{table_name="block_ref"} 677 +garage_table_put_request_counter{table_name="block_ref"} 677 +garage_table_put_request_duration_bucket{table_name="block_ref",le="0.5"} 677 +garage_table_put_request_duration_sum{table_name="block_ref"} 61.617528636 +garage_table_put_request_duration_count{table_name="block_ref"} 677 ``` -#### `table_internal_delete_counter` (counter) +#### `garage_table_internal_delete_counter` (counter) Number of value deletions in the tree (due to GC or repartitioning) ``` -table_internal_delete_counter{table_name="block_ref"} 2296 +garage_table_internal_delete_counter{table_name="block_ref"} 2296 ``` -#### `table_internal_update_counter` (counter) +#### `garage_table_internal_update_counter` (counter) Number of value updates where the value actually changes (includes creation of new key and update of existing key) ``` -table_internal_update_counter{table_name="block_ref"} 5996 +garage_table_internal_update_counter{table_name="block_ref"} 5996 ``` -#### `table_merkle_updater_todo_queue_length` (gauge) +#### `garage_table_merkle_updater_todo_queue_length` (gauge) Merkle tree updater TODO queue length (should fall to zero rapidly) ``` -table_merkle_updater_todo_queue_length{table_name="block_ref"} 0 +garage_table_merkle_updater_todo_queue_length{table_name="block_ref"} 0 ``` -#### `table_sync_items_received`, `table_sync_items_sent` (counters) +#### `garage_table_sync_items_received`, `garage_table_sync_items_sent` (counters) Number of data items sent to/received from other nodes during resync procedures ``` -table_sync_items_received{from="",table_name="bucket_v2"} 3 -table_sync_items_sent{table_name="block_ref",to=""} 2 +garage_table_sync_items_received{from="",table_name="bucket_v2"} 3 +garage_table_sync_items_sent{table_name="block_ref",to=""} 2 ``` - - diff --git a/script/telemetry/grafana-garage-dashboard-elasticsearch.json b/script/telemetry/grafana-garage-dashboard-elasticsearch.json index 456ff84d..d079c161 100644 --- a/script/telemetry/grafana-garage-dashboard-elasticsearch.json +++ b/script/telemetry/grafana-garage-dashboard-elasticsearch.json @@ -161,7 +161,7 @@ }, "metrics": [ { - "field": "api_request_counter", + "field": "garage_api_request_counter", "hide": true, "id": "1", "type": "sum" @@ -284,7 +284,7 @@ "hide": false, "metrics": [ { - "field": "api_request_duration", + "field": "garage_api_request_duration", "id": "1", "type": "avg" } @@ -412,7 +412,7 @@ }, "metrics": [ { - "field": "api_error_counter", + "field": "garage_api_error_counter", "hide": true, "id": "1", "type": "sum" @@ -540,7 +540,7 @@ }, "metrics": [ { - "field": "web_request_counter", + "field": "garage_web_request_counter", "hide": true, "id": "1", "type": "sum" @@ -666,7 +666,7 @@ "hide": false, "metrics": [ { - "field": "web_request_duration", + "field": "garage_web_request_duration", "id": "1", "type": "avg" } @@ -794,7 +794,7 @@ }, "metrics": [ { - "field": "web_error_counter", + "field": "garage_web_error_counter", "hide": true, "id": "1", "type": "sum" @@ -918,7 +918,7 @@ "hide": false, "metrics": [ { - "field": "table_get_request_counter", + "field": "garage_table_get_request_counter", "hide": true, "id": "1", "type": "sum" @@ -1042,7 +1042,7 @@ "hide": false, "metrics": [ { - "field": "table_put_request_counter", + "field": "garage_table_put_request_counter", "hide": true, "id": "1", "type": "sum" @@ -1154,7 +1154,7 @@ "hide": false, "metrics": [ { - "field": "block_bytes_read", + "field": "garage_block_bytes_read", "hide": true, "id": "1", "type": "sum" @@ -1270,7 +1270,7 @@ }, "metrics": [ { - "field": "block_bytes_written", + "field": "garage_block_bytes_written", "hide": true, "id": "1", "type": "sum" @@ -1386,7 +1386,7 @@ }, "metrics": [ { - "field": "block_resync_counter", + "field": "garage_block_resync_counter", "hide": true, "id": "1", "type": "sum" @@ -1500,7 +1500,7 @@ "hide": false, "metrics": [ { - "field": "block_resync_queue_length", + "field": "garage_block_resync_queue_length", "id": "1", "type": "avg" } @@ -1610,7 +1610,7 @@ }, "metrics": [ { - "field": "table_merkle_updater_todo_queue_length", + "field": "garage_table_merkle_updater_todo_queue_length", "id": "1", "type": "avg" } @@ -1724,7 +1724,7 @@ }, "metrics": [ { - "field": "table_gc_todo_queue_length", + "field": "garage_table_gc_todo_queue_length", "id": "1", "type": "avg" } @@ -1824,7 +1824,7 @@ }, "metrics": [ { - "field": "block_resync_error_counter", + "field": "garage_block_resync_error_counter", "hide": true, "id": "1", "settings": {}, @@ -1938,7 +1938,7 @@ }, "metrics": [ { - "field": "block_resync_errored_blocks", + "field": "garage_block_resync_errored_blocks", "hide": false, "id": "1", "type": "sum" @@ -2041,7 +2041,7 @@ }, "metrics": [ { - "field": "block_corruption_counter", + "field": "garage_block_corruption_counter", "hide": true, "id": "1", "type": "sum" @@ -2165,7 +2165,7 @@ }, "metrics": [ { - "field": "rpc_netapp_error_counter", + "field": "garage_rpc_netapp_error_counter", "hide": true, "id": "1", "type": "sum" @@ -2292,7 +2292,7 @@ }, "metrics": [ { - "field": "rpc_request_counter", + "field": "garage_rpc_request_counter", "hide": true, "id": "1", "type": "sum" @@ -2418,7 +2418,7 @@ }, "metrics": [ { - "field": "rpc_duration", + "field": "garage_rpc_duration", "id": "1", "type": "avg" } @@ -2521,7 +2521,7 @@ }, "metrics": [ { - "field": "admin_http_requests_total", + "field": "garage_admin_http_requests_total", "hide": true, "id": "1", "type": "sum" @@ -2654,7 +2654,7 @@ }, "metrics": [ { - "field": "rpc_garage_error_counter", + "field": "garage_rpc_garage_error_counter", "hide": true, "id": "1", "type": "sum" @@ -2765,7 +2765,7 @@ }, "metrics": [ { - "field": "rpc_duration", + "field": "garage_rpc_duration", "id": "1", "type": "avg" } diff --git a/script/telemetry/grafana-garage-dashboard-prometheus.json b/script/telemetry/grafana-garage-dashboard-prometheus.json index 1e127f8a..421cb4db 100644 --- a/script/telemetry/grafana-garage-dashboard-prometheus.json +++ b/script/telemetry/grafana-garage-dashboard-prometheus.json @@ -143,7 +143,7 @@ "uid": "${DS_DS_PROMETHEUS}" }, "exemplar": true, - "expr": "sum(rate(block_bytes_read{job=\"garage\"}[$__rate_interval]) )", + "expr": "sum(rate(garage_block_bytes_read{job=\"garage\"}[$__rate_interval]) )", "hide": false, "interval": "", "legendFormat": "Disk bytes read", @@ -155,7 +155,7 @@ "uid": "${DS_DS_PROMETHEUS}" }, "exemplar": true, - "expr": "-sum(rate(block_bytes_written{job=\"garage\"}[$__rate_interval]) )", + "expr": "-sum(rate(garage_block_bytes_written{job=\"garage\"}[$__rate_interval]) )", "hide": false, "interval": "", "legendFormat": "Disk bytes written", @@ -250,7 +250,7 @@ }, "editorMode": "code", "exemplar": true, - "expr": "sum by (api_endpoint) (rate(api_s3_request_counter {job=\"garage\"}[$__rate_interval]))", + "expr": "sum by (api_endpoint) (rate(garage_api_s3_request_counter {job=\"garage\"}[$__rate_interval]))", "hide": false, "interval": "", "legendFormat": "{{api_endpoint}}", @@ -345,7 +345,7 @@ "uid": "${DS_DS_PROMETHEUS}" }, "exemplar": true, - "expr": "sum(rate(web_request_counter {job=\"garage\"}[$__rate_interval]))", + "expr": "sum(rate(garage_web_request_counter {job=\"garage\"}[$__rate_interval]))", "hide": false, "interval": "", "legendFormat": "Web request rate", @@ -439,7 +439,7 @@ "uid": "${DS_DS_PROMETHEUS}" }, "exemplar": true, - "expr": "sum by (rpc_endpoint) (rate(rpc_request_counter {job=\"garage\"}[$__rate_interval]))", + "expr": "sum by (rpc_endpoint) (rate(garage_rpc_request_counter {job=\"garage\"}[$__rate_interval]))", "hide": false, "interval": "", "legendFormat": "{{rpc_endpoint}}", @@ -534,7 +534,7 @@ }, "editorMode": "code", "exemplar": true, - "expr": "sum by (api_endpoint, status_code) (rate(api_s3_error_counter {job=\"garage\"}[$__rate_interval]))", + "expr": "sum by (api_endpoint, status_code) (rate(garage_api_s3_error_counter {job=\"garage\"}[$__rate_interval]))", "hide": false, "interval": "", "legendFormat": "{{api_endpoint}} {{status_code}}", @@ -629,7 +629,7 @@ "uid": "${DS_DS_PROMETHEUS}" }, "exemplar": true, - "expr": "sum by(status_code) (rate(web_error_counter {job=\"garage\"}[$__rate_interval]))", + "expr": "sum by(status_code) (rate(garage_web_error_count {job=\"garage\"}[$__rate_interval]))", "hide": false, "interval": "", "legendFormat": "{{status_code}}", @@ -722,7 +722,7 @@ "uid": "${DS_DS_PROMETHEUS}" }, "exemplar": true, - "expr": "block_resync_queue_length{job=\"garage\"}", + "expr": "garage_block_resync_queue_length{job=\"garage\"}", "interval": "", "legendFormat": "{{instance}}", "refId": "A" @@ -814,7 +814,7 @@ "uid": "${DS_DS_PROMETHEUS}" }, "exemplar": true, - "expr": "sum by(table_name) (table_gc_todo_queue_length{job=\"garage\"})", + "expr": "sum by(table_name) (garage_table_gc_todo_queue_length{job=\"garage\"})", "interval": "", "legendFormat": "{{ table_name}}", "refId": "A" @@ -906,7 +906,7 @@ "uid": "${DS_DS_PROMETHEUS}" }, "exemplar": true, - "expr": "sum by(table_name) (table_merkle_updater_todo_queue_length{job=\"garage\"})", + "expr": "sum by(table_name) (garage_table_merkle_updater_todo_queue_length{job=\"garage\"})", "interval": "", "legendFormat": "{{ table_name}}", "refId": "A" @@ -998,7 +998,7 @@ "uid": "${DS_DS_PROMETHEUS}" }, "exemplar": true, - "expr": "block_resync_errored_blocks{job=\"garage\"}", + "expr": "garage_block_resync_errored_blocks{job=\"garage\"}", "interval": "", "legendFormat": "{{instance}}", "refId": "A" @@ -1025,4 +1025,4 @@ "uid": "ys3pnpZ4k", "version": 26, "weekStart": "" -} \ No newline at end of file +} diff --git a/src/api/common/generic_server.rs b/src/api/common/generic_server.rs index 8a09d647..87a5637d 100644 --- a/src/api/common/generic_server.rs +++ b/src/api/common/generic_server.rs @@ -84,21 +84,21 @@ impl ApiServer { region, api_handler, request_counter: meter - .u64_counter(format!("api.{}.request_counter", A::API_NAME)) + .u64_counter(format!("garage_api.{}.request_counter", A::API_NAME)) .with_description(format!( "Number of API calls to the various {} API endpoints", A::API_NAME_DISPLAY )) .init(), error_counter: meter - .u64_counter(format!("api.{}.error_counter", A::API_NAME)) + .u64_counter(format!("garage_api.{}.error_counter", A::API_NAME)) .with_description(format!( "Number of API calls to the various {} API endpoints that resulted in errors", A::API_NAME_DISPLAY )) .init(), request_duration: meter - .f64_value_recorder(format!("api.{}.request_duration", A::API_NAME)) + .f64_value_recorder(format!("garage_api.{}.request_duration", A::API_NAME)) .with_description(format!( "Duration of API calls to the various {} API endpoints", A::API_NAME_DISPLAY diff --git a/src/block/metrics.rs b/src/block/metrics.rs index 7a07018c..e2cabbd3 100644 --- a/src/block/metrics.rs +++ b/src/block/metrics.rs @@ -41,7 +41,7 @@ impl BlockManagerMetrics { let meter = global::meter("garage_model/block"); Self { _compression_level: meter - .u64_value_observer("block.compression_level", move |observer| { + .u64_value_observer("garage_block.compression_level", move |observer| { match compression_level { Some(v) => observer.observe(v as u64, &[]), None => observer.observe(0_u64, &[]), @@ -50,7 +50,7 @@ impl BlockManagerMetrics { .with_description("Garage compression level for node") .init(), _rc_size: meter - .u64_value_observer("block.rc_size", move |observer| { + .u64_value_observer("garage_block.rc_size", move |observer| { if let Ok(value) = rc_tree.approximate_len() { observer.observe(value as u64, &[]); } @@ -58,7 +58,7 @@ impl BlockManagerMetrics { .with_description("Number of blocks known to the reference counter") .init(), _resync_queue_len: meter - .u64_value_observer("block.resync_queue_length", move |observer| { + .u64_value_observer("garage_block.resync_queue_length", move |observer| { if let Ok(value) = resync_queue.approximate_len() { observer.observe(value as u64, &[]); } @@ -68,7 +68,7 @@ impl BlockManagerMetrics { ) .init(), _resync_errored_blocks: meter - .u64_value_observer("block.resync_errored_blocks", move |observer| { + .u64_value_observer("garage_block.resync_errored_blocks", move |observer| { if let Ok(value) = resync_errors.approximate_len() { observer.observe(value as u64, &[]); } @@ -77,7 +77,7 @@ impl BlockManagerMetrics { .init(), _buffer_free_kb: meter - .u64_value_observer("block.ram_buffer_free_kb", move |observer| { + .u64_value_observer("garage_block.ram_buffer_free_kb", move |observer| { observer.observe(buffer_semaphore.available_permits() as u64, &[]); }) .with_description( @@ -86,63 +86,63 @@ impl BlockManagerMetrics { .init(), resync_counter: meter - .u64_counter("block.resync_counter") + .u64_counter("garage_block.resync_counter") .with_description("Number of calls to resync_block") .init() .bind(&[]), resync_error_counter: meter - .u64_counter("block.resync_error_counter") + .u64_counter("garage_block.resync_error_counter") .with_description("Number of calls to resync_block that returned an error") .init() .bind(&[]), resync_duration: meter - .f64_value_recorder("block.resync_duration") + .f64_value_recorder("garage_block.resync_duration") .with_description("Duration of resync_block operations") .init() .bind(&[]), resync_send_counter: meter - .u64_counter("block.resync_send_counter") + .u64_counter("garage_block.resync_send_counter") .with_description("Number of blocks sent to another node in resync operations") .init(), resync_recv_counter: meter - .u64_counter("block.resync_recv_counter") + .u64_counter("garage_block.resync_recv_counter") .with_description("Number of blocks received from other nodes in resync operations") .init() .bind(&[]), bytes_read: meter - .u64_counter("block.bytes_read") + .u64_counter("garage_block.bytes_read") .with_description("Number of bytes read from disk") .init() .bind(&[]), block_read_duration: meter - .f64_value_recorder("block.read_duration") + .f64_value_recorder("garage_block.read_duration") .with_description("Duration of block read operations") .init() .bind(&[]), block_read_semaphore_timeouts: meter - .u64_counter("block.read_semaphore_timeouts") + .u64_counter("garage_block.read_semaphore_timeouts") .with_description("Number of block reads that failed due to semaphore acquire timeout") .init() .bind(&[]), bytes_written: meter - .u64_counter("block.bytes_written") + .u64_counter("garage_block.bytes_written") .with_description("Number of bytes written to disk") .init() .bind(&[]), block_write_duration: meter - .f64_value_recorder("block.write_duration") + .f64_value_recorder("garage_block.write_duration") .with_description("Duration of block write operations") .init() .bind(&[]), delete_counter: meter - .u64_counter("block.delete_counter") + .u64_counter("garage_block.delete_counter") .with_description("Number of blocks deleted") .init() .bind(&[]), corruption_counter: meter - .u64_counter("block.corruption_counter") + .u64_counter("garage_block.corruption_counter") .with_description("Data corruptions detected on block reads") .init() .bind(&[]), diff --git a/src/rpc/metrics.rs b/src/rpc/metrics.rs index 0676dec0..25a028ae 100644 --- a/src/rpc/metrics.rs +++ b/src/rpc/metrics.rs @@ -14,23 +14,23 @@ impl RpcMetrics { let meter = global::meter("garage_rpc"); RpcMetrics { rpc_counter: meter - .u64_counter("rpc.request_counter") + .u64_counter("garage_rpc.request_counter") .with_description("Number of RPC requests emitted") .init(), rpc_timeout_counter: meter - .u64_counter("rpc.timeout_counter") + .u64_counter("garage_rpc.timeout_counter") .with_description("Number of RPC timeouts") .init(), rpc_netapp_error_counter: meter - .u64_counter("rpc.netapp_error_counter") + .u64_counter("garage_rpc.netapp_error_counter") .with_description("Number of communication errors (errors in the Netapp library)") .init(), rpc_garage_error_counter: meter - .u64_counter("rpc.garage_error_counter") + .u64_counter("garage_rpc.garage_error_counter") .with_description("Number of RPC errors (errors happening when handling the RPC)") .init(), rpc_duration: meter - .f64_value_recorder("rpc.duration") + .f64_value_recorder("garage_rpc.duration") .with_description("Duration of RPCs") .init(), } diff --git a/src/rpc/system_metrics.rs b/src/rpc/system_metrics.rs index 3d6c6e8b..821e9033 100644 --- a/src/rpc/system_metrics.rs +++ b/src/rpc/system_metrics.rs @@ -110,7 +110,7 @@ impl SystemMetrics { _cluster_healthy: { let get_health = get_health.clone(); meter - .u64_value_observer("cluster_healthy", move |observer| { + .u64_value_observer("garage_cluster_healthy", move |observer| { let h = get_health(); if h.status == ClusterHealthStatus::Healthy { observer.observe(1, &[]); @@ -123,7 +123,7 @@ impl SystemMetrics { }, _cluster_available: { let get_health = get_health.clone(); - meter.u64_value_observer("cluster_available", move |observer| { + meter.u64_value_observer("garage_cluster_available", move |observer| { let h = get_health(); if h.status != ClusterHealthStatus::Unavailable { observer.observe(1, &[]); @@ -137,7 +137,7 @@ impl SystemMetrics { _known_nodes: { let get_health = get_health.clone(); meter - .u64_value_observer("cluster_known_nodes", move |observer| { + .u64_value_observer("garage_cluster_known_nodes", move |observer| { let h = get_health(); observer.observe(h.known_nodes as u64, &[]); }) @@ -147,7 +147,7 @@ impl SystemMetrics { _connected_nodes: { let get_health = get_health.clone(); meter - .u64_value_observer("cluster_connected_nodes", move |observer| { + .u64_value_observer("garage_cluster_connected_nodes", move |observer| { let h = get_health(); observer.observe(h.connected_nodes as u64, &[]); }) @@ -157,7 +157,7 @@ impl SystemMetrics { _storage_nodes: { let get_health = get_health.clone(); meter - .u64_value_observer("cluster_storage_nodes", move |observer| { + .u64_value_observer("garage_cluster_storage_nodes", move |observer| { let h = get_health(); observer.observe(h.storage_nodes as u64, &[]); }) @@ -167,7 +167,7 @@ impl SystemMetrics { _storage_nodes_ok: { let get_health = get_health.clone(); meter - .u64_value_observer("cluster_storage_nodes_ok", move |observer| { + .u64_value_observer("garage_cluster_storage_nodes_ok", move |observer| { let h = get_health(); observer.observe(h.storage_nodes_ok as u64, &[]); }) @@ -177,7 +177,7 @@ impl SystemMetrics { _partitions: { let get_health = get_health.clone(); meter - .u64_value_observer("cluster_partitions", move |observer| { + .u64_value_observer("garage_cluster_partitions", move |observer| { let h = get_health(); observer.observe(h.partitions as u64, &[]); }) @@ -187,7 +187,7 @@ impl SystemMetrics { _partitions_quorum: { let get_health = get_health.clone(); meter - .u64_value_observer("cluster_partitions_quorum", move |observer| { + .u64_value_observer("garage_cluster_partitions_quorum", move |observer| { let h = get_health(); observer.observe(h.partitions_quorum as u64, &[]); }) @@ -199,7 +199,7 @@ impl SystemMetrics { _partitions_all_ok: { let get_health = get_health.clone(); meter - .u64_value_observer("cluster_partitions_all_ok", move |observer| { + .u64_value_observer("garage_cluster_partitions_all_ok", move |observer| { let h = get_health(); observer.observe(h.partitions_all_ok as u64, &[]); }) @@ -213,7 +213,7 @@ impl SystemMetrics { _layout_node_connected: { let system = system.clone(); meter - .u64_value_observer("cluster_layout_node_connected", move |observer| { + .u64_value_observer("garage_cluster_layout_node_connected", move |observer| { let layout = system.cluster_layout(); let nodes = system.get_known_nodes(); for id in layout.all_nodes().unwrap_or_default().iter() { @@ -260,44 +260,47 @@ impl SystemMetrics { _layout_node_disconnected_time: { let system = system.clone(); meter - .u64_value_observer("cluster_layout_node_disconnected_time", move |observer| { - let layout = system.cluster_layout(); - let nodes = system.get_known_nodes(); - for id in layout.all_nodes().unwrap_or_default().iter() { - let mut kv = vec![KeyValue::new("id", format!("{:?}", id))]; - if let Some(role) = layout - .current() - .ok() - .and_then(|l| l.roles.get(id)) - .and_then(|r| r.0.as_ref()) - { - kv.push(KeyValue::new("role_zone", role.zone.clone())); - match role.capacity { - Some(cap) => { - kv.push(KeyValue::new("role_capacity", cap as i64)); - kv.push(KeyValue::new("role_gateway", 0)); - } - None => { - kv.push(KeyValue::new("role_gateway", 1)); + .u64_value_observer( + "garage_cluster_layout_node_disconnected_time", + move |observer| { + let layout = system.cluster_layout(); + let nodes = system.get_known_nodes(); + for id in layout.all_nodes().unwrap_or_default().iter() { + let mut kv = vec![KeyValue::new("id", format!("{:?}", id))]; + if let Some(role) = layout + .current() + .ok() + .and_then(|l| l.roles.get(id)) + .and_then(|r| r.0.as_ref()) + { + kv.push(KeyValue::new("role_zone", role.zone.clone())); + match role.capacity { + Some(cap) => { + kv.push(KeyValue::new("role_capacity", cap as i64)); + kv.push(KeyValue::new("role_gateway", 0)); + } + None => { + kv.push(KeyValue::new("role_gateway", 1)); + } } } - } - if let Some(node) = nodes.iter().find(|n| n.id == *id) { - // TODO: see comment above - // kv.push(KeyValue::new("address", node.addr.to_string())); - // kv.push(KeyValue::new( - // "hostname", - // node.status.hostname.clone(), - // )); - if node.is_up { - observer.observe(0, &kv); - } else if let Some(secs) = node.last_seen_secs_ago { - observer.observe(secs, &kv); + if let Some(node) = nodes.iter().find(|n| n.id == *id) { + // TODO: see comment above + // kv.push(KeyValue::new("address", node.addr.to_string())); + // kv.push(KeyValue::new( + // "hostname", + // node.status.hostname.clone(), + // )); + if node.is_up { + observer.observe(0, &kv); + } else if let Some(secs) = node.last_seen_secs_ago { + observer.observe(secs, &kv); + } } } - } - }) + }, + ) .with_description( "Time (in seconds) since last connection to nodes in the cluster layout", ) diff --git a/src/table/metrics.rs b/src/table/metrics.rs index b1a787f2..b9caaf7b 100644 --- a/src/table/metrics.rs +++ b/src/table/metrics.rs @@ -34,7 +34,7 @@ impl TableMetrics { TableMetrics { _table_size: meter .u64_value_observer( - "table.size", + "garage_table.size", move |observer| { if let Ok(value) = store.approximate_len() { observer.observe( @@ -48,7 +48,7 @@ impl TableMetrics { .init(), _merkle_tree_size: meter .u64_value_observer( - "table.merkle_tree_size", + "garage_table.merkle_tree_size", move |observer| { if let Ok(value) = merkle_tree.approximate_len() { observer.observe( @@ -62,7 +62,7 @@ impl TableMetrics { .init(), _merkle_todo_len: meter .u64_value_observer( - "table.merkle_updater_todo_queue_length", + "garage_table.merkle_updater_todo_queue_length", move |observer| { if let Ok(v) = merkle_todo.approximate_len() { observer.observe( @@ -76,7 +76,7 @@ impl TableMetrics { .init(), _insert_queue_len: meter .u64_value_observer( - "table.insert_queue_length", + "garage_table.insert_queue_length", move |observer| { if let Ok(v) = insert_queue.approximate_len() { observer.observe( @@ -90,7 +90,7 @@ impl TableMetrics { .init(), _gc_todo_len: meter .u64_value_observer( - "table.gc_todo_queue_length", + "garage_table.gc_todo_queue_length", move |observer| { if let Ok(value) = gc_todo.approximate_len() { observer.observe( @@ -104,43 +104,43 @@ impl TableMetrics { .init(), get_request_counter: meter - .u64_counter("table.get_request_counter") + .u64_counter("garage_table.get_request_counter") .with_description("Number of get/get_range requests internally made on this table") .init() .bind(&[KeyValue::new("table_name", table_name)]), get_request_duration: meter - .f64_value_recorder("table.get_request_duration") + .f64_value_recorder("garage_table.get_request_duration") .with_description("Duration of get/get_range requests internally made on this table, in seconds") .init() .bind(&[KeyValue::new("table_name", table_name)]), put_request_counter: meter - .u64_counter("table.put_request_counter") + .u64_counter("garage_table.put_request_counter") .with_description("Number of insert/insert_many requests internally made on this table") .init() .bind(&[KeyValue::new("table_name", table_name)]), put_request_duration: meter - .f64_value_recorder("table.put_request_duration") + .f64_value_recorder("garage_table.put_request_duration") .with_description("Duration of insert/insert_many requests internally made on this table, in seconds") .init() .bind(&[KeyValue::new("table_name", table_name)]), internal_update_counter: meter - .u64_counter("table.internal_update_counter") + .u64_counter("garage_table.internal_update_counter") .with_description("Number of value updates where the value actually changes (includes creation of new key and update of existing key)") .init() .bind(&[KeyValue::new("table_name", table_name)]), internal_delete_counter: meter - .u64_counter("table.internal_delete_counter") + .u64_counter("garage_table.internal_delete_counter") .with_description("Number of value deletions in the tree (due to GC or repartitioning)") .init() .bind(&[KeyValue::new("table_name", table_name)]), sync_items_sent: meter - .u64_counter("table.sync_items_sent") + .u64_counter("garage_table.sync_items_sent") .with_description("Number of data items sent to other nodes during resync procedures") .init(), sync_items_received: meter - .u64_counter("table.sync_items_received") + .u64_counter("garage_table.sync_items_received") .with_description("Number of data items received from other nodes during resync procedures") .init(), } diff --git a/src/web/web_server.rs b/src/web/web_server.rs index eaff9dc7..e32266dc 100644 --- a/src/web/web_server.rs +++ b/src/web/web_server.rs @@ -54,15 +54,15 @@ impl WebMetrics { let meter = global::meter("garage/web"); Self { request_counter: meter - .u64_counter("web.request_counter") + .u64_counter("garage_web.request_counter") .with_description("Number of requests to the web endpoint") .init(), error_counter: meter - .u64_counter("web.error_counter") + .u64_counter("garage_web.error_counter") .with_description("Number of requests to the web endpoint resulting in errors") .init(), request_duration: meter - .f64_value_recorder("web.request_duration") + .f64_value_recorder("garage_web.request_duration") .with_description("Duration of requests to the web endpoint") .init(), }