mirror of
https://github.com/deuxfleurs-org/garage.git
synced 2026-08-11 06:46:53 +00:00
chore: add garage_ prefix for metrics who didn't have it
update: - monitoring doc - grafana dashboard elasticsearch.json
This commit is contained in:
Gwen Lg
@@ -182,15 +182,15 @@ content-type: text/plain; version=0.0.4
|
||||
content-length: 12145
|
||||
date: Tue, 08 Aug 2023 07:25:05 GMT
|
||||
|
||||
# HELP api_admin_error_counter Number of API calls to the various Admin API endpoints that resulted in errors
|
||||
# TYPE api_admin_error_counter counter
|
||||
api_admin_error_counter{api_endpoint="CheckWebsiteEnabled",status_code="400"} 1
|
||||
api_admin_error_counter{api_endpoint="CheckWebsiteEnabled",status_code="404"} 3
|
||||
# HELP api_admin_request_counter Number of API calls to the various Admin API endpoints
|
||||
# TYPE api_admin_request_counter counter
|
||||
api_admin_request_counter{api_endpoint="CheckWebsiteEnabled"} 7
|
||||
api_admin_request_counter{api_endpoint="Health"} 3
|
||||
# HELP api_admin_request_duration Duration of API calls to the various Admin API endpoints
|
||||
# HELP garage_api_admin_error_counter Number of API calls to the various Admin API endpoints that resulted in errors
|
||||
# TYPE garage_api_admin_error_counter counter
|
||||
garage_api_admin_error_counter{api_endpoint="CheckWebsiteEnabled",status_code="400"} 1
|
||||
garage_api_admin_error_counter{api_endpoint="CheckWebsiteEnabled",status_code="404"} 3
|
||||
# HELP garage_api_admin_request_counter Number of API calls to the various Admin API endpoints
|
||||
# TYPE garage_api_admin_request_counter counter
|
||||
garage_api_admin_request_counter{api_endpoint="CheckWebsiteEnabled"} 7
|
||||
garage_api_admin_request_counter{api_endpoint="Health"} 3
|
||||
# HELP garage_api_admin_request_duration Duration of API calls to the various Admin API endpoints
|
||||
...
|
||||
```
|
||||
|
||||
|
||||
@@ -40,146 +40,146 @@ garage_local_disk_total{volume="metadata"} 763063566336
|
||||
|
||||
### Cluster health status metrics
|
||||
|
||||
#### `cluster_healthy` (gauge)
|
||||
#### `garage_cluster_healthy` (gauge)
|
||||
|
||||
Whether all storage nodes are connected (0 or 1)
|
||||
|
||||
```
|
||||
cluster_healthy 0
|
||||
garage_cluster_healthy 0
|
||||
```
|
||||
|
||||
#### `cluster_available` (gauge)
|
||||
#### `garage_cluster_available` (gauge)
|
||||
|
||||
Whether all requests can be served, even if some storage nodes are disconnected
|
||||
|
||||
```
|
||||
cluster_available 1
|
||||
garage_cluster_available 1
|
||||
```
|
||||
|
||||
#### `cluster_connected_nodes` (gauge)
|
||||
#### `garage_cluster_connected_nodes` (gauge)
|
||||
|
||||
Number of nodes currently connected
|
||||
|
||||
```
|
||||
cluster_connected_nodes 3
|
||||
garage_cluster_connected_nodes 3
|
||||
```
|
||||
|
||||
#### `cluster_known_nodes` (gauge)
|
||||
#### `garage_cluster_known_nodes` (gauge)
|
||||
|
||||
Number of nodes already seen once in the cluster
|
||||
|
||||
```
|
||||
cluster_known_nodes 3
|
||||
garage_cluster_known_nodes 3
|
||||
```
|
||||
|
||||
#### `cluster_layout_node_connected` (gauge)
|
||||
#### `garage_cluster_layout_node_connected` (gauge)
|
||||
|
||||
Connection status for individual nodes of the cluster layout
|
||||
|
||||
```
|
||||
cluster_layout_node_connected{id="62b218d848e86a64",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 1
|
||||
cluster_layout_node_connected{id="a11c7cf18af29737",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 0
|
||||
cluster_layout_node_connected{id="a235ac7695e0c54d",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 1
|
||||
cluster_layout_node_connected{id="b10c110e4e854e5a",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 1
|
||||
garage_cluster_layout_node_connected{id="62b218d848e86a64",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 1
|
||||
garage_cluster_layout_node_connected{id="a11c7cf18af29737",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 0
|
||||
garage_cluster_layout_node_connected{id="a235ac7695e0c54d",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 1
|
||||
garage_cluster_layout_node_connected{id="b10c110e4e854e5a",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 1
|
||||
```
|
||||
|
||||
#### `cluster_layout_node_disconnected_time` (gauge)
|
||||
#### `garage_cluster_layout_node_disconnected_time` (gauge)
|
||||
|
||||
Time (in seconds) since last connection to individual nodes of the cluster layout
|
||||
|
||||
```
|
||||
cluster_layout_node_disconnected_time{id="62b218d848e86a64",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 0
|
||||
cluster_layout_node_disconnected_time{id="a235ac7695e0c54d",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 0
|
||||
cluster_layout_node_disconnected_time{id="b10c110e4e854e5a",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 0
|
||||
garage_cluster_layout_node_disconnected_time{id="62b218d848e86a64",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 0
|
||||
garage_cluster_layout_node_disconnected_time{id="a235ac7695e0c54d",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 0
|
||||
garage_cluster_layout_node_disconnected_time{id="b10c110e4e854e5a",role_capacity="1000000000",role_gateway="0",role_zone="dc1"} 0
|
||||
```
|
||||
|
||||
#### `cluster_storage_nodes` (gauge)
|
||||
#### `garage_cluster_storage_nodes` (gauge)
|
||||
|
||||
Number of storage nodes declared in the current layout
|
||||
|
||||
```
|
||||
cluster_storage_nodes 4
|
||||
garage_cluster_storage_nodes 4
|
||||
```
|
||||
|
||||
#### `cluster_storage_nodes_ok` (gauge)
|
||||
#### `garage_cluster_storage_nodes_ok` (gauge)
|
||||
|
||||
Number of storage nodes currently connected
|
||||
|
||||
```
|
||||
cluster_storage_nodes_ok 3
|
||||
garage_cluster_storage_nodes_ok 3
|
||||
```
|
||||
|
||||
#### `cluster_partitions` (gauge)
|
||||
#### `garage_cluster_partitions` (gauge)
|
||||
|
||||
Number of partitions in the layout (this is always 256)
|
||||
|
||||
```
|
||||
cluster_partitions 256
|
||||
garage_cluster_partitions 256
|
||||
```
|
||||
|
||||
#### `cluster_partitions_all_ok` (gauge)
|
||||
#### `garage_cluster_partitions_all_ok` (gauge)
|
||||
|
||||
Number of partitions for which all storage nodes are connected
|
||||
|
||||
```
|
||||
cluster_partitions_all_ok 64
|
||||
garage_cluster_partitions_all_ok 64
|
||||
```
|
||||
|
||||
#### `cluster_partitions_quorum` (gauge)
|
||||
#### `garage_cluster_partitions_quorum` (gauge)
|
||||
|
||||
Number of partitions for which we have a quorum of connected nodes and all requests can be served
|
||||
|
||||
```
|
||||
cluster_partitions_quorum 256
|
||||
garage_cluster_partitions_quorum 256
|
||||
```
|
||||
|
||||
### Metrics of the API endpoints
|
||||
|
||||
#### `api_admin_request_counter` (counter)
|
||||
#### `garage_api_admin_request_counter` (counter)
|
||||
|
||||
Counts the number of requests to a given endpoint of the administration API. Example:
|
||||
|
||||
```
|
||||
api_admin_request_counter{api_endpoint="Metrics"} 127041
|
||||
garage_api_admin_request_counter{api_endpoint="Metrics"} 127041
|
||||
```
|
||||
|
||||
#### `api_admin_request_duration` (histogram)
|
||||
#### `garage_api_admin_request_duration` (histogram)
|
||||
|
||||
Evaluates the duration of API calls to the various administration API endpoint. Example:
|
||||
|
||||
```
|
||||
api_admin_request_duration_bucket{api_endpoint="Metrics",le="0.5"} 127041
|
||||
api_admin_request_duration_sum{api_endpoint="Metrics"} 605.250344830999
|
||||
api_admin_request_duration_count{api_endpoint="Metrics"} 127041
|
||||
garage_api_admin_request_duration_bucket{api_endpoint="Metrics",le="0.5"} 127041
|
||||
garage_api_admin_request_duration_sum{api_endpoint="Metrics"} 605.250344830999
|
||||
garage_api_admin_request_duration_count{api_endpoint="Metrics"} 127041
|
||||
```
|
||||
|
||||
#### `api_s3_request_counter` (counter)
|
||||
#### `garage_api_s3_request_counter` (counter)
|
||||
|
||||
Counts the number of requests to a given endpoint of the S3 API. Example:
|
||||
|
||||
```
|
||||
api_s3_request_counter{api_endpoint="CreateMultipartUpload"} 1
|
||||
garage_api_s3_request_counter{api_endpoint="CreateMultipartUpload"} 1
|
||||
```
|
||||
|
||||
#### `api_s3_error_counter` (counter)
|
||||
#### `garage_api_s3_error_counter` (counter)
|
||||
|
||||
Counts the number of requests to a given endpoint of the S3 API that returned an error. Example:
|
||||
|
||||
```
|
||||
api_s3_error_counter{api_endpoint="GetObject",status_code="404"} 39
|
||||
garage_api_s3_error_counter{api_endpoint="GetObject",status_code="404"} 39
|
||||
```
|
||||
|
||||
#### `api_s3_request_duration` (histogram)
|
||||
#### `garage_api_s3_request_duration` (histogram)
|
||||
|
||||
Evaluates the duration of API calls to the various S3 API endpoints. Example:
|
||||
|
||||
```
|
||||
api_s3_request_duration_bucket{api_endpoint="CreateMultipartUpload",le="0.5"} 1
|
||||
api_s3_request_duration_sum{api_endpoint="CreateMultipartUpload"} 0.046340762
|
||||
api_s3_request_duration_count{api_endpoint="CreateMultipartUpload"} 1
|
||||
garage_api_s3_request_duration_bucket{api_endpoint="CreateMultipartUpload",le="0.5"} 1
|
||||
garage_api_s3_request_duration_sum{api_endpoint="CreateMultipartUpload"} 0.046340762
|
||||
garage_api_s3_request_duration_count{api_endpoint="CreateMultipartUpload"} 1
|
||||
```
|
||||
|
||||
#### `api_k2v_request_counter` (counter), `api_k2v_error_counter` (counter), `api_k2v_error_duration` (histogram)
|
||||
#### `garage_api_k2v_request_counter` (counter), `garage_api_k2v_error_counter` (counter), `garage_api_k2v_error_duration` (histogram)
|
||||
|
||||
Same as for S3, for the K2V API.
|
||||
|
||||
@@ -187,45 +187,45 @@ Same as for S3, for the K2V API.
|
||||
### Metrics of the Web endpoint
|
||||
|
||||
|
||||
#### `web_request_counter` (counter)
|
||||
#### `garage_web_request_counter` (counter)
|
||||
|
||||
Number of requests to the web endpoint
|
||||
|
||||
```
|
||||
web_request_counter{method="GET"} 80
|
||||
garage_web_request_counter{method="GET"} 80
|
||||
```
|
||||
|
||||
#### `web_request_duration` (histogram)
|
||||
#### `garage_web_request_duration` (histogram)
|
||||
|
||||
Duration of requests to the web endpoint
|
||||
|
||||
```
|
||||
web_request_duration_bucket{method="GET",le="0.5"} 80
|
||||
web_request_duration_sum{method="GET"} 1.0528433229999998
|
||||
web_request_duration_count{method="GET"} 80
|
||||
garage_web_request_duration_bucket{method="GET",le="0.5"} 80
|
||||
garage_web_request_duration_sum{method="GET"} 1.0528433229999998
|
||||
garage_web_request_duration_count{method="GET"} 80
|
||||
```
|
||||
|
||||
#### `web_error_counter` (counter)
|
||||
#### `garage_web_error_counter` (counter)
|
||||
|
||||
Number of requests to the web endpoint resulting in errors
|
||||
|
||||
```
|
||||
web_error_counter{method="GET",status_code="404 Not Found"} 64
|
||||
garage_web_error_counter{method="GET",status_code="404 Not Found"} 64
|
||||
```
|
||||
|
||||
|
||||
### Metrics of the data block manager
|
||||
|
||||
#### `block_bytes_read`, `block_bytes_written` (counter)
|
||||
#### `garage_block_bytes_read`, `garage_block_bytes_written` (counter)
|
||||
|
||||
Number of bytes read/written to/from disk in the data storage directory.
|
||||
|
||||
```
|
||||
block_bytes_read 120586322022
|
||||
block_bytes_written 3386618077
|
||||
garage_block_bytes_read 120586322022
|
||||
garage_block_bytes_written 3386618077
|
||||
```
|
||||
|
||||
#### `block_ram_buffer_free_kb` (gauge)
|
||||
#### `garage_block_ram_buffer_free_kb` (gauge)
|
||||
|
||||
Kibibytes available for buffering blocks that have to be sent to remote nodes.
|
||||
When clients send too much data to this node and a storage node is not receiving
|
||||
@@ -233,170 +233,168 @@ data fast enough due to slower network conditions, this will decrease down to
|
||||
zero and backpressure will be applied.
|
||||
|
||||
```
|
||||
block_ram_buffer_free_kb 219829
|
||||
garage_block_ram_buffer_free_kb 219829
|
||||
```
|
||||
|
||||
#### `block_compression_level` (counter)
|
||||
#### `garage_block_compression_level` (counter)
|
||||
|
||||
Exposes the block compression level configured for the Garage node.
|
||||
|
||||
```
|
||||
block_compression_level 3
|
||||
garage_block_compression_level 3
|
||||
```
|
||||
|
||||
#### `block_read_duration`, `block_write_duration` (histograms)
|
||||
#### `garage_block_read_duration`, `garage_block_write_duration` (histograms)
|
||||
|
||||
Evaluates the duration of the reading/writing of individual data blocks in the data storage directory.
|
||||
|
||||
```
|
||||
block_read_duration_bucket{le="0.5"} 169229
|
||||
block_read_duration_sum 2761.6902550310056
|
||||
block_read_duration_count 169240
|
||||
block_write_duration_bucket{le="0.5"} 3559
|
||||
block_write_duration_sum 195.59170078500006
|
||||
block_write_duration_count 3571
|
||||
garage_block_read_duration_bucket{le="0.5"} 169229
|
||||
garage_block_read_duration_sum 2761.6902550310056
|
||||
garage_block_read_duration_count 169240
|
||||
garage_block_write_duration_bucket{le="0.5"} 3559
|
||||
garage_block_write_duration_sum 195.59170078500006
|
||||
garage_block_write_duration_count 3571
|
||||
```
|
||||
|
||||
#### `block_delete_counter` (counter)
|
||||
#### `garage_block_delete_counter` (counter)
|
||||
|
||||
Counts the number of data blocks that have been deleted from storage.
|
||||
|
||||
```
|
||||
block_delete_counter 122
|
||||
garage_block_delete_counter 122
|
||||
```
|
||||
|
||||
#### `block_resync_counter` (counter), `block_resync_duration` (histogram)
|
||||
#### `garage_block_resync_counter` (counter), `garage_block_resync_duration` (histogram)
|
||||
|
||||
Counts the number of resync operations the node has executed, and evaluates their duration.
|
||||
|
||||
```
|
||||
block_resync_counter 308897
|
||||
block_resync_duration_bucket{le="0.5"} 308892
|
||||
block_resync_duration_sum 139.64204196100016
|
||||
block_resync_duration_count 308897
|
||||
garage_block_resync_counter 308897
|
||||
garage_block_resync_duration_bucket{le="0.5"} 308892
|
||||
garage_block_resync_duration_sum 139.64204196100016
|
||||
garage_block_resync_duration_count 308897
|
||||
```
|
||||
|
||||
#### `block_resync_queue_length` (gauge)
|
||||
#### `garage_block_resync_queue_length` (gauge)
|
||||
|
||||
The number of block hashes currently queued for a resync.
|
||||
This is normal to be nonzero for long periods of time.
|
||||
|
||||
```
|
||||
block_resync_queue_length 0
|
||||
garage_block_resync_queue_length 0
|
||||
```
|
||||
|
||||
#### `block_resync_errored_blocks` (gauge)
|
||||
#### `garage_block_resync_errored_blocks` (gauge)
|
||||
|
||||
The number of block hashes that we were unable to resync last time we tried.
|
||||
**THIS SHOULD BE ZERO, OR FALL BACK TO ZERO RAPIDLY, IN A HEALTHY CLUSTER.**
|
||||
Persistent nonzero values indicate that some data is likely to be lost.
|
||||
|
||||
```
|
||||
block_resync_errored_blocks 0
|
||||
garage_block_resync_errored_blocks 0
|
||||
```
|
||||
|
||||
|
||||
### Metrics related to RPCs (remote procedure calls) between nodes
|
||||
|
||||
#### `rpc_netapp_request_counter` (counter)
|
||||
#### `garage_rpc_netapp_request_counter` (counter)
|
||||
|
||||
Number of RPC requests emitted
|
||||
|
||||
```
|
||||
rpc_request_counter{from="<this node>",rpc_endpoint="garage_block/manager.rs/Rpc",to="<remote node>"} 176
|
||||
garage_rpc_request_counter{from="<this node>",rpc_endpoint="garage_block/manager.rs/Rpc",to="<remote node>"} 176
|
||||
```
|
||||
|
||||
#### `rpc_netapp_error_counter` (counter)
|
||||
#### `garage_rpc_netapp_error_counter` (counter)
|
||||
|
||||
Number of communication errors (errors in the Netapp library, generally due to disconnected nodes)
|
||||
|
||||
```
|
||||
rpc_netapp_error_counter{from="<this node>",rpc_endpoint="garage_block/manager.rs/Rpc",to="<remote node>"} 354
|
||||
garage_rpc_netapp_error_counter{from="<this node>",rpc_endpoint="garage_block/manager.rs/Rpc",to="<remote node>"} 354
|
||||
```
|
||||
|
||||
#### `rpc_timeout_counter` (counter)
|
||||
#### `garage_rpc_timeout_counter` (counter)
|
||||
|
||||
Number of RPC timeouts, should be close to zero in a healthy cluster.
|
||||
|
||||
```
|
||||
rpc_timeout_counter{from="<this node>",rpc_endpoint="garage_rpc/membership.rs/SystemRpc",to="<remote node>"} 1
|
||||
garage_rpc_timeout_counter{from="<this node>",rpc_endpoint="garage_rpc/membership.rs/SystemRpc",to="<remote node>"} 1
|
||||
```
|
||||
|
||||
#### `rpc_duration` (histogram)
|
||||
#### `garage_rpc_duration` (histogram)
|
||||
|
||||
The duration of internal RPC calls between Garage nodes.
|
||||
|
||||
```
|
||||
rpc_duration_bucket{from="<this node>",rpc_endpoint="garage_block/manager.rs/Rpc",to="<remote node>",le="0.5"} 166
|
||||
rpc_duration_sum{from="<this node>",rpc_endpoint="garage_block/manager.rs/Rpc",to="<remote node>"} 35.172253716
|
||||
rpc_duration_count{from="<this node>",rpc_endpoint="garage_block/manager.rs/Rpc",to="<remote node>"} 174
|
||||
garage_rpc_duration_bucket{from="<this node>",rpc_endpoint="garage_block/manager.rs/Rpc",to="<remote node>",le="0.5"} 166
|
||||
garage_rpc_duration_sum{from="<this node>",rpc_endpoint="garage_block/manager.rs/Rpc",to="<remote node>"} 35.172253716
|
||||
garage_rpc_duration_count{from="<this node>",rpc_endpoint="garage_block/manager.rs/Rpc",to="<remote node>"} 174
|
||||
```
|
||||
|
||||
|
||||
### Metrics of the metadata table manager
|
||||
|
||||
#### `table_gc_todo_queue_length` (gauge)
|
||||
#### `garage_table_gc_todo_queue_length` (gauge)
|
||||
|
||||
Table garbage collector TODO queue length
|
||||
|
||||
```
|
||||
table_gc_todo_queue_length{table_name="block_ref"} 0
|
||||
garage_table_gc_todo_queue_length{table_name="block_ref"} 0
|
||||
```
|
||||
|
||||
#### `table_get_request_counter` (counter), `table_get_request_duration` (histogram)
|
||||
#### `garage_table_get_request_counter` (counter), `garage_table_get_request_duration` (histogram)
|
||||
|
||||
Number of get/get_range requests internally made on each table, and their duration.
|
||||
|
||||
```
|
||||
table_get_request_counter{table_name="bucket_alias"} 315
|
||||
table_get_request_duration_bucket{table_name="bucket_alias",le="0.5"} 315
|
||||
table_get_request_duration_sum{table_name="bucket_alias"} 0.048509778000000024
|
||||
table_get_request_duration_count{table_name="bucket_alias"} 315
|
||||
garage_table_get_request_counter{table_name="bucket_alias"} 315
|
||||
garage_table_get_request_duration_bucket{table_name="bucket_alias",le="0.5"} 315
|
||||
garage_table_get_request_duration_sum{table_name="bucket_alias"} 0.048509778000000024
|
||||
garage_table_get_request_duration_count{table_name="bucket_alias"} 315
|
||||
```
|
||||
|
||||
|
||||
#### `table_put_request_counter` (counter), `table_put_request_duration` (histogram)
|
||||
#### `garage_table_put_request_counter` (counter), `garage_table_put_request_duration` (histogram)
|
||||
|
||||
Number of insert/insert_many requests internally made on this table, and their duration
|
||||
|
||||
```
|
||||
table_put_request_counter{table_name="block_ref"} 677
|
||||
table_put_request_duration_bucket{table_name="block_ref",le="0.5"} 677
|
||||
table_put_request_duration_sum{table_name="block_ref"} 61.617528636
|
||||
table_put_request_duration_count{table_name="block_ref"} 677
|
||||
garage_table_put_request_counter{table_name="block_ref"} 677
|
||||
garage_table_put_request_duration_bucket{table_name="block_ref",le="0.5"} 677
|
||||
garage_table_put_request_duration_sum{table_name="block_ref"} 61.617528636
|
||||
garage_table_put_request_duration_count{table_name="block_ref"} 677
|
||||
```
|
||||
|
||||
#### `table_internal_delete_counter` (counter)
|
||||
#### `garage_table_internal_delete_counter` (counter)
|
||||
|
||||
Number of value deletions in the tree (due to GC or repartitioning)
|
||||
|
||||
```
|
||||
table_internal_delete_counter{table_name="block_ref"} 2296
|
||||
garage_table_internal_delete_counter{table_name="block_ref"} 2296
|
||||
```
|
||||
|
||||
#### `table_internal_update_counter` (counter)
|
||||
#### `garage_table_internal_update_counter` (counter)
|
||||
|
||||
Number of value updates where the value actually changes (includes creation of new key and update of existing key)
|
||||
|
||||
```
|
||||
table_internal_update_counter{table_name="block_ref"} 5996
|
||||
garage_table_internal_update_counter{table_name="block_ref"} 5996
|
||||
```
|
||||
|
||||
#### `table_merkle_updater_todo_queue_length` (gauge)
|
||||
#### `garage_table_merkle_updater_todo_queue_length` (gauge)
|
||||
|
||||
Merkle tree updater TODO queue length (should fall to zero rapidly)
|
||||
|
||||
```
|
||||
table_merkle_updater_todo_queue_length{table_name="block_ref"} 0
|
||||
garage_table_merkle_updater_todo_queue_length{table_name="block_ref"} 0
|
||||
```
|
||||
|
||||
#### `table_sync_items_received`, `table_sync_items_sent` (counters)
|
||||
#### `garage_table_sync_items_received`, `garage_table_sync_items_sent` (counters)
|
||||
|
||||
Number of data items sent to/received from other nodes during resync procedures
|
||||
|
||||
```
|
||||
table_sync_items_received{from="<remote node>",table_name="bucket_v2"} 3
|
||||
table_sync_items_sent{table_name="block_ref",to="<remote node>"} 2
|
||||
garage_table_sync_items_received{from="<remote node>",table_name="bucket_v2"} 3
|
||||
garage_table_sync_items_sent{table_name="block_ref",to="<remote node>"} 2
|
||||
```
|
||||
|
||||
|
||||
|
||||
@@ -161,7 +161,7 @@
|
||||
},
|
||||
"metrics": [
|
||||
{
|
||||
"field": "api_request_counter",
|
||||
"field": "garage_api_request_counter",
|
||||
"hide": true,
|
||||
"id": "1",
|
||||
"type": "sum"
|
||||
@@ -284,7 +284,7 @@
|
||||
"hide": false,
|
||||
"metrics": [
|
||||
{
|
||||
"field": "api_request_duration",
|
||||
"field": "garage_api_request_duration",
|
||||
"id": "1",
|
||||
"type": "avg"
|
||||
}
|
||||
@@ -412,7 +412,7 @@
|
||||
},
|
||||
"metrics": [
|
||||
{
|
||||
"field": "api_error_counter",
|
||||
"field": "garage_api_error_counter",
|
||||
"hide": true,
|
||||
"id": "1",
|
||||
"type": "sum"
|
||||
@@ -540,7 +540,7 @@
|
||||
},
|
||||
"metrics": [
|
||||
{
|
||||
"field": "web_request_counter",
|
||||
"field": "garage_web_request_counter",
|
||||
"hide": true,
|
||||
"id": "1",
|
||||
"type": "sum"
|
||||
@@ -666,7 +666,7 @@
|
||||
"hide": false,
|
||||
"metrics": [
|
||||
{
|
||||
"field": "web_request_duration",
|
||||
"field": "garage_web_request_duration",
|
||||
"id": "1",
|
||||
"type": "avg"
|
||||
}
|
||||
@@ -794,7 +794,7 @@
|
||||
},
|
||||
"metrics": [
|
||||
{
|
||||
"field": "web_error_counter",
|
||||
"field": "garage_web_error_counter",
|
||||
"hide": true,
|
||||
"id": "1",
|
||||
"type": "sum"
|
||||
@@ -918,7 +918,7 @@
|
||||
"hide": false,
|
||||
"metrics": [
|
||||
{
|
||||
"field": "table_get_request_counter",
|
||||
"field": "garage_table_get_request_counter",
|
||||
"hide": true,
|
||||
"id": "1",
|
||||
"type": "sum"
|
||||
@@ -1042,7 +1042,7 @@
|
||||
"hide": false,
|
||||
"metrics": [
|
||||
{
|
||||
"field": "table_put_request_counter",
|
||||
"field": "garage_table_put_request_counter",
|
||||
"hide": true,
|
||||
"id": "1",
|
||||
"type": "sum"
|
||||
@@ -1154,7 +1154,7 @@
|
||||
"hide": false,
|
||||
"metrics": [
|
||||
{
|
||||
"field": "block_bytes_read",
|
||||
"field": "garage_block_bytes_read",
|
||||
"hide": true,
|
||||
"id": "1",
|
||||
"type": "sum"
|
||||
@@ -1270,7 +1270,7 @@
|
||||
},
|
||||
"metrics": [
|
||||
{
|
||||
"field": "block_bytes_written",
|
||||
"field": "garage_block_bytes_written",
|
||||
"hide": true,
|
||||
"id": "1",
|
||||
"type": "sum"
|
||||
@@ -1386,7 +1386,7 @@
|
||||
},
|
||||
"metrics": [
|
||||
{
|
||||
"field": "block_resync_counter",
|
||||
"field": "garage_block_resync_counter",
|
||||
"hide": true,
|
||||
"id": "1",
|
||||
"type": "sum"
|
||||
@@ -1500,7 +1500,7 @@
|
||||
"hide": false,
|
||||
"metrics": [
|
||||
{
|
||||
"field": "block_resync_queue_length",
|
||||
"field": "garage_block_resync_queue_length",
|
||||
"id": "1",
|
||||
"type": "avg"
|
||||
}
|
||||
@@ -1610,7 +1610,7 @@
|
||||
},
|
||||
"metrics": [
|
||||
{
|
||||
"field": "table_merkle_updater_todo_queue_length",
|
||||
"field": "garage_table_merkle_updater_todo_queue_length",
|
||||
"id": "1",
|
||||
"type": "avg"
|
||||
}
|
||||
@@ -1724,7 +1724,7 @@
|
||||
},
|
||||
"metrics": [
|
||||
{
|
||||
"field": "table_gc_todo_queue_length",
|
||||
"field": "garage_table_gc_todo_queue_length",
|
||||
"id": "1",
|
||||
"type": "avg"
|
||||
}
|
||||
@@ -1824,7 +1824,7 @@
|
||||
},
|
||||
"metrics": [
|
||||
{
|
||||
"field": "block_resync_error_counter",
|
||||
"field": "garage_block_resync_error_counter",
|
||||
"hide": true,
|
||||
"id": "1",
|
||||
"settings": {},
|
||||
@@ -1938,7 +1938,7 @@
|
||||
},
|
||||
"metrics": [
|
||||
{
|
||||
"field": "block_resync_errored_blocks",
|
||||
"field": "garage_block_resync_errored_blocks",
|
||||
"hide": false,
|
||||
"id": "1",
|
||||
"type": "sum"
|
||||
@@ -2041,7 +2041,7 @@
|
||||
},
|
||||
"metrics": [
|
||||
{
|
||||
"field": "block_corruption_counter",
|
||||
"field": "garage_block_corruption_counter",
|
||||
"hide": true,
|
||||
"id": "1",
|
||||
"type": "sum"
|
||||
@@ -2165,7 +2165,7 @@
|
||||
},
|
||||
"metrics": [
|
||||
{
|
||||
"field": "rpc_netapp_error_counter",
|
||||
"field": "garage_rpc_netapp_error_counter",
|
||||
"hide": true,
|
||||
"id": "1",
|
||||
"type": "sum"
|
||||
@@ -2292,7 +2292,7 @@
|
||||
},
|
||||
"metrics": [
|
||||
{
|
||||
"field": "rpc_request_counter",
|
||||
"field": "garage_rpc_request_counter",
|
||||
"hide": true,
|
||||
"id": "1",
|
||||
"type": "sum"
|
||||
@@ -2418,7 +2418,7 @@
|
||||
},
|
||||
"metrics": [
|
||||
{
|
||||
"field": "rpc_duration",
|
||||
"field": "garage_rpc_duration",
|
||||
"id": "1",
|
||||
"type": "avg"
|
||||
}
|
||||
@@ -2521,7 +2521,7 @@
|
||||
},
|
||||
"metrics": [
|
||||
{
|
||||
"field": "admin_http_requests_total",
|
||||
"field": "garage_admin_http_requests_total",
|
||||
"hide": true,
|
||||
"id": "1",
|
||||
"type": "sum"
|
||||
@@ -2654,7 +2654,7 @@
|
||||
},
|
||||
"metrics": [
|
||||
{
|
||||
"field": "rpc_garage_error_counter",
|
||||
"field": "garage_rpc_garage_error_counter",
|
||||
"hide": true,
|
||||
"id": "1",
|
||||
"type": "sum"
|
||||
@@ -2765,7 +2765,7 @@
|
||||
},
|
||||
"metrics": [
|
||||
{
|
||||
"field": "rpc_duration",
|
||||
"field": "garage_rpc_duration",
|
||||
"id": "1",
|
||||
"type": "avg"
|
||||
}
|
||||
|
||||
@@ -143,7 +143,7 @@
|
||||
"uid": "${DS_DS_PROMETHEUS}"
|
||||
},
|
||||
"exemplar": true,
|
||||
"expr": "sum(rate(block_bytes_read{job=\"garage\"}[$__rate_interval]) )",
|
||||
"expr": "sum(rate(garage_block_bytes_read{job=\"garage\"}[$__rate_interval]) )",
|
||||
"hide": false,
|
||||
"interval": "",
|
||||
"legendFormat": "Disk bytes read",
|
||||
@@ -155,7 +155,7 @@
|
||||
"uid": "${DS_DS_PROMETHEUS}"
|
||||
},
|
||||
"exemplar": true,
|
||||
"expr": "-sum(rate(block_bytes_written{job=\"garage\"}[$__rate_interval]) )",
|
||||
"expr": "-sum(rate(garage_block_bytes_written{job=\"garage\"}[$__rate_interval]) )",
|
||||
"hide": false,
|
||||
"interval": "",
|
||||
"legendFormat": "Disk bytes written",
|
||||
@@ -250,7 +250,7 @@
|
||||
},
|
||||
"editorMode": "code",
|
||||
"exemplar": true,
|
||||
"expr": "sum by (api_endpoint) (rate(api_s3_request_counter {job=\"garage\"}[$__rate_interval]))",
|
||||
"expr": "sum by (api_endpoint) (rate(garage_api_s3_request_counter {job=\"garage\"}[$__rate_interval]))",
|
||||
"hide": false,
|
||||
"interval": "",
|
||||
"legendFormat": "{{api_endpoint}}",
|
||||
@@ -345,7 +345,7 @@
|
||||
"uid": "${DS_DS_PROMETHEUS}"
|
||||
},
|
||||
"exemplar": true,
|
||||
"expr": "sum(rate(web_request_counter {job=\"garage\"}[$__rate_interval]))",
|
||||
"expr": "sum(rate(garage_web_request_counter {job=\"garage\"}[$__rate_interval]))",
|
||||
"hide": false,
|
||||
"interval": "",
|
||||
"legendFormat": "Web request rate",
|
||||
@@ -439,7 +439,7 @@
|
||||
"uid": "${DS_DS_PROMETHEUS}"
|
||||
},
|
||||
"exemplar": true,
|
||||
"expr": "sum by (rpc_endpoint) (rate(rpc_request_counter {job=\"garage\"}[$__rate_interval]))",
|
||||
"expr": "sum by (rpc_endpoint) (rate(garage_rpc_request_counter {job=\"garage\"}[$__rate_interval]))",
|
||||
"hide": false,
|
||||
"interval": "",
|
||||
"legendFormat": "{{rpc_endpoint}}",
|
||||
@@ -534,7 +534,7 @@
|
||||
},
|
||||
"editorMode": "code",
|
||||
"exemplar": true,
|
||||
"expr": "sum by (api_endpoint, status_code) (rate(api_s3_error_counter {job=\"garage\"}[$__rate_interval]))",
|
||||
"expr": "sum by (api_endpoint, status_code) (rate(garage_api_s3_error_counter {job=\"garage\"}[$__rate_interval]))",
|
||||
"hide": false,
|
||||
"interval": "",
|
||||
"legendFormat": "{{api_endpoint}} {{status_code}}",
|
||||
@@ -629,7 +629,7 @@
|
||||
"uid": "${DS_DS_PROMETHEUS}"
|
||||
},
|
||||
"exemplar": true,
|
||||
"expr": "sum by(status_code) (rate(web_error_counter {job=\"garage\"}[$__rate_interval]))",
|
||||
"expr": "sum by(status_code) (rate(garage_web_error_count {job=\"garage\"}[$__rate_interval]))",
|
||||
"hide": false,
|
||||
"interval": "",
|
||||
"legendFormat": "{{status_code}}",
|
||||
@@ -722,7 +722,7 @@
|
||||
"uid": "${DS_DS_PROMETHEUS}"
|
||||
},
|
||||
"exemplar": true,
|
||||
"expr": "block_resync_queue_length{job=\"garage\"}",
|
||||
"expr": "garage_block_resync_queue_length{job=\"garage\"}",
|
||||
"interval": "",
|
||||
"legendFormat": "{{instance}}",
|
||||
"refId": "A"
|
||||
@@ -814,7 +814,7 @@
|
||||
"uid": "${DS_DS_PROMETHEUS}"
|
||||
},
|
||||
"exemplar": true,
|
||||
"expr": "sum by(table_name) (table_gc_todo_queue_length{job=\"garage\"})",
|
||||
"expr": "sum by(table_name) (garage_table_gc_todo_queue_length{job=\"garage\"})",
|
||||
"interval": "",
|
||||
"legendFormat": "{{ table_name}}",
|
||||
"refId": "A"
|
||||
@@ -906,7 +906,7 @@
|
||||
"uid": "${DS_DS_PROMETHEUS}"
|
||||
},
|
||||
"exemplar": true,
|
||||
"expr": "sum by(table_name) (table_merkle_updater_todo_queue_length{job=\"garage\"})",
|
||||
"expr": "sum by(table_name) (garage_table_merkle_updater_todo_queue_length{job=\"garage\"})",
|
||||
"interval": "",
|
||||
"legendFormat": "{{ table_name}}",
|
||||
"refId": "A"
|
||||
@@ -998,7 +998,7 @@
|
||||
"uid": "${DS_DS_PROMETHEUS}"
|
||||
},
|
||||
"exemplar": true,
|
||||
"expr": "block_resync_errored_blocks{job=\"garage\"}",
|
||||
"expr": "garage_block_resync_errored_blocks{job=\"garage\"}",
|
||||
"interval": "",
|
||||
"legendFormat": "{{instance}}",
|
||||
"refId": "A"
|
||||
@@ -1025,4 +1025,4 @@
|
||||
"uid": "ys3pnpZ4k",
|
||||
"version": 26,
|
||||
"weekStart": ""
|
||||
}
|
||||
}
|
||||
|
||||
@@ -84,21 +84,21 @@ impl<A: ApiHandler> ApiServer<A> {
|
||||
region,
|
||||
api_handler,
|
||||
request_counter: meter
|
||||
.u64_counter(format!("api.{}.request_counter", A::API_NAME))
|
||||
.u64_counter(format!("garage_api.{}.request_counter", A::API_NAME))
|
||||
.with_description(format!(
|
||||
"Number of API calls to the various {} API endpoints",
|
||||
A::API_NAME_DISPLAY
|
||||
))
|
||||
.init(),
|
||||
error_counter: meter
|
||||
.u64_counter(format!("api.{}.error_counter", A::API_NAME))
|
||||
.u64_counter(format!("garage_api.{}.error_counter", A::API_NAME))
|
||||
.with_description(format!(
|
||||
"Number of API calls to the various {} API endpoints that resulted in errors",
|
||||
A::API_NAME_DISPLAY
|
||||
))
|
||||
.init(),
|
||||
request_duration: meter
|
||||
.f64_value_recorder(format!("api.{}.request_duration", A::API_NAME))
|
||||
.f64_value_recorder(format!("garage_api.{}.request_duration", A::API_NAME))
|
||||
.with_description(format!(
|
||||
"Duration of API calls to the various {} API endpoints",
|
||||
A::API_NAME_DISPLAY
|
||||
|
||||
+17
-17
@@ -41,7 +41,7 @@ impl BlockManagerMetrics {
|
||||
let meter = global::meter("garage_model/block");
|
||||
Self {
|
||||
_compression_level: meter
|
||||
.u64_value_observer("block.compression_level", move |observer| {
|
||||
.u64_value_observer("garage_block.compression_level", move |observer| {
|
||||
match compression_level {
|
||||
Some(v) => observer.observe(v as u64, &[]),
|
||||
None => observer.observe(0_u64, &[]),
|
||||
@@ -50,7 +50,7 @@ impl BlockManagerMetrics {
|
||||
.with_description("Garage compression level for node")
|
||||
.init(),
|
||||
_rc_size: meter
|
||||
.u64_value_observer("block.rc_size", move |observer| {
|
||||
.u64_value_observer("garage_block.rc_size", move |observer| {
|
||||
if let Ok(value) = rc_tree.approximate_len() {
|
||||
observer.observe(value as u64, &[]);
|
||||
}
|
||||
@@ -58,7 +58,7 @@ impl BlockManagerMetrics {
|
||||
.with_description("Number of blocks known to the reference counter")
|
||||
.init(),
|
||||
_resync_queue_len: meter
|
||||
.u64_value_observer("block.resync_queue_length", move |observer| {
|
||||
.u64_value_observer("garage_block.resync_queue_length", move |observer| {
|
||||
if let Ok(value) = resync_queue.approximate_len() {
|
||||
observer.observe(value as u64, &[]);
|
||||
}
|
||||
@@ -68,7 +68,7 @@ impl BlockManagerMetrics {
|
||||
)
|
||||
.init(),
|
||||
_resync_errored_blocks: meter
|
||||
.u64_value_observer("block.resync_errored_blocks", move |observer| {
|
||||
.u64_value_observer("garage_block.resync_errored_blocks", move |observer| {
|
||||
if let Ok(value) = resync_errors.approximate_len() {
|
||||
observer.observe(value as u64, &[]);
|
||||
}
|
||||
@@ -77,7 +77,7 @@ impl BlockManagerMetrics {
|
||||
.init(),
|
||||
|
||||
_buffer_free_kb: meter
|
||||
.u64_value_observer("block.ram_buffer_free_kb", move |observer| {
|
||||
.u64_value_observer("garage_block.ram_buffer_free_kb", move |observer| {
|
||||
observer.observe(buffer_semaphore.available_permits() as u64, &[]);
|
||||
})
|
||||
.with_description(
|
||||
@@ -86,63 +86,63 @@ impl BlockManagerMetrics {
|
||||
.init(),
|
||||
|
||||
resync_counter: meter
|
||||
.u64_counter("block.resync_counter")
|
||||
.u64_counter("garage_block.resync_counter")
|
||||
.with_description("Number of calls to resync_block")
|
||||
.init()
|
||||
.bind(&[]),
|
||||
resync_error_counter: meter
|
||||
.u64_counter("block.resync_error_counter")
|
||||
.u64_counter("garage_block.resync_error_counter")
|
||||
.with_description("Number of calls to resync_block that returned an error")
|
||||
.init()
|
||||
.bind(&[]),
|
||||
resync_duration: meter
|
||||
.f64_value_recorder("block.resync_duration")
|
||||
.f64_value_recorder("garage_block.resync_duration")
|
||||
.with_description("Duration of resync_block operations")
|
||||
.init()
|
||||
.bind(&[]),
|
||||
resync_send_counter: meter
|
||||
.u64_counter("block.resync_send_counter")
|
||||
.u64_counter("garage_block.resync_send_counter")
|
||||
.with_description("Number of blocks sent to another node in resync operations")
|
||||
.init(),
|
||||
resync_recv_counter: meter
|
||||
.u64_counter("block.resync_recv_counter")
|
||||
.u64_counter("garage_block.resync_recv_counter")
|
||||
.with_description("Number of blocks received from other nodes in resync operations")
|
||||
.init()
|
||||
.bind(&[]),
|
||||
|
||||
bytes_read: meter
|
||||
.u64_counter("block.bytes_read")
|
||||
.u64_counter("garage_block.bytes_read")
|
||||
.with_description("Number of bytes read from disk")
|
||||
.init()
|
||||
.bind(&[]),
|
||||
block_read_duration: meter
|
||||
.f64_value_recorder("block.read_duration")
|
||||
.f64_value_recorder("garage_block.read_duration")
|
||||
.with_description("Duration of block read operations")
|
||||
.init()
|
||||
.bind(&[]),
|
||||
block_read_semaphore_timeouts: meter
|
||||
.u64_counter("block.read_semaphore_timeouts")
|
||||
.u64_counter("garage_block.read_semaphore_timeouts")
|
||||
.with_description("Number of block reads that failed due to semaphore acquire timeout")
|
||||
.init()
|
||||
.bind(&[]),
|
||||
bytes_written: meter
|
||||
.u64_counter("block.bytes_written")
|
||||
.u64_counter("garage_block.bytes_written")
|
||||
.with_description("Number of bytes written to disk")
|
||||
.init()
|
||||
.bind(&[]),
|
||||
block_write_duration: meter
|
||||
.f64_value_recorder("block.write_duration")
|
||||
.f64_value_recorder("garage_block.write_duration")
|
||||
.with_description("Duration of block write operations")
|
||||
.init()
|
||||
.bind(&[]),
|
||||
delete_counter: meter
|
||||
.u64_counter("block.delete_counter")
|
||||
.u64_counter("garage_block.delete_counter")
|
||||
.with_description("Number of blocks deleted")
|
||||
.init()
|
||||
.bind(&[]),
|
||||
|
||||
corruption_counter: meter
|
||||
.u64_counter("block.corruption_counter")
|
||||
.u64_counter("garage_block.corruption_counter")
|
||||
.with_description("Data corruptions detected on block reads")
|
||||
.init()
|
||||
.bind(&[]),
|
||||
|
||||
+5
-5
@@ -14,23 +14,23 @@ impl RpcMetrics {
|
||||
let meter = global::meter("garage_rpc");
|
||||
RpcMetrics {
|
||||
rpc_counter: meter
|
||||
.u64_counter("rpc.request_counter")
|
||||
.u64_counter("garage_rpc.request_counter")
|
||||
.with_description("Number of RPC requests emitted")
|
||||
.init(),
|
||||
rpc_timeout_counter: meter
|
||||
.u64_counter("rpc.timeout_counter")
|
||||
.u64_counter("garage_rpc.timeout_counter")
|
||||
.with_description("Number of RPC timeouts")
|
||||
.init(),
|
||||
rpc_netapp_error_counter: meter
|
||||
.u64_counter("rpc.netapp_error_counter")
|
||||
.u64_counter("garage_rpc.netapp_error_counter")
|
||||
.with_description("Number of communication errors (errors in the Netapp library)")
|
||||
.init(),
|
||||
rpc_garage_error_counter: meter
|
||||
.u64_counter("rpc.garage_error_counter")
|
||||
.u64_counter("garage_rpc.garage_error_counter")
|
||||
.with_description("Number of RPC errors (errors happening when handling the RPC)")
|
||||
.init(),
|
||||
rpc_duration: meter
|
||||
.f64_value_recorder("rpc.duration")
|
||||
.f64_value_recorder("garage_rpc.duration")
|
||||
.with_description("Duration of RPCs")
|
||||
.init(),
|
||||
}
|
||||
|
||||
+46
-43
@@ -110,7 +110,7 @@ impl SystemMetrics {
|
||||
_cluster_healthy: {
|
||||
let get_health = get_health.clone();
|
||||
meter
|
||||
.u64_value_observer("cluster_healthy", move |observer| {
|
||||
.u64_value_observer("garage_cluster_healthy", move |observer| {
|
||||
let h = get_health();
|
||||
if h.status == ClusterHealthStatus::Healthy {
|
||||
observer.observe(1, &[]);
|
||||
@@ -123,7 +123,7 @@ impl SystemMetrics {
|
||||
},
|
||||
_cluster_available: {
|
||||
let get_health = get_health.clone();
|
||||
meter.u64_value_observer("cluster_available", move |observer| {
|
||||
meter.u64_value_observer("garage_cluster_available", move |observer| {
|
||||
let h = get_health();
|
||||
if h.status != ClusterHealthStatus::Unavailable {
|
||||
observer.observe(1, &[]);
|
||||
@@ -137,7 +137,7 @@ impl SystemMetrics {
|
||||
_known_nodes: {
|
||||
let get_health = get_health.clone();
|
||||
meter
|
||||
.u64_value_observer("cluster_known_nodes", move |observer| {
|
||||
.u64_value_observer("garage_cluster_known_nodes", move |observer| {
|
||||
let h = get_health();
|
||||
observer.observe(h.known_nodes as u64, &[]);
|
||||
})
|
||||
@@ -147,7 +147,7 @@ impl SystemMetrics {
|
||||
_connected_nodes: {
|
||||
let get_health = get_health.clone();
|
||||
meter
|
||||
.u64_value_observer("cluster_connected_nodes", move |observer| {
|
||||
.u64_value_observer("garage_cluster_connected_nodes", move |observer| {
|
||||
let h = get_health();
|
||||
observer.observe(h.connected_nodes as u64, &[]);
|
||||
})
|
||||
@@ -157,7 +157,7 @@ impl SystemMetrics {
|
||||
_storage_nodes: {
|
||||
let get_health = get_health.clone();
|
||||
meter
|
||||
.u64_value_observer("cluster_storage_nodes", move |observer| {
|
||||
.u64_value_observer("garage_cluster_storage_nodes", move |observer| {
|
||||
let h = get_health();
|
||||
observer.observe(h.storage_nodes as u64, &[]);
|
||||
})
|
||||
@@ -167,7 +167,7 @@ impl SystemMetrics {
|
||||
_storage_nodes_ok: {
|
||||
let get_health = get_health.clone();
|
||||
meter
|
||||
.u64_value_observer("cluster_storage_nodes_ok", move |observer| {
|
||||
.u64_value_observer("garage_cluster_storage_nodes_ok", move |observer| {
|
||||
let h = get_health();
|
||||
observer.observe(h.storage_nodes_ok as u64, &[]);
|
||||
})
|
||||
@@ -177,7 +177,7 @@ impl SystemMetrics {
|
||||
_partitions: {
|
||||
let get_health = get_health.clone();
|
||||
meter
|
||||
.u64_value_observer("cluster_partitions", move |observer| {
|
||||
.u64_value_observer("garage_cluster_partitions", move |observer| {
|
||||
let h = get_health();
|
||||
observer.observe(h.partitions as u64, &[]);
|
||||
})
|
||||
@@ -187,7 +187,7 @@ impl SystemMetrics {
|
||||
_partitions_quorum: {
|
||||
let get_health = get_health.clone();
|
||||
meter
|
||||
.u64_value_observer("cluster_partitions_quorum", move |observer| {
|
||||
.u64_value_observer("garage_cluster_partitions_quorum", move |observer| {
|
||||
let h = get_health();
|
||||
observer.observe(h.partitions_quorum as u64, &[]);
|
||||
})
|
||||
@@ -199,7 +199,7 @@ impl SystemMetrics {
|
||||
_partitions_all_ok: {
|
||||
let get_health = get_health.clone();
|
||||
meter
|
||||
.u64_value_observer("cluster_partitions_all_ok", move |observer| {
|
||||
.u64_value_observer("garage_cluster_partitions_all_ok", move |observer| {
|
||||
let h = get_health();
|
||||
observer.observe(h.partitions_all_ok as u64, &[]);
|
||||
})
|
||||
@@ -213,7 +213,7 @@ impl SystemMetrics {
|
||||
_layout_node_connected: {
|
||||
let system = system.clone();
|
||||
meter
|
||||
.u64_value_observer("cluster_layout_node_connected", move |observer| {
|
||||
.u64_value_observer("garage_cluster_layout_node_connected", move |observer| {
|
||||
let layout = system.cluster_layout();
|
||||
let nodes = system.get_known_nodes();
|
||||
for id in layout.all_nodes().unwrap_or_default().iter() {
|
||||
@@ -260,44 +260,47 @@ impl SystemMetrics {
|
||||
_layout_node_disconnected_time: {
|
||||
let system = system.clone();
|
||||
meter
|
||||
.u64_value_observer("cluster_layout_node_disconnected_time", move |observer| {
|
||||
let layout = system.cluster_layout();
|
||||
let nodes = system.get_known_nodes();
|
||||
for id in layout.all_nodes().unwrap_or_default().iter() {
|
||||
let mut kv = vec![KeyValue::new("id", format!("{:?}", id))];
|
||||
if let Some(role) = layout
|
||||
.current()
|
||||
.ok()
|
||||
.and_then(|l| l.roles.get(id))
|
||||
.and_then(|r| r.0.as_ref())
|
||||
{
|
||||
kv.push(KeyValue::new("role_zone", role.zone.clone()));
|
||||
match role.capacity {
|
||||
Some(cap) => {
|
||||
kv.push(KeyValue::new("role_capacity", cap as i64));
|
||||
kv.push(KeyValue::new("role_gateway", 0));
|
||||
}
|
||||
None => {
|
||||
kv.push(KeyValue::new("role_gateway", 1));
|
||||
.u64_value_observer(
|
||||
"garage_cluster_layout_node_disconnected_time",
|
||||
move |observer| {
|
||||
let layout = system.cluster_layout();
|
||||
let nodes = system.get_known_nodes();
|
||||
for id in layout.all_nodes().unwrap_or_default().iter() {
|
||||
let mut kv = vec![KeyValue::new("id", format!("{:?}", id))];
|
||||
if let Some(role) = layout
|
||||
.current()
|
||||
.ok()
|
||||
.and_then(|l| l.roles.get(id))
|
||||
.and_then(|r| r.0.as_ref())
|
||||
{
|
||||
kv.push(KeyValue::new("role_zone", role.zone.clone()));
|
||||
match role.capacity {
|
||||
Some(cap) => {
|
||||
kv.push(KeyValue::new("role_capacity", cap as i64));
|
||||
kv.push(KeyValue::new("role_gateway", 0));
|
||||
}
|
||||
None => {
|
||||
kv.push(KeyValue::new("role_gateway", 1));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(node) = nodes.iter().find(|n| n.id == *id) {
|
||||
// TODO: see comment above
|
||||
// kv.push(KeyValue::new("address", node.addr.to_string()));
|
||||
// kv.push(KeyValue::new(
|
||||
// "hostname",
|
||||
// node.status.hostname.clone(),
|
||||
// ));
|
||||
if node.is_up {
|
||||
observer.observe(0, &kv);
|
||||
} else if let Some(secs) = node.last_seen_secs_ago {
|
||||
observer.observe(secs, &kv);
|
||||
if let Some(node) = nodes.iter().find(|n| n.id == *id) {
|
||||
// TODO: see comment above
|
||||
// kv.push(KeyValue::new("address", node.addr.to_string()));
|
||||
// kv.push(KeyValue::new(
|
||||
// "hostname",
|
||||
// node.status.hostname.clone(),
|
||||
// ));
|
||||
if node.is_up {
|
||||
observer.observe(0, &kv);
|
||||
} else if let Some(secs) = node.last_seen_secs_ago {
|
||||
observer.observe(secs, &kv);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
},
|
||||
)
|
||||
.with_description(
|
||||
"Time (in seconds) since last connection to nodes in the cluster layout",
|
||||
)
|
||||
|
||||
+13
-13
@@ -34,7 +34,7 @@ impl TableMetrics {
|
||||
TableMetrics {
|
||||
_table_size: meter
|
||||
.u64_value_observer(
|
||||
"table.size",
|
||||
"garage_table.size",
|
||||
move |observer| {
|
||||
if let Ok(value) = store.approximate_len() {
|
||||
observer.observe(
|
||||
@@ -48,7 +48,7 @@ impl TableMetrics {
|
||||
.init(),
|
||||
_merkle_tree_size: meter
|
||||
.u64_value_observer(
|
||||
"table.merkle_tree_size",
|
||||
"garage_table.merkle_tree_size",
|
||||
move |observer| {
|
||||
if let Ok(value) = merkle_tree.approximate_len() {
|
||||
observer.observe(
|
||||
@@ -62,7 +62,7 @@ impl TableMetrics {
|
||||
.init(),
|
||||
_merkle_todo_len: meter
|
||||
.u64_value_observer(
|
||||
"table.merkle_updater_todo_queue_length",
|
||||
"garage_table.merkle_updater_todo_queue_length",
|
||||
move |observer| {
|
||||
if let Ok(v) = merkle_todo.approximate_len() {
|
||||
observer.observe(
|
||||
@@ -76,7 +76,7 @@ impl TableMetrics {
|
||||
.init(),
|
||||
_insert_queue_len: meter
|
||||
.u64_value_observer(
|
||||
"table.insert_queue_length",
|
||||
"garage_table.insert_queue_length",
|
||||
move |observer| {
|
||||
if let Ok(v) = insert_queue.approximate_len() {
|
||||
observer.observe(
|
||||
@@ -90,7 +90,7 @@ impl TableMetrics {
|
||||
.init(),
|
||||
_gc_todo_len: meter
|
||||
.u64_value_observer(
|
||||
"table.gc_todo_queue_length",
|
||||
"garage_table.gc_todo_queue_length",
|
||||
move |observer| {
|
||||
if let Ok(value) = gc_todo.approximate_len() {
|
||||
observer.observe(
|
||||
@@ -104,43 +104,43 @@ impl TableMetrics {
|
||||
.init(),
|
||||
|
||||
get_request_counter: meter
|
||||
.u64_counter("table.get_request_counter")
|
||||
.u64_counter("garage_table.get_request_counter")
|
||||
.with_description("Number of get/get_range requests internally made on this table")
|
||||
.init()
|
||||
.bind(&[KeyValue::new("table_name", table_name)]),
|
||||
get_request_duration: meter
|
||||
.f64_value_recorder("table.get_request_duration")
|
||||
.f64_value_recorder("garage_table.get_request_duration")
|
||||
.with_description("Duration of get/get_range requests internally made on this table, in seconds")
|
||||
.init()
|
||||
.bind(&[KeyValue::new("table_name", table_name)]),
|
||||
put_request_counter: meter
|
||||
.u64_counter("table.put_request_counter")
|
||||
.u64_counter("garage_table.put_request_counter")
|
||||
.with_description("Number of insert/insert_many requests internally made on this table")
|
||||
.init()
|
||||
.bind(&[KeyValue::new("table_name", table_name)]),
|
||||
put_request_duration: meter
|
||||
.f64_value_recorder("table.put_request_duration")
|
||||
.f64_value_recorder("garage_table.put_request_duration")
|
||||
.with_description("Duration of insert/insert_many requests internally made on this table, in seconds")
|
||||
.init()
|
||||
.bind(&[KeyValue::new("table_name", table_name)]),
|
||||
|
||||
internal_update_counter: meter
|
||||
.u64_counter("table.internal_update_counter")
|
||||
.u64_counter("garage_table.internal_update_counter")
|
||||
.with_description("Number of value updates where the value actually changes (includes creation of new key and update of existing key)")
|
||||
.init()
|
||||
.bind(&[KeyValue::new("table_name", table_name)]),
|
||||
internal_delete_counter: meter
|
||||
.u64_counter("table.internal_delete_counter")
|
||||
.u64_counter("garage_table.internal_delete_counter")
|
||||
.with_description("Number of value deletions in the tree (due to GC or repartitioning)")
|
||||
.init()
|
||||
.bind(&[KeyValue::new("table_name", table_name)]),
|
||||
|
||||
sync_items_sent: meter
|
||||
.u64_counter("table.sync_items_sent")
|
||||
.u64_counter("garage_table.sync_items_sent")
|
||||
.with_description("Number of data items sent to other nodes during resync procedures")
|
||||
.init(),
|
||||
sync_items_received: meter
|
||||
.u64_counter("table.sync_items_received")
|
||||
.u64_counter("garage_table.sync_items_received")
|
||||
.with_description("Number of data items received from other nodes during resync procedures")
|
||||
.init(),
|
||||
}
|
||||
|
||||
@@ -54,15 +54,15 @@ impl WebMetrics {
|
||||
let meter = global::meter("garage/web");
|
||||
Self {
|
||||
request_counter: meter
|
||||
.u64_counter("web.request_counter")
|
||||
.u64_counter("garage_web.request_counter")
|
||||
.with_description("Number of requests to the web endpoint")
|
||||
.init(),
|
||||
error_counter: meter
|
||||
.u64_counter("web.error_counter")
|
||||
.u64_counter("garage_web.error_counter")
|
||||
.with_description("Number of requests to the web endpoint resulting in errors")
|
||||
.init(),
|
||||
request_duration: meter
|
||||
.f64_value_recorder("web.request_duration")
|
||||
.f64_value_recorder("garage_web.request_duration")
|
||||
.with_description("Duration of requests to the web endpoint")
|
||||
.init(),
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user