gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-05 12:57:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(obs): add bounded metrics dimensions (#5645)

Browse Source 
* feat(obs): add drive topology detail metrics

Expose additive drive info, topology, state, and per-drive API metrics while preserving the existing drive metric label sets.

Backlog: rustfs/backlog#1655

Co-Authored-By: heihutu <heihutu@gmail.com>

* fix(obs): preserve suspect drive runtime state

Keep suspect as a bounded drive runtime state and avoid all-zero runtime_state samples for that storage health state.

Co-Authored-By: heihutu <heihutu@gmail.com>

* fix(obs): skip unknown drive inode samples

Avoid exporting zero inode gauges for missing or stale drive snapshots and ignore zero-count API latency buckets.

Co-Authored-By: heihutu <heihutu@gmail.com>

* feat(obs): add scanner source work detail metrics

Expose additive scanner source and cycle work metrics with bounded server/source/state labels while leaving the existing aggregate scanner metrics unchanged.

Co-Authored-By: heihutu <heihutu@gmail.com>

* feat(obs): add ilm action detail metrics

Expose additive ILM action/state task metrics with a server label while preserving the existing aggregate ILM series.

Co-Authored-By: heihutu <heihutu@gmail.com>

* feat(obs): add delivery target server metrics

Expose additive audit and notification delivery target metrics with server labels and extend removed-target tombstones for the server-aware series.

Co-Authored-By: heihutu <heihutu@gmail.com>

* feat(obs): add replication target flow metrics

Expose additive bucket replication target sent and failed-flow metrics while preserving existing bucket aggregates and target backlog series.

Co-Authored-By: heihutu <heihutu@gmail.com>

* feat(obs): add request server metrics

Expose additive API request metrics with server labels while preserving the existing request and traffic metric label sets.

Co-Authored-By: heihutu <heihutu@gmail.com>

* style(obs): apply rustfmt to metrics changes

Apply rustfmt output to the metrics dimension changes without altering behavior.

Co-Authored-By: heihutu <heihutu@gmail.com>

* style(obs): reuse audit target label constant

Use the exported audit target_id label constant for legacy audit target metrics.

Co-Authored-By: heihutu <heihutu@gmail.com>

* feat(obs): populate drive disk metrics

Co-Authored-By: heihutu <heihutu@gmail.com>

* feat(obs): add scanner bucket drive result metrics

Co-Authored-By: heihutu <heihutu@gmail.com>

* feat(obs): add replication proxy server metrics

Co-Authored-By: heihutu <heihutu@gmail.com>

* fix(obs): address metric liveness review

Use checked division for drive API latency aggregation and keep recovered drive, scanner current-cycle, replication flow, audit target, and notification target series from retaining stale values.

Co-Authored-By: heihutu <heihutu@gmail.com>

* fix(obs): address metric dimension review

Co-Authored-By: heihutu <heihutu@gmail.com>

* fix(obs): address additional metric review

Co-Authored-By: heihutu <heihutu@gmail.com>

* fix(obs): count drive calls at start

Co-Authored-By: heihutu <heihutu@gmail.com>

* fix(obs): address metrics dimension review

Co-Authored-By: heihutu <heihutu@gmail.com>

* fix(metrics): address dimension review gaps

Co-Authored-By: heihutu <heihutu@gmail.com>

* fix(metrics): address scanner review follow-ups

Co-Authored-By: heihutu <heihutu@gmail.com>

* fix(metrics): address runtime review follow-ups

Co-Authored-By: heihutu <heihutu@gmail.com>

* fix(metrics): reduce disk metric contention

Co-Authored-By: heihutu <heihutu@gmail.com>

* fix(metrics): address runtime review follow-ups

Co-Authored-By: heihutu <heihutu@gmail.com>

* fix(metrics): retire stale dimension series

Co-Authored-By: heihutu <heihutu@gmail.com>

---------

Co-authored-by: heihutu <heihutu@gmail.com>
This commit is contained in:
houseme
2026-08-03 09:03:34 +08:00
committed by GitHub
parent 988cd8adbb
commit 035ce5d784
36 changed files with 6282 additions and 666 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.docker/observability/grafana/dashboards/rustfs.json
+947
View File
@@ -11500,6 +11500,797 @@
],
"title": "Compression Operations Rate",
"type": "timeseries"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 332
},
"id": 531,
"panels": [],
"title": "Metrics Dimensions Drilldown",
"type": "row"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "smooth",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"showValues": false,
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
}
]
},
"unit": "ops"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 333
},
"id": 532,
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max"
],
"displayMode": "table",
"placement": "right",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "multi",
"sort": "desc"
}
},
"pluginVersion": "12.3.2",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"range": true,
"refId": "A",
"expr": "sum by (server, name, type) (rate(rustfs_api_requests_requests_total_by_server{job=~\"$job\",server=~\"$server\",name=~\"$api\"}[$__rate_interval]))",
"legendFormat": "{{server}} | {{name}} | {{type}}"
}
],
"title": "API Requests by Server and API",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "smooth",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"showValues": false,
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
}
]
},
"unit": "s"
},
"overrides": [
{
"matcher": {
"id": "byFrameRefID",
"options": "A"
},
"properties": [
{
"id": "unit",
"value": "none"
}
]
}
]
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 333
},
"id": 533,
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max"
],
"displayMode": "table",
"placement": "right",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "multi",
"sort": "desc"
}
},
"pluginVersion": "12.3.2",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"range": true,
"refId": "A",
"expr": "max by (server, drive, pool_index, set_index, drive_index, state) (rustfs_system_drive_runtime_state{job=~\"$job\",server=~\"$server\",drive=~\"$drive\"})",
"legendFormat": "{{server}} | {{drive}} | p{{pool_index}}/s{{set_index}}/d{{drive_index}} | {{state}}"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"range": true,
"refId": "B",
"expr": "max by (server, drive, pool_index, set_index, drive_index) (rustfs_system_drive_offline_duration_seconds{job=~\"$job\",server=~\"$server\",drive=~\"$drive\"})",
"legendFormat": "{{server}} | {{drive}} | offline seconds"
}
],
"title": "Drive Runtime State and Offline Duration",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "smooth",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"showValues": false,
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
}
]
},
"unit": "ops"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 341
},
"id": 534,
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max"
],
"displayMode": "table",
"placement": "right",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "multi",
"sort": "desc"
}
},
"pluginVersion": "12.3.2",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"range": true,
"refId": "A",
"expr": "sum by (server, drive, pool_index, set_index, drive_index, api) (rate(rustfs_system_drive_api_calls_total{job=~\"$job\",server=~\"$server\",drive=~\"$drive\",api=~\"$drive_api\"}[$__rate_interval]))",
"legendFormat": "{{server}} | {{drive}} | p{{pool_index}}/s{{set_index}}/d{{drive_index}} | {{api}}"
}
],
"title": "Drive API Calls by Operation",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "smooth",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"showValues": false,
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
}
]
},
"unit": "ops"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 341
},
"id": 535,
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max"
],
"displayMode": "table",
"placement": "right",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "multi",
"sort": "desc"
}
},
"pluginVersion": "12.3.2",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"range": true,
"refId": "A",
"expr": "sum by (server, source, state) (rate(rustfs_scanner_source_work_total{job=~\"$job\",server=~\"$server\",source=~\"$scanner_source\"}[$__rate_interval]))",
"legendFormat": "{{server}} | {{source}} | {{state}}"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"range": true,
"refId": "B",
"expr": "sum by (server, cycle_scope, source, state) (rustfs_scanner_cycle_source_work{job=~\"$job\",server=~\"$server\",source=~\"$scanner_source\"})",
"legendFormat": "{{server}} | {{cycle_scope}} | {{source}} | {{state}}"
}
],
"title": "Scanner Source Work by Server",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "smooth",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"showValues": false,
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
}
]
},
"unit": "ops"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 349
},
"id": 536,
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max"
],
"displayMode": "table",
"placement": "right",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "multi",
"sort": "desc"
}
},
"pluginVersion": "12.3.2",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"range": true,
"refId": "A",
"expr": "sum by (server, bucket, drive, result) (rate(rustfs_scanner_bucket_drive_result_total{job=~\"$job\",server=~\"$server\",bucket=~\"$bucket\",drive=~\"$drive\",result=~\"$scanner_result\"}[$__rate_interval]))",
"legendFormat": "{{server}} | {{bucket}} | {{drive}} | {{result}}"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"range": true,
"refId": "B",
"expr": "sum by (server, cycle_scope, bucket, drive, result) (rustfs_scanner_cycle_bucket_drive_result{job=~\"$job\",server=~\"$server\",bucket=~\"$bucket\",drive=~\"$drive\",result=~\"$scanner_result\"})",
"legendFormat": "{{server}} | {{cycle_scope}} | {{bucket}} | {{drive}} | {{result}}"
}
],
"title": "Scanner Bucket Drive Results",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "smooth",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"showValues": false,
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
}
]
},
"unit": "ops"
},
"overrides": [
{
"matcher": {
"id": "byFrameRefID",
"options": "B"
},
"properties": [
{
"id": "unit",
"value": "Bps"
}
]
}
]
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 349
},
"id": 537,
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max"
],
"displayMode": "table",
"placement": "right",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "multi",
"sort": "desc"
}
},
"pluginVersion": "12.3.2",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"range": true,
"refId": "A",
"expr": "sum by (bucket, target_arn) (rate(rustfs_bucket_replication_target_sent_count{job=~\"$job\",bucket=~\"$bucket\",target_arn=~\"$target_arn\"}[$__rate_interval]))",
"legendFormat": "sent objects | {{bucket}} | {{target_arn}}"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"range": true,
"refId": "B",
"expr": "sum by (bucket, target_arn) (rate(rustfs_bucket_replication_target_sent_bytes{job=~\"$job\",bucket=~\"$bucket\",target_arn=~\"$target_arn\"}[$__rate_interval]))",
"legendFormat": "sent bytes | {{bucket}} | {{target_arn}}"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"range": true,
"refId": "C",
"expr": "sum by (bucket, target_arn) (rate(rustfs_bucket_replication_target_total_failed_count{job=~\"$job\",bucket=~\"$bucket\",target_arn=~\"$target_arn\"}[$__rate_interval]))",
"legendFormat": "failed objects | {{bucket}} | {{target_arn}}"
}
],
"title": "Bucket Replication Target Flow",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "smooth",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"showValues": false,
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 357
},
"id": 538,
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max"
],
"displayMode": "table",
"placement": "right",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "multi",
"sort": "desc"
}
},
"pluginVersion": "12.3.2",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"range": true,
"refId": "A",
"expr": "max by (server, target_id) (rustfs_audit_target_queue_length_by_server{job=~\"$job\",server=~\"$server\"})",
"legendFormat": "audit queue | {{server}} | {{target_id}}"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"range": true,
"refId": "B",
"expr": "max by (server, action, state) (rustfs_ilm_action_tasks{job=~\"$job\",server=~\"$server\"})",
"legendFormat": "ilm | {{server}} | {{action}} | {{state}}"
}
],
"title": "Audit and ILM by Server",
"type": "timeseries"
}
],
"preload": false,
@@ -11551,6 +12342,32 @@
"sort": 1,
"type": "query"
},
{
"allValue": ".*",
"current": {
"text": "All",
"value": "$__all"
},
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"definition": "label_values(rustfs_system_drive_api_calls_total,api)",
"includeAll": true,
"label": "Drive API",
"multi": true,
"name": "drive_api",
"options": [],
"query": {
"qryType": 1,
"query": "label_values(rustfs_system_drive_api_calls_total,api)",
"refId": "PrometheusVariableQueryEditor-drive_api"
},
"refresh": 2,
"regex": "",
"sort": 1,
"type": "query"
},
{
"allValue": ".*",
"current": {
@@ -11670,6 +12487,136 @@
"regex": "",
"sort": 1,
"type": "query"
},
{
"allValue": ".*",
"current": {
"text": "All",
"value": "$__all"
},
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"definition": "label_values(rustfs_api_requests_requests_total_by_server,server)",
"includeAll": true,
"label": "Server",
"multi": true,
"name": "server",
"options": [],
"query": {
"qryType": 1,
"query": "label_values(rustfs_api_requests_requests_total_by_server,server)",
"refId": "PrometheusVariableQueryEditor-server"
},
"refresh": 2,
"regex": "",
"sort": 1,
"type": "query"
},
{
"allValue": ".*",
"current": {
"text": "All",
"value": "$__all"
},
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"definition": "label_values(rustfs_api_requests_requests_total_by_server,name)",
"includeAll": true,
"label": "API",
"multi": true,
"name": "api",
"options": [],
"query": {
"qryType": 1,
"query": "label_values(rustfs_api_requests_requests_total_by_server,name)",
"refId": "PrometheusVariableQueryEditor-api"
},
"refresh": 2,
"regex": "",
"sort": 1,
"type": "query"
},
{
"allValue": ".*",
"current": {
"text": "All",
"value": "$__all"
},
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"definition": "label_values({__name__=\"rustfs_bucket_replication_target_sent_count\",bucket=~\"$bucket\"},target_arn)",
"includeAll": true,
"label": "Target ARN",
"multi": true,
"name": "target_arn",
"options": [],
"query": {
"qryType": 1,
"query": "label_values({__name__=\"rustfs_bucket_replication_target_sent_count\",bucket=~\"$bucket\"},target_arn)",
"refId": "PrometheusVariableQueryEditor-target_arn"
},
"refresh": 2,
"regex": "",
"sort": 1,
"type": "query"
},
{
"allValue": ".*",
"current": {
"text": "All",
"value": "$__all"
},
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"definition": "label_values(rustfs_scanner_source_work_total,source)",
"includeAll": true,
"label": "Scanner Source",
"multi": true,
"name": "scanner_source",
"options": [],
"query": {
"qryType": 1,
"query": "label_values(rustfs_scanner_source_work_total,source)",
"refId": "PrometheusVariableQueryEditor-scanner_source"
},
"refresh": 2,
"regex": "",
"sort": 1,
"type": "query"
},
{
"allValue": ".*",
"current": {
"text": "All",
"value": "$__all"
},
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"definition": "label_values(rustfs_scanner_bucket_drive_result_total,result)",
"includeAll": true,
"label": "Scanner Result",
"multi": true,
"name": "scanner_result",
"options": [],
"query": {
"qryType": 1,
"query": "label_values(rustfs_scanner_bucket_drive_result_total,result)",
"refId": "PrometheusVariableQueryEditor-scanner_result"
},
"refresh": 2,
"regex": "",
"sort": 1,
"type": "query"
}
]
},
crates/common/src/metrics.rs
+316 -13
View File
@@ -708,6 +708,42 @@ struct ScannerDiskBucketScanState {
active: u64,
}
type ScannerDiskBucketScanKey = (String, String);
#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
pub struct ScannerDiskBucketScanSnapshot {
pub pool: String,
pub set: String,
pub concurrency_limit: u64,
pub queued: u64,
pub active: u64,
}
#[derive(Clone, Debug, PartialEq, Eq, Hash)]
struct ScannerBucketDriveResultKey {
bucket: String,
drive: String,
result: String,
}
impl ScannerBucketDriveResultKey {
fn new(bucket: impl Into<String>, drive: impl Into<String>, result: impl Into<String>) -> Self {
Self {
bucket: bucket.into(),
drive: drive.into(),
result: result.into(),
}
}
}
const MAX_SCANNER_BUCKET_DRIVE_RESULT_KEYS: usize = 4096;
#[derive(Clone, Copy, Debug)]
struct ScannerBucketDriveResultValue {
count: u64,
last_seen: u64,
}
// ---------------------------------------------------------------------------
// Metrics
// ---------------------------------------------------------------------------
@@ -738,7 +774,11 @@ pub struct Metrics {
scanner_set_scan_concurrency_limit: AtomicU64,
scanner_set_scans_queued: AtomicU64,
scanner_set_scans_active: AtomicU64,
scanner_disk_bucket_scan_states: Mutex<HashMap<String, ScannerDiskBucketScanState>>,
scanner_disk_bucket_scan_states: Mutex<HashMap<ScannerDiskBucketScanKey, ScannerDiskBucketScanState>>,
scanner_bucket_drive_results: Mutex<HashMap<ScannerBucketDriveResultKey, ScannerBucketDriveResultValue>>,
scanner_bucket_drive_result_clock: AtomicU64,
current_scan_cycle_bucket_drive_results_start: Mutex<HashMap<ScannerBucketDriveResultKey, u64>>,
last_scan_cycle_bucket_drive_results: Mutex<Vec<ScannerBucketDriveResultSnapshot>>,
scanner_leader_lock_state: RwLock<String>,
scanner_leader_lock_held: AtomicBool,
scanner_leader_lock_last_error: RwLock<String>,
@@ -958,6 +998,14 @@ pub struct ScannerSourceWorkSnapshot {
pub missed: u64,
}
#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
pub struct ScannerBucketDriveResultSnapshot {
pub bucket: String,
pub drive: String,
pub result: String,
pub count: u64,
}
#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
pub struct ScannerReplicationRepairSnapshot {
pub source: String,
@@ -1290,6 +1338,18 @@ pub struct ScannerMetricsReport {
pub partial_cycles: u64,
}
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
pub struct ScannerRuntimeDetailsReport {
#[serde(default)]
pub disk_bucket_scan_states: Vec<ScannerDiskBucketScanSnapshot>,
#[serde(default)]
pub bucket_drive_results: Vec<ScannerBucketDriveResultSnapshot>,
#[serde(default)]
pub current_cycle_bucket_drive_results: Vec<ScannerBucketDriveResultSnapshot>,
#[serde(default)]
pub last_cycle_bucket_drive_results: Vec<ScannerBucketDriveResultSnapshot>,
}
impl CurrentCycle {
pub fn unmarshal(&mut self, buf: &[u8]) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
*self = rmp_serde::from_slice(buf)?;
@@ -1657,6 +1717,7 @@ pub fn emit_scan_cycle_superseded(duration: Duration) {
pub fn emit_scan_bucket_drive_complete(success: bool, bucket: &str, disk: &str, duration: Duration) {
let result = if success { "success" } else { "error" };
global_metrics().record_scanner_bucket_drive_result(bucket, disk, result);
metrics::counter!(
OTEL_SCANNER_BUCKETS_SCANNED,
"result" => result,
@@ -1673,6 +1734,7 @@ pub fn emit_scan_bucket_drive_complete(success: bool, bucket: &str, disk: &str,
}
pub fn emit_scan_bucket_drive_partial(bucket: &str, disk: &str, duration: Duration) {
global_metrics().record_scanner_bucket_drive_result(bucket, disk, SCAN_CYCLE_RESULT_PARTIAL_LABEL);
metrics::counter!(
OTEL_SCANNER_BUCKETS_SCANNED,
"result" => SCAN_CYCLE_RESULT_PARTIAL_LABEL,
@@ -1723,6 +1785,10 @@ impl Metrics {
scanner_set_scans_queued: AtomicU64::new(0),
scanner_set_scans_active: AtomicU64::new(0),
scanner_disk_bucket_scan_states: Mutex::new(HashMap::new()),
scanner_bucket_drive_results: Mutex::new(HashMap::new()),
scanner_bucket_drive_result_clock: AtomicU64::new(0),
current_scan_cycle_bucket_drive_results_start: Mutex::new(HashMap::new()),
last_scan_cycle_bucket_drive_results: Mutex::new(Vec::new()),
scanner_leader_lock_state: RwLock::new("unknown".to_string()),
scanner_leader_lock_held: AtomicBool::new(false),
scanner_leader_lock_last_error: RwLock::new(String::new()),
@@ -2293,7 +2359,7 @@ impl Metrics {
queued: Option<usize>,
active: Option<usize>,
) {
let key = format!("{pool}/{set}");
let key = (pool.to_string(), set.to_string());
let mut states = self
.scanner_disk_bucket_scan_states
.lock()
@@ -2310,6 +2376,44 @@ impl Metrics {
}
}
pub fn record_scanner_bucket_drive_result(&self, bucket: &str, drive: &str, result: &str) {
if bucket.is_empty() || drive.is_empty() || result.is_empty() {
return;
}
let key = ScannerBucketDriveResultKey::new(bucket, drive, result);
let mut results = self
.scanner_bucket_drive_results
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
let last_seen = self.scanner_bucket_drive_result_clock.fetch_add(1, Ordering::Relaxed);
if !results.contains_key(&key)
&& results.len() >= MAX_SCANNER_BUCKET_DRIVE_RESULT_KEYS
&& let Some(stale_key) = results
.iter()
.min_by(|left, right| {
left.1
.last_seen
.cmp(&right.1.last_seen)
.then_with(|| left.0.bucket.cmp(&right.0.bucket))
.then_with(|| left.0.drive.cmp(&right.0.drive))
.then_with(|| left.0.result.cmp(&right.0.result))
})
.map(|(key, _)| key.clone())
{
results.remove(&stale_key);
}
match results.entry(key) {
std::collections::hash_map::Entry::Occupied(mut entry) => {
let value = entry.get_mut();
value.count = value.count.saturating_add(1);
value.last_seen = last_seen;
}
std::collections::hash_map::Entry::Vacant(entry) => {
entry.insert(ScannerBucketDriveResultValue { count: 1, last_seen });
}
}
}
// -----------------------------------------------------------------------
// Read-side helpers
// -----------------------------------------------------------------------
@@ -2481,6 +2585,11 @@ impl Metrics {
&self.current_scan_cycle_replication_repair_work_start,
&replication_repair_snapshot,
);
let bucket_drive_results = self.scanner_bucket_drive_result_counts();
match self.current_scan_cycle_bucket_drive_results_start.lock() {
Ok(mut start) => *start = bucket_drive_results,
Err(poisoned) => *poisoned.into_inner() = bucket_drive_results,
}
self.current_scan_cycle_work_active.store(true, Ordering::Release);
snapshot
}
@@ -2493,6 +2602,11 @@ impl Metrics {
self.record_scan_cycle_work(work);
self.record_scan_cycle_source_work(&source_work);
self.record_scan_cycle_replication_repair_work(&replication_repair_work);
let bucket_drive_results = self.current_cycle_bucket_drive_result_snapshots();
match self.last_scan_cycle_bucket_drive_results.lock() {
Ok(mut last) => *last = bucket_drive_results,
Err(poisoned) => *poisoned.into_inner() = bucket_drive_results,
}
self.current_scan_cycle_work_active.store(false, Ordering::Release);
}
@@ -2576,6 +2690,104 @@ impl Metrics {
}
}
fn scanner_bucket_drive_result_counts(&self) -> HashMap<ScannerBucketDriveResultKey, u64> {
self.scanner_bucket_drive_results
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.iter()
.map(|(key, value)| (key.clone(), value.count))
.collect()
}
fn scanner_bucket_drive_result_snapshots(
counts: impl IntoIterator<Item = (ScannerBucketDriveResultKey, u64)>,
) -> Vec<ScannerBucketDriveResultSnapshot> {
let mut snapshots = counts
.into_iter()
.filter(|(_, count)| *count > 0)
.map(|(key, count)| ScannerBucketDriveResultSnapshot {
bucket: key.bucket,
drive: key.drive,
result: key.result,
count,
})
.collect::<Vec<_>>();
snapshots.sort_by(|left, right| {
left.bucket
.cmp(&right.bucket)
.then_with(|| left.drive.cmp(&right.drive))
.then_with(|| left.result.cmp(&right.result))
});
snapshots
}
fn scanner_bucket_drive_result_counter_snapshots(&self) -> Vec<ScannerBucketDriveResultSnapshot> {
Self::scanner_bucket_drive_result_snapshots(self.scanner_bucket_drive_result_counts())
}
fn current_cycle_bucket_drive_result_snapshots(&self) -> Vec<ScannerBucketDriveResultSnapshot> {
let current = self.scanner_bucket_drive_result_counts();
let start = self
.current_scan_cycle_bucket_drive_results_start
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.clone();
Self::scanner_bucket_drive_result_snapshots(current.into_iter().filter_map(|(key, count)| {
let delta = count.saturating_sub(start.get(&key).copied().unwrap_or_default());
(delta > 0).then_some((key, delta))
}))
}
pub fn scanner_runtime_details_report(&self) -> ScannerRuntimeDetailsReport {
self.scanner_runtime_details_report_for_active(self.current_scan_cycle_work_active.load(Ordering::Acquire))
}
fn scanner_runtime_details_report_for_active(&self, current_cycle_active: bool) -> ScannerRuntimeDetailsReport {
let current_cycle_bucket_drive_results = if current_cycle_active {
self.current_cycle_bucket_drive_result_snapshots()
} else {
Vec::new()
};
ScannerRuntimeDetailsReport {
disk_bucket_scan_states: self.scanner_disk_bucket_scan_state_snapshots(),
bucket_drive_results: self.scanner_bucket_drive_result_counter_snapshots(),
current_cycle_bucket_drive_results,
last_cycle_bucket_drive_results: self
.last_scan_cycle_bucket_drive_results
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.clone(),
}
}
fn scanner_disk_bucket_scan_state_snapshots(&self) -> Vec<ScannerDiskBucketScanSnapshot> {
let mut disk_bucket_scan_states = match self.scanner_disk_bucket_scan_states.lock() {
Ok(states) => states
.iter()
.map(|((pool, set), state)| ScannerDiskBucketScanSnapshot {
pool: pool.clone(),
set: set.clone(),
concurrency_limit: state.concurrency_limit,
queued: state.queued,
active: state.active,
})
.collect::<Vec<_>>(),
Err(poisoned) => poisoned
.into_inner()
.iter()
.map(|((pool, set), state)| ScannerDiskBucketScanSnapshot {
pool: pool.clone(),
set: set.clone(),
concurrency_limit: state.concurrency_limit,
queued: state.queued,
active: state.active,
})
.collect::<Vec<_>>(),
};
disk_bucket_scan_states.sort_by(|left, right| left.pool.cmp(&right.pool).then_with(|| left.set.cmp(&right.set)));
disk_bucket_scan_states
}
fn scanner_source_work_values(&self) -> Vec<ScannerSourceWorkValues> {
ScannerWorkSource::all()
.iter()
@@ -2761,7 +2973,12 @@ impl Metrics {
/// Build a full metrics report snapshot.
pub async fn report(&self) -> ScannerMetricsReport {
self.report_with_runtime_details().await.0
}
pub async fn report_with_runtime_details(&self) -> (ScannerMetricsReport, ScannerRuntimeDetailsReport) {
let mut m = ScannerMetricsReport::default();
let runtime_details;
let has_cycle = {
let cycle = self.cycle_info.read().await;
@@ -2775,6 +2992,7 @@ impl Metrics {
};
m.current_cycle_active = self.current_scan_cycle_work_active.load(Ordering::Acquire);
if m.current_cycle_active {
// Keep cycle_info before cycle-baseline locks so active scrapes cannot mix two cycle identities.
let current_work = self.scan_cycle_work_since(self.current_scan_cycle_work_start());
let current_source_work = self.scanner_source_work_since(&self.current_scan_cycle_source_work_start_values());
let current_replication_repair_work =
@@ -2797,6 +3015,7 @@ impl Metrics {
m.current_cycle_replication_repair =
self.scanner_replication_repair_work_snapshots(&current_replication_repair_work);
}
runtime_details = self.scanner_runtime_details_report_for_active(m.current_cycle_active);
has_cycle
};
@@ -2826,15 +3045,11 @@ impl Metrics {
m.current_set_scan_concurrency_limit = self.scanner_set_scan_concurrency_limit.load(Ordering::Relaxed);
m.current_set_scans_queued = self.scanner_set_scans_queued.load(Ordering::Relaxed);
m.current_set_scans_active = self.scanner_set_scans_active.load(Ordering::Relaxed);
let disk_bucket_scan_states = self.scanner_disk_bucket_scan_state_snapshots();
let (disk_scan_concurrency_limit, disk_bucket_scans_queued, disk_bucket_scans_active) =
match self.scanner_disk_bucket_scan_states.lock() {
Ok(states) => states.values().fold((0, 0, 0), |acc, state| {
(acc.0 + state.concurrency_limit, acc.1 + state.queued, acc.2 + state.active)
}),
Err(poisoned) => poisoned.into_inner().values().fold((0, 0, 0), |acc, state| {
(acc.0 + state.concurrency_limit, acc.1 + state.queued, acc.2 + state.active)
}),
};
disk_bucket_scan_states.iter().fold((0, 0, 0), |acc, state| {
(acc.0 + state.concurrency_limit, acc.1 + state.queued, acc.2 + state.active)
});
m.current_disk_scan_concurrency_limit = disk_scan_concurrency_limit;
m.current_disk_bucket_scans_queued = disk_bucket_scans_queued;
m.current_disk_bucket_scans_active = disk_bucket_scans_active;
@@ -3003,7 +3218,7 @@ impl Metrics {
m.pacing_pressure = scanner_pacing_pressure(&m);
m.maintenance_control = scanner_maintenance_control(&m);
m
(m, runtime_details)
}
}
@@ -4100,6 +4315,83 @@ mod tests {
assert_eq!(report.life_time_ops.get("scan_bucket_drive_failure"), Some(&1));
}
#[tokio::test]
async fn report_includes_structured_bucket_drive_results() {
let metrics = Metrics::new();
metrics.record_scanner_bucket_drive_result("photos", "/data1", "success");
let cycle_start = metrics.start_scan_cycle_work();
metrics.record_scanner_bucket_drive_result("photos", "/data1", "partial");
let active_report = metrics.scanner_runtime_details_report();
assert_eq!(
active_report.current_cycle_bucket_drive_results,
vec![ScannerBucketDriveResultSnapshot {
bucket: "photos".to_string(),
drive: "/data1".to_string(),
result: "partial".to_string(),
count: 1,
}]
);
metrics.finish_scan_cycle_work(cycle_start);
let report = metrics.scanner_runtime_details_report();
assert_eq!(
report.bucket_drive_results,
vec![
ScannerBucketDriveResultSnapshot {
bucket: "photos".to_string(),
drive: "/data1".to_string(),
result: "partial".to_string(),
count: 1,
},
ScannerBucketDriveResultSnapshot {
bucket: "photos".to_string(),
drive: "/data1".to_string(),
result: "success".to_string(),
count: 1,
},
]
);
assert!(report.current_cycle_bucket_drive_results.is_empty());
assert_eq!(
report.last_cycle_bucket_drive_results,
vec![ScannerBucketDriveResultSnapshot {
bucket: "photos".to_string(),
drive: "/data1".to_string(),
result: "partial".to_string(),
count: 1,
}]
);
}
#[tokio::test]
async fn scanner_bucket_drive_results_are_bounded() {
let metrics = Metrics::new();
for index in 0..MAX_SCANNER_BUCKET_DRIVE_RESULT_KEYS {
metrics.record_scanner_bucket_drive_result(&format!("bucket-{index}"), "/data1", "success");
}
metrics.record_scanner_bucket_drive_result("overflow", "/data1", "success");
let report = metrics.scanner_runtime_details_report();
assert_eq!(report.bucket_drive_results.len(), MAX_SCANNER_BUCKET_DRIVE_RESULT_KEYS);
assert!(
report
.bucket_drive_results
.iter()
.any(|snapshot| snapshot.bucket == "overflow")
);
assert!(
report
.bucket_drive_results
.iter()
.all(|snapshot| snapshot.bucket != "bucket-0")
);
}
#[tokio::test]
async fn report_includes_usage_freshness_status() {
let metrics = Metrics::new();
@@ -4265,9 +4557,10 @@ mod tests {
};
let cycle_ten_start = metrics.start_scan_cycle_work_with_cycle(cycle_ten.clone()).await;
metrics.operations[Metric::ScanObject as usize].store(1, Ordering::Relaxed);
metrics.record_scanner_bucket_drive_result("cycle-ten", "/data1", "partial");
let paths = metrics.current_paths.write().await;
let mut report = Box::pin(metrics.report());
let mut report = Box::pin(metrics.report_with_runtime_details());
let waker = std::task::Waker::noop();
let mut context = std::task::Context::from_waker(waker);
assert!(report.as_mut().poll(&mut context).is_pending());
@@ -4284,12 +4577,22 @@ mod tests {
})
.await;
metrics.operations[Metric::ScanObject as usize].store(101, Ordering::Relaxed);
metrics.record_scanner_bucket_drive_result("cycle-eleven", "/data1", "partial");
drop(paths);
let snapshot = report.await;
let (snapshot, runtime_details) = report.await;
assert_eq!(snapshot.current_cycle, 10);
assert_eq!(snapshot.current_cycle_objects_scanned, 1);
assert_eq!(
runtime_details.current_cycle_bucket_drive_results,
vec![ScannerBucketDriveResultSnapshot {
bucket: "cycle-ten".to_string(),
drive: "/data1".to_string(),
result: "partial".to_string(),
count: 1,
}]
);
metrics
.finish_scan_cycle_work_with_cycle(cycle_eleven_start, CurrentCycle::default())
crates/ecstore/src/api/mod.rs
+15 -15
View File
@@ -178,21 +178,21 @@ pub mod bucket {
mrf_backlog_observability_snapshot,
};
pub use crate::bucket::replication::{
BucketReplicationResyncStatus, BucketStats, DeleteReplicationConfigSnapshot, DeletedObjectReplicationInfo,
DurableMrfBacklog, DynReplicationPool, MrfOpKind, MrfReplicateEntry, MustReplicateOptions, ObjectOpts,
REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS,
REPLICATE_INCOMING_DELETE, REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS,
REPLICATION_WRITABLE_FIELDS, ReplicateDecision, ReplicateObjectInfo, ReplicationBatchAdmission, ReplicationConfig,
ReplicationConfigurationExt, ReplicationDeleteScheduleInput, ReplicationDeleteStateSource,
ReplicationHealQueueResult, ReplicationObjectBridge, ReplicationObjectIO, ReplicationOperation, ReplicationPoolTrait,
ReplicationPriority, ReplicationQueueAdmission, ReplicationScannerBridge, ReplicationState, ReplicationStats,
ReplicationStatusType, ReplicationStorage, ReplicationTargetValidationError, ReplicationType, ResyncOpts,
ResyncStatusType, RuntimeReplicationTargetBacklog, TargetReplicationResyncStatus, VersionPurgeStatusType,
delete_replication_state_from_config, delete_replication_version_id, get_global_replication_pool,
get_global_replication_stats, init_background_replication, invalid_replication_config_status_field,
read_durable_mrf_backlog, replication_state_to_filemeta, replication_status_to_filemeta, replication_statuses_map,
replication_target_arns, resync_start_conflict_id, should_remove_replication_target,
should_schedule_delete_replication, should_use_existing_delete_replication_info,
BucketReplicationResyncStatus, BucketReplicationStats, BucketStats, DeleteReplicationConfigSnapshot,
DeletedObjectReplicationInfo, DurableMrfBacklog, DynReplicationPool, MrfOpKind, MrfReplicateEntry,
MustReplicateOptions, ObjectOpts, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS,
REMOTE_TARGET_WRITABLE_FIELDS, REPLICATE_INCOMING_DELETE, REPLICATION_CAPABILITY_CONTRACT_VERSION,
REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS, ReplicateDecision, ReplicateObjectInfo,
ReplicationBatchAdmission, ReplicationConfig, ReplicationConfigurationExt, ReplicationDeleteScheduleInput,
ReplicationDeleteStateSource, ReplicationHealQueueResult, ReplicationObjectBridge, ReplicationObjectIO,
ReplicationOperation, ReplicationPoolTrait, ReplicationPriority, ReplicationQueueAdmission, ReplicationScannerBridge,
ReplicationState, ReplicationStats, ReplicationStatusType, ReplicationStorage, ReplicationTargetValidationError,
ReplicationType, ResyncOpts, ResyncStatusType, RuntimeReplicationTargetBacklog, TargetReplicationResyncStatus,
VersionPurgeStatusType, delete_replication_state_from_config, delete_replication_version_id,
get_global_replication_pool, get_global_replication_stats, init_background_replication,
invalid_replication_config_status_field, read_durable_mrf_backlog, replication_state_to_filemeta,
replication_status_to_filemeta, replication_statuses_map, replication_target_arns, resync_start_conflict_id,
should_remove_replication_target, should_schedule_delete_replication, should_use_existing_delete_replication_info,
should_use_existing_delete_replication_source, unsupported_replication_config_field,
validate_replication_config_target_arns, version_purge_status_to_filemeta,
};
crates/ecstore/src/bucket/replication/mod.rs
+1 -1
View File
@@ -80,6 +80,6 @@ pub use replication_queue_boundary::{
pub use replication_resync_boundary::{BucketReplicationResyncStatus, ResyncOpts, TargetReplicationResyncStatus};
pub use replication_scanner_bridge::ReplicationScannerBridge;
pub use replication_state::{ReplicationStats, RuntimeReplicationTargetBacklog};
pub use replication_stats_boundary::BucketStats;
pub use replication_stats_boundary::{BucketReplicationStats, BucketStats};
pub use replication_storage_boundary::{ReplicationObjectIO, ReplicationStorage};
pub(crate) use replication_target_config_bridge::ReplicationTargetConfigBridge;
crates/ecstore/src/bucket/replication/replication_stats_boundary.rs
+3 -3
View File
@@ -12,10 +12,10 @@
// See the License for the specific language governing permissions and
// limitations under the License.
pub use rustfs_replication::BucketStats;
#[cfg(test)]
pub(crate) use rustfs_replication::FailStats;
pub(crate) use rustfs_replication::{
ActiveWorkerStat, BucketReplicationStat, BucketReplicationStats, InQueueMetric, ProxyMetric, ProxyStatsCache, QueueCache,
ReplicationMetricScope, SRMetricsSummary, XferStats,
ActiveWorkerStat, BucketReplicationStat, InQueueMetric, ProxyMetric, ProxyStatsCache, QueueCache, ReplicationMetricScope,
SRMetricsSummary, XferStats,
};
pub use rustfs_replication::{BucketReplicationStats, BucketStats};
crates/ecstore/src/disk/disk_store.rs
+882 -66
View File
File diff suppressed because it is too large Load Diff
crates/ecstore/src/disk/mod.rs
+27 -1
View File
@@ -142,6 +142,13 @@ impl Disk {
Disk::Remote(remote_disk) => remote_disk.set_disk_id(id).await,
}
}
pub(crate) fn local_health_tracker_epoch_for_reconnect(&self) -> Option<disk_store::ReconnectDiskHealthState> {
match self {
Disk::Local(local_disk) => Some(local_disk.health_tracker_epoch_for_reconnect()),
Disk::Remote(_) => None,
}
}
}
#[async_trait::async_trait]
@@ -606,6 +613,13 @@ impl Disk {
}
}
pub fn metrics_snapshot(&self) -> Option<DiskMetrics> {
match self {
Disk::Local(local_disk) => Some(local_disk.metrics_snapshot()),
Disk::Remote(_) => None,
}
}
#[cfg(test)]
pub fn health_check_enabled_for_test(&self) -> bool {
match self {
@@ -664,9 +678,21 @@ impl Disk {
}
pub async fn new_disk(ep: &Endpoint, opt: &DiskOption) -> Result<DiskStore> {
new_disk_with_health_tracker(ep, opt, None).await
}
pub(crate) async fn new_disk_with_health_tracker(
ep: &Endpoint,
opt: &DiskOption,
reconnect: Option<disk_store::ReconnectDiskHealthState>,
) -> Result<DiskStore> {
if ep.is_local {
let s = LocalDisk::new(ep, opt.cleanup).await?;
Ok(Arc::new(Disk::Local(Box::new(LocalDiskWrapper::new(Arc::new(s), opt.health_check)))))
Ok(Arc::new(Disk::Local(Box::new(LocalDiskWrapper::new_with_reconnect_state(
Arc::new(s),
opt.health_check,
reconnect,
)))))
} else {
let data_transport = build_internode_data_transport_from_env();
let remote_disk = RemoteDisk::new(ep, opt, data_transport?).await?;
crates/ecstore/src/set_disk/mod.rs
+45 -8
View File
@@ -4359,7 +4359,13 @@ async fn get_disks_info(disks: &[Option<DiskStore>], eps: &[Endpoint]) -> Vec<ru
let offline_duration_seconds = disk.offline_duration_secs();
let capacity_snapshot = disk.last_capacity_snapshot();
if runtime_state.should_probe_for_admin() || runtime_state == disk::health_state::RuntimeDriveHealthState::Suspect {
match disk.disk_info(&DiskInfoOptions::default()).await {
match disk
.disk_info(&DiskInfoOptions {
metrics: true,
..Default::default()
})
.await
{
Ok(res) => {
disk.record_capacity_probe(res.total, res.used, res.free);
ret.push(rustfs_madmin::Disk {
@@ -4390,6 +4396,7 @@ async fn get_disks_info(disks: &[Option<DiskStore>], eps: &[Endpoint]) -> Vec<ru
utilization: utilization_percent(res.total, res.used),
used_inodes: res.used_inodes,
free_inodes: res.free_inodes,
metrics: Some(res.metrics),
..Default::default()
});
}
@@ -4397,12 +4404,14 @@ async fn get_disks_info(disks: &[Option<DiskStore>], eps: &[Endpoint]) -> Vec<ru
let mut disk_info = rustfs_madmin::Disk {
state: err.to_string(),
endpoint: eps[i].to_string(),
drive_path: eps[i].get_file_path(),
local: eps[i].is_local,
pool_index: eps[i].pool_idx,
set_index: eps[i].set_idx,
disk_index: eps[i].disk_idx,
runtime_state: Some(runtime_state.as_str().to_string()),
offline_duration_seconds,
metrics: disk.metrics_snapshot(),
..Default::default()
};
if let Some((total, used, free, _)) = capacity_snapshot {
@@ -4421,16 +4430,15 @@ async fn get_disks_info(disks: &[Option<DiskStore>], eps: &[Endpoint]) -> Vec<ru
}
}
} else {
ret.push(build_runtime_snapshot_disk(
&eps[i],
runtime_state,
offline_duration_seconds,
capacity_snapshot,
));
let mut disk_info =
build_runtime_snapshot_disk(&eps[i], runtime_state, offline_duration_seconds, capacity_snapshot);
disk_info.metrics = disk.metrics_snapshot();
ret.push(disk_info);
}
} else {
ret.push(rustfs_madmin::Disk {
endpoint: eps[i].to_string(),
drive_path: eps[i].get_file_path(),
local: eps[i].is_local,
pool_index: eps[i].pool_idx,
set_index: eps[i].set_idx,
@@ -4456,6 +4464,7 @@ fn build_runtime_snapshot_disk(
) -> rustfs_madmin::Disk {
let mut disk = rustfs_madmin::Disk {
endpoint: endpoint.to_string(),
drive_path: endpoint.get_file_path(),
local: endpoint.is_local,
pool_index: endpoint.pool_idx,
set_index: endpoint.set_idx,
@@ -7709,14 +7718,42 @@ mod tests {
assert_eq!(info[0].state, "ok");
assert_eq!(info[0].runtime_state.as_deref(), Some("online"));
assert!(!info[0].drive_path.is_empty(), "online disk should keep immediate disk_info probe");
assert!(
info[0]
.metrics
.as_ref()
.and_then(|metrics| metrics.api_calls.get("disk_info"))
.copied()
.unwrap_or_default()
> 0,
"online disk should expose disk_info operation metrics"
);
assert_eq!(info[1].state, "ok");
assert_eq!(info[1].runtime_state.as_deref(), Some("suspect"));
assert!(!info[1].drive_path.is_empty(), "suspect disk should still probe for fresher disk info");
assert!(
info[1]
.metrics
.as_ref()
.and_then(|metrics| metrics.last_minute.get("disk_info"))
.map(|action| action.count)
.unwrap_or_default()
> 0,
"suspect disk should expose last-minute disk_info latency"
);
assert_eq!(info[2].state, "offline");
assert_eq!(info[2].runtime_state.as_deref(), Some("offline"));
assert!(info[2].drive_path.is_empty(), "offline disk should use runtime snapshot fallback");
assert_eq!(
info[2].drive_path,
endpoints[2].get_file_path(),
"offline disk should keep stable endpoint path"
);
assert!(
info[2].metrics.is_some(),
"offline runtime fallback should preserve disk metrics snapshot"
);
}
#[tokio::test]
crates/ecstore/src/set_disk/ops/locking.rs
+35 -3
View File
@@ -322,7 +322,16 @@ impl SetDisks {
pub async fn renew_disk(&self, ep: &Endpoint) {
debug!("renew_disk: start {:?}", ep);
let (new_disk, fm) = match Self::connect_endpoint(ep).await {
let previous_health = {
let disks = self.disks.read().await;
disks
.iter()
.filter_map(|disk| disk.as_ref())
.find(|disk| disk.endpoint() == *ep)
.and_then(|disk| disk.local_health_tracker_epoch_for_reconnect())
};
let (new_disk, fm) = match Self::connect_endpoint(ep, previous_health).await {
Ok(res) => res,
Err(e) => {
warn!("renew_disk: connect_endpoint err {:?}", &e);
@@ -400,13 +409,17 @@ impl SetDisks {
Err(Error::other("DriveID: not found"))
}
pub(in crate::set_disk) async fn connect_endpoint(ep: &Endpoint) -> disk::error::Result<(DiskStore, FormatV3)> {
let disk = new_disk(
pub(in crate::set_disk) async fn connect_endpoint(
ep: &Endpoint,
reconnect: Option<disk::disk_store::ReconnectDiskHealthState>,
) -> disk::error::Result<(DiskStore, FormatV3)> {
let disk = crate::disk::new_disk_with_health_tracker(
ep,
&DiskOption {
cleanup: false,
health_check: true,
},
reconnect,
)
.await?;
@@ -714,6 +727,25 @@ mod tests {
renewed_disk.health_check_enabled_for_test(),
"renewed disks must keep health monitoring enabled so later faulty marks can recover"
);
renewed_disk
.disk_info(&DiskInfoOptions::default())
.await
.expect("renewed disk_info should record a drive API metric");
renewed_disk.force_runtime_state_for_test(disk::health_state::RuntimeDriveHealthState::Offline);
set_disks.renew_disk(&endpoints[0]).await;
let disks = set_disks.get_disks_internal().await;
let renewed_again = disks[0]
.as_ref()
.expect("second renew_disk should keep the recovered disk attached");
assert_eq!(
renewed_again
.metrics_snapshot()
.and_then(|metrics| metrics.api_calls.get("disk_info").copied()),
Some(1),
"disk reconnect must preserve the local drive metrics tracker epoch"
);
drop(temp_dirs);
}
crates/io-metrics/src/lib.rs
+1 -1
View File
@@ -246,7 +246,7 @@ pub use process_lock_metrics::{
record_write_lock_held_acquire, record_write_lock_held_release, snapshot_process_lock_counts, snapshot_process_lock_events,
snapshot_process_platform_stats,
};
pub use s3_api_metrics::{init_s3_metrics, record_s3_op};
pub use s3_api_metrics::{S3OperationMetricSnapshot, init_s3_metrics, record_s3_op, s3_op_metrics_snapshot};
pub use sampler::{
ProcessResourceSnapshot, ProcessSampler, ProcessStatusSnapshot, ProcessSystemSnapshot, snapshot_process_platform,
snapshot_process_resource, snapshot_process_resource_and_system, snapshot_process_resource_and_system_with,
crates/io-metrics/src/s3_api_metrics.rs
+55
View File
@@ -14,8 +14,25 @@
use rustfs_s3_ops::S3Operation;
use std::sync::OnceLock;
use std::sync::atomic::{AtomicU64, Ordering};
const S3_OPS_METRIC: &str = "rustfs_s3_operations_total";
static S3_OP_COUNTERS: OnceLock<Box<[AtomicU64]>> = OnceLock::new();
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct S3OperationMetricSnapshot {
pub op: &'static str,
pub total: u64,
}
fn s3_op_counters() -> &'static [AtomicU64] {
S3_OP_COUNTERS.get_or_init(|| {
std::iter::repeat_with(|| AtomicU64::new(0))
.take(S3Operation::ALL.len())
.collect::<Vec<_>>()
.into_boxed_slice()
})
}
/// Record a handled S3 API operation.
///
@@ -26,9 +43,22 @@ const S3_OPS_METRIC: &str = "rustfs_s3_operations_total";
/// This mirrors MinIO, which never labels its default operation counters with
/// bucket. The `op` dimension is bounded (<= 122 variants).
pub fn record_s3_op(op: S3Operation) {
if let Some(counter) = s3_op_counters().get(op.metric_index()) {
counter.fetch_add(1, Ordering::Relaxed);
}
counter!(S3_OPS_METRIC, "op" => op.as_str()).increment(1);
}
pub fn s3_op_metrics_snapshot() -> Vec<S3OperationMetricSnapshot> {
S3Operation::ALL
.iter()
.filter_map(|op| {
let total = s3_op_counters().get(op.metric_index())?.load(Ordering::Relaxed);
(total > 0).then_some(S3OperationMetricSnapshot { op: op.as_str(), total })
})
.collect()
}
pub fn init_s3_metrics() {
static METRICS_DESC_INIT: OnceLock<()> = OnceLock::new();
METRICS_DESC_INIT.get_or_init(|| {
@@ -42,6 +72,9 @@ mod tests {
use metrics::with_local_recorder;
use metrics_util::debugging::DebuggingRecorder;
use std::collections::HashSet;
use std::sync::Mutex;
static S3_OP_TEST_LOCK: Mutex<()> = Mutex::new(());
/// Collect the label-key sets recorded against `rustfs_s3_operations_total`.
fn ops_metric_label_key_sets(recorder: &DebuggingRecorder) -> Vec<HashSet<String>> {
@@ -60,6 +93,7 @@ mod tests {
#[test]
fn record_s3_op_labels_by_op_only_no_bucket() {
let _guard = S3_OP_TEST_LOCK.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
let recorder = DebuggingRecorder::new();
let label_key_sets = ops_metric_label_key_sets(&recorder);
@@ -75,6 +109,7 @@ mod tests {
#[test]
fn record_s3_op_cardinality_bounded_by_distinct_ops() {
let _guard = S3_OP_TEST_LOCK.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
let recorder = DebuggingRecorder::new();
let snapshotter = recorder.snapshotter();
@@ -113,4 +148,24 @@ mod tests {
"series count must equal the number of distinct ops, never the bucket count"
);
}
#[test]
fn s3_op_metrics_snapshot_reports_recorded_totals() {
let _guard = S3_OP_TEST_LOCK.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
let before = s3_op_metrics_snapshot()
.into_iter()
.find(|snapshot| snapshot.op == S3Operation::GetObject.as_str())
.map(|snapshot| snapshot.total)
.unwrap_or_default();
record_s3_op(S3Operation::GetObject);
record_s3_op(S3Operation::GetObject);
let after = s3_op_metrics_snapshot()
.into_iter()
.find(|snapshot| snapshot.op == S3Operation::GetObject.as_str())
.map(|snapshot| snapshot.total)
.expect("GetObject snapshot should be present after recording");
assert_eq!(after, before + 2);
}
}
crates/madmin/src/info_commands.rs
+88 -2
View File
@@ -14,7 +14,8 @@
use std::{collections::HashMap, time::SystemTime};
use serde::{Deserialize, Serialize};
use serde::ser::SerializeStruct;
use serde::{Deserialize, Serialize, Serializer};
use time::OffsetDateTime;
use crate::metrics::TimedAction;
@@ -61,17 +62,41 @@ impl ItemState {
}
}
#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
#[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq)]
pub struct DiskMetrics {
#[serde(rename = "lastMinute", alias = "last_minute")]
pub last_minute: HashMap<String, TimedAction>,
#[serde(rename = "apiCalls", alias = "api_calls")]
pub api_calls: HashMap<String, u64>,
#[serde(rename = "totalWaiting", alias = "total_waiting")]
pub total_waiting: u32,
#[serde(rename = "totalErrsAvailability", alias = "total_errors_availability")]
pub total_errors_availability: u64,
#[serde(rename = "totalErrsTimeout", alias = "total_errors_timeout")]
pub total_errors_timeout: u64,
#[serde(rename = "totalWrites", alias = "total_writes")]
pub total_writes: u64,
#[serde(rename = "totalDeletes", alias = "total_deletes")]
pub total_deletes: u64,
}
impl Serialize for DiskMetrics {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
let mut state = serializer.serialize_struct("DiskMetrics", 7)?;
state.serialize_field("last_minute", &self.last_minute)?;
state.serialize_field("api_calls", &self.api_calls)?;
state.serialize_field("total_waiting", &self.total_waiting)?;
state.serialize_field("total_errors_availability", &self.total_errors_availability)?;
state.serialize_field("total_errors_timeout", &self.total_errors_timeout)?;
state.serialize_field("total_writes", &self.total_writes)?;
state.serialize_field("total_deletes", &self.total_deletes)?;
state.end()
}
}
#[derive(Serialize, Deserialize, Debug, Default, Clone)]
pub struct Disk {
pub endpoint: String,
@@ -432,6 +457,17 @@ mod tests {
disk_index: i32,
}
#[derive(Deserialize)]
struct LegacyDiskMetricsCompat {
last_minute: HashMap<String, TimedAction>,
api_calls: HashMap<String, u64>,
total_waiting: u32,
total_errors_availability: u64,
total_errors_timeout: u64,
total_writes: u64,
total_deletes: u64,
}
#[test]
fn test_item_state_to_string() {
assert_eq!(ItemState::Offline.to_string(), ITEM_OFFLINE);
@@ -495,6 +531,56 @@ mod tests {
assert_eq!(metrics.total_deletes, 50);
}
#[test]
fn test_disk_metrics_json_preserves_internode_legacy_fields() {
let metrics = DiskMetrics {
total_waiting: 5,
total_errors_availability: 2,
total_errors_timeout: 1,
total_writes: 1000,
total_deletes: 50,
..Default::default()
};
let json = serde_json::to_value(metrics).expect("disk metrics should serialize");
assert!(json.get("last_minute").is_some());
assert!(json.get("api_calls").is_some());
assert_eq!(json["total_waiting"], serde_json::json!(5));
assert_eq!(json["total_errors_availability"], serde_json::json!(2));
assert_eq!(json["total_errors_timeout"], serde_json::json!(1));
assert_eq!(json["total_writes"], serde_json::json!(1000));
assert_eq!(json["total_deletes"], serde_json::json!(50));
assert!(json.get("lastMinute").is_none());
assert!(json.get("totalErrsTimeout").is_none());
}
#[test]
fn test_disk_metrics_msgpack_uses_internode_legacy_fields() {
let metrics = DiskMetrics {
total_waiting: 5,
total_errors_availability: 2,
total_errors_timeout: 1,
total_writes: 1000,
total_deletes: 50,
..Default::default()
};
let mut encoded = Vec::new();
metrics
.serialize(&mut Serializer::new(&mut encoded).with_struct_map())
.expect("disk metrics should encode as named msgpack");
let decoded: LegacyDiskMetricsCompat = rmp_serde::from_slice(&encoded).expect("legacy disk metrics should decode");
assert!(decoded.last_minute.is_empty());
assert!(decoded.api_calls.is_empty());
assert_eq!(decoded.total_waiting, 5);
assert_eq!(decoded.total_errors_availability, 2);
assert_eq!(decoded.total_errors_timeout, 1);
assert_eq!(decoded.total_writes, 1000);
assert_eq!(decoded.total_deletes, 50);
}
#[test]
fn test_disk_default() {
let disk = Disk::default();
crates/obs/src/metrics/collectors/audit.rs
+77 -5
View File
@@ -41,6 +41,13 @@ pub struct AuditTargetStats {
pub total_messages: u64,
}
/// Audit target statistics with runtime-local node identity.
#[derive(Debug, Clone, Default)]
pub(crate) struct AuditTargetRuntimeStats {
pub(crate) server: String,
pub(crate) target: AuditTargetStats,
}
/// Collects audit metrics from the provided audit target statistics.
///
/// Uses the metric descriptors from `metrics_type::audit` module.
@@ -56,19 +63,53 @@ pub fn collect_audit_metrics(stats: &[AuditTargetStats]) -> Vec<PrometheusMetric
metrics.push(
PrometheusMetric::from_descriptor(&AUDIT_FAILED_MESSAGES_MD, stat.failed_messages as f64)
.with_label("target_id", target_id_label.clone()),
.with_label(TARGET_ID, target_id_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&AUDIT_FAILED_STORE_LENGTH_MD, stat.failed_store_length as f64)
.with_label("target_id", target_id_label.clone()),
.with_label(TARGET_ID, target_id_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&AUDIT_TARGET_QUEUE_LENGTH_MD, stat.queue_length as f64)
.with_label("target_id", target_id_label.clone()),
.with_label(TARGET_ID, target_id_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&AUDIT_TOTAL_MESSAGES_MD, stat.total_messages as f64)
.with_label("target_id", target_id_label),
.with_label(TARGET_ID, target_id_label),
);
}
metrics
}
pub(crate) fn collect_audit_runtime_metrics(stats: &[AuditTargetRuntimeStats]) -> Vec<PrometheusMetric> {
let legacy_stats = stats.iter().map(|stat| stat.target.clone()).collect::<Vec<_>>();
let mut metrics = collect_audit_metrics(&legacy_stats);
metrics.reserve(stats.len() * 4);
for stat in stats.iter().filter(|stat| !stat.server.is_empty()) {
let server_label: Cow<'static, str> = Cow::Owned(stat.server.clone());
let target_id_label: Cow<'static, str> = Cow::Owned(stat.target.target_id.clone());
metrics.push(
PrometheusMetric::from_descriptor(&AUDIT_FAILED_MESSAGES_BY_SERVER_MD, stat.target.failed_messages as f64)
.with_label(SERVER, server_label.clone())
.with_label(TARGET_ID, target_id_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&AUDIT_FAILED_STORE_LENGTH_BY_SERVER_MD, stat.target.failed_store_length as f64)
.with_label(SERVER, server_label.clone())
.with_label(TARGET_ID, target_id_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&AUDIT_TARGET_QUEUE_LENGTH_BY_SERVER_MD, stat.target.queue_length as f64)
.with_label(SERVER, server_label.clone())
.with_label(TARGET_ID, target_id_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&AUDIT_TOTAL_MESSAGES_BY_SERVER_MD, stat.target.total_messages as f64)
.with_label(SERVER, server_label)
.with_label(TARGET_ID, target_id_label),
);
}
@@ -101,7 +142,7 @@ mod tests {
let metrics = collect_audit_metrics(&stats);
assert_eq!(metrics.len(), 8); // 2 targets * 4 metrics each
assert_eq!(metrics.len(), 8);
let failed = metrics
.iter()
@@ -114,6 +155,37 @@ mod tests {
&& m.labels.iter().any(|(k, v)| *k == "target_id" && v == "target-1")
});
assert!(failed_store.is_some());
assert!(
metrics
.iter()
.all(|m| m.name != AUDIT_TARGET_QUEUE_LENGTH_BY_SERVER_MD.get_full_metric_name())
);
}
#[test]
fn collect_audit_runtime_metrics_adds_server_dimensions() {
let stats = vec![AuditTargetRuntimeStats {
server: "node1:9000".to_string(),
target: AuditTargetStats {
target_id: "target-1".to_string(),
failed_messages: 5,
failed_store_length: 3,
queue_length: 10,
total_messages: 1000,
},
}];
let metrics = collect_audit_runtime_metrics(&stats);
assert_eq!(metrics.len(), 8);
let server_queue = metrics.iter().find(|m| {
m.value == 10.0
&& m.name == AUDIT_TARGET_QUEUE_LENGTH_BY_SERVER_MD.get_full_metric_name()
&& m.labels.iter().any(|(k, v)| *k == SERVER && v == "node1:9000")
&& m.labels.iter().any(|(k, v)| *k == TARGET_ID && v == "target-1")
});
assert!(server_queue.is_some());
}
#[test]
crates/obs/src/metrics/collectors/bucket_replication.rs
+253 -38
View File
@@ -30,14 +30,18 @@ use crate::metrics::schema::bucket_replication::{
BUCKET_REPL_PROXIED_HEAD_REQUESTS_FAILURES_MD, BUCKET_REPL_PROXIED_HEAD_REQUESTS_TOTAL_MD,
BUCKET_REPL_PROXIED_PUT_REQUESTS_FAILURES_MD, BUCKET_REPL_PROXIED_PUT_REQUESTS_TOTAL_MD,
BUCKET_REPL_PROXIED_PUT_TAGGING_REQUESTS_FAILURES_MD, BUCKET_REPL_PROXIED_PUT_TAGGING_REQUESTS_TOTAL_MD,
BUCKET_REPL_RESYNC_CANCELED_TOTAL_MD, BUCKET_REPL_RESYNC_COMPLETED_TOTAL_MD, BUCKET_REPL_RESYNC_DURATION_MS_TOTAL_MD,
BUCKET_REPL_RESYNC_FAILED_TOTAL_MD, BUCKET_REPL_RESYNC_STARTED_TOTAL_MD, BUCKET_REPL_SENT_BYTES_MD,
BUCKET_REPL_SENT_COUNT_MD, BUCKET_REPL_TOTAL_FAILED_BYTES_MD, BUCKET_REPL_TOTAL_FAILED_COUNT_MD, OPERATION_L, RANGE_L,
TARGET_ARN_L,
BUCKET_REPL_PROXY_REQUESTS_TOTAL_MD, BUCKET_REPL_RESYNC_CANCELED_TOTAL_MD, BUCKET_REPL_RESYNC_COMPLETED_TOTAL_MD,
BUCKET_REPL_RESYNC_DURATION_MS_TOTAL_MD, BUCKET_REPL_RESYNC_FAILED_TOTAL_MD, BUCKET_REPL_RESYNC_STARTED_TOTAL_MD,
BUCKET_REPL_SENT_BYTES_MD, BUCKET_REPL_SENT_COUNT_MD, BUCKET_REPL_TARGET_LAST_HOUR_FAILED_BYTES_MD,
BUCKET_REPL_TARGET_LAST_HOUR_FAILED_COUNT_MD, BUCKET_REPL_TARGET_LAST_MIN_FAILED_BYTES_MD,
BUCKET_REPL_TARGET_LAST_MIN_FAILED_COUNT_MD, BUCKET_REPL_TARGET_SENT_BYTES_MD, BUCKET_REPL_TARGET_SENT_COUNT_MD,
BUCKET_REPL_TARGET_TOTAL_FAILED_BYTES_MD, BUCKET_REPL_TARGET_TOTAL_FAILED_COUNT_MD, BUCKET_REPL_TOTAL_FAILED_BYTES_MD,
BUCKET_REPL_TOTAL_FAILED_COUNT_MD, OPERATION_L, RANGE_L, RESULT_L, TARGET_ARN_L,
};
use std::borrow::Cow;
const BASE_BUCKET_REPLICATION_METRICS_PER_BUCKET: usize = 25;
const BASE_BUCKET_REPLICATION_METRICS_PER_BUCKET: usize = 37;
const BUCKET_REPLICATION_RUNTIME_FLOW_METRICS_PER_TARGET: usize = 8;
const BASE_BUCKET_REPLICATION_BACKLOG_METRICS_PER_BUCKET: usize = 11;
const BUCKET_REPLICATION_BACKLOG_METRICS_PER_TARGET: usize = 4;
@@ -49,6 +53,19 @@ pub struct BucketReplicationTargetStats {
pub latency_ms: f64,
}
#[derive(Debug, Clone, Default)]
pub(crate) struct BucketReplicationTargetFlowStats {
pub(crate) target_arn: String,
pub sent_bytes: u64,
pub sent_count: u64,
pub total_failed_bytes: u64,
pub total_failed_count: u64,
pub last_min_failed_bytes: u64,
pub last_min_failed_count: u64,
pub last_hour_failed_bytes: u64,
pub last_hour_failed_count: u64,
}
#[derive(Debug, Clone, Default)]
pub struct BucketReplicationBandwidthStats {
pub bucket: String,
@@ -88,6 +105,12 @@ pub struct BucketReplicationStats {
pub targets: Vec<BucketReplicationTargetStats>,
}
#[derive(Debug, Clone, Default)]
pub(crate) struct BucketReplicationRuntimeStats {
pub(crate) stats: BucketReplicationStats,
pub(crate) target_flows: Vec<BucketReplicationTargetFlowStats>,
}
#[derive(Debug, Clone, Default)]
pub(crate) struct BucketReplicationBacklogStats {
pub(crate) bucket: String,
@@ -140,6 +163,25 @@ pub fn collect_bucket_replication_bandwidth_metrics(stats: &[BucketReplicationBa
metrics
}
fn push_proxy_request_result_metrics(
metrics: &mut Vec<PrometheusMetric>,
bucket_label: Cow<'static, str>,
operation: &'static str,
total: u64,
failures: u64,
) {
let failure_count = failures.min(total);
let success_count = total.saturating_sub(failure_count);
for (result, value) in [("success", success_count), ("failure", failure_count)] {
metrics.push(
PrometheusMetric::from_descriptor(&BUCKET_REPL_PROXY_REQUESTS_TOTAL_MD, value as f64)
.with_label(BUCKET_L, bucket_label.clone())
.with_label(OPERATION_L, operation)
.with_label(RESULT_L, result),
);
}
}
pub fn collect_bucket_replication_metrics(stats: &[BucketReplicationStats]) -> Vec<PrometheusMetric> {
if stats.is_empty() {
return Vec::new();
@@ -263,6 +305,48 @@ pub fn collect_bucket_replication_metrics(stats: &[BucketReplicationStats]) -> V
)
.with_label(BUCKET_L, bucket_label.clone()),
);
push_proxy_request_result_metrics(
&mut metrics,
bucket_label.clone(),
"get",
stat.proxied_get_requests_total,
stat.proxied_get_requests_failures,
);
push_proxy_request_result_metrics(
&mut metrics,
bucket_label.clone(),
"head",
stat.proxied_head_requests_total,
stat.proxied_head_requests_failures,
);
push_proxy_request_result_metrics(
&mut metrics,
bucket_label.clone(),
"put",
stat.proxied_put_requests_total,
stat.proxied_put_requests_failures,
);
push_proxy_request_result_metrics(
&mut metrics,
bucket_label.clone(),
"put_tagging",
stat.proxied_put_tagging_requests_total,
stat.proxied_put_tagging_requests_failures,
);
push_proxy_request_result_metrics(
&mut metrics,
bucket_label.clone(),
"get_tagging",
stat.proxied_get_tagging_requests_total,
stat.proxied_get_tagging_requests_failures,
);
push_proxy_request_result_metrics(
&mut metrics,
bucket_label.clone(),
"delete_tagging",
stat.proxied_delete_tagging_requests_total,
stat.proxied_delete_tagging_requests_failures,
);
metrics.push(
PrometheusMetric::from_descriptor(&BUCKET_REPL_RESYNC_STARTED_TOTAL_MD, stat.resync_started_count as f64)
.with_label(BUCKET_L, bucket_label.clone()),
@@ -298,6 +382,81 @@ pub fn collect_bucket_replication_metrics(stats: &[BucketReplicationStats]) -> V
metrics
}
pub(crate) fn collect_bucket_replication_runtime_metrics(stats: &[BucketReplicationRuntimeStats]) -> Vec<PrometheusMetric> {
if stats.is_empty() {
return Vec::new();
}
let legacy_stats = stats.iter().map(|stat| stat.stats.clone()).collect::<Vec<_>>();
let mut metrics = collect_bucket_replication_metrics(&legacy_stats);
let flow_count = stats
.iter()
.map(|stat| stat.target_flows.len() * BUCKET_REPLICATION_RUNTIME_FLOW_METRICS_PER_TARGET)
.sum();
metrics.reserve(flow_count);
for stat in stats {
let bucket_label: Cow<'static, str> = Cow::Owned(stat.stats.bucket.clone());
for target in &stat.target_flows {
let target_label: Cow<'static, str> = Cow::Owned(target.target_arn.clone());
metrics.push(
PrometheusMetric::from_descriptor(&BUCKET_REPL_TARGET_SENT_BYTES_MD, target.sent_bytes as f64)
.with_label(BUCKET_L, bucket_label.clone())
.with_label(TARGET_ARN_L, target_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&BUCKET_REPL_TARGET_SENT_COUNT_MD, target.sent_count as f64)
.with_label(BUCKET_L, bucket_label.clone())
.with_label(TARGET_ARN_L, target_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&BUCKET_REPL_TARGET_TOTAL_FAILED_BYTES_MD, target.total_failed_bytes as f64)
.with_label(BUCKET_L, bucket_label.clone())
.with_label(TARGET_ARN_L, target_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&BUCKET_REPL_TARGET_TOTAL_FAILED_COUNT_MD, target.total_failed_count as f64)
.with_label(BUCKET_L, bucket_label.clone())
.with_label(TARGET_ARN_L, target_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(
&BUCKET_REPL_TARGET_LAST_MIN_FAILED_BYTES_MD,
target.last_min_failed_bytes as f64,
)
.with_label(BUCKET_L, bucket_label.clone())
.with_label(TARGET_ARN_L, target_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(
&BUCKET_REPL_TARGET_LAST_MIN_FAILED_COUNT_MD,
target.last_min_failed_count as f64,
)
.with_label(BUCKET_L, bucket_label.clone())
.with_label(TARGET_ARN_L, target_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(
&BUCKET_REPL_TARGET_LAST_HOUR_FAILED_BYTES_MD,
target.last_hour_failed_bytes as f64,
)
.with_label(BUCKET_L, bucket_label.clone())
.with_label(TARGET_ARN_L, target_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(
&BUCKET_REPL_TARGET_LAST_HOUR_FAILED_COUNT_MD,
target.last_hour_failed_count as f64,
)
.with_label(BUCKET_L, bucket_label.clone())
.with_label(TARGET_ARN_L, target_label.clone()),
);
}
}
metrics
}
pub(crate) fn collect_bucket_replication_backlog_metrics(stats: &[BucketReplicationBacklogStats]) -> Vec<PrometheusMetric> {
if stats.is_empty() {
return Vec::new();
@@ -412,43 +571,56 @@ mod tests {
#[test]
fn test_collect_bucket_replication_metrics() {
let stats = vec![BucketReplicationStats {
bucket: "b1".to_string(),
total_failed_bytes: 64,
total_failed_count: 2,
last_min_failed_bytes: 32,
last_min_failed_count: 1,
last_hour_failed_bytes: 64,
last_hour_failed_count: 2,
sent_bytes: 1024,
sent_count: 8,
proxied_get_requests_total: 5,
proxied_get_requests_failures: 1,
proxied_head_requests_total: 4,
proxied_head_requests_failures: 0,
proxied_put_requests_total: 6,
proxied_put_requests_failures: 2,
proxied_put_tagging_requests_total: 3,
proxied_put_tagging_requests_failures: 1,
proxied_get_tagging_requests_total: 2,
proxied_get_tagging_requests_failures: 0,
proxied_delete_tagging_requests_total: 1,
proxied_delete_tagging_requests_failures: 1,
resync_started_count: 2,
resync_completed_count: 1,
resync_failed_count: 1,
resync_canceled_count: 0,
resync_duration_ms: 1500,
targets: vec![BucketReplicationTargetStats {
let stats = vec![BucketReplicationRuntimeStats {
stats: BucketReplicationStats {
bucket: "b1".to_string(),
total_failed_bytes: 64,
total_failed_count: 2,
last_min_failed_bytes: 32,
last_min_failed_count: 1,
last_hour_failed_bytes: 64,
last_hour_failed_count: 2,
sent_bytes: 1024,
sent_count: 8,
proxied_get_requests_total: 5,
proxied_get_requests_failures: 1,
proxied_head_requests_total: 4,
proxied_head_requests_failures: 0,
proxied_put_requests_total: 6,
proxied_put_requests_failures: 2,
proxied_put_tagging_requests_total: 3,
proxied_put_tagging_requests_failures: 1,
proxied_get_tagging_requests_total: 2,
proxied_get_tagging_requests_failures: 0,
proxied_delete_tagging_requests_total: 1,
proxied_delete_tagging_requests_failures: 1,
resync_started_count: 2,
resync_completed_count: 1,
resync_failed_count: 1,
resync_canceled_count: 0,
resync_duration_ms: 1500,
targets: vec![BucketReplicationTargetStats {
target_arn: "arn:rustfs:replication:us-east-1:1:target".to_string(),
bandwidth_limit_bytes_per_sec: 2048,
current_bandwidth_bytes_per_sec: 1024.0,
latency_ms: 15.0,
}],
},
target_flows: vec![BucketReplicationTargetFlowStats {
target_arn: "arn:rustfs:replication:us-east-1:1:target".to_string(),
bandwidth_limit_bytes_per_sec: 2048,
current_bandwidth_bytes_per_sec: 1024.0,
latency_ms: 15.0,
sent_bytes: 512,
sent_count: 4,
total_failed_bytes: 96,
total_failed_count: 3,
last_min_failed_bytes: 32,
last_min_failed_count: 1,
last_hour_failed_bytes: 64,
last_hour_failed_count: 2,
}],
}];
let metrics = collect_bucket_replication_metrics(&stats);
assert_eq!(metrics.len(), 26);
let metrics = collect_bucket_replication_runtime_metrics(&stats);
assert_eq!(metrics.len(), 46);
let sent_name = BUCKET_REPL_SENT_COUNT_MD.get_full_metric_name();
assert!(metrics.iter().any(|metric| {
@@ -471,6 +643,28 @@ mod tests {
&& metric.labels.iter().any(|(key, value)| *key == BUCKET_L && value == "b1")
}));
let proxy_requests_name = BUCKET_REPL_PROXY_REQUESTS_TOTAL_MD.get_full_metric_name();
assert!(metrics.iter().any(|metric| {
metric.name == proxy_requests_name
&& metric.value == 4.0
&& metric.labels.iter().any(|(key, value)| *key == BUCKET_L && value == "b1")
&& metric.labels.iter().any(|(key, value)| *key == OPERATION_L && value == "put")
&& metric
.labels
.iter()
.any(|(key, value)| *key == RESULT_L && value == "success")
}));
assert!(metrics.iter().any(|metric| {
metric.name == proxy_requests_name
&& metric.value == 2.0
&& metric.labels.iter().any(|(key, value)| *key == BUCKET_L && value == "b1")
&& metric.labels.iter().any(|(key, value)| *key == OPERATION_L && value == "put")
&& metric
.labels
.iter()
.any(|(key, value)| *key == RESULT_L && value == "failure")
}));
let latency_name = BUCKET_REPL_LATENCY_MS_MD.get_full_metric_name();
assert!(metrics.iter().any(|metric| {
metric.name == latency_name
@@ -481,6 +675,27 @@ mod tests {
.any(|(key, value)| *key == TARGET_ARN_L && value == "arn:rustfs:replication:us-east-1:1:target")
}));
let target_sent_name = BUCKET_REPL_TARGET_SENT_COUNT_MD.get_full_metric_name();
assert!(metrics.iter().any(|metric| {
metric.name == target_sent_name
&& metric.value == 4.0
&& metric
.labels
.iter()
.any(|(key, value)| *key == TARGET_ARN_L && value == "arn:rustfs:replication:us-east-1:1:target")
}));
let target_last_min_failed_name = BUCKET_REPL_TARGET_LAST_MIN_FAILED_BYTES_MD.get_full_metric_name();
assert!(metrics.iter().any(|metric| {
metric.name == target_last_min_failed_name
&& metric.value == 32.0
&& metric.labels.iter().any(|(key, value)| *key == BUCKET_L && value == "b1")
&& metric
.labels
.iter()
.any(|(key, value)| *key == TARGET_ARN_L && value == "arn:rustfs:replication:us-east-1:1:target")
}));
let delete_tagging_total_name = BUCKET_REPL_PROXIED_DELETE_TAGGING_REQUESTS_TOTAL_MD.get_full_metric_name();
assert!(metrics.iter().any(|metric| {
metric.name == delete_tagging_total_name
crates/obs/src/metrics/collectors/ilm.rs
+87 -2
View File
@@ -48,6 +48,26 @@ pub struct IlmStats {
pub versions_scanned: u64,
}
/// ILM task metrics by action and state.
#[derive(Debug, Clone, Default)]
pub(crate) struct IlmActionTaskStats {
pub(crate) action: String,
pub(crate) state: String,
pub(crate) value: u64,
}
/// ILM statistics with runtime-local node identity and bounded action/state details.
#[derive(Debug, Clone, Default)]
pub(crate) struct IlmRuntimeStats {
pub(crate) server: String,
pub(crate) stats: IlmStats,
pub(crate) action_tasks: Vec<IlmActionTaskStats>,
}
fn is_live_action_task_state(state: &str) -> bool {
matches!(state, "pending" | "active" | "compensation_running")
}
/// Collects ILM metrics from the given stats.
///
/// Uses the metric descriptors from `metrics_type::ilm` module.
@@ -78,6 +98,25 @@ pub fn collect_ilm_metrics(stats: &IlmStats) -> Vec<PrometheusMetric> {
]
}
pub(crate) fn collect_ilm_runtime_metrics(stats: &IlmRuntimeStats) -> Vec<PrometheusMetric> {
let mut metrics = collect_ilm_metrics(&stats.stats);
metrics.extend(
stats
.action_tasks
.iter()
.filter(|task| is_live_action_task_state(&task.state))
.map(|task| {
PrometheusMetric::from_descriptor(&ILM_ACTION_TASKS_MD, task.value as f64)
.with_label_owned(SERVER_LABEL, stats.server.clone())
.with_label_owned(ACTION_LABEL, task.action.clone())
.with_label_owned(STATE_LABEL, task.state.clone())
}),
);
metrics
}
#[cfg(test)]
mod tests {
use super::*;
@@ -95,16 +134,62 @@ mod tests {
transition_compensation_running_tasks: 1,
versions_scanned: 1000000,
};
let runtime_stats = IlmRuntimeStats {
server: "node1:9000".to_string(),
stats,
action_tasks: vec![
IlmActionTaskStats {
action: "expiry".to_string(),
state: "pending".to_string(),
value: 100,
},
IlmActionTaskStats {
action: "transition".to_string(),
state: "queue_send_timeout".to_string(),
value: 3,
},
IlmActionTaskStats {
action: "transition".to_string(),
state: "active".to_string(),
value: 5,
},
],
};
let metrics = collect_ilm_metrics(&stats);
let metrics = collect_ilm_runtime_metrics(&runtime_stats);
assert_eq!(metrics.len(), 9);
assert_eq!(metrics.len(), 11);
let pending = metrics.iter().find(|m| m.value == 100.0);
assert!(pending.is_some());
let scanned = metrics.iter().find(|m| m.value == 1000000.0);
assert!(scanned.is_some());
let transition_timeout = metrics.iter().find(|m| {
m.name == ILM_ACTION_TASKS_MD.get_full_metric_name()
&& m.labels
.iter()
.any(|(name, value)| *name == SERVER_LABEL && value.as_ref() == "node1:9000")
&& m.labels
.iter()
.any(|(name, value)| *name == ACTION_LABEL && value.as_ref() == "transition")
&& m.labels
.iter()
.any(|(name, value)| *name == STATE_LABEL && value.as_ref() == "queue_send_timeout")
});
assert!(transition_timeout.is_none());
let transition_active = metrics.iter().find(|m| {
m.name == ILM_ACTION_TASKS_MD.get_full_metric_name()
&& m.labels
.iter()
.any(|(name, value)| *name == ACTION_LABEL && value.as_ref() == "transition")
&& m.labels
.iter()
.any(|(name, value)| *name == STATE_LABEL && value.as_ref() == "active")
});
assert_eq!(transition_active.map(|m| m.value), Some(5.0));
}
#[test]
crates/obs/src/metrics/collectors/mod.rs
+9 -1
View File
@@ -40,10 +40,12 @@ pub mod system_network;
pub mod system_network_host;
pub mod system_process;
pub(crate) use audit::{AuditTargetRuntimeStats, collect_audit_runtime_metrics};
pub use audit::{AuditTargetStats, collect_audit_metrics};
pub use bucket::{BucketStats, collect_bucket_metrics};
pub(crate) use bucket_replication::{
BucketReplicationBacklogStats, BucketReplicationTargetBacklogStats, collect_bucket_replication_backlog_metrics,
BucketReplicationBacklogStats, BucketReplicationRuntimeStats, BucketReplicationTargetBacklogStats,
BucketReplicationTargetFlowStats, collect_bucket_replication_backlog_metrics, collect_bucket_replication_runtime_metrics,
};
pub use bucket_replication::{
BucketReplicationBandwidthStats, BucketReplicationStats, BucketReplicationTargetStats,
@@ -57,18 +59,24 @@ pub use cluster_iam::{IamStats, collect_iam_metrics};
pub use cluster_usage::{BucketUsageStats, ClusterUsageStats, collect_bucket_usage_metrics, collect_cluster_usage_metrics};
pub use compression::{CompressionClusterStats, collect_compression_cluster_metrics};
pub use dial9::{Dial9Stats, collect_current_dial9_metrics, collect_dial9_metrics, is_dial9_enabled};
pub(crate) use ilm::{IlmActionTaskStats, IlmRuntimeStats, collect_ilm_runtime_metrics};
pub use ilm::{IlmStats, collect_ilm_metrics};
pub use node::{DiskStats, collect_node_metrics};
pub use notification::{NotificationStats, collect_notification_metrics};
pub(crate) use notification_target::{NotificationTargetRuntimeStats, collect_notification_target_runtime_metrics};
pub use notification_target::{NotificationTargetStats, collect_notification_target_metrics};
pub(crate) use replication::{ReplicationRuntimeStats, collect_replication_runtime_metrics};
pub use replication::{ReplicationStats, collect_replication_metrics};
pub(crate) use request::{ApiRequestMetricSupport, ApiRequestStats, collect_request_metrics};
pub use resource::{ResourceStats, collect_resource_metrics};
pub(crate) use scanner::{ScannerRuntimeStats, collect_scanner_runtime_metrics};
pub use scanner::{ScannerStats, collect_scanner_metrics};
pub use system_cpu::{CpuStats, ProcessCpuStats, collect_cpu_metrics, collect_process_cpu_metrics};
pub use system_drive::{
DriveCountStats, DriveDetailedStats, ProcessDiskStats, collect_drive_count_metrics, collect_drive_detailed_metrics,
collect_process_disk_metrics,
};
pub(crate) use system_drive::{DriveRuntimeDetailedStats, collect_drive_runtime_detailed_metrics};
#[cfg(feature = "gpu")]
pub use system_gpu::{GpuCollector, GpuError, GpuStats, collect_gpu_metrics};
pub use system_memory::{MemoryStats, ProcessMemoryStats, collect_memory_metrics, collect_process_memory_metrics};
crates/obs/src/metrics/collectors/notification_target.rs
+98 -6
View File
@@ -16,8 +16,10 @@
use crate::metrics::report::PrometheusMetric;
use crate::metrics::schema::notification_target::{
NOTIFICATION_TARGET_FAILED_MESSAGES_MD, NOTIFICATION_TARGET_FAILED_STORE_LENGTH_MD, NOTIFICATION_TARGET_QUEUE_LENGTH_MD,
NOTIFICATION_TARGET_TOTAL_MESSAGES_MD, TARGET_ID, TARGET_TYPE,
NOTIFICATION_TARGET_FAILED_MESSAGES_BY_SERVER_MD, NOTIFICATION_TARGET_FAILED_MESSAGES_MD,
NOTIFICATION_TARGET_FAILED_STORE_LENGTH_BY_SERVER_MD, NOTIFICATION_TARGET_FAILED_STORE_LENGTH_MD,
NOTIFICATION_TARGET_QUEUE_LENGTH_BY_SERVER_MD, NOTIFICATION_TARGET_QUEUE_LENGTH_MD,
NOTIFICATION_TARGET_TOTAL_MESSAGES_BY_SERVER_MD, NOTIFICATION_TARGET_TOTAL_MESSAGES_MD, SERVER, TARGET_ID, TARGET_TYPE,
};
use std::borrow::Cow;
@@ -31,12 +33,18 @@ pub struct NotificationTargetStats {
pub total_messages: u64,
}
#[derive(Debug, Clone, Default)]
pub(crate) struct NotificationTargetRuntimeStats {
pub(crate) server: String,
pub(crate) target: NotificationTargetStats,
}
pub fn collect_notification_target_metrics(stats: &[NotificationTargetStats]) -> Vec<PrometheusMetric> {
if stats.is_empty() {
return Vec::new();
}
let mut metrics = Vec::with_capacity(stats.len() * 4);
let mut metrics = Vec::with_capacity(stats.len() * 8);
for stat in stats {
let target_id: Cow<'static, str> = Cow::Owned(stat.target_id.clone());
let target_type: Cow<'static, str> = Cow::Owned(stat.target_type.clone());
@@ -66,6 +74,57 @@ pub fn collect_notification_target_metrics(stats: &[NotificationTargetStats]) ->
metrics
}
pub(crate) fn collect_notification_target_runtime_metrics(stats: &[NotificationTargetRuntimeStats]) -> Vec<PrometheusMetric> {
if stats.is_empty() {
return Vec::new();
}
let legacy_stats = stats.iter().map(|stat| stat.target.clone()).collect::<Vec<_>>();
let mut metrics = collect_notification_target_metrics(&legacy_stats);
metrics.reserve(stats.len() * 4);
for stat in stats {
let server: Cow<'static, str> = Cow::Owned(stat.server.clone());
let target_id: Cow<'static, str> = Cow::Owned(stat.target.target_id.clone());
let target_type: Cow<'static, str> = Cow::Owned(stat.target.target_type.clone());
metrics.push(
PrometheusMetric::from_descriptor(
&NOTIFICATION_TARGET_FAILED_MESSAGES_BY_SERVER_MD,
stat.target.failed_messages as f64,
)
.with_label(SERVER, server.clone())
.with_label(TARGET_ID, target_id.clone())
.with_label(TARGET_TYPE, target_type.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(
&NOTIFICATION_TARGET_FAILED_STORE_LENGTH_BY_SERVER_MD,
stat.target.failed_store_length as f64,
)
.with_label(SERVER, server.clone())
.with_label(TARGET_ID, target_id.clone())
.with_label(TARGET_TYPE, target_type.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&NOTIFICATION_TARGET_QUEUE_LENGTH_BY_SERVER_MD, stat.target.queue_length as f64)
.with_label(SERVER, server.clone())
.with_label(TARGET_ID, target_id.clone())
.with_label(TARGET_TYPE, target_type.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(
&NOTIFICATION_TARGET_TOTAL_MESSAGES_BY_SERVER_MD,
stat.target.total_messages as f64,
)
.with_label(SERVER, server)
.with_label(TARGET_ID, target_id)
.with_label(TARGET_TYPE, target_type),
);
}
metrics
}
#[cfg(test)]
mod tests {
use super::*;
@@ -73,7 +132,7 @@ mod tests {
#[test]
fn test_collect_notification_target_metrics() {
let stats = vec![NotificationTargetStats {
let stats = [NotificationTargetStats {
failed_messages: 2,
failed_store_length: 3,
queue_length: 4,
@@ -82,9 +141,12 @@ mod tests {
total_messages: 42,
}];
let metrics = collect_notification_target_metrics(&stats);
let metrics = collect_notification_target_runtime_metrics(&[NotificationTargetRuntimeStats {
server: "node1:9000".to_string(),
target: stats[0].clone(),
}]);
assert_eq!(metrics.len(), 4);
assert_eq!(metrics.len(), 8);
assert!(metrics.iter().any(|metric| {
metric.value == 3.0
&& metric.name == NOTIFICATION_TARGET_FAILED_STORE_LENGTH_MD.get_full_metric_name()
@@ -104,6 +166,22 @@ mod tests {
.iter()
.any(|(key, value)| *key == TARGET_TYPE && value == "webhook")
}));
assert!(metrics.iter().any(|metric| {
metric.value == 4.0
&& metric.name == NOTIFICATION_TARGET_QUEUE_LENGTH_BY_SERVER_MD.get_full_metric_name()
&& metric
.labels
.iter()
.any(|(key, value)| *key == SERVER && value == "node1:9000")
&& metric
.labels
.iter()
.any(|(key, value)| *key == TARGET_ID && value == "primary:webhook")
&& metric
.labels
.iter()
.any(|(key, value)| *key == TARGET_TYPE && value == "webhook")
}));
}
#[test]
@@ -113,4 +191,18 @@ mod tests {
assert_eq!(NOTIFICATION_TARGET_QUEUE_LENGTH_MD.metric_type, MetricType::Gauge);
assert_eq!(NOTIFICATION_TARGET_TOTAL_MESSAGES_MD.metric_type, MetricType::Gauge);
}
#[test]
fn notification_target_stats_struct_literal_keeps_legacy_fields() {
let stats = vec![NotificationTargetStats {
failed_messages: 2,
failed_store_length: 3,
queue_length: 4,
target_id: "primary:webhook".to_string(),
target_type: "webhook".to_string(),
total_messages: 42,
}];
assert_eq!(collect_notification_target_metrics(&stats).len(), 4);
}
}
crates/obs/src/metrics/collectors/replication.rs
+92 -2
View File
@@ -53,6 +53,12 @@ pub struct ReplicationStats {
pub recent_backlog_count: u64,
}
#[derive(Debug, Clone, Default)]
pub(crate) struct ReplicationRuntimeStats {
pub(crate) server: String,
pub(crate) stats: ReplicationStats,
}
/// Collects replication metrics from the given stats.
///
/// Returns a vector of Prometheus metrics for replication statistics.
@@ -74,6 +80,41 @@ pub fn collect_replication_metrics(stats: &ReplicationStats) -> Vec<PrometheusMe
]
}
pub(crate) fn collect_replication_runtime_metrics(runtime: &ReplicationRuntimeStats) -> Vec<PrometheusMetric> {
let stats = &runtime.stats;
let mut metrics = collect_replication_metrics(stats);
metrics.extend([
PrometheusMetric::from_descriptor(&REPLICATION_AVERAGE_ACTIVE_WORKERS_BY_SERVER_MD, stats.average_active_workers)
.with_label_owned(SERVER_LABEL, runtime.server.clone()),
PrometheusMetric::from_descriptor(&REPLICATION_AVERAGE_QUEUED_BYTES_BY_SERVER_MD, stats.average_queued_bytes as f64)
.with_label_owned(SERVER_LABEL, runtime.server.clone()),
PrometheusMetric::from_descriptor(&REPLICATION_AVERAGE_QUEUED_COUNT_BY_SERVER_MD, stats.average_queued_count as f64)
.with_label_owned(SERVER_LABEL, runtime.server.clone()),
PrometheusMetric::from_descriptor(&REPLICATION_CURRENT_ACTIVE_WORKERS_BY_SERVER_MD, stats.active_workers as f64)
.with_label_owned(SERVER_LABEL, runtime.server.clone()),
PrometheusMetric::from_descriptor(&REPLICATION_CURRENT_DATA_TRANSFER_RATE_BY_SERVER_MD, stats.current_data_transfer_rate)
.with_label_owned(SERVER_LABEL, runtime.server.clone()),
PrometheusMetric::from_descriptor(
&REPLICATION_LAST_MINUTE_QUEUED_BYTES_BY_SERVER_MD,
stats.last_minute_queued_bytes as f64,
)
.with_label_owned(SERVER_LABEL, runtime.server.clone()),
PrometheusMetric::from_descriptor(
&REPLICATION_LAST_MINUTE_QUEUED_COUNT_BY_SERVER_MD,
stats.last_minute_queued_count as f64,
)
.with_label_owned(SERVER_LABEL, runtime.server.clone()),
PrometheusMetric::from_descriptor(&REPLICATION_MAX_ACTIVE_WORKERS_BY_SERVER_MD, stats.max_active_workers as f64)
.with_label_owned(SERVER_LABEL, runtime.server.clone()),
PrometheusMetric::from_descriptor(&REPLICATION_MAX_QUEUED_BYTES_BY_SERVER_MD, stats.max_queued_bytes as f64)
.with_label_owned(SERVER_LABEL, runtime.server.clone()),
PrometheusMetric::from_descriptor(&REPLICATION_MAX_QUEUED_COUNT_BY_SERVER_MD, stats.max_queued_count as f64)
.with_label_owned(SERVER_LABEL, runtime.server.clone()),
]);
metrics
}
#[cfg(test)]
mod tests {
use super::*;
@@ -97,10 +138,13 @@ mod tests {
recent_backlog_count: 1500,
};
let metrics = collect_replication_metrics(&stats);
let metrics = collect_replication_runtime_metrics(&ReplicationRuntimeStats {
server: "node-a:9000".to_string(),
stats,
});
report_metrics(&metrics);
assert_eq!(metrics.len(), 13);
assert_eq!(metrics.len(), 23);
// Verify active workers
let active_name = REPLICATION_CURRENT_ACTIVE_WORKERS_MD.get_full_metric_name();
@@ -111,6 +155,31 @@ mod tests {
let avg_active_name = REPLICATION_AVERAGE_ACTIVE_WORKERS_MD.get_full_metric_name();
let avg_active = metrics.iter().find(|m| m.name == avg_active_name);
assert_eq!(avg_active.map(|m| m.value), Some(8.5));
let active_by_server_name = REPLICATION_CURRENT_ACTIVE_WORKERS_BY_SERVER_MD.get_full_metric_name();
let active_by_server = metrics.iter().find(|m| m.name == active_by_server_name);
assert_eq!(active_by_server.map(|m| m.value), Some(10.0));
assert_eq!(
active_by_server
.and_then(|m| m.labels.iter().find(|(name, _)| *name == SERVER_LABEL))
.map(|(_, value)| value.as_ref()),
Some("node-a:9000")
);
assert!(
metrics
.iter()
.all(|m| m.name != REPLICATION_AVERAGE_DATA_TRANSFER_RATE_BY_SERVER_MD.get_full_metric_name())
);
assert!(
metrics
.iter()
.all(|m| m.name != REPLICATION_MAX_DATA_TRANSFER_RATE_BY_SERVER_MD.get_full_metric_name())
);
assert!(
metrics
.iter()
.all(|m| m.name != REPLICATION_RECENT_BACKLOG_COUNT_BY_SERVER_MD.get_full_metric_name())
);
}
#[test]
@@ -124,4 +193,25 @@ mod tests {
assert!(metric.labels.is_empty());
}
}
#[test]
fn replication_stats_struct_literal_keeps_legacy_fields() {
let stats = ReplicationStats {
average_active_workers: 1.0,
average_queued_bytes: 2,
average_queued_count: 3,
average_data_transfer_rate: 4.0,
active_workers: 5,
current_data_transfer_rate: 6.0,
last_minute_queued_bytes: 7,
last_minute_queued_count: 8,
max_active_workers: 9,
max_queued_bytes: 10,
max_queued_count: 11,
max_data_transfer_rate: 12.0,
recent_backlog_count: 13,
};
assert_eq!(collect_replication_metrics(&stats).len(), 13);
}
}
crates/obs/src/metrics/collectors/request.rs
+239 -47
View File
@@ -23,9 +23,32 @@ use crate::metrics::report::PrometheusMetric;
use crate::metrics::schema::request::*;
use std::collections::HashMap;
#[derive(Debug, Clone, Copy, Eq, PartialEq)]
pub(crate) struct ApiRequestMetricSupport {
pub(crate) lifecycle: bool,
pub(crate) traffic: bool,
pub(crate) ttfb: bool,
}
impl ApiRequestMetricSupport {
pub(crate) const ALL: Self = Self {
lifecycle: true,
traffic: true,
ttfb: true,
};
pub(crate) const TOTALS_ONLY: Self = Self {
lifecycle: false,
traffic: false,
ttfb: false,
};
}
/// API request statistics for a specific API endpoint.
#[derive(Debug, Clone, Default)]
#[derive(Debug, Clone)]
pub struct ApiRequestStats {
/// Server identifier
pub server: String,
/// API name (e.g., "GetObject", "PutObject")
pub name: String,
/// Request type (e.g., "s3", "admin")
@@ -48,6 +71,27 @@ pub struct ApiRequestStats {
pub sent_bytes: u64,
/// Bytes received
pub recv_bytes: u64,
pub(crate) supported_metrics: ApiRequestMetricSupport,
}
impl Default for ApiRequestStats {
fn default() -> Self {
Self {
server: String::new(),
name: String::new(),
req_type: String::new(),
in_flight: 0,
total: 0,
errors_total: 0,
errors_5xx: 0,
errors_4xx: 0,
canceled: 0,
ttfb_distribution: Vec::new(),
sent_bytes: 0,
recv_bytes: 0,
supported_metrics: ApiRequestMetricSupport::ALL,
}
}
}
/// Collects API request metrics from the given stats.
@@ -56,62 +100,119 @@ pub struct ApiRequestStats {
pub fn collect_request_metrics(stats: &[ApiRequestStats]) -> Vec<PrometheusMetric> {
let mut metrics = Vec::new();
let mut traffic_by_type: HashMap<&str, (u64, u64)> = HashMap::with_capacity(stats.len());
let mut traffic_by_server_type: HashMap<(&str, &str), (u64, u64)> = HashMap::with_capacity(stats.len());
for stat in stats {
let entry = traffic_by_type.entry(stat.req_type.as_str()).or_default();
entry.0 = entry.0.saturating_add(stat.sent_bytes);
entry.1 = entry.1.saturating_add(stat.recv_bytes);
if stat.supported_metrics.traffic {
let entry = traffic_by_type.entry(stat.req_type.as_str()).or_default();
entry.0 = entry.0.saturating_add(stat.sent_bytes);
entry.1 = entry.1.saturating_add(stat.recv_bytes);
if !stat.server.is_empty() {
let entry = traffic_by_server_type
.entry((stat.server.as_str(), stat.req_type.as_str()))
.or_default();
entry.0 = entry.0.saturating_add(stat.sent_bytes);
entry.1 = entry.1.saturating_add(stat.recv_bytes);
}
}
// In-flight requests
metrics.push(
PrometheusMetric::from_descriptor(&API_REQUESTS_IN_FLIGHT_TOTAL_MD, stat.in_flight as f64)
.with_label_owned(NAME_LABEL, stat.name.clone())
.with_label_owned(TYPE_LABEL, stat.req_type.clone()),
);
// Total requests
metrics.push(
PrometheusMetric::from_descriptor(&API_REQUESTS_TOTAL_MD, stat.total as f64)
.with_label_owned(NAME_LABEL, stat.name.clone())
.with_label_owned(TYPE_LABEL, stat.req_type.clone()),
);
// Total errors
metrics.push(
PrometheusMetric::from_descriptor(&API_REQUESTS_ERRORS_TOTAL_MD, stat.errors_total as f64)
.with_label_owned(NAME_LABEL, stat.name.clone())
.with_label_owned(TYPE_LABEL, stat.req_type.clone()),
);
// 5xx errors
metrics.push(
PrometheusMetric::from_descriptor(&API_REQUESTS_5XX_ERRORS_TOTAL_MD, stat.errors_5xx as f64)
.with_label_owned(NAME_LABEL, stat.name.clone())
.with_label_owned(TYPE_LABEL, stat.req_type.clone()),
);
// 4xx errors
metrics.push(
PrometheusMetric::from_descriptor(&API_REQUESTS_4XX_ERRORS_TOTAL_MD, stat.errors_4xx as f64)
.with_label_owned(NAME_LABEL, stat.name.clone())
.with_label_owned(TYPE_LABEL, stat.req_type.clone()),
);
// Canceled requests
metrics.push(
PrometheusMetric::from_descriptor(&API_REQUESTS_CANCELED_TOTAL_MD, stat.canceled as f64)
.with_label_owned(NAME_LABEL, stat.name.clone())
.with_label_owned(TYPE_LABEL, stat.req_type.clone()),
);
// TTFB distribution (histogram buckets)
for (le, value) in &stat.ttfb_distribution {
if stat.supported_metrics.lifecycle {
metrics.push(
PrometheusMetric::from_descriptor(&API_REQUESTS_TTFB_SECONDS_DISTRIBUTION_MD, *value)
PrometheusMetric::from_descriptor(&API_REQUESTS_IN_FLIGHT_TOTAL_MD, stat.in_flight as f64)
.with_label_owned(NAME_LABEL, stat.name.clone())
.with_label_owned(TYPE_LABEL, stat.req_type.clone())
.with_label_owned(LE_LABEL, le.clone()),
.with_label_owned(TYPE_LABEL, stat.req_type.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&API_REQUESTS_ERRORS_TOTAL_MD, stat.errors_total as f64)
.with_label_owned(NAME_LABEL, stat.name.clone())
.with_label_owned(TYPE_LABEL, stat.req_type.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&API_REQUESTS_5XX_ERRORS_TOTAL_MD, stat.errors_5xx as f64)
.with_label_owned(NAME_LABEL, stat.name.clone())
.with_label_owned(TYPE_LABEL, stat.req_type.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&API_REQUESTS_4XX_ERRORS_TOTAL_MD, stat.errors_4xx as f64)
.with_label_owned(NAME_LABEL, stat.name.clone())
.with_label_owned(TYPE_LABEL, stat.req_type.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&API_REQUESTS_CANCELED_TOTAL_MD, stat.canceled as f64)
.with_label_owned(NAME_LABEL, stat.name.clone())
.with_label_owned(TYPE_LABEL, stat.req_type.clone()),
);
}
if stat.supported_metrics.ttfb {
for (le, value) in &stat.ttfb_distribution {
metrics.push(
PrometheusMetric::from_descriptor(&API_REQUESTS_TTFB_SECONDS_DISTRIBUTION_MD, *value)
.with_label_owned(NAME_LABEL, stat.name.clone())
.with_label_owned(TYPE_LABEL, stat.req_type.clone())
.with_label_owned(LE_LABEL, le.clone()),
);
}
}
if !stat.server.is_empty() {
metrics.push(
PrometheusMetric::from_descriptor(&API_REQUESTS_TOTAL_BY_SERVER_MD, stat.total as f64)
.with_label_owned(SERVER_LABEL, stat.server.clone())
.with_label_owned(NAME_LABEL, stat.name.clone())
.with_label_owned(TYPE_LABEL, stat.req_type.clone()),
);
if stat.supported_metrics.lifecycle {
metrics.push(
PrometheusMetric::from_descriptor(&API_REQUESTS_IN_FLIGHT_TOTAL_BY_SERVER_MD, stat.in_flight as f64)
.with_label_owned(SERVER_LABEL, stat.server.clone())
.with_label_owned(NAME_LABEL, stat.name.clone())
.with_label_owned(TYPE_LABEL, stat.req_type.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&API_REQUESTS_ERRORS_TOTAL_BY_SERVER_MD, stat.errors_total as f64)
.with_label_owned(SERVER_LABEL, stat.server.clone())
.with_label_owned(NAME_LABEL, stat.name.clone())
.with_label_owned(TYPE_LABEL, stat.req_type.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&API_REQUESTS_5XX_ERRORS_TOTAL_BY_SERVER_MD, stat.errors_5xx as f64)
.with_label_owned(SERVER_LABEL, stat.server.clone())
.with_label_owned(NAME_LABEL, stat.name.clone())
.with_label_owned(TYPE_LABEL, stat.req_type.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&API_REQUESTS_4XX_ERRORS_TOTAL_BY_SERVER_MD, stat.errors_4xx as f64)
.with_label_owned(SERVER_LABEL, stat.server.clone())
.with_label_owned(NAME_LABEL, stat.name.clone())
.with_label_owned(TYPE_LABEL, stat.req_type.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&API_REQUESTS_CANCELED_TOTAL_BY_SERVER_MD, stat.canceled as f64)
.with_label_owned(SERVER_LABEL, stat.server.clone())
.with_label_owned(NAME_LABEL, stat.name.clone())
.with_label_owned(TYPE_LABEL, stat.req_type.clone()),
);
}
if stat.supported_metrics.ttfb {
for (le, value) in &stat.ttfb_distribution {
metrics.push(
PrometheusMetric::from_descriptor(&API_REQUESTS_TTFB_SECONDS_DISTRIBUTION_BY_SERVER_MD, *value)
.with_label_owned(SERVER_LABEL, stat.server.clone())
.with_label_owned(NAME_LABEL, stat.name.clone())
.with_label_owned(TYPE_LABEL, stat.req_type.clone())
.with_label_owned(LE_LABEL, le.clone()),
);
}
}
}
}
@@ -126,6 +227,19 @@ pub fn collect_request_metrics(stats: &[ApiRequestStats]) -> Vec<PrometheusMetri
);
}
for ((server, req_type), (sent_bytes, recv_bytes)) in traffic_by_server_type {
metrics.push(
PrometheusMetric::from_descriptor(&API_TRAFFIC_SENT_BYTES_BY_SERVER_MD, sent_bytes as f64)
.with_label_owned(SERVER_LABEL, server.to_string())
.with_label_owned(TYPE_LABEL, req_type.to_string()),
);
metrics.push(
PrometheusMetric::from_descriptor(&API_TRAFFIC_RECV_BYTES_BY_SERVER_MD, recv_bytes as f64)
.with_label_owned(SERVER_LABEL, server.to_string())
.with_label_owned(TYPE_LABEL, req_type.to_string()),
);
}
metrics
}
@@ -137,6 +251,7 @@ mod tests {
#[test]
fn test_collect_request_metrics() {
let stats = vec![ApiRequestStats {
server: "node1:9000".to_string(),
name: "GetObject".to_string(),
req_type: "s3".to_string(),
in_flight: 10,
@@ -153,13 +268,13 @@ mod tests {
],
sent_bytes: 1024 * 1024 * 500, // 500 MB
recv_bytes: 1024 * 1024 * 100, // 100 MB
supported_metrics: ApiRequestMetricSupport::ALL,
}];
let metrics = collect_request_metrics(&stats);
report_metrics(&metrics);
// 6 base metrics + 4 TTFB buckets + 2 traffic metrics = 12
assert_eq!(metrics.len(), 12);
assert_eq!(metrics.len(), 24);
let total_name = API_REQUESTS_TOTAL_MD.get_full_metric_name();
let total = metrics.iter().find(|m| m.name == total_name);
@@ -170,6 +285,27 @@ mod tests {
let in_flight = metrics.iter().find(|m| m.name == in_flight_name);
assert!(in_flight.is_some());
assert_eq!(in_flight.map(|m| m.value), Some(10.0));
let by_server_total_name = API_REQUESTS_TOTAL_BY_SERVER_MD.get_full_metric_name();
let by_server_total = metrics.iter().find(|m| {
m.name == by_server_total_name
&& m.labels
.iter()
.any(|(key, value)| *key == SERVER_LABEL && value == "node1:9000")
&& m.labels.iter().any(|(key, value)| *key == NAME_LABEL && value == "GetObject")
&& m.labels.iter().any(|(key, value)| *key == TYPE_LABEL && value == "s3")
});
assert_eq!(by_server_total.map(|m| m.value), Some(10000.0));
let by_server_sent_name = API_TRAFFIC_SENT_BYTES_BY_SERVER_MD.get_full_metric_name();
let by_server_sent = metrics.iter().find(|m| {
m.name == by_server_sent_name
&& m.labels
.iter()
.any(|(key, value)| *key == SERVER_LABEL && value == "node1:9000")
&& m.labels.iter().any(|(key, value)| *key == TYPE_LABEL && value == "s3")
});
assert_eq!(by_server_sent.map(|m| m.value), Some((1024 * 1024 * 500) as f64));
}
#[test]
@@ -179,10 +315,63 @@ mod tests {
assert!(metrics.is_empty());
}
#[test]
fn test_collect_request_metrics_totals_only_skips_unsupported_dimensions() {
let stats = vec![ApiRequestStats {
server: "node1:9000".to_string(),
name: "GetObject".to_string(),
req_type: "s3".to_string(),
in_flight: 10,
total: 100,
errors_total: 5,
errors_5xx: 2,
errors_4xx: 3,
canceled: 1,
ttfb_distribution: vec![("+Inf".to_string(), 100.0)],
sent_bytes: 2048,
recv_bytes: 1024,
supported_metrics: ApiRequestMetricSupport::TOTALS_ONLY,
}];
let metrics = collect_request_metrics(&stats);
assert!(
metrics
.iter()
.any(|metric| metric.name == API_REQUESTS_TOTAL_MD.get_full_metric_name())
);
assert!(
metrics
.iter()
.any(|metric| metric.name == API_REQUESTS_TOTAL_BY_SERVER_MD.get_full_metric_name())
);
assert!(
!metrics
.iter()
.any(|metric| metric.name == API_REQUESTS_IN_FLIGHT_TOTAL_MD.get_full_metric_name())
);
assert!(
!metrics
.iter()
.any(|metric| metric.name == API_REQUESTS_ERRORS_TOTAL_MD.get_full_metric_name())
);
assert!(
!metrics
.iter()
.any(|metric| metric.name == API_TRAFFIC_SENT_BYTES_MD.get_full_metric_name())
);
assert!(
!metrics
.iter()
.any(|metric| metric.name == API_REQUESTS_TTFB_SECONDS_DISTRIBUTION_MD.get_full_metric_name())
);
}
#[test]
fn test_collect_request_metrics_aggregates_traffic_per_type() {
let stats = vec![
ApiRequestStats {
server: String::new(),
name: "GetObject".to_string(),
req_type: "s3".to_string(),
in_flight: 1,
@@ -194,8 +383,10 @@ mod tests {
ttfb_distribution: vec![],
sent_bytes: 100,
recv_bytes: 10,
supported_metrics: ApiRequestMetricSupport::ALL,
},
ApiRequestStats {
server: String::new(),
name: "HeadObject".to_string(),
req_type: "s3".to_string(),
in_flight: 2,
@@ -207,6 +398,7 @@ mod tests {
ttfb_distribution: vec![],
sent_bytes: 200,
recv_bytes: 20,
supported_metrics: ApiRequestMetricSupport::ALL,
},
];
crates/obs/src/metrics/collectors/scanner.rs
+318 -102
View File
@@ -20,34 +20,7 @@
//! directory scans, and object scans.
use crate::metrics::report::PrometheusMetric;
use crate::metrics::schema::scanner::{
SCANNER_ACTIVE_PATHS_MD, SCANNER_BITROT_CYCLE_ENABLED_MD, SCANNER_BITROT_CYCLE_SECONDS_MD, SCANNER_BUCKET_SCANS_FAILED_MD,
SCANNER_BUCKET_SCANS_FINISHED_MD, SCANNER_BUCKET_SCANS_STARTED_MD, SCANNER_COMPLETED_CYCLES_MD,
SCANNER_CURRENT_CYCLE_AGE_SECONDS_MD, SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_FAILURES_MD,
SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_MD, SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD,
SCANNER_CURRENT_CYCLE_DIRECTORIES_PER_SECOND_MD, SCANNER_CURRENT_CYCLE_DIRECTORIES_SCANNED_MD,
SCANNER_CURRENT_CYCLE_HEAL_OBJECTS_MD, SCANNER_CURRENT_CYCLE_ILM_ACTIONS_MD, SCANNER_CURRENT_CYCLE_MD,
SCANNER_CURRENT_CYCLE_OBJECTS_PER_SECOND_MD, SCANNER_CURRENT_CYCLE_OBJECTS_SCANNED_MD,
SCANNER_CURRENT_CYCLE_REPLICATION_CHECKS_MD, SCANNER_CURRENT_CYCLE_THROTTLE_SLEEP_DURATION_SECONDS_MD,
SCANNER_CURRENT_CYCLE_THROTTLE_SLEEP_EVENTS_MD, SCANNER_CURRENT_CYCLE_USAGE_SAVES_MD,
SCANNER_CURRENT_CYCLE_YIELD_DURATION_SECONDS_MD, SCANNER_CURRENT_CYCLE_YIELD_EVENTS_MD,
SCANNER_CURRENT_DISK_BUCKET_SCANS_ACTIVE_MD, SCANNER_CURRENT_DISK_BUCKET_SCANS_QUEUED_MD,
SCANNER_CURRENT_DISK_SCAN_CONCURRENCY_LIMIT_MD, SCANNER_CURRENT_SCAN_MODE_MD, SCANNER_CURRENT_SET_SCAN_CONCURRENCY_LIMIT_MD,
SCANNER_CURRENT_SET_SCANS_ACTIVE_MD, SCANNER_CURRENT_SET_SCANS_QUEUED_MD, SCANNER_CYCLE_INTERVAL_SECONDS_MD,
SCANNER_CYCLE_MAX_DIRECTORIES_MD, SCANNER_CYCLE_MAX_DURATION_SECONDS_MD, SCANNER_CYCLE_MAX_OBJECTS_MD,
SCANNER_DIRECTORIES_SCANNED_MD, SCANNER_FAILED_CYCLES_MD, SCANNER_LAST_ACTIVITY_SECONDS_MD,
SCANNER_LAST_CYCLE_BUCKET_DRIVE_FAILURES_MD, SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_MD,
SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD, SCANNER_LAST_CYCLE_DIRECTORIES_PER_SECOND_MD,
SCANNER_LAST_CYCLE_DIRECTORIES_SCANNED_MD, SCANNER_LAST_CYCLE_DURATION_SECONDS_MD, SCANNER_LAST_CYCLE_HEAL_OBJECTS_MD,
SCANNER_LAST_CYCLE_ILM_ACTIONS_MD, SCANNER_LAST_CYCLE_OBJECTS_PER_SECOND_MD, SCANNER_LAST_CYCLE_OBJECTS_SCANNED_MD,
SCANNER_LAST_CYCLE_PARTIAL_REASON_MD, SCANNER_LAST_CYCLE_REPLICATION_CHECKS_MD, SCANNER_LAST_CYCLE_RESULT_MD,
SCANNER_LAST_CYCLE_THROTTLE_SLEEP_DURATION_SECONDS_MD, SCANNER_LAST_CYCLE_THROTTLE_SLEEP_EVENTS_MD,
SCANNER_LAST_CYCLE_USAGE_SAVES_MD, SCANNER_LAST_CYCLE_YIELD_DURATION_SECONDS_MD, SCANNER_LAST_CYCLE_YIELD_EVENTS_MD,
SCANNER_OBJECTS_SCANNED_MD, SCANNER_OLDEST_ACTIVE_PATH_AGE_SECONDS_MD, SCANNER_PARTIAL_CYCLES_BY_REASON_MD,
SCANNER_PARTIAL_CYCLES_MD, SCANNER_SUPERSEDED_CYCLES_MD, SCANNER_THROTTLE_IDLE_MODE_ENABLED_MD,
SCANNER_THROTTLE_MAX_SLEEP_SECONDS_MD, SCANNER_THROTTLE_SLEEP_FACTOR_MD, SCANNER_VERSIONS_SCANNED_MD,
SCANNER_YIELD_EVERY_N_OBJECTS_MD,
};
use crate::metrics::schema::scanner::*;
/// Scanner statistics.
#[derive(Debug, Clone, Default)]
@@ -192,12 +165,123 @@ pub struct ScannerStats {
pub partial_cycles_directories: u64,
}
/// Scanner source-work metrics for a source.
#[derive(Debug, Clone, Default)]
pub struct ScannerSourceWorkStats {
pub source: String,
pub checked: u64,
pub queued: u64,
pub executed: u64,
pub failed: u64,
pub skipped: u64,
pub missed: u64,
}
/// Scanner bucket-drive result metrics for a structured bucket/drive pair.
#[derive(Debug, Clone, Default)]
pub struct ScannerBucketDriveResultStats {
pub bucket: String,
pub drive: String,
pub result: String,
pub count: u64,
}
/// Scanner statistics with runtime-local node identity and bounded source/result details.
#[derive(Debug, Clone, Default)]
pub(crate) struct ScannerRuntimeStats {
pub(crate) server: String,
pub(crate) stats: ScannerStats,
pub(crate) source_work: Vec<ScannerSourceWorkStats>,
pub(crate) current_cycle_source_work: Vec<ScannerSourceWorkStats>,
pub(crate) last_cycle_source_work: Vec<ScannerSourceWorkStats>,
pub(crate) bucket_drive_results: Vec<ScannerBucketDriveResultStats>,
pub(crate) current_cycle_bucket_drive_results: Vec<ScannerBucketDriveResultStats>,
pub(crate) last_cycle_bucket_drive_results: Vec<ScannerBucketDriveResultStats>,
}
/// Collects scanner metrics from the given stats.
///
/// Uses the metric descriptors from `metrics_type::scanner` module.
/// Returns a vector of Prometheus metrics for scanner statistics.
pub fn collect_scanner_metrics(stats: &ScannerStats) -> Vec<PrometheusMetric> {
vec![
collect_scanner_metrics_with_runtime(stats, None)
}
pub(crate) fn collect_scanner_runtime_metrics(stats: &ScannerRuntimeStats) -> Vec<PrometheusMetric> {
collect_scanner_metrics_with_runtime(&stats.stats, Some(stats))
}
fn collect_scanner_metrics_with_runtime(stats: &ScannerStats, runtime: Option<&ScannerRuntimeStats>) -> Vec<PrometheusMetric> {
fn push_source_work_metric(
metrics: &mut Vec<PrometheusMetric>,
descriptor: &'static crate::metrics::schema::MetricDescriptor,
server: &str,
source: &str,
state: &str,
value: u64,
cycle_scope: Option<&str>,
) {
let mut metric =
PrometheusMetric::from_descriptor(descriptor, value as f64).with_label_owned(SERVER_LABEL, server.to_string());
if let Some(cycle_scope) = cycle_scope {
metric = metric.with_label_owned(CYCLE_SCOPE_LABEL, cycle_scope.to_string());
}
metric = metric
.with_label_owned(SOURCE_LABEL, source.to_string())
.with_label_owned(STATE_LABEL, state.to_string());
metrics.push(metric);
}
fn push_source_work_metrics(
metrics: &mut Vec<PrometheusMetric>,
descriptor: &'static crate::metrics::schema::MetricDescriptor,
server: &str,
source_work: &[ScannerSourceWorkStats],
cycle_scope: Option<&str>,
) {
for work in source_work {
push_source_work_metric(metrics, descriptor, server, &work.source, "checked", work.checked, cycle_scope);
push_source_work_metric(metrics, descriptor, server, &work.source, "queued", work.queued, cycle_scope);
push_source_work_metric(metrics, descriptor, server, &work.source, "executed", work.executed, cycle_scope);
push_source_work_metric(metrics, descriptor, server, &work.source, "failed", work.failed, cycle_scope);
push_source_work_metric(metrics, descriptor, server, &work.source, "skipped", work.skipped, cycle_scope);
push_source_work_metric(metrics, descriptor, server, &work.source, "missed", work.missed, cycle_scope);
}
}
fn push_bucket_drive_result_metric(
metrics: &mut Vec<PrometheusMetric>,
descriptor: &'static crate::metrics::schema::MetricDescriptor,
server: &str,
result: &ScannerBucketDriveResultStats,
cycle_scope: Option<&str>,
) {
let mut metric =
PrometheusMetric::from_descriptor(descriptor, result.count as f64).with_label_owned(SERVER_LABEL, server.to_string());
if let Some(cycle_scope) = cycle_scope {
metric = metric.with_label_owned(CYCLE_SCOPE_LABEL, cycle_scope.to_string());
}
metrics.push(
metric
.with_label_owned(BUCKET_LABEL, result.bucket.clone())
.with_label_owned(DRIVE_LABEL, result.drive.clone())
.with_label_owned(RESULT_LABEL, result.result.clone()),
);
}
fn push_bucket_drive_result_metrics(
metrics: &mut Vec<PrometheusMetric>,
descriptor: &'static crate::metrics::schema::MetricDescriptor,
server: &str,
results: &[ScannerBucketDriveResultStats],
cycle_scope: Option<&str>,
) {
for result in results {
push_bucket_drive_result_metric(metrics, descriptor, server, result, cycle_scope);
}
}
let mut metrics = vec![
PrometheusMetric::from_descriptor(&SCANNER_BUCKET_SCANS_FINISHED_MD, stats.bucket_scans_finished as f64),
PrometheusMetric::from_descriptor(&SCANNER_BUCKET_SCANS_STARTED_MD, stats.bucket_scans_started as f64),
PrometheusMetric::from_descriptor(&SCANNER_BUCKET_SCANS_FAILED_MD, stats.bucket_scans_failed as f64),
@@ -331,7 +415,48 @@ pub fn collect_scanner_metrics(stats: &ScannerStats) -> Vec<PrometheusMetric> {
.with_label("reason", "objects"),
PrometheusMetric::from_descriptor(&SCANNER_PARTIAL_CYCLES_BY_REASON_MD, stats.partial_cycles_directories as f64)
.with_label("reason", "directories"),
]
];
if let Some(runtime) = runtime {
push_source_work_metrics(&mut metrics, &SCANNER_SOURCE_WORK_TOTAL_MD, &runtime.server, &runtime.source_work, None);
push_source_work_metrics(
&mut metrics,
&SCANNER_CYCLE_SOURCE_WORK_MD,
&runtime.server,
&runtime.current_cycle_source_work,
Some("current"),
);
push_source_work_metrics(
&mut metrics,
&SCANNER_CYCLE_SOURCE_WORK_MD,
&runtime.server,
&runtime.last_cycle_source_work,
Some("last"),
);
push_bucket_drive_result_metrics(
&mut metrics,
&SCANNER_BUCKET_DRIVE_RESULT_TOTAL_MD,
&runtime.server,
&runtime.bucket_drive_results,
None,
);
push_bucket_drive_result_metrics(
&mut metrics,
&SCANNER_CYCLE_BUCKET_DRIVE_RESULT_MD,
&runtime.server,
&runtime.current_cycle_bucket_drive_results,
Some("current"),
);
push_bucket_drive_result_metrics(
&mut metrics,
&SCANNER_CYCLE_BUCKET_DRIVE_RESULT_MD,
&runtime.server,
&runtime.last_cycle_bucket_drive_results,
Some("last"),
);
}
metrics
}
fn bool_metric_value(enabled: bool) -> f64 {
@@ -345,82 +470,130 @@ mod tests {
#[test]
fn test_collect_scanner_metrics() {
let stats = ScannerStats {
bucket_scans_finished: 100,
bucket_scans_started: 100,
bucket_scans_failed: 2,
directories_scanned: 50000,
objects_scanned: 1000000,
versions_scanned: 1500000,
last_activity_seconds: 30,
active_paths: 4,
oldest_active_path_age_seconds: 17,
current_set_scan_concurrency_limit: 3,
current_set_scans_queued: 5,
current_set_scans_active: 2,
current_disk_scan_concurrency_limit: 6,
current_disk_bucket_scans_queued: 18,
current_disk_bucket_scans_active: 4,
throttle_idle_mode_enabled: true,
throttle_sleep_factor: 10.0,
throttle_max_sleep_seconds: 15.0,
yield_every_n_objects: 128,
cycle_interval_seconds: 3600.0,
cycle_max_duration_seconds: 1800.0,
cycle_max_objects: 1_000_000,
cycle_max_directories: 100_000,
bitrot_cycle_enabled: true,
bitrot_cycle_seconds: 86400.0,
current_cycle: 12,
completed_cycles: 11,
current_cycle_age_seconds: 90,
current_cycle_objects_scanned: 250,
current_cycle_directories_scanned: 20,
current_cycle_bucket_drive_scans: 2,
current_cycle_bucket_drive_failures: 1,
current_cycle_objects_per_second: 12.5,
current_cycle_directories_per_second: 1.0,
current_cycle_bucket_drive_scans_per_second: 0.1,
current_cycle_yield_events: 8,
current_cycle_yield_duration_seconds: 1.25,
current_cycle_throttle_sleep_events: 4,
current_cycle_throttle_sleep_duration_seconds: 2.5,
current_cycle_ilm_actions: 6,
current_cycle_heal_objects: 2,
current_cycle_replication_checks: 5,
current_cycle_usage_saves: 3,
current_scan_mode: 2,
last_cycle_result: 1,
last_cycle_partial_reason: 3,
last_cycle_duration_seconds: 42.5,
last_cycle_objects_scanned: 900,
last_cycle_directories_scanned: 80,
last_cycle_bucket_drive_scans: 6,
last_cycle_bucket_drive_failures: 2,
last_cycle_objects_per_second: 18.0,
last_cycle_directories_per_second: 1.6,
last_cycle_bucket_drive_scans_per_second: 0.12,
last_cycle_yield_events: 30,
last_cycle_yield_duration_seconds: 9.5,
last_cycle_throttle_sleep_events: 12,
last_cycle_throttle_sleep_duration_seconds: 6.75,
last_cycle_ilm_actions: 44,
last_cycle_heal_objects: 7,
last_cycle_replication_checks: 12,
last_cycle_usage_saves: 9,
failed_cycles: 3,
superseded_cycles: 5,
partial_cycles: 10,
partial_cycles_unknown: 1,
partial_cycles_runtime: 2,
partial_cycles_objects: 3,
partial_cycles_directories: 4,
let stats = ScannerRuntimeStats {
server: "node1:9000".to_string(),
source_work: vec![ScannerSourceWorkStats {
source: "lifecycle".to_string(),
checked: 11,
queued: 2,
executed: 3,
failed: 4,
skipped: 5,
missed: 6,
}],
current_cycle_source_work: vec![ScannerSourceWorkStats {
source: "usage".to_string(),
checked: 21,
queued: 7,
executed: 8,
failed: 9,
skipped: 10,
missed: 11,
}],
last_cycle_source_work: vec![ScannerSourceWorkStats {
source: "heal".to_string(),
checked: 31,
queued: 12,
executed: 13,
failed: 14,
skipped: 15,
missed: 16,
}],
bucket_drive_results: vec![ScannerBucketDriveResultStats {
bucket: "photos".to_string(),
drive: "/data1".to_string(),
result: "success".to_string(),
count: 3,
}],
current_cycle_bucket_drive_results: vec![ScannerBucketDriveResultStats {
bucket: "photos".to_string(),
drive: "/data1".to_string(),
result: "partial".to_string(),
count: 1,
}],
last_cycle_bucket_drive_results: vec![ScannerBucketDriveResultStats {
bucket: "videos".to_string(),
drive: "/data2".to_string(),
result: "error".to_string(),
count: 2,
}],
stats: ScannerStats {
bucket_scans_finished: 100,
bucket_scans_started: 100,
bucket_scans_failed: 2,
directories_scanned: 50000,
objects_scanned: 1000000,
versions_scanned: 1500000,
last_activity_seconds: 30,
active_paths: 4,
oldest_active_path_age_seconds: 17,
current_set_scan_concurrency_limit: 3,
current_set_scans_queued: 5,
current_set_scans_active: 2,
current_disk_scan_concurrency_limit: 6,
current_disk_bucket_scans_queued: 18,
current_disk_bucket_scans_active: 4,
throttle_idle_mode_enabled: true,
throttle_sleep_factor: 10.0,
throttle_max_sleep_seconds: 15.0,
yield_every_n_objects: 128,
cycle_interval_seconds: 3600.0,
cycle_max_duration_seconds: 1800.0,
cycle_max_objects: 1_000_000,
cycle_max_directories: 100_000,
bitrot_cycle_enabled: true,
bitrot_cycle_seconds: 86400.0,
current_cycle: 12,
completed_cycles: 11,
current_cycle_age_seconds: 90,
current_cycle_objects_scanned: 250,
current_cycle_directories_scanned: 20,
current_cycle_bucket_drive_scans: 2,
current_cycle_bucket_drive_failures: 1,
current_cycle_objects_per_second: 12.5,
current_cycle_directories_per_second: 1.0,
current_cycle_bucket_drive_scans_per_second: 0.1,
current_cycle_yield_events: 8,
current_cycle_yield_duration_seconds: 1.25,
current_cycle_throttle_sleep_events: 4,
current_cycle_throttle_sleep_duration_seconds: 2.5,
current_cycle_ilm_actions: 6,
current_cycle_heal_objects: 2,
current_cycle_replication_checks: 5,
current_cycle_usage_saves: 3,
current_scan_mode: 2,
last_cycle_result: 1,
last_cycle_partial_reason: 3,
last_cycle_duration_seconds: 42.5,
last_cycle_objects_scanned: 900,
last_cycle_directories_scanned: 80,
last_cycle_bucket_drive_scans: 6,
last_cycle_bucket_drive_failures: 2,
last_cycle_objects_per_second: 18.0,
last_cycle_directories_per_second: 1.6,
last_cycle_bucket_drive_scans_per_second: 0.12,
last_cycle_yield_events: 30,
last_cycle_yield_duration_seconds: 9.5,
last_cycle_throttle_sleep_events: 12,
last_cycle_throttle_sleep_duration_seconds: 6.75,
last_cycle_ilm_actions: 44,
last_cycle_heal_objects: 7,
last_cycle_replication_checks: 12,
last_cycle_usage_saves: 9,
failed_cycles: 3,
superseded_cycles: 5,
partial_cycles: 10,
partial_cycles_unknown: 1,
partial_cycles_runtime: 2,
partial_cycles_objects: 3,
partial_cycles_directories: 4,
},
};
let metrics = collect_scanner_metrics(&stats);
let metrics = collect_scanner_runtime_metrics(&stats);
report_metrics(&metrics);
assert_eq!(metrics.len(), 69);
assert_eq!(metrics.len(), 90);
let objects = metrics.iter().find(|m| m.value == 1000000.0);
assert!(objects.is_some());
@@ -432,6 +605,18 @@ mod tests {
.iter()
.find(|m| m.name == SCANNER_ACTIVE_PATHS_MD.get_full_metric_name());
assert_eq!(active_paths.map(|m| m.value), Some(4.0));
assert_eq!(active_paths.map(|m| m.labels.len()), Some(0));
let bucket_drive_result = metrics
.iter()
.find(|m| m.name == SCANNER_BUCKET_DRIVE_RESULT_TOTAL_MD.get_full_metric_name());
assert_eq!(bucket_drive_result.map(|m| m.value), Some(3.0));
assert_eq!(
bucket_drive_result
.and_then(|m| m.labels.iter().find(|(name, _)| *name == BUCKET_LABEL))
.map(|(_, value)| value.as_ref()),
Some("photos")
);
let oldest_active_path_age = metrics
.iter()
@@ -746,6 +931,37 @@ mod tests {
.any(|(name, value)| *name == "reason" && value.as_ref() == "directories")
});
assert_eq!(partial_cycles_directories.map(|m| m.value), Some(4.0));
let lifecycle_failed = metrics.iter().find(|m| {
m.name == SCANNER_SOURCE_WORK_TOTAL_MD.get_full_metric_name()
&& m.labels
.iter()
.any(|(name, value)| *name == SERVER_LABEL && value.as_ref() == "node1:9000")
&& m.labels
.iter()
.any(|(name, value)| *name == SOURCE_LABEL && value.as_ref() == "lifecycle")
&& m.labels
.iter()
.any(|(name, value)| *name == STATE_LABEL && value.as_ref() == "failed")
});
assert_eq!(lifecycle_failed.map(|m| m.value), Some(4.0));
let current_usage_executed = metrics.iter().find(|m| {
m.name == SCANNER_CYCLE_SOURCE_WORK_MD.get_full_metric_name()
&& m.labels
.iter()
.any(|(name, value)| *name == SERVER_LABEL && value.as_ref() == "node1:9000")
&& m.labels
.iter()
.any(|(name, value)| *name == CYCLE_SCOPE_LABEL && value.as_ref() == "current")
&& m.labels
.iter()
.any(|(name, value)| *name == SOURCE_LABEL && value.as_ref() == "usage")
&& m.labels
.iter()
.any(|(name, value)| *name == STATE_LABEL && value.as_ref() == "executed")
});
assert_eq!(current_usage_executed.map(|m| m.value), Some(8.0));
}
#[test]
crates/obs/src/metrics/collectors/system_drive.rs
+321 -50
View File
@@ -78,6 +78,24 @@ pub struct DriveDetailedStats {
pub perc_util: Option<f64>,
}
/// Detailed drive statistics with runtime topology and per-operation dimensions.
#[derive(Debug, Clone, Default)]
pub(crate) struct DriveRuntimeDetailedStats {
pub(crate) stats: DriveDetailedStats,
pub(crate) pool_index: Option<String>,
pub(crate) set_index: Option<String>,
pub(crate) drive_index: Option<String>,
pub(crate) disk_id: Option<String>,
pub(crate) runtime_state: Option<String>,
pub(crate) healing: bool,
pub(crate) scanning: bool,
pub(crate) offline_duration_seconds: Option<u64>,
/// Drive API calls by operation
pub(crate) api_calls: Vec<(String, u64)>,
/// Last-minute API latency by operation, in microseconds
pub(crate) api_latency_by_api_micros: Vec<(String, u64)>,
}
/// Aggregate drive count statistics.
#[derive(Debug, Clone, Default)]
pub struct DriveCountStats {
@@ -93,6 +111,58 @@ pub struct DriveCountStats {
///
/// Returns a vector of Prometheus metrics for each drive.
pub fn collect_drive_detailed_metrics(stats: &[DriveDetailedStats]) -> Vec<PrometheusMetric> {
let runtime_stats = stats
.iter()
.cloned()
.map(|stats| DriveRuntimeDetailedStats {
stats,
..Default::default()
})
.collect::<Vec<_>>();
collect_drive_runtime_detailed_metrics(&runtime_stats)
}
pub(crate) fn collect_drive_runtime_detailed_metrics(stats: &[DriveRuntimeDetailedStats]) -> Vec<PrometheusMetric> {
const DRIVE_RUNTIME_STATES: [&str; 5] = ["online", "offline", "returning", "suspect", "unknown"];
fn topology_labels(stat: &DriveRuntimeDetailedStats) -> Option<[Cow<'static, str>; 5]> {
Some([
Cow::Owned(stat.stats.server.clone()),
Cow::Owned(stat.stats.drive.clone()),
Cow::Owned(stat.pool_index.as_ref()?.clone()),
Cow::Owned(stat.set_index.as_ref()?.clone()),
Cow::Owned(stat.drive_index.as_ref()?.clone()),
])
}
fn has_topology_labels(stat: &DriveRuntimeDetailedStats) -> bool {
stat.pool_index.is_some() && stat.set_index.is_some() && stat.drive_index.is_some()
}
fn normalized_runtime_state(runtime_state: &str) -> &str {
DRIVE_RUNTIME_STATES
.iter()
.copied()
.find(|state| state.eq_ignore_ascii_case(runtime_state))
.unwrap_or("unknown")
}
fn push_topology_metric(
metrics: &mut Vec<PrometheusMetric>,
descriptor: &'static crate::metrics::schema::MetricDescriptor,
value: f64,
labels: &[Cow<'static, str>; 5],
) {
metrics.push(
PrometheusMetric::from_descriptor(descriptor, value)
.with_label(SERVER_LABEL, labels[0].clone())
.with_label(DRIVE_LABEL, labels[1].clone())
.with_label(POOL_INDEX_LABEL, labels[2].clone())
.with_label(SET_INDEX_LABEL, labels[3].clone())
.with_label(DRIVE_INDEX_LABEL, labels[4].clone()),
);
}
fn push_drive_metric(
metrics: &mut Vec<PrometheusMetric>,
descriptor: &'static crate::metrics::schema::MetricDescriptor,
@@ -107,19 +177,49 @@ pub fn collect_drive_detailed_metrics(stats: &[DriveDetailedStats]) -> Vec<Prome
);
}
let mut metrics = Vec::with_capacity(stats.len() * 23);
let metric_capacity = stats
.iter()
.map(|stat| {
let api_metrics = if has_topology_labels(stat) {
stat.api_calls.len() + stat.api_latency_by_api_micros.len()
} else {
0
};
31 + api_metrics
})
.sum();
let mut metrics = Vec::with_capacity(metric_capacity);
for stat in stats {
let server_label = stat.server.as_str();
let drive_label = stat.drive.as_str();
let server_label = stat.stats.server.as_str();
let drive_label = stat.stats.drive.as_str();
let topology_labels = topology_labels(stat);
push_drive_metric(&mut metrics, &DRIVE_TOTAL_BYTES_MD, stat.total_bytes as f64, server_label, drive_label);
push_drive_metric(&mut metrics, &DRIVE_USED_BYTES_MD, stat.used_bytes as f64, server_label, drive_label);
push_drive_metric(&mut metrics, &DRIVE_FREE_BYTES_MD, stat.free_bytes as f64, server_label, drive_label);
push_drive_metric(
&mut metrics,
&DRIVE_TOTAL_BYTES_MD,
stat.stats.total_bytes as f64,
server_label,
drive_label,
);
push_drive_metric(
&mut metrics,
&DRIVE_USED_BYTES_MD,
stat.stats.used_bytes as f64,
server_label,
drive_label,
);
push_drive_metric(
&mut metrics,
&DRIVE_FREE_BYTES_MD,
stat.stats.free_bytes as f64,
server_label,
drive_label,
);
push_drive_metric(
&mut metrics,
&DRIVE_CAPACITY_OBSERVATION_AGE_SECONDS_MD,
stat.capacity_observation_age_seconds as f64,
stat.stats.capacity_observation_age_seconds as f64,
server_label,
drive_label,
);
@@ -127,59 +227,123 @@ pub fn collect_drive_detailed_metrics(stats: &[DriveDetailedStats]) -> Vec<Prome
metrics.push(
PrometheusMetric::from_descriptor(
&DRIVE_CAPACITY_OBSERVATION_STATE_MD,
if state == stat.capacity_observation_state { 1.0 } else { 0.0 },
if state == stat.stats.capacity_observation_state {
1.0
} else {
0.0
},
)
.with_label_owned(SERVER_LABEL, server_label.to_string())
.with_label_owned(DRIVE_LABEL, drive_label.to_string())
.with_label_owned("state", state.to_string()),
);
}
if let Some(value) = stat.used_inodes {
if let Some(value) = stat.stats.used_inodes {
push_drive_metric(&mut metrics, &DRIVE_USED_INODES_MD, value as f64, server_label, drive_label);
}
if let Some(value) = stat.free_inodes {
if let Some(value) = stat.stats.free_inodes {
push_drive_metric(&mut metrics, &DRIVE_FREE_INODES_MD, value as f64, server_label, drive_label);
}
if let Some(value) = stat.total_inodes {
if let Some(value) = stat.stats.total_inodes {
push_drive_metric(&mut metrics, &DRIVE_TOTAL_INODES_MD, value as f64, server_label, drive_label);
}
if let Some(value) = stat.timeout_errors_total {
if let Some(value) = stat.stats.timeout_errors_total {
push_drive_metric(&mut metrics, &DRIVE_TIMEOUT_ERRORS_MD, value as f64, server_label, drive_label);
}
if let Some(value) = stat.io_errors_total {
if let Some(value) = stat.stats.io_errors_total {
push_drive_metric(&mut metrics, &DRIVE_IO_ERRORS_MD, value as f64, server_label, drive_label);
}
if let Some(value) = stat.availability_errors_total {
if let Some(value) = stat.stats.availability_errors_total {
push_drive_metric(&mut metrics, &DRIVE_AVAILABILITY_ERRORS_MD, value as f64, server_label, drive_label);
}
if let Some(value) = stat.waiting_io {
if let Some(value) = stat.stats.waiting_io {
push_drive_metric(&mut metrics, &DRIVE_WAITING_IO_MD, value as f64, server_label, drive_label);
}
if let Some(value) = stat.api_latency_micros {
if let Some(value) = stat.stats.api_latency_micros {
push_drive_metric(&mut metrics, &DRIVE_API_LATENCY_MD, value as f64, server_label, drive_label);
}
push_drive_metric(&mut metrics, &DRIVE_HEALTH_MD, stat.health as f64, server_label, drive_label);
if let Some(value) = stat.reads_per_sec {
push_drive_metric(&mut metrics, &DRIVE_HEALTH_MD, stat.stats.health as f64, server_label, drive_label);
if let Some(value) = stat.stats.reads_per_sec {
push_drive_metric(&mut metrics, &DRIVE_READS_PER_SEC_MD, value, server_label, drive_label);
}
if let Some(value) = stat.reads_kb_per_sec {
if let Some(value) = stat.stats.reads_kb_per_sec {
push_drive_metric(&mut metrics, &DRIVE_READS_KB_PER_SEC_MD, value, server_label, drive_label);
}
if let Some(value) = stat.reads_await {
if let Some(value) = stat.stats.reads_await {
push_drive_metric(&mut metrics, &DRIVE_READS_AWAIT_MD, value, server_label, drive_label);
}
if let Some(value) = stat.writes_per_sec {
if let Some(value) = stat.stats.writes_per_sec {
push_drive_metric(&mut metrics, &DRIVE_WRITES_PER_SEC_MD, value, server_label, drive_label);
}
if let Some(value) = stat.writes_kb_per_sec {
if let Some(value) = stat.stats.writes_kb_per_sec {
push_drive_metric(&mut metrics, &DRIVE_WRITES_KB_PER_SEC_MD, value, server_label, drive_label);
}
if let Some(value) = stat.writes_await {
if let Some(value) = stat.stats.writes_await {
push_drive_metric(&mut metrics, &DRIVE_WRITES_AWAIT_MD, value, server_label, drive_label);
}
if let Some(value) = stat.perc_util {
if let Some(value) = stat.stats.perc_util {
push_drive_metric(&mut metrics, &DRIVE_PERC_UTIL_MD, value, server_label, drive_label);
}
if let Some(labels) = &topology_labels {
if let Some(disk_id) = stat.disk_id.as_ref().filter(|disk_id| !disk_id.is_empty()) {
metrics.push(
PrometheusMetric::from_descriptor(&DRIVE_INFO_MD, 1.0)
.with_label(SERVER_LABEL, labels[0].clone())
.with_label(DRIVE_LABEL, labels[1].clone())
.with_label(POOL_INDEX_LABEL, labels[2].clone())
.with_label(SET_INDEX_LABEL, labels[3].clone())
.with_label(DRIVE_INDEX_LABEL, labels[4].clone())
.with_label_owned(DISK_ID_LABEL, disk_id.clone()),
);
}
if let Some(runtime_state) = stat.runtime_state.as_ref().filter(|state| !state.is_empty()) {
let runtime_state = normalized_runtime_state(runtime_state);
for state in DRIVE_RUNTIME_STATES {
metrics.push(
PrometheusMetric::from_descriptor(
&DRIVE_RUNTIME_STATE_MD,
if state == runtime_state { 1.0 } else { 0.0 },
)
.with_label(SERVER_LABEL, labels[0].clone())
.with_label(DRIVE_LABEL, labels[1].clone())
.with_label(POOL_INDEX_LABEL, labels[2].clone())
.with_label(SET_INDEX_LABEL, labels[3].clone())
.with_label(DRIVE_INDEX_LABEL, labels[4].clone())
.with_label(STATE_LABEL, state),
);
}
}
push_topology_metric(&mut metrics, &DRIVE_HEALING_MD, if stat.healing { 1.0 } else { 0.0 }, labels);
push_topology_metric(&mut metrics, &DRIVE_SCANNING_MD, if stat.scanning { 1.0 } else { 0.0 }, labels);
push_topology_metric(
&mut metrics,
&DRIVE_OFFLINE_DURATION_SECONDS_MD,
stat.offline_duration_seconds.unwrap_or(0) as f64,
labels,
);
for (api, value) in &stat.api_calls {
metrics.push(
PrometheusMetric::from_descriptor(&DRIVE_API_CALLS_MD, *value as f64)
.with_label(SERVER_LABEL, labels[0].clone())
.with_label(DRIVE_LABEL, labels[1].clone())
.with_label(POOL_INDEX_LABEL, labels[2].clone())
.with_label(SET_INDEX_LABEL, labels[3].clone())
.with_label(DRIVE_INDEX_LABEL, labels[4].clone())
.with_label_owned(API_LABEL, api.clone()),
);
}
for (api, value) in &stat.api_latency_by_api_micros {
metrics.push(
PrometheusMetric::from_descriptor(&DRIVE_API_LATENCY_BY_API_MD, *value as f64)
.with_label(SERVER_LABEL, labels[0].clone())
.with_label(DRIVE_LABEL, labels[1].clone())
.with_label(POOL_INDEX_LABEL, labels[2].clone())
.with_label(SET_INDEX_LABEL, labels[3].clone())
.with_label(DRIVE_INDEX_LABEL, labels[4].clone())
.with_label_owned(API_LABEL, api.clone()),
);
}
}
}
metrics
@@ -259,36 +423,48 @@ mod tests {
#[test]
fn test_collect_drive_detailed_metrics() {
let stats = vec![DriveDetailedStats {
server: "node1:9000".to_string(),
drive: "/data/disk1".to_string(),
total_bytes: 1024 * 1024 * 1024 * 100, // 100 GB
used_bytes: 1024 * 1024 * 1024 * 50, // 50 GB
free_bytes: 1024 * 1024 * 1024 * 50, // 50 GB
capacity_observation_state: "live",
capacity_observation_age_seconds: 0,
used_inodes: Some(100000),
free_inodes: Some(900000),
total_inodes: Some(1000000),
timeout_errors_total: Some(5),
io_errors_total: Some(10),
availability_errors_total: Some(2),
waiting_io: Some(3),
api_latency_micros: Some(1500),
health: 1,
reads_per_sec: Some(100.0),
reads_kb_per_sec: Some(1024.0),
reads_await: Some(5.5),
writes_per_sec: Some(50.0),
writes_kb_per_sec: Some(512.0),
writes_await: Some(10.2),
perc_util: Some(75.5),
let stats = vec![DriveRuntimeDetailedStats {
pool_index: Some("0".to_string()),
set_index: Some("1".to_string()),
drive_index: Some("2".to_string()),
disk_id: Some("disk-uuid-1".to_string()),
runtime_state: Some("online".to_string()),
healing: true,
scanning: false,
offline_duration_seconds: Some(0),
api_calls: vec![("read".to_string(), 7)],
api_latency_by_api_micros: vec![("read".to_string(), 2500)],
stats: DriveDetailedStats {
server: "node1:9000".to_string(),
drive: "/data/disk1".to_string(),
total_bytes: 1024 * 1024 * 1024 * 100, // 100 GB
used_bytes: 1024 * 1024 * 1024 * 50, // 50 GB
free_bytes: 1024 * 1024 * 1024 * 50, // 50 GB
capacity_observation_state: "live",
capacity_observation_age_seconds: 0,
used_inodes: Some(100000),
free_inodes: Some(900000),
total_inodes: Some(1000000),
timeout_errors_total: Some(5),
io_errors_total: Some(10),
availability_errors_total: Some(2),
waiting_io: Some(3),
api_latency_micros: Some(1500),
health: 1,
reads_per_sec: Some(100.0),
reads_kb_per_sec: Some(1024.0),
reads_await: Some(5.5),
writes_per_sec: Some(50.0),
writes_kb_per_sec: Some(512.0),
writes_await: Some(10.2),
perc_util: Some(75.5),
},
}];
let metrics = collect_drive_detailed_metrics(&stats);
let metrics = collect_drive_runtime_detailed_metrics(&stats);
report_metrics(&metrics);
assert_eq!(metrics.len(), 23);
assert_eq!(metrics.len(), 34);
// Verify total bytes metric
let total_bytes_name = DRIVE_TOTAL_BYTES_MD.get_full_metric_name();
@@ -303,6 +479,32 @@ mod tests {
);
assert_metric_label_keys(&metrics, &DRIVE_API_LATENCY_MD, 1500.0, &[SERVER_LABEL, DRIVE_LABEL]);
assert_metric_label_keys(&metrics, &DRIVE_CAPACITY_OBSERVATION_STATE_MD, 1.0, &[SERVER_LABEL, DRIVE_LABEL, "state"]);
assert_metric_label_keys(
&metrics,
&DRIVE_INFO_MD,
1.0,
&[
SERVER_LABEL,
DRIVE_LABEL,
POOL_INDEX_LABEL,
SET_INDEX_LABEL,
DRIVE_INDEX_LABEL,
DISK_ID_LABEL,
],
);
assert_metric_label_keys(
&metrics,
&DRIVE_API_CALLS_MD,
7.0,
&[
SERVER_LABEL,
DRIVE_LABEL,
POOL_INDEX_LABEL,
SET_INDEX_LABEL,
DRIVE_INDEX_LABEL,
API_LABEL,
],
);
}
#[test]
@@ -353,6 +555,75 @@ mod tests {
);
}
#[test]
fn drive_runtime_state_metrics_keep_suspect_state_active() {
let stats = vec![DriveRuntimeDetailedStats {
pool_index: Some("0".to_string()),
set_index: Some("1".to_string()),
drive_index: Some("2".to_string()),
runtime_state: Some("suspect".to_string()),
stats: DriveDetailedStats {
server: "node1:9000".to_string(),
drive: "/data/disk1".to_string(),
..Default::default()
},
..Default::default()
}];
let metrics = collect_drive_runtime_detailed_metrics(&stats);
let state_metrics = metrics
.iter()
.filter(|metric| metric.name == DRIVE_RUNTIME_STATE_MD.get_full_metric_name())
.collect::<Vec<_>>();
assert_eq!(state_metrics.len(), 5);
assert!(state_metrics.iter().any(|metric| {
metric.value == 1.0
&& metric
.labels
.iter()
.any(|(name, value)| *name == STATE_LABEL && value.as_ref() == "suspect")
}));
assert!(state_metrics.iter().filter(|metric| metric.value == 1.0).all(|metric| {
metric
.labels
.iter()
.any(|(name, value)| *name == STATE_LABEL && value.as_ref() == "suspect")
}));
}
#[test]
fn drive_offline_duration_zeroes_recovered_topology_drive() {
let stats = vec![DriveRuntimeDetailedStats {
pool_index: Some("0".to_string()),
set_index: Some("1".to_string()),
drive_index: Some("2".to_string()),
runtime_state: Some("online".to_string()),
offline_duration_seconds: None,
stats: DriveDetailedStats {
server: "node1:9000".to_string(),
drive: "/data/disk1".to_string(),
..Default::default()
},
..Default::default()
}];
let metrics = collect_drive_runtime_detailed_metrics(&stats);
assert!(metrics.iter().any(|metric| {
metric.name == DRIVE_OFFLINE_DURATION_SECONDS_MD.get_full_metric_name()
&& metric.value == 0.0
&& metric
.labels
.iter()
.any(|(name, value)| *name == SERVER_LABEL && value == "node1:9000")
&& metric
.labels
.iter()
.any(|(name, value)| *name == DRIVE_INDEX_LABEL && value == "2")
}));
}
#[test]
fn test_collect_drive_count_metrics() {
let stats = DriveCountStats {
crates/obs/src/metrics/scheduler.rs
+973 -62
View File
File diff suppressed because it is too large Load Diff
crates/obs/src/metrics/schema/audit.rs
+45 -5
View File
@@ -17,18 +17,31 @@
use crate::{MetricDescriptor, MetricName, new_gauge_md, subsystems};
use std::sync::LazyLock;
const TARGET_ID: &str = "target_id";
pub const TARGET_ID: &str = "target_id";
pub const SERVER: &str = "server";
pub const RESULT: &str = "result"; // success / failure
pub const STATUS: &str = "status"; // success / failure
pub const SUCCESS: &str = "success";
pub const FAILURE: &str = "failure";
const TARGET_LABELS: [&str; 1] = [TARGET_ID];
const TARGET_SERVER_LABELS: [&str; 2] = [SERVER, TARGET_ID];
pub static AUDIT_FAILED_MESSAGES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::AuditFailedMessages,
"Total number of messages that failed to send since start",
&[TARGET_ID],
&TARGET_LABELS,
subsystems::AUDIT,
)
});
pub static AUDIT_FAILED_MESSAGES_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("failed_messages_by_server".to_string()),
"Total number of messages that failed to send since start by server and target",
&TARGET_SERVER_LABELS,
subsystems::AUDIT,
)
});
@@ -37,7 +50,16 @@ pub static AUDIT_FAILED_STORE_LENGTH_MD: LazyLock<MetricDescriptor> = LazyLock::
new_gauge_md(
MetricName::AuditFailedStoreLength,
"Number of audit messages held in the failed-events store for target",
&[TARGET_ID],
&TARGET_LABELS,
subsystems::AUDIT,
)
});
pub static AUDIT_FAILED_STORE_LENGTH_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("failed_store_length_by_server".to_string()),
"Number of audit messages held in the failed-events store by server and target",
&TARGET_SERVER_LABELS,
subsystems::AUDIT,
)
});
@@ -46,7 +68,16 @@ pub static AUDIT_TARGET_QUEUE_LENGTH_MD: LazyLock<MetricDescriptor> = LazyLock::
new_gauge_md(
MetricName::AuditTargetQueueLength,
"Number of unsent messages in queue for target",
&[TARGET_ID],
&TARGET_LABELS,
subsystems::AUDIT,
)
});
pub static AUDIT_TARGET_QUEUE_LENGTH_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("target_queue_length_by_server".to_string()),
"Number of unsent audit messages in queue by server and target",
&TARGET_SERVER_LABELS,
subsystems::AUDIT,
)
});
@@ -55,7 +86,16 @@ pub static AUDIT_TOTAL_MESSAGES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|
new_gauge_md(
MetricName::AuditTotalMessages,
"Total number of messages sent since start",
&[TARGET_ID],
&TARGET_LABELS,
subsystems::AUDIT,
)
});
pub static AUDIT_TOTAL_MESSAGES_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("total_messages_by_server".to_string()),
"Total number of messages sent since start by server and target",
&TARGET_SERVER_LABELS,
subsystems::AUDIT,
)
});
crates/obs/src/metrics/schema/bucket_replication.rs
+93
View File
@@ -21,6 +21,8 @@ use std::sync::LazyLock;
pub const BUCKET_L: &str = "bucket";
/// Replication operation
pub const OPERATION_L: &str = "operation";
/// Replication proxy result
pub const RESULT_L: &str = "result";
/// Replication target ARN
pub const TARGET_ARN_L: &str = "target_arn";
/// Replication range
@@ -48,6 +50,16 @@ const MRF_DROPPED_COUNT: &str = "mrf_dropped_count";
const MRF_MISSED_COUNT: &str = "mrf_missed_count";
const MRF_FLUSH_FAILURES: &str = "mrf_flush_failures";
const MRF_LAST_FLUSH_DURATION_MILLIS: &str = "mrf_last_flush_duration_millis";
const TARGET_SENT_BYTES: &str = "target_sent_bytes";
const TARGET_SENT_COUNT: &str = "target_sent_count";
const TARGET_TOTAL_FAILED_BYTES: &str = "target_total_failed_bytes";
const TARGET_TOTAL_FAILED_COUNT: &str = "target_total_failed_count";
const TARGET_LAST_MIN_FAILED_BYTES: &str = "target_last_min_failed_bytes";
const TARGET_LAST_MIN_FAILED_COUNT: &str = "target_last_min_failed_count";
const TARGET_LAST_HOUR_FAILED_BYTES: &str = "target_last_hour_failed_bytes";
const TARGET_LAST_HOUR_FAILED_COUNT: &str = "target_last_hour_failed_count";
const BUCKET_TARGET_LABELS: [&str; 2] = [BUCKET_L, TARGET_ARN_L];
pub static BUCKET_REPL_LAST_HR_FAILED_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
@@ -58,6 +70,15 @@ pub static BUCKET_REPL_LAST_HR_FAILED_BYTES_MD: LazyLock<MetricDescriptor> = Laz
)
});
pub static BUCKET_REPL_TARGET_LAST_HOUR_FAILED_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::from(TARGET_LAST_HOUR_FAILED_BYTES),
"Total number of bytes failed at least once to replicate in the last hour on a bucket and target ARN",
&BUCKET_TARGET_LABELS,
subsystems::BUCKET_REPLICATION,
)
});
pub static BUCKET_REPL_LAST_HR_FAILED_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::LastHourFailedCount,
@@ -67,6 +88,15 @@ pub static BUCKET_REPL_LAST_HR_FAILED_COUNT_MD: LazyLock<MetricDescriptor> = Laz
)
});
pub static BUCKET_REPL_TARGET_LAST_HOUR_FAILED_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::from(TARGET_LAST_HOUR_FAILED_COUNT),
"Total number of objects which failed replication in the last hour on a bucket and target ARN",
&BUCKET_TARGET_LABELS,
subsystems::BUCKET_REPLICATION,
)
});
pub static BUCKET_REPL_LAST_MIN_FAILED_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::LastMinFailedBytes,
@@ -76,6 +106,15 @@ pub static BUCKET_REPL_LAST_MIN_FAILED_BYTES_MD: LazyLock<MetricDescriptor> = La
)
});
pub static BUCKET_REPL_TARGET_LAST_MIN_FAILED_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::from(TARGET_LAST_MIN_FAILED_BYTES),
"Total number of bytes failed at least once to replicate in the last full minute on a bucket and target ARN",
&BUCKET_TARGET_LABELS,
subsystems::BUCKET_REPLICATION,
)
});
pub static BUCKET_REPL_LAST_MIN_FAILED_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::LastMinFailedCount,
@@ -85,6 +124,15 @@ pub static BUCKET_REPL_LAST_MIN_FAILED_COUNT_MD: LazyLock<MetricDescriptor> = La
)
});
pub static BUCKET_REPL_TARGET_LAST_MIN_FAILED_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::from(TARGET_LAST_MIN_FAILED_COUNT),
"Total number of objects which failed replication in the last full minute on a bucket and target ARN",
&BUCKET_TARGET_LABELS,
subsystems::BUCKET_REPLICATION,
)
});
pub static BUCKET_REPL_LATENCY_MS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::LatencyMilliSec,
@@ -337,6 +385,15 @@ pub static BUCKET_REPL_SENT_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new
)
});
pub static BUCKET_REPL_TARGET_SENT_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_counter_md(
MetricName::from(TARGET_SENT_BYTES),
"Total number of bytes replicated to a bucket replication target ARN",
&BUCKET_TARGET_LABELS,
subsystems::BUCKET_REPLICATION,
)
});
pub static BUCKET_REPL_SENT_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_counter_md(
MetricName::SentCount,
@@ -346,6 +403,15 @@ pub static BUCKET_REPL_SENT_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new
)
});
pub static BUCKET_REPL_TARGET_SENT_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_counter_md(
MetricName::from(TARGET_SENT_COUNT),
"Total number of objects replicated to a bucket replication target ARN",
&BUCKET_TARGET_LABELS,
subsystems::BUCKET_REPLICATION,
)
});
pub static BUCKET_REPL_RESYNC_STARTED_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_counter_md(
MetricName::from(RESYNC_STARTED_TOTAL),
@@ -400,6 +466,15 @@ pub static BUCKET_REPL_TOTAL_FAILED_BYTES_MD: LazyLock<MetricDescriptor> = LazyL
)
});
pub static BUCKET_REPL_TARGET_TOTAL_FAILED_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_counter_md(
MetricName::from(TARGET_TOTAL_FAILED_BYTES),
"Total number of bytes failed at least once to replicate since server start by bucket and target ARN",
&BUCKET_TARGET_LABELS,
subsystems::BUCKET_REPLICATION,
)
});
pub static BUCKET_REPL_TOTAL_FAILED_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_counter_md(
MetricName::TotalFailedCount,
@@ -409,6 +484,15 @@ pub static BUCKET_REPL_TOTAL_FAILED_COUNT_MD: LazyLock<MetricDescriptor> = LazyL
)
});
pub static BUCKET_REPL_TARGET_TOTAL_FAILED_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_counter_md(
MetricName::from(TARGET_TOTAL_FAILED_COUNT),
"Total number of objects which failed replication since server start by bucket and target ARN",
&BUCKET_TARGET_LABELS,
subsystems::BUCKET_REPLICATION,
)
});
pub static BUCKET_REPL_BANDWIDTH_LIMIT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::BandwidthLimitBytesPerSecond,
@@ -435,3 +519,12 @@ pub static BUCKET_REPL_PROXIED_DELETE_TAGGING_REQUESTS_FAILURES_MD: LazyLock<Met
subsystems::BUCKET_REPLICATION,
)
});
pub static BUCKET_REPL_PROXY_REQUESTS_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_counter_md(
MetricName::Custom("proxy_requests_total".to_string()),
"Total number of bucket replication proxy requests by operation and result",
&[BUCKET_L, OPERATION_L, RESULT_L],
subsystems::BUCKET_REPLICATION,
)
});
crates/obs/src/metrics/schema/ilm.rs
+13
View File
@@ -17,6 +17,19 @@
use crate::{MetricDescriptor, MetricName, new_counter_md, new_gauge_md, subsystems};
use std::sync::LazyLock;
pub const SERVER_LABEL: &str = "server";
pub const ACTION_LABEL: &str = "action";
pub const STATE_LABEL: &str = "state";
pub static ILM_ACTION_TASKS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("action_tasks".to_string()),
"ILM task counts by server, action, and state",
&[SERVER_LABEL, ACTION_LABEL, STATE_LABEL],
subsystems::ILM,
)
});
pub static ILM_EXPIRY_PENDING_TASKS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::IlmExpiryPendingTasks,
crates/obs/src/metrics/schema/notification_target.rs
+38
View File
@@ -19,8 +19,10 @@ use std::sync::LazyLock;
pub const TARGET_ID: &str = "target_id";
pub const TARGET_TYPE: &str = "target_type";
pub const SERVER: &str = "server";
const NOTIFICATION_TARGET_LABELS: [&str; 2] = [TARGET_ID, TARGET_TYPE];
const NOTIFICATION_TARGET_SERVER_LABELS: [&str; 3] = [SERVER, TARGET_ID, TARGET_TYPE];
pub static NOTIFICATION_TARGET_FAILED_MESSAGES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
@@ -31,6 +33,15 @@ pub static NOTIFICATION_TARGET_FAILED_MESSAGES_MD: LazyLock<MetricDescriptor> =
)
});
pub static NOTIFICATION_TARGET_FAILED_MESSAGES_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("target_failed_messages_by_server".to_string()),
"Total number of notification messages that permanently failed to send by server and target",
&NOTIFICATION_TARGET_SERVER_LABELS,
subsystems::NOTIFICATION,
)
});
pub static NOTIFICATION_TARGET_FAILED_STORE_LENGTH_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::NotificationTargetFailedStoreLength,
@@ -40,6 +51,15 @@ pub static NOTIFICATION_TARGET_FAILED_STORE_LENGTH_MD: LazyLock<MetricDescriptor
)
});
pub static NOTIFICATION_TARGET_FAILED_STORE_LENGTH_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("target_failed_store_length_by_server".to_string()),
"Number of notification messages held in the failed-events store by server and target",
&NOTIFICATION_TARGET_SERVER_LABELS,
subsystems::NOTIFICATION,
)
});
pub static NOTIFICATION_TARGET_QUEUE_LENGTH_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::NotificationTargetQueueLength,
@@ -49,6 +69,15 @@ pub static NOTIFICATION_TARGET_QUEUE_LENGTH_MD: LazyLock<MetricDescriptor> = Laz
)
});
pub static NOTIFICATION_TARGET_QUEUE_LENGTH_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("target_queue_length_by_server".to_string()),
"Number of queued notification messages pending delivery by server and target",
&NOTIFICATION_TARGET_SERVER_LABELS,
subsystems::NOTIFICATION,
)
});
pub static NOTIFICATION_TARGET_TOTAL_MESSAGES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::NotificationTargetTotalMessages,
@@ -57,3 +86,12 @@ pub static NOTIFICATION_TARGET_TOTAL_MESSAGES_MD: LazyLock<MetricDescriptor> = L
subsystems::NOTIFICATION,
)
});
pub static NOTIFICATION_TARGET_TOTAL_MESSAGES_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("target_total_messages_by_server".to_string()),
"Total number of notification messages successfully delivered by server and target",
&NOTIFICATION_TARGET_SERVER_LABELS,
subsystems::NOTIFICATION,
)
});
crates/obs/src/metrics/schema/replication.rs
+119
View File
@@ -17,6 +17,8 @@
use crate::{MetricDescriptor, MetricName, new_gauge_md, subsystems};
use std::sync::LazyLock;
pub const SERVER_LABEL: &str = "server";
pub static REPLICATION_AVERAGE_ACTIVE_WORKERS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::ReplicationAverageActiveWorkers,
@@ -26,6 +28,15 @@ pub static REPLICATION_AVERAGE_ACTIVE_WORKERS_MD: LazyLock<MetricDescriptor> = L
)
});
pub static REPLICATION_AVERAGE_ACTIVE_WORKERS_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("average_active_workers_by_server".to_string()),
"Average number of active replication workers by server",
&[SERVER_LABEL],
subsystems::REPLICATION,
)
});
pub static REPLICATION_AVERAGE_QUEUED_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::ReplicationAverageQueuedBytes,
@@ -35,6 +46,15 @@ pub static REPLICATION_AVERAGE_QUEUED_BYTES_MD: LazyLock<MetricDescriptor> = Laz
)
});
pub static REPLICATION_AVERAGE_QUEUED_BYTES_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("average_queued_bytes_by_server".to_string()),
"Average number of bytes queued for replication since server start by server",
&[SERVER_LABEL],
subsystems::REPLICATION,
)
});
pub static REPLICATION_AVERAGE_QUEUED_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::ReplicationAverageQueuedCount,
@@ -44,6 +64,15 @@ pub static REPLICATION_AVERAGE_QUEUED_COUNT_MD: LazyLock<MetricDescriptor> = Laz
)
});
pub static REPLICATION_AVERAGE_QUEUED_COUNT_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("average_queued_count_by_server".to_string()),
"Average number of objects queued for replication since server start by server",
&[SERVER_LABEL],
subsystems::REPLICATION,
)
});
pub static REPLICATION_AVERAGE_DATA_TRANSFER_RATE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::ReplicationAverageDataTransferRate,
@@ -53,6 +82,15 @@ pub static REPLICATION_AVERAGE_DATA_TRANSFER_RATE_MD: LazyLock<MetricDescriptor>
)
});
pub static REPLICATION_AVERAGE_DATA_TRANSFER_RATE_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("average_data_transfer_rate_by_server".to_string()),
"Average replication data transfer rate in bytes/sec by server",
&[SERVER_LABEL],
subsystems::REPLICATION,
)
});
pub static REPLICATION_CURRENT_ACTIVE_WORKERS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::ReplicationCurrentActiveWorkers,
@@ -62,6 +100,15 @@ pub static REPLICATION_CURRENT_ACTIVE_WORKERS_MD: LazyLock<MetricDescriptor> = L
)
});
pub static REPLICATION_CURRENT_ACTIVE_WORKERS_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("current_active_workers_by_server".to_string()),
"Total number of active replication workers by server",
&[SERVER_LABEL],
subsystems::REPLICATION,
)
});
pub static REPLICATION_CURRENT_DATA_TRANSFER_RATE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::ReplicationCurrentDataTransferRate,
@@ -71,6 +118,15 @@ pub static REPLICATION_CURRENT_DATA_TRANSFER_RATE_MD: LazyLock<MetricDescriptor>
)
});
pub static REPLICATION_CURRENT_DATA_TRANSFER_RATE_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("current_data_transfer_rate_by_server".to_string()),
"Current replication data transfer rate in bytes/sec by server",
&[SERVER_LABEL],
subsystems::REPLICATION,
)
});
pub static REPLICATION_LAST_MINUTE_QUEUED_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::ReplicationLastMinuteQueuedBytes,
@@ -80,6 +136,15 @@ pub static REPLICATION_LAST_MINUTE_QUEUED_BYTES_MD: LazyLock<MetricDescriptor> =
)
});
pub static REPLICATION_LAST_MINUTE_QUEUED_BYTES_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("last_minute_queued_bytes_by_server".to_string()),
"Number of bytes queued for replication in the last full minute by server",
&[SERVER_LABEL],
subsystems::REPLICATION,
)
});
pub static REPLICATION_LAST_MINUTE_QUEUED_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::ReplicationLastMinuteQueuedCount,
@@ -89,6 +154,15 @@ pub static REPLICATION_LAST_MINUTE_QUEUED_COUNT_MD: LazyLock<MetricDescriptor> =
)
});
pub static REPLICATION_LAST_MINUTE_QUEUED_COUNT_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("last_minute_queued_count_by_server".to_string()),
"Number of objects queued for replication in the last full minute by server",
&[SERVER_LABEL],
subsystems::REPLICATION,
)
});
pub static REPLICATION_MAX_ACTIVE_WORKERS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::ReplicationMaxActiveWorkers,
@@ -98,6 +172,15 @@ pub static REPLICATION_MAX_ACTIVE_WORKERS_MD: LazyLock<MetricDescriptor> = LazyL
)
});
pub static REPLICATION_MAX_ACTIVE_WORKERS_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("max_active_workers_by_server".to_string()),
"Maximum number of active replication workers seen since server start by server",
&[SERVER_LABEL],
subsystems::REPLICATION,
)
});
pub static REPLICATION_MAX_QUEUED_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::ReplicationMaxQueuedBytes,
@@ -107,6 +190,15 @@ pub static REPLICATION_MAX_QUEUED_BYTES_MD: LazyLock<MetricDescriptor> = LazyLoc
)
});
pub static REPLICATION_MAX_QUEUED_BYTES_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("max_queued_bytes_by_server".to_string()),
"Maximum number of bytes queued for replication since server start by server",
&[SERVER_LABEL],
subsystems::REPLICATION,
)
});
pub static REPLICATION_MAX_QUEUED_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::ReplicationMaxQueuedCount,
@@ -116,6 +208,15 @@ pub static REPLICATION_MAX_QUEUED_COUNT_MD: LazyLock<MetricDescriptor> = LazyLoc
)
});
pub static REPLICATION_MAX_QUEUED_COUNT_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("max_queued_count_by_server".to_string()),
"Maximum number of objects queued for replication since server start by server",
&[SERVER_LABEL],
subsystems::REPLICATION,
)
});
pub static REPLICATION_MAX_DATA_TRANSFER_RATE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::ReplicationMaxDataTransferRate,
@@ -125,6 +226,15 @@ pub static REPLICATION_MAX_DATA_TRANSFER_RATE_MD: LazyLock<MetricDescriptor> = L
)
});
pub static REPLICATION_MAX_DATA_TRANSFER_RATE_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("max_data_transfer_rate_by_server".to_string()),
"Maximum replication data transfer rate in bytes/sec seen since server start by server",
&[SERVER_LABEL],
subsystems::REPLICATION,
)
});
pub static REPLICATION_RECENT_BACKLOG_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::ReplicationRecentBacklogCount,
@@ -133,3 +243,12 @@ pub static REPLICATION_RECENT_BACKLOG_COUNT_MD: LazyLock<MetricDescriptor> = Laz
subsystems::REPLICATION,
)
});
pub static REPLICATION_RECENT_BACKLOG_COUNT_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("recent_backlog_count_by_server".to_string()),
"Objects currently in replication backlog by server",
&[SERVER_LABEL],
subsystems::REPLICATION,
)
});
crates/obs/src/metrics/schema/request.rs
+99 -9
View File
@@ -22,6 +22,15 @@ pub const NAME_LABEL: &str = "name";
pub const TYPE_LABEL: &str = "type";
/// le label (for histogram buckets)
pub const LE_LABEL: &str = "le";
/// server label
pub const SERVER_LABEL: &str = "server";
const API_NAME_TYPE_LABELS: [&str; 2] = [NAME_LABEL, TYPE_LABEL];
const API_SERVER_NAME_TYPE_LABELS: [&str; 3] = [SERVER_LABEL, NAME_LABEL, TYPE_LABEL];
const API_NAME_TYPE_LE_LABELS: [&str; 3] = [NAME_LABEL, TYPE_LABEL, LE_LABEL];
const API_SERVER_NAME_TYPE_LE_LABELS: [&str; 4] = [SERVER_LABEL, NAME_LABEL, TYPE_LABEL, LE_LABEL];
const API_TYPE_LABELS: [&str; 1] = [TYPE_LABEL];
const API_SERVER_TYPE_LABELS: [&str; 2] = [SERVER_LABEL, TYPE_LABEL];
pub static API_REJECTED_AUTH_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_counter_md(
@@ -81,7 +90,16 @@ pub static API_REQUESTS_IN_FLIGHT_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLoc
new_gauge_md(
MetricName::ApiRequestsInFlightTotal,
"Total number of requests currently in flight",
&["name", "type"],
&API_NAME_TYPE_LABELS,
MetricSubsystem::ApiRequests,
)
});
pub static API_REQUESTS_IN_FLIGHT_TOTAL_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("requests_in_flight_total_by_server".to_string()),
"Total number of requests currently in flight by server",
&API_SERVER_NAME_TYPE_LABELS,
MetricSubsystem::ApiRequests,
)
});
@@ -90,7 +108,16 @@ pub static API_REQUESTS_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(||
new_counter_md(
MetricName::ApiRequestsTotal,
"Total number of requests",
&["name", "type"],
&API_NAME_TYPE_LABELS,
MetricSubsystem::ApiRequests,
)
});
pub static API_REQUESTS_TOTAL_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_counter_md(
MetricName::Custom("requests_total_by_server".to_string()),
"Total number of requests by server",
&API_SERVER_NAME_TYPE_LABELS,
MetricSubsystem::ApiRequests,
)
});
@@ -99,7 +126,16 @@ pub static API_REQUESTS_ERRORS_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::
new_counter_md(
MetricName::ApiRequestsErrorsTotal,
"Total number of requests with (4xx and 5xx) errors",
&["name", "type"],
&API_NAME_TYPE_LABELS,
MetricSubsystem::ApiRequests,
)
});
pub static API_REQUESTS_ERRORS_TOTAL_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_counter_md(
MetricName::Custom("requests_errors_total_by_server".to_string()),
"Total number of requests with (4xx and 5xx) errors by server",
&API_SERVER_NAME_TYPE_LABELS,
MetricSubsystem::ApiRequests,
)
});
@@ -108,7 +144,16 @@ pub static API_REQUESTS_5XX_ERRORS_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLo
new_counter_md(
MetricName::ApiRequests5xxErrorsTotal,
"Total number of requests with 5xx errors",
&["name", "type"],
&API_NAME_TYPE_LABELS,
MetricSubsystem::ApiRequests,
)
});
pub static API_REQUESTS_5XX_ERRORS_TOTAL_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_counter_md(
MetricName::Custom("requests_5xx_errors_total_by_server".to_string()),
"Total number of requests with 5xx errors by server",
&API_SERVER_NAME_TYPE_LABELS,
MetricSubsystem::ApiRequests,
)
});
@@ -117,7 +162,16 @@ pub static API_REQUESTS_4XX_ERRORS_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLo
new_counter_md(
MetricName::ApiRequests4xxErrorsTotal,
"Total number of requests with 4xx errors",
&["name", "type"],
&API_NAME_TYPE_LABELS,
MetricSubsystem::ApiRequests,
)
});
pub static API_REQUESTS_4XX_ERRORS_TOTAL_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_counter_md(
MetricName::Custom("requests_4xx_errors_total_by_server".to_string()),
"Total number of requests with 4xx errors by server",
&API_SERVER_NAME_TYPE_LABELS,
MetricSubsystem::ApiRequests,
)
});
@@ -126,7 +180,16 @@ pub static API_REQUESTS_CANCELED_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock
new_counter_md(
MetricName::ApiRequestsCanceledTotal,
"Total number of requests canceled by the client",
&["name", "type"],
&API_NAME_TYPE_LABELS,
MetricSubsystem::ApiRequests,
)
});
pub static API_REQUESTS_CANCELED_TOTAL_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_counter_md(
MetricName::Custom("requests_canceled_total_by_server".to_string()),
"Total number of requests canceled by the client by server",
&API_SERVER_NAME_TYPE_LABELS,
MetricSubsystem::ApiRequests,
)
});
@@ -135,7 +198,16 @@ pub static API_REQUESTS_TTFB_SECONDS_DISTRIBUTION_MD: LazyLock<MetricDescriptor>
new_counter_md(
MetricName::ApiRequestsTTFBSecondsDistribution,
"Distribution of time to first byte across API calls",
&["name", "type", "le"],
&API_NAME_TYPE_LE_LABELS,
MetricSubsystem::ApiRequests,
)
});
pub static API_REQUESTS_TTFB_SECONDS_DISTRIBUTION_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_counter_md(
MetricName::Custom("requests_ttfb_seconds_distribution_by_server".to_string()),
"Distribution of time to first byte across API calls by server",
&API_SERVER_NAME_TYPE_LE_LABELS,
MetricSubsystem::ApiRequests,
)
});
@@ -144,7 +216,16 @@ pub static API_TRAFFIC_SENT_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new
new_counter_md(
MetricName::ApiTrafficSentBytes,
"Total number of bytes sent",
&["type"],
&API_TYPE_LABELS,
MetricSubsystem::ApiRequests,
)
});
pub static API_TRAFFIC_SENT_BYTES_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_counter_md(
MetricName::Custom("traffic_sent_bytes_by_server".to_string()),
"Total number of bytes sent by server",
&API_SERVER_TYPE_LABELS,
MetricSubsystem::ApiRequests,
)
});
@@ -153,7 +234,16 @@ pub static API_TRAFFIC_RECV_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new
new_counter_md(
MetricName::ApiTrafficRecvBytes,
"Total number of bytes received",
&["type"],
&API_TYPE_LABELS,
MetricSubsystem::ApiRequests,
)
});
pub static API_TRAFFIC_RECV_BYTES_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_counter_md(
MetricName::Custom("traffic_recv_bytes_by_server".to_string()),
"Total number of bytes received by server",
&API_SERVER_TYPE_LABELS,
MetricSubsystem::ApiRequests,
)
});
crates/obs/src/metrics/schema/scanner.rs
+44
View File
@@ -17,6 +17,50 @@
use crate::{MetricDescriptor, MetricName, new_counter_md, new_gauge_md, subsystems};
use std::sync::LazyLock;
pub const SERVER_LABEL: &str = "server";
pub const SOURCE_LABEL: &str = "source";
pub const STATE_LABEL: &str = "state";
pub const CYCLE_SCOPE_LABEL: &str = "cycle_scope";
pub const BUCKET_LABEL: &str = "bucket";
pub const DRIVE_LABEL: &str = "drive";
pub const RESULT_LABEL: &str = "result";
pub static SCANNER_SOURCE_WORK_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_counter_md(
MetricName::Custom("source_work_total".to_string()),
"Total scanner work by source and state since server start",
&[SERVER_LABEL, SOURCE_LABEL, STATE_LABEL],
subsystems::SCANNER,
)
});
pub static SCANNER_CYCLE_SOURCE_WORK_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("cycle_source_work".to_string()),
"Scanner work by cycle scope, source, and state",
&[SERVER_LABEL, CYCLE_SCOPE_LABEL, SOURCE_LABEL, STATE_LABEL],
subsystems::SCANNER,
)
});
pub static SCANNER_BUCKET_DRIVE_RESULT_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_counter_md(
MetricName::Custom("bucket_drive_result_total".to_string()),
"Total scanner bucket-drive scan results by server, bucket, drive, and result",
&[SERVER_LABEL, BUCKET_LABEL, DRIVE_LABEL, RESULT_LABEL],
subsystems::SCANNER,
)
});
pub static SCANNER_CYCLE_BUCKET_DRIVE_RESULT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("cycle_bucket_drive_result".to_string()),
"Scanner bucket-drive scan results by cycle scope, server, bucket, drive, and result",
&[SERVER_LABEL, CYCLE_SCOPE_LABEL, BUCKET_LABEL, DRIVE_LABEL, RESULT_LABEL],
subsystems::SCANNER,
)
});
pub static SCANNER_BUCKET_SCANS_FINISHED_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_counter_md(
MetricName::ScannerBucketScansFinished,
crates/obs/src/metrics/schema/system_drive.rs
+102
View File
@@ -29,9 +29,111 @@ pub const SET_INDEX_LABEL: &str = "set_index";
pub const DRIVE_INDEX_LABEL: &str = "drive_index";
/// API label
pub const API_LABEL: &str = "api";
/// Disk id label
pub const DISK_ID_LABEL: &str = "disk_id";
/// State label
pub const STATE_LABEL: &str = "state";
/// All drive-related labels
pub const ALL_DRIVE_LABELS: [&str; 2] = [SERVER_LABEL, DRIVE_LABEL];
/// Drive labels with erasure-set topology.
pub const DRIVE_TOPOLOGY_LABELS: [&str; 5] = [
SERVER_LABEL,
DRIVE_LABEL,
POOL_INDEX_LABEL,
SET_INDEX_LABEL,
DRIVE_INDEX_LABEL,
];
/// Drive info labels.
pub const DRIVE_INFO_LABELS: [&str; 6] = [
SERVER_LABEL,
DRIVE_LABEL,
POOL_INDEX_LABEL,
SET_INDEX_LABEL,
DRIVE_INDEX_LABEL,
DISK_ID_LABEL,
];
/// Drive topology labels with a state dimension.
pub const DRIVE_TOPOLOGY_STATE_LABELS: [&str; 6] = [
SERVER_LABEL,
DRIVE_LABEL,
POOL_INDEX_LABEL,
SET_INDEX_LABEL,
DRIVE_INDEX_LABEL,
STATE_LABEL,
];
/// Drive topology labels with an API dimension.
pub const DRIVE_TOPOLOGY_API_LABELS: [&str; 6] = [
SERVER_LABEL,
DRIVE_LABEL,
POOL_INDEX_LABEL,
SET_INDEX_LABEL,
DRIVE_INDEX_LABEL,
API_LABEL,
];
pub static DRIVE_INFO_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("info".to_string()),
"Drive topology and stable disk identity information",
&DRIVE_INFO_LABELS,
subsystems::SYSTEM_DRIVE,
)
});
pub static DRIVE_RUNTIME_STATE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("runtime_state".to_string()),
"Drive runtime state (1 for the active state label, 0 otherwise)",
&DRIVE_TOPOLOGY_STATE_LABELS,
subsystems::SYSTEM_DRIVE,
)
});
pub static DRIVE_HEALING_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("healing".to_string()),
"Whether the drive is currently healing",
&DRIVE_TOPOLOGY_LABELS,
subsystems::SYSTEM_DRIVE,
)
});
pub static DRIVE_SCANNING_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("scanning".to_string()),
"Whether the drive is currently being scanned",
&DRIVE_TOPOLOGY_LABELS,
subsystems::SYSTEM_DRIVE,
)
});
pub static DRIVE_OFFLINE_DURATION_SECONDS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("offline_duration_seconds".to_string()),
"Duration in seconds the drive has been offline",
&DRIVE_TOPOLOGY_LABELS,
subsystems::SYSTEM_DRIVE,
)
});
pub static DRIVE_API_CALLS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_counter_md(
MetricName::Custom("api_calls_total".to_string()),
"Total drive API calls by operation",
&DRIVE_TOPOLOGY_API_LABELS,
subsystems::SYSTEM_DRIVE,
)
});
pub static DRIVE_API_LATENCY_BY_API_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("api_latency_by_api_micros".to_string()),
"Average last minute drive API latency in microseconds by operation",
&DRIVE_TOPOLOGY_API_LABELS,
subsystems::SYSTEM_DRIVE,
)
});
pub static DRIVE_USED_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
crates/obs/src/metrics/stats_collector.rs
+608 -166
View File
@@ -20,12 +20,15 @@
//! RustFS internal sources (storage layer, bucket monitor, system info)
//! and convert them to the Stats structs used by collectors.
use crate::metrics::collectors::scanner::{ScannerBucketDriveResultStats, ScannerSourceWorkStats};
use crate::metrics::collectors::{
BucketReplicationBacklogStats, BucketReplicationBandwidthStats, BucketReplicationStats, BucketReplicationTargetBacklogStats,
ApiRequestMetricSupport, ApiRequestStats, BucketReplicationBacklogStats, BucketReplicationBandwidthStats,
BucketReplicationRuntimeStats, BucketReplicationStats, BucketReplicationTargetBacklogStats, BucketReplicationTargetFlowStats,
BucketReplicationTargetStats, BucketStats, BucketUsageStats, ClusterConfigStats, ClusterHealthStats, ClusterStats,
ClusterUsageStats, CompressionClusterStats, CpuStats, DiskStats, DriveCountStats, DriveDetailedStats, ErasureSetStats,
HostNetworkStats, IamStats, IlmStats, MemoryStats, NetworkStats, ProcessStats, ProcessStatusType, ReplicationStats,
ResourceStats, ScannerStats,
ClusterUsageStats, CompressionClusterStats, CpuStats, DiskStats, DriveCountStats, DriveDetailedStats,
DriveRuntimeDetailedStats, ErasureSetStats, HostNetworkStats, IamStats, IlmActionTaskStats, IlmRuntimeStats, IlmStats,
MemoryStats, NetworkStats, ProcessStats, ProcessStatusType, ReplicationStats, ResourceStats, ScannerRuntimeStats,
ScannerStats,
};
use crate::metrics::runtime_sources::{ObsIlmRuntimeSnapshot, bucket_monitor_handle, iam_metrics_snapshot, ilm_runtime_snapshot};
use crate::metrics::{
@@ -37,11 +40,11 @@ use crate::metrics::{
use crate::node_identity::current_local_node_identity;
use chrono::Utc;
use rustfs_common::heal_channel::HealScanMode;
use rustfs_common::metrics::{ScannerMetricsReport, global_metrics};
use rustfs_common::metrics::{ScannerBucketDriveResultSnapshot, ScannerMetricsReport, ScannerSourceWorkSnapshot, global_metrics};
use rustfs_io_metrics::internode_metrics::global_internode_metrics;
use rustfs_io_metrics::{
ProcessResourceSnapshot, ProcessSampler, ProcessStatusSnapshot, ProcessSystemSnapshot, snapshot_process_resource_and_system,
snapshot_process_resource_and_system_with,
ProcessResourceSnapshot, ProcessSampler, ProcessStatusSnapshot, ProcessSystemSnapshot, s3_op_metrics_snapshot,
snapshot_process_resource_and_system, snapshot_process_resource_and_system_with,
};
use std::{
collections::{HashMap, HashSet},
@@ -193,7 +196,7 @@ async fn obs_ilm_runtime_snapshot() -> ObsIlmRuntimeSnapshot {
ilm_runtime_snapshot().await
}
async fn obs_bucket_replication_stats_bundle() -> (Vec<BucketReplicationStats>, Vec<BucketReplicationBacklogStats>) {
async fn obs_bucket_replication_stats_bundle() -> (Vec<BucketReplicationRuntimeStats>, Vec<BucketReplicationBacklogStats>) {
let snapshots = obs_bucket_replication_stats_snapshot().await;
let mut detail_stats = Vec::with_capacity(snapshots.len());
let mut backlog_stats = Vec::with_capacity(snapshots.len());
@@ -230,44 +233,65 @@ async fn obs_bucket_replication_stats_bundle() -> (Vec<BucketReplicationStats>,
(detail_stats, backlog_stats)
}
fn bucket_replication_detail_from_snapshot(stats: ObsBucketReplicationStatsSnapshot) -> BucketReplicationStats {
BucketReplicationStats {
bucket: stats.bucket,
total_failed_bytes: stats.total_failed_bytes,
total_failed_count: stats.total_failed_count,
last_min_failed_bytes: stats.last_min_failed_bytes,
last_min_failed_count: stats.last_min_failed_count,
last_hour_failed_bytes: stats.last_hour_failed_bytes,
last_hour_failed_count: stats.last_hour_failed_count,
sent_bytes: stats.sent_bytes,
sent_count: stats.sent_count,
proxied_get_requests_total: stats.proxied_get_requests_total,
proxied_get_requests_failures: stats.proxied_get_requests_failures,
proxied_head_requests_total: stats.proxied_head_requests_total,
proxied_head_requests_failures: stats.proxied_head_requests_failures,
proxied_put_requests_total: stats.proxied_put_requests_total,
proxied_put_requests_failures: stats.proxied_put_requests_failures,
proxied_put_tagging_requests_total: stats.proxied_put_tagging_requests_total,
proxied_put_tagging_requests_failures: stats.proxied_put_tagging_requests_failures,
proxied_get_tagging_requests_total: stats.proxied_get_tagging_requests_total,
proxied_get_tagging_requests_failures: stats.proxied_get_tagging_requests_failures,
proxied_delete_tagging_requests_total: stats.proxied_delete_tagging_requests_total,
proxied_delete_tagging_requests_failures: stats.proxied_delete_tagging_requests_failures,
resync_started_count: stats.resync_started_count,
resync_completed_count: stats.resync_completed_count,
resync_failed_count: stats.resync_failed_count,
resync_canceled_count: stats.resync_canceled_count,
resync_duration_ms: stats.resync_duration_ms,
targets: stats
.targets
.into_iter()
.map(|target| BucketReplicationTargetStats {
target_arn: target.target_arn,
bandwidth_limit_bytes_per_sec: target.bandwidth_limit_bytes_per_sec,
current_bandwidth_bytes_per_sec: target.current_bandwidth_bytes_per_sec,
latency_ms: target.latency_ms,
})
.collect(),
fn bucket_replication_detail_from_snapshot(stats: ObsBucketReplicationStatsSnapshot) -> BucketReplicationRuntimeStats {
let bucket = stats.bucket;
let (targets, target_flows): (Vec<_>, Vec<_>) = stats
.targets
.into_iter()
.map(|target| {
(
BucketReplicationTargetStats {
target_arn: target.target_arn.clone(),
bandwidth_limit_bytes_per_sec: target.bandwidth_limit_bytes_per_sec,
current_bandwidth_bytes_per_sec: target.current_bandwidth_bytes_per_sec,
latency_ms: target.latency_ms,
},
BucketReplicationTargetFlowStats {
target_arn: target.target_arn,
sent_bytes: target.sent_bytes,
sent_count: target.sent_count,
total_failed_bytes: target.total_failed_bytes,
total_failed_count: target.total_failed_count,
last_min_failed_bytes: target.last_min_failed_bytes,
last_min_failed_count: target.last_min_failed_count,
last_hour_failed_bytes: target.last_hour_failed_bytes,
last_hour_failed_count: target.last_hour_failed_count,
},
)
})
.unzip();
BucketReplicationRuntimeStats {
target_flows,
stats: BucketReplicationStats {
bucket,
total_failed_bytes: stats.total_failed_bytes,
total_failed_count: stats.total_failed_count,
last_min_failed_bytes: stats.last_min_failed_bytes,
last_min_failed_count: stats.last_min_failed_count,
last_hour_failed_bytes: stats.last_hour_failed_bytes,
last_hour_failed_count: stats.last_hour_failed_count,
sent_bytes: stats.sent_bytes,
sent_count: stats.sent_count,
proxied_get_requests_total: stats.proxied_get_requests_total,
proxied_get_requests_failures: stats.proxied_get_requests_failures,
proxied_head_requests_total: stats.proxied_head_requests_total,
proxied_head_requests_failures: stats.proxied_head_requests_failures,
proxied_put_requests_total: stats.proxied_put_requests_total,
proxied_put_requests_failures: stats.proxied_put_requests_failures,
proxied_put_tagging_requests_total: stats.proxied_put_tagging_requests_total,
proxied_put_tagging_requests_failures: stats.proxied_put_tagging_requests_failures,
proxied_get_tagging_requests_total: stats.proxied_get_tagging_requests_total,
proxied_get_tagging_requests_failures: stats.proxied_get_tagging_requests_failures,
proxied_delete_tagging_requests_total: stats.proxied_delete_tagging_requests_total,
proxied_delete_tagging_requests_failures: stats.proxied_delete_tagging_requests_failures,
resync_started_count: stats.resync_started_count,
resync_completed_count: stats.resync_completed_count,
resync_failed_count: stats.resync_failed_count,
resync_canceled_count: stats.resync_canceled_count,
resync_duration_ms: stats.resync_duration_ms,
targets,
},
}
}
@@ -364,6 +388,63 @@ fn disk_capacity_observation_state(source: Option<&str>, age_seconds: Option<u64
}
}
fn disk_topology_label(index: i32) -> Option<String> {
if index >= 0 { Some(index.to_string()) } else { None }
}
fn non_empty_disk_id(uuid: &str) -> Option<String> {
let uuid = uuid.trim();
if uuid.is_empty() { None } else { Some(uuid.to_string()) }
}
fn drive_inode_stats(used_inodes: u64, free_inodes: u64) -> (Option<u64>, Option<u64>, Option<u64>) {
let total_inodes = used_inodes.saturating_add(free_inodes);
if total_inodes == 0 {
(None, None, None)
} else {
(Some(used_inodes), Some(free_inodes), Some(total_inodes))
}
}
fn drive_api_latency_micros(actions: impl Iterator<Item = (u64, u64)>) -> Option<u64> {
let mut count = 0u64;
let mut acc_time_ns = 0u64;
let mut saw_action = false;
for (action_count, action_acc_time_ns) in actions {
saw_action = true;
if action_count > 0 {
count = count.saturating_add(action_count);
acc_time_ns = acc_time_ns.saturating_add(action_acc_time_ns);
}
}
saw_action.then(|| acc_time_ns.checked_div(count).unwrap_or_default() / 1_000)
}
fn drive_api_latency_by_api_micros<'a>(actions: impl Iterator<Item = (&'a String, u64, u64)>) -> Vec<(String, u64)> {
let mut values = actions
.map(|(api, count, acc_time)| (api.clone(), acc_time.checked_div(count).unwrap_or_default() / 1_000))
.collect::<Vec<_>>();
values.sort_by(|left, right| left.0.cmp(&right.0));
values
}
fn drive_api_calls<'a>(api_calls: impl Iterator<Item = (&'a String, &'a u64)>) -> Vec<(String, u64)> {
let mut values = api_calls.map(|(api, calls)| (api.clone(), *calls)).collect::<Vec<_>>();
values.sort_by(|left, right| left.0.cmp(&right.0));
values
}
fn drive_server_label(endpoint: &str, local_server: &str) -> String {
endpoint
.strip_prefix("http://")
.or_else(|| endpoint.strip_prefix("https://"))
.and_then(|rest| rest.split('/').next())
.filter(|authority| !authority.is_empty())
.unwrap_or(local_server)
.to_string()
}
fn derive_erasure_set_quorum_shape(set_drive_count: usize, parity: usize) -> ErasureSetQuorumShape {
let data_shards = set_drive_count.saturating_sub(parity);
let read_quorum = data_shards.max(1);
@@ -563,12 +644,12 @@ pub async fn collect_bucket_replication_detail_stats() -> Vec<BucketReplicationS
obs_bucket_replication_stats_snapshot()
.await
.into_iter()
.map(bucket_replication_detail_from_snapshot)
.map(|snapshot| bucket_replication_detail_from_snapshot(snapshot).stats)
.collect()
}
pub(crate) async fn collect_bucket_replication_stats_bundle() -> (Vec<BucketReplicationStats>, Vec<BucketReplicationBacklogStats>)
{
pub(crate) async fn collect_bucket_replication_stats_bundle()
-> (Vec<BucketReplicationRuntimeStats>, Vec<BucketReplicationBacklogStats>) {
obs_bucket_replication_stats_bundle().await
}
@@ -577,6 +658,22 @@ pub async fn collect_replication_stats() -> ReplicationStats {
obs_site_replication_stats().await
}
/// Collect S3 API request totals from the in-process operation recorder.
pub(crate) fn collect_api_request_stats() -> Vec<ApiRequestStats> {
let server = current_local_node_identity();
s3_op_metrics_snapshot()
.into_iter()
.map(|snapshot| ApiRequestStats {
server: server.clone(),
name: snapshot.op.to_string(),
req_type: "s3".to_string(),
total: snapshot.total,
supported_metrics: ApiRequestMetricSupport::TOTALS_ONLY,
..Default::default()
})
.collect()
}
/// Collect disk statistics from the storage layer.
pub async fn collect_disk_stats() -> Vec<DiskStats> {
let (disk_stats, _, _) = collect_disk_and_system_drive_stats().await;
@@ -646,16 +743,23 @@ pub fn collect_system_memory_stats() -> MemoryStats {
/// Collect node disk stats and drive stats from a single storage snapshot.
pub async fn collect_disk_and_system_drive_stats() -> (Vec<DiskStats>, Vec<DriveDetailedStats>, DriveCountStats) {
let (disk_stats, drive_stats, drive_count_stats) = collect_disk_and_system_drive_runtime_stats().await;
(disk_stats, drive_stats.into_iter().map(|stat| stat.stats).collect(), drive_count_stats)
}
pub(crate) async fn collect_disk_and_system_drive_runtime_stats()
-> (Vec<DiskStats>, Vec<DriveRuntimeDetailedStats>, DriveCountStats) {
let Some(store) = resolve_obs_object_store_handle() else {
return (Vec::new(), Vec::new(), DriveCountStats::default());
};
let storage_info = StorageAdminApi::storage_info(store.as_ref()).await;
let local_server = current_local_node_identity();
let disk_stats = storage_info
.disks
.iter()
.map(|disk| DiskStats {
server: disk.endpoint.clone(),
server: drive_server_label(&disk.endpoint, &local_server),
drive: disk.drive_path.clone(),
total_bytes: disk.total_space,
used_bytes: disk.used_space,
@@ -679,31 +783,61 @@ pub async fn collect_disk_and_system_drive_stats() -> (Vec<DiskStats>, Vec<Drive
} else {
offline_count += 1;
}
let (used_inodes, free_inodes, total_inodes) = drive_inode_stats(disk.used_inodes, disk.free_inodes);
DriveDetailedStats {
server: disk.endpoint.clone(),
drive: disk.drive_path.clone(),
total_bytes: disk.total_space,
used_bytes: disk.used_space,
free_bytes: disk.available_space,
capacity_observation_state,
capacity_observation_age_seconds,
used_inodes: None,
free_inodes: None,
total_inodes: None,
timeout_errors_total: None,
io_errors_total: None,
availability_errors_total: None,
waiting_io: None,
api_latency_micros: None,
health: if is_online { 1 } else { 0 },
reads_per_sec: None,
reads_kb_per_sec: None,
reads_await: None,
writes_per_sec: None,
writes_kb_per_sec: None,
writes_await: None,
perc_util: None,
DriveRuntimeDetailedStats {
pool_index: disk_topology_label(disk.pool_index),
set_index: disk_topology_label(disk.set_index),
drive_index: disk_topology_label(disk.disk_index),
disk_id: non_empty_disk_id(&disk.uuid),
runtime_state: Some(disk.runtime_state.as_deref().unwrap_or("unknown").to_ascii_lowercase()),
healing: disk.healing,
scanning: disk.scanning,
offline_duration_seconds: disk.offline_duration_seconds,
api_calls: disk
.metrics
.as_ref()
.map(|metrics| drive_api_calls(metrics.api_calls.iter()))
.unwrap_or_default(),
api_latency_by_api_micros: disk
.metrics
.as_ref()
.map(|metrics| {
drive_api_latency_by_api_micros(
metrics
.last_minute
.iter()
.map(|(api, action)| (api, action.count, action.acc_time)),
)
})
.unwrap_or_default(),
stats: DriveDetailedStats {
server: drive_server_label(&disk.endpoint, &local_server),
drive: disk.drive_path.clone(),
total_bytes: disk.total_space,
used_bytes: disk.used_space,
free_bytes: disk.available_space,
capacity_observation_state,
capacity_observation_age_seconds,
used_inodes,
free_inodes,
total_inodes,
timeout_errors_total: disk.metrics.as_ref().map(|metrics| metrics.total_errors_timeout),
io_errors_total: None,
availability_errors_total: disk.metrics.as_ref().map(|metrics| metrics.total_errors_availability),
waiting_io: disk.metrics.as_ref().map(|metrics| u64::from(metrics.total_waiting)),
api_latency_micros: disk.metrics.as_ref().and_then(|metrics| {
drive_api_latency_micros(metrics.last_minute.values().map(|action| (action.count, action.acc_time)))
}),
health: if is_online { 1 } else { 0 },
reads_per_sec: None,
reads_kb_per_sec: None,
reads_await: None,
writes_per_sec: None,
writes_kb_per_sec: None,
writes_await: None,
perc_util: None,
},
}
})
.collect();
@@ -1090,22 +1224,75 @@ async fn collect_cluster_usage_metric_stats_from_data_usage(
))
}
fn ilm_action_task_stats(ilm: &ObsIlmRuntimeSnapshot) -> Vec<IlmActionTaskStats> {
vec![
IlmActionTaskStats {
action: "expiry".to_string(),
state: "pending".to_string(),
value: ilm.expiry_pending_tasks,
},
IlmActionTaskStats {
action: "transition".to_string(),
state: "active".to_string(),
value: ilm.transition_active_tasks,
},
IlmActionTaskStats {
action: "transition".to_string(),
state: "pending".to_string(),
value: ilm.transition_pending_tasks,
},
IlmActionTaskStats {
action: "transition".to_string(),
state: "missed_immediate".to_string(),
value: ilm.transition_missed_immediate_tasks,
},
IlmActionTaskStats {
action: "transition".to_string(),
state: "queue_full".to_string(),
value: ilm.transition_queue_full_tasks,
},
IlmActionTaskStats {
action: "transition".to_string(),
state: "queue_send_timeout".to_string(),
value: ilm.transition_queue_send_timeout_tasks,
},
IlmActionTaskStats {
action: "transition".to_string(),
state: "compensation_scheduled".to_string(),
value: ilm.transition_compensation_scheduled_tasks,
},
IlmActionTaskStats {
action: "transition".to_string(),
state: "compensation_running".to_string(),
value: ilm.transition_compensation_running_tasks,
},
]
}
/// Collect ILM metrics from the current lifecycle runtime state.
pub async fn collect_ilm_metric_stats() -> Option<IlmStats> {
collect_ilm_runtime_metric_stats().await.map(|stats| stats.stats)
}
pub(crate) async fn collect_ilm_runtime_metric_stats() -> Option<IlmRuntimeStats> {
let ilm = obs_ilm_runtime_snapshot().await;
let metrics = global_metrics().report().await;
let versions_scanned = scanner_lifecycle_checked_versions(&metrics);
Some(IlmStats {
expiry_pending_tasks: ilm.expiry_pending_tasks,
transition_active_tasks: ilm.transition_active_tasks,
transition_pending_tasks: ilm.transition_pending_tasks,
transition_missed_immediate_tasks: ilm.transition_missed_immediate_tasks,
transition_queue_full_tasks: ilm.transition_queue_full_tasks,
transition_queue_send_timeout_tasks: ilm.transition_queue_send_timeout_tasks,
transition_compensation_scheduled_tasks: ilm.transition_compensation_scheduled_tasks,
transition_compensation_running_tasks: ilm.transition_compensation_running_tasks,
versions_scanned,
Some(IlmRuntimeStats {
server: current_local_node_identity(),
action_tasks: ilm_action_task_stats(&ilm),
stats: IlmStats {
expiry_pending_tasks: ilm.expiry_pending_tasks,
transition_active_tasks: ilm.transition_active_tasks,
transition_pending_tasks: ilm.transition_pending_tasks,
transition_missed_immediate_tasks: ilm.transition_missed_immediate_tasks,
transition_queue_full_tasks: ilm.transition_queue_full_tasks,
transition_queue_send_timeout_tasks: ilm.transition_queue_send_timeout_tasks,
transition_compensation_scheduled_tasks: ilm.transition_compensation_scheduled_tasks,
transition_compensation_running_tasks: ilm.transition_compensation_running_tasks,
versions_scanned,
},
})
}
@@ -1120,8 +1307,76 @@ fn scanner_bucket_scans_started(life_time_ops: &HashMap<String, u64>, bucket_sca
.unwrap_or(bucket_scans_finished)
}
fn scanner_source_work_stats(source_work: &[ScannerSourceWorkSnapshot]) -> Vec<ScannerSourceWorkStats> {
let mut stats = source_work
.iter()
.filter(|work| !work.source.is_empty())
.map(|work| ScannerSourceWorkStats {
source: work.source.clone(),
checked: work.checked,
queued: work.queued,
executed: work.executed,
failed: work.failed,
skipped: work.skipped,
missed: work.missed,
})
.collect::<Vec<_>>();
stats.sort_by(|left, right| left.source.cmp(&right.source));
stats
}
fn scanner_current_cycle_source_work_stats(metrics: &ScannerMetricsReport) -> Vec<ScannerSourceWorkStats> {
let current = scanner_source_work_stats(&metrics.current_cycle_source_work);
if !current.is_empty() {
return current;
}
let mut sources = scanner_source_work_stats(&metrics.last_cycle_source_work)
.into_iter()
.map(|work| work.source)
.chain(
scanner_source_work_stats(&metrics.source_work)
.into_iter()
.map(|work| work.source),
)
.collect::<Vec<_>>();
sources.sort();
sources.dedup();
sources
.into_iter()
.map(|source| ScannerSourceWorkStats {
source,
..Default::default()
})
.collect()
}
fn scanner_bucket_drive_result_stats(results: &[ScannerBucketDriveResultSnapshot]) -> Vec<ScannerBucketDriveResultStats> {
let mut stats = results
.iter()
.filter(|result| !result.bucket.is_empty() && !result.drive.is_empty() && !result.result.is_empty() && result.count > 0)
.map(|result| ScannerBucketDriveResultStats {
bucket: result.bucket.clone(),
drive: result.drive.clone(),
result: result.result.clone(),
count: result.count,
})
.collect::<Vec<_>>();
stats.sort_by(|left, right| {
left.bucket
.cmp(&right.bucket)
.then_with(|| left.drive.cmp(&right.drive))
.then_with(|| left.result.cmp(&right.result))
});
stats
}
pub async fn collect_scanner_metric_stats() -> Option<ScannerStats> {
let metrics = global_metrics().report().await;
collect_scanner_runtime_metric_stats().await.map(|stats| stats.stats)
}
pub(crate) async fn collect_scanner_runtime_metric_stats() -> Option<ScannerRuntimeStats> {
let (metrics, runtime_details) = global_metrics().report_with_runtime_details().await;
let now = Utc::now();
let bucket_scans_finished = metrics.life_time_ops.get("scan_bucket_drive").copied().unwrap_or_default();
let bucket_scans_started = scanner_bucket_scans_started(&metrics.life_time_ops, bucket_scans_finished);
@@ -1147,88 +1402,102 @@ pub async fn collect_scanner_metric_stats() -> Option<ScannerStats> {
let current_cycle_age = current_cycle_age_seconds as f64;
let last_cycle_duration = metrics.last_cycle_duration_seconds;
Some(ScannerStats {
bucket_scans_finished,
bucket_scans_started,
bucket_scans_failed,
directories_scanned,
objects_scanned,
versions_scanned,
last_activity_seconds,
active_paths,
oldest_active_path_age_seconds: metrics.oldest_active_path_age_seconds,
current_set_scan_concurrency_limit: metrics.current_set_scan_concurrency_limit,
current_set_scans_queued: metrics.current_set_scans_queued,
current_set_scans_active: metrics.current_set_scans_active,
current_disk_scan_concurrency_limit: metrics.current_disk_scan_concurrency_limit,
current_disk_bucket_scans_queued: metrics.current_disk_bucket_scans_queued,
current_disk_bucket_scans_active: metrics.current_disk_bucket_scans_active,
throttle_idle_mode_enabled: metrics.throttle_idle_mode_enabled,
throttle_sleep_factor: metrics.throttle_sleep_factor,
throttle_max_sleep_seconds: metrics.throttle_max_sleep_seconds,
yield_every_n_objects: metrics.yield_every_n_objects,
cycle_interval_seconds: metrics.cycle_interval_seconds,
cycle_max_duration_seconds: metrics.cycle_max_duration_seconds,
cycle_max_objects: metrics.cycle_max_objects,
cycle_max_directories: metrics.cycle_max_directories,
bitrot_cycle_enabled: metrics.bitrot_cycle_enabled,
bitrot_cycle_seconds: metrics.bitrot_cycle_seconds,
current_cycle: metrics.current_cycle,
completed_cycles,
current_cycle_age_seconds,
current_cycle_objects_scanned: metrics.current_cycle_objects_scanned,
current_cycle_directories_scanned: metrics.current_cycle_directories_scanned,
current_cycle_bucket_drive_scans: metrics.current_cycle_bucket_drive_scans,
current_cycle_bucket_drive_failures: metrics.current_cycle_bucket_drive_failures,
current_cycle_objects_per_second: scanner_work_rate_per_second(metrics.current_cycle_objects_scanned, current_cycle_age),
current_cycle_directories_per_second: scanner_work_rate_per_second(
metrics.current_cycle_directories_scanned,
current_cycle_age,
Some(ScannerRuntimeStats {
server: current_local_node_identity(),
source_work: scanner_source_work_stats(&metrics.source_work),
current_cycle_source_work: scanner_current_cycle_source_work_stats(&metrics),
last_cycle_source_work: scanner_source_work_stats(&metrics.last_cycle_source_work),
bucket_drive_results: scanner_bucket_drive_result_stats(&runtime_details.bucket_drive_results),
current_cycle_bucket_drive_results: scanner_bucket_drive_result_stats(
&runtime_details.current_cycle_bucket_drive_results,
),
current_cycle_bucket_drive_scans_per_second: scanner_work_rate_per_second(
metrics.current_cycle_bucket_drive_scans,
current_cycle_age,
),
current_cycle_yield_events: metrics.current_cycle_yield_events,
current_cycle_yield_duration_seconds: metrics.current_cycle_yield_duration_seconds,
current_cycle_throttle_sleep_events: metrics.current_cycle_throttle_sleep_events,
current_cycle_throttle_sleep_duration_seconds: metrics.current_cycle_throttle_sleep_duration_seconds,
current_cycle_ilm_actions: metrics.current_cycle_ilm_actions,
current_cycle_heal_objects: metrics.current_cycle_heal_objects,
current_cycle_replication_checks: metrics.current_cycle_replication_checks,
current_cycle_usage_saves: metrics.current_cycle_usage_saves,
current_scan_mode,
last_cycle_result: metrics.last_cycle_result_code,
last_cycle_partial_reason: metrics.last_cycle_partial_reason_code,
last_cycle_duration_seconds: metrics.last_cycle_duration_seconds,
last_cycle_objects_scanned: metrics.last_cycle_objects_scanned,
last_cycle_directories_scanned: metrics.last_cycle_directories_scanned,
last_cycle_bucket_drive_scans: metrics.last_cycle_bucket_drive_scans,
last_cycle_bucket_drive_failures: metrics.last_cycle_bucket_drive_failures,
last_cycle_objects_per_second: scanner_work_rate_per_second(metrics.last_cycle_objects_scanned, last_cycle_duration),
last_cycle_directories_per_second: scanner_work_rate_per_second(
metrics.last_cycle_directories_scanned,
last_cycle_duration,
),
last_cycle_bucket_drive_scans_per_second: scanner_work_rate_per_second(
metrics.last_cycle_bucket_drive_scans,
last_cycle_duration,
),
last_cycle_yield_events: metrics.last_cycle_yield_events,
last_cycle_yield_duration_seconds: metrics.last_cycle_yield_duration_seconds,
last_cycle_throttle_sleep_events: metrics.last_cycle_throttle_sleep_events,
last_cycle_throttle_sleep_duration_seconds: metrics.last_cycle_throttle_sleep_duration_seconds,
last_cycle_ilm_actions: metrics.last_cycle_ilm_actions,
last_cycle_heal_objects: metrics.last_cycle_heal_objects,
last_cycle_replication_checks: metrics.last_cycle_replication_checks,
last_cycle_usage_saves: metrics.last_cycle_usage_saves,
failed_cycles: metrics.failed_cycles,
superseded_cycles: metrics.superseded_cycles,
partial_cycles: metrics.partial_cycles,
partial_cycles_unknown: metrics.partial_cycles_unknown,
partial_cycles_runtime: metrics.partial_cycles_runtime,
partial_cycles_objects: metrics.partial_cycles_objects,
partial_cycles_directories: metrics.partial_cycles_directories,
last_cycle_bucket_drive_results: scanner_bucket_drive_result_stats(&runtime_details.last_cycle_bucket_drive_results),
stats: ScannerStats {
bucket_scans_finished,
bucket_scans_started,
bucket_scans_failed,
directories_scanned,
objects_scanned,
versions_scanned,
last_activity_seconds,
active_paths,
oldest_active_path_age_seconds: metrics.oldest_active_path_age_seconds,
current_set_scan_concurrency_limit: metrics.current_set_scan_concurrency_limit,
current_set_scans_queued: metrics.current_set_scans_queued,
current_set_scans_active: metrics.current_set_scans_active,
current_disk_scan_concurrency_limit: metrics.current_disk_scan_concurrency_limit,
current_disk_bucket_scans_queued: metrics.current_disk_bucket_scans_queued,
current_disk_bucket_scans_active: metrics.current_disk_bucket_scans_active,
throttle_idle_mode_enabled: metrics.throttle_idle_mode_enabled,
throttle_sleep_factor: metrics.throttle_sleep_factor,
throttle_max_sleep_seconds: metrics.throttle_max_sleep_seconds,
yield_every_n_objects: metrics.yield_every_n_objects,
cycle_interval_seconds: metrics.cycle_interval_seconds,
cycle_max_duration_seconds: metrics.cycle_max_duration_seconds,
cycle_max_objects: metrics.cycle_max_objects,
cycle_max_directories: metrics.cycle_max_directories,
bitrot_cycle_enabled: metrics.bitrot_cycle_enabled,
bitrot_cycle_seconds: metrics.bitrot_cycle_seconds,
current_cycle: metrics.current_cycle,
completed_cycles,
current_cycle_age_seconds,
current_cycle_objects_scanned: metrics.current_cycle_objects_scanned,
current_cycle_directories_scanned: metrics.current_cycle_directories_scanned,
current_cycle_bucket_drive_scans: metrics.current_cycle_bucket_drive_scans,
current_cycle_bucket_drive_failures: metrics.current_cycle_bucket_drive_failures,
current_cycle_objects_per_second: scanner_work_rate_per_second(
metrics.current_cycle_objects_scanned,
current_cycle_age,
),
current_cycle_directories_per_second: scanner_work_rate_per_second(
metrics.current_cycle_directories_scanned,
current_cycle_age,
),
current_cycle_bucket_drive_scans_per_second: scanner_work_rate_per_second(
metrics.current_cycle_bucket_drive_scans,
current_cycle_age,
),
current_cycle_yield_events: metrics.current_cycle_yield_events,
current_cycle_yield_duration_seconds: metrics.current_cycle_yield_duration_seconds,
current_cycle_throttle_sleep_events: metrics.current_cycle_throttle_sleep_events,
current_cycle_throttle_sleep_duration_seconds: metrics.current_cycle_throttle_sleep_duration_seconds,
current_cycle_ilm_actions: metrics.current_cycle_ilm_actions,
current_cycle_heal_objects: metrics.current_cycle_heal_objects,
current_cycle_replication_checks: metrics.current_cycle_replication_checks,
current_cycle_usage_saves: metrics.current_cycle_usage_saves,
current_scan_mode,
last_cycle_result: metrics.last_cycle_result_code,
last_cycle_partial_reason: metrics.last_cycle_partial_reason_code,
last_cycle_duration_seconds: metrics.last_cycle_duration_seconds,
last_cycle_objects_scanned: metrics.last_cycle_objects_scanned,
last_cycle_directories_scanned: metrics.last_cycle_directories_scanned,
last_cycle_bucket_drive_scans: metrics.last_cycle_bucket_drive_scans,
last_cycle_bucket_drive_failures: metrics.last_cycle_bucket_drive_failures,
last_cycle_objects_per_second: scanner_work_rate_per_second(metrics.last_cycle_objects_scanned, last_cycle_duration),
last_cycle_directories_per_second: scanner_work_rate_per_second(
metrics.last_cycle_directories_scanned,
last_cycle_duration,
),
last_cycle_bucket_drive_scans_per_second: scanner_work_rate_per_second(
metrics.last_cycle_bucket_drive_scans,
last_cycle_duration,
),
last_cycle_yield_events: metrics.last_cycle_yield_events,
last_cycle_yield_duration_seconds: metrics.last_cycle_yield_duration_seconds,
last_cycle_throttle_sleep_events: metrics.last_cycle_throttle_sleep_events,
last_cycle_throttle_sleep_duration_seconds: metrics.last_cycle_throttle_sleep_duration_seconds,
last_cycle_ilm_actions: metrics.last_cycle_ilm_actions,
last_cycle_heal_objects: metrics.last_cycle_heal_objects,
last_cycle_replication_checks: metrics.last_cycle_replication_checks,
last_cycle_usage_saves: metrics.last_cycle_usage_saves,
failed_cycles: metrics.failed_cycles,
superseded_cycles: metrics.superseded_cycles,
partial_cycles: metrics.partial_cycles,
partial_cycles_unknown: metrics.partial_cycles_unknown,
partial_cycles_runtime: metrics.partial_cycles_runtime,
partial_cycles_objects: metrics.partial_cycles_objects,
partial_cycles_directories: metrics.partial_cycles_directories,
},
})
}
@@ -1480,6 +1749,60 @@ mod tests {
assert!(!disk_is_online_for_metrics(DRIVE_STATE_OK, Some("offline")));
}
#[test]
fn disk_topology_label_rejects_unknown_negative_index() {
assert_eq!(disk_topology_label(-1), None);
assert_eq!(disk_topology_label(3), Some("3".to_string()));
}
#[test]
fn non_empty_disk_id_rejects_blank_uuid() {
assert_eq!(non_empty_disk_id(" "), None);
assert_eq!(non_empty_disk_id("disk-1"), Some("disk-1".to_string()));
}
#[test]
fn drive_server_label_uses_node_identity_for_urls_and_local_paths() {
assert_eq!(drive_server_label("http://node1:9000/data", "local:9000"), "node1:9000");
assert_eq!(drive_server_label("https://node2:9443/export/d1", "local:9000"), "node2:9443");
assert_eq!(drive_server_label("/mnt/data1", "local:9000"), "local:9000");
}
#[test]
fn drive_inode_stats_skip_unknown_zero_inode_totals() {
assert_eq!(drive_inode_stats(0, 0), (None, None, None));
assert_eq!(drive_inode_stats(2, 3), (Some(2), Some(3), Some(5)));
}
#[test]
fn drive_api_metrics_are_sorted_and_average_latency() {
let last_minute = HashMap::from([
("write".to_string(), (2, 6_000)),
("read".to_string(), (1, 3_000)),
("zero".to_string(), (0, 9_000)),
]);
let api_calls = HashMap::from([("write".to_string(), 9), ("read".to_string(), 4)]);
assert_eq!(drive_api_latency_micros(last_minute.values().copied()), Some(3));
assert_eq!(
drive_api_latency_by_api_micros(last_minute.iter().map(|(api, (count, acc_time))| (api, *count, *acc_time))),
vec![("read".to_string(), 3), ("write".to_string(), 3), ("zero".to_string(), 0)]
);
assert_eq!(drive_api_calls(api_calls.iter()), vec![("read".to_string(), 4), ("write".to_string(), 9)]);
}
#[test]
fn drive_api_latency_skips_zero_denominators() {
let last_minute = HashMap::from([("zero".to_string(), (0, 9_000))]);
assert_eq!(drive_api_latency_micros(last_minute.values().copied()), Some(0));
assert_eq!(
drive_api_latency_by_api_micros(last_minute.iter().map(|(api, (count, acc_time))| (api, *count, *acc_time))),
vec![("zero".to_string(), 0)]
);
assert_eq!(drive_api_latency_micros([].into_iter()), None);
}
#[test]
fn derive_erasure_set_quorum_shape_handles_standard_layout() {
let shape = derive_erasure_set_quorum_shape(16, 4);
@@ -1580,6 +1903,35 @@ mod tests {
assert_eq!(scanner_bucket_scans_started(&life_time_ops, 5), 5);
}
#[test]
fn ilm_action_task_stats_maps_runtime_states() {
let stats = ilm_action_task_stats(&ObsIlmRuntimeSnapshot {
expiry_pending_tasks: 1,
transition_active_tasks: 2,
transition_pending_tasks: 3,
transition_missed_immediate_tasks: 4,
transition_queue_full_tasks: 5,
transition_queue_send_timeout_tasks: 6,
transition_compensation_scheduled_tasks: 7,
transition_compensation_running_tasks: 8,
});
assert_eq!(stats.len(), 8);
assert_eq!(stats[0].action, "expiry");
assert_eq!(stats[0].state, "pending");
assert_eq!(stats[0].value, 1);
assert!(
stats
.iter()
.any(|task| { task.action == "transition" && task.state == "queue_send_timeout" && task.value == 6 })
);
assert!(
stats
.iter()
.any(|task| { task.action == "transition" && task.state == "compensation_running" && task.value == 8 })
);
}
#[test]
fn scanner_lifecycle_checked_versions_uses_lifecycle_checked_source_work() {
let report = ScannerMetricsReport {
@@ -1602,6 +1954,96 @@ mod tests {
assert_eq!(scanner_lifecycle_checked_versions(&report), 37);
}
#[test]
fn scanner_source_work_stats_sorts_and_skips_empty_source() {
let stats = scanner_source_work_stats(&[
ScannerSourceWorkSnapshot {
source: "usage".to_string(),
checked: 11,
queued: 2,
executed: 3,
failed: 4,
skipped: 5,
missed: 6,
},
ScannerSourceWorkSnapshot {
checked: 99,
queued: 99,
executed: 99,
failed: 99,
skipped: 99,
missed: 99,
..Default::default()
},
ScannerSourceWorkSnapshot {
source: "lifecycle".to_string(),
checked: 21,
queued: 7,
executed: 8,
failed: 9,
skipped: 10,
missed: 12,
},
]);
assert_eq!(stats.len(), 2);
assert_eq!(stats[0].source, "lifecycle");
assert_eq!(stats[0].checked, 21);
assert_eq!(stats[0].missed, 12);
assert_eq!(stats[1].source, "usage");
assert_eq!(stats[1].failed, 4);
}
#[test]
fn scanner_current_cycle_source_work_stats_zeroes_idle_sources() {
let report = ScannerMetricsReport {
source_work: vec![ScannerSourceWorkSnapshot {
source: "usage".to_string(),
checked: 11,
queued: 2,
..Default::default()
}],
last_cycle_source_work: vec![ScannerSourceWorkSnapshot {
source: "lifecycle".to_string(),
checked: 21,
queued: 7,
..Default::default()
}],
..Default::default()
};
let stats = scanner_current_cycle_source_work_stats(&report);
assert_eq!(stats.len(), 2);
assert_eq!(stats[0].source, "lifecycle");
assert_eq!(stats[0].checked, 0);
assert_eq!(stats[1].source, "usage");
assert_eq!(stats[1].queued, 0);
}
#[test]
fn scanner_current_cycle_source_work_stats_keeps_active_values() {
let report = ScannerMetricsReport {
source_work: vec![ScannerSourceWorkSnapshot {
source: "usage".to_string(),
checked: 11,
..Default::default()
}],
current_cycle_source_work: vec![ScannerSourceWorkSnapshot {
source: "usage".to_string(),
checked: 3,
..Default::default()
}],
..Default::default()
};
let stats = scanner_current_cycle_source_work_stats(&report);
assert_eq!(stats.len(), 1);
assert_eq!(stats[0].source, "usage");
assert_eq!(stats[0].checked, 3);
}
#[test]
fn scanner_lifecycle_checked_versions_defaults_to_zero_when_lifecycle_missing() {
let report = ScannerMetricsReport {
crates/obs/src/metrics/storage_api.rs
+120 -55
View File
@@ -18,9 +18,9 @@ use std::time::Duration;
pub(crate) use rustfs_ecstore::api::bucket::bandwidth::monitor::Monitor as ObsBucketBandwidthMonitor;
pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::get_quota_config as obs_get_quota_config;
use rustfs_ecstore::api::bucket::replication::{
DurableMrfBucketBacklog, DurableMrfTargetBacklog, MrfBucketBacklogObservability, RuntimeReplicationTargetBacklog,
durable_mrf_backlog_summary_snapshot, durable_mrf_target_backlog_snapshot, get_global_replication_stats,
mrf_backlog_observability_snapshot,
BucketReplicationStats as SourceBucketReplicationStats, DurableMrfBucketBacklog, DurableMrfTargetBacklog,
MrfBucketBacklogObservability, RuntimeReplicationTargetBacklog, durable_mrf_backlog_summary_snapshot,
durable_mrf_target_backlog_snapshot, get_global_replication_stats, mrf_backlog_observability_snapshot,
};
pub(crate) use rustfs_ecstore::api::capacity::{
get_total_usable_capacity as obs_get_total_usable_capacity,
@@ -43,6 +43,14 @@ pub(crate) struct ObsBucketReplicationTargetStatsSnapshot {
pub(crate) bandwidth_limit_bytes_per_sec: u64,
pub(crate) current_bandwidth_bytes_per_sec: f64,
pub(crate) latency_ms: f64,
pub(crate) sent_bytes: u64,
pub(crate) sent_count: u64,
pub(crate) total_failed_bytes: u64,
pub(crate) total_failed_count: u64,
pub(crate) last_min_failed_bytes: u64,
pub(crate) last_min_failed_count: u64,
pub(crate) last_hour_failed_bytes: u64,
pub(crate) last_hour_failed_count: u64,
}
#[derive(Debug, Clone, PartialEq)]
@@ -173,6 +181,73 @@ fn replication_backlog_count(failed_counts: impl Iterator<Item = i64>, queued_co
failed_backlog.saturating_add(i64_to_u64_floor_zero(queued_count))
}
fn bucket_replication_runtime_snapshot_from_source(
bucket_stats: Option<&SourceBucketReplicationStats>,
) -> ObsBucketReplicationRuntimeSnapshot {
let mut runtime = ObsBucketReplicationRuntimeSnapshot {
targets: Vec::with_capacity(bucket_stats.map(|stats| stats.stats.len()).unwrap_or(0)),
..Default::default()
};
if let Some(bucket_stats) = bucket_stats {
for (target_arn, target_stats) in &bucket_stats.stats {
runtime.total_failed_bytes = runtime
.total_failed_bytes
.saturating_add(i64_to_u64_floor_zero(target_stats.fail_stats.size));
runtime.total_failed_count = runtime
.total_failed_count
.saturating_add(i64_to_u64_floor_zero(target_stats.fail_stats.count));
let last_min = target_stats.fail_stats.recent_since(Duration::from_secs(60));
runtime.last_min_failed_bytes = runtime
.last_min_failed_bytes
.saturating_add(i64_to_u64_floor_zero(last_min.size));
runtime.last_min_failed_count = runtime
.last_min_failed_count
.saturating_add(i64_to_u64_floor_zero(last_min.count));
let last_hour = target_stats.fail_stats.recent_since(Duration::from_secs(60 * 60));
runtime.last_hour_failed_bytes = runtime
.last_hour_failed_bytes
.saturating_add(i64_to_u64_floor_zero(last_hour.size));
runtime.last_hour_failed_count = runtime
.last_hour_failed_count
.saturating_add(i64_to_u64_floor_zero(last_hour.count));
runtime.sent_bytes = runtime
.sent_bytes
.saturating_add(i64_to_u64_floor_zero(target_stats.replicated_size));
runtime.sent_count = runtime
.sent_count
.saturating_add(i64_to_u64_floor_zero(target_stats.replicated_count));
runtime.targets.push(ObsBucketReplicationTargetStatsSnapshot {
target_arn: target_arn.clone(),
bandwidth_limit_bytes_per_sec: i64_to_u64_floor_zero(target_stats.bandwidth_limit_bytes_per_sec),
current_bandwidth_bytes_per_sec: target_stats.current_bandwidth_bytes_per_sec,
latency_ms: target_stats.latency.curr,
sent_bytes: i64_to_u64_floor_zero(target_stats.replicated_size),
sent_count: i64_to_u64_floor_zero(target_stats.replicated_count),
total_failed_bytes: i64_to_u64_floor_zero(target_stats.fail_stats.size),
total_failed_count: i64_to_u64_floor_zero(target_stats.fail_stats.count),
last_min_failed_bytes: i64_to_u64_floor_zero(last_min.size),
last_min_failed_count: i64_to_u64_floor_zero(last_min.count),
last_hour_failed_bytes: i64_to_u64_floor_zero(last_hour.size),
last_hour_failed_count: i64_to_u64_floor_zero(last_hour.count),
});
}
runtime.resync_started_count = i64_to_u64_floor_zero(bucket_stats.resync_started_count);
runtime.resync_completed_count = i64_to_u64_floor_zero(bucket_stats.resync_completed_count);
runtime.resync_failed_count = i64_to_u64_floor_zero(bucket_stats.resync_failed_count);
runtime.resync_canceled_count = i64_to_u64_floor_zero(bucket_stats.resync_canceled_count);
runtime.resync_duration_ms = i64_to_u64_floor_zero(bucket_stats.resync_duration_ms);
runtime.current_backlog_count = i64_to_u64_floor_zero(bucket_stats.q_stat.curr.count);
runtime.current_backlog_bytes = i64_to_u64_floor_zero(bucket_stats.q_stat.curr.bytes);
}
runtime
}
fn bucket_replication_stats_snapshot_from_parts(
bucket: String,
runtime: ObsBucketReplicationRuntimeSnapshot,
@@ -357,58 +432,7 @@ pub(crate) async fn obs_bucket_replication_stats_snapshot() -> Vec<ObsBucketRepl
proxied_delete_tagging_requests_total: i64_to_u64_floor_zero(proxy.delete_tag_total),
proxied_delete_tagging_requests_failures: i64_to_u64_floor_zero(proxy.delete_tag_failed),
};
let mut runtime = ObsBucketReplicationRuntimeSnapshot {
targets: Vec::with_capacity(bucket_stats.map(|stats| stats.stats.len()).unwrap_or(0)),
..Default::default()
};
if let Some(bucket_stats) = bucket_stats {
for (target_arn, target_stats) in &bucket_stats.stats {
runtime.total_failed_bytes = runtime
.total_failed_bytes
.saturating_add(i64_to_u64_floor_zero(target_stats.fail_stats.size));
runtime.total_failed_count = runtime
.total_failed_count
.saturating_add(i64_to_u64_floor_zero(target_stats.fail_stats.count));
let last_min = target_stats.fail_stats.recent_since(Duration::from_secs(60));
runtime.last_min_failed_bytes = runtime
.last_min_failed_bytes
.saturating_add(i64_to_u64_floor_zero(last_min.size));
runtime.last_min_failed_count = runtime
.last_min_failed_count
.saturating_add(i64_to_u64_floor_zero(last_min.count));
let last_hour = target_stats.fail_stats.recent_since(Duration::from_secs(60 * 60));
runtime.last_hour_failed_bytes = runtime
.last_hour_failed_bytes
.saturating_add(i64_to_u64_floor_zero(last_hour.size));
runtime.last_hour_failed_count = runtime
.last_hour_failed_count
.saturating_add(i64_to_u64_floor_zero(last_hour.count));
runtime.sent_bytes = runtime
.sent_bytes
.saturating_add(i64_to_u64_floor_zero(target_stats.replicated_size));
runtime.sent_count = runtime
.sent_count
.saturating_add(i64_to_u64_floor_zero(target_stats.replicated_count));
runtime.targets.push(ObsBucketReplicationTargetStatsSnapshot {
target_arn: target_arn.clone(),
bandwidth_limit_bytes_per_sec: i64_to_u64_floor_zero(target_stats.bandwidth_limit_bytes_per_sec),
current_bandwidth_bytes_per_sec: target_stats.current_bandwidth_bytes_per_sec,
latency_ms: target_stats.latency.curr,
});
}
runtime.resync_started_count = i64_to_u64_floor_zero(bucket_stats.resync_started_count);
runtime.resync_completed_count = i64_to_u64_floor_zero(bucket_stats.resync_completed_count);
runtime.resync_failed_count = i64_to_u64_floor_zero(bucket_stats.resync_failed_count);
runtime.resync_canceled_count = i64_to_u64_floor_zero(bucket_stats.resync_canceled_count);
runtime.resync_duration_ms = i64_to_u64_floor_zero(bucket_stats.resync_duration_ms);
runtime.current_backlog_count = i64_to_u64_floor_zero(bucket_stats.q_stat.curr.count);
runtime.current_backlog_bytes = i64_to_u64_floor_zero(bucket_stats.q_stat.curr.bytes);
}
let runtime = bucket_replication_runtime_snapshot_from_source(bucket_stats);
let durable_bucket = durable_buckets.get(&bucket).cloned().unwrap_or_default();
let runtime_targets = runtime_targets_by_bucket.remove(&bucket).unwrap_or_default();
let durable_targets = durable_targets_by_bucket.remove(&bucket).unwrap_or_default();
@@ -500,6 +524,47 @@ mod tests {
assert_eq!(replication_backlog_count([9].into_iter(), 0), 9);
}
#[test]
fn bucket_replication_runtime_snapshot_maps_target_flow_fields_from_source() {
let mut source = SourceBucketReplicationStats::new();
let target = source.stats.entry("arn:rustfs:replication:target-a".to_string()).or_default();
target.fail_stats.add_size::<()>(100, None);
target.fail_stats.add_size::<()>(200, None);
target.fail_stats.count = 7;
target.fail_stats.size = 900;
target.replicated_size = 1234;
target.replicated_count = 12;
target.bandwidth_limit_bytes_per_sec = 4096;
target.current_bandwidth_bytes_per_sec = 512.5;
target.latency.curr = 45.0;
let snapshot = bucket_replication_runtime_snapshot_from_source(Some(&source));
assert_eq!(snapshot.sent_bytes, 1234);
assert_eq!(snapshot.sent_count, 12);
assert_eq!(snapshot.total_failed_bytes, 900);
assert_eq!(snapshot.total_failed_count, 7);
assert_eq!(snapshot.last_min_failed_bytes, 300);
assert_eq!(snapshot.last_min_failed_count, 2);
assert_eq!(snapshot.last_hour_failed_bytes, 300);
assert_eq!(snapshot.last_hour_failed_count, 2);
assert_eq!(snapshot.targets.len(), 1);
let target = &snapshot.targets[0];
assert_eq!(target.target_arn, "arn:rustfs:replication:target-a");
assert_eq!(target.bandwidth_limit_bytes_per_sec, 4096);
assert_eq!(target.current_bandwidth_bytes_per_sec, 512.5);
assert_eq!(target.latency_ms, 45.0);
assert_eq!(target.sent_bytes, 1234);
assert_eq!(target.sent_count, 12);
assert_eq!(target.total_failed_bytes, 900);
assert_eq!(target.total_failed_count, 7);
assert_eq!(target.last_min_failed_bytes, 300);
assert_eq!(target.last_min_failed_count, 2);
assert_eq!(target.last_hour_failed_bytes, 300);
assert_eq!(target.last_hour_failed_count, 2);
}
#[test]
fn bucket_replication_snapshot_maps_runtime_and_durable_backlog() {
let snapshot = bucket_replication_stats_snapshot_from_parts(
crates/s3-ops/src/lib.rs
+75
View File
@@ -15,6 +15,7 @@
use rustfs_s3_types::EventName;
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
#[repr(usize)]
pub enum S3Operation {
AbortMultipartUpload,
CompleteMultipartUpload,
@@ -125,6 +126,80 @@ impl EventMapping {
}
impl S3Operation {
pub const ALL: [Self; 66] = [
Self::AbortMultipartUpload,
Self::CompleteMultipartUpload,
Self::CopyObject,
Self::CreateBucket,
Self::CreateMultipartUpload,
Self::DeleteBucket,
Self::DeleteBucketCors,
Self::DeleteBucketEncryption,
Self::DeleteBucketLifecycle,
Self::DeleteBucketPolicy,
Self::DeleteBucketReplication,
Self::DeleteBucketTagging,
Self::DeleteObject,
Self::DeleteObjectTagging,
Self::DeleteObjects,
Self::DeletePublicAccessBlock,
Self::GetBucketAcl,
Self::GetBucketCors,
Self::GetBucketEncryption,
Self::GetBucketLifecycleConfiguration,
Self::GetBucketLocation,
Self::GetBucketLogging,
Self::GetBucketNotificationConfiguration,
Self::GetBucketPolicy,
Self::GetBucketPolicyStatus,
Self::GetBucketReplication,
Self::GetBucketTagging,
Self::GetBucketVersioning,
Self::GetObject,
Self::GetObjectAcl,
Self::GetObjectAttributes,
Self::GetObjectLegalHold,
Self::GetObjectLockConfiguration,
Self::GetObjectRetention,
Self::GetObjectTagging,
Self::GetObjectTorrent,
Self::GetPublicAccessBlock,
Self::HeadBucket,
Self::HeadObject,
Self::ListBuckets,
Self::ListMultipartUploads,
Self::ListObjectVersions,
Self::ListObjects,
Self::ListObjectsV2,
Self::ListParts,
Self::PutBucketAcl,
Self::PutBucketCors,
Self::PutBucketEncryption,
Self::PutBucketLifecycleConfiguration,
Self::PutBucketLogging,
Self::PutBucketNotificationConfiguration,
Self::PutBucketPolicy,
Self::PutBucketReplication,
Self::PutBucketTagging,
Self::PutBucketVersioning,
Self::PutObject,
Self::PutObjectAcl,
Self::PutObjectLegalHold,
Self::PutObjectLockConfiguration,
Self::PutObjectRetention,
Self::PutObjectTagging,
Self::PutPublicAccessBlock,
Self::RestoreObject,
Self::SelectObjectContent,
Self::UploadPart,
Self::UploadPartCopy,
];
#[inline]
pub const fn metric_index(self) -> usize {
self as usize
}
pub fn as_str(self) -> &'static str {
match self {
Self::AbortMultipartUpload => "s3:AbortMultipartUpload",
rustfs/src/app/bucket_usecase.rs
+32
View File
@@ -75,6 +75,7 @@ use futures::StreamExt;
use http::StatusCode;
use metrics::counter;
use rustfs_config::RUSTFS_REGION;
use rustfs_io_metrics::record_s3_op;
use rustfs_madmin::{SITE_REPL_API_VERSION, SRBucketMeta};
use rustfs_policy::policy::{
action::{Action, S3Action},
@@ -1313,6 +1314,7 @@ impl DefaultBucketUsecase {
#[instrument(level = "debug", skip(self, req))]
pub async fn execute_head_bucket(&self, req: S3Request<HeadBucketInput>) -> S3Result<S3Response<HeadBucketOutput>> {
record_s3_op(S3Operation::HeadBucket);
let input = req.input;
let Some(store) = self.object_store() else {
@@ -1502,6 +1504,7 @@ impl DefaultBucketUsecase {
&self,
req: S3Request<DeleteBucketPolicyInput>,
) -> S3Result<S3Response<DeleteBucketPolicyOutput>> {
record_s3_op(S3Operation::DeleteBucketPolicy);
let request_context = req.extensions.get::<request_context::RequestContext>().cloned();
let DeleteBucketPolicyInput { bucket, .. } = req.input;
@@ -2265,6 +2268,7 @@ impl DefaultBucketUsecase {
&self,
req: S3Request<PutBucketPolicyInput>,
) -> S3Result<S3Response<PutBucketPolicyOutput>> {
record_s3_op(S3Operation::PutBucketPolicy);
let request_context = req.extensions.get::<request_context::RequestContext>().cloned();
let PutBucketPolicyInput { bucket, policy, .. } = req.input;
@@ -2714,6 +2718,14 @@ mod tests {
use std::time::Duration;
use tokio::sync::Notify;
fn s3_op_total(op: S3Operation) -> u64 {
rustfs_io_metrics::s3_op_metrics_snapshot()
.into_iter()
.find(|snapshot| snapshot.op == op.as_str())
.map(|snapshot| snapshot.total)
.unwrap_or_default()
}
#[tokio::test]
async fn bucket_usecase_task_finishes_post_commit_hooks_after_parent_cancellation() {
let admission = Arc::new(Semaphore::new(1));
@@ -3337,9 +3349,27 @@ mod tests {
let req = build_request(input, Method::HEAD);
let usecase = DefaultBucketUsecase::without_context();
let before = s3_op_total(S3Operation::HeadBucket);
let err = usecase.execute_head_bucket(req).await.unwrap_err();
assert_eq!(err.code(), &S3ErrorCode::InternalError);
assert_eq!(s3_op_total(S3Operation::HeadBucket), before + 1);
}
#[tokio::test]
async fn execute_delete_bucket_policy_records_s3_operation_before_store_lookup() {
let input = DeleteBucketPolicyInput::builder()
.bucket("test-bucket".to_string())
.build()
.unwrap();
let req = build_request(input, Method::DELETE);
let usecase = DefaultBucketUsecase::without_context();
let before = s3_op_total(S3Operation::DeleteBucketPolicy);
let err = usecase.execute_delete_bucket_policy(req).await.unwrap_err();
assert_eq!(err.code(), &S3ErrorCode::InternalError);
assert_eq!(s3_op_total(S3Operation::DeleteBucketPolicy), before + 1);
}
#[tokio::test]
@@ -4327,9 +4357,11 @@ mod tests {
let req = build_request(input, Method::PUT);
let usecase = DefaultBucketUsecase::without_context();
let before = s3_op_total(S3Operation::PutBucketPolicy);
let err = usecase.execute_put_bucket_policy(req).await.unwrap_err();
assert_eq!(err.code(), &S3ErrorCode::InternalError);
assert_eq!(s3_op_total(S3Operation::PutBucketPolicy), before + 1);
}
#[tokio::test]
rustfs/src/app/multipart_usecase.rs
+12
View File
@@ -72,6 +72,7 @@ use crate::table_catalog;
use bytes::Bytes;
use futures::StreamExt;
use http::{HeaderMap, HeaderValue, Uri};
use rustfs_io_metrics::record_s3_op;
use rustfs_s3_ops::S3Operation;
use rustfs_targets::EventName;
use rustfs_utils::CompressionAlgorithm;
@@ -341,6 +342,7 @@ impl DefaultMultipartUsecase {
&self,
req: S3Request<AbortMultipartUploadInput>,
) -> S3Result<S3Response<AbortMultipartUploadOutput>> {
record_s3_op(S3Operation::AbortMultipartUpload);
let AbortMultipartUploadInput {
bucket, key, upload_id, ..
} = req.input;
@@ -1449,6 +1451,14 @@ mod tests {
use temp_env::async_with_vars;
use tokio::io::AsyncReadExt;
fn s3_op_total(op: S3Operation) -> u64 {
rustfs_io_metrics::s3_op_metrics_snapshot()
.into_iter()
.find(|snapshot| snapshot.op == op.as_str())
.map(|snapshot| snapshot.total)
.unwrap_or_default()
}
fn build_request<T>(input: T, method: Method) -> S3Request<T> {
S3Request {
input,
@@ -1755,9 +1765,11 @@ mod tests {
.build()
.unwrap();
let req = build_request(input, Method::DELETE);
let before = s3_op_total(S3Operation::AbortMultipartUpload);
let err = make_usecase().execute_abort_multipart_upload(req).await.unwrap_err();
assert_eq!(err.code(), &S3ErrorCode::InternalError);
assert_eq!(s3_op_total(S3Operation::AbortMultipartUpload), before + 1);
}
#[tokio::test]
rustfs/src/storage/ecfs.rs
-3
View File
@@ -409,7 +409,6 @@ impl S3 for FS {
&self,
req: S3Request<DeleteObjectTaggingInput>,
) -> S3Result<S3Response<DeleteObjectTaggingOutput>> {
record_s3_op(S3Operation::DeleteObjectTagging);
let start_time = std::time::Instant::now();
let mut helper = OperationHelper::new(&req, EventName::ObjectTaggingDelete, S3Operation::DeleteObjectTagging);
let DeleteObjectTaggingInput {
@@ -1229,7 +1228,6 @@ impl S3 for FS {
}
async fn put_object_acl(&self, req: S3Request<PutObjectAclInput>) -> S3Result<S3Response<PutObjectAclOutput>> {
record_s3_op(S3Operation::PutObjectAcl);
let mut helper = OperationHelper::new(&req, EventName::ObjectAclPut, S3Operation::PutObjectAcl);
let bucket = &req.input.bucket;
let key = &req.input.key;
@@ -1558,7 +1556,6 @@ impl S3 for FS {
#[instrument(level = "debug", skip(self, req))]
async fn put_object_tagging(&self, req: S3Request<PutObjectTaggingInput>) -> S3Result<S3Response<PutObjectTaggingOutput>> {
record_s3_op(S3Operation::PutObjectTagging);
let start_time = std::time::Instant::now();
let mut helper = OperationHelper::new(&req, EventName::ObjectTaggingPut, S3Operation::PutObjectTagging);
let PutObjectTaggingInput {