diff --git a/.docker/observability/grafana/dashboards/rustfs.json b/.docker/observability/grafana/dashboards/rustfs.json index a9a17ae02..5b7f047a8 100644 --- a/.docker/observability/grafana/dashboards/rustfs.json +++ b/.docker/observability/grafana/dashboards/rustfs.json @@ -11500,6 +11500,797 @@ ], "title": "Compression Operations Rate", "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 332 + }, + "id": 531, + "panels": [], + "title": "Metrics Dimensions Drilldown", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 333 + }, + "id": 532, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "range": true, + "refId": "A", + "expr": "sum by (server, name, type) (rate(rustfs_api_requests_requests_total_by_server{job=~\"$job\",server=~\"$server\",name=~\"$api\"}[$__rate_interval]))", + "legendFormat": "{{server}} | {{name}} | {{type}}" + } + ], + "title": "API Requests by Server and API", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "unit", + "value": "none" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 333 + }, + "id": 533, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "range": true, + "refId": "A", + "expr": "max by (server, drive, pool_index, set_index, drive_index, state) (rustfs_system_drive_runtime_state{job=~\"$job\",server=~\"$server\",drive=~\"$drive\"})", + "legendFormat": "{{server}} | {{drive}} | p{{pool_index}}/s{{set_index}}/d{{drive_index}} | {{state}}" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "range": true, + "refId": "B", + "expr": "max by (server, drive, pool_index, set_index, drive_index) (rustfs_system_drive_offline_duration_seconds{job=~\"$job\",server=~\"$server\",drive=~\"$drive\"})", + "legendFormat": "{{server}} | {{drive}} | offline seconds" + } + ], + "title": "Drive Runtime State and Offline Duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 341 + }, + "id": 534, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "range": true, + "refId": "A", + "expr": "sum by (server, drive, pool_index, set_index, drive_index, api) (rate(rustfs_system_drive_api_calls_total{job=~\"$job\",server=~\"$server\",drive=~\"$drive\",api=~\"$drive_api\"}[$__rate_interval]))", + "legendFormat": "{{server}} | {{drive}} | p{{pool_index}}/s{{set_index}}/d{{drive_index}} | {{api}}" + } + ], + "title": "Drive API Calls by Operation", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 341 + }, + "id": 535, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "range": true, + "refId": "A", + "expr": "sum by (server, source, state) (rate(rustfs_scanner_source_work_total{job=~\"$job\",server=~\"$server\",source=~\"$scanner_source\"}[$__rate_interval]))", + "legendFormat": "{{server}} | {{source}} | {{state}}" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "range": true, + "refId": "B", + "expr": "sum by (server, cycle_scope, source, state) (rustfs_scanner_cycle_source_work{job=~\"$job\",server=~\"$server\",source=~\"$scanner_source\"})", + "legendFormat": "{{server}} | {{cycle_scope}} | {{source}} | {{state}}" + } + ], + "title": "Scanner Source Work by Server", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 349 + }, + "id": 536, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "range": true, + "refId": "A", + "expr": "sum by (server, bucket, drive, result) (rate(rustfs_scanner_bucket_drive_result_total{job=~\"$job\",server=~\"$server\",bucket=~\"$bucket\",drive=~\"$drive\",result=~\"$scanner_result\"}[$__rate_interval]))", + "legendFormat": "{{server}} | {{bucket}} | {{drive}} | {{result}}" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "range": true, + "refId": "B", + "expr": "sum by (server, cycle_scope, bucket, drive, result) (rustfs_scanner_cycle_bucket_drive_result{job=~\"$job\",server=~\"$server\",bucket=~\"$bucket\",drive=~\"$drive\",result=~\"$scanner_result\"})", + "legendFormat": "{{server}} | {{cycle_scope}} | {{bucket}} | {{drive}} | {{result}}" + } + ], + "title": "Scanner Bucket Drive Results", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "ops" + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "unit", + "value": "Bps" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 349 + }, + "id": 537, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "range": true, + "refId": "A", + "expr": "sum by (bucket, target_arn) (rate(rustfs_bucket_replication_target_sent_count{job=~\"$job\",bucket=~\"$bucket\",target_arn=~\"$target_arn\"}[$__rate_interval]))", + "legendFormat": "sent objects | {{bucket}} | {{target_arn}}" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "range": true, + "refId": "B", + "expr": "sum by (bucket, target_arn) (rate(rustfs_bucket_replication_target_sent_bytes{job=~\"$job\",bucket=~\"$bucket\",target_arn=~\"$target_arn\"}[$__rate_interval]))", + "legendFormat": "sent bytes | {{bucket}} | {{target_arn}}" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "range": true, + "refId": "C", + "expr": "sum by (bucket, target_arn) (rate(rustfs_bucket_replication_target_total_failed_count{job=~\"$job\",bucket=~\"$bucket\",target_arn=~\"$target_arn\"}[$__rate_interval]))", + "legendFormat": "failed objects | {{bucket}} | {{target_arn}}" + } + ], + "title": "Bucket Replication Target Flow", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 357 + }, + "id": 538, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "range": true, + "refId": "A", + "expr": "max by (server, target_id) (rustfs_audit_target_queue_length_by_server{job=~\"$job\",server=~\"$server\"})", + "legendFormat": "audit queue | {{server}} | {{target_id}}" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "range": true, + "refId": "B", + "expr": "max by (server, action, state) (rustfs_ilm_action_tasks{job=~\"$job\",server=~\"$server\"})", + "legendFormat": "ilm | {{server}} | {{action}} | {{state}}" + } + ], + "title": "Audit and ILM by Server", + "type": "timeseries" } ], "preload": false, @@ -11551,6 +12342,32 @@ "sort": 1, "type": "query" }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(rustfs_system_drive_api_calls_total,api)", + "includeAll": true, + "label": "Drive API", + "multi": true, + "name": "drive_api", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rustfs_system_drive_api_calls_total,api)", + "refId": "PrometheusVariableQueryEditor-drive_api" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + }, { "allValue": ".*", "current": { @@ -11670,6 +12487,136 @@ "regex": "", "sort": 1, "type": "query" + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(rustfs_api_requests_requests_total_by_server,server)", + "includeAll": true, + "label": "Server", + "multi": true, + "name": "server", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rustfs_api_requests_requests_total_by_server,server)", + "refId": "PrometheusVariableQueryEditor-server" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(rustfs_api_requests_requests_total_by_server,name)", + "includeAll": true, + "label": "API", + "multi": true, + "name": "api", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rustfs_api_requests_requests_total_by_server,name)", + "refId": "PrometheusVariableQueryEditor-api" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values({__name__=\"rustfs_bucket_replication_target_sent_count\",bucket=~\"$bucket\"},target_arn)", + "includeAll": true, + "label": "Target ARN", + "multi": true, + "name": "target_arn", + "options": [], + "query": { + "qryType": 1, + "query": "label_values({__name__=\"rustfs_bucket_replication_target_sent_count\",bucket=~\"$bucket\"},target_arn)", + "refId": "PrometheusVariableQueryEditor-target_arn" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(rustfs_scanner_source_work_total,source)", + "includeAll": true, + "label": "Scanner Source", + "multi": true, + "name": "scanner_source", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rustfs_scanner_source_work_total,source)", + "refId": "PrometheusVariableQueryEditor-scanner_source" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(rustfs_scanner_bucket_drive_result_total,result)", + "includeAll": true, + "label": "Scanner Result", + "multi": true, + "name": "scanner_result", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rustfs_scanner_bucket_drive_result_total,result)", + "refId": "PrometheusVariableQueryEditor-scanner_result" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" } ] }, diff --git a/crates/common/src/metrics.rs b/crates/common/src/metrics.rs index 24f35678b..61879b591 100644 --- a/crates/common/src/metrics.rs +++ b/crates/common/src/metrics.rs @@ -708,6 +708,42 @@ struct ScannerDiskBucketScanState { active: u64, } +type ScannerDiskBucketScanKey = (String, String); + +#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)] +pub struct ScannerDiskBucketScanSnapshot { + pub pool: String, + pub set: String, + pub concurrency_limit: u64, + pub queued: u64, + pub active: u64, +} + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +struct ScannerBucketDriveResultKey { + bucket: String, + drive: String, + result: String, +} + +impl ScannerBucketDriveResultKey { + fn new(bucket: impl Into, drive: impl Into, result: impl Into) -> Self { + Self { + bucket: bucket.into(), + drive: drive.into(), + result: result.into(), + } + } +} + +const MAX_SCANNER_BUCKET_DRIVE_RESULT_KEYS: usize = 4096; + +#[derive(Clone, Copy, Debug)] +struct ScannerBucketDriveResultValue { + count: u64, + last_seen: u64, +} + // --------------------------------------------------------------------------- // Metrics // --------------------------------------------------------------------------- @@ -738,7 +774,11 @@ pub struct Metrics { scanner_set_scan_concurrency_limit: AtomicU64, scanner_set_scans_queued: AtomicU64, scanner_set_scans_active: AtomicU64, - scanner_disk_bucket_scan_states: Mutex>, + scanner_disk_bucket_scan_states: Mutex>, + scanner_bucket_drive_results: Mutex>, + scanner_bucket_drive_result_clock: AtomicU64, + current_scan_cycle_bucket_drive_results_start: Mutex>, + last_scan_cycle_bucket_drive_results: Mutex>, scanner_leader_lock_state: RwLock, scanner_leader_lock_held: AtomicBool, scanner_leader_lock_last_error: RwLock, @@ -958,6 +998,14 @@ pub struct ScannerSourceWorkSnapshot { pub missed: u64, } +#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)] +pub struct ScannerBucketDriveResultSnapshot { + pub bucket: String, + pub drive: String, + pub result: String, + pub count: u64, +} + #[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)] pub struct ScannerReplicationRepairSnapshot { pub source: String, @@ -1290,6 +1338,18 @@ pub struct ScannerMetricsReport { pub partial_cycles: u64, } +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub struct ScannerRuntimeDetailsReport { + #[serde(default)] + pub disk_bucket_scan_states: Vec, + #[serde(default)] + pub bucket_drive_results: Vec, + #[serde(default)] + pub current_cycle_bucket_drive_results: Vec, + #[serde(default)] + pub last_cycle_bucket_drive_results: Vec, +} + impl CurrentCycle { pub fn unmarshal(&mut self, buf: &[u8]) -> Result<(), Box> { *self = rmp_serde::from_slice(buf)?; @@ -1657,6 +1717,7 @@ pub fn emit_scan_cycle_superseded(duration: Duration) { pub fn emit_scan_bucket_drive_complete(success: bool, bucket: &str, disk: &str, duration: Duration) { let result = if success { "success" } else { "error" }; + global_metrics().record_scanner_bucket_drive_result(bucket, disk, result); metrics::counter!( OTEL_SCANNER_BUCKETS_SCANNED, "result" => result, @@ -1673,6 +1734,7 @@ pub fn emit_scan_bucket_drive_complete(success: bool, bucket: &str, disk: &str, } pub fn emit_scan_bucket_drive_partial(bucket: &str, disk: &str, duration: Duration) { + global_metrics().record_scanner_bucket_drive_result(bucket, disk, SCAN_CYCLE_RESULT_PARTIAL_LABEL); metrics::counter!( OTEL_SCANNER_BUCKETS_SCANNED, "result" => SCAN_CYCLE_RESULT_PARTIAL_LABEL, @@ -1723,6 +1785,10 @@ impl Metrics { scanner_set_scans_queued: AtomicU64::new(0), scanner_set_scans_active: AtomicU64::new(0), scanner_disk_bucket_scan_states: Mutex::new(HashMap::new()), + scanner_bucket_drive_results: Mutex::new(HashMap::new()), + scanner_bucket_drive_result_clock: AtomicU64::new(0), + current_scan_cycle_bucket_drive_results_start: Mutex::new(HashMap::new()), + last_scan_cycle_bucket_drive_results: Mutex::new(Vec::new()), scanner_leader_lock_state: RwLock::new("unknown".to_string()), scanner_leader_lock_held: AtomicBool::new(false), scanner_leader_lock_last_error: RwLock::new(String::new()), @@ -2293,7 +2359,7 @@ impl Metrics { queued: Option, active: Option, ) { - let key = format!("{pool}/{set}"); + let key = (pool.to_string(), set.to_string()); let mut states = self .scanner_disk_bucket_scan_states .lock() @@ -2310,6 +2376,44 @@ impl Metrics { } } + pub fn record_scanner_bucket_drive_result(&self, bucket: &str, drive: &str, result: &str) { + if bucket.is_empty() || drive.is_empty() || result.is_empty() { + return; + } + let key = ScannerBucketDriveResultKey::new(bucket, drive, result); + let mut results = self + .scanner_bucket_drive_results + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + let last_seen = self.scanner_bucket_drive_result_clock.fetch_add(1, Ordering::Relaxed); + if !results.contains_key(&key) + && results.len() >= MAX_SCANNER_BUCKET_DRIVE_RESULT_KEYS + && let Some(stale_key) = results + .iter() + .min_by(|left, right| { + left.1 + .last_seen + .cmp(&right.1.last_seen) + .then_with(|| left.0.bucket.cmp(&right.0.bucket)) + .then_with(|| left.0.drive.cmp(&right.0.drive)) + .then_with(|| left.0.result.cmp(&right.0.result)) + }) + .map(|(key, _)| key.clone()) + { + results.remove(&stale_key); + } + match results.entry(key) { + std::collections::hash_map::Entry::Occupied(mut entry) => { + let value = entry.get_mut(); + value.count = value.count.saturating_add(1); + value.last_seen = last_seen; + } + std::collections::hash_map::Entry::Vacant(entry) => { + entry.insert(ScannerBucketDriveResultValue { count: 1, last_seen }); + } + } + } + // ----------------------------------------------------------------------- // Read-side helpers // ----------------------------------------------------------------------- @@ -2481,6 +2585,11 @@ impl Metrics { &self.current_scan_cycle_replication_repair_work_start, &replication_repair_snapshot, ); + let bucket_drive_results = self.scanner_bucket_drive_result_counts(); + match self.current_scan_cycle_bucket_drive_results_start.lock() { + Ok(mut start) => *start = bucket_drive_results, + Err(poisoned) => *poisoned.into_inner() = bucket_drive_results, + } self.current_scan_cycle_work_active.store(true, Ordering::Release); snapshot } @@ -2493,6 +2602,11 @@ impl Metrics { self.record_scan_cycle_work(work); self.record_scan_cycle_source_work(&source_work); self.record_scan_cycle_replication_repair_work(&replication_repair_work); + let bucket_drive_results = self.current_cycle_bucket_drive_result_snapshots(); + match self.last_scan_cycle_bucket_drive_results.lock() { + Ok(mut last) => *last = bucket_drive_results, + Err(poisoned) => *poisoned.into_inner() = bucket_drive_results, + } self.current_scan_cycle_work_active.store(false, Ordering::Release); } @@ -2576,6 +2690,104 @@ impl Metrics { } } + fn scanner_bucket_drive_result_counts(&self) -> HashMap { + self.scanner_bucket_drive_results + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .iter() + .map(|(key, value)| (key.clone(), value.count)) + .collect() + } + + fn scanner_bucket_drive_result_snapshots( + counts: impl IntoIterator, + ) -> Vec { + let mut snapshots = counts + .into_iter() + .filter(|(_, count)| *count > 0) + .map(|(key, count)| ScannerBucketDriveResultSnapshot { + bucket: key.bucket, + drive: key.drive, + result: key.result, + count, + }) + .collect::>(); + snapshots.sort_by(|left, right| { + left.bucket + .cmp(&right.bucket) + .then_with(|| left.drive.cmp(&right.drive)) + .then_with(|| left.result.cmp(&right.result)) + }); + snapshots + } + + fn scanner_bucket_drive_result_counter_snapshots(&self) -> Vec { + Self::scanner_bucket_drive_result_snapshots(self.scanner_bucket_drive_result_counts()) + } + + fn current_cycle_bucket_drive_result_snapshots(&self) -> Vec { + let current = self.scanner_bucket_drive_result_counts(); + let start = self + .current_scan_cycle_bucket_drive_results_start + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .clone(); + Self::scanner_bucket_drive_result_snapshots(current.into_iter().filter_map(|(key, count)| { + let delta = count.saturating_sub(start.get(&key).copied().unwrap_or_default()); + (delta > 0).then_some((key, delta)) + })) + } + + pub fn scanner_runtime_details_report(&self) -> ScannerRuntimeDetailsReport { + self.scanner_runtime_details_report_for_active(self.current_scan_cycle_work_active.load(Ordering::Acquire)) + } + + fn scanner_runtime_details_report_for_active(&self, current_cycle_active: bool) -> ScannerRuntimeDetailsReport { + let current_cycle_bucket_drive_results = if current_cycle_active { + self.current_cycle_bucket_drive_result_snapshots() + } else { + Vec::new() + }; + ScannerRuntimeDetailsReport { + disk_bucket_scan_states: self.scanner_disk_bucket_scan_state_snapshots(), + bucket_drive_results: self.scanner_bucket_drive_result_counter_snapshots(), + current_cycle_bucket_drive_results, + last_cycle_bucket_drive_results: self + .last_scan_cycle_bucket_drive_results + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .clone(), + } + } + + fn scanner_disk_bucket_scan_state_snapshots(&self) -> Vec { + let mut disk_bucket_scan_states = match self.scanner_disk_bucket_scan_states.lock() { + Ok(states) => states + .iter() + .map(|((pool, set), state)| ScannerDiskBucketScanSnapshot { + pool: pool.clone(), + set: set.clone(), + concurrency_limit: state.concurrency_limit, + queued: state.queued, + active: state.active, + }) + .collect::>(), + Err(poisoned) => poisoned + .into_inner() + .iter() + .map(|((pool, set), state)| ScannerDiskBucketScanSnapshot { + pool: pool.clone(), + set: set.clone(), + concurrency_limit: state.concurrency_limit, + queued: state.queued, + active: state.active, + }) + .collect::>(), + }; + disk_bucket_scan_states.sort_by(|left, right| left.pool.cmp(&right.pool).then_with(|| left.set.cmp(&right.set))); + disk_bucket_scan_states + } + fn scanner_source_work_values(&self) -> Vec { ScannerWorkSource::all() .iter() @@ -2761,7 +2973,12 @@ impl Metrics { /// Build a full metrics report snapshot. pub async fn report(&self) -> ScannerMetricsReport { + self.report_with_runtime_details().await.0 + } + + pub async fn report_with_runtime_details(&self) -> (ScannerMetricsReport, ScannerRuntimeDetailsReport) { let mut m = ScannerMetricsReport::default(); + let runtime_details; let has_cycle = { let cycle = self.cycle_info.read().await; @@ -2775,6 +2992,7 @@ impl Metrics { }; m.current_cycle_active = self.current_scan_cycle_work_active.load(Ordering::Acquire); if m.current_cycle_active { + // Keep cycle_info before cycle-baseline locks so active scrapes cannot mix two cycle identities. let current_work = self.scan_cycle_work_since(self.current_scan_cycle_work_start()); let current_source_work = self.scanner_source_work_since(&self.current_scan_cycle_source_work_start_values()); let current_replication_repair_work = @@ -2797,6 +3015,7 @@ impl Metrics { m.current_cycle_replication_repair = self.scanner_replication_repair_work_snapshots(¤t_replication_repair_work); } + runtime_details = self.scanner_runtime_details_report_for_active(m.current_cycle_active); has_cycle }; @@ -2826,15 +3045,11 @@ impl Metrics { m.current_set_scan_concurrency_limit = self.scanner_set_scan_concurrency_limit.load(Ordering::Relaxed); m.current_set_scans_queued = self.scanner_set_scans_queued.load(Ordering::Relaxed); m.current_set_scans_active = self.scanner_set_scans_active.load(Ordering::Relaxed); + let disk_bucket_scan_states = self.scanner_disk_bucket_scan_state_snapshots(); let (disk_scan_concurrency_limit, disk_bucket_scans_queued, disk_bucket_scans_active) = - match self.scanner_disk_bucket_scan_states.lock() { - Ok(states) => states.values().fold((0, 0, 0), |acc, state| { - (acc.0 + state.concurrency_limit, acc.1 + state.queued, acc.2 + state.active) - }), - Err(poisoned) => poisoned.into_inner().values().fold((0, 0, 0), |acc, state| { - (acc.0 + state.concurrency_limit, acc.1 + state.queued, acc.2 + state.active) - }), - }; + disk_bucket_scan_states.iter().fold((0, 0, 0), |acc, state| { + (acc.0 + state.concurrency_limit, acc.1 + state.queued, acc.2 + state.active) + }); m.current_disk_scan_concurrency_limit = disk_scan_concurrency_limit; m.current_disk_bucket_scans_queued = disk_bucket_scans_queued; m.current_disk_bucket_scans_active = disk_bucket_scans_active; @@ -3003,7 +3218,7 @@ impl Metrics { m.pacing_pressure = scanner_pacing_pressure(&m); m.maintenance_control = scanner_maintenance_control(&m); - m + (m, runtime_details) } } @@ -4100,6 +4315,83 @@ mod tests { assert_eq!(report.life_time_ops.get("scan_bucket_drive_failure"), Some(&1)); } + #[tokio::test] + async fn report_includes_structured_bucket_drive_results() { + let metrics = Metrics::new(); + metrics.record_scanner_bucket_drive_result("photos", "/data1", "success"); + + let cycle_start = metrics.start_scan_cycle_work(); + metrics.record_scanner_bucket_drive_result("photos", "/data1", "partial"); + + let active_report = metrics.scanner_runtime_details_report(); + assert_eq!( + active_report.current_cycle_bucket_drive_results, + vec![ScannerBucketDriveResultSnapshot { + bucket: "photos".to_string(), + drive: "/data1".to_string(), + result: "partial".to_string(), + count: 1, + }] + ); + + metrics.finish_scan_cycle_work(cycle_start); + + let report = metrics.scanner_runtime_details_report(); + + assert_eq!( + report.bucket_drive_results, + vec![ + ScannerBucketDriveResultSnapshot { + bucket: "photos".to_string(), + drive: "/data1".to_string(), + result: "partial".to_string(), + count: 1, + }, + ScannerBucketDriveResultSnapshot { + bucket: "photos".to_string(), + drive: "/data1".to_string(), + result: "success".to_string(), + count: 1, + }, + ] + ); + assert!(report.current_cycle_bucket_drive_results.is_empty()); + assert_eq!( + report.last_cycle_bucket_drive_results, + vec![ScannerBucketDriveResultSnapshot { + bucket: "photos".to_string(), + drive: "/data1".to_string(), + result: "partial".to_string(), + count: 1, + }] + ); + } + + #[tokio::test] + async fn scanner_bucket_drive_results_are_bounded() { + let metrics = Metrics::new(); + for index in 0..MAX_SCANNER_BUCKET_DRIVE_RESULT_KEYS { + metrics.record_scanner_bucket_drive_result(&format!("bucket-{index}"), "/data1", "success"); + } + metrics.record_scanner_bucket_drive_result("overflow", "/data1", "success"); + + let report = metrics.scanner_runtime_details_report(); + + assert_eq!(report.bucket_drive_results.len(), MAX_SCANNER_BUCKET_DRIVE_RESULT_KEYS); + assert!( + report + .bucket_drive_results + .iter() + .any(|snapshot| snapshot.bucket == "overflow") + ); + assert!( + report + .bucket_drive_results + .iter() + .all(|snapshot| snapshot.bucket != "bucket-0") + ); + } + #[tokio::test] async fn report_includes_usage_freshness_status() { let metrics = Metrics::new(); @@ -4265,9 +4557,10 @@ mod tests { }; let cycle_ten_start = metrics.start_scan_cycle_work_with_cycle(cycle_ten.clone()).await; metrics.operations[Metric::ScanObject as usize].store(1, Ordering::Relaxed); + metrics.record_scanner_bucket_drive_result("cycle-ten", "/data1", "partial"); let paths = metrics.current_paths.write().await; - let mut report = Box::pin(metrics.report()); + let mut report = Box::pin(metrics.report_with_runtime_details()); let waker = std::task::Waker::noop(); let mut context = std::task::Context::from_waker(waker); assert!(report.as_mut().poll(&mut context).is_pending()); @@ -4284,12 +4577,22 @@ mod tests { }) .await; metrics.operations[Metric::ScanObject as usize].store(101, Ordering::Relaxed); + metrics.record_scanner_bucket_drive_result("cycle-eleven", "/data1", "partial"); drop(paths); - let snapshot = report.await; + let (snapshot, runtime_details) = report.await; assert_eq!(snapshot.current_cycle, 10); assert_eq!(snapshot.current_cycle_objects_scanned, 1); + assert_eq!( + runtime_details.current_cycle_bucket_drive_results, + vec![ScannerBucketDriveResultSnapshot { + bucket: "cycle-ten".to_string(), + drive: "/data1".to_string(), + result: "partial".to_string(), + count: 1, + }] + ); metrics .finish_scan_cycle_work_with_cycle(cycle_eleven_start, CurrentCycle::default()) diff --git a/crates/ecstore/src/api/mod.rs b/crates/ecstore/src/api/mod.rs index ce1d5ec3e..952409d1f 100644 --- a/crates/ecstore/src/api/mod.rs +++ b/crates/ecstore/src/api/mod.rs @@ -178,21 +178,21 @@ pub mod bucket { mrf_backlog_observability_snapshot, }; pub use crate::bucket::replication::{ - BucketReplicationResyncStatus, BucketStats, DeleteReplicationConfigSnapshot, DeletedObjectReplicationInfo, - DurableMrfBacklog, DynReplicationPool, MrfOpKind, MrfReplicateEntry, MustReplicateOptions, ObjectOpts, - REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS, - REPLICATE_INCOMING_DELETE, REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS, - REPLICATION_WRITABLE_FIELDS, ReplicateDecision, ReplicateObjectInfo, ReplicationBatchAdmission, ReplicationConfig, - ReplicationConfigurationExt, ReplicationDeleteScheduleInput, ReplicationDeleteStateSource, - ReplicationHealQueueResult, ReplicationObjectBridge, ReplicationObjectIO, ReplicationOperation, ReplicationPoolTrait, - ReplicationPriority, ReplicationQueueAdmission, ReplicationScannerBridge, ReplicationState, ReplicationStats, - ReplicationStatusType, ReplicationStorage, ReplicationTargetValidationError, ReplicationType, ResyncOpts, - ResyncStatusType, RuntimeReplicationTargetBacklog, TargetReplicationResyncStatus, VersionPurgeStatusType, - delete_replication_state_from_config, delete_replication_version_id, get_global_replication_pool, - get_global_replication_stats, init_background_replication, invalid_replication_config_status_field, - read_durable_mrf_backlog, replication_state_to_filemeta, replication_status_to_filemeta, replication_statuses_map, - replication_target_arns, resync_start_conflict_id, should_remove_replication_target, - should_schedule_delete_replication, should_use_existing_delete_replication_info, + BucketReplicationResyncStatus, BucketReplicationStats, BucketStats, DeleteReplicationConfigSnapshot, + DeletedObjectReplicationInfo, DurableMrfBacklog, DynReplicationPool, MrfOpKind, MrfReplicateEntry, + MustReplicateOptions, ObjectOpts, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS, + REMOTE_TARGET_WRITABLE_FIELDS, REPLICATE_INCOMING_DELETE, REPLICATION_CAPABILITY_CONTRACT_VERSION, + REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS, ReplicateDecision, ReplicateObjectInfo, + ReplicationBatchAdmission, ReplicationConfig, ReplicationConfigurationExt, ReplicationDeleteScheduleInput, + ReplicationDeleteStateSource, ReplicationHealQueueResult, ReplicationObjectBridge, ReplicationObjectIO, + ReplicationOperation, ReplicationPoolTrait, ReplicationPriority, ReplicationQueueAdmission, ReplicationScannerBridge, + ReplicationState, ReplicationStats, ReplicationStatusType, ReplicationStorage, ReplicationTargetValidationError, + ReplicationType, ResyncOpts, ResyncStatusType, RuntimeReplicationTargetBacklog, TargetReplicationResyncStatus, + VersionPurgeStatusType, delete_replication_state_from_config, delete_replication_version_id, + get_global_replication_pool, get_global_replication_stats, init_background_replication, + invalid_replication_config_status_field, read_durable_mrf_backlog, replication_state_to_filemeta, + replication_status_to_filemeta, replication_statuses_map, replication_target_arns, resync_start_conflict_id, + should_remove_replication_target, should_schedule_delete_replication, should_use_existing_delete_replication_info, should_use_existing_delete_replication_source, unsupported_replication_config_field, validate_replication_config_target_arns, version_purge_status_to_filemeta, }; diff --git a/crates/ecstore/src/bucket/replication/mod.rs b/crates/ecstore/src/bucket/replication/mod.rs index 18c6345a0..faf24b3e4 100644 --- a/crates/ecstore/src/bucket/replication/mod.rs +++ b/crates/ecstore/src/bucket/replication/mod.rs @@ -80,6 +80,6 @@ pub use replication_queue_boundary::{ pub use replication_resync_boundary::{BucketReplicationResyncStatus, ResyncOpts, TargetReplicationResyncStatus}; pub use replication_scanner_bridge::ReplicationScannerBridge; pub use replication_state::{ReplicationStats, RuntimeReplicationTargetBacklog}; -pub use replication_stats_boundary::BucketStats; +pub use replication_stats_boundary::{BucketReplicationStats, BucketStats}; pub use replication_storage_boundary::{ReplicationObjectIO, ReplicationStorage}; pub(crate) use replication_target_config_bridge::ReplicationTargetConfigBridge; diff --git a/crates/ecstore/src/bucket/replication/replication_stats_boundary.rs b/crates/ecstore/src/bucket/replication/replication_stats_boundary.rs index e5bdfc66a..ad3ef737f 100644 --- a/crates/ecstore/src/bucket/replication/replication_stats_boundary.rs +++ b/crates/ecstore/src/bucket/replication/replication_stats_boundary.rs @@ -12,10 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -pub use rustfs_replication::BucketStats; #[cfg(test)] pub(crate) use rustfs_replication::FailStats; pub(crate) use rustfs_replication::{ - ActiveWorkerStat, BucketReplicationStat, BucketReplicationStats, InQueueMetric, ProxyMetric, ProxyStatsCache, QueueCache, - ReplicationMetricScope, SRMetricsSummary, XferStats, + ActiveWorkerStat, BucketReplicationStat, InQueueMetric, ProxyMetric, ProxyStatsCache, QueueCache, ReplicationMetricScope, + SRMetricsSummary, XferStats, }; +pub use rustfs_replication::{BucketReplicationStats, BucketStats}; diff --git a/crates/ecstore/src/disk/disk_store.rs b/crates/ecstore/src/disk/disk_store.rs index 29c76765d..fb8d43299 100644 --- a/crates/ecstore/src/disk/disk_store.rs +++ b/crates/ecstore/src/disk/disk_store.rs @@ -27,15 +27,17 @@ use crate::runtime::sources as runtime_sources; use bytes::Bytes; use metrics::counter; use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo}; +use rustfs_madmin::{info_commands::DiskMetrics, metrics::TimedAction}; #[cfg(not(test))] use std::sync::OnceLock; use std::{ + collections::HashMap, path::PathBuf, sync::{ - Arc, + Arc, LazyLock, RwLock as StdRwLock, atomic::{AtomicI64, AtomicU32, AtomicU64, Ordering}, }, - time::{Duration, SystemTime, UNIX_EPOCH}, + time::{Duration, Instant, SystemTime, UNIX_EPOCH}, }; use tokio::{sync::RwLock, time}; use tokio_util::sync::CancellationToken; @@ -57,6 +59,13 @@ enum TimeoutHealthAction { IgnoreFailure, } +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum DiskMetricMutation { + None, + Write, + Delete, +} + #[derive(Clone, Copy, Debug, Eq, PartialEq)] enum TimeoutHealthPolicy { MarkFailure, @@ -105,6 +114,56 @@ static DRIVE_TIMEOUT_PROFILE_CACHE: OnceLock = OnceLock::ne #[cfg(not(test))] static DRIVE_TIMEOUT_HEALTH_POLICY_CACHE: OnceLock = OnceLock::new(); +const DISK_OPERATION_NAMES: &[&str] = &[ + "read_metadata", + "disk_info", + "make_volume", + "make_volumes", + "list_volumes", + "stat_volume", + "delete_volume", + "walk_dir", + "delete_version", + "delete_versions", + "delete_paths", + "acquire_snapshot_lease", + "release_snapshot_lease", + "renew_snapshot_lease", + "delete_data_dir", + "write_metadata", + "update_metadata", + "read_version", + "read_xl", + "rename_data", + "list_dir", + "read_file", + "read_file_stream", + "read_file_mmap_copy", + "read_file_mmap_copy_with_metrics", + "append_file", + "create_file", + "rename_file", + "rename_part", + "prepare_part_transaction", + "settle_part_transaction", + "delete", + "verify_file", + "check_parts", + "read_parts", + "read_multiple", + "write_all", + "read_all", +]; + +static DISK_OPERATION_INDEX: LazyLock> = LazyLock::new(|| { + DISK_OPERATION_NAMES + .iter() + .copied() + .enumerate() + .map(|(index, name)| (name, index)) + .collect() +}); + lazy_static::lazy_static! { static ref TEST_DATA: Bytes = Bytes::from(vec![42u8; 2048]); static ref TEST_BUCKET: String = ".rustfs.sys/tmp".to_string(); @@ -326,6 +385,286 @@ pub struct DiskHealthTracker { pub last_capacity_probe_unix_secs: AtomicI64, } +#[derive(Debug)] +pub(crate) struct DiskHealthMetricEpoch { + /// Preallocated per-operation metrics for built-in disk operation names. + operation_metrics: Box<[DiskOperationMetricEntry]>, + /// Fallback for tests or future extension operations outside DISK_OPERATION_NAMES. + fallback_operation_metrics: StdRwLock>>, + /// Caller API operations currently executing through the disk health wrapper. + api_waiting: AtomicU32, + /// Operations rejected because the disk was unavailable for the caller. + total_errors_availability: AtomicU64, + /// Operations that timed out in the disk health wrapper. + total_errors_timeout: AtomicU64, + /// Completed disk write mutations. + total_writes: AtomicU64, + /// Completed disk delete mutations. + total_deletes: AtomicU64, +} + +impl Default for DiskHealthMetricEpoch { + fn default() -> Self { + Self { + operation_metrics: DISK_OPERATION_NAMES + .iter() + .copied() + .map(|name| DiskOperationMetricEntry { + name, + metrics: DiskOperationMetrics::default(), + }) + .collect::>() + .into_boxed_slice(), + fallback_operation_metrics: StdRwLock::new(HashMap::new()), + api_waiting: AtomicU32::new(0), + total_errors_availability: AtomicU64::new(0), + total_errors_timeout: AtomicU64::new(0), + total_writes: AtomicU64::new(0), + total_deletes: AtomicU64::new(0), + } + } +} + +impl DiskHealthMetricEpoch { + fn fallback_operation_metrics_read( + &self, + ) -> std::sync::RwLockReadGuard<'_, HashMap<&'static str, Arc>> { + self.fallback_operation_metrics + .read() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + } + + fn fallback_operation_metrics_write( + &self, + ) -> std::sync::RwLockWriteGuard<'_, HashMap<&'static str, Arc>> { + self.fallback_operation_metrics + .write() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + } + + fn predefined_operation_metric(&self, op: &'static str) -> Option<&DiskOperationMetrics> { + DISK_OPERATION_INDEX + .get(op) + .and_then(|index| self.operation_metrics.get(*index)) + .map(|entry| &entry.metrics) + } + + fn fallback_operation_metric(&self, op: &'static str) -> Arc { + if let Some(metrics) = self.fallback_operation_metrics_read().get(op).cloned() { + return metrics; + } + + let mut operation_metrics = self.fallback_operation_metrics_write(); + operation_metrics.entry(op).or_default().clone() + } + + fn record_operation_call(&self, op: &'static str) { + if let Some(metrics) = self.predefined_operation_metric(op) { + metrics.record_call_atomic(); + } else { + self.fallback_operation_metric(op).record_call_atomic(); + } + } + + fn record_operation_latency(&self, op: &'static str, elapsed: Duration) { + let now_sec = current_unix_secs(); + if let Some(metrics) = self.predefined_operation_metric(op) { + metrics.record_latency_atomic(now_sec, elapsed); + } else { + self.fallback_operation_metric(op).record_latency_atomic(now_sec, elapsed); + } + } + + fn record_availability_error(&self) { + self.total_errors_availability.fetch_add(1, Ordering::Relaxed); + } + + fn record_timeout_error(&self) { + self.total_errors_timeout.fetch_add(1, Ordering::Relaxed); + } + + fn record_mutation_success(&self, mutation: DiskMetricMutation) { + match mutation { + DiskMetricMutation::None => {} + DiskMetricMutation::Write => { + self.total_writes.fetch_add(1, Ordering::Relaxed); + } + DiskMetricMutation::Delete => { + self.total_deletes.fetch_add(1, Ordering::Relaxed); + } + } + } + + fn waiting_guard(&self) -> DiskMetricWaitingGuard<'_> { + self.api_waiting.fetch_add(1, Ordering::Relaxed); + DiskMetricWaitingGuard { metrics: self } + } + + fn waiting_count(&self) -> u32 { + self.api_waiting.load(Ordering::Relaxed) + } + + fn metrics_snapshot(&self) -> DiskMetrics { + let now_sec = current_unix_secs(); + let fallback_operation_metrics = self.fallback_operation_metrics_read(); + let mut last_minute = HashMap::with_capacity(self.operation_metrics.len() + fallback_operation_metrics.len()); + let mut api_calls = HashMap::with_capacity(self.operation_metrics.len() + fallback_operation_metrics.len()); + for entry in self.operation_metrics.iter() { + Self::insert_operation_snapshot(entry.name, &entry.metrics, now_sec, &mut last_minute, &mut api_calls); + } + for (op, action) in fallback_operation_metrics.iter() { + Self::insert_operation_snapshot(op, action, now_sec, &mut last_minute, &mut api_calls); + } + + DiskMetrics { + last_minute, + api_calls, + total_waiting: self.waiting_count(), + total_errors_availability: self.total_errors_availability.load(Ordering::Relaxed), + total_errors_timeout: self.total_errors_timeout.load(Ordering::Relaxed), + total_writes: self.total_writes.load(Ordering::Relaxed), + total_deletes: self.total_deletes.load(Ordering::Relaxed), + } + } + + fn insert_operation_snapshot( + op: &str, + action: &DiskOperationMetrics, + now_sec: u64, + last_minute: &mut HashMap, + api_calls: &mut HashMap, + ) { + let lifetime_calls = action.lifetime_calls.load(Ordering::Relaxed); + if lifetime_calls > 0 { + last_minute.insert(op.to_string(), action.last_minute_snapshot(now_sec)); + api_calls.insert(op.to_string(), lifetime_calls); + } + } +} + +struct DiskMetricWaitingGuard<'a> { + metrics: &'a DiskHealthMetricEpoch, +} + +impl Drop for DiskMetricWaitingGuard<'_> { + fn drop(&mut self) { + self.metrics.api_waiting.fetch_sub(1, Ordering::Relaxed); + } +} + +#[derive(Clone, Debug)] +pub(crate) struct ReconnectDiskHealthState { + pub(crate) health: Arc, + pub(crate) metrics: Arc, +} + +#[derive(Debug)] +struct DiskOperationMetricEntry { + name: &'static str, + metrics: DiskOperationMetrics, +} + +#[derive(Debug)] +struct TimedActionSlot { + version: AtomicU64, + unix_sec: AtomicU64, + count: AtomicU64, + acc_time: AtomicU64, +} + +impl Default for TimedActionSlot { + fn default() -> Self { + Self { + version: AtomicU64::new(0), + unix_sec: AtomicU64::new(0), + count: AtomicU64::new(0), + acc_time: AtomicU64::new(0), + } + } +} + +#[derive(Debug)] +struct DiskOperationMetrics { + lifetime_calls: AtomicU64, + last_minute: Box<[TimedActionSlot]>, +} + +impl Default for DiskOperationMetrics { + fn default() -> Self { + Self { + lifetime_calls: AtomicU64::new(0), + last_minute: std::iter::repeat_with(TimedActionSlot::default) + .take(60) + .collect::>() + .into_boxed_slice(), + } + } +} + +impl DiskOperationMetrics { + fn record_call(&mut self) { + self.lifetime_calls.fetch_add(1, Ordering::Relaxed); + } + + fn record_latency(&mut self, now_sec: u64, elapsed: Duration) { + self.record_latency_atomic(now_sec, elapsed); + } + + fn record(&mut self, now_sec: u64, elapsed: Duration) { + self.record_call(); + self.record_latency(now_sec, elapsed); + } + + fn record_call_atomic(&self) { + self.lifetime_calls.fetch_add(1, Ordering::Relaxed); + } + + fn record_latency_atomic(&self, now_sec: u64, elapsed: Duration) { + let elapsed_nanos = u64::try_from(elapsed.as_nanos()).unwrap_or(u64::MAX); + let slot = &self.last_minute[(now_sec % 60) as usize]; + loop { + let version = slot.version.load(Ordering::Acquire); + if !version.is_multiple_of(2) { + std::hint::spin_loop(); + continue; + } + if slot + .version + .compare_exchange(version, version.wrapping_add(1), Ordering::AcqRel, Ordering::Acquire) + .is_ok() + { + if slot.unix_sec.load(Ordering::Acquire) != now_sec { + slot.count.store(0, Ordering::Relaxed); + slot.acc_time.store(0, Ordering::Relaxed); + slot.unix_sec.store(now_sec, Ordering::Release); + } + slot.count.fetch_add(1, Ordering::Relaxed); + slot.acc_time.fetch_add(elapsed_nanos, Ordering::Relaxed); + slot.version.store(version.wrapping_add(2), Ordering::Release); + break; + } + } + } + + fn last_minute_snapshot(&self, now_sec: u64) -> TimedAction { + let mut snapshot = TimedAction::default(); + for slot in &self.last_minute { + let version = slot.version.load(Ordering::Acquire); + if !version.is_multiple_of(2) { + continue; + } + let slot_sec = slot.unix_sec.load(Ordering::Acquire); + let count = slot.count.load(Ordering::Acquire); + let acc_time = slot.acc_time.load(Ordering::Acquire); + if slot.version.load(Ordering::Acquire) == version && slot_sec <= now_sec && now_sec.saturating_sub(slot_sec) < 60 { + snapshot.count = snapshot.count.saturating_add(count); + snapshot.acc_time = snapshot.acc_time.saturating_add(acc_time); + } + } + snapshot + } +} + pub(crate) struct DiskHealthWaitingGuard<'a> { health: &'a DiskHealthTracker, } @@ -372,6 +711,10 @@ impl DiskHealthTracker { .store(current_unix_secs() as i64, Ordering::Release); } + pub(crate) fn metric_epoch_for_reconnect(&self) -> Self { + Self::new() + } + pub fn last_capacity_snapshot(&self) -> Option<(u64, u64, u64, u64)> { let ts = self.last_capacity_probe_unix_secs.load(Ordering::Acquire); if ts <= 0 { @@ -671,6 +1014,8 @@ pub struct LocalDiskWrapper { disk: Arc, /// Health tracker health: Arc, + /// Internal metrics epoch preserved across local disk reconnects. + metrics: Arc, /// Whether health checking is enabled health_check: bool, /// Cancellation token for monitoring tasks @@ -684,6 +1029,36 @@ pub struct LocalDiskWrapper { impl LocalDiskWrapper { /// Create a new LocalDiskWrapper pub fn new(disk: Arc, health_check: bool) -> Self { + Self::new_with_health_and_metrics( + disk, + health_check, + Arc::new(DiskHealthTracker::new()), + Arc::new(DiskHealthMetricEpoch::default()), + ) + } + + pub(crate) fn new_with_health(disk: Arc, health_check: bool, health: Arc) -> Self { + Self::new_with_health_and_metrics(disk, health_check, health, Arc::new(DiskHealthMetricEpoch::default())) + } + + pub(crate) fn new_with_reconnect_state( + disk: Arc, + health_check: bool, + reconnect: Option, + ) -> Self { + let reconnect = reconnect.unwrap_or_else(|| ReconnectDiskHealthState { + health: Arc::new(DiskHealthTracker::new()), + metrics: Arc::new(DiskHealthMetricEpoch::default()), + }); + Self::new_with_health_and_metrics(disk, health_check, reconnect.health, reconnect.metrics) + } + + fn new_with_health_and_metrics( + disk: Arc, + health_check: bool, + health: Arc, + metrics: Arc, + ) -> Self { // Check environment variable for health check override. // Only enable if both param and env are true. let env_health_check = @@ -691,7 +1066,8 @@ impl LocalDiskWrapper { let wrapper = Self { disk, - health: Arc::new(DiskHealthTracker::new()), + health, + metrics, health_check: health_check && env_health_check, cancel_token: CancellationToken::new(), disk_id: Arc::new(RwLock::new(None)), @@ -701,6 +1077,13 @@ impl LocalDiskWrapper { wrapper } + pub(crate) fn health_tracker_epoch_for_reconnect(&self) -> ReconnectDiskHealthState { + ReconnectDiskHealthState { + health: Arc::new(self.health.metric_epoch_for_reconnect()), + metrics: self.metrics.clone(), + } + } + pub fn get_disk(&self) -> Arc { self.disk.clone() } @@ -1053,6 +1436,38 @@ impl LocalDiskWrapper { *self.disk_id.write().await = id; } + pub(crate) fn metrics_snapshot(&self) -> DiskMetrics { + self.metrics.metrics_snapshot() + } + + fn record_result_error_metrics(&self, result: &Result) { + match result { + Err(DiskError::Timeout) => self.metrics.record_timeout_error(), + Err(DiskError::FaultyDisk | DiskError::FaultyRemoteDisk | DiskError::DiskNotFound) => { + self.metrics.record_availability_error(); + } + _ => {} + } + } + + fn record_batch_delete_error_metrics(&self, result: &[Option]) { + let mut saw_timeout = false; + let mut saw_availability = false; + for error in result.iter().flatten() { + match error { + DiskError::Timeout => saw_timeout = true, + DiskError::FaultyDisk | DiskError::FaultyRemoteDisk | DiskError::DiskNotFound => saw_availability = true, + _ => {} + } + } + if saw_timeout { + self.metrics.record_timeout_error(); + } + if saw_availability { + self.metrics.record_availability_error(); + } + } + /// Get the current disk ID pub async fn get_current_disk_id(&self) -> Option { *self.disk_id.read().await @@ -1068,6 +1483,27 @@ impl LocalDiskWrapper { self.track_disk_health_with_op("unknown", operation, timeout_duration).await } + async fn track_disk_health_mutation( + &self, + op: &'static str, + mutation: DiskMetricMutation, + operation: F, + timeout_duration: Duration, + ) -> Result + where + F: FnOnce() -> Fut, + Fut: std::future::Future>, + { + self.track_disk_health_with_op_timeout_action_and_mutation( + op, + operation, + timeout_duration, + TimeoutHealthAction::MarkFailure, + mutation, + ) + .await + } + pub async fn track_disk_health_with_op( &self, op: &'static str, @@ -1089,12 +1525,35 @@ impl LocalDiskWrapper { timeout_duration: Duration, timeout_health_action: TimeoutHealthAction, ) -> Result + where + F: FnOnce() -> Fut, + Fut: std::future::Future>, + { + self.track_disk_health_with_op_timeout_action_and_mutation( + op, + operation, + timeout_duration, + timeout_health_action, + DiskMetricMutation::None, + ) + .await + } + + async fn track_disk_health_with_op_timeout_action_and_mutation( + &self, + op: &'static str, + operation: F, + timeout_duration: Duration, + timeout_health_action: TimeoutHealthAction, + mutation: DiskMetricMutation, + ) -> Result where F: FnOnce() -> Fut, Fut: std::future::Future>, { // Check if disk is faulty if self.health.is_faulty() { + self.metrics.record_availability_error(); warn!( event = EVENT_DISK_HEALTH_CHECK_FAILED, component = LOG_COMPONENT_ECSTORE, @@ -1107,16 +1566,25 @@ impl LocalDiskWrapper { } // Check if disk is stale - self.check_disk_stale().await?; + if let Err(err) = self.check_disk_stale().await { + self.metrics.record_availability_error(); + return Err(err); + } // Record operation start self.health.last_started.store(current_unix_nanos(), Ordering::Relaxed); let _waiting_guard = self.health.waiting_guard(); + let _metric_waiting_guard = self.metrics.waiting_guard(); + self.metrics.record_operation_call(op); + let started = Instant::now(); if timeout_duration == Duration::ZERO { let result = operation().await; + self.metrics.record_operation_latency(op, started.elapsed()); + self.record_result_error_metrics(&result); if result.is_ok() { self.health.record_operation_success(&self.endpoint(), "operation_success"); + self.metrics.record_mutation_success(mutation); } return result; } @@ -1125,13 +1593,18 @@ impl LocalDiskWrapper { match result { Ok(operation_result) => { + self.metrics.record_operation_latency(op, started.elapsed()); + self.record_result_error_metrics(&operation_result); // Log success; the waiting guard balances every exit path. if operation_result.is_ok() { self.health.record_operation_success(&self.endpoint(), "operation_success"); + self.metrics.record_mutation_success(mutation); } operation_result } Err(_) => { + self.metrics.record_operation_latency(op, started.elapsed()); + self.metrics.record_timeout_error(); // Timeout occurred, mark disk as potentially faulty. if timeout_health_action == TimeoutHealthAction::MarkFailure && self.health.mark_failure(&self.endpoint(), "operation_timeout") @@ -1231,46 +1704,67 @@ impl DiskAPI for LocalDiskWrapper { async fn disk_info(&self, opts: &DiskInfoOptions) -> Result { if opts.noop && opts.metrics { - let mut info = DiskInfo::default(); - // Add health metrics - info.metrics.total_waiting = self.health.waiting_count(); + let info = DiskInfo { + metrics: self.metrics_snapshot(), + ..Default::default() + }; if self.health.is_faulty() { + self.metrics.record_availability_error(); return Err(DiskError::FaultyDisk); } return Ok(info); } if self.health.is_faulty() { + self.metrics.record_availability_error(); return Err(DiskError::FaultyDisk); } - self.track_disk_health_with_op_and_timeout_action( - "disk_info", - || async { - let result = self.disk.disk_info(opts).await?; + let result = self + .track_disk_health_with_op_and_timeout_action( + "disk_info", + || async { + let result = self.disk.disk_info(opts).await?; - if let Some(current_disk_id) = *self.disk_id.read().await - && Some(current_disk_id) != result.id - { - return Err(DiskError::DiskNotFound); - }; + if let Some(current_disk_id) = *self.disk_id.read().await + && Some(current_disk_id) != result.id + { + return Err(DiskError::DiskNotFound); + }; - Ok(result) - }, - get_drive_disk_info_timeout(), - self.scanner_timeout_health_action(), + Ok(result) + }, + get_drive_disk_info_timeout(), + self.scanner_timeout_health_action(), + ) + .await; + + result.map(|mut info| { + if opts.metrics { + info.metrics = self.metrics_snapshot(); + } + info + }) + } + + async fn make_volume(&self, volume: &str) -> Result<()> { + self.track_disk_health_mutation( + "make_volume", + DiskMetricMutation::Write, + || async { self.disk.make_volume(volume).await }, + get_max_timeout_duration(), ) .await } - async fn make_volume(&self, volume: &str) -> Result<()> { - self.track_disk_health(|| async { self.disk.make_volume(volume).await }, get_max_timeout_duration()) - .await - } - async fn make_volumes(&self, volumes: Vec<&str>) -> Result<()> { - self.track_disk_health(|| async { self.disk.make_volumes(volumes).await }, get_max_timeout_duration()) - .await + self.track_disk_health_mutation( + "make_volumes", + DiskMetricMutation::Write, + || async { self.disk.make_volumes(volumes).await }, + get_max_timeout_duration(), + ) + .await } async fn list_volumes(&self) -> Result> { @@ -1279,13 +1773,22 @@ impl DiskAPI for LocalDiskWrapper { } async fn stat_volume(&self, volume: &str) -> Result { - self.track_disk_health(|| async { self.disk.stat_volume(volume).await }, get_max_timeout_duration()) - .await + self.track_disk_health_with_op( + "stat_volume", + || async { self.disk.stat_volume(volume).await }, + get_max_timeout_duration(), + ) + .await } async fn delete_volume(&self, volume: &str, force_delete: bool) -> Result<()> { - self.track_disk_health(|| async { self.disk.delete_volume(volume, force_delete).await }, Duration::ZERO) - .await + self.track_disk_health_mutation( + "delete_volume", + DiskMetricMutation::Delete, + || async { self.disk.delete_volume(volume, force_delete).await }, + Duration::ZERO, + ) + .await } async fn walk_dir(&self, opts: WalkDirOptions, wr: &mut W) -> Result<()> { @@ -1313,7 +1816,9 @@ impl DiskAPI for LocalDiskWrapper { force_del_marker: bool, opts: DeleteOptions, ) -> Result<()> { - self.track_disk_health( + self.track_disk_health_mutation( + "delete_version", + DiskMetricMutation::Delete, || async { self.disk.delete_version(volume, path, fi, force_del_marker, opts).await }, get_max_timeout_duration(), ) @@ -1323,38 +1828,53 @@ impl DiskAPI for LocalDiskWrapper { async fn delete_versions(&self, volume: &str, versions: Vec, opts: DeleteOptions) -> Vec> { // Check if disk is faulty before proceeding if self.health.is_faulty() { + self.metrics.record_availability_error(); return vec![Some(DiskError::FaultyDisk); versions.len()]; } // Check if disk is stale if let Err(e) = self.check_disk_stale().await { + self.metrics.record_availability_error(); return vec![Some(e); versions.len()]; } // Record operation start self.health.last_started.store(current_unix_nanos(), Ordering::Relaxed); self.health.increment_waiting(); + let metric_waiting_guard = self.metrics.waiting_guard(); + self.metrics.record_operation_call("delete_versions"); + let started = Instant::now(); // Execute the operation let result = self.disk.delete_versions(volume, versions, opts).await; + self.metrics.record_operation_latency("delete_versions", started.elapsed()); + self.record_batch_delete_error_metrics(&result); self.health.decrement_waiting(); + drop(metric_waiting_guard); let has_err = result.iter().any(|e| e.is_some()); if !has_err { // Log success and decrement waiting counter self.health.record_operation_success(&self.endpoint(), "operation_success"); + self.metrics.record_mutation_success(DiskMetricMutation::Delete); } result } async fn delete_paths(&self, volume: &str, paths: &[String]) -> Result<()> { - self.track_disk_health(|| async { self.disk.delete_paths(volume, paths).await }, get_max_timeout_duration()) - .await + self.track_disk_health_mutation( + "delete_paths", + DiskMetricMutation::Delete, + || async { self.disk.delete_paths(volume, paths).await }, + get_max_timeout_duration(), + ) + .await } async fn acquire_snapshot_lease(&self, volume: &str, path: &str) -> Result { - self.track_disk_health( + self.track_disk_health_with_op( + "acquire_snapshot_lease", || async { self.disk.acquire_snapshot_lease(volume, path).await }, get_max_timeout_duration(), ) @@ -1362,7 +1882,8 @@ impl DiskAPI for LocalDiskWrapper { } async fn release_snapshot_lease(&self, volume: &str, path: &str, token: SnapshotLeaseToken) -> Result<()> { - self.track_disk_health( + self.track_disk_health_with_op( + "release_snapshot_lease", || async { self.disk.release_snapshot_lease(volume, path, token).await }, get_max_timeout_duration(), ) @@ -1370,7 +1891,8 @@ impl DiskAPI for LocalDiskWrapper { } async fn renew_snapshot_lease(&self, volume: &str, path: &str, token: SnapshotLeaseToken) -> Result { - self.track_disk_health( + self.track_disk_health_with_op( + "renew_snapshot_lease", || async { self.disk.renew_snapshot_lease(volume, path, token).await }, get_max_timeout_duration(), ) @@ -1378,7 +1900,9 @@ impl DiskAPI for LocalDiskWrapper { } async fn delete_data_dir(&self, volume: &str, path: &str, opts: DeleteOptions) -> Result { - self.track_disk_health( + self.track_disk_health_mutation( + "delete_data_dir", + DiskMetricMutation::Delete, || async { self.disk.delete_data_dir(volume, path, opts).await }, get_max_timeout_duration(), ) @@ -1386,7 +1910,9 @@ impl DiskAPI for LocalDiskWrapper { } async fn write_metadata(&self, org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()> { - self.track_disk_health( + self.track_disk_health_mutation( + "write_metadata", + DiskMetricMutation::Write, || async { self.disk.write_metadata(org_volume, volume, path, fi).await }, get_max_timeout_duration(), ) @@ -1394,7 +1920,9 @@ impl DiskAPI for LocalDiskWrapper { } async fn update_metadata(&self, volume: &str, path: &str, fi: FileInfo, opts: &UpdateMetadataOpts) -> Result<()> { - self.track_disk_health( + self.track_disk_health_mutation( + "update_metadata", + DiskMetricMutation::Write, || async { self.disk.update_metadata(volume, path, fi, opts).await }, get_max_timeout_duration(), ) @@ -1409,7 +1937,8 @@ impl DiskAPI for LocalDiskWrapper { version_id: &str, opts: &ReadOptions, ) -> Result { - self.track_disk_health( + self.track_disk_health_with_op( + "read_version", || async { self.disk.read_version(org_volume, volume, path, version_id, opts).await }, get_max_timeout_duration(), ) @@ -1417,8 +1946,12 @@ impl DiskAPI for LocalDiskWrapper { } async fn read_xl(&self, volume: &str, path: &str, read_data: bool) -> Result { - self.track_disk_health(|| async { self.disk.read_xl(volume, path, read_data).await }, get_max_timeout_duration()) - .await + self.track_disk_health_with_op( + "read_xl", + || async { self.disk.read_xl(volume, path, read_data).await }, + get_max_timeout_duration(), + ) + .await } async fn rename_data( @@ -1429,8 +1962,9 @@ impl DiskAPI for LocalDiskWrapper { dst_volume: &str, dst_path: &str, ) -> Result { - self.track_disk_health_with_op( + self.track_disk_health_mutation( "rename_data", + DiskMetricMutation::Write, || async { self.disk.rename_data(src_volume, src_path, fi, dst_volume, dst_path).await }, get_max_timeout_duration(), ) @@ -1448,12 +1982,17 @@ impl DiskAPI for LocalDiskWrapper { } async fn read_file(&self, volume: &str, path: &str) -> Result { - self.track_disk_health(|| async { self.disk.read_file(volume, path).await }, get_max_timeout_duration()) - .await + self.track_disk_health_with_op( + "read_file", + || async { self.disk.read_file(volume, path).await }, + get_max_timeout_duration(), + ) + .await } async fn read_file_stream(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result { - self.track_disk_health( + self.track_disk_health_with_op( + "read_file_stream", || async { self.disk.read_file_stream(volume, path, offset, length).await }, get_max_timeout_duration(), ) @@ -1461,7 +2000,8 @@ impl DiskAPI for LocalDiskWrapper { } async fn read_file_mmap_copy(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result { - self.track_disk_health( + self.track_disk_health_with_op( + "read_file_mmap_copy", || async { self.disk.read_file_mmap_copy(volume, path, offset, length).await }, get_max_timeout_duration(), ) @@ -1476,7 +2016,8 @@ impl DiskAPI for LocalDiskWrapper { length: usize, metrics: Option, ) -> Result { - self.track_disk_health( + self.track_disk_health_with_op( + "read_file_mmap_copy_with_metrics", || async { self.disk .read_file_mmap_copy_with_metrics(volume, path, offset, length, metrics) @@ -1488,12 +2029,14 @@ impl DiskAPI for LocalDiskWrapper { } async fn append_file(&self, volume: &str, path: &str) -> Result { - self.track_disk_health(|| async { self.disk.append_file(volume, path).await }, Duration::ZERO) + self.track_disk_health_with_op("append_file", || async { self.disk.append_file(volume, path).await }, Duration::ZERO) .await } async fn create_file(&self, origvolume: &str, volume: &str, path: &str, file_size: i64) -> Result { - self.track_disk_health( + self.track_disk_health_mutation( + "create_file", + DiskMetricMutation::Write, || async { self.disk.create_file(origvolume, volume, path, file_size).await }, Duration::ZERO, ) @@ -1501,7 +2044,9 @@ impl DiskAPI for LocalDiskWrapper { } async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()> { - self.track_disk_health( + self.track_disk_health_mutation( + "rename_file", + DiskMetricMutation::Write, || async { self.disk.rename_file(src_volume, src_path, dst_volume, dst_path).await }, get_max_timeout_duration(), ) @@ -1509,7 +2054,9 @@ impl DiskAPI for LocalDiskWrapper { } async fn rename_part(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str, meta: Bytes) -> Result<()> { - self.track_disk_health( + self.track_disk_health_mutation( + "rename_part", + DiskMetricMutation::Write, || async { self.disk.rename_part(src_volume, src_path, dst_volume, dst_path, meta).await }, get_max_timeout_duration(), ) @@ -1524,7 +2071,9 @@ impl DiskAPI for LocalDiskWrapper { dst_path: &str, meta: Bytes, ) -> Result<()> { - self.track_disk_health( + self.track_disk_health_mutation( + "prepare_part_transaction", + DiskMetricMutation::Write, || async { self.disk .prepare_part_transaction(src_volume, src_path, dst_volume, dst_path, meta) @@ -1536,7 +2085,9 @@ impl DiskAPI for LocalDiskWrapper { } async fn settle_part_transaction(&self, volume: &str, path: &str, action: crate::disk::PartTransactionAction) -> Result<()> { - self.track_disk_health( + self.track_disk_health_mutation( + "settle_part_transaction", + DiskMetricMutation::Write, || async { self.disk.settle_part_transaction(volume, path, action).await }, get_max_timeout_duration(), ) @@ -1544,38 +2095,52 @@ impl DiskAPI for LocalDiskWrapper { } async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()> { - self.track_disk_health(|| async { self.disk.delete(volume, path, opt).await }, get_max_timeout_duration()) - .await + self.track_disk_health_mutation( + "delete", + DiskMetricMutation::Delete, + || async { self.disk.delete(volume, path, opt).await }, + get_max_timeout_duration(), + ) + .await } async fn verify_file(&self, volume: &str, path: &str, fi: &FileInfo) -> Result { - self.track_disk_health(|| async { self.disk.verify_file(volume, path, fi).await }, Duration::ZERO) + self.track_disk_health_with_op("verify_file", || async { self.disk.verify_file(volume, path, fi).await }, Duration::ZERO) .await } async fn check_parts(&self, volume: &str, path: &str, fi: &FileInfo) -> Result { - self.track_disk_health(|| async { self.disk.check_parts(volume, path, fi).await }, Duration::ZERO) + self.track_disk_health_with_op("check_parts", || async { self.disk.check_parts(volume, path, fi).await }, Duration::ZERO) .await } async fn read_parts(&self, bucket: &str, paths: &[String]) -> Result> { - self.track_disk_health(|| async { self.disk.read_parts(bucket, paths).await }, Duration::ZERO) + self.track_disk_health_with_op("read_parts", || async { self.disk.read_parts(bucket, paths).await }, Duration::ZERO) .await } async fn read_multiple(&self, req: ReadMultipleReq) -> Result> { - self.track_disk_health(|| async { self.disk.read_multiple(req).await }, Duration::ZERO) + self.track_disk_health_with_op("read_multiple", || async { self.disk.read_multiple(req).await }, Duration::ZERO) .await } async fn write_all(&self, volume: &str, path: &str, data: Bytes) -> Result<()> { - self.track_disk_health(|| async { self.disk.write_all(volume, path, data).await }, get_max_timeout_duration()) - .await + self.track_disk_health_mutation( + "write_all", + DiskMetricMutation::Write, + || async { self.disk.write_all(volume, path, data).await }, + get_max_timeout_duration(), + ) + .await } async fn read_all(&self, volume: &str, path: &str) -> Result { - self.track_disk_health(|| async { self.disk.read_all(volume, path).await }, get_max_timeout_duration()) - .await + self.track_disk_health_with_op( + "read_all", + || async { self.disk.read_all(volume, path).await }, + get_max_timeout_duration(), + ) + .await } } @@ -1586,6 +2151,7 @@ mod tests { use crate::disk::health_state::RuntimeDriveHealthState; use std::{ io, + panic::{AssertUnwindSafe, catch_unwind}, pin::Pin, task::{Context, Poll}, }; @@ -1603,6 +2169,106 @@ mod tests { assert_eq!(health.waiting_count(), 0); } + #[test] + fn disk_operation_metrics_keep_lifetime_calls_separate_from_last_minute_latency() { + let mut metrics = DiskOperationMetrics::default(); + metrics.record(10, Duration::from_micros(5)); + metrics.record(69, Duration::from_micros(7)); + metrics.record(70, Duration::from_micros(11)); + + assert_eq!(metrics.lifetime_calls.load(Ordering::Relaxed), 3); + + let window = metrics.last_minute_snapshot(70); + assert_eq!(window.count, 2); + assert_eq!(window.acc_time, 18_000); + } + + #[test] + fn disk_health_metrics_snapshot_exports_waiting_errors_and_operation_windows() { + let metrics = DiskHealthMetricEpoch::default(); + metrics.record_operation_call("read_all"); + metrics.record_operation_latency("read_all", Duration::from_micros(13)); + metrics.record_availability_error(); + metrics.record_timeout_error(); + { + let _guard = metrics.waiting_guard(); + let snapshot = metrics.metrics_snapshot(); + assert_eq!(snapshot.total_waiting, 1); + assert_eq!(snapshot.total_errors_availability, 1); + assert_eq!(snapshot.total_errors_timeout, 1); + assert_eq!(snapshot.api_calls.get("read_all"), Some(&1)); + assert_eq!(snapshot.last_minute.get("read_all").map(|action| action.count), Some(1)); + } + } + + #[test] + fn disk_health_metrics_snapshot_excludes_recovery_monitor_waiting() { + let health = DiskHealthTracker::new(); + let metrics = DiskHealthMetricEpoch::default(); + + health.increment_waiting(); + let snapshot = metrics.metrics_snapshot(); + + assert_eq!(health.waiting_count(), 1); + assert_eq!(snapshot.total_waiting, 0); + } + + #[test] + fn disk_health_metrics_snapshot_keeps_expired_operation_windows() { + let epoch = DiskHealthMetricEpoch::default(); + let metrics = epoch + .predefined_operation_metric("read_all") + .expect("read_all should have a preallocated metrics slot"); + metrics.record_call_atomic(); + metrics.record_latency_atomic(current_unix_secs().saturating_sub(60), Duration::from_micros(13)); + + let snapshot = epoch.metrics_snapshot(); + + assert_eq!(snapshot.api_calls.get("read_all"), Some(&1)); + assert_eq!(snapshot.last_minute.get("read_all").map(|action| action.count), Some(0)); + } + + #[test] + fn disk_metric_epoch_recovers_poisoned_map_lock() { + let epoch = DiskHealthMetricEpoch::default(); + let panic_result = catch_unwind(AssertUnwindSafe(|| { + let _guard = epoch + .fallback_operation_metrics + .write() + .expect("test should lock fallback operation metrics"); + panic!("poison disk operation metrics lock"); + })); + + assert!(panic_result.is_err()); + epoch.record_operation_call("custom_test_op"); + epoch.record_operation_latency("custom_test_op", Duration::from_micros(13)); + let snapshot = epoch.metrics_snapshot(); + + assert_eq!(snapshot.api_calls.get("custom_test_op"), Some(&1)); + } + + #[test] + fn reconnect_health_tracker_shares_metric_epoch_without_health_state() { + let health = DiskHealthTracker::new(); + let metrics = Arc::new(DiskHealthMetricEpoch::default()); + metrics.record_operation_call("read_all"); + metrics.record_availability_error(); + health.set_faulty(); + + let reconnect = ReconnectDiskHealthState { + health: Arc::new(health.metric_epoch_for_reconnect()), + metrics: metrics.clone(), + }; + metrics.record_operation_call("read_all"); + reconnect.metrics.record_timeout_error(); + + assert!(!reconnect.health.is_faulty()); + let snapshot = reconnect.metrics.metrics_snapshot(); + assert_eq!(snapshot.api_calls.get("read_all"), Some(&2)); + assert_eq!(snapshot.total_errors_availability, 1); + assert_eq!(snapshot.total_errors_timeout, 1); + } + #[tokio::test] async fn local_disk_health_wrapper_balances_task_cancellation() { let dir = tempfile::tempdir().expect("temp dir should be created"); @@ -1613,7 +2279,11 @@ mod tests { let task_wrapper = Arc::clone(&wrapper); let task = tokio::spawn(async move { task_wrapper - .track_disk_health(|| async { std::future::pending::>().await }, Duration::ZERO) + .track_disk_health_with_op( + "test_pending", + || async { std::future::pending::>().await }, + Duration::ZERO, + ) .await }); @@ -1628,6 +2298,151 @@ mod tests { let _ = task.await; assert_eq!(wrapper.health.waiting_count(), 0); + assert_eq!(wrapper.metrics_snapshot().api_calls.get("test_pending"), Some(&1)); + } + + #[tokio::test] + async fn local_disk_health_wrapper_preserves_legacy_call_shape() { + let dir = tempfile::tempdir().expect("temp dir should be created"); + let endpoint = + Endpoint::try_from(dir.path().to_str().expect("temp dir should be valid UTF-8")).expect("endpoint should parse"); + let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk should be created")); + let wrapper = LocalDiskWrapper::new(disk, false); + + wrapper + .track_disk_health(|| async { Ok(()) }, Duration::ZERO) + .await + .expect("legacy health wrapper call should succeed"); + + assert_eq!(wrapper.metrics_snapshot().api_calls.get("unknown"), Some(&1)); + } + + #[tokio::test] + async fn local_disk_health_wrapper_counts_returned_availability_errors() { + let dir = tempfile::tempdir().expect("temp dir should be created"); + let endpoint = + Endpoint::try_from(dir.path().to_str().expect("temp dir should be valid UTF-8")).expect("endpoint should parse"); + let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk should be created")); + let wrapper = LocalDiskWrapper::new(disk, false); + + let err = wrapper + .track_disk_health_with_op("read_all", || async { Err::<(), DiskError>(DiskError::DiskNotFound) }, Duration::ZERO) + .await + .expect_err("returned availability error should propagate"); + + assert_eq!(err, DiskError::DiskNotFound); + assert_eq!(wrapper.metrics_snapshot().total_errors_availability, 1); + } + + #[tokio::test] + async fn delete_versions_counts_returned_batch_error_classes() { + let dir = tempfile::tempdir().expect("temp dir should be created"); + let endpoint = + Endpoint::try_from(dir.path().to_str().expect("temp dir should be valid UTF-8")).expect("endpoint should parse"); + let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk should be created")); + let wrapper = LocalDiskWrapper::new(disk, false); + + wrapper.record_batch_delete_error_metrics(&[ + Some(DiskError::DiskNotFound), + Some(DiskError::FaultyDisk), + Some(DiskError::Timeout), + None, + ]); + let snapshot = wrapper.metrics_snapshot(); + + assert_eq!(snapshot.total_errors_availability, 1); + assert_eq!(snapshot.total_errors_timeout, 1); + } + + #[tokio::test] + async fn local_disk_metrics_count_successful_write_and_delete_mutations() { + let dir = tempfile::tempdir().expect("temp dir should be created"); + let endpoint = + Endpoint::try_from(dir.path().to_str().expect("temp dir should be valid UTF-8")).expect("endpoint should parse"); + let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk should be created")); + let wrapper = LocalDiskWrapper::new(disk, false); + + wrapper.make_volume("bucket").await.expect("volume should be created"); + wrapper + .write_all("bucket", "object", Bytes::from_static(b"data")) + .await + .expect("object should be written"); + wrapper + .delete("bucket", "object", DeleteOptions::default()) + .await + .expect("object should be deleted"); + + let snapshot = wrapper.metrics_snapshot(); + assert_eq!(snapshot.total_writes, 2); + assert_eq!(snapshot.total_deletes, 1); + } + + #[tokio::test] + async fn delete_versions_counts_faulty_drive_availability_rejection() { + let dir = tempfile::tempdir().expect("temp dir should be created"); + let endpoint = + Endpoint::try_from(dir.path().to_str().expect("temp dir should be valid UTF-8")).expect("endpoint should parse"); + let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk should be created")); + let wrapper = LocalDiskWrapper::new(disk, false); + wrapper.health.force_runtime_state_for_test(RuntimeDriveHealthState::Offline); + + let result = wrapper + .delete_versions("bucket", vec![FileInfoVersions::default()], DeleteOptions::default()) + .await; + + assert_eq!(result.len(), 1); + assert!(matches!(result.first(), Some(Some(DiskError::FaultyDisk)))); + assert_eq!(wrapper.metrics_snapshot().total_errors_availability, 1); + } + + #[tokio::test] + async fn disk_info_counts_faulty_noop_metrics_rejection() { + let dir = tempfile::tempdir().expect("temp dir should be created"); + let endpoint = + Endpoint::try_from(dir.path().to_str().expect("temp dir should be valid UTF-8")).expect("endpoint should parse"); + let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk should be created")); + let wrapper = LocalDiskWrapper::new(disk, false); + wrapper.health.force_runtime_state_for_test(RuntimeDriveHealthState::Offline); + + let err = wrapper + .disk_info(&DiskInfoOptions { + noop: true, + metrics: true, + ..Default::default() + }) + .await + .expect_err("faulty disk_info should be rejected"); + + assert_eq!(err, DiskError::FaultyDisk); + assert_eq!(wrapper.metrics_snapshot().total_errors_availability, 1); + } + + #[tokio::test] + async fn delete_versions_counts_stale_drive_availability_rejection() { + let dir = tempfile::tempdir().expect("temp dir should be created"); + let endpoint = + Endpoint::try_from(dir.path().to_str().expect("temp dir should be valid UTF-8")).expect("endpoint should parse"); + let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk should be created")); + { + let mut format_info = disk.format_info.write().await; + format_info.id = Some(Uuid::new_v4()); + format_info.file_info = Some( + tokio::fs::metadata(dir.path()) + .await + .expect("temp dir metadata should be readable"), + ); + format_info.last_check = Some(::time::OffsetDateTime::now_utc()); + } + let wrapper = LocalDiskWrapper::new(disk, false); + wrapper.set_disk_id_state(Some(Uuid::new_v4())).await; + + let result = wrapper + .delete_versions("bucket", vec![FileInfoVersions::default()], DeleteOptions::default()) + .await; + + assert_eq!(result.len(), 1); + assert!(matches!(result.first(), Some(Some(DiskError::DiskNotFound)))); + assert_eq!(wrapper.metrics_snapshot().total_errors_availability, 1); } impl AsyncWrite for PendingWriter { @@ -2228,6 +3043,7 @@ mod tests { assert_eq!(walk_err, DiskError::Timeout); assert_eq!(wrapper.runtime_state(), RuntimeDriveHealthState::Online); assert!(!wrapper.health.is_faulty()); + assert_eq!(wrapper.metrics_snapshot().total_errors_timeout, 1); let info = wrapper .stat_volume(bucket) diff --git a/crates/ecstore/src/disk/mod.rs b/crates/ecstore/src/disk/mod.rs index a8074ddd9..8d9909a17 100644 --- a/crates/ecstore/src/disk/mod.rs +++ b/crates/ecstore/src/disk/mod.rs @@ -142,6 +142,13 @@ impl Disk { Disk::Remote(remote_disk) => remote_disk.set_disk_id(id).await, } } + + pub(crate) fn local_health_tracker_epoch_for_reconnect(&self) -> Option { + match self { + Disk::Local(local_disk) => Some(local_disk.health_tracker_epoch_for_reconnect()), + Disk::Remote(_) => None, + } + } } #[async_trait::async_trait] @@ -606,6 +613,13 @@ impl Disk { } } + pub fn metrics_snapshot(&self) -> Option { + match self { + Disk::Local(local_disk) => Some(local_disk.metrics_snapshot()), + Disk::Remote(_) => None, + } + } + #[cfg(test)] pub fn health_check_enabled_for_test(&self) -> bool { match self { @@ -664,9 +678,21 @@ impl Disk { } pub async fn new_disk(ep: &Endpoint, opt: &DiskOption) -> Result { + new_disk_with_health_tracker(ep, opt, None).await +} + +pub(crate) async fn new_disk_with_health_tracker( + ep: &Endpoint, + opt: &DiskOption, + reconnect: Option, +) -> Result { if ep.is_local { let s = LocalDisk::new(ep, opt.cleanup).await?; - Ok(Arc::new(Disk::Local(Box::new(LocalDiskWrapper::new(Arc::new(s), opt.health_check))))) + Ok(Arc::new(Disk::Local(Box::new(LocalDiskWrapper::new_with_reconnect_state( + Arc::new(s), + opt.health_check, + reconnect, + ))))) } else { let data_transport = build_internode_data_transport_from_env(); let remote_disk = RemoteDisk::new(ep, opt, data_transport?).await?; diff --git a/crates/ecstore/src/set_disk/mod.rs b/crates/ecstore/src/set_disk/mod.rs index 2a6e3895a..3356cad93 100644 --- a/crates/ecstore/src/set_disk/mod.rs +++ b/crates/ecstore/src/set_disk/mod.rs @@ -4359,7 +4359,13 @@ async fn get_disks_info(disks: &[Option], eps: &[Endpoint]) -> Vec { disk.record_capacity_probe(res.total, res.used, res.free); ret.push(rustfs_madmin::Disk { @@ -4390,6 +4396,7 @@ async fn get_disks_info(disks: &[Option], eps: &[Endpoint]) -> Vec], eps: &[Endpoint]) -> Vec], eps: &[Endpoint]) -> Vec rustfs_madmin::Disk { let mut disk = rustfs_madmin::Disk { endpoint: endpoint.to_string(), + drive_path: endpoint.get_file_path(), local: endpoint.is_local, pool_index: endpoint.pool_idx, set_index: endpoint.set_idx, @@ -7709,14 +7718,42 @@ mod tests { assert_eq!(info[0].state, "ok"); assert_eq!(info[0].runtime_state.as_deref(), Some("online")); assert!(!info[0].drive_path.is_empty(), "online disk should keep immediate disk_info probe"); + assert!( + info[0] + .metrics + .as_ref() + .and_then(|metrics| metrics.api_calls.get("disk_info")) + .copied() + .unwrap_or_default() + > 0, + "online disk should expose disk_info operation metrics" + ); assert_eq!(info[1].state, "ok"); assert_eq!(info[1].runtime_state.as_deref(), Some("suspect")); assert!(!info[1].drive_path.is_empty(), "suspect disk should still probe for fresher disk info"); + assert!( + info[1] + .metrics + .as_ref() + .and_then(|metrics| metrics.last_minute.get("disk_info")) + .map(|action| action.count) + .unwrap_or_default() + > 0, + "suspect disk should expose last-minute disk_info latency" + ); assert_eq!(info[2].state, "offline"); assert_eq!(info[2].runtime_state.as_deref(), Some("offline")); - assert!(info[2].drive_path.is_empty(), "offline disk should use runtime snapshot fallback"); + assert_eq!( + info[2].drive_path, + endpoints[2].get_file_path(), + "offline disk should keep stable endpoint path" + ); + assert!( + info[2].metrics.is_some(), + "offline runtime fallback should preserve disk metrics snapshot" + ); } #[tokio::test] diff --git a/crates/ecstore/src/set_disk/ops/locking.rs b/crates/ecstore/src/set_disk/ops/locking.rs index 6d842633b..7b120723a 100644 --- a/crates/ecstore/src/set_disk/ops/locking.rs +++ b/crates/ecstore/src/set_disk/ops/locking.rs @@ -322,7 +322,16 @@ impl SetDisks { pub async fn renew_disk(&self, ep: &Endpoint) { debug!("renew_disk: start {:?}", ep); - let (new_disk, fm) = match Self::connect_endpoint(ep).await { + let previous_health = { + let disks = self.disks.read().await; + disks + .iter() + .filter_map(|disk| disk.as_ref()) + .find(|disk| disk.endpoint() == *ep) + .and_then(|disk| disk.local_health_tracker_epoch_for_reconnect()) + }; + + let (new_disk, fm) = match Self::connect_endpoint(ep, previous_health).await { Ok(res) => res, Err(e) => { warn!("renew_disk: connect_endpoint err {:?}", &e); @@ -400,13 +409,17 @@ impl SetDisks { Err(Error::other("DriveID: not found")) } - pub(in crate::set_disk) async fn connect_endpoint(ep: &Endpoint) -> disk::error::Result<(DiskStore, FormatV3)> { - let disk = new_disk( + pub(in crate::set_disk) async fn connect_endpoint( + ep: &Endpoint, + reconnect: Option, + ) -> disk::error::Result<(DiskStore, FormatV3)> { + let disk = crate::disk::new_disk_with_health_tracker( ep, &DiskOption { cleanup: false, health_check: true, }, + reconnect, ) .await?; @@ -714,6 +727,25 @@ mod tests { renewed_disk.health_check_enabled_for_test(), "renewed disks must keep health monitoring enabled so later faulty marks can recover" ); + renewed_disk + .disk_info(&DiskInfoOptions::default()) + .await + .expect("renewed disk_info should record a drive API metric"); + renewed_disk.force_runtime_state_for_test(disk::health_state::RuntimeDriveHealthState::Offline); + + set_disks.renew_disk(&endpoints[0]).await; + + let disks = set_disks.get_disks_internal().await; + let renewed_again = disks[0] + .as_ref() + .expect("second renew_disk should keep the recovered disk attached"); + assert_eq!( + renewed_again + .metrics_snapshot() + .and_then(|metrics| metrics.api_calls.get("disk_info").copied()), + Some(1), + "disk reconnect must preserve the local drive metrics tracker epoch" + ); drop(temp_dirs); } diff --git a/crates/io-metrics/src/lib.rs b/crates/io-metrics/src/lib.rs index 994777b49..4414291cb 100644 --- a/crates/io-metrics/src/lib.rs +++ b/crates/io-metrics/src/lib.rs @@ -246,7 +246,7 @@ pub use process_lock_metrics::{ record_write_lock_held_acquire, record_write_lock_held_release, snapshot_process_lock_counts, snapshot_process_lock_events, snapshot_process_platform_stats, }; -pub use s3_api_metrics::{init_s3_metrics, record_s3_op}; +pub use s3_api_metrics::{S3OperationMetricSnapshot, init_s3_metrics, record_s3_op, s3_op_metrics_snapshot}; pub use sampler::{ ProcessResourceSnapshot, ProcessSampler, ProcessStatusSnapshot, ProcessSystemSnapshot, snapshot_process_platform, snapshot_process_resource, snapshot_process_resource_and_system, snapshot_process_resource_and_system_with, diff --git a/crates/io-metrics/src/s3_api_metrics.rs b/crates/io-metrics/src/s3_api_metrics.rs index 251d76312..e9da74a9a 100644 --- a/crates/io-metrics/src/s3_api_metrics.rs +++ b/crates/io-metrics/src/s3_api_metrics.rs @@ -14,8 +14,25 @@ use rustfs_s3_ops::S3Operation; use std::sync::OnceLock; +use std::sync::atomic::{AtomicU64, Ordering}; const S3_OPS_METRIC: &str = "rustfs_s3_operations_total"; +static S3_OP_COUNTERS: OnceLock> = OnceLock::new(); + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct S3OperationMetricSnapshot { + pub op: &'static str, + pub total: u64, +} + +fn s3_op_counters() -> &'static [AtomicU64] { + S3_OP_COUNTERS.get_or_init(|| { + std::iter::repeat_with(|| AtomicU64::new(0)) + .take(S3Operation::ALL.len()) + .collect::>() + .into_boxed_slice() + }) +} /// Record a handled S3 API operation. /// @@ -26,9 +43,22 @@ const S3_OPS_METRIC: &str = "rustfs_s3_operations_total"; /// This mirrors MinIO, which never labels its default operation counters with /// bucket. The `op` dimension is bounded (<= 122 variants). pub fn record_s3_op(op: S3Operation) { + if let Some(counter) = s3_op_counters().get(op.metric_index()) { + counter.fetch_add(1, Ordering::Relaxed); + } counter!(S3_OPS_METRIC, "op" => op.as_str()).increment(1); } +pub fn s3_op_metrics_snapshot() -> Vec { + S3Operation::ALL + .iter() + .filter_map(|op| { + let total = s3_op_counters().get(op.metric_index())?.load(Ordering::Relaxed); + (total > 0).then_some(S3OperationMetricSnapshot { op: op.as_str(), total }) + }) + .collect() +} + pub fn init_s3_metrics() { static METRICS_DESC_INIT: OnceLock<()> = OnceLock::new(); METRICS_DESC_INIT.get_or_init(|| { @@ -42,6 +72,9 @@ mod tests { use metrics::with_local_recorder; use metrics_util::debugging::DebuggingRecorder; use std::collections::HashSet; + use std::sync::Mutex; + + static S3_OP_TEST_LOCK: Mutex<()> = Mutex::new(()); /// Collect the label-key sets recorded against `rustfs_s3_operations_total`. fn ops_metric_label_key_sets(recorder: &DebuggingRecorder) -> Vec> { @@ -60,6 +93,7 @@ mod tests { #[test] fn record_s3_op_labels_by_op_only_no_bucket() { + let _guard = S3_OP_TEST_LOCK.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); let recorder = DebuggingRecorder::new(); let label_key_sets = ops_metric_label_key_sets(&recorder); @@ -75,6 +109,7 @@ mod tests { #[test] fn record_s3_op_cardinality_bounded_by_distinct_ops() { + let _guard = S3_OP_TEST_LOCK.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); let recorder = DebuggingRecorder::new(); let snapshotter = recorder.snapshotter(); @@ -113,4 +148,24 @@ mod tests { "series count must equal the number of distinct ops, never the bucket count" ); } + + #[test] + fn s3_op_metrics_snapshot_reports_recorded_totals() { + let _guard = S3_OP_TEST_LOCK.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + let before = s3_op_metrics_snapshot() + .into_iter() + .find(|snapshot| snapshot.op == S3Operation::GetObject.as_str()) + .map(|snapshot| snapshot.total) + .unwrap_or_default(); + + record_s3_op(S3Operation::GetObject); + record_s3_op(S3Operation::GetObject); + + let after = s3_op_metrics_snapshot() + .into_iter() + .find(|snapshot| snapshot.op == S3Operation::GetObject.as_str()) + .map(|snapshot| snapshot.total) + .expect("GetObject snapshot should be present after recording"); + assert_eq!(after, before + 2); + } } diff --git a/crates/madmin/src/info_commands.rs b/crates/madmin/src/info_commands.rs index 2c2295166..ee4572f04 100644 --- a/crates/madmin/src/info_commands.rs +++ b/crates/madmin/src/info_commands.rs @@ -14,7 +14,8 @@ use std::{collections::HashMap, time::SystemTime}; -use serde::{Deserialize, Serialize}; +use serde::ser::SerializeStruct; +use serde::{Deserialize, Serialize, Serializer}; use time::OffsetDateTime; use crate::metrics::TimedAction; @@ -61,17 +62,41 @@ impl ItemState { } } -#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)] +#[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq)] pub struct DiskMetrics { + #[serde(rename = "lastMinute", alias = "last_minute")] pub last_minute: HashMap, + #[serde(rename = "apiCalls", alias = "api_calls")] pub api_calls: HashMap, + #[serde(rename = "totalWaiting", alias = "total_waiting")] pub total_waiting: u32, + #[serde(rename = "totalErrsAvailability", alias = "total_errors_availability")] pub total_errors_availability: u64, + #[serde(rename = "totalErrsTimeout", alias = "total_errors_timeout")] pub total_errors_timeout: u64, + #[serde(rename = "totalWrites", alias = "total_writes")] pub total_writes: u64, + #[serde(rename = "totalDeletes", alias = "total_deletes")] pub total_deletes: u64, } +impl Serialize for DiskMetrics { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let mut state = serializer.serialize_struct("DiskMetrics", 7)?; + state.serialize_field("last_minute", &self.last_minute)?; + state.serialize_field("api_calls", &self.api_calls)?; + state.serialize_field("total_waiting", &self.total_waiting)?; + state.serialize_field("total_errors_availability", &self.total_errors_availability)?; + state.serialize_field("total_errors_timeout", &self.total_errors_timeout)?; + state.serialize_field("total_writes", &self.total_writes)?; + state.serialize_field("total_deletes", &self.total_deletes)?; + state.end() + } +} + #[derive(Serialize, Deserialize, Debug, Default, Clone)] pub struct Disk { pub endpoint: String, @@ -432,6 +457,17 @@ mod tests { disk_index: i32, } + #[derive(Deserialize)] + struct LegacyDiskMetricsCompat { + last_minute: HashMap, + api_calls: HashMap, + total_waiting: u32, + total_errors_availability: u64, + total_errors_timeout: u64, + total_writes: u64, + total_deletes: u64, + } + #[test] fn test_item_state_to_string() { assert_eq!(ItemState::Offline.to_string(), ITEM_OFFLINE); @@ -495,6 +531,56 @@ mod tests { assert_eq!(metrics.total_deletes, 50); } + #[test] + fn test_disk_metrics_json_preserves_internode_legacy_fields() { + let metrics = DiskMetrics { + total_waiting: 5, + total_errors_availability: 2, + total_errors_timeout: 1, + total_writes: 1000, + total_deletes: 50, + ..Default::default() + }; + + let json = serde_json::to_value(metrics).expect("disk metrics should serialize"); + + assert!(json.get("last_minute").is_some()); + assert!(json.get("api_calls").is_some()); + assert_eq!(json["total_waiting"], serde_json::json!(5)); + assert_eq!(json["total_errors_availability"], serde_json::json!(2)); + assert_eq!(json["total_errors_timeout"], serde_json::json!(1)); + assert_eq!(json["total_writes"], serde_json::json!(1000)); + assert_eq!(json["total_deletes"], serde_json::json!(50)); + assert!(json.get("lastMinute").is_none()); + assert!(json.get("totalErrsTimeout").is_none()); + } + + #[test] + fn test_disk_metrics_msgpack_uses_internode_legacy_fields() { + let metrics = DiskMetrics { + total_waiting: 5, + total_errors_availability: 2, + total_errors_timeout: 1, + total_writes: 1000, + total_deletes: 50, + ..Default::default() + }; + + let mut encoded = Vec::new(); + metrics + .serialize(&mut Serializer::new(&mut encoded).with_struct_map()) + .expect("disk metrics should encode as named msgpack"); + + let decoded: LegacyDiskMetricsCompat = rmp_serde::from_slice(&encoded).expect("legacy disk metrics should decode"); + assert!(decoded.last_minute.is_empty()); + assert!(decoded.api_calls.is_empty()); + assert_eq!(decoded.total_waiting, 5); + assert_eq!(decoded.total_errors_availability, 2); + assert_eq!(decoded.total_errors_timeout, 1); + assert_eq!(decoded.total_writes, 1000); + assert_eq!(decoded.total_deletes, 50); + } + #[test] fn test_disk_default() { let disk = Disk::default(); diff --git a/crates/obs/src/metrics/collectors/audit.rs b/crates/obs/src/metrics/collectors/audit.rs index 0f7e1f600..eb6d75f30 100644 --- a/crates/obs/src/metrics/collectors/audit.rs +++ b/crates/obs/src/metrics/collectors/audit.rs @@ -41,6 +41,13 @@ pub struct AuditTargetStats { pub total_messages: u64, } +/// Audit target statistics with runtime-local node identity. +#[derive(Debug, Clone, Default)] +pub(crate) struct AuditTargetRuntimeStats { + pub(crate) server: String, + pub(crate) target: AuditTargetStats, +} + /// Collects audit metrics from the provided audit target statistics. /// /// Uses the metric descriptors from `metrics_type::audit` module. @@ -56,19 +63,53 @@ pub fn collect_audit_metrics(stats: &[AuditTargetStats]) -> Vec Vec { + let legacy_stats = stats.iter().map(|stat| stat.target.clone()).collect::>(); + let mut metrics = collect_audit_metrics(&legacy_stats); + metrics.reserve(stats.len() * 4); + + for stat in stats.iter().filter(|stat| !stat.server.is_empty()) { + let server_label: Cow<'static, str> = Cow::Owned(stat.server.clone()); + let target_id_label: Cow<'static, str> = Cow::Owned(stat.target.target_id.clone()); + + metrics.push( + PrometheusMetric::from_descriptor(&AUDIT_FAILED_MESSAGES_BY_SERVER_MD, stat.target.failed_messages as f64) + .with_label(SERVER, server_label.clone()) + .with_label(TARGET_ID, target_id_label.clone()), + ); + metrics.push( + PrometheusMetric::from_descriptor(&AUDIT_FAILED_STORE_LENGTH_BY_SERVER_MD, stat.target.failed_store_length as f64) + .with_label(SERVER, server_label.clone()) + .with_label(TARGET_ID, target_id_label.clone()), + ); + metrics.push( + PrometheusMetric::from_descriptor(&AUDIT_TARGET_QUEUE_LENGTH_BY_SERVER_MD, stat.target.queue_length as f64) + .with_label(SERVER, server_label.clone()) + .with_label(TARGET_ID, target_id_label.clone()), + ); + metrics.push( + PrometheusMetric::from_descriptor(&AUDIT_TOTAL_MESSAGES_BY_SERVER_MD, stat.target.total_messages as f64) + .with_label(SERVER, server_label) + .with_label(TARGET_ID, target_id_label), ); } @@ -101,7 +142,7 @@ mod tests { let metrics = collect_audit_metrics(&stats); - assert_eq!(metrics.len(), 8); // 2 targets * 4 metrics each + assert_eq!(metrics.len(), 8); let failed = metrics .iter() @@ -114,6 +155,37 @@ mod tests { && m.labels.iter().any(|(k, v)| *k == "target_id" && v == "target-1") }); assert!(failed_store.is_some()); + + assert!( + metrics + .iter() + .all(|m| m.name != AUDIT_TARGET_QUEUE_LENGTH_BY_SERVER_MD.get_full_metric_name()) + ); + } + + #[test] + fn collect_audit_runtime_metrics_adds_server_dimensions() { + let stats = vec![AuditTargetRuntimeStats { + server: "node1:9000".to_string(), + target: AuditTargetStats { + target_id: "target-1".to_string(), + failed_messages: 5, + failed_store_length: 3, + queue_length: 10, + total_messages: 1000, + }, + }]; + + let metrics = collect_audit_runtime_metrics(&stats); + + assert_eq!(metrics.len(), 8); + let server_queue = metrics.iter().find(|m| { + m.value == 10.0 + && m.name == AUDIT_TARGET_QUEUE_LENGTH_BY_SERVER_MD.get_full_metric_name() + && m.labels.iter().any(|(k, v)| *k == SERVER && v == "node1:9000") + && m.labels.iter().any(|(k, v)| *k == TARGET_ID && v == "target-1") + }); + assert!(server_queue.is_some()); } #[test] diff --git a/crates/obs/src/metrics/collectors/bucket_replication.rs b/crates/obs/src/metrics/collectors/bucket_replication.rs index a3b764451..a393c8e38 100644 --- a/crates/obs/src/metrics/collectors/bucket_replication.rs +++ b/crates/obs/src/metrics/collectors/bucket_replication.rs @@ -30,14 +30,18 @@ use crate::metrics::schema::bucket_replication::{ BUCKET_REPL_PROXIED_HEAD_REQUESTS_FAILURES_MD, BUCKET_REPL_PROXIED_HEAD_REQUESTS_TOTAL_MD, BUCKET_REPL_PROXIED_PUT_REQUESTS_FAILURES_MD, BUCKET_REPL_PROXIED_PUT_REQUESTS_TOTAL_MD, BUCKET_REPL_PROXIED_PUT_TAGGING_REQUESTS_FAILURES_MD, BUCKET_REPL_PROXIED_PUT_TAGGING_REQUESTS_TOTAL_MD, - BUCKET_REPL_RESYNC_CANCELED_TOTAL_MD, BUCKET_REPL_RESYNC_COMPLETED_TOTAL_MD, BUCKET_REPL_RESYNC_DURATION_MS_TOTAL_MD, - BUCKET_REPL_RESYNC_FAILED_TOTAL_MD, BUCKET_REPL_RESYNC_STARTED_TOTAL_MD, BUCKET_REPL_SENT_BYTES_MD, - BUCKET_REPL_SENT_COUNT_MD, BUCKET_REPL_TOTAL_FAILED_BYTES_MD, BUCKET_REPL_TOTAL_FAILED_COUNT_MD, OPERATION_L, RANGE_L, - TARGET_ARN_L, + BUCKET_REPL_PROXY_REQUESTS_TOTAL_MD, BUCKET_REPL_RESYNC_CANCELED_TOTAL_MD, BUCKET_REPL_RESYNC_COMPLETED_TOTAL_MD, + BUCKET_REPL_RESYNC_DURATION_MS_TOTAL_MD, BUCKET_REPL_RESYNC_FAILED_TOTAL_MD, BUCKET_REPL_RESYNC_STARTED_TOTAL_MD, + BUCKET_REPL_SENT_BYTES_MD, BUCKET_REPL_SENT_COUNT_MD, BUCKET_REPL_TARGET_LAST_HOUR_FAILED_BYTES_MD, + BUCKET_REPL_TARGET_LAST_HOUR_FAILED_COUNT_MD, BUCKET_REPL_TARGET_LAST_MIN_FAILED_BYTES_MD, + BUCKET_REPL_TARGET_LAST_MIN_FAILED_COUNT_MD, BUCKET_REPL_TARGET_SENT_BYTES_MD, BUCKET_REPL_TARGET_SENT_COUNT_MD, + BUCKET_REPL_TARGET_TOTAL_FAILED_BYTES_MD, BUCKET_REPL_TARGET_TOTAL_FAILED_COUNT_MD, BUCKET_REPL_TOTAL_FAILED_BYTES_MD, + BUCKET_REPL_TOTAL_FAILED_COUNT_MD, OPERATION_L, RANGE_L, RESULT_L, TARGET_ARN_L, }; use std::borrow::Cow; -const BASE_BUCKET_REPLICATION_METRICS_PER_BUCKET: usize = 25; +const BASE_BUCKET_REPLICATION_METRICS_PER_BUCKET: usize = 37; +const BUCKET_REPLICATION_RUNTIME_FLOW_METRICS_PER_TARGET: usize = 8; const BASE_BUCKET_REPLICATION_BACKLOG_METRICS_PER_BUCKET: usize = 11; const BUCKET_REPLICATION_BACKLOG_METRICS_PER_TARGET: usize = 4; @@ -49,6 +53,19 @@ pub struct BucketReplicationTargetStats { pub latency_ms: f64, } +#[derive(Debug, Clone, Default)] +pub(crate) struct BucketReplicationTargetFlowStats { + pub(crate) target_arn: String, + pub sent_bytes: u64, + pub sent_count: u64, + pub total_failed_bytes: u64, + pub total_failed_count: u64, + pub last_min_failed_bytes: u64, + pub last_min_failed_count: u64, + pub last_hour_failed_bytes: u64, + pub last_hour_failed_count: u64, +} + #[derive(Debug, Clone, Default)] pub struct BucketReplicationBandwidthStats { pub bucket: String, @@ -88,6 +105,12 @@ pub struct BucketReplicationStats { pub targets: Vec, } +#[derive(Debug, Clone, Default)] +pub(crate) struct BucketReplicationRuntimeStats { + pub(crate) stats: BucketReplicationStats, + pub(crate) target_flows: Vec, +} + #[derive(Debug, Clone, Default)] pub(crate) struct BucketReplicationBacklogStats { pub(crate) bucket: String, @@ -140,6 +163,25 @@ pub fn collect_bucket_replication_bandwidth_metrics(stats: &[BucketReplicationBa metrics } +fn push_proxy_request_result_metrics( + metrics: &mut Vec, + bucket_label: Cow<'static, str>, + operation: &'static str, + total: u64, + failures: u64, +) { + let failure_count = failures.min(total); + let success_count = total.saturating_sub(failure_count); + for (result, value) in [("success", success_count), ("failure", failure_count)] { + metrics.push( + PrometheusMetric::from_descriptor(&BUCKET_REPL_PROXY_REQUESTS_TOTAL_MD, value as f64) + .with_label(BUCKET_L, bucket_label.clone()) + .with_label(OPERATION_L, operation) + .with_label(RESULT_L, result), + ); + } +} + pub fn collect_bucket_replication_metrics(stats: &[BucketReplicationStats]) -> Vec { if stats.is_empty() { return Vec::new(); @@ -263,6 +305,48 @@ pub fn collect_bucket_replication_metrics(stats: &[BucketReplicationStats]) -> V ) .with_label(BUCKET_L, bucket_label.clone()), ); + push_proxy_request_result_metrics( + &mut metrics, + bucket_label.clone(), + "get", + stat.proxied_get_requests_total, + stat.proxied_get_requests_failures, + ); + push_proxy_request_result_metrics( + &mut metrics, + bucket_label.clone(), + "head", + stat.proxied_head_requests_total, + stat.proxied_head_requests_failures, + ); + push_proxy_request_result_metrics( + &mut metrics, + bucket_label.clone(), + "put", + stat.proxied_put_requests_total, + stat.proxied_put_requests_failures, + ); + push_proxy_request_result_metrics( + &mut metrics, + bucket_label.clone(), + "put_tagging", + stat.proxied_put_tagging_requests_total, + stat.proxied_put_tagging_requests_failures, + ); + push_proxy_request_result_metrics( + &mut metrics, + bucket_label.clone(), + "get_tagging", + stat.proxied_get_tagging_requests_total, + stat.proxied_get_tagging_requests_failures, + ); + push_proxy_request_result_metrics( + &mut metrics, + bucket_label.clone(), + "delete_tagging", + stat.proxied_delete_tagging_requests_total, + stat.proxied_delete_tagging_requests_failures, + ); metrics.push( PrometheusMetric::from_descriptor(&BUCKET_REPL_RESYNC_STARTED_TOTAL_MD, stat.resync_started_count as f64) .with_label(BUCKET_L, bucket_label.clone()), @@ -298,6 +382,81 @@ pub fn collect_bucket_replication_metrics(stats: &[BucketReplicationStats]) -> V metrics } +pub(crate) fn collect_bucket_replication_runtime_metrics(stats: &[BucketReplicationRuntimeStats]) -> Vec { + if stats.is_empty() { + return Vec::new(); + } + + let legacy_stats = stats.iter().map(|stat| stat.stats.clone()).collect::>(); + let mut metrics = collect_bucket_replication_metrics(&legacy_stats); + let flow_count = stats + .iter() + .map(|stat| stat.target_flows.len() * BUCKET_REPLICATION_RUNTIME_FLOW_METRICS_PER_TARGET) + .sum(); + metrics.reserve(flow_count); + + for stat in stats { + let bucket_label: Cow<'static, str> = Cow::Owned(stat.stats.bucket.clone()); + for target in &stat.target_flows { + let target_label: Cow<'static, str> = Cow::Owned(target.target_arn.clone()); + metrics.push( + PrometheusMetric::from_descriptor(&BUCKET_REPL_TARGET_SENT_BYTES_MD, target.sent_bytes as f64) + .with_label(BUCKET_L, bucket_label.clone()) + .with_label(TARGET_ARN_L, target_label.clone()), + ); + metrics.push( + PrometheusMetric::from_descriptor(&BUCKET_REPL_TARGET_SENT_COUNT_MD, target.sent_count as f64) + .with_label(BUCKET_L, bucket_label.clone()) + .with_label(TARGET_ARN_L, target_label.clone()), + ); + metrics.push( + PrometheusMetric::from_descriptor(&BUCKET_REPL_TARGET_TOTAL_FAILED_BYTES_MD, target.total_failed_bytes as f64) + .with_label(BUCKET_L, bucket_label.clone()) + .with_label(TARGET_ARN_L, target_label.clone()), + ); + metrics.push( + PrometheusMetric::from_descriptor(&BUCKET_REPL_TARGET_TOTAL_FAILED_COUNT_MD, target.total_failed_count as f64) + .with_label(BUCKET_L, bucket_label.clone()) + .with_label(TARGET_ARN_L, target_label.clone()), + ); + metrics.push( + PrometheusMetric::from_descriptor( + &BUCKET_REPL_TARGET_LAST_MIN_FAILED_BYTES_MD, + target.last_min_failed_bytes as f64, + ) + .with_label(BUCKET_L, bucket_label.clone()) + .with_label(TARGET_ARN_L, target_label.clone()), + ); + metrics.push( + PrometheusMetric::from_descriptor( + &BUCKET_REPL_TARGET_LAST_MIN_FAILED_COUNT_MD, + target.last_min_failed_count as f64, + ) + .with_label(BUCKET_L, bucket_label.clone()) + .with_label(TARGET_ARN_L, target_label.clone()), + ); + metrics.push( + PrometheusMetric::from_descriptor( + &BUCKET_REPL_TARGET_LAST_HOUR_FAILED_BYTES_MD, + target.last_hour_failed_bytes as f64, + ) + .with_label(BUCKET_L, bucket_label.clone()) + .with_label(TARGET_ARN_L, target_label.clone()), + ); + metrics.push( + PrometheusMetric::from_descriptor( + &BUCKET_REPL_TARGET_LAST_HOUR_FAILED_COUNT_MD, + target.last_hour_failed_count as f64, + ) + .with_label(BUCKET_L, bucket_label.clone()) + .with_label(TARGET_ARN_L, target_label.clone()), + ); + } + } + + metrics +} + pub(crate) fn collect_bucket_replication_backlog_metrics(stats: &[BucketReplicationBacklogStats]) -> Vec { if stats.is_empty() { return Vec::new(); @@ -412,43 +571,56 @@ mod tests { #[test] fn test_collect_bucket_replication_metrics() { - let stats = vec![BucketReplicationStats { - bucket: "b1".to_string(), - total_failed_bytes: 64, - total_failed_count: 2, - last_min_failed_bytes: 32, - last_min_failed_count: 1, - last_hour_failed_bytes: 64, - last_hour_failed_count: 2, - sent_bytes: 1024, - sent_count: 8, - proxied_get_requests_total: 5, - proxied_get_requests_failures: 1, - proxied_head_requests_total: 4, - proxied_head_requests_failures: 0, - proxied_put_requests_total: 6, - proxied_put_requests_failures: 2, - proxied_put_tagging_requests_total: 3, - proxied_put_tagging_requests_failures: 1, - proxied_get_tagging_requests_total: 2, - proxied_get_tagging_requests_failures: 0, - proxied_delete_tagging_requests_total: 1, - proxied_delete_tagging_requests_failures: 1, - resync_started_count: 2, - resync_completed_count: 1, - resync_failed_count: 1, - resync_canceled_count: 0, - resync_duration_ms: 1500, - targets: vec![BucketReplicationTargetStats { + let stats = vec![BucketReplicationRuntimeStats { + stats: BucketReplicationStats { + bucket: "b1".to_string(), + total_failed_bytes: 64, + total_failed_count: 2, + last_min_failed_bytes: 32, + last_min_failed_count: 1, + last_hour_failed_bytes: 64, + last_hour_failed_count: 2, + sent_bytes: 1024, + sent_count: 8, + proxied_get_requests_total: 5, + proxied_get_requests_failures: 1, + proxied_head_requests_total: 4, + proxied_head_requests_failures: 0, + proxied_put_requests_total: 6, + proxied_put_requests_failures: 2, + proxied_put_tagging_requests_total: 3, + proxied_put_tagging_requests_failures: 1, + proxied_get_tagging_requests_total: 2, + proxied_get_tagging_requests_failures: 0, + proxied_delete_tagging_requests_total: 1, + proxied_delete_tagging_requests_failures: 1, + resync_started_count: 2, + resync_completed_count: 1, + resync_failed_count: 1, + resync_canceled_count: 0, + resync_duration_ms: 1500, + targets: vec![BucketReplicationTargetStats { + target_arn: "arn:rustfs:replication:us-east-1:1:target".to_string(), + bandwidth_limit_bytes_per_sec: 2048, + current_bandwidth_bytes_per_sec: 1024.0, + latency_ms: 15.0, + }], + }, + target_flows: vec![BucketReplicationTargetFlowStats { target_arn: "arn:rustfs:replication:us-east-1:1:target".to_string(), - bandwidth_limit_bytes_per_sec: 2048, - current_bandwidth_bytes_per_sec: 1024.0, - latency_ms: 15.0, + sent_bytes: 512, + sent_count: 4, + total_failed_bytes: 96, + total_failed_count: 3, + last_min_failed_bytes: 32, + last_min_failed_count: 1, + last_hour_failed_bytes: 64, + last_hour_failed_count: 2, }], }]; - let metrics = collect_bucket_replication_metrics(&stats); - assert_eq!(metrics.len(), 26); + let metrics = collect_bucket_replication_runtime_metrics(&stats); + assert_eq!(metrics.len(), 46); let sent_name = BUCKET_REPL_SENT_COUNT_MD.get_full_metric_name(); assert!(metrics.iter().any(|metric| { @@ -471,6 +643,28 @@ mod tests { && metric.labels.iter().any(|(key, value)| *key == BUCKET_L && value == "b1") })); + let proxy_requests_name = BUCKET_REPL_PROXY_REQUESTS_TOTAL_MD.get_full_metric_name(); + assert!(metrics.iter().any(|metric| { + metric.name == proxy_requests_name + && metric.value == 4.0 + && metric.labels.iter().any(|(key, value)| *key == BUCKET_L && value == "b1") + && metric.labels.iter().any(|(key, value)| *key == OPERATION_L && value == "put") + && metric + .labels + .iter() + .any(|(key, value)| *key == RESULT_L && value == "success") + })); + assert!(metrics.iter().any(|metric| { + metric.name == proxy_requests_name + && metric.value == 2.0 + && metric.labels.iter().any(|(key, value)| *key == BUCKET_L && value == "b1") + && metric.labels.iter().any(|(key, value)| *key == OPERATION_L && value == "put") + && metric + .labels + .iter() + .any(|(key, value)| *key == RESULT_L && value == "failure") + })); + let latency_name = BUCKET_REPL_LATENCY_MS_MD.get_full_metric_name(); assert!(metrics.iter().any(|metric| { metric.name == latency_name @@ -481,6 +675,27 @@ mod tests { .any(|(key, value)| *key == TARGET_ARN_L && value == "arn:rustfs:replication:us-east-1:1:target") })); + let target_sent_name = BUCKET_REPL_TARGET_SENT_COUNT_MD.get_full_metric_name(); + assert!(metrics.iter().any(|metric| { + metric.name == target_sent_name + && metric.value == 4.0 + && metric + .labels + .iter() + .any(|(key, value)| *key == TARGET_ARN_L && value == "arn:rustfs:replication:us-east-1:1:target") + })); + + let target_last_min_failed_name = BUCKET_REPL_TARGET_LAST_MIN_FAILED_BYTES_MD.get_full_metric_name(); + assert!(metrics.iter().any(|metric| { + metric.name == target_last_min_failed_name + && metric.value == 32.0 + && metric.labels.iter().any(|(key, value)| *key == BUCKET_L && value == "b1") + && metric + .labels + .iter() + .any(|(key, value)| *key == TARGET_ARN_L && value == "arn:rustfs:replication:us-east-1:1:target") + })); + let delete_tagging_total_name = BUCKET_REPL_PROXIED_DELETE_TAGGING_REQUESTS_TOTAL_MD.get_full_metric_name(); assert!(metrics.iter().any(|metric| { metric.name == delete_tagging_total_name diff --git a/crates/obs/src/metrics/collectors/ilm.rs b/crates/obs/src/metrics/collectors/ilm.rs index 73d7dd0ba..664f3a36b 100644 --- a/crates/obs/src/metrics/collectors/ilm.rs +++ b/crates/obs/src/metrics/collectors/ilm.rs @@ -48,6 +48,26 @@ pub struct IlmStats { pub versions_scanned: u64, } +/// ILM task metrics by action and state. +#[derive(Debug, Clone, Default)] +pub(crate) struct IlmActionTaskStats { + pub(crate) action: String, + pub(crate) state: String, + pub(crate) value: u64, +} + +/// ILM statistics with runtime-local node identity and bounded action/state details. +#[derive(Debug, Clone, Default)] +pub(crate) struct IlmRuntimeStats { + pub(crate) server: String, + pub(crate) stats: IlmStats, + pub(crate) action_tasks: Vec, +} + +fn is_live_action_task_state(state: &str) -> bool { + matches!(state, "pending" | "active" | "compensation_running") +} + /// Collects ILM metrics from the given stats. /// /// Uses the metric descriptors from `metrics_type::ilm` module. @@ -78,6 +98,25 @@ pub fn collect_ilm_metrics(stats: &IlmStats) -> Vec { ] } +pub(crate) fn collect_ilm_runtime_metrics(stats: &IlmRuntimeStats) -> Vec { + let mut metrics = collect_ilm_metrics(&stats.stats); + + metrics.extend( + stats + .action_tasks + .iter() + .filter(|task| is_live_action_task_state(&task.state)) + .map(|task| { + PrometheusMetric::from_descriptor(&ILM_ACTION_TASKS_MD, task.value as f64) + .with_label_owned(SERVER_LABEL, stats.server.clone()) + .with_label_owned(ACTION_LABEL, task.action.clone()) + .with_label_owned(STATE_LABEL, task.state.clone()) + }), + ); + + metrics +} + #[cfg(test)] mod tests { use super::*; @@ -95,16 +134,62 @@ mod tests { transition_compensation_running_tasks: 1, versions_scanned: 1000000, }; + let runtime_stats = IlmRuntimeStats { + server: "node1:9000".to_string(), + stats, + action_tasks: vec![ + IlmActionTaskStats { + action: "expiry".to_string(), + state: "pending".to_string(), + value: 100, + }, + IlmActionTaskStats { + action: "transition".to_string(), + state: "queue_send_timeout".to_string(), + value: 3, + }, + IlmActionTaskStats { + action: "transition".to_string(), + state: "active".to_string(), + value: 5, + }, + ], + }; - let metrics = collect_ilm_metrics(&stats); + let metrics = collect_ilm_runtime_metrics(&runtime_stats); - assert_eq!(metrics.len(), 9); + assert_eq!(metrics.len(), 11); let pending = metrics.iter().find(|m| m.value == 100.0); assert!(pending.is_some()); let scanned = metrics.iter().find(|m| m.value == 1000000.0); assert!(scanned.is_some()); + + let transition_timeout = metrics.iter().find(|m| { + m.name == ILM_ACTION_TASKS_MD.get_full_metric_name() + && m.labels + .iter() + .any(|(name, value)| *name == SERVER_LABEL && value.as_ref() == "node1:9000") + && m.labels + .iter() + .any(|(name, value)| *name == ACTION_LABEL && value.as_ref() == "transition") + && m.labels + .iter() + .any(|(name, value)| *name == STATE_LABEL && value.as_ref() == "queue_send_timeout") + }); + assert!(transition_timeout.is_none()); + + let transition_active = metrics.iter().find(|m| { + m.name == ILM_ACTION_TASKS_MD.get_full_metric_name() + && m.labels + .iter() + .any(|(name, value)| *name == ACTION_LABEL && value.as_ref() == "transition") + && m.labels + .iter() + .any(|(name, value)| *name == STATE_LABEL && value.as_ref() == "active") + }); + assert_eq!(transition_active.map(|m| m.value), Some(5.0)); } #[test] diff --git a/crates/obs/src/metrics/collectors/mod.rs b/crates/obs/src/metrics/collectors/mod.rs index a674e1e5b..1a356025f 100644 --- a/crates/obs/src/metrics/collectors/mod.rs +++ b/crates/obs/src/metrics/collectors/mod.rs @@ -40,10 +40,12 @@ pub mod system_network; pub mod system_network_host; pub mod system_process; +pub(crate) use audit::{AuditTargetRuntimeStats, collect_audit_runtime_metrics}; pub use audit::{AuditTargetStats, collect_audit_metrics}; pub use bucket::{BucketStats, collect_bucket_metrics}; pub(crate) use bucket_replication::{ - BucketReplicationBacklogStats, BucketReplicationTargetBacklogStats, collect_bucket_replication_backlog_metrics, + BucketReplicationBacklogStats, BucketReplicationRuntimeStats, BucketReplicationTargetBacklogStats, + BucketReplicationTargetFlowStats, collect_bucket_replication_backlog_metrics, collect_bucket_replication_runtime_metrics, }; pub use bucket_replication::{ BucketReplicationBandwidthStats, BucketReplicationStats, BucketReplicationTargetStats, @@ -57,18 +59,24 @@ pub use cluster_iam::{IamStats, collect_iam_metrics}; pub use cluster_usage::{BucketUsageStats, ClusterUsageStats, collect_bucket_usage_metrics, collect_cluster_usage_metrics}; pub use compression::{CompressionClusterStats, collect_compression_cluster_metrics}; pub use dial9::{Dial9Stats, collect_current_dial9_metrics, collect_dial9_metrics, is_dial9_enabled}; +pub(crate) use ilm::{IlmActionTaskStats, IlmRuntimeStats, collect_ilm_runtime_metrics}; pub use ilm::{IlmStats, collect_ilm_metrics}; pub use node::{DiskStats, collect_node_metrics}; pub use notification::{NotificationStats, collect_notification_metrics}; +pub(crate) use notification_target::{NotificationTargetRuntimeStats, collect_notification_target_runtime_metrics}; pub use notification_target::{NotificationTargetStats, collect_notification_target_metrics}; +pub(crate) use replication::{ReplicationRuntimeStats, collect_replication_runtime_metrics}; pub use replication::{ReplicationStats, collect_replication_metrics}; +pub(crate) use request::{ApiRequestMetricSupport, ApiRequestStats, collect_request_metrics}; pub use resource::{ResourceStats, collect_resource_metrics}; +pub(crate) use scanner::{ScannerRuntimeStats, collect_scanner_runtime_metrics}; pub use scanner::{ScannerStats, collect_scanner_metrics}; pub use system_cpu::{CpuStats, ProcessCpuStats, collect_cpu_metrics, collect_process_cpu_metrics}; pub use system_drive::{ DriveCountStats, DriveDetailedStats, ProcessDiskStats, collect_drive_count_metrics, collect_drive_detailed_metrics, collect_process_disk_metrics, }; +pub(crate) use system_drive::{DriveRuntimeDetailedStats, collect_drive_runtime_detailed_metrics}; #[cfg(feature = "gpu")] pub use system_gpu::{GpuCollector, GpuError, GpuStats, collect_gpu_metrics}; pub use system_memory::{MemoryStats, ProcessMemoryStats, collect_memory_metrics, collect_process_memory_metrics}; diff --git a/crates/obs/src/metrics/collectors/notification_target.rs b/crates/obs/src/metrics/collectors/notification_target.rs index 8216dc855..797f00ff4 100644 --- a/crates/obs/src/metrics/collectors/notification_target.rs +++ b/crates/obs/src/metrics/collectors/notification_target.rs @@ -16,8 +16,10 @@ use crate::metrics::report::PrometheusMetric; use crate::metrics::schema::notification_target::{ - NOTIFICATION_TARGET_FAILED_MESSAGES_MD, NOTIFICATION_TARGET_FAILED_STORE_LENGTH_MD, NOTIFICATION_TARGET_QUEUE_LENGTH_MD, - NOTIFICATION_TARGET_TOTAL_MESSAGES_MD, TARGET_ID, TARGET_TYPE, + NOTIFICATION_TARGET_FAILED_MESSAGES_BY_SERVER_MD, NOTIFICATION_TARGET_FAILED_MESSAGES_MD, + NOTIFICATION_TARGET_FAILED_STORE_LENGTH_BY_SERVER_MD, NOTIFICATION_TARGET_FAILED_STORE_LENGTH_MD, + NOTIFICATION_TARGET_QUEUE_LENGTH_BY_SERVER_MD, NOTIFICATION_TARGET_QUEUE_LENGTH_MD, + NOTIFICATION_TARGET_TOTAL_MESSAGES_BY_SERVER_MD, NOTIFICATION_TARGET_TOTAL_MESSAGES_MD, SERVER, TARGET_ID, TARGET_TYPE, }; use std::borrow::Cow; @@ -31,12 +33,18 @@ pub struct NotificationTargetStats { pub total_messages: u64, } +#[derive(Debug, Clone, Default)] +pub(crate) struct NotificationTargetRuntimeStats { + pub(crate) server: String, + pub(crate) target: NotificationTargetStats, +} + pub fn collect_notification_target_metrics(stats: &[NotificationTargetStats]) -> Vec { if stats.is_empty() { return Vec::new(); } - let mut metrics = Vec::with_capacity(stats.len() * 4); + let mut metrics = Vec::with_capacity(stats.len() * 8); for stat in stats { let target_id: Cow<'static, str> = Cow::Owned(stat.target_id.clone()); let target_type: Cow<'static, str> = Cow::Owned(stat.target_type.clone()); @@ -66,6 +74,57 @@ pub fn collect_notification_target_metrics(stats: &[NotificationTargetStats]) -> metrics } +pub(crate) fn collect_notification_target_runtime_metrics(stats: &[NotificationTargetRuntimeStats]) -> Vec { + if stats.is_empty() { + return Vec::new(); + } + + let legacy_stats = stats.iter().map(|stat| stat.target.clone()).collect::>(); + let mut metrics = collect_notification_target_metrics(&legacy_stats); + metrics.reserve(stats.len() * 4); + for stat in stats { + let server: Cow<'static, str> = Cow::Owned(stat.server.clone()); + let target_id: Cow<'static, str> = Cow::Owned(stat.target.target_id.clone()); + let target_type: Cow<'static, str> = Cow::Owned(stat.target.target_type.clone()); + + metrics.push( + PrometheusMetric::from_descriptor( + &NOTIFICATION_TARGET_FAILED_MESSAGES_BY_SERVER_MD, + stat.target.failed_messages as f64, + ) + .with_label(SERVER, server.clone()) + .with_label(TARGET_ID, target_id.clone()) + .with_label(TARGET_TYPE, target_type.clone()), + ); + metrics.push( + PrometheusMetric::from_descriptor( + &NOTIFICATION_TARGET_FAILED_STORE_LENGTH_BY_SERVER_MD, + stat.target.failed_store_length as f64, + ) + .with_label(SERVER, server.clone()) + .with_label(TARGET_ID, target_id.clone()) + .with_label(TARGET_TYPE, target_type.clone()), + ); + metrics.push( + PrometheusMetric::from_descriptor(&NOTIFICATION_TARGET_QUEUE_LENGTH_BY_SERVER_MD, stat.target.queue_length as f64) + .with_label(SERVER, server.clone()) + .with_label(TARGET_ID, target_id.clone()) + .with_label(TARGET_TYPE, target_type.clone()), + ); + metrics.push( + PrometheusMetric::from_descriptor( + &NOTIFICATION_TARGET_TOTAL_MESSAGES_BY_SERVER_MD, + stat.target.total_messages as f64, + ) + .with_label(SERVER, server) + .with_label(TARGET_ID, target_id) + .with_label(TARGET_TYPE, target_type), + ); + } + + metrics +} + #[cfg(test)] mod tests { use super::*; @@ -73,7 +132,7 @@ mod tests { #[test] fn test_collect_notification_target_metrics() { - let stats = vec![NotificationTargetStats { + let stats = [NotificationTargetStats { failed_messages: 2, failed_store_length: 3, queue_length: 4, @@ -82,9 +141,12 @@ mod tests { total_messages: 42, }]; - let metrics = collect_notification_target_metrics(&stats); + let metrics = collect_notification_target_runtime_metrics(&[NotificationTargetRuntimeStats { + server: "node1:9000".to_string(), + target: stats[0].clone(), + }]); - assert_eq!(metrics.len(), 4); + assert_eq!(metrics.len(), 8); assert!(metrics.iter().any(|metric| { metric.value == 3.0 && metric.name == NOTIFICATION_TARGET_FAILED_STORE_LENGTH_MD.get_full_metric_name() @@ -104,6 +166,22 @@ mod tests { .iter() .any(|(key, value)| *key == TARGET_TYPE && value == "webhook") })); + assert!(metrics.iter().any(|metric| { + metric.value == 4.0 + && metric.name == NOTIFICATION_TARGET_QUEUE_LENGTH_BY_SERVER_MD.get_full_metric_name() + && metric + .labels + .iter() + .any(|(key, value)| *key == SERVER && value == "node1:9000") + && metric + .labels + .iter() + .any(|(key, value)| *key == TARGET_ID && value == "primary:webhook") + && metric + .labels + .iter() + .any(|(key, value)| *key == TARGET_TYPE && value == "webhook") + })); } #[test] @@ -113,4 +191,18 @@ mod tests { assert_eq!(NOTIFICATION_TARGET_QUEUE_LENGTH_MD.metric_type, MetricType::Gauge); assert_eq!(NOTIFICATION_TARGET_TOTAL_MESSAGES_MD.metric_type, MetricType::Gauge); } + + #[test] + fn notification_target_stats_struct_literal_keeps_legacy_fields() { + let stats = vec![NotificationTargetStats { + failed_messages: 2, + failed_store_length: 3, + queue_length: 4, + target_id: "primary:webhook".to_string(), + target_type: "webhook".to_string(), + total_messages: 42, + }]; + + assert_eq!(collect_notification_target_metrics(&stats).len(), 4); + } } diff --git a/crates/obs/src/metrics/collectors/replication.rs b/crates/obs/src/metrics/collectors/replication.rs index 88b4b19a1..c1311696b 100644 --- a/crates/obs/src/metrics/collectors/replication.rs +++ b/crates/obs/src/metrics/collectors/replication.rs @@ -53,6 +53,12 @@ pub struct ReplicationStats { pub recent_backlog_count: u64, } +#[derive(Debug, Clone, Default)] +pub(crate) struct ReplicationRuntimeStats { + pub(crate) server: String, + pub(crate) stats: ReplicationStats, +} + /// Collects replication metrics from the given stats. /// /// Returns a vector of Prometheus metrics for replication statistics. @@ -74,6 +80,41 @@ pub fn collect_replication_metrics(stats: &ReplicationStats) -> Vec Vec { + let stats = &runtime.stats; + let mut metrics = collect_replication_metrics(stats); + metrics.extend([ + PrometheusMetric::from_descriptor(&REPLICATION_AVERAGE_ACTIVE_WORKERS_BY_SERVER_MD, stats.average_active_workers) + .with_label_owned(SERVER_LABEL, runtime.server.clone()), + PrometheusMetric::from_descriptor(&REPLICATION_AVERAGE_QUEUED_BYTES_BY_SERVER_MD, stats.average_queued_bytes as f64) + .with_label_owned(SERVER_LABEL, runtime.server.clone()), + PrometheusMetric::from_descriptor(&REPLICATION_AVERAGE_QUEUED_COUNT_BY_SERVER_MD, stats.average_queued_count as f64) + .with_label_owned(SERVER_LABEL, runtime.server.clone()), + PrometheusMetric::from_descriptor(&REPLICATION_CURRENT_ACTIVE_WORKERS_BY_SERVER_MD, stats.active_workers as f64) + .with_label_owned(SERVER_LABEL, runtime.server.clone()), + PrometheusMetric::from_descriptor(&REPLICATION_CURRENT_DATA_TRANSFER_RATE_BY_SERVER_MD, stats.current_data_transfer_rate) + .with_label_owned(SERVER_LABEL, runtime.server.clone()), + PrometheusMetric::from_descriptor( + &REPLICATION_LAST_MINUTE_QUEUED_BYTES_BY_SERVER_MD, + stats.last_minute_queued_bytes as f64, + ) + .with_label_owned(SERVER_LABEL, runtime.server.clone()), + PrometheusMetric::from_descriptor( + &REPLICATION_LAST_MINUTE_QUEUED_COUNT_BY_SERVER_MD, + stats.last_minute_queued_count as f64, + ) + .with_label_owned(SERVER_LABEL, runtime.server.clone()), + PrometheusMetric::from_descriptor(&REPLICATION_MAX_ACTIVE_WORKERS_BY_SERVER_MD, stats.max_active_workers as f64) + .with_label_owned(SERVER_LABEL, runtime.server.clone()), + PrometheusMetric::from_descriptor(&REPLICATION_MAX_QUEUED_BYTES_BY_SERVER_MD, stats.max_queued_bytes as f64) + .with_label_owned(SERVER_LABEL, runtime.server.clone()), + PrometheusMetric::from_descriptor(&REPLICATION_MAX_QUEUED_COUNT_BY_SERVER_MD, stats.max_queued_count as f64) + .with_label_owned(SERVER_LABEL, runtime.server.clone()), + ]); + + metrics +} + #[cfg(test)] mod tests { use super::*; @@ -97,10 +138,13 @@ mod tests { recent_backlog_count: 1500, }; - let metrics = collect_replication_metrics(&stats); + let metrics = collect_replication_runtime_metrics(&ReplicationRuntimeStats { + server: "node-a:9000".to_string(), + stats, + }); report_metrics(&metrics); - assert_eq!(metrics.len(), 13); + assert_eq!(metrics.len(), 23); // Verify active workers let active_name = REPLICATION_CURRENT_ACTIVE_WORKERS_MD.get_full_metric_name(); @@ -111,6 +155,31 @@ mod tests { let avg_active_name = REPLICATION_AVERAGE_ACTIVE_WORKERS_MD.get_full_metric_name(); let avg_active = metrics.iter().find(|m| m.name == avg_active_name); assert_eq!(avg_active.map(|m| m.value), Some(8.5)); + + let active_by_server_name = REPLICATION_CURRENT_ACTIVE_WORKERS_BY_SERVER_MD.get_full_metric_name(); + let active_by_server = metrics.iter().find(|m| m.name == active_by_server_name); + assert_eq!(active_by_server.map(|m| m.value), Some(10.0)); + assert_eq!( + active_by_server + .and_then(|m| m.labels.iter().find(|(name, _)| *name == SERVER_LABEL)) + .map(|(_, value)| value.as_ref()), + Some("node-a:9000") + ); + assert!( + metrics + .iter() + .all(|m| m.name != REPLICATION_AVERAGE_DATA_TRANSFER_RATE_BY_SERVER_MD.get_full_metric_name()) + ); + assert!( + metrics + .iter() + .all(|m| m.name != REPLICATION_MAX_DATA_TRANSFER_RATE_BY_SERVER_MD.get_full_metric_name()) + ); + assert!( + metrics + .iter() + .all(|m| m.name != REPLICATION_RECENT_BACKLOG_COUNT_BY_SERVER_MD.get_full_metric_name()) + ); } #[test] @@ -124,4 +193,25 @@ mod tests { assert!(metric.labels.is_empty()); } } + + #[test] + fn replication_stats_struct_literal_keeps_legacy_fields() { + let stats = ReplicationStats { + average_active_workers: 1.0, + average_queued_bytes: 2, + average_queued_count: 3, + average_data_transfer_rate: 4.0, + active_workers: 5, + current_data_transfer_rate: 6.0, + last_minute_queued_bytes: 7, + last_minute_queued_count: 8, + max_active_workers: 9, + max_queued_bytes: 10, + max_queued_count: 11, + max_data_transfer_rate: 12.0, + recent_backlog_count: 13, + }; + + assert_eq!(collect_replication_metrics(&stats).len(), 13); + } } diff --git a/crates/obs/src/metrics/collectors/request.rs b/crates/obs/src/metrics/collectors/request.rs index b7e0b8987..2aae53666 100644 --- a/crates/obs/src/metrics/collectors/request.rs +++ b/crates/obs/src/metrics/collectors/request.rs @@ -23,9 +23,32 @@ use crate::metrics::report::PrometheusMetric; use crate::metrics::schema::request::*; use std::collections::HashMap; +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub(crate) struct ApiRequestMetricSupport { + pub(crate) lifecycle: bool, + pub(crate) traffic: bool, + pub(crate) ttfb: bool, +} + +impl ApiRequestMetricSupport { + pub(crate) const ALL: Self = Self { + lifecycle: true, + traffic: true, + ttfb: true, + }; + + pub(crate) const TOTALS_ONLY: Self = Self { + lifecycle: false, + traffic: false, + ttfb: false, + }; +} + /// API request statistics for a specific API endpoint. -#[derive(Debug, Clone, Default)] +#[derive(Debug, Clone)] pub struct ApiRequestStats { + /// Server identifier + pub server: String, /// API name (e.g., "GetObject", "PutObject") pub name: String, /// Request type (e.g., "s3", "admin") @@ -48,6 +71,27 @@ pub struct ApiRequestStats { pub sent_bytes: u64, /// Bytes received pub recv_bytes: u64, + pub(crate) supported_metrics: ApiRequestMetricSupport, +} + +impl Default for ApiRequestStats { + fn default() -> Self { + Self { + server: String::new(), + name: String::new(), + req_type: String::new(), + in_flight: 0, + total: 0, + errors_total: 0, + errors_5xx: 0, + errors_4xx: 0, + canceled: 0, + ttfb_distribution: Vec::new(), + sent_bytes: 0, + recv_bytes: 0, + supported_metrics: ApiRequestMetricSupport::ALL, + } + } } /// Collects API request metrics from the given stats. @@ -56,62 +100,119 @@ pub struct ApiRequestStats { pub fn collect_request_metrics(stats: &[ApiRequestStats]) -> Vec { let mut metrics = Vec::new(); let mut traffic_by_type: HashMap<&str, (u64, u64)> = HashMap::with_capacity(stats.len()); + let mut traffic_by_server_type: HashMap<(&str, &str), (u64, u64)> = HashMap::with_capacity(stats.len()); for stat in stats { - let entry = traffic_by_type.entry(stat.req_type.as_str()).or_default(); - entry.0 = entry.0.saturating_add(stat.sent_bytes); - entry.1 = entry.1.saturating_add(stat.recv_bytes); + if stat.supported_metrics.traffic { + let entry = traffic_by_type.entry(stat.req_type.as_str()).or_default(); + entry.0 = entry.0.saturating_add(stat.sent_bytes); + entry.1 = entry.1.saturating_add(stat.recv_bytes); + if !stat.server.is_empty() { + let entry = traffic_by_server_type + .entry((stat.server.as_str(), stat.req_type.as_str())) + .or_default(); + entry.0 = entry.0.saturating_add(stat.sent_bytes); + entry.1 = entry.1.saturating_add(stat.recv_bytes); + } + } - // In-flight requests - metrics.push( - PrometheusMetric::from_descriptor(&API_REQUESTS_IN_FLIGHT_TOTAL_MD, stat.in_flight as f64) - .with_label_owned(NAME_LABEL, stat.name.clone()) - .with_label_owned(TYPE_LABEL, stat.req_type.clone()), - ); - - // Total requests metrics.push( PrometheusMetric::from_descriptor(&API_REQUESTS_TOTAL_MD, stat.total as f64) .with_label_owned(NAME_LABEL, stat.name.clone()) .with_label_owned(TYPE_LABEL, stat.req_type.clone()), ); - // Total errors - metrics.push( - PrometheusMetric::from_descriptor(&API_REQUESTS_ERRORS_TOTAL_MD, stat.errors_total as f64) - .with_label_owned(NAME_LABEL, stat.name.clone()) - .with_label_owned(TYPE_LABEL, stat.req_type.clone()), - ); - - // 5xx errors - metrics.push( - PrometheusMetric::from_descriptor(&API_REQUESTS_5XX_ERRORS_TOTAL_MD, stat.errors_5xx as f64) - .with_label_owned(NAME_LABEL, stat.name.clone()) - .with_label_owned(TYPE_LABEL, stat.req_type.clone()), - ); - - // 4xx errors - metrics.push( - PrometheusMetric::from_descriptor(&API_REQUESTS_4XX_ERRORS_TOTAL_MD, stat.errors_4xx as f64) - .with_label_owned(NAME_LABEL, stat.name.clone()) - .with_label_owned(TYPE_LABEL, stat.req_type.clone()), - ); - - // Canceled requests - metrics.push( - PrometheusMetric::from_descriptor(&API_REQUESTS_CANCELED_TOTAL_MD, stat.canceled as f64) - .with_label_owned(NAME_LABEL, stat.name.clone()) - .with_label_owned(TYPE_LABEL, stat.req_type.clone()), - ); - - // TTFB distribution (histogram buckets) - for (le, value) in &stat.ttfb_distribution { + if stat.supported_metrics.lifecycle { metrics.push( - PrometheusMetric::from_descriptor(&API_REQUESTS_TTFB_SECONDS_DISTRIBUTION_MD, *value) + PrometheusMetric::from_descriptor(&API_REQUESTS_IN_FLIGHT_TOTAL_MD, stat.in_flight as f64) .with_label_owned(NAME_LABEL, stat.name.clone()) - .with_label_owned(TYPE_LABEL, stat.req_type.clone()) - .with_label_owned(LE_LABEL, le.clone()), + .with_label_owned(TYPE_LABEL, stat.req_type.clone()), ); + metrics.push( + PrometheusMetric::from_descriptor(&API_REQUESTS_ERRORS_TOTAL_MD, stat.errors_total as f64) + .with_label_owned(NAME_LABEL, stat.name.clone()) + .with_label_owned(TYPE_LABEL, stat.req_type.clone()), + ); + metrics.push( + PrometheusMetric::from_descriptor(&API_REQUESTS_5XX_ERRORS_TOTAL_MD, stat.errors_5xx as f64) + .with_label_owned(NAME_LABEL, stat.name.clone()) + .with_label_owned(TYPE_LABEL, stat.req_type.clone()), + ); + metrics.push( + PrometheusMetric::from_descriptor(&API_REQUESTS_4XX_ERRORS_TOTAL_MD, stat.errors_4xx as f64) + .with_label_owned(NAME_LABEL, stat.name.clone()) + .with_label_owned(TYPE_LABEL, stat.req_type.clone()), + ); + metrics.push( + PrometheusMetric::from_descriptor(&API_REQUESTS_CANCELED_TOTAL_MD, stat.canceled as f64) + .with_label_owned(NAME_LABEL, stat.name.clone()) + .with_label_owned(TYPE_LABEL, stat.req_type.clone()), + ); + } + + if stat.supported_metrics.ttfb { + for (le, value) in &stat.ttfb_distribution { + metrics.push( + PrometheusMetric::from_descriptor(&API_REQUESTS_TTFB_SECONDS_DISTRIBUTION_MD, *value) + .with_label_owned(NAME_LABEL, stat.name.clone()) + .with_label_owned(TYPE_LABEL, stat.req_type.clone()) + .with_label_owned(LE_LABEL, le.clone()), + ); + } + } + + if !stat.server.is_empty() { + metrics.push( + PrometheusMetric::from_descriptor(&API_REQUESTS_TOTAL_BY_SERVER_MD, stat.total as f64) + .with_label_owned(SERVER_LABEL, stat.server.clone()) + .with_label_owned(NAME_LABEL, stat.name.clone()) + .with_label_owned(TYPE_LABEL, stat.req_type.clone()), + ); + + if stat.supported_metrics.lifecycle { + metrics.push( + PrometheusMetric::from_descriptor(&API_REQUESTS_IN_FLIGHT_TOTAL_BY_SERVER_MD, stat.in_flight as f64) + .with_label_owned(SERVER_LABEL, stat.server.clone()) + .with_label_owned(NAME_LABEL, stat.name.clone()) + .with_label_owned(TYPE_LABEL, stat.req_type.clone()), + ); + metrics.push( + PrometheusMetric::from_descriptor(&API_REQUESTS_ERRORS_TOTAL_BY_SERVER_MD, stat.errors_total as f64) + .with_label_owned(SERVER_LABEL, stat.server.clone()) + .with_label_owned(NAME_LABEL, stat.name.clone()) + .with_label_owned(TYPE_LABEL, stat.req_type.clone()), + ); + metrics.push( + PrometheusMetric::from_descriptor(&API_REQUESTS_5XX_ERRORS_TOTAL_BY_SERVER_MD, stat.errors_5xx as f64) + .with_label_owned(SERVER_LABEL, stat.server.clone()) + .with_label_owned(NAME_LABEL, stat.name.clone()) + .with_label_owned(TYPE_LABEL, stat.req_type.clone()), + ); + metrics.push( + PrometheusMetric::from_descriptor(&API_REQUESTS_4XX_ERRORS_TOTAL_BY_SERVER_MD, stat.errors_4xx as f64) + .with_label_owned(SERVER_LABEL, stat.server.clone()) + .with_label_owned(NAME_LABEL, stat.name.clone()) + .with_label_owned(TYPE_LABEL, stat.req_type.clone()), + ); + metrics.push( + PrometheusMetric::from_descriptor(&API_REQUESTS_CANCELED_TOTAL_BY_SERVER_MD, stat.canceled as f64) + .with_label_owned(SERVER_LABEL, stat.server.clone()) + .with_label_owned(NAME_LABEL, stat.name.clone()) + .with_label_owned(TYPE_LABEL, stat.req_type.clone()), + ); + } + + if stat.supported_metrics.ttfb { + for (le, value) in &stat.ttfb_distribution { + metrics.push( + PrometheusMetric::from_descriptor(&API_REQUESTS_TTFB_SECONDS_DISTRIBUTION_BY_SERVER_MD, *value) + .with_label_owned(SERVER_LABEL, stat.server.clone()) + .with_label_owned(NAME_LABEL, stat.name.clone()) + .with_label_owned(TYPE_LABEL, stat.req_type.clone()) + .with_label_owned(LE_LABEL, le.clone()), + ); + } + } } } @@ -126,6 +227,19 @@ pub fn collect_request_metrics(stats: &[ApiRequestStats]) -> Vec, + pub(crate) current_cycle_source_work: Vec, + pub(crate) last_cycle_source_work: Vec, + pub(crate) bucket_drive_results: Vec, + pub(crate) current_cycle_bucket_drive_results: Vec, + pub(crate) last_cycle_bucket_drive_results: Vec, +} + /// Collects scanner metrics from the given stats. /// /// Uses the metric descriptors from `metrics_type::scanner` module. /// Returns a vector of Prometheus metrics for scanner statistics. pub fn collect_scanner_metrics(stats: &ScannerStats) -> Vec { - vec![ + collect_scanner_metrics_with_runtime(stats, None) +} + +pub(crate) fn collect_scanner_runtime_metrics(stats: &ScannerRuntimeStats) -> Vec { + collect_scanner_metrics_with_runtime(&stats.stats, Some(stats)) +} + +fn collect_scanner_metrics_with_runtime(stats: &ScannerStats, runtime: Option<&ScannerRuntimeStats>) -> Vec { + fn push_source_work_metric( + metrics: &mut Vec, + descriptor: &'static crate::metrics::schema::MetricDescriptor, + server: &str, + source: &str, + state: &str, + value: u64, + cycle_scope: Option<&str>, + ) { + let mut metric = + PrometheusMetric::from_descriptor(descriptor, value as f64).with_label_owned(SERVER_LABEL, server.to_string()); + if let Some(cycle_scope) = cycle_scope { + metric = metric.with_label_owned(CYCLE_SCOPE_LABEL, cycle_scope.to_string()); + } + metric = metric + .with_label_owned(SOURCE_LABEL, source.to_string()) + .with_label_owned(STATE_LABEL, state.to_string()); + metrics.push(metric); + } + + fn push_source_work_metrics( + metrics: &mut Vec, + descriptor: &'static crate::metrics::schema::MetricDescriptor, + server: &str, + source_work: &[ScannerSourceWorkStats], + cycle_scope: Option<&str>, + ) { + for work in source_work { + push_source_work_metric(metrics, descriptor, server, &work.source, "checked", work.checked, cycle_scope); + push_source_work_metric(metrics, descriptor, server, &work.source, "queued", work.queued, cycle_scope); + push_source_work_metric(metrics, descriptor, server, &work.source, "executed", work.executed, cycle_scope); + push_source_work_metric(metrics, descriptor, server, &work.source, "failed", work.failed, cycle_scope); + push_source_work_metric(metrics, descriptor, server, &work.source, "skipped", work.skipped, cycle_scope); + push_source_work_metric(metrics, descriptor, server, &work.source, "missed", work.missed, cycle_scope); + } + } + + fn push_bucket_drive_result_metric( + metrics: &mut Vec, + descriptor: &'static crate::metrics::schema::MetricDescriptor, + server: &str, + result: &ScannerBucketDriveResultStats, + cycle_scope: Option<&str>, + ) { + let mut metric = + PrometheusMetric::from_descriptor(descriptor, result.count as f64).with_label_owned(SERVER_LABEL, server.to_string()); + if let Some(cycle_scope) = cycle_scope { + metric = metric.with_label_owned(CYCLE_SCOPE_LABEL, cycle_scope.to_string()); + } + metrics.push( + metric + .with_label_owned(BUCKET_LABEL, result.bucket.clone()) + .with_label_owned(DRIVE_LABEL, result.drive.clone()) + .with_label_owned(RESULT_LABEL, result.result.clone()), + ); + } + + fn push_bucket_drive_result_metrics( + metrics: &mut Vec, + descriptor: &'static crate::metrics::schema::MetricDescriptor, + server: &str, + results: &[ScannerBucketDriveResultStats], + cycle_scope: Option<&str>, + ) { + for result in results { + push_bucket_drive_result_metric(metrics, descriptor, server, result, cycle_scope); + } + } + + let mut metrics = vec![ PrometheusMetric::from_descriptor(&SCANNER_BUCKET_SCANS_FINISHED_MD, stats.bucket_scans_finished as f64), PrometheusMetric::from_descriptor(&SCANNER_BUCKET_SCANS_STARTED_MD, stats.bucket_scans_started as f64), PrometheusMetric::from_descriptor(&SCANNER_BUCKET_SCANS_FAILED_MD, stats.bucket_scans_failed as f64), @@ -331,7 +415,48 @@ pub fn collect_scanner_metrics(stats: &ScannerStats) -> Vec { .with_label("reason", "objects"), PrometheusMetric::from_descriptor(&SCANNER_PARTIAL_CYCLES_BY_REASON_MD, stats.partial_cycles_directories as f64) .with_label("reason", "directories"), - ] + ]; + + if let Some(runtime) = runtime { + push_source_work_metrics(&mut metrics, &SCANNER_SOURCE_WORK_TOTAL_MD, &runtime.server, &runtime.source_work, None); + push_source_work_metrics( + &mut metrics, + &SCANNER_CYCLE_SOURCE_WORK_MD, + &runtime.server, + &runtime.current_cycle_source_work, + Some("current"), + ); + push_source_work_metrics( + &mut metrics, + &SCANNER_CYCLE_SOURCE_WORK_MD, + &runtime.server, + &runtime.last_cycle_source_work, + Some("last"), + ); + push_bucket_drive_result_metrics( + &mut metrics, + &SCANNER_BUCKET_DRIVE_RESULT_TOTAL_MD, + &runtime.server, + &runtime.bucket_drive_results, + None, + ); + push_bucket_drive_result_metrics( + &mut metrics, + &SCANNER_CYCLE_BUCKET_DRIVE_RESULT_MD, + &runtime.server, + &runtime.current_cycle_bucket_drive_results, + Some("current"), + ); + push_bucket_drive_result_metrics( + &mut metrics, + &SCANNER_CYCLE_BUCKET_DRIVE_RESULT_MD, + &runtime.server, + &runtime.last_cycle_bucket_drive_results, + Some("last"), + ); + } + + metrics } fn bool_metric_value(enabled: bool) -> f64 { @@ -345,82 +470,130 @@ mod tests { #[test] fn test_collect_scanner_metrics() { - let stats = ScannerStats { - bucket_scans_finished: 100, - bucket_scans_started: 100, - bucket_scans_failed: 2, - directories_scanned: 50000, - objects_scanned: 1000000, - versions_scanned: 1500000, - last_activity_seconds: 30, - active_paths: 4, - oldest_active_path_age_seconds: 17, - current_set_scan_concurrency_limit: 3, - current_set_scans_queued: 5, - current_set_scans_active: 2, - current_disk_scan_concurrency_limit: 6, - current_disk_bucket_scans_queued: 18, - current_disk_bucket_scans_active: 4, - throttle_idle_mode_enabled: true, - throttle_sleep_factor: 10.0, - throttle_max_sleep_seconds: 15.0, - yield_every_n_objects: 128, - cycle_interval_seconds: 3600.0, - cycle_max_duration_seconds: 1800.0, - cycle_max_objects: 1_000_000, - cycle_max_directories: 100_000, - bitrot_cycle_enabled: true, - bitrot_cycle_seconds: 86400.0, - current_cycle: 12, - completed_cycles: 11, - current_cycle_age_seconds: 90, - current_cycle_objects_scanned: 250, - current_cycle_directories_scanned: 20, - current_cycle_bucket_drive_scans: 2, - current_cycle_bucket_drive_failures: 1, - current_cycle_objects_per_second: 12.5, - current_cycle_directories_per_second: 1.0, - current_cycle_bucket_drive_scans_per_second: 0.1, - current_cycle_yield_events: 8, - current_cycle_yield_duration_seconds: 1.25, - current_cycle_throttle_sleep_events: 4, - current_cycle_throttle_sleep_duration_seconds: 2.5, - current_cycle_ilm_actions: 6, - current_cycle_heal_objects: 2, - current_cycle_replication_checks: 5, - current_cycle_usage_saves: 3, - current_scan_mode: 2, - last_cycle_result: 1, - last_cycle_partial_reason: 3, - last_cycle_duration_seconds: 42.5, - last_cycle_objects_scanned: 900, - last_cycle_directories_scanned: 80, - last_cycle_bucket_drive_scans: 6, - last_cycle_bucket_drive_failures: 2, - last_cycle_objects_per_second: 18.0, - last_cycle_directories_per_second: 1.6, - last_cycle_bucket_drive_scans_per_second: 0.12, - last_cycle_yield_events: 30, - last_cycle_yield_duration_seconds: 9.5, - last_cycle_throttle_sleep_events: 12, - last_cycle_throttle_sleep_duration_seconds: 6.75, - last_cycle_ilm_actions: 44, - last_cycle_heal_objects: 7, - last_cycle_replication_checks: 12, - last_cycle_usage_saves: 9, - failed_cycles: 3, - superseded_cycles: 5, - partial_cycles: 10, - partial_cycles_unknown: 1, - partial_cycles_runtime: 2, - partial_cycles_objects: 3, - partial_cycles_directories: 4, + let stats = ScannerRuntimeStats { + server: "node1:9000".to_string(), + source_work: vec![ScannerSourceWorkStats { + source: "lifecycle".to_string(), + checked: 11, + queued: 2, + executed: 3, + failed: 4, + skipped: 5, + missed: 6, + }], + current_cycle_source_work: vec![ScannerSourceWorkStats { + source: "usage".to_string(), + checked: 21, + queued: 7, + executed: 8, + failed: 9, + skipped: 10, + missed: 11, + }], + last_cycle_source_work: vec![ScannerSourceWorkStats { + source: "heal".to_string(), + checked: 31, + queued: 12, + executed: 13, + failed: 14, + skipped: 15, + missed: 16, + }], + bucket_drive_results: vec![ScannerBucketDriveResultStats { + bucket: "photos".to_string(), + drive: "/data1".to_string(), + result: "success".to_string(), + count: 3, + }], + current_cycle_bucket_drive_results: vec![ScannerBucketDriveResultStats { + bucket: "photos".to_string(), + drive: "/data1".to_string(), + result: "partial".to_string(), + count: 1, + }], + last_cycle_bucket_drive_results: vec![ScannerBucketDriveResultStats { + bucket: "videos".to_string(), + drive: "/data2".to_string(), + result: "error".to_string(), + count: 2, + }], + stats: ScannerStats { + bucket_scans_finished: 100, + bucket_scans_started: 100, + bucket_scans_failed: 2, + directories_scanned: 50000, + objects_scanned: 1000000, + versions_scanned: 1500000, + last_activity_seconds: 30, + active_paths: 4, + oldest_active_path_age_seconds: 17, + current_set_scan_concurrency_limit: 3, + current_set_scans_queued: 5, + current_set_scans_active: 2, + current_disk_scan_concurrency_limit: 6, + current_disk_bucket_scans_queued: 18, + current_disk_bucket_scans_active: 4, + throttle_idle_mode_enabled: true, + throttle_sleep_factor: 10.0, + throttle_max_sleep_seconds: 15.0, + yield_every_n_objects: 128, + cycle_interval_seconds: 3600.0, + cycle_max_duration_seconds: 1800.0, + cycle_max_objects: 1_000_000, + cycle_max_directories: 100_000, + bitrot_cycle_enabled: true, + bitrot_cycle_seconds: 86400.0, + current_cycle: 12, + completed_cycles: 11, + current_cycle_age_seconds: 90, + current_cycle_objects_scanned: 250, + current_cycle_directories_scanned: 20, + current_cycle_bucket_drive_scans: 2, + current_cycle_bucket_drive_failures: 1, + current_cycle_objects_per_second: 12.5, + current_cycle_directories_per_second: 1.0, + current_cycle_bucket_drive_scans_per_second: 0.1, + current_cycle_yield_events: 8, + current_cycle_yield_duration_seconds: 1.25, + current_cycle_throttle_sleep_events: 4, + current_cycle_throttle_sleep_duration_seconds: 2.5, + current_cycle_ilm_actions: 6, + current_cycle_heal_objects: 2, + current_cycle_replication_checks: 5, + current_cycle_usage_saves: 3, + current_scan_mode: 2, + last_cycle_result: 1, + last_cycle_partial_reason: 3, + last_cycle_duration_seconds: 42.5, + last_cycle_objects_scanned: 900, + last_cycle_directories_scanned: 80, + last_cycle_bucket_drive_scans: 6, + last_cycle_bucket_drive_failures: 2, + last_cycle_objects_per_second: 18.0, + last_cycle_directories_per_second: 1.6, + last_cycle_bucket_drive_scans_per_second: 0.12, + last_cycle_yield_events: 30, + last_cycle_yield_duration_seconds: 9.5, + last_cycle_throttle_sleep_events: 12, + last_cycle_throttle_sleep_duration_seconds: 6.75, + last_cycle_ilm_actions: 44, + last_cycle_heal_objects: 7, + last_cycle_replication_checks: 12, + last_cycle_usage_saves: 9, + failed_cycles: 3, + superseded_cycles: 5, + partial_cycles: 10, + partial_cycles_unknown: 1, + partial_cycles_runtime: 2, + partial_cycles_objects: 3, + partial_cycles_directories: 4, + }, }; - let metrics = collect_scanner_metrics(&stats); + let metrics = collect_scanner_runtime_metrics(&stats); report_metrics(&metrics); - assert_eq!(metrics.len(), 69); + assert_eq!(metrics.len(), 90); let objects = metrics.iter().find(|m| m.value == 1000000.0); assert!(objects.is_some()); @@ -432,6 +605,18 @@ mod tests { .iter() .find(|m| m.name == SCANNER_ACTIVE_PATHS_MD.get_full_metric_name()); assert_eq!(active_paths.map(|m| m.value), Some(4.0)); + assert_eq!(active_paths.map(|m| m.labels.len()), Some(0)); + + let bucket_drive_result = metrics + .iter() + .find(|m| m.name == SCANNER_BUCKET_DRIVE_RESULT_TOTAL_MD.get_full_metric_name()); + assert_eq!(bucket_drive_result.map(|m| m.value), Some(3.0)); + assert_eq!( + bucket_drive_result + .and_then(|m| m.labels.iter().find(|(name, _)| *name == BUCKET_LABEL)) + .map(|(_, value)| value.as_ref()), + Some("photos") + ); let oldest_active_path_age = metrics .iter() @@ -746,6 +931,37 @@ mod tests { .any(|(name, value)| *name == "reason" && value.as_ref() == "directories") }); assert_eq!(partial_cycles_directories.map(|m| m.value), Some(4.0)); + + let lifecycle_failed = metrics.iter().find(|m| { + m.name == SCANNER_SOURCE_WORK_TOTAL_MD.get_full_metric_name() + && m.labels + .iter() + .any(|(name, value)| *name == SERVER_LABEL && value.as_ref() == "node1:9000") + && m.labels + .iter() + .any(|(name, value)| *name == SOURCE_LABEL && value.as_ref() == "lifecycle") + && m.labels + .iter() + .any(|(name, value)| *name == STATE_LABEL && value.as_ref() == "failed") + }); + assert_eq!(lifecycle_failed.map(|m| m.value), Some(4.0)); + + let current_usage_executed = metrics.iter().find(|m| { + m.name == SCANNER_CYCLE_SOURCE_WORK_MD.get_full_metric_name() + && m.labels + .iter() + .any(|(name, value)| *name == SERVER_LABEL && value.as_ref() == "node1:9000") + && m.labels + .iter() + .any(|(name, value)| *name == CYCLE_SCOPE_LABEL && value.as_ref() == "current") + && m.labels + .iter() + .any(|(name, value)| *name == SOURCE_LABEL && value.as_ref() == "usage") + && m.labels + .iter() + .any(|(name, value)| *name == STATE_LABEL && value.as_ref() == "executed") + }); + assert_eq!(current_usage_executed.map(|m| m.value), Some(8.0)); } #[test] diff --git a/crates/obs/src/metrics/collectors/system_drive.rs b/crates/obs/src/metrics/collectors/system_drive.rs index 7cda25a07..936a88c4e 100644 --- a/crates/obs/src/metrics/collectors/system_drive.rs +++ b/crates/obs/src/metrics/collectors/system_drive.rs @@ -78,6 +78,24 @@ pub struct DriveDetailedStats { pub perc_util: Option, } +/// Detailed drive statistics with runtime topology and per-operation dimensions. +#[derive(Debug, Clone, Default)] +pub(crate) struct DriveRuntimeDetailedStats { + pub(crate) stats: DriveDetailedStats, + pub(crate) pool_index: Option, + pub(crate) set_index: Option, + pub(crate) drive_index: Option, + pub(crate) disk_id: Option, + pub(crate) runtime_state: Option, + pub(crate) healing: bool, + pub(crate) scanning: bool, + pub(crate) offline_duration_seconds: Option, + /// Drive API calls by operation + pub(crate) api_calls: Vec<(String, u64)>, + /// Last-minute API latency by operation, in microseconds + pub(crate) api_latency_by_api_micros: Vec<(String, u64)>, +} + /// Aggregate drive count statistics. #[derive(Debug, Clone, Default)] pub struct DriveCountStats { @@ -93,6 +111,58 @@ pub struct DriveCountStats { /// /// Returns a vector of Prometheus metrics for each drive. pub fn collect_drive_detailed_metrics(stats: &[DriveDetailedStats]) -> Vec { + let runtime_stats = stats + .iter() + .cloned() + .map(|stats| DriveRuntimeDetailedStats { + stats, + ..Default::default() + }) + .collect::>(); + collect_drive_runtime_detailed_metrics(&runtime_stats) +} + +pub(crate) fn collect_drive_runtime_detailed_metrics(stats: &[DriveRuntimeDetailedStats]) -> Vec { + const DRIVE_RUNTIME_STATES: [&str; 5] = ["online", "offline", "returning", "suspect", "unknown"]; + + fn topology_labels(stat: &DriveRuntimeDetailedStats) -> Option<[Cow<'static, str>; 5]> { + Some([ + Cow::Owned(stat.stats.server.clone()), + Cow::Owned(stat.stats.drive.clone()), + Cow::Owned(stat.pool_index.as_ref()?.clone()), + Cow::Owned(stat.set_index.as_ref()?.clone()), + Cow::Owned(stat.drive_index.as_ref()?.clone()), + ]) + } + + fn has_topology_labels(stat: &DriveRuntimeDetailedStats) -> bool { + stat.pool_index.is_some() && stat.set_index.is_some() && stat.drive_index.is_some() + } + + fn normalized_runtime_state(runtime_state: &str) -> &str { + DRIVE_RUNTIME_STATES + .iter() + .copied() + .find(|state| state.eq_ignore_ascii_case(runtime_state)) + .unwrap_or("unknown") + } + + fn push_topology_metric( + metrics: &mut Vec, + descriptor: &'static crate::metrics::schema::MetricDescriptor, + value: f64, + labels: &[Cow<'static, str>; 5], + ) { + metrics.push( + PrometheusMetric::from_descriptor(descriptor, value) + .with_label(SERVER_LABEL, labels[0].clone()) + .with_label(DRIVE_LABEL, labels[1].clone()) + .with_label(POOL_INDEX_LABEL, labels[2].clone()) + .with_label(SET_INDEX_LABEL, labels[3].clone()) + .with_label(DRIVE_INDEX_LABEL, labels[4].clone()), + ); + } + fn push_drive_metric( metrics: &mut Vec, descriptor: &'static crate::metrics::schema::MetricDescriptor, @@ -107,19 +177,49 @@ pub fn collect_drive_detailed_metrics(stats: &[DriveDetailedStats]) -> Vec Vec>(); + + assert_eq!(state_metrics.len(), 5); + assert!(state_metrics.iter().any(|metric| { + metric.value == 1.0 + && metric + .labels + .iter() + .any(|(name, value)| *name == STATE_LABEL && value.as_ref() == "suspect") + })); + assert!(state_metrics.iter().filter(|metric| metric.value == 1.0).all(|metric| { + metric + .labels + .iter() + .any(|(name, value)| *name == STATE_LABEL && value.as_ref() == "suspect") + })); + } + + #[test] + fn drive_offline_duration_zeroes_recovered_topology_drive() { + let stats = vec![DriveRuntimeDetailedStats { + pool_index: Some("0".to_string()), + set_index: Some("1".to_string()), + drive_index: Some("2".to_string()), + runtime_state: Some("online".to_string()), + offline_duration_seconds: None, + stats: DriveDetailedStats { + server: "node1:9000".to_string(), + drive: "/data/disk1".to_string(), + ..Default::default() + }, + ..Default::default() + }]; + + let metrics = collect_drive_runtime_detailed_metrics(&stats); + + assert!(metrics.iter().any(|metric| { + metric.name == DRIVE_OFFLINE_DURATION_SECONDS_MD.get_full_metric_name() + && metric.value == 0.0 + && metric + .labels + .iter() + .any(|(name, value)| *name == SERVER_LABEL && value == "node1:9000") + && metric + .labels + .iter() + .any(|(name, value)| *name == DRIVE_INDEX_LABEL && value == "2") + })); + } + #[test] fn test_collect_drive_count_metrics() { let stats = DriveCountStats { diff --git a/crates/obs/src/metrics/scheduler.rs b/crates/obs/src/metrics/scheduler.rs index eddd04538..a7e4f8325 100644 --- a/crates/obs/src/metrics/scheduler.rs +++ b/crates/obs/src/metrics/scheduler.rs @@ -24,21 +24,27 @@ //! - Host network I/O metrics use crate::metrics::collectors::{ + AuditTargetRuntimeStats, AuditTargetStats, BucketReplicationBacklogStats, BucketReplicationBandwidthStats, + BucketReplicationRuntimeStats, + DriveRuntimeDetailedStats, NotificationStats, + NotificationTargetRuntimeStats, NotificationTargetStats, // System monitoring collectors (migrated from rustfs-obs::system) ProcessAttributeError, ProcessCpuStats, ProcessDiskStats, ProcessMemoryStats, - collect_audit_metrics, + ReplicationRuntimeStats, + ScannerRuntimeStats, + collect_audit_runtime_metrics, collect_bucket_metrics, collect_bucket_replication_backlog_metrics, collect_bucket_replication_bandwidth_metrics, - collect_bucket_replication_metrics, + collect_bucket_replication_runtime_metrics, collect_bucket_usage_metrics, collect_cluster_config_metrics, collect_cluster_health_metrics, @@ -48,24 +54,25 @@ use crate::metrics::collectors::{ collect_cpu_metrics, collect_current_dial9_metrics, collect_drive_count_metrics, - collect_drive_detailed_metrics, + collect_drive_runtime_detailed_metrics, collect_erasure_set_metrics, collect_host_network_metrics, collect_iam_metrics, - collect_ilm_metrics, + collect_ilm_runtime_metrics, collect_memory_metrics, collect_network_metrics, collect_node_metrics, collect_notification_metrics, - collect_notification_target_metrics, + collect_notification_target_runtime_metrics, collect_process_attributes, collect_process_cpu_metrics, collect_process_disk_metrics, collect_process_memory_metrics, collect_process_metrics, - collect_replication_metrics, + collect_replication_runtime_metrics, + collect_request_metrics, collect_resource_metrics, - collect_scanner_metrics, + collect_scanner_runtime_metrics, }; use crate::metrics::config::{ DEFAULT_AUDIT_METRICS_INTERVAL, DEFAULT_BUCKET_METRICS_INTERVAL, DEFAULT_BUCKET_REPLICATION_BANDWIDTH_METRICS_INTERVAL, @@ -77,15 +84,23 @@ use crate::metrics::config::{ use crate::metrics::obs_is_disk_compression_enabled; use crate::metrics::report::{PrometheusMetric, report_metrics}; use crate::metrics::runtime_sources::bucket_monitor_available; -use crate::metrics::schema::audit::{AUDIT_FAILED_MESSAGES_MD, AUDIT_TARGET_QUEUE_LENGTH_MD, AUDIT_TOTAL_MESSAGES_MD}; +use crate::metrics::schema::audit::{ + AUDIT_FAILED_MESSAGES_BY_SERVER_MD, AUDIT_FAILED_MESSAGES_MD, AUDIT_FAILED_STORE_LENGTH_BY_SERVER_MD, + AUDIT_FAILED_STORE_LENGTH_MD, AUDIT_TARGET_QUEUE_LENGTH_BY_SERVER_MD, AUDIT_TARGET_QUEUE_LENGTH_MD, + AUDIT_TOTAL_MESSAGES_BY_SERVER_MD, AUDIT_TOTAL_MESSAGES_MD, SERVER as AUDIT_SERVER_LABEL, +}; use crate::metrics::schema::bucket_replication::{ BUCKET_L, BUCKET_REPL_BANDWIDTH_CURRENT_MD, BUCKET_REPL_BANDWIDTH_LIMIT_MD, BUCKET_REPL_CURRENT_BACKLOG_BYTES_MD, BUCKET_REPL_CURRENT_BACKLOG_COUNT_MD, BUCKET_REPL_CURRENT_TARGET_BACKLOG_BYTES_MD, BUCKET_REPL_CURRENT_TARGET_BACKLOG_COUNT_MD, BUCKET_REPL_DURABLE_MRF_AVAILABLE_MD, BUCKET_REPL_DURABLE_MRF_BACKLOG_BYTES_MD, BUCKET_REPL_DURABLE_MRF_BACKLOG_COUNT_MD, BUCKET_REPL_DURABLE_MRF_TARGET_BACKLOG_BYTES_MD, - BUCKET_REPL_DURABLE_MRF_TARGET_BACKLOG_COUNT_MD, BUCKET_REPL_MRF_DROPPED_COUNT_MD, BUCKET_REPL_MRF_FLUSH_FAILURES_MD, - BUCKET_REPL_MRF_LAST_FLUSH_DURATION_MILLIS_MD, BUCKET_REPL_MRF_MISSED_COUNT_MD, BUCKET_REPL_MRF_PENDING_BYTES_MD, - BUCKET_REPL_MRF_PENDING_COUNT_MD, TARGET_ARN_L, + BUCKET_REPL_DURABLE_MRF_TARGET_BACKLOG_COUNT_MD, BUCKET_REPL_LATENCY_MS_MD, BUCKET_REPL_MRF_DROPPED_COUNT_MD, + BUCKET_REPL_MRF_FLUSH_FAILURES_MD, BUCKET_REPL_MRF_LAST_FLUSH_DURATION_MILLIS_MD, BUCKET_REPL_MRF_MISSED_COUNT_MD, + BUCKET_REPL_MRF_PENDING_BYTES_MD, BUCKET_REPL_MRF_PENDING_COUNT_MD, BUCKET_REPL_PROXY_REQUESTS_TOTAL_MD, + BUCKET_REPL_TARGET_LAST_HOUR_FAILED_BYTES_MD, BUCKET_REPL_TARGET_LAST_HOUR_FAILED_COUNT_MD, + BUCKET_REPL_TARGET_LAST_MIN_FAILED_BYTES_MD, BUCKET_REPL_TARGET_LAST_MIN_FAILED_COUNT_MD, BUCKET_REPL_TARGET_SENT_BYTES_MD, + BUCKET_REPL_TARGET_SENT_COUNT_MD, BUCKET_REPL_TARGET_TOTAL_FAILED_BYTES_MD, BUCKET_REPL_TARGET_TOTAL_FAILED_COUNT_MD, + OPERATION_L, RANGE_L, RESULT_L, TARGET_ARN_L, }; use crate::metrics::schema::cluster::{CLUSTER_BUCKETS_TOTAL_MD, CLUSTER_OBJECTS_TOTAL_MD}; use crate::metrics::schema::cluster_usage::{ @@ -97,17 +112,29 @@ use crate::metrics::schema::cluster_usage::{ }; use crate::metrics::schema::node_bucket::{BUCKET_OBJECTS_TOTAL_MD, BUCKET_QUOTA_BYTES_MD, BUCKET_USAGE_BYTES_MD}; use crate::metrics::schema::notification_target::{ - NOTIFICATION_TARGET_FAILED_MESSAGES_MD, NOTIFICATION_TARGET_QUEUE_LENGTH_MD, NOTIFICATION_TARGET_TOTAL_MESSAGES_MD, + NOTIFICATION_TARGET_FAILED_MESSAGES_BY_SERVER_MD, NOTIFICATION_TARGET_FAILED_MESSAGES_MD, + NOTIFICATION_TARGET_FAILED_STORE_LENGTH_BY_SERVER_MD, NOTIFICATION_TARGET_FAILED_STORE_LENGTH_MD, + NOTIFICATION_TARGET_QUEUE_LENGTH_BY_SERVER_MD, NOTIFICATION_TARGET_QUEUE_LENGTH_MD, + NOTIFICATION_TARGET_TOTAL_MESSAGES_BY_SERVER_MD, NOTIFICATION_TARGET_TOTAL_MESSAGES_MD, SERVER as NOTIFICATION_SERVER_LABEL, TARGET_ID as NOTIFICATION_TARGET_ID_LABEL, TARGET_TYPE as NOTIFICATION_TARGET_TYPE_LABEL, }; +use crate::metrics::schema::scanner::{ + BUCKET_LABEL as SCANNER_BUCKET_LABEL, CYCLE_SCOPE_LABEL as SCANNER_CYCLE_SCOPE_LABEL, DRIVE_LABEL as SCANNER_DRIVE_LABEL, + RESULT_LABEL as SCANNER_RESULT_LABEL, SCANNER_BUCKET_DRIVE_RESULT_TOTAL_MD, SCANNER_CYCLE_BUCKET_DRIVE_RESULT_MD, +}; +use crate::metrics::schema::system_drive::{ + API_LABEL as DRIVE_API_LABEL, DISK_ID_LABEL, DRIVE_API_CALLS_MD, DRIVE_API_LATENCY_BY_API_MD, DRIVE_HEALING_MD, + DRIVE_INDEX_LABEL, DRIVE_INFO_MD, DRIVE_LABEL, DRIVE_OFFLINE_DURATION_SECONDS_MD, DRIVE_RUNTIME_STATE_MD, DRIVE_SCANNING_MD, + POOL_INDEX_LABEL, SET_INDEX_LABEL, STATE_LABEL as DRIVE_STATE_LABEL, +}; use crate::metrics::schema::system_process::{PROCESS_EXECUTABLE_NAME_LABEL, PROCESS_PID_LABEL}; use crate::metrics::stats_collector::{ - ProcessMetricBundle, collect_bucket_replication_bandwidth_stats, collect_bucket_replication_stats_bundle, - collect_bucket_stats, collect_cluster_and_health_stats, collect_cluster_config_stats, collect_cluster_usage_metric_stats, - collect_compression_cluster_stats, collect_disk_and_system_drive_stats, collect_erasure_set_stats, - collect_host_network_stats, collect_iam_stats, collect_ilm_metric_stats, collect_internode_network_stats, - collect_process_metric_bundle_with, collect_replication_stats, collect_scanner_metric_stats, - collect_system_cpu_and_memory_stats_with, + ProcessMetricBundle, collect_api_request_stats, collect_bucket_replication_bandwidth_stats, + collect_bucket_replication_stats_bundle, collect_bucket_stats, collect_cluster_and_health_stats, + collect_cluster_config_stats, collect_cluster_usage_metric_stats, collect_compression_cluster_stats, + collect_disk_and_system_drive_runtime_stats, collect_erasure_set_stats, collect_host_network_stats, collect_iam_stats, + collect_ilm_runtime_metric_stats, collect_internode_network_stats, collect_process_metric_bundle_with, + collect_replication_stats, collect_scanner_runtime_metric_stats, collect_system_cpu_and_memory_stats_with, }; use crate::node_identity::{SERVER_LABEL, current_local_node_identity}; use crate::telemetry::retire_metric_series; @@ -153,7 +180,7 @@ const DEFAULT_REPL_BW_ZERO_TOMBSTONE_CYCLES: u8 = 3; /// Env var that overrides the zero-emission tombstone cycles for removed replication bandwidth series. const ENV_REPL_BW_ZERO_TOMBSTONE_CYCLES: &str = "RUSTFS_METRICS_REPL_BW_ZERO_TOMBSTONE_CYCLES"; const METRICS_RUNTIME_SERVICE_NAME: &str = "metrics_runtime"; -const METRICS_RUNTIME_BASE_COLLECTOR_TASKS: u8 = 10; +const METRICS_RUNTIME_BASE_COLLECTOR_TASKS: u8 = 11; #[derive(Debug, Clone, Copy, PartialEq, Eq)] #[repr(u8)] @@ -168,7 +195,8 @@ enum MetricsCollectorTaskId { BackgroundWorkflowStats = 7, ProcessMetrics = 8, InternodeNetworkStats = 9, - CompressionClusterStats = 10, + RequestStats = 10, + CompressionClusterStats = 11, } const BASE_COLLECTOR_TASK_IDS: [MetricsCollectorTaskId; METRICS_RUNTIME_BASE_COLLECTOR_TASKS as usize] = [ @@ -182,6 +210,7 @@ const BASE_COLLECTOR_TASK_IDS: [MetricsCollectorTaskId; METRICS_RUNTIME_BASE_COL MetricsCollectorTaskId::BackgroundWorkflowStats, MetricsCollectorTaskId::ProcessMetrics, MetricsCollectorTaskId::InternodeNetworkStats, + MetricsCollectorTaskId::RequestStats, ]; const ALL_COLLECTOR_TASK_IDS: [MetricsCollectorTaskId; METRICS_RUNTIME_BASE_COLLECTOR_TASKS as usize + 1] = [ @@ -195,6 +224,7 @@ const ALL_COLLECTOR_TASK_IDS: [MetricsCollectorTaskId; METRICS_RUNTIME_BASE_COLL MetricsCollectorTaskId::BackgroundWorkflowStats, MetricsCollectorTaskId::ProcessMetrics, MetricsCollectorTaskId::InternodeNetworkStats, + MetricsCollectorTaskId::RequestStats, MetricsCollectorTaskId::CompressionClusterStats, ]; @@ -268,8 +298,176 @@ static METRICS_RUNTIME_COLLECTOR_HEALTH: OnceLock type ReplBwKey = (String, String); // (bucket, target_arn) type BucketKey = String; type BucketRangeKey = (String, String); // (bucket, range) -type AuditTargetKey = String; -type NotificationTargetKey = (String, String); // (target_id, target_type) +type AuditLegacyTargetKey = String; +type AuditTargetKey = (String, String); // (server, target_id) +type NotificationLegacyTargetKey = (String, String); // (target_id, target_type) +type NotificationTargetKey = (String, String, String); // (server, target_id, target_type) +type DriveTopologyKey = (String, String, String, String, String); // (server, drive, pool, set, drive_index) +type DriveTopologyApiKey = (String, String, String, String, String, String); // (server, drive, pool, set, drive_index, api) +type DriveInfoKey = (String, String, String, String, String, String); // (server, drive, pool, set, drive_index, disk_id) +type ScannerCycleBucketDriveResultKey = (String, String, String, String, String); // (server, cycle_scope, bucket, drive, result) +type ScannerBucketDriveResultKey = (String, String, String, String); // (server, bucket, drive, result) + +fn drive_info_live_keys(stats: &[DriveRuntimeDetailedStats]) -> HashSet { + stats.iter().filter_map(drive_info_key).collect() +} + +fn drive_topology_live_keys(stats: &[DriveRuntimeDetailedStats]) -> HashSet { + stats.iter().filter_map(drive_topology_key).collect() +} + +fn drive_topology_api_live_keys(stats: &[DriveRuntimeDetailedStats]) -> HashSet { + stats + .iter() + .filter_map(|stat| { + let topology = drive_topology_key(stat)?; + Some( + stat.api_calls + .iter() + .map(|(api, _)| api) + .chain(stat.api_latency_by_api_micros.iter().map(|(api, _)| api)) + .map(move |api| { + ( + topology.0.clone(), + topology.1.clone(), + topology.2.clone(), + topology.3.clone(), + topology.4.clone(), + api.clone(), + ) + }), + ) + }) + .flatten() + .collect() +} + +fn drive_topology_key(stat: &DriveRuntimeDetailedStats) -> Option { + Some(( + stat.stats.server.clone(), + stat.stats.drive.clone(), + stat.pool_index.as_ref()?.clone(), + stat.set_index.as_ref()?.clone(), + stat.drive_index.as_ref()?.clone(), + )) +} + +fn drive_info_key(stat: &DriveRuntimeDetailedStats) -> Option { + let disk_id = stat.disk_id.as_ref().filter(|disk_id| !disk_id.is_empty())?; + Some(( + stat.stats.server.clone(), + stat.stats.drive.clone(), + stat.pool_index.as_ref()?.clone(), + stat.set_index.as_ref()?.clone(), + stat.drive_index.as_ref()?.clone(), + disk_id.clone(), + )) +} + +fn retire_drive_info_metric_series(key: &DriveInfoKey) -> usize { + let labels = [ + (SERVER_LABEL, Cow::Owned(key.0.clone())), + (DRIVE_LABEL, Cow::Owned(key.1.clone())), + (POOL_INDEX_LABEL, Cow::Owned(key.2.clone())), + (SET_INDEX_LABEL, Cow::Owned(key.3.clone())), + (DRIVE_INDEX_LABEL, Cow::Owned(key.4.clone())), + (DISK_ID_LABEL, Cow::Owned(key.5.clone())), + ]; + retire_metric_series(&DRIVE_INFO_MD.get_full_metric_name(), &labels) +} + +fn retire_drive_topology_metric_series(key: &DriveTopologyKey) -> usize { + let labels = [ + (SERVER_LABEL, Cow::Owned(key.0.clone())), + (DRIVE_LABEL, Cow::Owned(key.1.clone())), + (POOL_INDEX_LABEL, Cow::Owned(key.2.clone())), + (SET_INDEX_LABEL, Cow::Owned(key.3.clone())), + (DRIVE_INDEX_LABEL, Cow::Owned(key.4.clone())), + ]; + let mut retired = 0; + for descriptor in [&DRIVE_HEALING_MD, &DRIVE_SCANNING_MD, &DRIVE_OFFLINE_DURATION_SECONDS_MD] { + retired += retire_metric_series(&descriptor.get_full_metric_name(), &labels); + } + for state in ["online", "offline", "returning", "suspect", "unknown"] { + let state_labels = [ + (SERVER_LABEL, Cow::Owned(key.0.clone())), + (DRIVE_LABEL, Cow::Owned(key.1.clone())), + (POOL_INDEX_LABEL, Cow::Owned(key.2.clone())), + (SET_INDEX_LABEL, Cow::Owned(key.3.clone())), + (DRIVE_INDEX_LABEL, Cow::Owned(key.4.clone())), + (DRIVE_STATE_LABEL, Cow::Borrowed(state)), + ]; + retired += retire_metric_series(&DRIVE_RUNTIME_STATE_MD.get_full_metric_name(), &state_labels); + } + retired +} + +fn retire_drive_topology_api_metric_series(key: &DriveTopologyApiKey) -> usize { + let labels = [ + (SERVER_LABEL, Cow::Owned(key.0.clone())), + (DRIVE_LABEL, Cow::Owned(key.1.clone())), + (POOL_INDEX_LABEL, Cow::Owned(key.2.clone())), + (SET_INDEX_LABEL, Cow::Owned(key.3.clone())), + (DRIVE_INDEX_LABEL, Cow::Owned(key.4.clone())), + (DRIVE_API_LABEL, Cow::Owned(key.5.clone())), + ]; + retire_metric_series(&DRIVE_API_CALLS_MD.get_full_metric_name(), &labels) + + retire_metric_series(&DRIVE_API_LATENCY_BY_API_MD.get_full_metric_name(), &labels) +} + +fn scanner_cycle_bucket_drive_result_live_keys(stats: &ScannerRuntimeStats) -> HashSet { + stats + .current_cycle_bucket_drive_results + .iter() + .map(|result| { + ( + stats.server.clone(), + "current".to_string(), + result.bucket.clone(), + result.drive.clone(), + result.result.clone(), + ) + }) + .chain(stats.last_cycle_bucket_drive_results.iter().map(|result| { + ( + stats.server.clone(), + "last".to_string(), + result.bucket.clone(), + result.drive.clone(), + result.result.clone(), + ) + })) + .collect() +} + +fn scanner_bucket_drive_result_live_keys(stats: &ScannerRuntimeStats) -> HashSet { + stats + .bucket_drive_results + .iter() + .map(|result| (stats.server.clone(), result.bucket.clone(), result.drive.clone(), result.result.clone())) + .collect() +} + +fn retire_scanner_cycle_bucket_drive_result_metric_series(key: &ScannerCycleBucketDriveResultKey) -> usize { + let labels = [ + (SERVER_LABEL, Cow::Owned(key.0.clone())), + (SCANNER_CYCLE_SCOPE_LABEL, Cow::Owned(key.1.clone())), + (SCANNER_BUCKET_LABEL, Cow::Owned(key.2.clone())), + (SCANNER_DRIVE_LABEL, Cow::Owned(key.3.clone())), + (SCANNER_RESULT_LABEL, Cow::Owned(key.4.clone())), + ]; + retire_metric_series(&SCANNER_CYCLE_BUCKET_DRIVE_RESULT_MD.get_full_metric_name(), &labels) +} + +fn retire_scanner_bucket_drive_result_metric_series(key: &ScannerBucketDriveResultKey) -> usize { + let labels = [ + (SERVER_LABEL, Cow::Owned(key.0.clone())), + (SCANNER_BUCKET_LABEL, Cow::Owned(key.1.clone())), + (SCANNER_DRIVE_LABEL, Cow::Owned(key.2.clone())), + (SCANNER_RESULT_LABEL, Cow::Owned(key.3.clone())), + ]; + retire_metric_series(&SCANNER_BUCKET_DRIVE_RESULT_TOTAL_MD.get_full_metric_name(), &labels) +} #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Default)] pub struct MetricsRuntimeCollectorHealthSnapshot { @@ -646,6 +844,21 @@ fn repl_backlog_target_live_keys(stats: &[BucketReplicationBacklogStats]) -> Has .collect() } +fn repl_flow_live_keys(stats: &[BucketReplicationRuntimeStats]) -> HashSet { + stats + .iter() + .flat_map(|stat| { + stat.target_flows + .iter() + .map(|target| (stat.stats.bucket.clone(), target.target_arn.clone())) + }) + .collect() +} + +fn repl_proxy_bucket_live_keys(stats: &[BucketReplicationRuntimeStats]) -> HashSet { + stats.iter().map(|stat| stat.stats.bucket.clone()).collect() +} + fn update_series_zero_tombstones( has_seen_valid_snapshot: &mut bool, prev_live_keys: &mut HashSet, @@ -898,22 +1111,33 @@ fn retire_bucket_usage_distribution_series(metric_name: String, bucket: &str, ra retire_metric_series(&metric_name, &labels) } -fn audit_target_live_keys(stats: &[AuditTargetStats]) -> HashSet { - stats.iter().map(|stat| stat.target_id.clone()).collect() +fn audit_target_live_keys(stats: &[AuditTargetRuntimeStats]) -> HashSet { + stats + .iter() + .map(|stat| (stat.server.clone(), stat.target.target_id.clone())) + .collect() } -fn collect_audit_zero_tombstone_metrics(zero_tombstones: &HashMap) -> Vec { +fn audit_legacy_target_live_keys(stats: &[AuditTargetRuntimeStats]) -> HashSet { + stats.iter().map(|stat| stat.target.target_id.clone()).collect() +} + +fn collect_audit_legacy_zero_tombstone_metrics(zero_tombstones: &HashMap) -> Vec { if zero_tombstones.is_empty() { return Vec::new(); } - let mut zero_metrics = Vec::with_capacity(zero_tombstones.len() * 3); + let mut zero_metrics = Vec::with_capacity(zero_tombstones.len() * 4); for target_id in zero_tombstones.keys() { let target_id_label: Cow<'static, str> = Cow::Owned(target_id.clone()); zero_metrics.push( PrometheusMetric::from_descriptor(&AUDIT_FAILED_MESSAGES_MD, 0.0) .with_label(AUDIT_TARGET_ID_LABEL, target_id_label.clone()), ); + zero_metrics.push( + PrometheusMetric::from_descriptor(&AUDIT_FAILED_STORE_LENGTH_MD, 0.0) + .with_label(AUDIT_TARGET_ID_LABEL, target_id_label.clone()), + ); zero_metrics.push( PrometheusMetric::from_descriptor(&AUDIT_TARGET_QUEUE_LENGTH_MD, 0.0) .with_label(AUDIT_TARGET_ID_LABEL, target_id_label.clone()), @@ -926,28 +1150,81 @@ fn collect_audit_zero_tombstone_metrics(zero_tombstones: &HashMap usize { +fn collect_audit_zero_tombstone_metrics(zero_tombstones: &HashMap) -> Vec { + if zero_tombstones.is_empty() { + return Vec::new(); + } + + let mut zero_metrics = Vec::with_capacity(zero_tombstones.len() * 4); + for (server, target_id) in zero_tombstones.keys() { + let server_label: Cow<'static, str> = Cow::Owned(server.clone()); + let target_id_label: Cow<'static, str> = Cow::Owned(target_id.clone()); + zero_metrics.push( + PrometheusMetric::from_descriptor(&AUDIT_FAILED_MESSAGES_BY_SERVER_MD, 0.0) + .with_label(AUDIT_SERVER_LABEL, server_label.clone()) + .with_label(AUDIT_TARGET_ID_LABEL, target_id_label.clone()), + ); + zero_metrics.push( + PrometheusMetric::from_descriptor(&AUDIT_FAILED_STORE_LENGTH_BY_SERVER_MD, 0.0) + .with_label(AUDIT_SERVER_LABEL, server_label.clone()) + .with_label(AUDIT_TARGET_ID_LABEL, target_id_label.clone()), + ); + zero_metrics.push( + PrometheusMetric::from_descriptor(&AUDIT_TARGET_QUEUE_LENGTH_BY_SERVER_MD, 0.0) + .with_label(AUDIT_SERVER_LABEL, server_label.clone()) + .with_label(AUDIT_TARGET_ID_LABEL, target_id_label.clone()), + ); + zero_metrics.push( + PrometheusMetric::from_descriptor(&AUDIT_TOTAL_MESSAGES_BY_SERVER_MD, 0.0) + .with_label(AUDIT_SERVER_LABEL, server_label) + .with_label(AUDIT_TARGET_ID_LABEL, target_id_label), + ); + } + + zero_metrics +} + +fn retire_audit_legacy_target_metric_series(target_id: &str) -> usize { let labels = [(AUDIT_TARGET_ID_LABEL, Cow::Owned(target_id.to_string()))]; retire_metric_series(&AUDIT_FAILED_MESSAGES_MD.get_full_metric_name(), &labels) + + retire_metric_series(&AUDIT_FAILED_STORE_LENGTH_MD.get_full_metric_name(), &labels) + retire_metric_series(&AUDIT_TARGET_QUEUE_LENGTH_MD.get_full_metric_name(), &labels) + retire_metric_series(&AUDIT_TOTAL_MESSAGES_MD.get_full_metric_name(), &labels) } -fn notification_target_live_keys(stats: &[NotificationTargetStats]) -> HashSet { +fn retire_audit_target_metric_series(server: &str, target_id: &str) -> usize { + let server_labels = [ + (AUDIT_SERVER_LABEL, Cow::Owned(server.to_string())), + (AUDIT_TARGET_ID_LABEL, Cow::Owned(target_id.to_string())), + ]; + retire_metric_series(&AUDIT_FAILED_MESSAGES_BY_SERVER_MD.get_full_metric_name(), &server_labels) + + retire_metric_series(&AUDIT_FAILED_STORE_LENGTH_BY_SERVER_MD.get_full_metric_name(), &server_labels) + + retire_metric_series(&AUDIT_TARGET_QUEUE_LENGTH_BY_SERVER_MD.get_full_metric_name(), &server_labels) + + retire_metric_series(&AUDIT_TOTAL_MESSAGES_BY_SERVER_MD.get_full_metric_name(), &server_labels) +} + +fn notification_target_live_keys(stats: &[NotificationTargetRuntimeStats]) -> HashSet { stats .iter() - .map(|stat| (stat.target_id.clone(), stat.target_type.clone())) + .map(|stat| (stat.server.clone(), stat.target.target_id.clone(), stat.target.target_type.clone())) .collect() } -fn collect_notification_target_zero_tombstone_metrics( - zero_tombstones: &HashMap, +fn notification_legacy_target_live_keys(stats: &[NotificationTargetRuntimeStats]) -> HashSet { + stats + .iter() + .map(|stat| (stat.target.target_id.clone(), stat.target.target_type.clone())) + .collect() +} + +fn collect_notification_legacy_target_zero_tombstone_metrics( + zero_tombstones: &HashMap, ) -> Vec { if zero_tombstones.is_empty() { return Vec::new(); } - let mut zero_metrics = Vec::with_capacity(zero_tombstones.len() * 3); + let mut zero_metrics = Vec::with_capacity(zero_tombstones.len() * 4); for (target_id, target_type) in zero_tombstones.keys() { let target_id_label: Cow<'static, str> = Cow::Owned(target_id.clone()); let target_type_label: Cow<'static, str> = Cow::Owned(target_type.clone()); @@ -956,6 +1233,11 @@ fn collect_notification_target_zero_tombstone_metrics( .with_label(NOTIFICATION_TARGET_ID_LABEL, target_id_label.clone()) .with_label(NOTIFICATION_TARGET_TYPE_LABEL, target_type_label.clone()), ); + zero_metrics.push( + PrometheusMetric::from_descriptor(&NOTIFICATION_TARGET_FAILED_STORE_LENGTH_MD, 0.0) + .with_label(NOTIFICATION_TARGET_ID_LABEL, target_id_label.clone()) + .with_label(NOTIFICATION_TARGET_TYPE_LABEL, target_type_label.clone()), + ); zero_metrics.push( PrometheusMetric::from_descriptor(&NOTIFICATION_TARGET_QUEUE_LENGTH_MD, 0.0) .with_label(NOTIFICATION_TARGET_ID_LABEL, target_id_label.clone()) @@ -971,16 +1253,73 @@ fn collect_notification_target_zero_tombstone_metrics( zero_metrics } -fn retire_notification_target_metric_series(target_id: &str, target_type: &str) -> usize { +fn collect_notification_target_zero_tombstone_metrics( + zero_tombstones: &HashMap, +) -> Vec { + if zero_tombstones.is_empty() { + return Vec::new(); + } + + let mut zero_metrics = Vec::with_capacity(zero_tombstones.len() * 4); + for (server, target_id, target_type) in zero_tombstones.keys() { + let server_label: Cow<'static, str> = Cow::Owned(server.clone()); + let target_id_label: Cow<'static, str> = Cow::Owned(target_id.clone()); + let target_type_label: Cow<'static, str> = Cow::Owned(target_type.clone()); + zero_metrics.push( + PrometheusMetric::from_descriptor(&NOTIFICATION_TARGET_FAILED_MESSAGES_BY_SERVER_MD, 0.0) + .with_label(NOTIFICATION_SERVER_LABEL, server_label.clone()) + .with_label(NOTIFICATION_TARGET_ID_LABEL, target_id_label.clone()) + .with_label(NOTIFICATION_TARGET_TYPE_LABEL, target_type_label.clone()), + ); + zero_metrics.push( + PrometheusMetric::from_descriptor(&NOTIFICATION_TARGET_FAILED_STORE_LENGTH_BY_SERVER_MD, 0.0) + .with_label(NOTIFICATION_SERVER_LABEL, server_label.clone()) + .with_label(NOTIFICATION_TARGET_ID_LABEL, target_id_label.clone()) + .with_label(NOTIFICATION_TARGET_TYPE_LABEL, target_type_label.clone()), + ); + zero_metrics.push( + PrometheusMetric::from_descriptor(&NOTIFICATION_TARGET_QUEUE_LENGTH_BY_SERVER_MD, 0.0) + .with_label(NOTIFICATION_SERVER_LABEL, server_label.clone()) + .with_label(NOTIFICATION_TARGET_ID_LABEL, target_id_label.clone()) + .with_label(NOTIFICATION_TARGET_TYPE_LABEL, target_type_label.clone()), + ); + zero_metrics.push( + PrometheusMetric::from_descriptor(&NOTIFICATION_TARGET_TOTAL_MESSAGES_BY_SERVER_MD, 0.0) + .with_label(NOTIFICATION_SERVER_LABEL, server_label) + .with_label(NOTIFICATION_TARGET_ID_LABEL, target_id_label) + .with_label(NOTIFICATION_TARGET_TYPE_LABEL, target_type_label), + ); + } + + zero_metrics +} + +fn retire_notification_legacy_target_metric_series(target_id: &str, target_type: &str) -> usize { let labels = [ (NOTIFICATION_TARGET_ID_LABEL, Cow::Owned(target_id.to_string())), (NOTIFICATION_TARGET_TYPE_LABEL, Cow::Owned(target_type.to_string())), ]; retire_metric_series(&NOTIFICATION_TARGET_FAILED_MESSAGES_MD.get_full_metric_name(), &labels) + + retire_metric_series(&NOTIFICATION_TARGET_FAILED_STORE_LENGTH_MD.get_full_metric_name(), &labels) + retire_metric_series(&NOTIFICATION_TARGET_QUEUE_LENGTH_MD.get_full_metric_name(), &labels) + retire_metric_series(&NOTIFICATION_TARGET_TOTAL_MESSAGES_MD.get_full_metric_name(), &labels) } +fn retire_notification_target_metric_series(server: &str, target_id: &str, target_type: &str) -> usize { + let server_labels = [ + (NOTIFICATION_SERVER_LABEL, Cow::Owned(server.to_string())), + (NOTIFICATION_TARGET_ID_LABEL, Cow::Owned(target_id.to_string())), + (NOTIFICATION_TARGET_TYPE_LABEL, Cow::Owned(target_type.to_string())), + ]; + retire_metric_series(&NOTIFICATION_TARGET_FAILED_MESSAGES_BY_SERVER_MD.get_full_metric_name(), &server_labels) + + retire_metric_series( + &NOTIFICATION_TARGET_FAILED_STORE_LENGTH_BY_SERVER_MD.get_full_metric_name(), + &server_labels, + ) + + retire_metric_series(&NOTIFICATION_TARGET_QUEUE_LENGTH_BY_SERVER_MD.get_full_metric_name(), &server_labels) + + retire_metric_series(&NOTIFICATION_TARGET_TOTAL_MESSAGES_BY_SERVER_MD.get_full_metric_name(), &server_labels) +} + fn update_repl_bw_zero_tombstones( monitor_available: bool, has_seen_valid_snapshot: &mut bool, @@ -1107,6 +1446,27 @@ fn collect_repl_backlog_target_zero_tombstone_metrics(zero_tombstones: &HashMap< zero_metrics } +fn collect_repl_flow_zero_tombstone_metrics(zero_tombstones: &HashMap) -> Vec { + if zero_tombstones.is_empty() { + return Vec::new(); + } + + let mut zero_metrics = Vec::with_capacity(zero_tombstones.len()); + for (bucket, target_arn) in zero_tombstones.keys() { + let bucket_label: Cow<'static, str> = Cow::Owned(bucket.clone()); + let target_arn_label: Cow<'static, str> = Cow::Owned(target_arn.clone()); + zero_metrics.push( + PrometheusMetric::from_descriptor(&BUCKET_REPL_LATENCY_MS_MD, 0.0) + .with_label(BUCKET_L, bucket_label) + .with_label(OPERATION_L, Cow::Borrowed("object_replication")) + .with_label(RANGE_L, Cow::Borrowed("all")) + .with_label(TARGET_ARN_L, target_arn_label), + ); + } + + zero_metrics +} + fn retire_repl_bw_metric_series(bucket: &str, target_arn: &str) -> usize { let labels = [ (BUCKET_L, Cow::Owned(bucket.to_string())), @@ -1116,6 +1476,33 @@ fn retire_repl_bw_metric_series(bucket: &str, target_arn: &str) -> usize { + retire_metric_series(&BUCKET_REPL_BANDWIDTH_CURRENT_MD.get_full_metric_name(), &labels) } +fn retire_repl_flow_metric_series(bucket: &str, target_arn: &str) -> usize { + let labels = [ + (BUCKET_L, Cow::Owned(bucket.to_string())), + (TARGET_ARN_L, Cow::Owned(target_arn.to_string())), + ]; + let latency_labels = [ + (BUCKET_L, Cow::Owned(bucket.to_string())), + (OPERATION_L, Cow::Borrowed("object_replication")), + (RANGE_L, Cow::Borrowed("all")), + (TARGET_ARN_L, Cow::Owned(target_arn.to_string())), + ]; + [ + BUCKET_REPL_TARGET_SENT_BYTES_MD.get_full_metric_name(), + BUCKET_REPL_TARGET_SENT_COUNT_MD.get_full_metric_name(), + BUCKET_REPL_TARGET_TOTAL_FAILED_BYTES_MD.get_full_metric_name(), + BUCKET_REPL_TARGET_TOTAL_FAILED_COUNT_MD.get_full_metric_name(), + BUCKET_REPL_TARGET_LAST_MIN_FAILED_BYTES_MD.get_full_metric_name(), + BUCKET_REPL_TARGET_LAST_MIN_FAILED_COUNT_MD.get_full_metric_name(), + BUCKET_REPL_TARGET_LAST_HOUR_FAILED_BYTES_MD.get_full_metric_name(), + BUCKET_REPL_TARGET_LAST_HOUR_FAILED_COUNT_MD.get_full_metric_name(), + ] + .iter() + .map(|name| retire_metric_series(name, &labels)) + .sum::() + + retire_metric_series(&BUCKET_REPL_LATENCY_MS_MD.get_full_metric_name(), &latency_labels) +} + fn retire_repl_backlog_metric_series(bucket: &str) -> usize { let labels = [(BUCKET_L, Cow::Owned(bucket.to_string()))]; [ @@ -1136,6 +1523,22 @@ fn retire_repl_backlog_metric_series(bucket: &str) -> usize { .sum() } +fn retire_bucket_replication_proxy_request_metric_series(bucket: &str) -> usize { + const PROXY_OPERATIONS: [&str; 6] = ["get", "head", "put", "put_tagging", "get_tagging", "delete_tagging"]; + let mut retired = 0; + for operation in PROXY_OPERATIONS { + for result in ["success", "failure"] { + let labels = [ + (BUCKET_L, Cow::Owned(bucket.to_string())), + (OPERATION_L, Cow::Borrowed(operation)), + (RESULT_L, Cow::Borrowed(result)), + ]; + retired += retire_metric_series(&BUCKET_REPL_PROXY_REQUESTS_TOTAL_MD.get_full_metric_name(), &labels); + } + } + retired +} + fn retire_repl_backlog_target_metric_series(bucket: &str, target_arn: &str) -> usize { let labels = [ (BUCKET_L, Cow::Owned(bucket.to_string())), @@ -1438,15 +1841,53 @@ pub fn init_metrics_runtime(token: CancellationToken) { let token_clone = token.clone(); tokio::spawn(async move { let mut interval = metrics_interval(node_interval, Duration::ZERO); + let mut prev_drive_info_keys: HashSet = HashSet::new(); + let mut prev_drive_topology_keys: HashSet = HashSet::new(); + let mut prev_drive_topology_api_keys: HashSet = HashSet::new(); + let mut has_seen_drive_info_snapshot = false; loop { tokio::select! { _ = interval.tick() => { run_metrics_collector_tick(health, MetricsCollectorTaskId::NodeDiskStats, "node_disk_stats", async { - let (disk_stats, drive_stats, drive_counts) = collect_disk_and_system_drive_stats().await; + let (disk_stats, drive_stats, drive_counts) = collect_disk_and_system_drive_runtime_stats().await; + let current_drive_info_keys = drive_info_live_keys(&drive_stats); + let current_drive_topology_keys = drive_topology_live_keys(&drive_stats); + let current_drive_topology_api_keys = drive_topology_api_live_keys(&drive_stats); + let retire_drive_info_keys = if has_seen_drive_info_snapshot { + prev_drive_info_keys.difference(¤t_drive_info_keys).cloned().collect::>() + } else { + Vec::new() + }; + let retire_drive_topology_keys = if has_seen_drive_info_snapshot { + prev_drive_topology_keys.difference(¤t_drive_topology_keys).cloned().collect::>() + } else { + Vec::new() + }; + let retire_drive_topology_api_keys = if has_seen_drive_info_snapshot { + prev_drive_topology_api_keys + .difference(¤t_drive_topology_api_keys) + .cloned() + .collect::>() + } else { + Vec::new() + }; + prev_drive_info_keys = current_drive_info_keys; + prev_drive_topology_keys = current_drive_topology_keys; + prev_drive_topology_api_keys = current_drive_topology_api_keys; + has_seen_drive_info_snapshot = true; let mut metrics = collect_node_metrics(&disk_stats); - metrics.extend(collect_drive_detailed_metrics(&drive_stats)); + metrics.extend(collect_drive_runtime_detailed_metrics(&drive_stats)); metrics.extend(collect_drive_count_metrics(&drive_counts)); report_metrics(&metrics); + for key in retire_drive_info_keys { + let _ = retire_drive_info_metric_series(&key); + } + for key in retire_drive_topology_keys { + let _ = retire_drive_topology_metric_series(&key); + } + for key in retire_drive_topology_api_keys { + let _ = retire_drive_topology_api_metric_series(&key); + } }).await; } _ = token_clone.cancelled() => { @@ -1471,6 +1912,11 @@ pub fn init_metrics_runtime(token: CancellationToken) { let mut prev_backlog_target_live_keys: HashSet = HashSet::new(); let mut backlog_target_zero_tombstones: HashMap = HashMap::new(); let mut has_seen_valid_backlog_target_snapshot = false; + let mut prev_flow_live_keys: HashSet = HashSet::new(); + let mut flow_zero_tombstones: HashMap = HashMap::new(); + let mut has_seen_valid_flow_snapshot = false; + let mut prev_proxy_bucket_live_keys: HashSet = HashSet::new(); + let mut has_seen_proxy_bucket_snapshot = false; loop { tokio::select! { _ = interval.tick() => { @@ -1501,6 +1947,17 @@ pub fn init_metrics_runtime(token: CancellationToken) { metrics.extend(collect_repl_bw_zero_tombstone_metrics(&zero_tombstones)); let (bucket_replication, bucket_replication_backlog) = collect_bucket_replication_stats_bundle().await; + let current_proxy_bucket_live_keys = repl_proxy_bucket_live_keys(&bucket_replication); + let retire_proxy_buckets = if has_seen_proxy_bucket_snapshot { + prev_proxy_bucket_live_keys + .difference(¤t_proxy_bucket_live_keys) + .cloned() + .collect::>() + } else { + Vec::new() + }; + prev_proxy_bucket_live_keys = current_proxy_bucket_live_keys; + has_seen_proxy_bucket_snapshot = true; let durable_mrf_available = bucket_replication_backlog.iter().any(|stat| stat.durable_mrf_available); let backlog_target_metrics_available = durable_mrf_available || monitor_available || !bucket_replication_backlog.is_empty(); @@ -1521,12 +1978,23 @@ pub fn init_metrics_runtime(token: CancellationToken) { repl_bw_zero_tombstone_cycles, ); } - metrics.extend(collect_bucket_replication_metrics(&bucket_replication)); + metrics.extend(collect_bucket_replication_runtime_metrics(&bucket_replication)); + update_series_zero_tombstones( + &mut has_seen_valid_flow_snapshot, + &mut prev_flow_live_keys, + &mut flow_zero_tombstones, + repl_flow_live_keys(&bucket_replication), + repl_bw_zero_tombstone_cycles, + ); metrics.extend(collect_bucket_replication_backlog_metrics(&bucket_replication_backlog)); metrics.extend(collect_repl_backlog_zero_tombstone_metrics(&backlog_zero_tombstones)); metrics.extend(collect_repl_backlog_target_zero_tombstone_metrics(&backlog_target_zero_tombstones)); + metrics.extend(collect_repl_flow_zero_tombstone_metrics(&flow_zero_tombstones)); let replication = collect_replication_stats().await; - metrics.extend(collect_replication_metrics(&replication)); + metrics.extend(collect_replication_runtime_metrics(&ReplicationRuntimeStats { + server: current_local_node_identity(), + stats: replication, + })); report_metrics(&metrics); // Phase-2: after N cycles, stop reporting -> series becomes absent after expiration. @@ -1542,6 +2010,12 @@ pub fn init_metrics_runtime(token: CancellationToken) { ) { let _ = retire_repl_backlog_target_metric_series(&bucket, &target_arn); } + for (bucket, target_arn) in expire_series_zero_tombstones(&mut flow_zero_tombstones) { + let _ = retire_repl_flow_metric_series(&bucket, &target_arn); + } + for bucket in retire_proxy_buckets { + let _ = retire_bucket_replication_proxy_request_metric_series(&bucket); + } }, ).await; } @@ -1561,18 +2035,25 @@ pub fn init_metrics_runtime(token: CancellationToken) { let mut has_seen_audit_snapshot = false; let mut prev_audit_target_keys: HashSet = HashSet::new(); let mut audit_zero_tombstones: HashMap = HashMap::new(); + let mut has_seen_audit_legacy_snapshot = false; + let mut prev_audit_legacy_target_keys: HashSet = HashSet::new(); + let mut audit_legacy_zero_tombstones: HashMap = HashMap::new(); loop { tokio::select! { _ = interval.tick() => { run_metrics_collector_tick(health, MetricsCollectorTaskId::AuditTargetStats, "audit_target_stats", async { + let server = current_local_node_identity(); let stats = audit_target_metrics().await .into_iter() - .map(|snapshot| AuditTargetStats { - failed_messages: snapshot.failed_messages, - failed_store_length: snapshot.failed_store_length, - queue_length: snapshot.queue_length, - target_id: snapshot.target_id, - total_messages: snapshot.total_messages, + .map(|snapshot| AuditTargetRuntimeStats { + server: server.clone(), + target: AuditTargetStats { + failed_messages: snapshot.failed_messages, + failed_store_length: snapshot.failed_store_length, + queue_length: snapshot.queue_length, + target_id: snapshot.target_id, + total_messages: snapshot.total_messages, + }, }) .collect::>(); update_series_zero_tombstones( @@ -1582,11 +2063,22 @@ pub fn init_metrics_runtime(token: CancellationToken) { audit_target_live_keys(&stats), tombstone_cycles, ); - let mut metrics = collect_audit_metrics(&stats); + update_series_zero_tombstones( + &mut has_seen_audit_legacy_snapshot, + &mut prev_audit_legacy_target_keys, + &mut audit_legacy_zero_tombstones, + audit_legacy_target_live_keys(&stats), + tombstone_cycles, + ); + let mut metrics = collect_audit_runtime_metrics(&stats); + metrics.extend(collect_audit_legacy_zero_tombstone_metrics(&audit_legacy_zero_tombstones)); metrics.extend(collect_audit_zero_tombstone_metrics(&audit_zero_tombstones)); report_metrics(&metrics); - for target_id in expire_series_zero_tombstones(&mut audit_zero_tombstones) { - let _ = retire_audit_target_metric_series(&target_id); + for target_id in expire_series_zero_tombstones(&mut audit_legacy_zero_tombstones) { + let _ = retire_audit_legacy_target_metric_series(&target_id); + } + for (server, target_id) in expire_series_zero_tombstones(&mut audit_zero_tombstones) { + let _ = retire_audit_target_metric_series(&server, &target_id); } }).await; } @@ -1606,6 +2098,9 @@ pub fn init_metrics_runtime(token: CancellationToken) { let mut has_seen_notification_target_snapshot = false; let mut prev_notification_target_keys: HashSet = HashSet::new(); let mut notification_target_zero_tombstones: HashMap = HashMap::new(); + let mut has_seen_notification_legacy_target_snapshot = false; + let mut prev_notification_legacy_target_keys: HashSet = HashSet::new(); + let mut notification_legacy_target_zero_tombstones: HashMap = HashMap::new(); loop { tokio::select! { _ = interval.tick() => { @@ -1618,15 +2113,19 @@ pub fn init_metrics_runtime(token: CancellationToken) { events_skipped_total: snapshot.events_skipped_total, }); + let server = current_local_node_identity(); let target_stats = notification_target_metrics().await .into_iter() - .map(|snapshot| NotificationTargetStats { - failed_messages: snapshot.failed_messages, - failed_store_length: snapshot.failed_store_length, - queue_length: snapshot.queue_length, - target_id: snapshot.target_id, - target_type: snapshot.target_type, - total_messages: snapshot.total_messages, + .map(|snapshot| NotificationTargetRuntimeStats { + server: server.clone(), + target: NotificationTargetStats { + failed_messages: snapshot.failed_messages, + failed_store_length: snapshot.failed_store_length, + queue_length: snapshot.queue_length, + target_id: snapshot.target_id, + target_type: snapshot.target_type, + total_messages: snapshot.total_messages, + }, }) .collect::>(); update_series_zero_tombstones( @@ -1636,13 +2135,26 @@ pub fn init_metrics_runtime(token: CancellationToken) { notification_target_live_keys(&target_stats), tombstone_cycles, ); - metrics.extend(collect_notification_target_metrics(&target_stats)); + update_series_zero_tombstones( + &mut has_seen_notification_legacy_target_snapshot, + &mut prev_notification_legacy_target_keys, + &mut notification_legacy_target_zero_tombstones, + notification_legacy_target_live_keys(&target_stats), + tombstone_cycles, + ); + metrics.extend(collect_notification_target_runtime_metrics(&target_stats)); + metrics.extend(collect_notification_legacy_target_zero_tombstone_metrics( + ¬ification_legacy_target_zero_tombstones, + )); metrics.extend(collect_notification_target_zero_tombstone_metrics( ¬ification_target_zero_tombstones, )); report_metrics(&metrics); - for (target_id, target_type) in expire_series_zero_tombstones(&mut notification_target_zero_tombstones) { - let _ = retire_notification_target_metric_series(&target_id, &target_type); + for (target_id, target_type) in expire_series_zero_tombstones(&mut notification_legacy_target_zero_tombstones) { + let _ = retire_notification_legacy_target_metric_series(&target_id, &target_type); + } + for (server, target_id, target_type) in expire_series_zero_tombstones(&mut notification_target_zero_tombstones) { + let _ = retire_notification_target_metric_series(&server, &target_id, &target_type); } }).await; } @@ -1658,6 +2170,9 @@ pub fn init_metrics_runtime(token: CancellationToken) { let token_clone = token.clone(); tokio::spawn(async move { let mut interval = metrics_interval(cluster_interval, stagger_duration(cluster_interval, 2, 3)); + let mut has_seen_scanner_snapshot = false; + let mut prev_scanner_cycle_bucket_drive_result_keys: HashSet = HashSet::new(); + let mut prev_scanner_bucket_drive_result_keys: HashSet = HashSet::new(); loop { tokio::select! { _ = interval.tick() => { @@ -1668,17 +2183,40 @@ pub fn init_metrics_runtime(token: CancellationToken) { async { let mut metrics = Vec::new(); - if let Some(stats) = collect_ilm_metric_stats().await { - metrics.extend(collect_ilm_metrics(&stats)); + if let Some(stats) = collect_ilm_runtime_metric_stats().await { + metrics.extend(collect_ilm_runtime_metrics(&stats)); } - if let Some(stats) = collect_scanner_metric_stats().await { - metrics.extend(collect_scanner_metrics(&stats)); + let mut retire_scanner_cycle_bucket_drive_result_keys = Vec::new(); + let mut retire_scanner_bucket_drive_result_keys = Vec::new(); + if let Some(stats) = collect_scanner_runtime_metric_stats().await { + let current_cycle_keys = scanner_cycle_bucket_drive_result_live_keys(&stats); + let current_keys = scanner_bucket_drive_result_live_keys(&stats); + if has_seen_scanner_snapshot { + retire_scanner_cycle_bucket_drive_result_keys = prev_scanner_cycle_bucket_drive_result_keys + .difference(¤t_cycle_keys) + .cloned() + .collect(); + retire_scanner_bucket_drive_result_keys = prev_scanner_bucket_drive_result_keys + .difference(¤t_keys) + .cloned() + .collect(); + } + prev_scanner_cycle_bucket_drive_result_keys = current_cycle_keys; + prev_scanner_bucket_drive_result_keys = current_keys; + has_seen_scanner_snapshot = true; + metrics.extend(collect_scanner_runtime_metrics(&stats)); } if !metrics.is_empty() { report_metrics(&metrics); } + for key in retire_scanner_cycle_bucket_drive_result_keys { + let _ = retire_scanner_cycle_bucket_drive_result_metric_series(&key); + } + for key in retire_scanner_bucket_drive_result_keys { + let _ = retire_scanner_bucket_drive_result_metric_series(&key); + } }, ).await; } @@ -1690,6 +2228,28 @@ pub fn init_metrics_runtime(token: CancellationToken) { } }); + // Spawn task for API request metrics. + let token_clone = token.clone(); + tokio::spawn(async move { + let mut interval = metrics_interval(resource_interval, stagger_duration(resource_interval, 1, 2)); + loop { + tokio::select! { + _ = interval.tick() => { + run_metrics_collector_tick(health, MetricsCollectorTaskId::RequestStats, "request_stats", async { + let metrics = collect_request_metrics(&collect_api_request_stats()); + if !metrics.is_empty() { + report_metrics(&metrics); + } + }).await; + } + _ = token_clone.cancelled() => { + warn!(event = EVENT_METRICS_RUNTIME_STATE, component = LOG_COMPONENT_OBS, subsystem = LOG_SUBSYSTEM_METRICS_RUNTIME, collector = "request_stats", state = "cancelled", "metrics runtime state changed"); + return; + } + } + } + }); + // Spawn task for system monitoring metrics (migrated from rustfs-obs::system) let system_interval = config.system_interval; @@ -1985,6 +2545,206 @@ mod tests { keys.iter().map(|bucket| bucket_key(bucket)).collect() } + fn audit_target_key(server: &str, target_id: &str) -> AuditTargetKey { + (server.to_string(), target_id.to_string()) + } + + fn notification_target_key(server: &str, target_id: &str, target_type: &str) -> NotificationTargetKey { + (server.to_string(), target_id.to_string(), target_type.to_string()) + } + + fn drive_info_stat(disk_id: &str) -> DriveRuntimeDetailedStats { + DriveRuntimeDetailedStats { + pool_index: Some("0".to_string()), + set_index: Some("1".to_string()), + drive_index: Some("2".to_string()), + disk_id: Some(disk_id.to_string()), + runtime_state: Some("online".to_string()), + api_calls: vec![("read_all".to_string(), 1)], + api_latency_by_api_micros: vec![("write_all".to_string(), 2)], + stats: crate::metrics::DriveDetailedStats { + server: "server-a".to_string(), + drive: "/data1".to_string(), + ..Default::default() + }, + ..Default::default() + } + } + + fn scanner_stats_with_last_result(bucket: &str) -> ScannerRuntimeStats { + ScannerRuntimeStats { + server: "server-a".to_string(), + last_cycle_bucket_drive_results: vec![crate::metrics::scanner::ScannerBucketDriveResultStats { + bucket: bucket.to_string(), + drive: "/data1".to_string(), + result: "success".to_string(), + count: 1, + }], + ..Default::default() + } + } + + fn scanner_stats_with_lifetime_result(bucket: &str) -> ScannerRuntimeStats { + ScannerRuntimeStats { + server: "server-a".to_string(), + bucket_drive_results: vec![crate::metrics::scanner::ScannerBucketDriveResultStats { + bucket: bucket.to_string(), + drive: "/data1".to_string(), + result: "success".to_string(), + count: 1, + }], + ..Default::default() + } + } + + fn scanner_stats_with_current_result(bucket: &str) -> ScannerRuntimeStats { + ScannerRuntimeStats { + server: "server-a".to_string(), + current_cycle_bucket_drive_results: vec![crate::metrics::scanner::ScannerBucketDriveResultStats { + bucket: bucket.to_string(), + drive: "/data1".to_string(), + result: "success".to_string(), + count: 1, + }], + ..Default::default() + } + } + + #[test] + fn drive_info_live_keys_detect_disk_identity_replacement() { + let previous = drive_info_live_keys(&[drive_info_stat("disk-old")]); + let current = drive_info_live_keys(&[drive_info_stat("disk-new")]); + let retired = previous.difference(¤t).cloned().collect::>(); + + assert!(current.contains(&( + "server-a".to_string(), + "/data1".to_string(), + "0".to_string(), + "1".to_string(), + "2".to_string(), + "disk-new".to_string(), + ))); + assert!(retired.contains(&( + "server-a".to_string(), + "/data1".to_string(), + "0".to_string(), + "1".to_string(), + "2".to_string(), + "disk-old".to_string(), + ))); + } + + #[test] + fn drive_topology_keys_detect_removed_drives() { + let previous = drive_topology_live_keys(&[drive_info_stat("disk-old")]); + let current = drive_topology_live_keys(&[]); + let retired = previous.difference(¤t).cloned().collect::>(); + + assert!(retired.contains(&( + "server-a".to_string(), + "/data1".to_string(), + "0".to_string(), + "1".to_string(), + "2".to_string(), + ))); + } + + #[test] + fn drive_topology_api_keys_detect_removed_drives() { + let previous = drive_topology_api_live_keys(&[drive_info_stat("disk-old")]); + let current = drive_topology_api_live_keys(&[]); + let retired = previous.difference(¤t).cloned().collect::>(); + + assert!(retired.contains(&( + "server-a".to_string(), + "/data1".to_string(), + "0".to_string(), + "1".to_string(), + "2".to_string(), + "read_all".to_string(), + ))); + assert!(retired.contains(&( + "server-a".to_string(), + "/data1".to_string(), + "0".to_string(), + "1".to_string(), + "2".to_string(), + "write_all".to_string(), + ))); + } + + #[test] + fn scanner_last_bucket_drive_result_keys_detect_superseded_cycle_results() { + let previous = scanner_cycle_bucket_drive_result_live_keys(&scanner_stats_with_last_result("photos")); + let current = scanner_cycle_bucket_drive_result_live_keys(&scanner_stats_with_last_result("logs")); + let retired = previous.difference(¤t).cloned().collect::>(); + + assert!(retired.contains(&( + "server-a".to_string(), + "last".to_string(), + "photos".to_string(), + "/data1".to_string(), + "success".to_string(), + ))); + assert!(current.contains(&( + "server-a".to_string(), + "last".to_string(), + "logs".to_string(), + "/data1".to_string(), + "success".to_string(), + ))); + } + + #[test] + fn scanner_bucket_drive_result_keys_detect_completed_current_cycle_results() { + let previous = scanner_cycle_bucket_drive_result_live_keys(&scanner_stats_with_current_result("photos")); + let current = scanner_cycle_bucket_drive_result_live_keys(&ScannerRuntimeStats { + server: "server-a".to_string(), + ..Default::default() + }); + let retired = previous.difference(¤t).cloned().collect::>(); + + assert!(retired.contains(&( + "server-a".to_string(), + "current".to_string(), + "photos".to_string(), + "/data1".to_string(), + "success".to_string(), + ))); + } + + #[test] + fn scanner_bucket_drive_result_keys_detect_evicted_lifetime_results() { + let previous = scanner_bucket_drive_result_live_keys(&scanner_stats_with_lifetime_result("photos")); + let current = scanner_bucket_drive_result_live_keys(&scanner_stats_with_lifetime_result("logs")); + let retired = previous.difference(¤t).cloned().collect::>(); + + assert!(retired.contains(&("server-a".to_string(), "photos".to_string(), "/data1".to_string(), "success".to_string(),))); + assert!(current.contains(&("server-a".to_string(), "logs".to_string(), "/data1".to_string(), "success".to_string(),))); + } + + #[test] + fn replication_proxy_bucket_keys_detect_removed_buckets() { + let previous = repl_proxy_bucket_live_keys(&[BucketReplicationRuntimeStats { + stats: crate::metrics::BucketReplicationStats { + bucket: "photos".to_string(), + ..Default::default() + }, + ..Default::default() + }]); + let current = repl_proxy_bucket_live_keys(&[BucketReplicationRuntimeStats { + stats: crate::metrics::BucketReplicationStats { + bucket: "logs".to_string(), + ..Default::default() + }, + ..Default::default() + }]); + let retired = previous.difference(¤t).cloned().collect::>(); + + assert_eq!(retired, bucket_keys(&["photos"])); + assert_eq!(current, bucket_keys(&["logs"])); + } + #[test] fn metrics_runtime_status_reports_disabled_state() { let snapshot = build_metrics_runtime_status_snapshot(false, false, fixed_metrics_runtime_config(), false); @@ -2066,8 +2826,14 @@ mod tests { #[test] fn metrics_runtime_collector_tasks_expand_when_compression_enabled() { - assert_eq!(metrics_runtime_collector_tasks(false), 10); - assert_eq!(metrics_runtime_collector_tasks(true), 11); + assert_eq!( + metrics_runtime_collector_tasks(false), + u8::try_from(BASE_COLLECTOR_TASK_IDS.len()).expect("base collector count fits in u8") + ); + assert_eq!( + metrics_runtime_collector_tasks(true), + u8::try_from(ALL_COLLECTOR_TASK_IDS.len()).expect("all collector count fits in u8") + ); } #[tokio::test] @@ -2146,6 +2912,151 @@ mod tests { assert!(zero_tombstones.is_empty()); } + #[test] + fn repl_flow_tombstones_zero_latency_and_retire_target_counters() { + let zero_tombstones = HashMap::from([(repl_bw_key("photos", "arn:rustfs:replication:target-a"), 2)]); + + let metrics = collect_repl_flow_zero_tombstone_metrics(&zero_tombstones); + + assert_eq!(metrics.len(), 1); + assert!(metrics.iter().all(|metric| metric.value == 0.0)); + let names = metrics.iter().map(|metric| metric.name.to_string()).collect::>(); + assert!(names.contains(&BUCKET_REPL_LATENCY_MS_MD.get_full_metric_name())); + for metric in metrics { + let labels = metric + .labels + .into_iter() + .map(|(key, value)| (key, value.to_string())) + .collect::>(); + assert_eq!(labels.get(BUCKET_L).map(String::as_str), Some("photos")); + assert_eq!(labels.get(TARGET_ARN_L).map(String::as_str), Some("arn:rustfs:replication:target-a")); + if metric.name == BUCKET_REPL_LATENCY_MS_MD.get_full_metric_name() { + assert_eq!(labels.get(OPERATION_L).map(String::as_str), Some("object_replication")); + assert_eq!(labels.get(RANGE_L).map(String::as_str), Some("all")); + } + } + } + + #[test] + fn audit_target_tombstones_include_server_detail_metrics() { + let zero_tombstones = HashMap::from([(audit_target_key("node1:9000", "audit-webhook"), 2)]); + let metrics = collect_audit_zero_tombstone_metrics(&zero_tombstones); + + assert_eq!(metrics.len(), 4); + assert!(metrics.iter().any(|metric| { + metric.name == AUDIT_TARGET_QUEUE_LENGTH_BY_SERVER_MD.get_full_metric_name() + && metric.value == 0.0 + && metric + .labels + .iter() + .any(|(name, value)| *name == AUDIT_SERVER_LABEL && value == "node1:9000") + && metric + .labels + .iter() + .any(|(name, value)| *name == AUDIT_TARGET_ID_LABEL && value == "audit-webhook") + })); + } + + #[test] + fn audit_legacy_tombstones_follow_target_only_liveness() { + let mut has_seen_snapshot = true; + let mut prev_live_keys = HashSet::from(["audit-webhook".to_string()]); + let mut zero_tombstones = HashMap::new(); + let stats = vec![AuditTargetRuntimeStats { + server: "node2:9000".to_string(), + target: AuditTargetStats { + target_id: "audit-webhook".to_string(), + ..Default::default() + }, + }]; + + update_series_zero_tombstones( + &mut has_seen_snapshot, + &mut prev_live_keys, + &mut zero_tombstones, + audit_legacy_target_live_keys(&stats), + 2, + ); + + assert!(zero_tombstones.is_empty()); + let metrics = collect_audit_legacy_zero_tombstone_metrics(&HashMap::from([("removed-webhook".to_string(), 2)])); + assert_eq!(metrics.len(), 4); + assert!(metrics.iter().any(|metric| { + metric.name == AUDIT_FAILED_STORE_LENGTH_MD.get_full_metric_name() + && metric.value == 0.0 + && metric + .labels + .iter() + .any(|(name, value)| *name == AUDIT_TARGET_ID_LABEL && value == "removed-webhook") + })); + } + + #[test] + fn notification_target_tombstones_include_server_detail_metrics() { + let zero_tombstones = HashMap::from([(notification_target_key("node1:9000", "primary:webhook", "webhook"), 2)]); + let metrics = collect_notification_target_zero_tombstone_metrics(&zero_tombstones); + + assert_eq!(metrics.len(), 4); + assert!(metrics.iter().any(|metric| { + metric.name == NOTIFICATION_TARGET_QUEUE_LENGTH_BY_SERVER_MD.get_full_metric_name() + && metric.value == 0.0 + && metric + .labels + .iter() + .any(|(name, value)| *name == NOTIFICATION_SERVER_LABEL && value == "node1:9000") + && metric + .labels + .iter() + .any(|(name, value)| *name == NOTIFICATION_TARGET_ID_LABEL && value == "primary:webhook") + && metric + .labels + .iter() + .any(|(name, value)| *name == NOTIFICATION_TARGET_TYPE_LABEL && value == "webhook") + })); + } + + #[test] + fn notification_legacy_tombstones_follow_target_only_liveness() { + let mut has_seen_snapshot = true; + let mut prev_live_keys = HashSet::from([("primary:webhook".to_string(), "webhook".to_string())]); + let mut zero_tombstones = HashMap::new(); + let stats = vec![NotificationTargetRuntimeStats { + server: "node2:9000".to_string(), + target: NotificationTargetStats { + target_id: "primary:webhook".to_string(), + target_type: "webhook".to_string(), + ..Default::default() + }, + }]; + + update_series_zero_tombstones( + &mut has_seen_snapshot, + &mut prev_live_keys, + &mut zero_tombstones, + notification_legacy_target_live_keys(&stats), + 2, + ); + + assert!(zero_tombstones.is_empty()); + let metrics = collect_notification_legacy_target_zero_tombstone_metrics(&HashMap::from([( + ("removed:webhook".to_string(), "webhook".to_string()), + 2, + )])); + assert_eq!(metrics.len(), 4); + assert!(metrics.iter().any(|metric| { + metric.name == NOTIFICATION_TARGET_FAILED_STORE_LENGTH_MD.get_full_metric_name() + && metric.value == 0.0 + && metric + .labels + .iter() + .any(|(name, value)| *name == NOTIFICATION_TARGET_ID_LABEL && value == "removed:webhook") + && metric + .labels + .iter() + .any(|(name, value)| *name == NOTIFICATION_TARGET_TYPE_LABEL && value == "webhook") + })); + } + #[test] fn repl_bw_tombstones_stop_zeroing_when_key_becomes_live_again() { let mut has_seen_valid_snapshot = false; diff --git a/crates/obs/src/metrics/schema/audit.rs b/crates/obs/src/metrics/schema/audit.rs index ceeb9045e..b08e95269 100644 --- a/crates/obs/src/metrics/schema/audit.rs +++ b/crates/obs/src/metrics/schema/audit.rs @@ -17,18 +17,31 @@ use crate::{MetricDescriptor, MetricName, new_gauge_md, subsystems}; use std::sync::LazyLock; -const TARGET_ID: &str = "target_id"; +pub const TARGET_ID: &str = "target_id"; +pub const SERVER: &str = "server"; pub const RESULT: &str = "result"; // success / failure pub const STATUS: &str = "status"; // success / failure pub const SUCCESS: &str = "success"; pub const FAILURE: &str = "failure"; +const TARGET_LABELS: [&str; 1] = [TARGET_ID]; +const TARGET_SERVER_LABELS: [&str; 2] = [SERVER, TARGET_ID]; + pub static AUDIT_FAILED_MESSAGES_MD: LazyLock = LazyLock::new(|| { new_gauge_md( MetricName::AuditFailedMessages, "Total number of messages that failed to send since start", - &[TARGET_ID], + &TARGET_LABELS, + subsystems::AUDIT, + ) +}); + +pub static AUDIT_FAILED_MESSAGES_BY_SERVER_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::Custom("failed_messages_by_server".to_string()), + "Total number of messages that failed to send since start by server and target", + &TARGET_SERVER_LABELS, subsystems::AUDIT, ) }); @@ -37,7 +50,16 @@ pub static AUDIT_FAILED_STORE_LENGTH_MD: LazyLock = LazyLock:: new_gauge_md( MetricName::AuditFailedStoreLength, "Number of audit messages held in the failed-events store for target", - &[TARGET_ID], + &TARGET_LABELS, + subsystems::AUDIT, + ) +}); + +pub static AUDIT_FAILED_STORE_LENGTH_BY_SERVER_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::Custom("failed_store_length_by_server".to_string()), + "Number of audit messages held in the failed-events store by server and target", + &TARGET_SERVER_LABELS, subsystems::AUDIT, ) }); @@ -46,7 +68,16 @@ pub static AUDIT_TARGET_QUEUE_LENGTH_MD: LazyLock = LazyLock:: new_gauge_md( MetricName::AuditTargetQueueLength, "Number of unsent messages in queue for target", - &[TARGET_ID], + &TARGET_LABELS, + subsystems::AUDIT, + ) +}); + +pub static AUDIT_TARGET_QUEUE_LENGTH_BY_SERVER_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::Custom("target_queue_length_by_server".to_string()), + "Number of unsent audit messages in queue by server and target", + &TARGET_SERVER_LABELS, subsystems::AUDIT, ) }); @@ -55,7 +86,16 @@ pub static AUDIT_TOTAL_MESSAGES_MD: LazyLock = LazyLock::new(| new_gauge_md( MetricName::AuditTotalMessages, "Total number of messages sent since start", - &[TARGET_ID], + &TARGET_LABELS, + subsystems::AUDIT, + ) +}); + +pub static AUDIT_TOTAL_MESSAGES_BY_SERVER_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::Custom("total_messages_by_server".to_string()), + "Total number of messages sent since start by server and target", + &TARGET_SERVER_LABELS, subsystems::AUDIT, ) }); diff --git a/crates/obs/src/metrics/schema/bucket_replication.rs b/crates/obs/src/metrics/schema/bucket_replication.rs index 823edd980..1d7b1c924 100644 --- a/crates/obs/src/metrics/schema/bucket_replication.rs +++ b/crates/obs/src/metrics/schema/bucket_replication.rs @@ -21,6 +21,8 @@ use std::sync::LazyLock; pub const BUCKET_L: &str = "bucket"; /// Replication operation pub const OPERATION_L: &str = "operation"; +/// Replication proxy result +pub const RESULT_L: &str = "result"; /// Replication target ARN pub const TARGET_ARN_L: &str = "target_arn"; /// Replication range @@ -48,6 +50,16 @@ const MRF_DROPPED_COUNT: &str = "mrf_dropped_count"; const MRF_MISSED_COUNT: &str = "mrf_missed_count"; const MRF_FLUSH_FAILURES: &str = "mrf_flush_failures"; const MRF_LAST_FLUSH_DURATION_MILLIS: &str = "mrf_last_flush_duration_millis"; +const TARGET_SENT_BYTES: &str = "target_sent_bytes"; +const TARGET_SENT_COUNT: &str = "target_sent_count"; +const TARGET_TOTAL_FAILED_BYTES: &str = "target_total_failed_bytes"; +const TARGET_TOTAL_FAILED_COUNT: &str = "target_total_failed_count"; +const TARGET_LAST_MIN_FAILED_BYTES: &str = "target_last_min_failed_bytes"; +const TARGET_LAST_MIN_FAILED_COUNT: &str = "target_last_min_failed_count"; +const TARGET_LAST_HOUR_FAILED_BYTES: &str = "target_last_hour_failed_bytes"; +const TARGET_LAST_HOUR_FAILED_COUNT: &str = "target_last_hour_failed_count"; + +const BUCKET_TARGET_LABELS: [&str; 2] = [BUCKET_L, TARGET_ARN_L]; pub static BUCKET_REPL_LAST_HR_FAILED_BYTES_MD: LazyLock = LazyLock::new(|| { new_gauge_md( @@ -58,6 +70,15 @@ pub static BUCKET_REPL_LAST_HR_FAILED_BYTES_MD: LazyLock = Laz ) }); +pub static BUCKET_REPL_TARGET_LAST_HOUR_FAILED_BYTES_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::from(TARGET_LAST_HOUR_FAILED_BYTES), + "Total number of bytes failed at least once to replicate in the last hour on a bucket and target ARN", + &BUCKET_TARGET_LABELS, + subsystems::BUCKET_REPLICATION, + ) +}); + pub static BUCKET_REPL_LAST_HR_FAILED_COUNT_MD: LazyLock = LazyLock::new(|| { new_gauge_md( MetricName::LastHourFailedCount, @@ -67,6 +88,15 @@ pub static BUCKET_REPL_LAST_HR_FAILED_COUNT_MD: LazyLock = Laz ) }); +pub static BUCKET_REPL_TARGET_LAST_HOUR_FAILED_COUNT_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::from(TARGET_LAST_HOUR_FAILED_COUNT), + "Total number of objects which failed replication in the last hour on a bucket and target ARN", + &BUCKET_TARGET_LABELS, + subsystems::BUCKET_REPLICATION, + ) +}); + pub static BUCKET_REPL_LAST_MIN_FAILED_BYTES_MD: LazyLock = LazyLock::new(|| { new_gauge_md( MetricName::LastMinFailedBytes, @@ -76,6 +106,15 @@ pub static BUCKET_REPL_LAST_MIN_FAILED_BYTES_MD: LazyLock = La ) }); +pub static BUCKET_REPL_TARGET_LAST_MIN_FAILED_BYTES_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::from(TARGET_LAST_MIN_FAILED_BYTES), + "Total number of bytes failed at least once to replicate in the last full minute on a bucket and target ARN", + &BUCKET_TARGET_LABELS, + subsystems::BUCKET_REPLICATION, + ) +}); + pub static BUCKET_REPL_LAST_MIN_FAILED_COUNT_MD: LazyLock = LazyLock::new(|| { new_gauge_md( MetricName::LastMinFailedCount, @@ -85,6 +124,15 @@ pub static BUCKET_REPL_LAST_MIN_FAILED_COUNT_MD: LazyLock = La ) }); +pub static BUCKET_REPL_TARGET_LAST_MIN_FAILED_COUNT_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::from(TARGET_LAST_MIN_FAILED_COUNT), + "Total number of objects which failed replication in the last full minute on a bucket and target ARN", + &BUCKET_TARGET_LABELS, + subsystems::BUCKET_REPLICATION, + ) +}); + pub static BUCKET_REPL_LATENCY_MS_MD: LazyLock = LazyLock::new(|| { new_gauge_md( MetricName::LatencyMilliSec, @@ -337,6 +385,15 @@ pub static BUCKET_REPL_SENT_BYTES_MD: LazyLock = LazyLock::new ) }); +pub static BUCKET_REPL_TARGET_SENT_BYTES_MD: LazyLock = LazyLock::new(|| { + new_counter_md( + MetricName::from(TARGET_SENT_BYTES), + "Total number of bytes replicated to a bucket replication target ARN", + &BUCKET_TARGET_LABELS, + subsystems::BUCKET_REPLICATION, + ) +}); + pub static BUCKET_REPL_SENT_COUNT_MD: LazyLock = LazyLock::new(|| { new_counter_md( MetricName::SentCount, @@ -346,6 +403,15 @@ pub static BUCKET_REPL_SENT_COUNT_MD: LazyLock = LazyLock::new ) }); +pub static BUCKET_REPL_TARGET_SENT_COUNT_MD: LazyLock = LazyLock::new(|| { + new_counter_md( + MetricName::from(TARGET_SENT_COUNT), + "Total number of objects replicated to a bucket replication target ARN", + &BUCKET_TARGET_LABELS, + subsystems::BUCKET_REPLICATION, + ) +}); + pub static BUCKET_REPL_RESYNC_STARTED_TOTAL_MD: LazyLock = LazyLock::new(|| { new_counter_md( MetricName::from(RESYNC_STARTED_TOTAL), @@ -400,6 +466,15 @@ pub static BUCKET_REPL_TOTAL_FAILED_BYTES_MD: LazyLock = LazyL ) }); +pub static BUCKET_REPL_TARGET_TOTAL_FAILED_BYTES_MD: LazyLock = LazyLock::new(|| { + new_counter_md( + MetricName::from(TARGET_TOTAL_FAILED_BYTES), + "Total number of bytes failed at least once to replicate since server start by bucket and target ARN", + &BUCKET_TARGET_LABELS, + subsystems::BUCKET_REPLICATION, + ) +}); + pub static BUCKET_REPL_TOTAL_FAILED_COUNT_MD: LazyLock = LazyLock::new(|| { new_counter_md( MetricName::TotalFailedCount, @@ -409,6 +484,15 @@ pub static BUCKET_REPL_TOTAL_FAILED_COUNT_MD: LazyLock = LazyL ) }); +pub static BUCKET_REPL_TARGET_TOTAL_FAILED_COUNT_MD: LazyLock = LazyLock::new(|| { + new_counter_md( + MetricName::from(TARGET_TOTAL_FAILED_COUNT), + "Total number of objects which failed replication since server start by bucket and target ARN", + &BUCKET_TARGET_LABELS, + subsystems::BUCKET_REPLICATION, + ) +}); + pub static BUCKET_REPL_BANDWIDTH_LIMIT_MD: LazyLock = LazyLock::new(|| { new_gauge_md( MetricName::BandwidthLimitBytesPerSecond, @@ -435,3 +519,12 @@ pub static BUCKET_REPL_PROXIED_DELETE_TAGGING_REQUESTS_FAILURES_MD: LazyLock = LazyLock::new(|| { + new_counter_md( + MetricName::Custom("proxy_requests_total".to_string()), + "Total number of bucket replication proxy requests by operation and result", + &[BUCKET_L, OPERATION_L, RESULT_L], + subsystems::BUCKET_REPLICATION, + ) +}); diff --git a/crates/obs/src/metrics/schema/ilm.rs b/crates/obs/src/metrics/schema/ilm.rs index 253464552..01316de37 100644 --- a/crates/obs/src/metrics/schema/ilm.rs +++ b/crates/obs/src/metrics/schema/ilm.rs @@ -17,6 +17,19 @@ use crate::{MetricDescriptor, MetricName, new_counter_md, new_gauge_md, subsystems}; use std::sync::LazyLock; +pub const SERVER_LABEL: &str = "server"; +pub const ACTION_LABEL: &str = "action"; +pub const STATE_LABEL: &str = "state"; + +pub static ILM_ACTION_TASKS_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::Custom("action_tasks".to_string()), + "ILM task counts by server, action, and state", + &[SERVER_LABEL, ACTION_LABEL, STATE_LABEL], + subsystems::ILM, + ) +}); + pub static ILM_EXPIRY_PENDING_TASKS_MD: LazyLock = LazyLock::new(|| { new_gauge_md( MetricName::IlmExpiryPendingTasks, diff --git a/crates/obs/src/metrics/schema/notification_target.rs b/crates/obs/src/metrics/schema/notification_target.rs index 3a1c6e491..32c4c4502 100644 --- a/crates/obs/src/metrics/schema/notification_target.rs +++ b/crates/obs/src/metrics/schema/notification_target.rs @@ -19,8 +19,10 @@ use std::sync::LazyLock; pub const TARGET_ID: &str = "target_id"; pub const TARGET_TYPE: &str = "target_type"; +pub const SERVER: &str = "server"; const NOTIFICATION_TARGET_LABELS: [&str; 2] = [TARGET_ID, TARGET_TYPE]; +const NOTIFICATION_TARGET_SERVER_LABELS: [&str; 3] = [SERVER, TARGET_ID, TARGET_TYPE]; pub static NOTIFICATION_TARGET_FAILED_MESSAGES_MD: LazyLock = LazyLock::new(|| { new_gauge_md( @@ -31,6 +33,15 @@ pub static NOTIFICATION_TARGET_FAILED_MESSAGES_MD: LazyLock = ) }); +pub static NOTIFICATION_TARGET_FAILED_MESSAGES_BY_SERVER_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::Custom("target_failed_messages_by_server".to_string()), + "Total number of notification messages that permanently failed to send by server and target", + &NOTIFICATION_TARGET_SERVER_LABELS, + subsystems::NOTIFICATION, + ) +}); + pub static NOTIFICATION_TARGET_FAILED_STORE_LENGTH_MD: LazyLock = LazyLock::new(|| { new_gauge_md( MetricName::NotificationTargetFailedStoreLength, @@ -40,6 +51,15 @@ pub static NOTIFICATION_TARGET_FAILED_STORE_LENGTH_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::Custom("target_failed_store_length_by_server".to_string()), + "Number of notification messages held in the failed-events store by server and target", + &NOTIFICATION_TARGET_SERVER_LABELS, + subsystems::NOTIFICATION, + ) +}); + pub static NOTIFICATION_TARGET_QUEUE_LENGTH_MD: LazyLock = LazyLock::new(|| { new_gauge_md( MetricName::NotificationTargetQueueLength, @@ -49,6 +69,15 @@ pub static NOTIFICATION_TARGET_QUEUE_LENGTH_MD: LazyLock = Laz ) }); +pub static NOTIFICATION_TARGET_QUEUE_LENGTH_BY_SERVER_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::Custom("target_queue_length_by_server".to_string()), + "Number of queued notification messages pending delivery by server and target", + &NOTIFICATION_TARGET_SERVER_LABELS, + subsystems::NOTIFICATION, + ) +}); + pub static NOTIFICATION_TARGET_TOTAL_MESSAGES_MD: LazyLock = LazyLock::new(|| { new_gauge_md( MetricName::NotificationTargetTotalMessages, @@ -57,3 +86,12 @@ pub static NOTIFICATION_TARGET_TOTAL_MESSAGES_MD: LazyLock = L subsystems::NOTIFICATION, ) }); + +pub static NOTIFICATION_TARGET_TOTAL_MESSAGES_BY_SERVER_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::Custom("target_total_messages_by_server".to_string()), + "Total number of notification messages successfully delivered by server and target", + &NOTIFICATION_TARGET_SERVER_LABELS, + subsystems::NOTIFICATION, + ) +}); diff --git a/crates/obs/src/metrics/schema/replication.rs b/crates/obs/src/metrics/schema/replication.rs index db1563255..93cd0cbd7 100644 --- a/crates/obs/src/metrics/schema/replication.rs +++ b/crates/obs/src/metrics/schema/replication.rs @@ -17,6 +17,8 @@ use crate::{MetricDescriptor, MetricName, new_gauge_md, subsystems}; use std::sync::LazyLock; +pub const SERVER_LABEL: &str = "server"; + pub static REPLICATION_AVERAGE_ACTIVE_WORKERS_MD: LazyLock = LazyLock::new(|| { new_gauge_md( MetricName::ReplicationAverageActiveWorkers, @@ -26,6 +28,15 @@ pub static REPLICATION_AVERAGE_ACTIVE_WORKERS_MD: LazyLock = L ) }); +pub static REPLICATION_AVERAGE_ACTIVE_WORKERS_BY_SERVER_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::Custom("average_active_workers_by_server".to_string()), + "Average number of active replication workers by server", + &[SERVER_LABEL], + subsystems::REPLICATION, + ) +}); + pub static REPLICATION_AVERAGE_QUEUED_BYTES_MD: LazyLock = LazyLock::new(|| { new_gauge_md( MetricName::ReplicationAverageQueuedBytes, @@ -35,6 +46,15 @@ pub static REPLICATION_AVERAGE_QUEUED_BYTES_MD: LazyLock = Laz ) }); +pub static REPLICATION_AVERAGE_QUEUED_BYTES_BY_SERVER_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::Custom("average_queued_bytes_by_server".to_string()), + "Average number of bytes queued for replication since server start by server", + &[SERVER_LABEL], + subsystems::REPLICATION, + ) +}); + pub static REPLICATION_AVERAGE_QUEUED_COUNT_MD: LazyLock = LazyLock::new(|| { new_gauge_md( MetricName::ReplicationAverageQueuedCount, @@ -44,6 +64,15 @@ pub static REPLICATION_AVERAGE_QUEUED_COUNT_MD: LazyLock = Laz ) }); +pub static REPLICATION_AVERAGE_QUEUED_COUNT_BY_SERVER_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::Custom("average_queued_count_by_server".to_string()), + "Average number of objects queued for replication since server start by server", + &[SERVER_LABEL], + subsystems::REPLICATION, + ) +}); + pub static REPLICATION_AVERAGE_DATA_TRANSFER_RATE_MD: LazyLock = LazyLock::new(|| { new_gauge_md( MetricName::ReplicationAverageDataTransferRate, @@ -53,6 +82,15 @@ pub static REPLICATION_AVERAGE_DATA_TRANSFER_RATE_MD: LazyLock ) }); +pub static REPLICATION_AVERAGE_DATA_TRANSFER_RATE_BY_SERVER_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::Custom("average_data_transfer_rate_by_server".to_string()), + "Average replication data transfer rate in bytes/sec by server", + &[SERVER_LABEL], + subsystems::REPLICATION, + ) +}); + pub static REPLICATION_CURRENT_ACTIVE_WORKERS_MD: LazyLock = LazyLock::new(|| { new_gauge_md( MetricName::ReplicationCurrentActiveWorkers, @@ -62,6 +100,15 @@ pub static REPLICATION_CURRENT_ACTIVE_WORKERS_MD: LazyLock = L ) }); +pub static REPLICATION_CURRENT_ACTIVE_WORKERS_BY_SERVER_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::Custom("current_active_workers_by_server".to_string()), + "Total number of active replication workers by server", + &[SERVER_LABEL], + subsystems::REPLICATION, + ) +}); + pub static REPLICATION_CURRENT_DATA_TRANSFER_RATE_MD: LazyLock = LazyLock::new(|| { new_gauge_md( MetricName::ReplicationCurrentDataTransferRate, @@ -71,6 +118,15 @@ pub static REPLICATION_CURRENT_DATA_TRANSFER_RATE_MD: LazyLock ) }); +pub static REPLICATION_CURRENT_DATA_TRANSFER_RATE_BY_SERVER_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::Custom("current_data_transfer_rate_by_server".to_string()), + "Current replication data transfer rate in bytes/sec by server", + &[SERVER_LABEL], + subsystems::REPLICATION, + ) +}); + pub static REPLICATION_LAST_MINUTE_QUEUED_BYTES_MD: LazyLock = LazyLock::new(|| { new_gauge_md( MetricName::ReplicationLastMinuteQueuedBytes, @@ -80,6 +136,15 @@ pub static REPLICATION_LAST_MINUTE_QUEUED_BYTES_MD: LazyLock = ) }); +pub static REPLICATION_LAST_MINUTE_QUEUED_BYTES_BY_SERVER_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::Custom("last_minute_queued_bytes_by_server".to_string()), + "Number of bytes queued for replication in the last full minute by server", + &[SERVER_LABEL], + subsystems::REPLICATION, + ) +}); + pub static REPLICATION_LAST_MINUTE_QUEUED_COUNT_MD: LazyLock = LazyLock::new(|| { new_gauge_md( MetricName::ReplicationLastMinuteQueuedCount, @@ -89,6 +154,15 @@ pub static REPLICATION_LAST_MINUTE_QUEUED_COUNT_MD: LazyLock = ) }); +pub static REPLICATION_LAST_MINUTE_QUEUED_COUNT_BY_SERVER_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::Custom("last_minute_queued_count_by_server".to_string()), + "Number of objects queued for replication in the last full minute by server", + &[SERVER_LABEL], + subsystems::REPLICATION, + ) +}); + pub static REPLICATION_MAX_ACTIVE_WORKERS_MD: LazyLock = LazyLock::new(|| { new_gauge_md( MetricName::ReplicationMaxActiveWorkers, @@ -98,6 +172,15 @@ pub static REPLICATION_MAX_ACTIVE_WORKERS_MD: LazyLock = LazyL ) }); +pub static REPLICATION_MAX_ACTIVE_WORKERS_BY_SERVER_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::Custom("max_active_workers_by_server".to_string()), + "Maximum number of active replication workers seen since server start by server", + &[SERVER_LABEL], + subsystems::REPLICATION, + ) +}); + pub static REPLICATION_MAX_QUEUED_BYTES_MD: LazyLock = LazyLock::new(|| { new_gauge_md( MetricName::ReplicationMaxQueuedBytes, @@ -107,6 +190,15 @@ pub static REPLICATION_MAX_QUEUED_BYTES_MD: LazyLock = LazyLoc ) }); +pub static REPLICATION_MAX_QUEUED_BYTES_BY_SERVER_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::Custom("max_queued_bytes_by_server".to_string()), + "Maximum number of bytes queued for replication since server start by server", + &[SERVER_LABEL], + subsystems::REPLICATION, + ) +}); + pub static REPLICATION_MAX_QUEUED_COUNT_MD: LazyLock = LazyLock::new(|| { new_gauge_md( MetricName::ReplicationMaxQueuedCount, @@ -116,6 +208,15 @@ pub static REPLICATION_MAX_QUEUED_COUNT_MD: LazyLock = LazyLoc ) }); +pub static REPLICATION_MAX_QUEUED_COUNT_BY_SERVER_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::Custom("max_queued_count_by_server".to_string()), + "Maximum number of objects queued for replication since server start by server", + &[SERVER_LABEL], + subsystems::REPLICATION, + ) +}); + pub static REPLICATION_MAX_DATA_TRANSFER_RATE_MD: LazyLock = LazyLock::new(|| { new_gauge_md( MetricName::ReplicationMaxDataTransferRate, @@ -125,6 +226,15 @@ pub static REPLICATION_MAX_DATA_TRANSFER_RATE_MD: LazyLock = L ) }); +pub static REPLICATION_MAX_DATA_TRANSFER_RATE_BY_SERVER_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::Custom("max_data_transfer_rate_by_server".to_string()), + "Maximum replication data transfer rate in bytes/sec seen since server start by server", + &[SERVER_LABEL], + subsystems::REPLICATION, + ) +}); + pub static REPLICATION_RECENT_BACKLOG_COUNT_MD: LazyLock = LazyLock::new(|| { new_gauge_md( MetricName::ReplicationRecentBacklogCount, @@ -133,3 +243,12 @@ pub static REPLICATION_RECENT_BACKLOG_COUNT_MD: LazyLock = Laz subsystems::REPLICATION, ) }); + +pub static REPLICATION_RECENT_BACKLOG_COUNT_BY_SERVER_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::Custom("recent_backlog_count_by_server".to_string()), + "Objects currently in replication backlog by server", + &[SERVER_LABEL], + subsystems::REPLICATION, + ) +}); diff --git a/crates/obs/src/metrics/schema/request.rs b/crates/obs/src/metrics/schema/request.rs index be037d653..9f90a64c9 100644 --- a/crates/obs/src/metrics/schema/request.rs +++ b/crates/obs/src/metrics/schema/request.rs @@ -22,6 +22,15 @@ pub const NAME_LABEL: &str = "name"; pub const TYPE_LABEL: &str = "type"; /// le label (for histogram buckets) pub const LE_LABEL: &str = "le"; +/// server label +pub const SERVER_LABEL: &str = "server"; + +const API_NAME_TYPE_LABELS: [&str; 2] = [NAME_LABEL, TYPE_LABEL]; +const API_SERVER_NAME_TYPE_LABELS: [&str; 3] = [SERVER_LABEL, NAME_LABEL, TYPE_LABEL]; +const API_NAME_TYPE_LE_LABELS: [&str; 3] = [NAME_LABEL, TYPE_LABEL, LE_LABEL]; +const API_SERVER_NAME_TYPE_LE_LABELS: [&str; 4] = [SERVER_LABEL, NAME_LABEL, TYPE_LABEL, LE_LABEL]; +const API_TYPE_LABELS: [&str; 1] = [TYPE_LABEL]; +const API_SERVER_TYPE_LABELS: [&str; 2] = [SERVER_LABEL, TYPE_LABEL]; pub static API_REJECTED_AUTH_TOTAL_MD: LazyLock = LazyLock::new(|| { new_counter_md( @@ -81,7 +90,16 @@ pub static API_REQUESTS_IN_FLIGHT_TOTAL_MD: LazyLock = LazyLoc new_gauge_md( MetricName::ApiRequestsInFlightTotal, "Total number of requests currently in flight", - &["name", "type"], + &API_NAME_TYPE_LABELS, + MetricSubsystem::ApiRequests, + ) +}); + +pub static API_REQUESTS_IN_FLIGHT_TOTAL_BY_SERVER_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::Custom("requests_in_flight_total_by_server".to_string()), + "Total number of requests currently in flight by server", + &API_SERVER_NAME_TYPE_LABELS, MetricSubsystem::ApiRequests, ) }); @@ -90,7 +108,16 @@ pub static API_REQUESTS_TOTAL_MD: LazyLock = LazyLock::new(|| new_counter_md( MetricName::ApiRequestsTotal, "Total number of requests", - &["name", "type"], + &API_NAME_TYPE_LABELS, + MetricSubsystem::ApiRequests, + ) +}); + +pub static API_REQUESTS_TOTAL_BY_SERVER_MD: LazyLock = LazyLock::new(|| { + new_counter_md( + MetricName::Custom("requests_total_by_server".to_string()), + "Total number of requests by server", + &API_SERVER_NAME_TYPE_LABELS, MetricSubsystem::ApiRequests, ) }); @@ -99,7 +126,16 @@ pub static API_REQUESTS_ERRORS_TOTAL_MD: LazyLock = LazyLock:: new_counter_md( MetricName::ApiRequestsErrorsTotal, "Total number of requests with (4xx and 5xx) errors", - &["name", "type"], + &API_NAME_TYPE_LABELS, + MetricSubsystem::ApiRequests, + ) +}); + +pub static API_REQUESTS_ERRORS_TOTAL_BY_SERVER_MD: LazyLock = LazyLock::new(|| { + new_counter_md( + MetricName::Custom("requests_errors_total_by_server".to_string()), + "Total number of requests with (4xx and 5xx) errors by server", + &API_SERVER_NAME_TYPE_LABELS, MetricSubsystem::ApiRequests, ) }); @@ -108,7 +144,16 @@ pub static API_REQUESTS_5XX_ERRORS_TOTAL_MD: LazyLock = LazyLo new_counter_md( MetricName::ApiRequests5xxErrorsTotal, "Total number of requests with 5xx errors", - &["name", "type"], + &API_NAME_TYPE_LABELS, + MetricSubsystem::ApiRequests, + ) +}); + +pub static API_REQUESTS_5XX_ERRORS_TOTAL_BY_SERVER_MD: LazyLock = LazyLock::new(|| { + new_counter_md( + MetricName::Custom("requests_5xx_errors_total_by_server".to_string()), + "Total number of requests with 5xx errors by server", + &API_SERVER_NAME_TYPE_LABELS, MetricSubsystem::ApiRequests, ) }); @@ -117,7 +162,16 @@ pub static API_REQUESTS_4XX_ERRORS_TOTAL_MD: LazyLock = LazyLo new_counter_md( MetricName::ApiRequests4xxErrorsTotal, "Total number of requests with 4xx errors", - &["name", "type"], + &API_NAME_TYPE_LABELS, + MetricSubsystem::ApiRequests, + ) +}); + +pub static API_REQUESTS_4XX_ERRORS_TOTAL_BY_SERVER_MD: LazyLock = LazyLock::new(|| { + new_counter_md( + MetricName::Custom("requests_4xx_errors_total_by_server".to_string()), + "Total number of requests with 4xx errors by server", + &API_SERVER_NAME_TYPE_LABELS, MetricSubsystem::ApiRequests, ) }); @@ -126,7 +180,16 @@ pub static API_REQUESTS_CANCELED_TOTAL_MD: LazyLock = LazyLock new_counter_md( MetricName::ApiRequestsCanceledTotal, "Total number of requests canceled by the client", - &["name", "type"], + &API_NAME_TYPE_LABELS, + MetricSubsystem::ApiRequests, + ) +}); + +pub static API_REQUESTS_CANCELED_TOTAL_BY_SERVER_MD: LazyLock = LazyLock::new(|| { + new_counter_md( + MetricName::Custom("requests_canceled_total_by_server".to_string()), + "Total number of requests canceled by the client by server", + &API_SERVER_NAME_TYPE_LABELS, MetricSubsystem::ApiRequests, ) }); @@ -135,7 +198,16 @@ pub static API_REQUESTS_TTFB_SECONDS_DISTRIBUTION_MD: LazyLock new_counter_md( MetricName::ApiRequestsTTFBSecondsDistribution, "Distribution of time to first byte across API calls", - &["name", "type", "le"], + &API_NAME_TYPE_LE_LABELS, + MetricSubsystem::ApiRequests, + ) +}); + +pub static API_REQUESTS_TTFB_SECONDS_DISTRIBUTION_BY_SERVER_MD: LazyLock = LazyLock::new(|| { + new_counter_md( + MetricName::Custom("requests_ttfb_seconds_distribution_by_server".to_string()), + "Distribution of time to first byte across API calls by server", + &API_SERVER_NAME_TYPE_LE_LABELS, MetricSubsystem::ApiRequests, ) }); @@ -144,7 +216,16 @@ pub static API_TRAFFIC_SENT_BYTES_MD: LazyLock = LazyLock::new new_counter_md( MetricName::ApiTrafficSentBytes, "Total number of bytes sent", - &["type"], + &API_TYPE_LABELS, + MetricSubsystem::ApiRequests, + ) +}); + +pub static API_TRAFFIC_SENT_BYTES_BY_SERVER_MD: LazyLock = LazyLock::new(|| { + new_counter_md( + MetricName::Custom("traffic_sent_bytes_by_server".to_string()), + "Total number of bytes sent by server", + &API_SERVER_TYPE_LABELS, MetricSubsystem::ApiRequests, ) }); @@ -153,7 +234,16 @@ pub static API_TRAFFIC_RECV_BYTES_MD: LazyLock = LazyLock::new new_counter_md( MetricName::ApiTrafficRecvBytes, "Total number of bytes received", - &["type"], + &API_TYPE_LABELS, + MetricSubsystem::ApiRequests, + ) +}); + +pub static API_TRAFFIC_RECV_BYTES_BY_SERVER_MD: LazyLock = LazyLock::new(|| { + new_counter_md( + MetricName::Custom("traffic_recv_bytes_by_server".to_string()), + "Total number of bytes received by server", + &API_SERVER_TYPE_LABELS, MetricSubsystem::ApiRequests, ) }); diff --git a/crates/obs/src/metrics/schema/scanner.rs b/crates/obs/src/metrics/schema/scanner.rs index b79ee6e15..965810ff7 100644 --- a/crates/obs/src/metrics/schema/scanner.rs +++ b/crates/obs/src/metrics/schema/scanner.rs @@ -17,6 +17,50 @@ use crate::{MetricDescriptor, MetricName, new_counter_md, new_gauge_md, subsystems}; use std::sync::LazyLock; +pub const SERVER_LABEL: &str = "server"; +pub const SOURCE_LABEL: &str = "source"; +pub const STATE_LABEL: &str = "state"; +pub const CYCLE_SCOPE_LABEL: &str = "cycle_scope"; +pub const BUCKET_LABEL: &str = "bucket"; +pub const DRIVE_LABEL: &str = "drive"; +pub const RESULT_LABEL: &str = "result"; + +pub static SCANNER_SOURCE_WORK_TOTAL_MD: LazyLock = LazyLock::new(|| { + new_counter_md( + MetricName::Custom("source_work_total".to_string()), + "Total scanner work by source and state since server start", + &[SERVER_LABEL, SOURCE_LABEL, STATE_LABEL], + subsystems::SCANNER, + ) +}); + +pub static SCANNER_CYCLE_SOURCE_WORK_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::Custom("cycle_source_work".to_string()), + "Scanner work by cycle scope, source, and state", + &[SERVER_LABEL, CYCLE_SCOPE_LABEL, SOURCE_LABEL, STATE_LABEL], + subsystems::SCANNER, + ) +}); + +pub static SCANNER_BUCKET_DRIVE_RESULT_TOTAL_MD: LazyLock = LazyLock::new(|| { + new_counter_md( + MetricName::Custom("bucket_drive_result_total".to_string()), + "Total scanner bucket-drive scan results by server, bucket, drive, and result", + &[SERVER_LABEL, BUCKET_LABEL, DRIVE_LABEL, RESULT_LABEL], + subsystems::SCANNER, + ) +}); + +pub static SCANNER_CYCLE_BUCKET_DRIVE_RESULT_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::Custom("cycle_bucket_drive_result".to_string()), + "Scanner bucket-drive scan results by cycle scope, server, bucket, drive, and result", + &[SERVER_LABEL, CYCLE_SCOPE_LABEL, BUCKET_LABEL, DRIVE_LABEL, RESULT_LABEL], + subsystems::SCANNER, + ) +}); + pub static SCANNER_BUCKET_SCANS_FINISHED_MD: LazyLock = LazyLock::new(|| { new_counter_md( MetricName::ScannerBucketScansFinished, diff --git a/crates/obs/src/metrics/schema/system_drive.rs b/crates/obs/src/metrics/schema/system_drive.rs index 9fbac5f5e..e32fab738 100644 --- a/crates/obs/src/metrics/schema/system_drive.rs +++ b/crates/obs/src/metrics/schema/system_drive.rs @@ -29,9 +29,111 @@ pub const SET_INDEX_LABEL: &str = "set_index"; pub const DRIVE_INDEX_LABEL: &str = "drive_index"; /// API label pub const API_LABEL: &str = "api"; +/// Disk id label +pub const DISK_ID_LABEL: &str = "disk_id"; +/// State label +pub const STATE_LABEL: &str = "state"; /// All drive-related labels pub const ALL_DRIVE_LABELS: [&str; 2] = [SERVER_LABEL, DRIVE_LABEL]; +/// Drive labels with erasure-set topology. +pub const DRIVE_TOPOLOGY_LABELS: [&str; 5] = [ + SERVER_LABEL, + DRIVE_LABEL, + POOL_INDEX_LABEL, + SET_INDEX_LABEL, + DRIVE_INDEX_LABEL, +]; +/// Drive info labels. +pub const DRIVE_INFO_LABELS: [&str; 6] = [ + SERVER_LABEL, + DRIVE_LABEL, + POOL_INDEX_LABEL, + SET_INDEX_LABEL, + DRIVE_INDEX_LABEL, + DISK_ID_LABEL, +]; +/// Drive topology labels with a state dimension. +pub const DRIVE_TOPOLOGY_STATE_LABELS: [&str; 6] = [ + SERVER_LABEL, + DRIVE_LABEL, + POOL_INDEX_LABEL, + SET_INDEX_LABEL, + DRIVE_INDEX_LABEL, + STATE_LABEL, +]; +/// Drive topology labels with an API dimension. +pub const DRIVE_TOPOLOGY_API_LABELS: [&str; 6] = [ + SERVER_LABEL, + DRIVE_LABEL, + POOL_INDEX_LABEL, + SET_INDEX_LABEL, + DRIVE_INDEX_LABEL, + API_LABEL, +]; + +pub static DRIVE_INFO_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::Custom("info".to_string()), + "Drive topology and stable disk identity information", + &DRIVE_INFO_LABELS, + subsystems::SYSTEM_DRIVE, + ) +}); + +pub static DRIVE_RUNTIME_STATE_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::Custom("runtime_state".to_string()), + "Drive runtime state (1 for the active state label, 0 otherwise)", + &DRIVE_TOPOLOGY_STATE_LABELS, + subsystems::SYSTEM_DRIVE, + ) +}); + +pub static DRIVE_HEALING_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::Custom("healing".to_string()), + "Whether the drive is currently healing", + &DRIVE_TOPOLOGY_LABELS, + subsystems::SYSTEM_DRIVE, + ) +}); + +pub static DRIVE_SCANNING_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::Custom("scanning".to_string()), + "Whether the drive is currently being scanned", + &DRIVE_TOPOLOGY_LABELS, + subsystems::SYSTEM_DRIVE, + ) +}); + +pub static DRIVE_OFFLINE_DURATION_SECONDS_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::Custom("offline_duration_seconds".to_string()), + "Duration in seconds the drive has been offline", + &DRIVE_TOPOLOGY_LABELS, + subsystems::SYSTEM_DRIVE, + ) +}); + +pub static DRIVE_API_CALLS_MD: LazyLock = LazyLock::new(|| { + new_counter_md( + MetricName::Custom("api_calls_total".to_string()), + "Total drive API calls by operation", + &DRIVE_TOPOLOGY_API_LABELS, + subsystems::SYSTEM_DRIVE, + ) +}); + +pub static DRIVE_API_LATENCY_BY_API_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::Custom("api_latency_by_api_micros".to_string()), + "Average last minute drive API latency in microseconds by operation", + &DRIVE_TOPOLOGY_API_LABELS, + subsystems::SYSTEM_DRIVE, + ) +}); pub static DRIVE_USED_BYTES_MD: LazyLock = LazyLock::new(|| { new_gauge_md( diff --git a/crates/obs/src/metrics/stats_collector.rs b/crates/obs/src/metrics/stats_collector.rs index 7ab0a7311..951447c79 100644 --- a/crates/obs/src/metrics/stats_collector.rs +++ b/crates/obs/src/metrics/stats_collector.rs @@ -20,12 +20,15 @@ //! RustFS internal sources (storage layer, bucket monitor, system info) //! and convert them to the Stats structs used by collectors. +use crate::metrics::collectors::scanner::{ScannerBucketDriveResultStats, ScannerSourceWorkStats}; use crate::metrics::collectors::{ - BucketReplicationBacklogStats, BucketReplicationBandwidthStats, BucketReplicationStats, BucketReplicationTargetBacklogStats, + ApiRequestMetricSupport, ApiRequestStats, BucketReplicationBacklogStats, BucketReplicationBandwidthStats, + BucketReplicationRuntimeStats, BucketReplicationStats, BucketReplicationTargetBacklogStats, BucketReplicationTargetFlowStats, BucketReplicationTargetStats, BucketStats, BucketUsageStats, ClusterConfigStats, ClusterHealthStats, ClusterStats, - ClusterUsageStats, CompressionClusterStats, CpuStats, DiskStats, DriveCountStats, DriveDetailedStats, ErasureSetStats, - HostNetworkStats, IamStats, IlmStats, MemoryStats, NetworkStats, ProcessStats, ProcessStatusType, ReplicationStats, - ResourceStats, ScannerStats, + ClusterUsageStats, CompressionClusterStats, CpuStats, DiskStats, DriveCountStats, DriveDetailedStats, + DriveRuntimeDetailedStats, ErasureSetStats, HostNetworkStats, IamStats, IlmActionTaskStats, IlmRuntimeStats, IlmStats, + MemoryStats, NetworkStats, ProcessStats, ProcessStatusType, ReplicationStats, ResourceStats, ScannerRuntimeStats, + ScannerStats, }; use crate::metrics::runtime_sources::{ObsIlmRuntimeSnapshot, bucket_monitor_handle, iam_metrics_snapshot, ilm_runtime_snapshot}; use crate::metrics::{ @@ -37,11 +40,11 @@ use crate::metrics::{ use crate::node_identity::current_local_node_identity; use chrono::Utc; use rustfs_common::heal_channel::HealScanMode; -use rustfs_common::metrics::{ScannerMetricsReport, global_metrics}; +use rustfs_common::metrics::{ScannerBucketDriveResultSnapshot, ScannerMetricsReport, ScannerSourceWorkSnapshot, global_metrics}; use rustfs_io_metrics::internode_metrics::global_internode_metrics; use rustfs_io_metrics::{ - ProcessResourceSnapshot, ProcessSampler, ProcessStatusSnapshot, ProcessSystemSnapshot, snapshot_process_resource_and_system, - snapshot_process_resource_and_system_with, + ProcessResourceSnapshot, ProcessSampler, ProcessStatusSnapshot, ProcessSystemSnapshot, s3_op_metrics_snapshot, + snapshot_process_resource_and_system, snapshot_process_resource_and_system_with, }; use std::{ collections::{HashMap, HashSet}, @@ -193,7 +196,7 @@ async fn obs_ilm_runtime_snapshot() -> ObsIlmRuntimeSnapshot { ilm_runtime_snapshot().await } -async fn obs_bucket_replication_stats_bundle() -> (Vec, Vec) { +async fn obs_bucket_replication_stats_bundle() -> (Vec, Vec) { let snapshots = obs_bucket_replication_stats_snapshot().await; let mut detail_stats = Vec::with_capacity(snapshots.len()); let mut backlog_stats = Vec::with_capacity(snapshots.len()); @@ -230,44 +233,65 @@ async fn obs_bucket_replication_stats_bundle() -> (Vec, (detail_stats, backlog_stats) } -fn bucket_replication_detail_from_snapshot(stats: ObsBucketReplicationStatsSnapshot) -> BucketReplicationStats { - BucketReplicationStats { - bucket: stats.bucket, - total_failed_bytes: stats.total_failed_bytes, - total_failed_count: stats.total_failed_count, - last_min_failed_bytes: stats.last_min_failed_bytes, - last_min_failed_count: stats.last_min_failed_count, - last_hour_failed_bytes: stats.last_hour_failed_bytes, - last_hour_failed_count: stats.last_hour_failed_count, - sent_bytes: stats.sent_bytes, - sent_count: stats.sent_count, - proxied_get_requests_total: stats.proxied_get_requests_total, - proxied_get_requests_failures: stats.proxied_get_requests_failures, - proxied_head_requests_total: stats.proxied_head_requests_total, - proxied_head_requests_failures: stats.proxied_head_requests_failures, - proxied_put_requests_total: stats.proxied_put_requests_total, - proxied_put_requests_failures: stats.proxied_put_requests_failures, - proxied_put_tagging_requests_total: stats.proxied_put_tagging_requests_total, - proxied_put_tagging_requests_failures: stats.proxied_put_tagging_requests_failures, - proxied_get_tagging_requests_total: stats.proxied_get_tagging_requests_total, - proxied_get_tagging_requests_failures: stats.proxied_get_tagging_requests_failures, - proxied_delete_tagging_requests_total: stats.proxied_delete_tagging_requests_total, - proxied_delete_tagging_requests_failures: stats.proxied_delete_tagging_requests_failures, - resync_started_count: stats.resync_started_count, - resync_completed_count: stats.resync_completed_count, - resync_failed_count: stats.resync_failed_count, - resync_canceled_count: stats.resync_canceled_count, - resync_duration_ms: stats.resync_duration_ms, - targets: stats - .targets - .into_iter() - .map(|target| BucketReplicationTargetStats { - target_arn: target.target_arn, - bandwidth_limit_bytes_per_sec: target.bandwidth_limit_bytes_per_sec, - current_bandwidth_bytes_per_sec: target.current_bandwidth_bytes_per_sec, - latency_ms: target.latency_ms, - }) - .collect(), +fn bucket_replication_detail_from_snapshot(stats: ObsBucketReplicationStatsSnapshot) -> BucketReplicationRuntimeStats { + let bucket = stats.bucket; + let (targets, target_flows): (Vec<_>, Vec<_>) = stats + .targets + .into_iter() + .map(|target| { + ( + BucketReplicationTargetStats { + target_arn: target.target_arn.clone(), + bandwidth_limit_bytes_per_sec: target.bandwidth_limit_bytes_per_sec, + current_bandwidth_bytes_per_sec: target.current_bandwidth_bytes_per_sec, + latency_ms: target.latency_ms, + }, + BucketReplicationTargetFlowStats { + target_arn: target.target_arn, + sent_bytes: target.sent_bytes, + sent_count: target.sent_count, + total_failed_bytes: target.total_failed_bytes, + total_failed_count: target.total_failed_count, + last_min_failed_bytes: target.last_min_failed_bytes, + last_min_failed_count: target.last_min_failed_count, + last_hour_failed_bytes: target.last_hour_failed_bytes, + last_hour_failed_count: target.last_hour_failed_count, + }, + ) + }) + .unzip(); + + BucketReplicationRuntimeStats { + target_flows, + stats: BucketReplicationStats { + bucket, + total_failed_bytes: stats.total_failed_bytes, + total_failed_count: stats.total_failed_count, + last_min_failed_bytes: stats.last_min_failed_bytes, + last_min_failed_count: stats.last_min_failed_count, + last_hour_failed_bytes: stats.last_hour_failed_bytes, + last_hour_failed_count: stats.last_hour_failed_count, + sent_bytes: stats.sent_bytes, + sent_count: stats.sent_count, + proxied_get_requests_total: stats.proxied_get_requests_total, + proxied_get_requests_failures: stats.proxied_get_requests_failures, + proxied_head_requests_total: stats.proxied_head_requests_total, + proxied_head_requests_failures: stats.proxied_head_requests_failures, + proxied_put_requests_total: stats.proxied_put_requests_total, + proxied_put_requests_failures: stats.proxied_put_requests_failures, + proxied_put_tagging_requests_total: stats.proxied_put_tagging_requests_total, + proxied_put_tagging_requests_failures: stats.proxied_put_tagging_requests_failures, + proxied_get_tagging_requests_total: stats.proxied_get_tagging_requests_total, + proxied_get_tagging_requests_failures: stats.proxied_get_tagging_requests_failures, + proxied_delete_tagging_requests_total: stats.proxied_delete_tagging_requests_total, + proxied_delete_tagging_requests_failures: stats.proxied_delete_tagging_requests_failures, + resync_started_count: stats.resync_started_count, + resync_completed_count: stats.resync_completed_count, + resync_failed_count: stats.resync_failed_count, + resync_canceled_count: stats.resync_canceled_count, + resync_duration_ms: stats.resync_duration_ms, + targets, + }, } } @@ -364,6 +388,63 @@ fn disk_capacity_observation_state(source: Option<&str>, age_seconds: Option Option { + if index >= 0 { Some(index.to_string()) } else { None } +} + +fn non_empty_disk_id(uuid: &str) -> Option { + let uuid = uuid.trim(); + if uuid.is_empty() { None } else { Some(uuid.to_string()) } +} + +fn drive_inode_stats(used_inodes: u64, free_inodes: u64) -> (Option, Option, Option) { + let total_inodes = used_inodes.saturating_add(free_inodes); + if total_inodes == 0 { + (None, None, None) + } else { + (Some(used_inodes), Some(free_inodes), Some(total_inodes)) + } +} + +fn drive_api_latency_micros(actions: impl Iterator) -> Option { + let mut count = 0u64; + let mut acc_time_ns = 0u64; + let mut saw_action = false; + for (action_count, action_acc_time_ns) in actions { + saw_action = true; + if action_count > 0 { + count = count.saturating_add(action_count); + acc_time_ns = acc_time_ns.saturating_add(action_acc_time_ns); + } + } + + saw_action.then(|| acc_time_ns.checked_div(count).unwrap_or_default() / 1_000) +} + +fn drive_api_latency_by_api_micros<'a>(actions: impl Iterator) -> Vec<(String, u64)> { + let mut values = actions + .map(|(api, count, acc_time)| (api.clone(), acc_time.checked_div(count).unwrap_or_default() / 1_000)) + .collect::>(); + values.sort_by(|left, right| left.0.cmp(&right.0)); + values +} + +fn drive_api_calls<'a>(api_calls: impl Iterator) -> Vec<(String, u64)> { + let mut values = api_calls.map(|(api, calls)| (api.clone(), *calls)).collect::>(); + values.sort_by(|left, right| left.0.cmp(&right.0)); + values +} + +fn drive_server_label(endpoint: &str, local_server: &str) -> String { + endpoint + .strip_prefix("http://") + .or_else(|| endpoint.strip_prefix("https://")) + .and_then(|rest| rest.split('/').next()) + .filter(|authority| !authority.is_empty()) + .unwrap_or(local_server) + .to_string() +} + fn derive_erasure_set_quorum_shape(set_drive_count: usize, parity: usize) -> ErasureSetQuorumShape { let data_shards = set_drive_count.saturating_sub(parity); let read_quorum = data_shards.max(1); @@ -563,12 +644,12 @@ pub async fn collect_bucket_replication_detail_stats() -> Vec (Vec, Vec) -{ +pub(crate) async fn collect_bucket_replication_stats_bundle() +-> (Vec, Vec) { obs_bucket_replication_stats_bundle().await } @@ -577,6 +658,22 @@ pub async fn collect_replication_stats() -> ReplicationStats { obs_site_replication_stats().await } +/// Collect S3 API request totals from the in-process operation recorder. +pub(crate) fn collect_api_request_stats() -> Vec { + let server = current_local_node_identity(); + s3_op_metrics_snapshot() + .into_iter() + .map(|snapshot| ApiRequestStats { + server: server.clone(), + name: snapshot.op.to_string(), + req_type: "s3".to_string(), + total: snapshot.total, + supported_metrics: ApiRequestMetricSupport::TOTALS_ONLY, + ..Default::default() + }) + .collect() +} + /// Collect disk statistics from the storage layer. pub async fn collect_disk_stats() -> Vec { let (disk_stats, _, _) = collect_disk_and_system_drive_stats().await; @@ -646,16 +743,23 @@ pub fn collect_system_memory_stats() -> MemoryStats { /// Collect node disk stats and drive stats from a single storage snapshot. pub async fn collect_disk_and_system_drive_stats() -> (Vec, Vec, DriveCountStats) { + let (disk_stats, drive_stats, drive_count_stats) = collect_disk_and_system_drive_runtime_stats().await; + (disk_stats, drive_stats.into_iter().map(|stat| stat.stats).collect(), drive_count_stats) +} + +pub(crate) async fn collect_disk_and_system_drive_runtime_stats() +-> (Vec, Vec, DriveCountStats) { let Some(store) = resolve_obs_object_store_handle() else { return (Vec::new(), Vec::new(), DriveCountStats::default()); }; let storage_info = StorageAdminApi::storage_info(store.as_ref()).await; + let local_server = current_local_node_identity(); let disk_stats = storage_info .disks .iter() .map(|disk| DiskStats { - server: disk.endpoint.clone(), + server: drive_server_label(&disk.endpoint, &local_server), drive: disk.drive_path.clone(), total_bytes: disk.total_space, used_bytes: disk.used_space, @@ -679,31 +783,61 @@ pub async fn collect_disk_and_system_drive_stats() -> (Vec, Vec Vec { + vec![ + IlmActionTaskStats { + action: "expiry".to_string(), + state: "pending".to_string(), + value: ilm.expiry_pending_tasks, + }, + IlmActionTaskStats { + action: "transition".to_string(), + state: "active".to_string(), + value: ilm.transition_active_tasks, + }, + IlmActionTaskStats { + action: "transition".to_string(), + state: "pending".to_string(), + value: ilm.transition_pending_tasks, + }, + IlmActionTaskStats { + action: "transition".to_string(), + state: "missed_immediate".to_string(), + value: ilm.transition_missed_immediate_tasks, + }, + IlmActionTaskStats { + action: "transition".to_string(), + state: "queue_full".to_string(), + value: ilm.transition_queue_full_tasks, + }, + IlmActionTaskStats { + action: "transition".to_string(), + state: "queue_send_timeout".to_string(), + value: ilm.transition_queue_send_timeout_tasks, + }, + IlmActionTaskStats { + action: "transition".to_string(), + state: "compensation_scheduled".to_string(), + value: ilm.transition_compensation_scheduled_tasks, + }, + IlmActionTaskStats { + action: "transition".to_string(), + state: "compensation_running".to_string(), + value: ilm.transition_compensation_running_tasks, + }, + ] +} + /// Collect ILM metrics from the current lifecycle runtime state. pub async fn collect_ilm_metric_stats() -> Option { + collect_ilm_runtime_metric_stats().await.map(|stats| stats.stats) +} + +pub(crate) async fn collect_ilm_runtime_metric_stats() -> Option { let ilm = obs_ilm_runtime_snapshot().await; let metrics = global_metrics().report().await; let versions_scanned = scanner_lifecycle_checked_versions(&metrics); - Some(IlmStats { - expiry_pending_tasks: ilm.expiry_pending_tasks, - transition_active_tasks: ilm.transition_active_tasks, - transition_pending_tasks: ilm.transition_pending_tasks, - transition_missed_immediate_tasks: ilm.transition_missed_immediate_tasks, - transition_queue_full_tasks: ilm.transition_queue_full_tasks, - transition_queue_send_timeout_tasks: ilm.transition_queue_send_timeout_tasks, - transition_compensation_scheduled_tasks: ilm.transition_compensation_scheduled_tasks, - transition_compensation_running_tasks: ilm.transition_compensation_running_tasks, - versions_scanned, + Some(IlmRuntimeStats { + server: current_local_node_identity(), + action_tasks: ilm_action_task_stats(&ilm), + stats: IlmStats { + expiry_pending_tasks: ilm.expiry_pending_tasks, + transition_active_tasks: ilm.transition_active_tasks, + transition_pending_tasks: ilm.transition_pending_tasks, + transition_missed_immediate_tasks: ilm.transition_missed_immediate_tasks, + transition_queue_full_tasks: ilm.transition_queue_full_tasks, + transition_queue_send_timeout_tasks: ilm.transition_queue_send_timeout_tasks, + transition_compensation_scheduled_tasks: ilm.transition_compensation_scheduled_tasks, + transition_compensation_running_tasks: ilm.transition_compensation_running_tasks, + versions_scanned, + }, }) } @@ -1120,8 +1307,76 @@ fn scanner_bucket_scans_started(life_time_ops: &HashMap, bucket_sca .unwrap_or(bucket_scans_finished) } +fn scanner_source_work_stats(source_work: &[ScannerSourceWorkSnapshot]) -> Vec { + let mut stats = source_work + .iter() + .filter(|work| !work.source.is_empty()) + .map(|work| ScannerSourceWorkStats { + source: work.source.clone(), + checked: work.checked, + queued: work.queued, + executed: work.executed, + failed: work.failed, + skipped: work.skipped, + missed: work.missed, + }) + .collect::>(); + stats.sort_by(|left, right| left.source.cmp(&right.source)); + stats +} + +fn scanner_current_cycle_source_work_stats(metrics: &ScannerMetricsReport) -> Vec { + let current = scanner_source_work_stats(&metrics.current_cycle_source_work); + if !current.is_empty() { + return current; + } + + let mut sources = scanner_source_work_stats(&metrics.last_cycle_source_work) + .into_iter() + .map(|work| work.source) + .chain( + scanner_source_work_stats(&metrics.source_work) + .into_iter() + .map(|work| work.source), + ) + .collect::>(); + sources.sort(); + sources.dedup(); + sources + .into_iter() + .map(|source| ScannerSourceWorkStats { + source, + ..Default::default() + }) + .collect() +} + +fn scanner_bucket_drive_result_stats(results: &[ScannerBucketDriveResultSnapshot]) -> Vec { + let mut stats = results + .iter() + .filter(|result| !result.bucket.is_empty() && !result.drive.is_empty() && !result.result.is_empty() && result.count > 0) + .map(|result| ScannerBucketDriveResultStats { + bucket: result.bucket.clone(), + drive: result.drive.clone(), + result: result.result.clone(), + count: result.count, + }) + .collect::>(); + stats.sort_by(|left, right| { + left.bucket + .cmp(&right.bucket) + .then_with(|| left.drive.cmp(&right.drive)) + .then_with(|| left.result.cmp(&right.result)) + }); + stats +} + pub async fn collect_scanner_metric_stats() -> Option { - let metrics = global_metrics().report().await; + collect_scanner_runtime_metric_stats().await.map(|stats| stats.stats) +} + +pub(crate) async fn collect_scanner_runtime_metric_stats() -> Option { + let (metrics, runtime_details) = global_metrics().report_with_runtime_details().await; let now = Utc::now(); let bucket_scans_finished = metrics.life_time_ops.get("scan_bucket_drive").copied().unwrap_or_default(); let bucket_scans_started = scanner_bucket_scans_started(&metrics.life_time_ops, bucket_scans_finished); @@ -1147,88 +1402,102 @@ pub async fn collect_scanner_metric_stats() -> Option { let current_cycle_age = current_cycle_age_seconds as f64; let last_cycle_duration = metrics.last_cycle_duration_seconds; - Some(ScannerStats { - bucket_scans_finished, - bucket_scans_started, - bucket_scans_failed, - directories_scanned, - objects_scanned, - versions_scanned, - last_activity_seconds, - active_paths, - oldest_active_path_age_seconds: metrics.oldest_active_path_age_seconds, - current_set_scan_concurrency_limit: metrics.current_set_scan_concurrency_limit, - current_set_scans_queued: metrics.current_set_scans_queued, - current_set_scans_active: metrics.current_set_scans_active, - current_disk_scan_concurrency_limit: metrics.current_disk_scan_concurrency_limit, - current_disk_bucket_scans_queued: metrics.current_disk_bucket_scans_queued, - current_disk_bucket_scans_active: metrics.current_disk_bucket_scans_active, - throttle_idle_mode_enabled: metrics.throttle_idle_mode_enabled, - throttle_sleep_factor: metrics.throttle_sleep_factor, - throttle_max_sleep_seconds: metrics.throttle_max_sleep_seconds, - yield_every_n_objects: metrics.yield_every_n_objects, - cycle_interval_seconds: metrics.cycle_interval_seconds, - cycle_max_duration_seconds: metrics.cycle_max_duration_seconds, - cycle_max_objects: metrics.cycle_max_objects, - cycle_max_directories: metrics.cycle_max_directories, - bitrot_cycle_enabled: metrics.bitrot_cycle_enabled, - bitrot_cycle_seconds: metrics.bitrot_cycle_seconds, - current_cycle: metrics.current_cycle, - completed_cycles, - current_cycle_age_seconds, - current_cycle_objects_scanned: metrics.current_cycle_objects_scanned, - current_cycle_directories_scanned: metrics.current_cycle_directories_scanned, - current_cycle_bucket_drive_scans: metrics.current_cycle_bucket_drive_scans, - current_cycle_bucket_drive_failures: metrics.current_cycle_bucket_drive_failures, - current_cycle_objects_per_second: scanner_work_rate_per_second(metrics.current_cycle_objects_scanned, current_cycle_age), - current_cycle_directories_per_second: scanner_work_rate_per_second( - metrics.current_cycle_directories_scanned, - current_cycle_age, + Some(ScannerRuntimeStats { + server: current_local_node_identity(), + source_work: scanner_source_work_stats(&metrics.source_work), + current_cycle_source_work: scanner_current_cycle_source_work_stats(&metrics), + last_cycle_source_work: scanner_source_work_stats(&metrics.last_cycle_source_work), + bucket_drive_results: scanner_bucket_drive_result_stats(&runtime_details.bucket_drive_results), + current_cycle_bucket_drive_results: scanner_bucket_drive_result_stats( + &runtime_details.current_cycle_bucket_drive_results, ), - current_cycle_bucket_drive_scans_per_second: scanner_work_rate_per_second( - metrics.current_cycle_bucket_drive_scans, - current_cycle_age, - ), - current_cycle_yield_events: metrics.current_cycle_yield_events, - current_cycle_yield_duration_seconds: metrics.current_cycle_yield_duration_seconds, - current_cycle_throttle_sleep_events: metrics.current_cycle_throttle_sleep_events, - current_cycle_throttle_sleep_duration_seconds: metrics.current_cycle_throttle_sleep_duration_seconds, - current_cycle_ilm_actions: metrics.current_cycle_ilm_actions, - current_cycle_heal_objects: metrics.current_cycle_heal_objects, - current_cycle_replication_checks: metrics.current_cycle_replication_checks, - current_cycle_usage_saves: metrics.current_cycle_usage_saves, - current_scan_mode, - last_cycle_result: metrics.last_cycle_result_code, - last_cycle_partial_reason: metrics.last_cycle_partial_reason_code, - last_cycle_duration_seconds: metrics.last_cycle_duration_seconds, - last_cycle_objects_scanned: metrics.last_cycle_objects_scanned, - last_cycle_directories_scanned: metrics.last_cycle_directories_scanned, - last_cycle_bucket_drive_scans: metrics.last_cycle_bucket_drive_scans, - last_cycle_bucket_drive_failures: metrics.last_cycle_bucket_drive_failures, - last_cycle_objects_per_second: scanner_work_rate_per_second(metrics.last_cycle_objects_scanned, last_cycle_duration), - last_cycle_directories_per_second: scanner_work_rate_per_second( - metrics.last_cycle_directories_scanned, - last_cycle_duration, - ), - last_cycle_bucket_drive_scans_per_second: scanner_work_rate_per_second( - metrics.last_cycle_bucket_drive_scans, - last_cycle_duration, - ), - last_cycle_yield_events: metrics.last_cycle_yield_events, - last_cycle_yield_duration_seconds: metrics.last_cycle_yield_duration_seconds, - last_cycle_throttle_sleep_events: metrics.last_cycle_throttle_sleep_events, - last_cycle_throttle_sleep_duration_seconds: metrics.last_cycle_throttle_sleep_duration_seconds, - last_cycle_ilm_actions: metrics.last_cycle_ilm_actions, - last_cycle_heal_objects: metrics.last_cycle_heal_objects, - last_cycle_replication_checks: metrics.last_cycle_replication_checks, - last_cycle_usage_saves: metrics.last_cycle_usage_saves, - failed_cycles: metrics.failed_cycles, - superseded_cycles: metrics.superseded_cycles, - partial_cycles: metrics.partial_cycles, - partial_cycles_unknown: metrics.partial_cycles_unknown, - partial_cycles_runtime: metrics.partial_cycles_runtime, - partial_cycles_objects: metrics.partial_cycles_objects, - partial_cycles_directories: metrics.partial_cycles_directories, + last_cycle_bucket_drive_results: scanner_bucket_drive_result_stats(&runtime_details.last_cycle_bucket_drive_results), + stats: ScannerStats { + bucket_scans_finished, + bucket_scans_started, + bucket_scans_failed, + directories_scanned, + objects_scanned, + versions_scanned, + last_activity_seconds, + active_paths, + oldest_active_path_age_seconds: metrics.oldest_active_path_age_seconds, + current_set_scan_concurrency_limit: metrics.current_set_scan_concurrency_limit, + current_set_scans_queued: metrics.current_set_scans_queued, + current_set_scans_active: metrics.current_set_scans_active, + current_disk_scan_concurrency_limit: metrics.current_disk_scan_concurrency_limit, + current_disk_bucket_scans_queued: metrics.current_disk_bucket_scans_queued, + current_disk_bucket_scans_active: metrics.current_disk_bucket_scans_active, + throttle_idle_mode_enabled: metrics.throttle_idle_mode_enabled, + throttle_sleep_factor: metrics.throttle_sleep_factor, + throttle_max_sleep_seconds: metrics.throttle_max_sleep_seconds, + yield_every_n_objects: metrics.yield_every_n_objects, + cycle_interval_seconds: metrics.cycle_interval_seconds, + cycle_max_duration_seconds: metrics.cycle_max_duration_seconds, + cycle_max_objects: metrics.cycle_max_objects, + cycle_max_directories: metrics.cycle_max_directories, + bitrot_cycle_enabled: metrics.bitrot_cycle_enabled, + bitrot_cycle_seconds: metrics.bitrot_cycle_seconds, + current_cycle: metrics.current_cycle, + completed_cycles, + current_cycle_age_seconds, + current_cycle_objects_scanned: metrics.current_cycle_objects_scanned, + current_cycle_directories_scanned: metrics.current_cycle_directories_scanned, + current_cycle_bucket_drive_scans: metrics.current_cycle_bucket_drive_scans, + current_cycle_bucket_drive_failures: metrics.current_cycle_bucket_drive_failures, + current_cycle_objects_per_second: scanner_work_rate_per_second( + metrics.current_cycle_objects_scanned, + current_cycle_age, + ), + current_cycle_directories_per_second: scanner_work_rate_per_second( + metrics.current_cycle_directories_scanned, + current_cycle_age, + ), + current_cycle_bucket_drive_scans_per_second: scanner_work_rate_per_second( + metrics.current_cycle_bucket_drive_scans, + current_cycle_age, + ), + current_cycle_yield_events: metrics.current_cycle_yield_events, + current_cycle_yield_duration_seconds: metrics.current_cycle_yield_duration_seconds, + current_cycle_throttle_sleep_events: metrics.current_cycle_throttle_sleep_events, + current_cycle_throttle_sleep_duration_seconds: metrics.current_cycle_throttle_sleep_duration_seconds, + current_cycle_ilm_actions: metrics.current_cycle_ilm_actions, + current_cycle_heal_objects: metrics.current_cycle_heal_objects, + current_cycle_replication_checks: metrics.current_cycle_replication_checks, + current_cycle_usage_saves: metrics.current_cycle_usage_saves, + current_scan_mode, + last_cycle_result: metrics.last_cycle_result_code, + last_cycle_partial_reason: metrics.last_cycle_partial_reason_code, + last_cycle_duration_seconds: metrics.last_cycle_duration_seconds, + last_cycle_objects_scanned: metrics.last_cycle_objects_scanned, + last_cycle_directories_scanned: metrics.last_cycle_directories_scanned, + last_cycle_bucket_drive_scans: metrics.last_cycle_bucket_drive_scans, + last_cycle_bucket_drive_failures: metrics.last_cycle_bucket_drive_failures, + last_cycle_objects_per_second: scanner_work_rate_per_second(metrics.last_cycle_objects_scanned, last_cycle_duration), + last_cycle_directories_per_second: scanner_work_rate_per_second( + metrics.last_cycle_directories_scanned, + last_cycle_duration, + ), + last_cycle_bucket_drive_scans_per_second: scanner_work_rate_per_second( + metrics.last_cycle_bucket_drive_scans, + last_cycle_duration, + ), + last_cycle_yield_events: metrics.last_cycle_yield_events, + last_cycle_yield_duration_seconds: metrics.last_cycle_yield_duration_seconds, + last_cycle_throttle_sleep_events: metrics.last_cycle_throttle_sleep_events, + last_cycle_throttle_sleep_duration_seconds: metrics.last_cycle_throttle_sleep_duration_seconds, + last_cycle_ilm_actions: metrics.last_cycle_ilm_actions, + last_cycle_heal_objects: metrics.last_cycle_heal_objects, + last_cycle_replication_checks: metrics.last_cycle_replication_checks, + last_cycle_usage_saves: metrics.last_cycle_usage_saves, + failed_cycles: metrics.failed_cycles, + superseded_cycles: metrics.superseded_cycles, + partial_cycles: metrics.partial_cycles, + partial_cycles_unknown: metrics.partial_cycles_unknown, + partial_cycles_runtime: metrics.partial_cycles_runtime, + partial_cycles_objects: metrics.partial_cycles_objects, + partial_cycles_directories: metrics.partial_cycles_directories, + }, }) } @@ -1480,6 +1749,60 @@ mod tests { assert!(!disk_is_online_for_metrics(DRIVE_STATE_OK, Some("offline"))); } + #[test] + fn disk_topology_label_rejects_unknown_negative_index() { + assert_eq!(disk_topology_label(-1), None); + assert_eq!(disk_topology_label(3), Some("3".to_string())); + } + + #[test] + fn non_empty_disk_id_rejects_blank_uuid() { + assert_eq!(non_empty_disk_id(" "), None); + assert_eq!(non_empty_disk_id("disk-1"), Some("disk-1".to_string())); + } + + #[test] + fn drive_server_label_uses_node_identity_for_urls_and_local_paths() { + assert_eq!(drive_server_label("http://node1:9000/data", "local:9000"), "node1:9000"); + assert_eq!(drive_server_label("https://node2:9443/export/d1", "local:9000"), "node2:9443"); + assert_eq!(drive_server_label("/mnt/data1", "local:9000"), "local:9000"); + } + + #[test] + fn drive_inode_stats_skip_unknown_zero_inode_totals() { + assert_eq!(drive_inode_stats(0, 0), (None, None, None)); + assert_eq!(drive_inode_stats(2, 3), (Some(2), Some(3), Some(5))); + } + + #[test] + fn drive_api_metrics_are_sorted_and_average_latency() { + let last_minute = HashMap::from([ + ("write".to_string(), (2, 6_000)), + ("read".to_string(), (1, 3_000)), + ("zero".to_string(), (0, 9_000)), + ]); + let api_calls = HashMap::from([("write".to_string(), 9), ("read".to_string(), 4)]); + + assert_eq!(drive_api_latency_micros(last_minute.values().copied()), Some(3)); + assert_eq!( + drive_api_latency_by_api_micros(last_minute.iter().map(|(api, (count, acc_time))| (api, *count, *acc_time))), + vec![("read".to_string(), 3), ("write".to_string(), 3), ("zero".to_string(), 0)] + ); + assert_eq!(drive_api_calls(api_calls.iter()), vec![("read".to_string(), 4), ("write".to_string(), 9)]); + } + + #[test] + fn drive_api_latency_skips_zero_denominators() { + let last_minute = HashMap::from([("zero".to_string(), (0, 9_000))]); + + assert_eq!(drive_api_latency_micros(last_minute.values().copied()), Some(0)); + assert_eq!( + drive_api_latency_by_api_micros(last_minute.iter().map(|(api, (count, acc_time))| (api, *count, *acc_time))), + vec![("zero".to_string(), 0)] + ); + assert_eq!(drive_api_latency_micros([].into_iter()), None); + } + #[test] fn derive_erasure_set_quorum_shape_handles_standard_layout() { let shape = derive_erasure_set_quorum_shape(16, 4); @@ -1580,6 +1903,35 @@ mod tests { assert_eq!(scanner_bucket_scans_started(&life_time_ops, 5), 5); } + #[test] + fn ilm_action_task_stats_maps_runtime_states() { + let stats = ilm_action_task_stats(&ObsIlmRuntimeSnapshot { + expiry_pending_tasks: 1, + transition_active_tasks: 2, + transition_pending_tasks: 3, + transition_missed_immediate_tasks: 4, + transition_queue_full_tasks: 5, + transition_queue_send_timeout_tasks: 6, + transition_compensation_scheduled_tasks: 7, + transition_compensation_running_tasks: 8, + }); + + assert_eq!(stats.len(), 8); + assert_eq!(stats[0].action, "expiry"); + assert_eq!(stats[0].state, "pending"); + assert_eq!(stats[0].value, 1); + assert!( + stats + .iter() + .any(|task| { task.action == "transition" && task.state == "queue_send_timeout" && task.value == 6 }) + ); + assert!( + stats + .iter() + .any(|task| { task.action == "transition" && task.state == "compensation_running" && task.value == 8 }) + ); + } + #[test] fn scanner_lifecycle_checked_versions_uses_lifecycle_checked_source_work() { let report = ScannerMetricsReport { @@ -1602,6 +1954,96 @@ mod tests { assert_eq!(scanner_lifecycle_checked_versions(&report), 37); } + #[test] + fn scanner_source_work_stats_sorts_and_skips_empty_source() { + let stats = scanner_source_work_stats(&[ + ScannerSourceWorkSnapshot { + source: "usage".to_string(), + checked: 11, + queued: 2, + executed: 3, + failed: 4, + skipped: 5, + missed: 6, + }, + ScannerSourceWorkSnapshot { + checked: 99, + queued: 99, + executed: 99, + failed: 99, + skipped: 99, + missed: 99, + ..Default::default() + }, + ScannerSourceWorkSnapshot { + source: "lifecycle".to_string(), + checked: 21, + queued: 7, + executed: 8, + failed: 9, + skipped: 10, + missed: 12, + }, + ]); + + assert_eq!(stats.len(), 2); + assert_eq!(stats[0].source, "lifecycle"); + assert_eq!(stats[0].checked, 21); + assert_eq!(stats[0].missed, 12); + assert_eq!(stats[1].source, "usage"); + assert_eq!(stats[1].failed, 4); + } + + #[test] + fn scanner_current_cycle_source_work_stats_zeroes_idle_sources() { + let report = ScannerMetricsReport { + source_work: vec![ScannerSourceWorkSnapshot { + source: "usage".to_string(), + checked: 11, + queued: 2, + ..Default::default() + }], + last_cycle_source_work: vec![ScannerSourceWorkSnapshot { + source: "lifecycle".to_string(), + checked: 21, + queued: 7, + ..Default::default() + }], + ..Default::default() + }; + + let stats = scanner_current_cycle_source_work_stats(&report); + + assert_eq!(stats.len(), 2); + assert_eq!(stats[0].source, "lifecycle"); + assert_eq!(stats[0].checked, 0); + assert_eq!(stats[1].source, "usage"); + assert_eq!(stats[1].queued, 0); + } + + #[test] + fn scanner_current_cycle_source_work_stats_keeps_active_values() { + let report = ScannerMetricsReport { + source_work: vec![ScannerSourceWorkSnapshot { + source: "usage".to_string(), + checked: 11, + ..Default::default() + }], + current_cycle_source_work: vec![ScannerSourceWorkSnapshot { + source: "usage".to_string(), + checked: 3, + ..Default::default() + }], + ..Default::default() + }; + + let stats = scanner_current_cycle_source_work_stats(&report); + + assert_eq!(stats.len(), 1); + assert_eq!(stats[0].source, "usage"); + assert_eq!(stats[0].checked, 3); + } + #[test] fn scanner_lifecycle_checked_versions_defaults_to_zero_when_lifecycle_missing() { let report = ScannerMetricsReport { diff --git a/crates/obs/src/metrics/storage_api.rs b/crates/obs/src/metrics/storage_api.rs index e3801276e..452471632 100644 --- a/crates/obs/src/metrics/storage_api.rs +++ b/crates/obs/src/metrics/storage_api.rs @@ -18,9 +18,9 @@ use std::time::Duration; pub(crate) use rustfs_ecstore::api::bucket::bandwidth::monitor::Monitor as ObsBucketBandwidthMonitor; pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::get_quota_config as obs_get_quota_config; use rustfs_ecstore::api::bucket::replication::{ - DurableMrfBucketBacklog, DurableMrfTargetBacklog, MrfBucketBacklogObservability, RuntimeReplicationTargetBacklog, - durable_mrf_backlog_summary_snapshot, durable_mrf_target_backlog_snapshot, get_global_replication_stats, - mrf_backlog_observability_snapshot, + BucketReplicationStats as SourceBucketReplicationStats, DurableMrfBucketBacklog, DurableMrfTargetBacklog, + MrfBucketBacklogObservability, RuntimeReplicationTargetBacklog, durable_mrf_backlog_summary_snapshot, + durable_mrf_target_backlog_snapshot, get_global_replication_stats, mrf_backlog_observability_snapshot, }; pub(crate) use rustfs_ecstore::api::capacity::{ get_total_usable_capacity as obs_get_total_usable_capacity, @@ -43,6 +43,14 @@ pub(crate) struct ObsBucketReplicationTargetStatsSnapshot { pub(crate) bandwidth_limit_bytes_per_sec: u64, pub(crate) current_bandwidth_bytes_per_sec: f64, pub(crate) latency_ms: f64, + pub(crate) sent_bytes: u64, + pub(crate) sent_count: u64, + pub(crate) total_failed_bytes: u64, + pub(crate) total_failed_count: u64, + pub(crate) last_min_failed_bytes: u64, + pub(crate) last_min_failed_count: u64, + pub(crate) last_hour_failed_bytes: u64, + pub(crate) last_hour_failed_count: u64, } #[derive(Debug, Clone, PartialEq)] @@ -173,6 +181,73 @@ fn replication_backlog_count(failed_counts: impl Iterator, queued_co failed_backlog.saturating_add(i64_to_u64_floor_zero(queued_count)) } +fn bucket_replication_runtime_snapshot_from_source( + bucket_stats: Option<&SourceBucketReplicationStats>, +) -> ObsBucketReplicationRuntimeSnapshot { + let mut runtime = ObsBucketReplicationRuntimeSnapshot { + targets: Vec::with_capacity(bucket_stats.map(|stats| stats.stats.len()).unwrap_or(0)), + ..Default::default() + }; + + if let Some(bucket_stats) = bucket_stats { + for (target_arn, target_stats) in &bucket_stats.stats { + runtime.total_failed_bytes = runtime + .total_failed_bytes + .saturating_add(i64_to_u64_floor_zero(target_stats.fail_stats.size)); + runtime.total_failed_count = runtime + .total_failed_count + .saturating_add(i64_to_u64_floor_zero(target_stats.fail_stats.count)); + + let last_min = target_stats.fail_stats.recent_since(Duration::from_secs(60)); + runtime.last_min_failed_bytes = runtime + .last_min_failed_bytes + .saturating_add(i64_to_u64_floor_zero(last_min.size)); + runtime.last_min_failed_count = runtime + .last_min_failed_count + .saturating_add(i64_to_u64_floor_zero(last_min.count)); + + let last_hour = target_stats.fail_stats.recent_since(Duration::from_secs(60 * 60)); + runtime.last_hour_failed_bytes = runtime + .last_hour_failed_bytes + .saturating_add(i64_to_u64_floor_zero(last_hour.size)); + runtime.last_hour_failed_count = runtime + .last_hour_failed_count + .saturating_add(i64_to_u64_floor_zero(last_hour.count)); + + runtime.sent_bytes = runtime + .sent_bytes + .saturating_add(i64_to_u64_floor_zero(target_stats.replicated_size)); + runtime.sent_count = runtime + .sent_count + .saturating_add(i64_to_u64_floor_zero(target_stats.replicated_count)); + + runtime.targets.push(ObsBucketReplicationTargetStatsSnapshot { + target_arn: target_arn.clone(), + bandwidth_limit_bytes_per_sec: i64_to_u64_floor_zero(target_stats.bandwidth_limit_bytes_per_sec), + current_bandwidth_bytes_per_sec: target_stats.current_bandwidth_bytes_per_sec, + latency_ms: target_stats.latency.curr, + sent_bytes: i64_to_u64_floor_zero(target_stats.replicated_size), + sent_count: i64_to_u64_floor_zero(target_stats.replicated_count), + total_failed_bytes: i64_to_u64_floor_zero(target_stats.fail_stats.size), + total_failed_count: i64_to_u64_floor_zero(target_stats.fail_stats.count), + last_min_failed_bytes: i64_to_u64_floor_zero(last_min.size), + last_min_failed_count: i64_to_u64_floor_zero(last_min.count), + last_hour_failed_bytes: i64_to_u64_floor_zero(last_hour.size), + last_hour_failed_count: i64_to_u64_floor_zero(last_hour.count), + }); + } + runtime.resync_started_count = i64_to_u64_floor_zero(bucket_stats.resync_started_count); + runtime.resync_completed_count = i64_to_u64_floor_zero(bucket_stats.resync_completed_count); + runtime.resync_failed_count = i64_to_u64_floor_zero(bucket_stats.resync_failed_count); + runtime.resync_canceled_count = i64_to_u64_floor_zero(bucket_stats.resync_canceled_count); + runtime.resync_duration_ms = i64_to_u64_floor_zero(bucket_stats.resync_duration_ms); + runtime.current_backlog_count = i64_to_u64_floor_zero(bucket_stats.q_stat.curr.count); + runtime.current_backlog_bytes = i64_to_u64_floor_zero(bucket_stats.q_stat.curr.bytes); + } + + runtime +} + fn bucket_replication_stats_snapshot_from_parts( bucket: String, runtime: ObsBucketReplicationRuntimeSnapshot, @@ -357,58 +432,7 @@ pub(crate) async fn obs_bucket_replication_stats_snapshot() -> Vec(100, None); + target.fail_stats.add_size::<()>(200, None); + target.fail_stats.count = 7; + target.fail_stats.size = 900; + target.replicated_size = 1234; + target.replicated_count = 12; + target.bandwidth_limit_bytes_per_sec = 4096; + target.current_bandwidth_bytes_per_sec = 512.5; + target.latency.curr = 45.0; + + let snapshot = bucket_replication_runtime_snapshot_from_source(Some(&source)); + + assert_eq!(snapshot.sent_bytes, 1234); + assert_eq!(snapshot.sent_count, 12); + assert_eq!(snapshot.total_failed_bytes, 900); + assert_eq!(snapshot.total_failed_count, 7); + assert_eq!(snapshot.last_min_failed_bytes, 300); + assert_eq!(snapshot.last_min_failed_count, 2); + assert_eq!(snapshot.last_hour_failed_bytes, 300); + assert_eq!(snapshot.last_hour_failed_count, 2); + + assert_eq!(snapshot.targets.len(), 1); + let target = &snapshot.targets[0]; + assert_eq!(target.target_arn, "arn:rustfs:replication:target-a"); + assert_eq!(target.bandwidth_limit_bytes_per_sec, 4096); + assert_eq!(target.current_bandwidth_bytes_per_sec, 512.5); + assert_eq!(target.latency_ms, 45.0); + assert_eq!(target.sent_bytes, 1234); + assert_eq!(target.sent_count, 12); + assert_eq!(target.total_failed_bytes, 900); + assert_eq!(target.total_failed_count, 7); + assert_eq!(target.last_min_failed_bytes, 300); + assert_eq!(target.last_min_failed_count, 2); + assert_eq!(target.last_hour_failed_bytes, 300); + assert_eq!(target.last_hour_failed_count, 2); + } + #[test] fn bucket_replication_snapshot_maps_runtime_and_durable_backlog() { let snapshot = bucket_replication_stats_snapshot_from_parts( diff --git a/crates/s3-ops/src/lib.rs b/crates/s3-ops/src/lib.rs index 6a16a6525..c638c1f75 100644 --- a/crates/s3-ops/src/lib.rs +++ b/crates/s3-ops/src/lib.rs @@ -15,6 +15,7 @@ use rustfs_s3_types::EventName; #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +#[repr(usize)] pub enum S3Operation { AbortMultipartUpload, CompleteMultipartUpload, @@ -125,6 +126,80 @@ impl EventMapping { } impl S3Operation { + pub const ALL: [Self; 66] = [ + Self::AbortMultipartUpload, + Self::CompleteMultipartUpload, + Self::CopyObject, + Self::CreateBucket, + Self::CreateMultipartUpload, + Self::DeleteBucket, + Self::DeleteBucketCors, + Self::DeleteBucketEncryption, + Self::DeleteBucketLifecycle, + Self::DeleteBucketPolicy, + Self::DeleteBucketReplication, + Self::DeleteBucketTagging, + Self::DeleteObject, + Self::DeleteObjectTagging, + Self::DeleteObjects, + Self::DeletePublicAccessBlock, + Self::GetBucketAcl, + Self::GetBucketCors, + Self::GetBucketEncryption, + Self::GetBucketLifecycleConfiguration, + Self::GetBucketLocation, + Self::GetBucketLogging, + Self::GetBucketNotificationConfiguration, + Self::GetBucketPolicy, + Self::GetBucketPolicyStatus, + Self::GetBucketReplication, + Self::GetBucketTagging, + Self::GetBucketVersioning, + Self::GetObject, + Self::GetObjectAcl, + Self::GetObjectAttributes, + Self::GetObjectLegalHold, + Self::GetObjectLockConfiguration, + Self::GetObjectRetention, + Self::GetObjectTagging, + Self::GetObjectTorrent, + Self::GetPublicAccessBlock, + Self::HeadBucket, + Self::HeadObject, + Self::ListBuckets, + Self::ListMultipartUploads, + Self::ListObjectVersions, + Self::ListObjects, + Self::ListObjectsV2, + Self::ListParts, + Self::PutBucketAcl, + Self::PutBucketCors, + Self::PutBucketEncryption, + Self::PutBucketLifecycleConfiguration, + Self::PutBucketLogging, + Self::PutBucketNotificationConfiguration, + Self::PutBucketPolicy, + Self::PutBucketReplication, + Self::PutBucketTagging, + Self::PutBucketVersioning, + Self::PutObject, + Self::PutObjectAcl, + Self::PutObjectLegalHold, + Self::PutObjectLockConfiguration, + Self::PutObjectRetention, + Self::PutObjectTagging, + Self::PutPublicAccessBlock, + Self::RestoreObject, + Self::SelectObjectContent, + Self::UploadPart, + Self::UploadPartCopy, + ]; + + #[inline] + pub const fn metric_index(self) -> usize { + self as usize + } + pub fn as_str(self) -> &'static str { match self { Self::AbortMultipartUpload => "s3:AbortMultipartUpload", diff --git a/rustfs/src/app/bucket_usecase.rs b/rustfs/src/app/bucket_usecase.rs index 7834e85c3..075514dcc 100644 --- a/rustfs/src/app/bucket_usecase.rs +++ b/rustfs/src/app/bucket_usecase.rs @@ -75,6 +75,7 @@ use futures::StreamExt; use http::StatusCode; use metrics::counter; use rustfs_config::RUSTFS_REGION; +use rustfs_io_metrics::record_s3_op; use rustfs_madmin::{SITE_REPL_API_VERSION, SRBucketMeta}; use rustfs_policy::policy::{ action::{Action, S3Action}, @@ -1313,6 +1314,7 @@ impl DefaultBucketUsecase { #[instrument(level = "debug", skip(self, req))] pub async fn execute_head_bucket(&self, req: S3Request) -> S3Result> { + record_s3_op(S3Operation::HeadBucket); let input = req.input; let Some(store) = self.object_store() else { @@ -1502,6 +1504,7 @@ impl DefaultBucketUsecase { &self, req: S3Request, ) -> S3Result> { + record_s3_op(S3Operation::DeleteBucketPolicy); let request_context = req.extensions.get::().cloned(); let DeleteBucketPolicyInput { bucket, .. } = req.input; @@ -2265,6 +2268,7 @@ impl DefaultBucketUsecase { &self, req: S3Request, ) -> S3Result> { + record_s3_op(S3Operation::PutBucketPolicy); let request_context = req.extensions.get::().cloned(); let PutBucketPolicyInput { bucket, policy, .. } = req.input; @@ -2714,6 +2718,14 @@ mod tests { use std::time::Duration; use tokio::sync::Notify; + fn s3_op_total(op: S3Operation) -> u64 { + rustfs_io_metrics::s3_op_metrics_snapshot() + .into_iter() + .find(|snapshot| snapshot.op == op.as_str()) + .map(|snapshot| snapshot.total) + .unwrap_or_default() + } + #[tokio::test] async fn bucket_usecase_task_finishes_post_commit_hooks_after_parent_cancellation() { let admission = Arc::new(Semaphore::new(1)); @@ -3337,9 +3349,27 @@ mod tests { let req = build_request(input, Method::HEAD); let usecase = DefaultBucketUsecase::without_context(); + let before = s3_op_total(S3Operation::HeadBucket); let err = usecase.execute_head_bucket(req).await.unwrap_err(); assert_eq!(err.code(), &S3ErrorCode::InternalError); + assert_eq!(s3_op_total(S3Operation::HeadBucket), before + 1); + } + + #[tokio::test] + async fn execute_delete_bucket_policy_records_s3_operation_before_store_lookup() { + let input = DeleteBucketPolicyInput::builder() + .bucket("test-bucket".to_string()) + .build() + .unwrap(); + + let req = build_request(input, Method::DELETE); + let usecase = DefaultBucketUsecase::without_context(); + let before = s3_op_total(S3Operation::DeleteBucketPolicy); + + let err = usecase.execute_delete_bucket_policy(req).await.unwrap_err(); + assert_eq!(err.code(), &S3ErrorCode::InternalError); + assert_eq!(s3_op_total(S3Operation::DeleteBucketPolicy), before + 1); } #[tokio::test] @@ -4327,9 +4357,11 @@ mod tests { let req = build_request(input, Method::PUT); let usecase = DefaultBucketUsecase::without_context(); + let before = s3_op_total(S3Operation::PutBucketPolicy); let err = usecase.execute_put_bucket_policy(req).await.unwrap_err(); assert_eq!(err.code(), &S3ErrorCode::InternalError); + assert_eq!(s3_op_total(S3Operation::PutBucketPolicy), before + 1); } #[tokio::test] diff --git a/rustfs/src/app/multipart_usecase.rs b/rustfs/src/app/multipart_usecase.rs index 7af8643c5..0a140f272 100644 --- a/rustfs/src/app/multipart_usecase.rs +++ b/rustfs/src/app/multipart_usecase.rs @@ -72,6 +72,7 @@ use crate::table_catalog; use bytes::Bytes; use futures::StreamExt; use http::{HeaderMap, HeaderValue, Uri}; +use rustfs_io_metrics::record_s3_op; use rustfs_s3_ops::S3Operation; use rustfs_targets::EventName; use rustfs_utils::CompressionAlgorithm; @@ -341,6 +342,7 @@ impl DefaultMultipartUsecase { &self, req: S3Request, ) -> S3Result> { + record_s3_op(S3Operation::AbortMultipartUpload); let AbortMultipartUploadInput { bucket, key, upload_id, .. } = req.input; @@ -1449,6 +1451,14 @@ mod tests { use temp_env::async_with_vars; use tokio::io::AsyncReadExt; + fn s3_op_total(op: S3Operation) -> u64 { + rustfs_io_metrics::s3_op_metrics_snapshot() + .into_iter() + .find(|snapshot| snapshot.op == op.as_str()) + .map(|snapshot| snapshot.total) + .unwrap_or_default() + } + fn build_request(input: T, method: Method) -> S3Request { S3Request { input, @@ -1755,9 +1765,11 @@ mod tests { .build() .unwrap(); let req = build_request(input, Method::DELETE); + let before = s3_op_total(S3Operation::AbortMultipartUpload); let err = make_usecase().execute_abort_multipart_upload(req).await.unwrap_err(); assert_eq!(err.code(), &S3ErrorCode::InternalError); + assert_eq!(s3_op_total(S3Operation::AbortMultipartUpload), before + 1); } #[tokio::test] diff --git a/rustfs/src/storage/ecfs.rs b/rustfs/src/storage/ecfs.rs index 1d139d591..3802ba555 100644 --- a/rustfs/src/storage/ecfs.rs +++ b/rustfs/src/storage/ecfs.rs @@ -409,7 +409,6 @@ impl S3 for FS { &self, req: S3Request, ) -> S3Result> { - record_s3_op(S3Operation::DeleteObjectTagging); let start_time = std::time::Instant::now(); let mut helper = OperationHelper::new(&req, EventName::ObjectTaggingDelete, S3Operation::DeleteObjectTagging); let DeleteObjectTaggingInput { @@ -1229,7 +1228,6 @@ impl S3 for FS { } async fn put_object_acl(&self, req: S3Request) -> S3Result> { - record_s3_op(S3Operation::PutObjectAcl); let mut helper = OperationHelper::new(&req, EventName::ObjectAclPut, S3Operation::PutObjectAcl); let bucket = &req.input.bucket; let key = &req.input.key; @@ -1558,7 +1556,6 @@ impl S3 for FS { #[instrument(level = "debug", skip(self, req))] async fn put_object_tagging(&self, req: S3Request) -> S3Result> { - record_s3_op(S3Operation::PutObjectTagging); let start_time = std::time::Instant::now(); let mut helper = OperationHelper::new(&req, EventName::ObjectTaggingPut, S3Operation::PutObjectTagging); let PutObjectTaggingInput {