From 9fe3d5621e6a100869d29da1aa0c806e529ee93b Mon Sep 17 00:00:00 2001 From: evan slack <51209817+evanofslack@users.noreply.github.com> Date: Sat, 14 Feb 2026 08:41:07 -0500 Subject: [PATCH] feat(observability): Add additional metric panels to grafana dashboard (#1778) Signed-off-by: evan slack <51209817+evanofslack@users.noreply.github.com> Co-authored-by: loverustfs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Co-authored-by: houseme --- .../grafana/dashboards/rustfs.yaml | 1757 ++++++++++++++++- 1 file changed, 1713 insertions(+), 44 deletions(-) diff --git a/.docker/observability/grafana/dashboards/rustfs.yaml b/.docker/observability/grafana/dashboards/rustfs.yaml index 29c2f9ddb..41bd5ad2b 100644 --- a/.docker/observability/grafana/dashboards/rustfs.yaml +++ b/.docker/observability/grafana/dashboards/rustfs.yaml @@ -42,7 +42,74 @@ "fieldConfig": { "defaults": { "color": { - "mode": "thresholds" + "fixedColor": "#ffffff", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 36, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rustfs_process_uptime_seconds{job=~\"$job\"}", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Uptime", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "fixed" }, "mappings": [], "thresholds": { @@ -61,7 +128,7 @@ "gridPos": { "h": 5, "w": 6, - "x": 0, + "x": 6, "y": 1 }, "id": 6, @@ -90,13 +157,145 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(rustfs_api_requests_total{job=~\"$job\", key_request_uri_path=~\"$path\"})", - "legendFormat": "Total Requests", + "expr": "sum(increase(rustfs_api_requests_total{job=~\"$job\"}[$__range]))", + "legendFormat": "__auto", "range": true, "refId": "A" } ], - "title": "Total Requests", + "title": "Requests", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "orange", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 1 + }, + "id": 19, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rustfs_cluster_buckets_total{job=~\"$job\"})", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Total Buckets", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "purple", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 1 + }, + "id": 18, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rustfs_cluster_objects_total{job=~\"$job\"})", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Total Objects", "type": "stat" }, { @@ -119,7 +318,171 @@ } ] }, - "unit": "recps" + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Used" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "text", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "text", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Percent" + }, + "properties": [ + { + "id": "unit", + "value": "percentunit" + }, + { + "id": "color", + "value": { + "mode": "thresholds" + } + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 0.6 + }, + { + "color": "orange", + "value": 0.7 + }, + { + "color": "red", + "value": 0.8 + } + ] + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 6 + }, + "id": 20, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rustfs_cluster_capacity_used_bytes{job=~\"$job\"})", + "legendFormat": "Used", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rustfs_cluster_capacity_raw_total_bytes{job=~\"$job\"})", + "hide": false, + "legendFormat": "Total", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rustfs_cluster_capacity_used_bytes{job=~\"$job\"}) / sum(rustfs_cluster_capacity_raw_total_bytes{job=~\"$job\"})", + "hide": false, + "instant": false, + "legendFormat": "Percent", + "range": true, + "refId": "B" + } + ], + "title": "Capacity", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "reqps" }, "overrides": [] }, @@ -127,7 +490,7 @@ "h": 5, "w": 6, "x": 6, - "y": 1 + "y": 6 }, "id": 14, "options": { @@ -155,7 +518,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(rate(rustfs_api_requests_total{job=~\"$job\", key_request_uri_path=~\"$path\"}[5m]))", + "expr": "sum(rate(rustfs_api_requests_total{job=~\"$job\"}[5m]))", "hide": false, "instant": false, "legendFormat": "__auto", @@ -174,7 +537,8 @@ "fieldConfig": { "defaults": { "color": { - "mode": "thresholds" + "fixedColor": "orange", + "mode": "fixed" }, "mappings": [], "thresholds": { @@ -194,7 +558,7 @@ "h": 5, "w": 6, "x": 12, - "y": 1 + "y": 6 }, "id": 15, "options": { @@ -222,8 +586,8 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(rustfs_create_bucket_total{job=~\"$job\"})", - "legendFormat": "Total Requests", + "expr": "sum(increase(rustfs_create_bucket_total{job=~\"$job\"}[$__range]))", + "legendFormat": "__auto", "range": true, "refId": "A" } @@ -239,7 +603,8 @@ "fieldConfig": { "defaults": { "color": { - "mode": "thresholds" + "fixedColor": "red", + "mode": "fixed" }, "mappings": [], "thresholds": { @@ -259,7 +624,7 @@ "h": 5, "w": 6, "x": 18, - "y": 1 + "y": 6 }, "id": 16, "options": { @@ -287,8 +652,8 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(rustfs_object_cache_invalidations_total{job=~\"$job\"})", - "legendFormat": "Total Requests", + "expr": "sum(increase(rustfs_object_cache_invalidations_total{job=~\"$job\"}[$__range]))", + "legendFormat": "__auto", "range": true, "refId": "A" } @@ -302,7 +667,7 @@ "h": 1, "w": 24, "x": 0, - "y": 6 + "y": 11 }, "id": 11, "panels": [], @@ -371,7 +736,7 @@ "h": 7, "w": 24, "x": 0, - "y": 7 + "y": 12 }, "id": 2, "options": { @@ -523,9 +888,9 @@ }, "gridPos": { "h": 7, - "w": 24, + "w": 12, "x": 0, - "y": 14 + "y": 19 }, "id": 3, "options": { @@ -697,9 +1062,9 @@ }, "gridPos": { "h": 7, - "w": 24, - "x": 0, - "y": 21 + "w": 12, + "x": 12, + "y": 19 }, "id": 8, "options": { @@ -763,11 +1128,11 @@ "h": 1, "w": 24, "x": 0, - "y": 28 + "y": 26 }, - "id": 12, + "id": 22, "panels": [], - "title": "Resource Useage", + "title": "GET Operations", "type": "row" }, { @@ -824,17 +1189,48 @@ } ] }, - "unit": "percent" + "unit": "reqps" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Started" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Completed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] }, "gridPos": { "h": 7, "w": 24, "x": 0, - "y": 29 + "y": 27 }, - "id": 9, + "id": 23, "options": { "legend": { "calcs": [ @@ -859,13 +1255,484 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (job) (process_cpu_usage_percent{job=~\"$job\"})", - "legendFormat": "{{job}}", + "expr": "sum(rate(rustfs_get_object_requests_completed_total{job=~\"$job\"}[$__rate_interval]))", + "hide": false, + "legendFormat": "Completed", + "range": true, + "refId": "B" + } + ], + "title": "GET Request Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*Duration.*" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*Total.*" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "shades" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 34 + }, + "id": 25, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.50, sum by (le) (rate(rustfs_get_object_duration_seconds_bucket{job=~\"$job\"}[5m])))", + "legendFormat": "P50 Duration", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by (le) (rate(rustfs_get_object_duration_seconds_bucket{job=~\"$job\"}[5m])))", + "hide": false, + "legendFormat": "P95 Duration", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(rustfs_get_object_duration_seconds_bucket{job=~\"$job\"}[5m])))", + "hide": false, + "legendFormat": "P99 Duration", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.50, sum by (le) (rate(rustfs_get_object_total_duration_seconds_bucket{job=~\"$job\"}[5m])))", + "hide": false, + "legendFormat": "P50 Total", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by (le) (rate(rustfs_get_object_total_duration_seconds_bucket{job=~\"$job\"}[5m])))", + "hide": false, + "legendFormat": "P95 Total", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(rustfs_get_object_total_duration_seconds_bucket{job=~\"$job\"}[5m])))", + "hide": false, + "legendFormat": "P99 Total", + "range": true, + "refId": "F" + } + ], + "title": "GET Latency Percentiles", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "P50" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "P95" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "P99" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 34 + }, + "id": 26, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.50, sum by (le) (rate(rustfs_get_object_response_size_bytes_bucket{job=~\"$job\"}[5m])))", + "legendFormat": "P50", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by (le) (rate(rustfs_get_object_response_size_bytes_bucket{job=~\"$job\"}[5m])))", + "hide": false, + "legendFormat": "P95", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(rustfs_get_object_response_size_bytes_bucket{job=~\"$job\"}[5m])))", + "hide": false, + "legendFormat": "P99", + "range": true, + "refId": "C" + } + ], + "title": "GET Response Size Percentiles", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 41 + }, + "id": 34, + "panels": [], + "title": "Buckets", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "objs" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 42 + }, + "id": 35, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (bucket) (rustfs_bucket_objects_total{job=~\"$job\", bucket=~\"$bucket\"})", + "legendFormat": "{{bucket}}", "range": true, "refId": "A" } ], - "title": "CPU", + "title": "Bucket Objects", "type": "timeseries" }, { @@ -928,9 +1795,757 @@ }, "gridPos": { "h": 7, + "w": 12, + "x": 12, + "y": 42 + }, + "id": 37, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (bucket) (rustfs_bucket_usage_bytes{job=~\"$job\", bucket=~\"$bucket\"})", + "legendFormat": "{{bucket}}", + "range": true, + "refId": "A" + } + ], + "title": "Bucket Size", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, "w": 24, "x": 0, - "y": 36 + "y": 49 + }, + "id": 27, + "panels": [], + "title": "Disks", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*/Percent" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "unit", + "value": "percentunit" + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "unit", + "value": "percentunit" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 50 + }, + "id": 28, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (drive) (rustfs_node_disk_used_bytes{job=~\"$job\", drive=~\"$drive\"})", + "legendFormat": "{{drive}} (bytes)", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (drive) (rustfs_node_disk_used_bytes{job=~\"$job\", drive=~\"$drive\"}) / sum by (drive)(rustfs_node_disk_total_bytes{job=~\"$job\", drive=~\"$drive\"})", + "hide": false, + "instant": false, + "legendFormat": "{{drive}} (percent)", + "range": true, + "refId": "B" + } + ], + "title": "Node Disk Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "P50" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "P95" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "P99" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 50 + }, + "id": 29, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.50, sum by (le) (rate(rustfs_disk_permit_wait_duration_seconds_bucket{job=~\"$job\"}[5m])))", + "legendFormat": "P50", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by (le) (rate(rustfs_disk_permit_wait_duration_seconds_bucket{job=~\"$job\"}[5m])))", + "hide": false, + "legendFormat": "P95", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(rustfs_disk_permit_wait_duration_seconds_bucket{job=~\"$job\"}[5m])))", + "hide": false, + "legendFormat": "P99", + "range": true, + "refId": "C" + } + ], + "title": "Disk Permit Wait Duration", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 57 + }, + "id": 30, + "panels": [], + "title": "I/O Performance", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 58 + }, + "id": 33, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (level) (rate(rustfs_io_strategy_selected_total{job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "{{level}}", + "range": true, + "refId": "A" + } + ], + "title": "I/O Strategy Selection Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 58 + }, + "id": 32, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rustfs_io_buffer_multiplier{job=~\"$job\"}", + "legendFormat": "{{job}}", + "range": true, + "refId": "A" + } + ], + "title": "I/O Buffer Multiplier", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 65 + }, + "id": 12, + "panels": [], + "title": "Resource Usage", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "shades" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 66 + }, + "id": 9, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (job) (process_cpu_usage_percent{job=~\"$job\"})", + "legendFormat": "{{job}}", + "range": true, + "refId": "A" + } + ], + "title": "CPU", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "purple", + "mode": "shades" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 66 }, "id": 10, "options": { @@ -957,7 +2572,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (job) (process_memory_usage_byte{job=~\"$job\"})", + "expr": "sum by (job) (rustfs_process_memory_bytes{job=~\"$job\"})", "legendFormat": "{{job}}", "range": true, "refId": "B" @@ -1020,7 +2635,7 @@ } ] }, - "unit": "bytes" + "unit": "Bps" }, "overrides": [] }, @@ -1028,7 +2643,7 @@ "h": 7, "w": 24, "x": 0, - "y": 43 + "y": 73 }, "id": 5, "options": { @@ -1077,7 +2692,7 @@ } ], "preload": false, - "refresh": "10s", + "refresh": "15s", "schemaVersion": 42, "tags": [ "rustfs", @@ -1103,9 +2718,7 @@ "allValue": ".*", "current": { "text": "All", - "value": [ - "$__all" - ] + "value": "$__all" }, "datasource": { "type": "prometheus", @@ -1122,7 +2735,7 @@ "query": "label_values(rustfs_api_requests_total, job)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, - "refresh": 1, + "refresh": 2, "regex": "", "sort": 1, "type": "query" @@ -1152,6 +2765,48 @@ "regex": "", "sort": 1, "type": "query" + }, + { + "allowCustomValue": false, + "current": { + "text": "All", + "value": "$__all" + }, + "definition": "label_values(rustfs_bucket_objects_total,bucket)", + "includeAll": true, + "label": "Bucket", + "multi": true, + "name": "bucket", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rustfs_bucket_objects_total,bucket)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "type": "query" + }, + { + "allowCustomValue": false, + "current": { + "text": "All", + "value": "$__all" + }, + "definition": "label_values(rustfs_node_disk_used_bytes,drive)", + "includeAll": true, + "label": "Drive", + "multi": true, + "name": "drive", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rustfs_node_disk_used_bytes,drive)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "type": "query" } ] }, @@ -1159,9 +2814,23 @@ "from": "now-1h", "to": "now" }, - "timepicker": {}, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "15s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, "timezone": "browser", "title": "RustFS", "uid": "rustfs-s3", - "version": 8 + "version": 10 }