mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-26 08:18:18 +00:00
perf(memory): add reclaim signals and cache controls (#2689)
This commit is contained in:
houseme
@@ -2534,7 +2534,7 @@
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum by (job) (rustfs_system_process_resident_memory_bytes{job=~\"$job\"})",
|
||||
"expr": "sum by (job) (rustfs_system_process_resident_memory_bytes{job=~\"$job\"}) or sum by (job) (rustfs_memory_process_resident_bytes{job=~\"$job\"})",
|
||||
"legendFormat": "process rss - {{job}}",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
@@ -2545,8 +2545,8 @@
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "rustfs_system_memory_used_bytes{job=~\"$job\"}",
|
||||
"legendFormat": "system used - {{job}}",
|
||||
"expr": "sum by (job) (rustfs_memory_process_virtual_bytes{job=~\"$job\"})",
|
||||
"legendFormat": "process virtual - {{job}}",
|
||||
"range": true,
|
||||
"refId": "B"
|
||||
},
|
||||
@@ -2556,13 +2556,755 @@
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "rustfs_system_memory_used_perc{job=~\"$job\"}",
|
||||
"legendFormat": "system used percent - {{job}}",
|
||||
"expr": "sum by (job) (rustfs_memory_cgroup_anon_bytes{job=~\"$job\"})",
|
||||
"legendFormat": "anon split - {{job}}",
|
||||
"range": true,
|
||||
"refId": "C"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum by (job) (rustfs_memory_cgroup_file_bytes{job=~\"$job\"})",
|
||||
"legendFormat": "file split - {{job}}",
|
||||
"range": true,
|
||||
"refId": "D"
|
||||
}
|
||||
],
|
||||
"title": "Memory Split",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"barWidthFactor": 0.6,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"showValues": false,
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": 0
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 80
|
||||
},
|
||||
"id": 501,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"lastNotNull",
|
||||
"max"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "right",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"hideZeros": false,
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "12.3.2",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum by (job) (rustfs_delete_tail_activity_total_inflight_current{job=~\"$job\"})",
|
||||
"legendFormat": "delete tail inflight - {{job}}",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum by (job) (rustfs_memory_allocator_reclaim_scanner_activity_current{job=~\"$job\"})",
|
||||
"legendFormat": "scanner activity - {{job}}",
|
||||
"range": true,
|
||||
"refId": "B"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum by (job) (rustfs_memory_allocator_reclaim_heal_activity_current{job=~\"$job\"})",
|
||||
"legendFormat": "heal activity - {{job}}",
|
||||
"range": true,
|
||||
"refId": "C"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum by (job) (rustfs_memory_allocator_reclaim_reclaimable_work_current{job=~\"$job\"})",
|
||||
"legendFormat": "reclaimable work - {{job}}",
|
||||
"range": true,
|
||||
"refId": "D"
|
||||
}
|
||||
],
|
||||
"title": "Tail / Reclaim Activity",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"barWidthFactor": 0.6,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"showValues": false,
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": 0
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 80
|
||||
},
|
||||
"id": 502,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"lastNotNull",
|
||||
"max"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "right",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"hideZeros": false,
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "12.3.2",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum by (job) (rustfs_memory_allocator_reclaim_idle_streak{job=~\"$job\"})",
|
||||
"legendFormat": "idle streak - {{job}}",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum by (job) (rate(rustfs_memory_allocator_reclaim_total{job=~\"$job\",result=\"ok\"}[5m]))",
|
||||
"legendFormat": "reclaim ok rate - {{job}}",
|
||||
"range": true,
|
||||
"refId": "B"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum by (job,reason) (rate(rustfs_memory_allocator_reclaim_skipped_total{job=~\"$job\"}[5m]))",
|
||||
"legendFormat": "reclaim skipped {{reason}} - {{job}}",
|
||||
"range": true,
|
||||
"refId": "C"
|
||||
}
|
||||
],
|
||||
"title": "Memory Usage",
|
||||
"title": "Allocator Reclaim Health",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"barWidthFactor": 0.6,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"showValues": false,
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": 0
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "bytes"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 87
|
||||
},
|
||||
"id": 503,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"lastNotNull",
|
||||
"max"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "right",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"hideZeros": false,
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "12.3.2",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum by (job) (rustfs_memory_cgroup_current_bytes{job=~\"$job\"})",
|
||||
"legendFormat": "cgroup current - {{job}}",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum by (job) (rustfs_memory_cgroup_limit_bytes{job=~\"$job\"})",
|
||||
"legendFormat": "cgroup limit - {{job}}",
|
||||
"range": true,
|
||||
"refId": "B"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum by (job) (rustfs_memory_cgroup_active_file_bytes{job=~\"$job\"})",
|
||||
"legendFormat": "active file - {{job}}",
|
||||
"range": true,
|
||||
"refId": "C"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum by (job) (rustfs_memory_cgroup_inactive_file_bytes{job=~\"$job\"})",
|
||||
"legendFormat": "inactive file - {{job}}",
|
||||
"range": true,
|
||||
"refId": "D"
|
||||
}
|
||||
],
|
||||
"title": "Memory Cgroup Detail",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"barWidthFactor": 0.6,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"showValues": false,
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": 0
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 87
|
||||
},
|
||||
"id": 504,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"lastNotNull",
|
||||
"max"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "right",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"hideZeros": false,
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "12.3.2",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum by (job) (rustfs_ec_encode_inflight_bytes_current{job=~\"$job\"})",
|
||||
"legendFormat": "ec inflight bytes - {{job}}",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum by (job) (rustfs_get_object_buffered_bytes_current{job=~\"$job\"})",
|
||||
"legendFormat": "get buffered bytes - {{job}}",
|
||||
"range": true,
|
||||
"refId": "B"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum by (job) (rustfs_memory_allocator_reclaim_active_requests{job=~\"$job\"})",
|
||||
"legendFormat": "allocator active requests - {{job}}",
|
||||
"range": true,
|
||||
"refId": "C"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum by (job) (rustfs_memory_allocator_reclaim_delete_tail_activity_current{job=~\"$job\"})",
|
||||
"legendFormat": "allocator tail activity - {{job}}",
|
||||
"range": true,
|
||||
"refId": "D"
|
||||
}
|
||||
],
|
||||
"title": "Heap Amplification Signals",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"barWidthFactor": 0.6,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"showValues": false,
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": 0
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 94
|
||||
},
|
||||
"id": 505,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"lastNotNull",
|
||||
"max"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "right",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"hideZeros": false,
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "12.3.2",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum by (job,stage) (rustfs_delete_tail_activity_inflight_current{job=~\"$job\"})",
|
||||
"legendFormat": "{{stage}} inflight - {{job}}",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum by (job,stage) (rate(rustfs_delete_tail_activity_started_total{job=~\"$job\"}[5m]))",
|
||||
"legendFormat": "{{stage}} started rate - {{job}}",
|
||||
"range": true,
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"title": "Delete Tail by Stage",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"barWidthFactor": 0.6,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"showValues": false,
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": 0
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "bytes"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 94
|
||||
},
|
||||
"id": 506,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"lastNotNull",
|
||||
"max"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "right",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"hideZeros": false,
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "12.3.2",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum by (job,kind,result) (rate(rustfs_page_cache_reclaim_requests_total{job=~\"$job\"}[5m]))",
|
||||
"legendFormat": "{{kind}} reclaim {{result}} rate - {{job}}",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum by (job,kind) (rate(rustfs_page_cache_reclaim_bytes_total{job=~\"$job\"}[5m]))",
|
||||
"legendFormat": "{{kind}} reclaim bytes/s - {{job}}",
|
||||
"range": true,
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"title": "Page Cache Reclaim",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
|
||||
Generated
+46
-75
@@ -40,18 +40,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6b657e772794c6b04730ea897b66a058ccd866c16d1967da05eeeecec39043fe"
|
||||
dependencies = [
|
||||
"crypto-common 0.2.1",
|
||||
"inout 0.2.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aes"
|
||||
version = "0.8.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"cipher 0.4.4",
|
||||
"cpufeatures 0.2.17",
|
||||
"inout",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -60,7 +49,7 @@ version = "0.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "66bd29a732b644c0431c6140f370d097879203d79b80c94a6747ba0872adaef8"
|
||||
dependencies = [
|
||||
"cipher 0.5.1",
|
||||
"cipher",
|
||||
"cpubits",
|
||||
"cpufeatures 0.3.0",
|
||||
]
|
||||
@@ -72,8 +61,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e22c0c90bbe8d4f77c3ca9ddabe41a1f8382d6fc1f7cea89459d0f320371f972"
|
||||
dependencies = [
|
||||
"aead",
|
||||
"aes 0.9.0",
|
||||
"cipher 0.5.1",
|
||||
"aes",
|
||||
"cipher",
|
||||
"ctr",
|
||||
"ghash",
|
||||
"subtle",
|
||||
@@ -203,7 +192,7 @@ version = "1.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
|
||||
dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.60.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -214,7 +203,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
|
||||
dependencies = [
|
||||
"anstyle",
|
||||
"once_cell_polyfill",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.60.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1323,6 +1312,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cdd35008169921d80bc60d3d0ab416eecb028c4cd653352907921d95084790be"
|
||||
dependencies = [
|
||||
"hybrid-array",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1508,7 +1498,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"cipher 0.5.1",
|
||||
"cipher",
|
||||
"cpufeatures 0.3.0",
|
||||
"rand_core 0.10.1",
|
||||
]
|
||||
@@ -1521,7 +1511,7 @@ checksum = "1c9ed179664f12fd6f155f6dd632edf5f3806d48c228c67ff78366f2a0eb6b5e"
|
||||
dependencies = [
|
||||
"aead",
|
||||
"chacha20",
|
||||
"cipher 0.5.1",
|
||||
"cipher",
|
||||
"poly1305",
|
||||
]
|
||||
|
||||
@@ -1576,16 +1566,6 @@ dependencies = [
|
||||
"half",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cipher"
|
||||
version = "0.4.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad"
|
||||
dependencies = [
|
||||
"crypto-common 0.1.6",
|
||||
"inout 0.1.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cipher"
|
||||
version = "0.5.1"
|
||||
@@ -1594,7 +1574,7 @@ checksum = "e34d8227fe1ba289043aeb13792056ff80fd6de1a9f49137a5f499de8e8c78ea"
|
||||
dependencies = [
|
||||
"block-buffer 0.12.0",
|
||||
"crypto-common 0.2.1",
|
||||
"inout 0.2.2",
|
||||
"inout",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2045,9 +2025,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "crypto-common"
|
||||
version = "0.1.6"
|
||||
version = "0.1.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
|
||||
checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
|
||||
dependencies = [
|
||||
"generic-array",
|
||||
"typenum",
|
||||
@@ -2102,7 +2082,7 @@ version = "0.10.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "17469f8eb9bdbfad10f71f4cfddfd38b01143520c0e717d8796ccb4d44d44e42"
|
||||
dependencies = [
|
||||
"cipher 0.5.1",
|
||||
"cipher",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2114,6 +2094,12 @@ dependencies = [
|
||||
"cmov",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cty"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b365fabc795046672053e29c954733ec3b05e4be654ab130fe8f1f94d7051f35"
|
||||
|
||||
[[package]]
|
||||
name = "curve25519-dalek"
|
||||
version = "4.1.3"
|
||||
@@ -3175,7 +3161,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
|
||||
dependencies = [
|
||||
"block-buffer 0.10.4",
|
||||
"const-oid 0.9.6",
|
||||
"crypto-common 0.1.6",
|
||||
"crypto-common 0.1.7",
|
||||
"subtle",
|
||||
]
|
||||
|
||||
@@ -3189,6 +3175,7 @@ dependencies = [
|
||||
"const-oid 0.10.2",
|
||||
"crypto-common 0.2.1",
|
||||
"ctutils",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3209,7 +3196,7 @@ dependencies = [
|
||||
"libc",
|
||||
"option-ext",
|
||||
"redox_users",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3488,7 +3475,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3777,9 +3764,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "generic-array"
|
||||
version = "0.14.9"
|
||||
version = "0.14.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2"
|
||||
checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
|
||||
dependencies = [
|
||||
"typenum",
|
||||
"version_check",
|
||||
@@ -4691,15 +4678,6 @@ dependencies = [
|
||||
"str_stack",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "inout"
|
||||
version = "0.1.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01"
|
||||
dependencies = [
|
||||
"generic-array",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "inout"
|
||||
version = "0.2.2"
|
||||
@@ -4768,7 +4746,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46"
|
||||
dependencies = [
|
||||
"hermit-abi",
|
||||
"libc",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4845,7 +4823,7 @@ dependencies = [
|
||||
"portable-atomic",
|
||||
"portable-atomic-util",
|
||||
"serde_core",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -5161,6 +5139,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2d1eacfa31c33ec25e873c136ba5669f00f9866d0688bea7be4d3f7e43067df6"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"cty",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -5814,7 +5793,7 @@ version = "0.50.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
|
||||
dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -6438,16 +6417,6 @@ dependencies = [
|
||||
"once_cell",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pbkdf2"
|
||||
version = "0.12.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2"
|
||||
dependencies = [
|
||||
"digest 0.10.7",
|
||||
"hmac 0.12.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pbkdf2"
|
||||
version = "0.13.0"
|
||||
@@ -7878,6 +7847,7 @@ dependencies = [
|
||||
"jemalloc_pprof",
|
||||
"jiff",
|
||||
"libc",
|
||||
"libmimalloc-sys",
|
||||
"libsystemd",
|
||||
"matchit 0.9.2",
|
||||
"md5",
|
||||
@@ -8066,7 +8036,7 @@ dependencies = [
|
||||
"cfg-if",
|
||||
"chacha20poly1305",
|
||||
"jsonwebtoken",
|
||||
"pbkdf2 0.13.0",
|
||||
"pbkdf2",
|
||||
"rand 0.10.1",
|
||||
"serde_json",
|
||||
"sha2 0.11.0",
|
||||
@@ -8145,6 +8115,7 @@ dependencies = [
|
||||
"rustfs-signer",
|
||||
"rustfs-utils",
|
||||
"rustfs-workers",
|
||||
"rustix 1.1.4",
|
||||
"rustls",
|
||||
"s3s",
|
||||
"serde",
|
||||
@@ -8294,7 +8265,7 @@ dependencies = [
|
||||
"indexmap 2.14.0",
|
||||
"kafka-protocol",
|
||||
"metrics",
|
||||
"pbkdf2 0.13.0",
|
||||
"pbkdf2",
|
||||
"rand 0.10.1",
|
||||
"rustls",
|
||||
"rustls-native-certs",
|
||||
@@ -8317,7 +8288,7 @@ dependencies = [
|
||||
"hmac 0.13.0",
|
||||
"kafka-protocol",
|
||||
"metrics",
|
||||
"pbkdf2 0.13.0",
|
||||
"pbkdf2",
|
||||
"rand 0.10.1",
|
||||
"rustfs-kafka",
|
||||
"rustls",
|
||||
@@ -8917,7 +8888,7 @@ dependencies = [
|
||||
"errno",
|
||||
"libc",
|
||||
"linux-raw-sys 0.12.1",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -8976,7 +8947,7 @@ dependencies = [
|
||||
"security-framework",
|
||||
"security-framework-sys",
|
||||
"webpki-root-certs",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -9647,7 +9618,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.60.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -9736,7 +9707,7 @@ dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"psm",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.60.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -9983,7 +9954,7 @@ dependencies = [
|
||||
"getrandom 0.4.2",
|
||||
"once_cell",
|
||||
"rustix 1.1.4",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -10973,7 +10944,7 @@ version = "0.1.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
|
||||
dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -11644,24 +11615,24 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "zip"
|
||||
version = "8.5.1"
|
||||
version = "8.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dcab981e19633ebcf0b001ddd37dd802996098bc1864f90b7c5d970ce76c1d59"
|
||||
checksum = "2d04a6b5381502aa6087c94c669499eb1602eb9c5e8198e534de571f7154809b"
|
||||
dependencies = [
|
||||
"aes 0.8.4",
|
||||
"aes",
|
||||
"bzip2",
|
||||
"constant_time_eq",
|
||||
"crc32fast",
|
||||
"deflate64",
|
||||
"flate2",
|
||||
"getrandom 0.4.2",
|
||||
"hmac 0.12.1",
|
||||
"hmac 0.13.0",
|
||||
"indexmap 2.14.0",
|
||||
"lzma-rust2",
|
||||
"memchr",
|
||||
"pbkdf2 0.12.2",
|
||||
"pbkdf2",
|
||||
"ppmd-rust",
|
||||
"sha1 0.10.6",
|
||||
"sha1 0.11.0",
|
||||
"time",
|
||||
"typed-path",
|
||||
"zeroize",
|
||||
|
||||
+1
-1
@@ -280,7 +280,7 @@ walkdir = "2.5.0"
|
||||
wildmatch = { version = "2.6.1", features = ["serde"] }
|
||||
windows = { version = "0.62.2" }
|
||||
xxhash-rust = { version = "0.8.15", features = ["xxh64", "xxh3"] }
|
||||
zip = "8.5.1"
|
||||
zip = "8.6.0"
|
||||
zstd = "0.13.3"
|
||||
|
||||
# Observability and Metrics
|
||||
|
||||
@@ -59,9 +59,28 @@ pub const DEFAULT_RUNTIME_DIAL9_ROTATION_COUNT: usize = 10;
|
||||
pub const DEFAULT_RUNTIME_DIAL9_SAMPLING_RATE: f64 = 1.0; // 100% sampling
|
||||
// Note: S3 bucket/prefix have no default; absence means upload is disabled (modeled as Option<String>)
|
||||
|
||||
// Allocator reclaim configuration
|
||||
pub const ENV_ALLOCATOR_RECLAIM_ENABLED: &str = "RUSTFS_ALLOCATOR_RECLAIM_ENABLED";
|
||||
pub const ENV_ALLOCATOR_RECLAIM_INTERVAL_SECS: &str = "RUSTFS_ALLOCATOR_RECLAIM_INTERVAL_SECS";
|
||||
pub const ENV_ALLOCATOR_RECLAIM_FORCE: &str = "RUSTFS_ALLOCATOR_RECLAIM_FORCE";
|
||||
pub const ENV_ALLOCATOR_RECLAIM_IDLE_INTERVALS: &str = "RUSTFS_ALLOCATOR_RECLAIM_IDLE_INTERVALS";
|
||||
pub const DEFAULT_ALLOCATOR_RECLAIM_ENABLED: bool = false;
|
||||
pub const DEFAULT_ALLOCATOR_RECLAIM_INTERVAL_SECS: u64 = 30;
|
||||
pub const DEFAULT_ALLOCATOR_RECLAIM_FORCE: bool = true;
|
||||
pub const DEFAULT_ALLOCATOR_RECLAIM_IDLE_INTERVALS: u64 = 3;
|
||||
|
||||
// File page-cache reclaim configuration
|
||||
pub const ENV_OBJECT_FILE_CACHE_RECLAIM_WRITE_ENABLE: &str = "RUSTFS_OBJECT_FILE_CACHE_RECLAIM_WRITE_ENABLE";
|
||||
pub const ENV_OBJECT_FILE_CACHE_RECLAIM_READ_ENABLE: &str = "RUSTFS_OBJECT_FILE_CACHE_RECLAIM_READ_ENABLE";
|
||||
pub const ENV_OBJECT_FILE_CACHE_RECLAIM_THRESHOLD: &str = "RUSTFS_OBJECT_FILE_CACHE_RECLAIM_THRESHOLD";
|
||||
pub const DEFAULT_OBJECT_FILE_CACHE_RECLAIM_WRITE_ENABLE: bool = false;
|
||||
pub const DEFAULT_OBJECT_FILE_CACHE_RECLAIM_READ_ENABLE: bool = false;
|
||||
pub const DEFAULT_OBJECT_FILE_CACHE_RECLAIM_THRESHOLD: usize = 4 * 1024 * 1024;
|
||||
|
||||
/// Threshold for small object seek support in megabytes.
|
||||
///
|
||||
/// When an object is smaller than this size, rustfs will provide seek support.
|
||||
///
|
||||
/// Default is set to 10MB.
|
||||
pub const ENV_OBJECT_SEEK_SUPPORT_THRESHOLD: &str = "RUSTFS_OBJECT_SEEK_SUPPORT_THRESHOLD";
|
||||
pub const DEFAULT_OBJECT_SEEK_SUPPORT_THRESHOLD: usize = 10 * 1024 * 1024;
|
||||
|
||||
@@ -100,6 +100,7 @@ pin-project-lite.workspace = true
|
||||
md-5.workspace = true
|
||||
memmap2 = { workspace = true }
|
||||
libc.workspace = true
|
||||
rustix = { workspace = true }
|
||||
rustfs-madmin.workspace = true
|
||||
rustfs-workers.workspace = true
|
||||
reqwest = { workspace = true }
|
||||
|
||||
@@ -31,6 +31,7 @@ use crate::disk::{
|
||||
use crate::erasure_coding::bitrot_verify;
|
||||
use crate::global::{GLOBAL_IsErasureSD, GLOBAL_RootDiskThreshold};
|
||||
use bytes::Bytes;
|
||||
use metrics::counter;
|
||||
use parking_lot::RwLock as ParkingLotRwLock;
|
||||
use rustfs_filemeta::{
|
||||
Cache, FileInfo, FileInfoOpts, FileMeta, MetaCacheEntry, MetacacheWriter, ObjectPartInfo, Opts, RawFileInfo, UpdateFn,
|
||||
@@ -79,6 +80,238 @@ pub enum InternalBuf<'a> {
|
||||
Owned(Bytes),
|
||||
}
|
||||
|
||||
struct FileCacheReclaimWriter {
|
||||
inner: File,
|
||||
reclaim_len: usize,
|
||||
reclaim_on_shutdown: bool,
|
||||
reclaimed: bool,
|
||||
}
|
||||
|
||||
struct FileCacheReclaimReader {
|
||||
inner: File,
|
||||
reclaim_offset: u64,
|
||||
reclaim_len: usize,
|
||||
reclaim_on_drop: bool,
|
||||
reclaimed: bool,
|
||||
}
|
||||
|
||||
fn record_file_cache_reclaim_success(kind: &'static str, reclaim_len: usize, started: std::time::Instant) {
|
||||
counter!("rustfs_page_cache_reclaim_requests_total", "kind" => kind.to_string(), "result" => "ok".to_string()).increment(1);
|
||||
counter!("rustfs_page_cache_reclaim_bytes_total", "kind" => kind.to_string()).increment(reclaim_len as u64);
|
||||
metrics::histogram!("rustfs_page_cache_reclaim_duration_seconds", "kind" => kind.to_string())
|
||||
.record(started.elapsed().as_secs_f64());
|
||||
}
|
||||
|
||||
fn record_file_cache_reclaim_error(kind: &'static str) {
|
||||
counter!("rustfs_page_cache_reclaim_requests_total", "kind" => kind.to_string(), "result" => "err".to_string()).increment(1);
|
||||
}
|
||||
|
||||
impl FileCacheReclaimReader {
|
||||
fn new(inner: File, reclaim_offset: u64, reclaim_len: usize, reclaim_on_drop: bool) -> Self {
|
||||
#[cfg(target_os = "macos")]
|
||||
if reclaim_on_drop {
|
||||
let _ = set_fd_nocache(&inner);
|
||||
}
|
||||
|
||||
Self {
|
||||
inner,
|
||||
reclaim_offset,
|
||||
reclaim_len,
|
||||
reclaim_on_drop,
|
||||
reclaimed: false,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn reclaim_file_cache(&mut self) -> std::io::Result<()> {
|
||||
use core::num::NonZeroU64;
|
||||
use rustix::fs::{Advice, fadvise};
|
||||
|
||||
if !self.reclaim_on_drop || self.reclaimed || self.reclaim_len == 0 {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let started = std::time::Instant::now();
|
||||
let reclaim_len =
|
||||
NonZeroU64::new(self.reclaim_len as u64).expect("reclaim_len is guaranteed non-zero by the early return");
|
||||
fadvise(&self.inner, self.reclaim_offset, Some(reclaim_len), Advice::DontNeed).map_err(std::io::Error::from)?;
|
||||
|
||||
self.reclaimed = true;
|
||||
record_file_cache_reclaim_success("read", self.reclaim_len, started);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
fn reclaim_file_cache(&mut self) -> std::io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
#[allow(unsafe_code)]
|
||||
fn set_fd_nocache(file: &File) -> std::io::Result<()> {
|
||||
use std::os::fd::AsRawFd;
|
||||
|
||||
// SAFETY: `fcntl` is called on a valid file descriptor owned by `file`.
|
||||
let ret = unsafe { libc::fcntl(file.as_raw_fd(), libc::F_NOCACHE, 1) };
|
||||
if ret == -1 {
|
||||
return Err(std::io::Error::last_os_error());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
#[allow(unsafe_code)]
|
||||
fn set_std_fd_nocache(file: &std::fs::File) -> std::io::Result<()> {
|
||||
use std::os::fd::AsRawFd;
|
||||
|
||||
// SAFETY: `fcntl` is called on a valid file descriptor owned by `file`.
|
||||
let ret = unsafe { libc::fcntl(file.as_raw_fd(), libc::F_NOCACHE, 1) };
|
||||
if ret == -1 {
|
||||
return Err(std::io::Error::last_os_error());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
impl Drop for FileCacheReclaimReader {
|
||||
fn drop(&mut self) {
|
||||
if let Err(err) = self.reclaim_file_cache() {
|
||||
record_file_cache_reclaim_error("read");
|
||||
debug!(error = ?err, reclaim_offset = self.reclaim_offset, reclaim_len = self.reclaim_len, "failed to reclaim file cache after read");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl tokio::io::AsyncRead for FileCacheReclaimReader {
|
||||
fn poll_read(
|
||||
mut self: std::pin::Pin<&mut Self>,
|
||||
cx: &mut std::task::Context<'_>,
|
||||
buf: &mut tokio::io::ReadBuf<'_>,
|
||||
) -> std::task::Poll<std::io::Result<()>> {
|
||||
std::pin::Pin::new(&mut self.inner).poll_read(cx, buf)
|
||||
}
|
||||
}
|
||||
|
||||
impl FileCacheReclaimWriter {
|
||||
fn new(inner: File, reclaim_len: usize, reclaim_on_shutdown: bool) -> Self {
|
||||
#[cfg(target_os = "macos")]
|
||||
if reclaim_on_shutdown {
|
||||
let _ = set_fd_nocache(&inner);
|
||||
}
|
||||
|
||||
Self {
|
||||
inner,
|
||||
reclaim_len,
|
||||
reclaim_on_shutdown,
|
||||
reclaimed: false,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn reclaim_file_cache(&mut self) -> std::io::Result<()> {
|
||||
use core::num::NonZeroU64;
|
||||
use rustix::fs::{Advice, fadvise};
|
||||
|
||||
if !self.reclaim_on_shutdown || self.reclaimed || self.reclaim_len == 0 {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let started = std::time::Instant::now();
|
||||
let reclaim_len =
|
||||
NonZeroU64::new(self.reclaim_len as u64).expect("reclaim_len is guaranteed non-zero by the early return");
|
||||
fadvise(&self.inner, 0, Some(reclaim_len), Advice::DontNeed).map_err(std::io::Error::from)?;
|
||||
|
||||
self.reclaimed = true;
|
||||
record_file_cache_reclaim_success("write", self.reclaim_len, started);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
fn reclaim_file_cache(&mut self) -> std::io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl AsyncWrite for FileCacheReclaimWriter {
|
||||
fn poll_write(
|
||||
mut self: std::pin::Pin<&mut Self>,
|
||||
cx: &mut std::task::Context<'_>,
|
||||
buf: &[u8],
|
||||
) -> std::task::Poll<std::io::Result<usize>> {
|
||||
std::pin::Pin::new(&mut self.inner).poll_write(cx, buf)
|
||||
}
|
||||
|
||||
fn poll_flush(mut self: std::pin::Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> std::task::Poll<std::io::Result<()>> {
|
||||
std::pin::Pin::new(&mut self.inner).poll_flush(cx)
|
||||
}
|
||||
|
||||
fn poll_shutdown(
|
||||
mut self: std::pin::Pin<&mut Self>,
|
||||
cx: &mut std::task::Context<'_>,
|
||||
) -> std::task::Poll<std::io::Result<()>> {
|
||||
match std::pin::Pin::new(&mut self.inner).poll_shutdown(cx) {
|
||||
std::task::Poll::Ready(Ok(())) => {
|
||||
if let Err(err) = self.reclaim_file_cache() {
|
||||
record_file_cache_reclaim_error("write");
|
||||
debug!(error = ?err, reclaim_len = self.reclaim_len, "failed to reclaim file cache after write");
|
||||
}
|
||||
std::task::Poll::Ready(Ok(()))
|
||||
}
|
||||
other => other,
|
||||
}
|
||||
}
|
||||
|
||||
fn poll_write_vectored(
|
||||
mut self: std::pin::Pin<&mut Self>,
|
||||
cx: &mut std::task::Context<'_>,
|
||||
bufs: &[std::io::IoSlice<'_>],
|
||||
) -> std::task::Poll<std::io::Result<usize>> {
|
||||
std::pin::Pin::new(&mut self.inner).poll_write_vectored(cx, bufs)
|
||||
}
|
||||
|
||||
fn is_write_vectored(&self) -> bool {
|
||||
self.inner.is_write_vectored()
|
||||
}
|
||||
}
|
||||
|
||||
fn should_reclaim_file_cache_after_write(file_size: i64) -> bool {
|
||||
if file_size <= 0 {
|
||||
return false;
|
||||
}
|
||||
|
||||
if !rustfs_utils::get_env_bool(
|
||||
rustfs_config::ENV_OBJECT_FILE_CACHE_RECLAIM_WRITE_ENABLE,
|
||||
rustfs_config::DEFAULT_OBJECT_FILE_CACHE_RECLAIM_WRITE_ENABLE,
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
|
||||
let threshold = rustfs_utils::get_env_usize(
|
||||
rustfs_config::ENV_OBJECT_FILE_CACHE_RECLAIM_THRESHOLD,
|
||||
rustfs_config::DEFAULT_OBJECT_FILE_CACHE_RECLAIM_THRESHOLD,
|
||||
);
|
||||
file_size as usize >= threshold
|
||||
}
|
||||
|
||||
fn should_reclaim_file_cache_after_read(length: usize) -> bool {
|
||||
if length == 0 {
|
||||
return false;
|
||||
}
|
||||
|
||||
if !rustfs_utils::get_env_bool(
|
||||
rustfs_config::ENV_OBJECT_FILE_CACHE_RECLAIM_READ_ENABLE,
|
||||
rustfs_config::DEFAULT_OBJECT_FILE_CACHE_RECLAIM_READ_ENABLE,
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
|
||||
let threshold = rustfs_utils::get_env_usize(
|
||||
rustfs_config::ENV_OBJECT_FILE_CACHE_RECLAIM_THRESHOLD,
|
||||
rustfs_config::DEFAULT_OBJECT_FILE_CACHE_RECLAIM_THRESHOLD,
|
||||
);
|
||||
length >= threshold
|
||||
}
|
||||
|
||||
pub struct LocalDisk {
|
||||
pub root: PathBuf,
|
||||
pub format_path: PathBuf,
|
||||
@@ -1847,8 +2080,9 @@ impl DiskAPI for LocalDisk {
|
||||
let f = super::fs::open_file(&file_path, O_CREATE | O_WRONLY)
|
||||
.await
|
||||
.map_err(to_file_error)?;
|
||||
let reclaim_on_shutdown = should_reclaim_file_cache_after_write(_file_size);
|
||||
|
||||
Ok(Box::new(f))
|
||||
Ok(Box::new(FileCacheReclaimWriter::new(f, _file_size.max(0) as usize, reclaim_on_shutdown)))
|
||||
|
||||
// Ok(())
|
||||
}
|
||||
@@ -1920,7 +2154,8 @@ impl DiskAPI for LocalDisk {
|
||||
f.seek(SeekFrom::Start(offset as u64)).await?;
|
||||
}
|
||||
|
||||
Ok(Box::new(f))
|
||||
let reclaim_on_drop = should_reclaim_file_cache_after_read(length);
|
||||
Ok(Box::new(FileCacheReclaimReader::new(f, offset as u64, length, reclaim_on_drop)))
|
||||
}
|
||||
|
||||
/// Zero-copy file read using memory mapping (Unix) or efficient read (non-Unix).
|
||||
@@ -1965,9 +2200,15 @@ impl DiskAPI for LocalDisk {
|
||||
use memmap2::MmapOptions;
|
||||
let file_path_clone = file_path.clone();
|
||||
|
||||
let should_reclaim_after_read = should_reclaim_file_cache_after_read(length);
|
||||
let bytes = tokio::task::spawn_blocking(move || {
|
||||
let file = std::fs::File::open(&file_path_clone).map_err(DiskError::from)?;
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
if should_reclaim_after_read {
|
||||
let _ = set_std_fd_nocache(&file);
|
||||
}
|
||||
|
||||
// mmap offsets on Unix must be page-size aligned. Align the
|
||||
// mapping down to the nearest page boundary, then slice out the
|
||||
// originally requested logical range.
|
||||
@@ -1995,7 +2236,21 @@ impl DiskAPI for LocalDisk {
|
||||
let end = logical_offset
|
||||
.checked_add(length)
|
||||
.ok_or_else(|| DiskError::other("mmap slice length overflow"))?;
|
||||
Ok::<Bytes, DiskError>(Bytes::copy_from_slice(&mmap[logical_offset..end]))
|
||||
let bytes = Bytes::copy_from_slice(&mmap[logical_offset..end]);
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
if should_reclaim_after_read {
|
||||
use core::num::NonZeroU64;
|
||||
use rustix::fs::{Advice, fadvise};
|
||||
|
||||
let reclaim_len =
|
||||
NonZeroU64::new(map_len as u64).ok_or_else(|| DiskError::other("mmap reclaim length overflow"))?;
|
||||
fadvise(&file, aligned_offset, Some(reclaim_len), Advice::DontNeed)
|
||||
.map_err(std::io::Error::from)
|
||||
.map_err(DiskError::from)?;
|
||||
}
|
||||
|
||||
Ok::<Bytes, DiskError>(bytes)
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)??;
|
||||
@@ -3372,4 +3627,32 @@ mod test {
|
||||
assert_eq!(normalize_path_components("C:\\a\\..\\b"), PathBuf::from("C:\\b"));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn should_reclaim_file_cache_after_write_respects_env_and_threshold() {
|
||||
temp_env::with_var_unset(rustfs_config::ENV_OBJECT_FILE_CACHE_RECLAIM_WRITE_ENABLE, || {
|
||||
assert!(!should_reclaim_file_cache_after_write(8 * 1024 * 1024));
|
||||
});
|
||||
|
||||
temp_env::with_var(rustfs_config::ENV_OBJECT_FILE_CACHE_RECLAIM_WRITE_ENABLE, Some("true"), || {
|
||||
temp_env::with_var(rustfs_config::ENV_OBJECT_FILE_CACHE_RECLAIM_THRESHOLD, Some("4194304"), || {
|
||||
assert!(should_reclaim_file_cache_after_write(8 * 1024 * 1024));
|
||||
assert!(!should_reclaim_file_cache_after_write(1024));
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn should_reclaim_file_cache_after_read_respects_env_and_threshold() {
|
||||
temp_env::with_var_unset(rustfs_config::ENV_OBJECT_FILE_CACHE_RECLAIM_READ_ENABLE, || {
|
||||
assert!(!should_reclaim_file_cache_after_read(8 * 1024 * 1024));
|
||||
});
|
||||
|
||||
temp_env::with_var(rustfs_config::ENV_OBJECT_FILE_CACHE_RECLAIM_READ_ENABLE, Some("true"), || {
|
||||
temp_env::with_var(rustfs_config::ENV_OBJECT_FILE_CACHE_RECLAIM_THRESHOLD, Some("4194304"), || {
|
||||
assert!(should_reclaim_file_cache_after_read(8 * 1024 * 1024));
|
||||
assert!(!should_reclaim_file_cache_after_read(1024));
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -27,7 +27,7 @@ use tokio::sync::mpsc;
|
||||
use tracing::error;
|
||||
|
||||
const ENV_RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES: &str = "RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES";
|
||||
const DEFAULT_RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES: usize = 32 * 1024 * 1024;
|
||||
const DEFAULT_RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES: usize = 8 * 1024 * 1024;
|
||||
const DEFAULT_RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BLOCKS: usize = 8;
|
||||
|
||||
fn encode_channel_capacity(expanded_block_bytes: usize, max_inflight_bytes: usize) -> usize {
|
||||
@@ -40,6 +40,16 @@ fn encode_channel_capacity(expanded_block_bytes: usize, max_inflight_bytes: usiz
|
||||
.clamp(1, DEFAULT_RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BLOCKS)
|
||||
}
|
||||
|
||||
fn queued_block_bytes(block: &[Bytes]) -> usize {
|
||||
block.iter().map(Bytes::len).sum()
|
||||
}
|
||||
|
||||
async fn drain_queued_inflight_bytes(rx: &mut mpsc::Receiver<Vec<Bytes>>) {
|
||||
while let Some(block) = rx.recv().await {
|
||||
rustfs_io_metrics::remove_ec_encode_inflight_bytes(queued_block_bytes(&block));
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct MultiWriter<'a> {
|
||||
writers: &'a mut [Option<BitrotWriterWrapper>],
|
||||
write_quorum: usize,
|
||||
@@ -217,7 +227,10 @@ impl Erasure {
|
||||
Ok(n) if n > 0 => {
|
||||
total += n;
|
||||
let res = self.encode_data(&buf[..n])?;
|
||||
let queued_bytes = queued_block_bytes(&res);
|
||||
rustfs_io_metrics::add_ec_encode_inflight_bytes(queued_bytes);
|
||||
if let Err(err) = tx.send(res).await {
|
||||
rustfs_io_metrics::remove_ec_encode_inflight_bytes(queued_bytes);
|
||||
return Err(std::io::Error::other(format!("Failed to send encoded data : {err}")));
|
||||
}
|
||||
}
|
||||
@@ -250,6 +263,8 @@ impl Erasure {
|
||||
if block.is_empty() {
|
||||
break;
|
||||
}
|
||||
let queued_bytes = queued_block_bytes(&block);
|
||||
rustfs_io_metrics::remove_ec_encode_inflight_bytes(queued_bytes);
|
||||
if let Err(err) = writers.write(block).await {
|
||||
write_err = Some(err);
|
||||
break;
|
||||
@@ -259,6 +274,7 @@ impl Erasure {
|
||||
if let Some(err) = write_err {
|
||||
task.abort();
|
||||
let _ = task.await;
|
||||
drain_queued_inflight_bytes(&mut rx).await;
|
||||
if let Err(shutdown_err) = writers.shutdown().await {
|
||||
error!("failed to shutdown erasure writers after write error: {:?}", shutdown_err);
|
||||
}
|
||||
|
||||
@@ -237,6 +237,14 @@ impl PriorityHealQueue {
|
||||
}
|
||||
}
|
||||
|
||||
fn publish_active_heal_count(active_heals: &HashMap<String, Arc<HealTask>>) {
|
||||
crate::set_heal_active_tasks(active_heals.len());
|
||||
}
|
||||
|
||||
fn publish_heal_queue_length(queue: &PriorityHealQueue) {
|
||||
crate::set_heal_queue_length(queue.len());
|
||||
}
|
||||
|
||||
/// Heal config
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct HealConfig {
|
||||
@@ -420,6 +428,8 @@ impl HealManager {
|
||||
}
|
||||
}
|
||||
active_heals.clear();
|
||||
publish_active_heal_count(&active_heals);
|
||||
crate::set_heal_queue_length(0);
|
||||
|
||||
// update state
|
||||
let mut state = self.state.write().await;
|
||||
@@ -435,6 +445,7 @@ impl HealManager {
|
||||
let mut queue = self.heal_queue.lock().await;
|
||||
|
||||
let queue_len = queue.len();
|
||||
publish_heal_queue_length(&queue);
|
||||
let queue_capacity = config.queue_size;
|
||||
|
||||
if queue.contains_key(&request) {
|
||||
@@ -505,6 +516,7 @@ impl HealManager {
|
||||
|
||||
let push_outcome = queue.push(request);
|
||||
debug_assert_eq!(push_outcome, QueuePushOutcome::Accepted);
|
||||
publish_heal_queue_length(&queue);
|
||||
|
||||
// Log queue statistics periodically (when adding high/urgent priority items)
|
||||
if matches!(priority, HealPriority::High | HealPriority::Urgent) {
|
||||
@@ -544,7 +556,9 @@ impl HealManager {
|
||||
|
||||
/// Get task progress
|
||||
pub async fn get_active_tasks_count(&self) -> usize {
|
||||
self.active_heals.lock().await.len()
|
||||
let active_heals = self.active_heals.lock().await;
|
||||
publish_active_heal_count(&active_heals);
|
||||
active_heals.len()
|
||||
}
|
||||
|
||||
pub async fn get_task_progress(&self, task_id: &str) -> Result<HealProgress> {
|
||||
@@ -564,6 +578,7 @@ impl HealManager {
|
||||
if let Some(task) = active_heals.get(task_id) {
|
||||
task.cancel().await?;
|
||||
active_heals.remove(task_id);
|
||||
publish_active_heal_count(&active_heals);
|
||||
info!("Cancelled heal task: {}", task_id);
|
||||
Ok(())
|
||||
} else {
|
||||
@@ -581,12 +596,14 @@ impl HealManager {
|
||||
/// Get active task count
|
||||
pub async fn get_active_task_count(&self) -> usize {
|
||||
let active_heals = self.active_heals.lock().await;
|
||||
publish_active_heal_count(&active_heals);
|
||||
active_heals.len()
|
||||
}
|
||||
|
||||
/// Get queue length
|
||||
pub async fn get_queue_length(&self) -> usize {
|
||||
let queue = self.heal_queue.lock().await;
|
||||
publish_heal_queue_length(&queue);
|
||||
queue.len()
|
||||
}
|
||||
|
||||
@@ -731,6 +748,7 @@ impl HealManager {
|
||||
);
|
||||
let mut queue = heal_queue.lock().await;
|
||||
if matches!(queue.push(req), QueuePushOutcome::Accepted) {
|
||||
publish_heal_queue_length(&queue);
|
||||
let config = config.read().await;
|
||||
if config.event_driven_scheduler_enable {
|
||||
notify.notify_one();
|
||||
@@ -757,6 +775,7 @@ impl HealManager {
|
||||
) {
|
||||
let config = config.read().await;
|
||||
let mut active_heals_guard = active_heals.lock().await;
|
||||
publish_active_heal_count(&active_heals_guard);
|
||||
|
||||
// Check if new heal tasks can be started
|
||||
let active_count = active_heals_guard.len();
|
||||
@@ -769,6 +788,7 @@ impl HealManager {
|
||||
|
||||
let mut queue = heal_queue.lock().await;
|
||||
let queue_len = queue.len();
|
||||
publish_heal_queue_length(&queue);
|
||||
|
||||
if queue_len == 0 {
|
||||
return;
|
||||
@@ -804,6 +824,7 @@ impl HealManager {
|
||||
let task = Arc::new(HealTask::from_request(request, storage.clone()));
|
||||
let task_id = task.id.clone();
|
||||
active_heals_guard.insert(task_id.clone(), task.clone());
|
||||
publish_active_heal_count(&active_heals_guard);
|
||||
update_task_running_metric_for_task(&active_heals_guard, task.as_ref());
|
||||
let active_heals_clone = active_heals.clone();
|
||||
let statistics_clone = statistics.clone();
|
||||
@@ -828,6 +849,7 @@ impl HealManager {
|
||||
}
|
||||
let mut active_heals_guard = active_heals_clone.lock().await;
|
||||
if let Some(completed_task) = active_heals_guard.remove(&task_id) {
|
||||
publish_active_heal_count(&active_heals_guard);
|
||||
update_task_running_metric_for_task(&active_heals_guard, completed_task.as_ref());
|
||||
// update statistics
|
||||
let mut stats = statistics_clone.write().await;
|
||||
@@ -853,6 +875,8 @@ impl HealManager {
|
||||
let mut stats = statistics.write().await;
|
||||
stats.total_tasks += tasks_started as u64;
|
||||
stats.update_running_tasks(active_heals_guard.len() as u64);
|
||||
publish_active_heal_count(&active_heals_guard);
|
||||
publish_heal_queue_length(&queue);
|
||||
|
||||
// Log queue status if items remain
|
||||
if !queue.is_empty() {
|
||||
|
||||
@@ -17,6 +17,7 @@ pub mod heal;
|
||||
|
||||
pub use error::{Error, Result};
|
||||
pub use heal::{HealManager, HealOptions, HealPriority, HealRequest, HealType, channel::HealChannelProcessor};
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::sync::{Arc, OnceLock};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::{error, info};
|
||||
@@ -55,6 +56,8 @@ static GLOBAL_HEAL_MANAGER: OnceLock<Arc<HealManager>> = OnceLock::new();
|
||||
|
||||
/// Global heal channel processor instance
|
||||
static GLOBAL_HEAL_CHANNEL_PROCESSOR: OnceLock<Arc<tokio::sync::Mutex<HealChannelProcessor>>> = OnceLock::new();
|
||||
static GLOBAL_HEAL_ACTIVE_TASKS: AtomicU64 = AtomicU64::new(0);
|
||||
static GLOBAL_HEAL_QUEUE_LENGTH: AtomicU64 = AtomicU64::new(0);
|
||||
|
||||
/// Initialize and start heal manager with channel processor
|
||||
pub async fn init_heal_manager(
|
||||
@@ -107,3 +110,19 @@ pub fn get_heal_manager() -> Option<&'static Arc<HealManager>> {
|
||||
pub fn get_heal_channel_processor() -> Option<&'static Arc<tokio::sync::Mutex<HealChannelProcessor>>> {
|
||||
GLOBAL_HEAL_CHANNEL_PROCESSOR.get()
|
||||
}
|
||||
|
||||
pub fn current_heal_active_tasks() -> u64 {
|
||||
GLOBAL_HEAL_ACTIVE_TASKS.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
pub fn current_heal_queue_length() -> u64 {
|
||||
GLOBAL_HEAL_QUEUE_LENGTH.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
pub(crate) fn set_heal_active_tasks(count: usize) {
|
||||
GLOBAL_HEAL_ACTIVE_TASKS.store(count as u64, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub(crate) fn set_heal_queue_length(count: usize) {
|
||||
GLOBAL_HEAL_QUEUE_LENGTH.store(count as u64, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
@@ -49,6 +49,8 @@
|
||||
#[macro_use]
|
||||
extern crate metrics;
|
||||
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
|
||||
// Public modules
|
||||
pub mod adaptive_ttl;
|
||||
pub mod autotuner;
|
||||
@@ -137,6 +139,43 @@ pub use config::{
|
||||
pub use collector::MetricsCollector;
|
||||
pub use performance::PerformanceMetrics;
|
||||
|
||||
static EC_ENCODE_INFLIGHT_BYTES: AtomicU64 = AtomicU64::new(0);
|
||||
static GET_OBJECT_BUFFERED_BYTES: AtomicU64 = AtomicU64::new(0);
|
||||
|
||||
fn saturating_sub_atomic(counter: &AtomicU64, bytes: u64) -> u64 {
|
||||
let mut current = counter.load(Ordering::Relaxed);
|
||||
loop {
|
||||
let next = current.saturating_sub(bytes);
|
||||
match counter.compare_exchange_weak(current, next, Ordering::Relaxed, Ordering::Relaxed) {
|
||||
Ok(_) => return next,
|
||||
Err(actual) => current = actual,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
enum TrackedMemoryGauge {
|
||||
GetObjectBufferedBytes,
|
||||
}
|
||||
|
||||
/// Drop-based guard for tracked in-memory payloads.
|
||||
#[derive(Debug)]
|
||||
pub struct MemoryGaugeGuard {
|
||||
gauge: TrackedMemoryGauge,
|
||||
bytes: u64,
|
||||
}
|
||||
|
||||
impl Drop for MemoryGaugeGuard {
|
||||
fn drop(&mut self) {
|
||||
match self.gauge {
|
||||
TrackedMemoryGauge::GetObjectBufferedBytes => {
|
||||
let next = saturating_sub_atomic(&GET_OBJECT_BUFFERED_BYTES, self.bytes);
|
||||
gauge!("rustfs_get_object_buffered_bytes_current").set(next as f64);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Record GetObject request start.
|
||||
#[inline(always)]
|
||||
pub fn record_get_object_request_start(concurrent_requests: usize) {
|
||||
@@ -544,6 +583,85 @@ pub fn record_memory_usage(used_bytes: u64, total_bytes: u64) {
|
||||
}
|
||||
}
|
||||
|
||||
/// Record process-level memory split metrics.
|
||||
#[inline(always)]
|
||||
pub fn record_process_memory_split(resident_bytes: u64, virtual_bytes: u64) {
|
||||
gauge!("rustfs_memory_process_resident_bytes").set(resident_bytes as f64);
|
||||
gauge!("rustfs_memory_process_virtual_bytes").set(virtual_bytes as f64);
|
||||
}
|
||||
|
||||
/// Record cgroup memory split metrics when available.
|
||||
#[inline(always)]
|
||||
pub fn record_cgroup_memory_split(
|
||||
current_bytes: Option<u64>,
|
||||
limit_bytes: Option<u64>,
|
||||
anon_bytes: Option<u64>,
|
||||
file_bytes: Option<u64>,
|
||||
active_file_bytes: Option<u64>,
|
||||
inactive_file_bytes: Option<u64>,
|
||||
) {
|
||||
if let Some(current_bytes) = current_bytes {
|
||||
gauge!("rustfs_memory_cgroup_current_bytes").set(current_bytes as f64);
|
||||
}
|
||||
if let Some(limit_bytes) = limit_bytes {
|
||||
gauge!("rustfs_memory_cgroup_limit_bytes").set(limit_bytes as f64);
|
||||
}
|
||||
if let Some(anon_bytes) = anon_bytes {
|
||||
gauge!("rustfs_memory_cgroup_anon_bytes").set(anon_bytes as f64);
|
||||
}
|
||||
if let Some(file_bytes) = file_bytes {
|
||||
gauge!("rustfs_memory_cgroup_file_bytes").set(file_bytes as f64);
|
||||
}
|
||||
if let Some(active_file_bytes) = active_file_bytes {
|
||||
gauge!("rustfs_memory_cgroup_active_file_bytes").set(active_file_bytes as f64);
|
||||
}
|
||||
if let Some(inactive_file_bytes) = inactive_file_bytes {
|
||||
gauge!("rustfs_memory_cgroup_inactive_file_bytes").set(inactive_file_bytes as f64);
|
||||
}
|
||||
}
|
||||
|
||||
/// Track encoded bytes currently queued between erasure encode and disk writers.
|
||||
#[inline(always)]
|
||||
pub fn add_ec_encode_inflight_bytes(bytes: usize) {
|
||||
let next = EC_ENCODE_INFLIGHT_BYTES.fetch_add(bytes as u64, Ordering::Relaxed) + bytes as u64;
|
||||
gauge!("rustfs_ec_encode_inflight_bytes_current").set(next as f64);
|
||||
}
|
||||
|
||||
/// Remove encoded bytes from the tracked erasure encode in-flight gauge.
|
||||
#[inline(always)]
|
||||
pub fn remove_ec_encode_inflight_bytes(bytes: usize) {
|
||||
let next = saturating_sub_atomic(&EC_ENCODE_INFLIGHT_BYTES, bytes as u64);
|
||||
gauge!("rustfs_ec_encode_inflight_bytes_current").set(next as f64);
|
||||
}
|
||||
|
||||
/// Return the current tracked EC encode in-flight bytes.
|
||||
#[inline(always)]
|
||||
pub fn current_ec_encode_inflight_bytes() -> u64 {
|
||||
EC_ENCODE_INFLIGHT_BYTES.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// Track whole-object buffering on the GET path.
|
||||
#[inline(always)]
|
||||
pub fn track_get_object_buffered_bytes(bytes: usize) -> Option<MemoryGaugeGuard> {
|
||||
if bytes == 0 {
|
||||
return None;
|
||||
}
|
||||
|
||||
let next = GET_OBJECT_BUFFERED_BYTES.fetch_add(bytes as u64, Ordering::Relaxed) + bytes as u64;
|
||||
gauge!("rustfs_get_object_buffered_bytes_current").set(next as f64);
|
||||
|
||||
Some(MemoryGaugeGuard {
|
||||
gauge: TrackedMemoryGauge::GetObjectBufferedBytes,
|
||||
bytes: bytes as u64,
|
||||
})
|
||||
}
|
||||
|
||||
/// Return the current tracked GET whole-buffered bytes.
|
||||
#[inline(always)]
|
||||
pub fn current_get_object_buffered_bytes() -> u64 {
|
||||
GET_OBJECT_BUFFERED_BYTES.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// Record CPU usage.
|
||||
///
|
||||
/// # Arguments
|
||||
@@ -758,6 +876,48 @@ mod tests {
|
||||
record_memory_usage(2 * 1024 * 1024 * 1024, 8 * 1024 * 1024 * 1024);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_record_process_memory_split() {
|
||||
record_process_memory_split(1024, 2048);
|
||||
record_process_memory_split(4096, 8192);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_record_cgroup_memory_split() {
|
||||
record_cgroup_memory_split(Some(1), Some(2), Some(3), Some(4), Some(5), Some(6));
|
||||
record_cgroup_memory_split(None, None, None, None, None, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_ec_encode_inflight_bytes_tracking() {
|
||||
EC_ENCODE_INFLIGHT_BYTES.store(0, Ordering::Relaxed);
|
||||
add_ec_encode_inflight_bytes(1024);
|
||||
add_ec_encode_inflight_bytes(2048);
|
||||
remove_ec_encode_inflight_bytes(1024);
|
||||
remove_ec_encode_inflight_bytes(2048);
|
||||
remove_ec_encode_inflight_bytes(4096);
|
||||
assert_eq!(current_ec_encode_inflight_bytes(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_get_object_buffered_bytes_guard() {
|
||||
GET_OBJECT_BUFFERED_BYTES.store(0, Ordering::Relaxed);
|
||||
drop(track_get_object_buffered_bytes(1024));
|
||||
let guard = track_get_object_buffered_bytes(2048);
|
||||
drop(guard);
|
||||
assert_eq!(current_get_object_buffered_bytes(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_get_object_buffered_bytes_guard_saturates_on_underflow() {
|
||||
GET_OBJECT_BUFFERED_BYTES.store(1024, Ordering::Relaxed);
|
||||
drop(MemoryGaugeGuard {
|
||||
gauge: TrackedMemoryGauge::GetObjectBufferedBytes,
|
||||
bytes: 2048,
|
||||
});
|
||||
assert_eq!(current_get_object_buffered_bytes(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_record_cpu_usage() {
|
||||
record_cpu_usage(25.5);
|
||||
|
||||
@@ -32,3 +32,26 @@ pub use data_usage_define::*;
|
||||
pub use error::ScannerError;
|
||||
pub use scanner::init_data_scanner;
|
||||
pub use sleeper::{DynamicSleeper, SCANNER_IDLE_MODE, SCANNER_SLEEPER};
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
|
||||
static SCANNER_ACTIVE_WORK_UNITS: AtomicU64 = AtomicU64::new(0);
|
||||
|
||||
pub fn current_scanner_activity() -> u64 {
|
||||
SCANNER_ACTIVE_WORK_UNITS.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
pub(crate) struct ScannerActivityGuard;
|
||||
|
||||
impl ScannerActivityGuard {
|
||||
pub(crate) fn new() -> Self {
|
||||
SCANNER_ACTIVE_WORK_UNITS.fetch_add(1, Ordering::Relaxed);
|
||||
Self
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ScannerActivityGuard {
|
||||
fn drop(&mut self) {
|
||||
let _ = SCANNER_ACTIVE_WORK_UNITS
|
||||
.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| Some(current.saturating_sub(1)));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,7 +18,7 @@ use crate::data_usage_define::{BACKGROUND_HEAL_INFO_PATH, DATA_USAGE_BLOOM_NAME_
|
||||
use crate::scanner_folder::data_usage_update_dir_cycles;
|
||||
use crate::scanner_io::ScannerIO;
|
||||
use crate::sleeper::SCANNER_SLEEPER;
|
||||
use crate::{DataUsageInfo, ScannerError};
|
||||
use crate::{DataUsageInfo, ScannerActivityGuard, ScannerError};
|
||||
use chrono::{DateTime, Utc};
|
||||
use rustfs_common::heal_channel::HealScanMode;
|
||||
use rustfs_common::metrics::{CurrentCycle, Metric, Metrics, emit_scan_cycle_complete, global_metrics};
|
||||
@@ -183,6 +183,7 @@ fn get_lock_acquire_timeout() -> Duration {
|
||||
|
||||
#[instrument(skip_all)]
|
||||
async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>, cycle_info: &mut CurrentCycle) {
|
||||
let _activity_guard = ScannerActivityGuard::new();
|
||||
SCANNER_SLEEPER.refresh_from_env();
|
||||
info!("Start run data scanner cycle");
|
||||
cycle_info.current = cycle_info.next;
|
||||
@@ -324,6 +325,7 @@ pub async fn store_data_usage_in_backend(
|
||||
let mut attempts = 1u32;
|
||||
|
||||
while let Some(data_usage_info) = receiver.recv().await {
|
||||
let _activity_guard = ScannerActivityGuard::new();
|
||||
if ctx.is_cancelled() {
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -171,6 +171,7 @@ libsystemd.workspace = true
|
||||
|
||||
[target.'cfg(not(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")))'.dependencies]
|
||||
mimalloc = { workspace = true }
|
||||
libmimalloc-sys = { version = "0.1.47", features = ["extended"] }
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,243 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#![allow(unsafe_code)]
|
||||
|
||||
use metrics::{counter, gauge, histogram};
|
||||
use std::time::Duration;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::{debug, warn};
|
||||
|
||||
pub fn allocator_backend() -> &'static str {
|
||||
#[cfg(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))]
|
||||
{
|
||||
"jemalloc"
|
||||
}
|
||||
|
||||
#[cfg(all(
|
||||
not(target_os = "windows"),
|
||||
not(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))
|
||||
))]
|
||||
{
|
||||
"mimalloc"
|
||||
}
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
{
|
||||
"mimalloc-windows"
|
||||
}
|
||||
}
|
||||
|
||||
fn active_requests() -> u64 {
|
||||
crate::server::active_http_requests()
|
||||
}
|
||||
|
||||
fn current_delete_tail_activity() -> u64 {
|
||||
crate::delete_tail_activity::current_delete_tail_activity()
|
||||
}
|
||||
|
||||
fn current_scanner_activity() -> u64 {
|
||||
rustfs_scanner::current_scanner_activity()
|
||||
}
|
||||
|
||||
fn current_heal_activity() -> u64 {
|
||||
rustfs_heal::current_heal_active_tasks() + rustfs_heal::current_heal_queue_length()
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default)]
|
||||
struct ReclaimableWorkSnapshot {
|
||||
active_requests: u64,
|
||||
delete_tail_activity: u64,
|
||||
scanner_activity: u64,
|
||||
heal_activity: u64,
|
||||
ec_inflight_bytes: u64,
|
||||
get_buffered_bytes: u64,
|
||||
}
|
||||
|
||||
impl ReclaimableWorkSnapshot {
|
||||
fn active_signal_count(self) -> u64 {
|
||||
u64::from(self.active_requests > 0)
|
||||
+ u64::from(self.delete_tail_activity > 0)
|
||||
+ u64::from(self.scanner_activity > 0)
|
||||
+ u64::from(self.heal_activity > 0)
|
||||
+ u64::from(self.ec_inflight_bytes > 0)
|
||||
+ u64::from(self.get_buffered_bytes > 0)
|
||||
}
|
||||
}
|
||||
|
||||
fn reclaimable_work_snapshot() -> ReclaimableWorkSnapshot {
|
||||
ReclaimableWorkSnapshot {
|
||||
active_requests: active_requests(),
|
||||
delete_tail_activity: current_delete_tail_activity(),
|
||||
scanner_activity: current_scanner_activity(),
|
||||
heal_activity: current_heal_activity(),
|
||||
ec_inflight_bytes: rustfs_io_metrics::current_ec_encode_inflight_bytes(),
|
||||
get_buffered_bytes: rustfs_io_metrics::current_get_object_buffered_bytes(),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(
|
||||
not(target_os = "windows"),
|
||||
not(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))
|
||||
))]
|
||||
fn collect_allocator_memory(force: bool) -> Result<(), String> {
|
||||
// SAFETY: `mi_collect` is provided by the active global allocator backend
|
||||
// on this target family. It is explicitly intended to reclaim retained
|
||||
// pages/segments and does not require additional invariants from the caller.
|
||||
unsafe {
|
||||
libmimalloc_sys::mi_collect(force);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))]
|
||||
fn collect_allocator_memory(_force: bool) -> Result<(), String> {
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
let _ = tikv_jemalloc_ctl::background_thread::write(true);
|
||||
tikv_jemalloc_ctl::epoch::advance().map_err(|err| err.to_string())?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
fn collect_allocator_memory(_force: bool) -> Result<(), String> {
|
||||
Err("allocator reclaim is not supported on Windows".to_string())
|
||||
}
|
||||
|
||||
fn run_allocator_reclaim(force: bool) {
|
||||
let backend = allocator_backend();
|
||||
let start = std::time::Instant::now();
|
||||
|
||||
match collect_allocator_memory(force) {
|
||||
Ok(()) => {
|
||||
counter!("rustfs_memory_allocator_reclaim_total", "backend" => backend.to_string(), "result" => "ok".to_string())
|
||||
.increment(1);
|
||||
histogram!(
|
||||
"rustfs_memory_allocator_reclaim_duration_seconds",
|
||||
"backend" => backend.to_string(),
|
||||
"result" => "ok".to_string()
|
||||
)
|
||||
.record(start.elapsed().as_secs_f64());
|
||||
}
|
||||
Err(err) => {
|
||||
counter!(
|
||||
"rustfs_memory_allocator_reclaim_total",
|
||||
"backend" => backend.to_string(),
|
||||
"result" => "err".to_string()
|
||||
)
|
||||
.increment(1);
|
||||
warn!(backend, force, error = %err, "allocator reclaim failed");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn init_allocator_reclaim(ctx: CancellationToken) {
|
||||
let backend = allocator_backend();
|
||||
let enabled = rustfs_utils::get_env_bool(
|
||||
rustfs_config::ENV_ALLOCATOR_RECLAIM_ENABLED,
|
||||
rustfs_config::DEFAULT_ALLOCATOR_RECLAIM_ENABLED,
|
||||
);
|
||||
gauge!("rustfs_memory_allocator_reclaim_enabled").set(if enabled { 1.0 } else { 0.0 });
|
||||
counter!("rustfs_memory_allocator_backend_info", "backend" => backend.to_string()).increment(1);
|
||||
|
||||
if !enabled {
|
||||
debug!("allocator reclaim loop disabled");
|
||||
return;
|
||||
}
|
||||
|
||||
let configured_force =
|
||||
rustfs_utils::get_env_bool(rustfs_config::ENV_ALLOCATOR_RECLAIM_FORCE, rustfs_config::DEFAULT_ALLOCATOR_RECLAIM_FORCE);
|
||||
let force = if backend == "jemalloc" && configured_force {
|
||||
warn!(
|
||||
backend,
|
||||
env = rustfs_config::ENV_ALLOCATOR_RECLAIM_FORCE,
|
||||
"allocator reclaim force mode is not supported on jemalloc backend; ignoring configured force flag"
|
||||
);
|
||||
false
|
||||
} else {
|
||||
configured_force
|
||||
};
|
||||
let idle_intervals = rustfs_utils::get_env_u64(
|
||||
rustfs_config::ENV_ALLOCATOR_RECLAIM_IDLE_INTERVALS,
|
||||
rustfs_config::DEFAULT_ALLOCATOR_RECLAIM_IDLE_INTERVALS,
|
||||
)
|
||||
.max(1);
|
||||
let interval = Duration::from_secs(
|
||||
rustfs_utils::get_env_u64(
|
||||
rustfs_config::ENV_ALLOCATOR_RECLAIM_INTERVAL_SECS,
|
||||
rustfs_config::DEFAULT_ALLOCATOR_RECLAIM_INTERVAL_SECS,
|
||||
)
|
||||
.max(1),
|
||||
);
|
||||
|
||||
tokio::spawn(async move {
|
||||
let mut ticker = tokio::time::interval(interval);
|
||||
ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
|
||||
let mut idle_streak = 0_u64;
|
||||
|
||||
loop {
|
||||
tokio::select! {
|
||||
_ = ctx.cancelled() => {
|
||||
debug!("allocator reclaim loop cancelled");
|
||||
break;
|
||||
}
|
||||
_ = ticker.tick() => {
|
||||
let snapshot = reclaimable_work_snapshot();
|
||||
let active_signal_count = snapshot.active_signal_count();
|
||||
gauge!("rustfs_memory_allocator_reclaim_active_requests").set(snapshot.active_requests as f64);
|
||||
gauge!("rustfs_memory_allocator_reclaim_delete_tail_activity_current").set(snapshot.delete_tail_activity as f64);
|
||||
gauge!("rustfs_memory_allocator_reclaim_scanner_activity_current").set(snapshot.scanner_activity as f64);
|
||||
gauge!("rustfs_memory_allocator_reclaim_heal_activity_current").set(snapshot.heal_activity as f64);
|
||||
gauge!("rustfs_memory_allocator_reclaim_ec_inflight_bytes_current").set(snapshot.ec_inflight_bytes as f64);
|
||||
gauge!("rustfs_memory_allocator_reclaim_get_buffered_bytes_current").set(snapshot.get_buffered_bytes as f64);
|
||||
gauge!("rustfs_memory_allocator_reclaim_reclaimable_work_current").set(active_signal_count as f64);
|
||||
if active_signal_count == 0 {
|
||||
idle_streak = idle_streak.saturating_add(1);
|
||||
gauge!("rustfs_memory_allocator_reclaim_idle_streak").set(idle_streak as f64);
|
||||
} else {
|
||||
idle_streak = 0;
|
||||
gauge!("rustfs_memory_allocator_reclaim_idle_streak").set(0.0);
|
||||
}
|
||||
|
||||
if idle_streak >= idle_intervals {
|
||||
run_allocator_reclaim(force);
|
||||
idle_streak = 0;
|
||||
gauge!("rustfs_memory_allocator_reclaim_idle_streak").set(0.0);
|
||||
} else {
|
||||
let reason = if active_signal_count > 0 {
|
||||
"work_inflight"
|
||||
} else {
|
||||
"idle_window"
|
||||
};
|
||||
counter!("rustfs_memory_allocator_reclaim_skipped_total", "reason" => reason.to_string()).increment(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{allocator_backend, reclaimable_work_snapshot};
|
||||
|
||||
#[test]
|
||||
fn allocator_backend_name_is_available() {
|
||||
assert!(!allocator_backend().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reclaimable_work_snapshot_is_collectable() {
|
||||
let _ = reclaimable_work_snapshot();
|
||||
}
|
||||
}
|
||||
@@ -16,6 +16,7 @@
|
||||
|
||||
use crate::app::context::{AppContext, default_notify_interface, get_global_app_context};
|
||||
use crate::config::RustFSBufferConfig;
|
||||
use crate::delete_tail_activity::{DeleteTailActivityGuard, DeleteTailStage};
|
||||
use crate::error::ApiError;
|
||||
use crate::storage::access::{PostObjectRequestMarker, authorize_request, has_bypass_governance_header, req_info_mut};
|
||||
use crate::storage::concurrency::{
|
||||
@@ -36,7 +37,7 @@ use bytes::Bytes;
|
||||
use datafusion::arrow::{
|
||||
csv::WriterBuilder as CsvWriterBuilder, json::WriterBuilder as JsonWriterBuilder, json::writer::JsonArray,
|
||||
};
|
||||
use futures::{StreamExt, stream};
|
||||
use futures::StreamExt;
|
||||
use http::{HeaderMap, HeaderValue, StatusCode};
|
||||
use md5::Context as Md5Context;
|
||||
use metrics::{counter, histogram};
|
||||
@@ -118,7 +119,7 @@ use std::collections::HashMap;
|
||||
use std::ops::Add;
|
||||
use std::path::Path;
|
||||
use std::str::FromStr;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::sync::{Arc, Mutex, OnceLock};
|
||||
use std::time::Duration;
|
||||
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
|
||||
use tokio::io::{AsyncRead, ReadBuf};
|
||||
@@ -213,6 +214,7 @@ async fn enqueue_transitioned_delete_cleanup(bucket: &str, object: &str, opts: &
|
||||
let Some(existing) = existing else {
|
||||
return;
|
||||
};
|
||||
let _activity_guard = DeleteTailActivityGuard::new(DeleteTailStage::Cleanup);
|
||||
|
||||
let je = if opts.delete_prefix {
|
||||
rustfs_ecstore::bucket::lifecycle::tier_sweeper::transitioned_force_delete_journal_entry(&existing.transitioned_object)
|
||||
@@ -255,6 +257,38 @@ pin_project! {
|
||||
}
|
||||
}
|
||||
|
||||
pin_project! {
|
||||
struct MemoryTrackedBytesStream {
|
||||
bytes: Bytes,
|
||||
emitted: bool,
|
||||
_guard: Option<rustfs_io_metrics::MemoryGaugeGuard>,
|
||||
}
|
||||
}
|
||||
|
||||
impl MemoryTrackedBytesStream {
|
||||
fn new(bytes: Bytes, guard: Option<rustfs_io_metrics::MemoryGaugeGuard>) -> Self {
|
||||
Self {
|
||||
bytes,
|
||||
emitted: false,
|
||||
_guard: guard,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl futures::Stream for MemoryTrackedBytesStream {
|
||||
type Item = std::io::Result<Bytes>;
|
||||
|
||||
fn poll_next(self: std::pin::Pin<&mut Self>, _cx: &mut std::task::Context<'_>) -> std::task::Poll<Option<Self::Item>> {
|
||||
let this = self.project();
|
||||
if *this.emitted {
|
||||
return std::task::Poll::Ready(None);
|
||||
}
|
||||
|
||||
*this.emitted = true;
|
||||
std::task::Poll::Ready(Some(Ok(this.bytes.clone())))
|
||||
}
|
||||
}
|
||||
|
||||
impl<R> ExtractArchiveEtagReader<R> {
|
||||
fn new(inner: R, etag: Arc<Mutex<Option<String>>>) -> Self {
|
||||
Self {
|
||||
@@ -348,6 +382,16 @@ fn should_use_zero_copy(size: i64, headers: &HeaderMap) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
fn object_seek_support_threshold() -> usize {
|
||||
static OBJECT_SEEK_SUPPORT_THRESHOLD: OnceLock<usize> = OnceLock::new();
|
||||
*OBJECT_SEEK_SUPPORT_THRESHOLD.get_or_init(|| {
|
||||
rustfs_utils::get_env_usize(
|
||||
rustfs_config::ENV_OBJECT_SEEK_SUPPORT_THRESHOLD,
|
||||
rustfs_config::DEFAULT_OBJECT_SEEK_SUPPORT_THRESHOLD,
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod deadlock_request_guard_tests {
|
||||
use super::DeadlockRequestGuard;
|
||||
@@ -994,8 +1038,10 @@ impl DefaultObjectUsecase {
|
||||
}
|
||||
|
||||
fn build_memory_blob(buf: Vec<u8>, response_content_length: i64, _optimal_buffer_size: usize) -> Option<StreamingBlob> {
|
||||
let guard = rustfs_io_metrics::track_get_object_buffered_bytes(buf.len());
|
||||
let bytes = Bytes::from(buf);
|
||||
Some(StreamingBlob::wrap(bytes_stream(
|
||||
stream::once(async move { Ok::<Bytes, std::io::Error>(Bytes::from(buf)) }),
|
||||
MemoryTrackedBytesStream::new(bytes, guard),
|
||||
response_content_length as usize,
|
||||
)))
|
||||
}
|
||||
@@ -1211,11 +1257,9 @@ impl DefaultObjectUsecase {
|
||||
|
||||
let info = reader.object_info;
|
||||
|
||||
use rustfs_io_metrics::{record_memory_copy_saved, record_zero_copy_read};
|
||||
use rustfs_io_metrics::record_zero_copy_read;
|
||||
let read_duration = read_start.elapsed();
|
||||
let estimated_saved = (info.size * 2) as usize;
|
||||
record_zero_copy_read(info.size as usize, read_duration.as_secs_f64() * 1000.0);
|
||||
record_memory_copy_saved(estimated_saved);
|
||||
|
||||
manager.record_disk_operation(info.size as u64, read_duration, true).await;
|
||||
|
||||
@@ -1483,7 +1527,7 @@ impl DefaultObjectUsecase {
|
||||
R: AsyncRead + Send + Sync + Unpin + 'static,
|
||||
{
|
||||
if encryption_applied {
|
||||
let seekable_object_size_threshold = rustfs_config::DEFAULT_OBJECT_SEEK_SUPPORT_THRESHOLD;
|
||||
let seekable_object_size_threshold = object_seek_support_threshold();
|
||||
let should_buffer_encrypted_object = response_content_length > 0
|
||||
&& response_content_length <= seekable_object_size_threshold as i64
|
||||
&& part_number.is_none()
|
||||
@@ -1514,7 +1558,7 @@ impl DefaultObjectUsecase {
|
||||
return Ok(Self::build_reader_blob(final_stream, response_content_length, optimal_buffer_size));
|
||||
}
|
||||
|
||||
let seekable_object_size_threshold = rustfs_config::DEFAULT_OBJECT_SEEK_SUPPORT_THRESHOLD;
|
||||
let seekable_object_size_threshold = object_seek_support_threshold();
|
||||
let should_provide_seek_support = response_content_length > 0
|
||||
&& response_content_length <= seekable_object_size_threshold as i64
|
||||
&& part_number.is_none()
|
||||
@@ -2114,9 +2158,9 @@ impl DefaultObjectUsecase {
|
||||
|
||||
let request_id = req
|
||||
.extensions
|
||||
.get::<crate::storage::request_context::RequestContext>()
|
||||
.get::<request_context::RequestContext>()
|
||||
.map(|ctx| ctx.request_id.clone())
|
||||
.unwrap_or_else(|| crate::storage::request_context::RequestContext::fallback().request_id);
|
||||
.unwrap_or_else(|| request_context::RequestContext::fallback().request_id);
|
||||
let bootstrap = Self::init_get_object_bootstrap(&req.input.bucket, &req.input.key, &request_id)?;
|
||||
let timeout_config = bootstrap.timeout_config;
|
||||
let wrapper = bootstrap.wrapper;
|
||||
@@ -3083,6 +3127,7 @@ impl DefaultObjectUsecase {
|
||||
&& (dobj.delete_marker_replication_status() == ReplicationStatusType::Pending
|
||||
|| dobj.version_purge_status() == VersionPurgeStatusType::Pending)
|
||||
{
|
||||
let _activity_guard = DeleteTailActivityGuard::new(DeleteTailStage::Replication);
|
||||
let mut dobj = dobj.clone();
|
||||
if is_dir_object(dobj.object_name.as_str()) && dobj.version_id.is_none() {
|
||||
dobj.version_id = Some(Uuid::nil());
|
||||
@@ -3104,11 +3149,9 @@ impl DefaultObjectUsecase {
|
||||
.as_ref()
|
||||
.map(|context| context.notify())
|
||||
.unwrap_or_else(default_notify_interface);
|
||||
let request_context = req
|
||||
.extensions
|
||||
.get::<crate::storage::request_context::RequestContext>()
|
||||
.cloned();
|
||||
let request_context = req.extensions.get::<request_context::RequestContext>().cloned();
|
||||
spawn_background_with_context(request_context, async move {
|
||||
let _activity_guard = DeleteTailActivityGuard::new(DeleteTailStage::Notify);
|
||||
for res in delete_results {
|
||||
if let Some(dobj) = res.delete_object {
|
||||
let event_name = if dobj.delete_marker {
|
||||
@@ -3298,6 +3341,7 @@ impl DefaultObjectUsecase {
|
||||
let deleted_replication_info = existing_object_info
|
||||
.as_ref()
|
||||
.filter(|_| should_use_existing_delete_replication_info(&opts));
|
||||
let _delete_tail_guard = DeleteTailActivityGuard::new(DeleteTailStage::Tail);
|
||||
let deleted_object_source = deleted_replication_info.unwrap_or(&obj_info);
|
||||
let replication_state_source =
|
||||
delete_replication_state_source(&opts, existing_object_info.as_ref(), deleted_object_source);
|
||||
@@ -3311,6 +3355,7 @@ impl DefaultObjectUsecase {
|
||||
};
|
||||
|
||||
if schedule_delete_replication {
|
||||
let _activity_guard = DeleteTailActivityGuard::new(DeleteTailStage::Replication);
|
||||
let mut deleted_object = DeletedObjectReplicationInfo {
|
||||
delete_object: rustfs_ecstore::store_api::DeletedObject {
|
||||
delete_marker: deleted_object_source.delete_marker && !deleted_delete_marker_version,
|
||||
@@ -4321,10 +4366,7 @@ impl DefaultObjectUsecase {
|
||||
};
|
||||
|
||||
let notify = notify.clone();
|
||||
let request_context = req
|
||||
.extensions
|
||||
.get::<crate::storage::request_context::RequestContext>()
|
||||
.cloned();
|
||||
let request_context = req.extensions.get::<request_context::RequestContext>().cloned();
|
||||
spawn_background_with_context(request_context, async move {
|
||||
notify.notify(event_args).await;
|
||||
});
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use metrics::{counter, gauge, histogram};
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
|
||||
static DELETE_TAIL_TOTAL: AtomicU64 = AtomicU64::new(0);
|
||||
static DELETE_CLEANUP_TOTAL: AtomicU64 = AtomicU64::new(0);
|
||||
static DELETE_REPLICATION_TOTAL: AtomicU64 = AtomicU64::new(0);
|
||||
static DELETE_NOTIFY_TOTAL: AtomicU64 = AtomicU64::new(0);
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub enum DeleteTailStage {
|
||||
Tail,
|
||||
Cleanup,
|
||||
Replication,
|
||||
Notify,
|
||||
}
|
||||
|
||||
impl DeleteTailStage {
|
||||
const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Tail => "tail",
|
||||
Self::Cleanup => "cleanup",
|
||||
Self::Replication => "replication",
|
||||
Self::Notify => "notify",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn stage_counter(stage: DeleteTailStage) -> &'static AtomicU64 {
|
||||
match stage {
|
||||
DeleteTailStage::Tail => &DELETE_TAIL_TOTAL,
|
||||
DeleteTailStage::Cleanup => &DELETE_CLEANUP_TOTAL,
|
||||
DeleteTailStage::Replication => &DELETE_REPLICATION_TOTAL,
|
||||
DeleteTailStage::Notify => &DELETE_NOTIFY_TOTAL,
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct DeleteTailActivityGuard {
|
||||
stage: DeleteTailStage,
|
||||
started_at: std::time::Instant,
|
||||
}
|
||||
|
||||
impl DeleteTailActivityGuard {
|
||||
pub fn new(stage: DeleteTailStage) -> Self {
|
||||
let total = stage_counter(stage).fetch_add(1, Ordering::Relaxed) + 1;
|
||||
gauge!(
|
||||
"rustfs_delete_tail_activity_inflight_current",
|
||||
"stage" => stage.as_str().to_string()
|
||||
)
|
||||
.set(total as f64);
|
||||
gauge!("rustfs_delete_tail_activity_total_inflight_current").set(current_delete_tail_activity() as f64);
|
||||
counter!(
|
||||
"rustfs_delete_tail_activity_started_total",
|
||||
"stage" => stage.as_str().to_string()
|
||||
)
|
||||
.increment(1);
|
||||
Self {
|
||||
stage,
|
||||
started_at: std::time::Instant::now(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for DeleteTailActivityGuard {
|
||||
fn drop(&mut self) {
|
||||
let previous = stage_counter(self.stage).fetch_sub(1, Ordering::Relaxed);
|
||||
let next = previous.saturating_sub(1);
|
||||
gauge!(
|
||||
"rustfs_delete_tail_activity_inflight_current",
|
||||
"stage" => self.stage.as_str().to_string()
|
||||
)
|
||||
.set(next as f64);
|
||||
gauge!("rustfs_delete_tail_activity_total_inflight_current").set(current_delete_tail_activity() as f64);
|
||||
histogram!(
|
||||
"rustfs_delete_tail_activity_duration_seconds",
|
||||
"stage" => self.stage.as_str().to_string()
|
||||
)
|
||||
.record(self.started_at.elapsed().as_secs_f64());
|
||||
}
|
||||
}
|
||||
|
||||
pub fn current_delete_tail_activity() -> u64 {
|
||||
DELETE_TAIL_TOTAL.load(Ordering::Relaxed)
|
||||
+ DELETE_CLEANUP_TOTAL.load(Ordering::Relaxed)
|
||||
+ DELETE_REPLICATION_TOTAL.load(Ordering::Relaxed)
|
||||
+ DELETE_NOTIFY_TOTAL.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{DeleteTailActivityGuard, DeleteTailStage, current_delete_tail_activity};
|
||||
|
||||
#[test]
|
||||
fn delete_tail_activity_guard_tracks_total_activity() {
|
||||
let before = current_delete_tail_activity();
|
||||
let guard = DeleteTailActivityGuard::new(DeleteTailStage::Cleanup);
|
||||
assert_eq!(current_delete_tail_activity(), before + 1);
|
||||
drop(guard);
|
||||
assert_eq!(current_delete_tail_activity(), before);
|
||||
}
|
||||
}
|
||||
@@ -51,15 +51,18 @@
|
||||
//! tests, and then shut it down.
|
||||
|
||||
pub mod admin;
|
||||
pub mod allocator_reclaim;
|
||||
pub mod app;
|
||||
pub mod auth;
|
||||
pub mod auth_keystone;
|
||||
pub mod capacity;
|
||||
pub mod config;
|
||||
pub mod delete_tail_activity;
|
||||
pub mod embedded;
|
||||
pub mod error;
|
||||
pub mod init;
|
||||
pub mod license;
|
||||
pub mod memory_observability;
|
||||
pub mod profiling;
|
||||
#[cfg(any(feature = "ftps", feature = "webdav"))]
|
||||
pub mod protocols;
|
||||
|
||||
@@ -560,10 +560,12 @@ async fn run(config: rustfs::config::Config) -> Result<()> {
|
||||
print_server_info();
|
||||
|
||||
init_update_check();
|
||||
rustfs::allocator_reclaim::init_allocator_reclaim(ctx.clone());
|
||||
|
||||
if rustfs_obs::observability_metric_enabled() {
|
||||
// Initialize metrics system
|
||||
init_metrics_runtime(ctx.clone());
|
||||
rustfs::memory_observability::init_memory_observability(ctx.clone());
|
||||
|
||||
// Initialize auto-tuner for performance optimization (optional)
|
||||
rustfs::init::init_auto_tuner(ctx.clone()).await;
|
||||
|
||||
@@ -0,0 +1,209 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use rustfs_io_metrics::{
|
||||
record_cgroup_memory_split, record_cpu_usage, record_memory_usage, record_process_memory_split,
|
||||
snapshot_process_resource_and_system,
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
use std::path::Path;
|
||||
use std::sync::{Mutex, OnceLock};
|
||||
use std::time::Duration;
|
||||
use sysinfo::System;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::debug;
|
||||
|
||||
static MEMORY_SYSTEM: OnceLock<Mutex<System>> = OnceLock::new();
|
||||
|
||||
const ENV_MEMORY_OBSERVABILITY_INTERVAL_SECS: &str = "RUSTFS_MEMORY_OBSERVABILITY_INTERVAL_SECS";
|
||||
const DEFAULT_MEMORY_OBSERVABILITY_INTERVAL_SECS: u64 = 15;
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
struct CgroupMemorySnapshot {
|
||||
current_bytes: Option<u64>,
|
||||
limit_bytes: Option<u64>,
|
||||
anon_bytes: Option<u64>,
|
||||
file_bytes: Option<u64>,
|
||||
active_file_bytes: Option<u64>,
|
||||
inactive_file_bytes: Option<u64>,
|
||||
}
|
||||
|
||||
fn memory_system() -> &'static Mutex<System> {
|
||||
MEMORY_SYSTEM.get_or_init(|| Mutex::new(System::new()))
|
||||
}
|
||||
|
||||
fn refresh_total_memory() -> u64 {
|
||||
let mut system = memory_system().lock().unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
system.refresh_memory();
|
||||
system.total_memory()
|
||||
}
|
||||
|
||||
fn read_optional_u64(path: &Path) -> Option<u64> {
|
||||
let content = std::fs::read_to_string(path).ok()?;
|
||||
let trimmed = content.trim();
|
||||
if trimmed.is_empty() || trimmed == "max" {
|
||||
return None;
|
||||
}
|
||||
trimmed.parse::<u64>().ok()
|
||||
}
|
||||
|
||||
fn parse_kv_stats(content: &str) -> HashMap<String, u64> {
|
||||
content
|
||||
.lines()
|
||||
.filter_map(|line| {
|
||||
let mut parts = line.split_whitespace();
|
||||
let key = parts.next()?;
|
||||
let value = parts.next()?.parse::<u64>().ok()?;
|
||||
Some((key.to_string(), value))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn read_cgroup_v2() -> Option<CgroupMemorySnapshot> {
|
||||
let root = Path::new("/sys/fs/cgroup");
|
||||
let stat_path = root.join("memory.stat");
|
||||
if !stat_path.exists() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let stats = parse_kv_stats(&std::fs::read_to_string(&stat_path).ok()?);
|
||||
Some(CgroupMemorySnapshot {
|
||||
current_bytes: read_optional_u64(&root.join("memory.current")),
|
||||
limit_bytes: read_optional_u64(&root.join("memory.max")),
|
||||
anon_bytes: stats.get("anon").copied(),
|
||||
file_bytes: stats.get("file").copied(),
|
||||
active_file_bytes: stats.get("active_file").copied(),
|
||||
inactive_file_bytes: stats.get("inactive_file").copied(),
|
||||
})
|
||||
}
|
||||
|
||||
fn read_cgroup_v1() -> Option<CgroupMemorySnapshot> {
|
||||
let root = Path::new("/sys/fs/cgroup/memory");
|
||||
let stat_path = root.join("memory.stat");
|
||||
if !stat_path.exists() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let stats = parse_kv_stats(&std::fs::read_to_string(&stat_path).ok()?);
|
||||
Some(CgroupMemorySnapshot {
|
||||
current_bytes: read_optional_u64(&root.join("memory.usage_in_bytes")),
|
||||
limit_bytes: read_optional_u64(&root.join("memory.limit_in_bytes")),
|
||||
anon_bytes: stats.get("total_rss").copied().or_else(|| stats.get("rss").copied()),
|
||||
file_bytes: stats.get("total_cache").copied().or_else(|| stats.get("cache").copied()),
|
||||
active_file_bytes: stats
|
||||
.get("total_active_file")
|
||||
.copied()
|
||||
.or_else(|| stats.get("active_file").copied()),
|
||||
inactive_file_bytes: stats
|
||||
.get("total_inactive_file")
|
||||
.copied()
|
||||
.or_else(|| stats.get("inactive_file").copied()),
|
||||
})
|
||||
}
|
||||
|
||||
fn read_cgroup_memory_snapshot() -> Option<CgroupMemorySnapshot> {
|
||||
read_cgroup_v2().or_else(read_cgroup_v1)
|
||||
}
|
||||
|
||||
async fn record_memory_snapshot() {
|
||||
match tokio::task::spawn_blocking(|| {
|
||||
let (resource, process) = snapshot_process_resource_and_system();
|
||||
let total_memory = refresh_total_memory();
|
||||
let cgroup = read_cgroup_memory_snapshot();
|
||||
(resource, process, total_memory, cgroup)
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok((resource, process, total_memory, cgroup)) => {
|
||||
record_memory_usage(process.resident_memory_bytes, total_memory);
|
||||
record_cpu_usage(resource.cpu_percent);
|
||||
record_process_memory_split(process.resident_memory_bytes, process.virtual_memory_bytes);
|
||||
|
||||
if let Some(cgroup) = cgroup {
|
||||
record_cgroup_memory_split(
|
||||
cgroup.current_bytes,
|
||||
cgroup.limit_bytes,
|
||||
cgroup.anon_bytes,
|
||||
cgroup.file_bytes,
|
||||
cgroup.active_file_bytes,
|
||||
cgroup.inactive_file_bytes,
|
||||
);
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
debug!(error = ?err, "memory observability sampler task failed");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn init_memory_observability(ctx: CancellationToken) {
|
||||
let interval_secs =
|
||||
rustfs_utils::get_env_u64(ENV_MEMORY_OBSERVABILITY_INTERVAL_SECS, DEFAULT_MEMORY_OBSERVABILITY_INTERVAL_SECS);
|
||||
let interval = Duration::from_secs(interval_secs.max(1));
|
||||
|
||||
tokio::spawn(async move {
|
||||
let mut ticker = tokio::time::interval(interval);
|
||||
ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
|
||||
|
||||
loop {
|
||||
tokio::select! {
|
||||
_ = ctx.cancelled() => {
|
||||
debug!("memory observability sampler cancelled");
|
||||
break;
|
||||
}
|
||||
_ = ticker.tick() => {
|
||||
record_memory_snapshot().await;
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{CgroupMemorySnapshot, parse_kv_stats, read_optional_u64};
|
||||
use std::fs;
|
||||
use std::path::PathBuf;
|
||||
|
||||
#[test]
|
||||
fn parse_kv_stats_extracts_numeric_pairs() {
|
||||
let parsed = parse_kv_stats("anon 12\nfile 34\nactive_file 56\n");
|
||||
assert_eq!(parsed.get("anon").copied(), Some(12));
|
||||
assert_eq!(parsed.get("file").copied(), Some(34));
|
||||
assert_eq!(parsed.get("active_file").copied(), Some(56));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn read_optional_u64_parses_numeric_and_max_values() {
|
||||
let tempdir = tempfile::tempdir().expect("tempdir");
|
||||
let value_path: PathBuf = tempdir.path().join("value");
|
||||
let max_path: PathBuf = tempdir.path().join("max");
|
||||
fs::write(&value_path, "123\n").expect("write numeric");
|
||||
fs::write(&max_path, "max\n").expect("write max");
|
||||
|
||||
assert_eq!(read_optional_u64(&value_path), Some(123));
|
||||
assert_eq!(read_optional_u64(&max_path), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cgroup_memory_snapshot_defaults_are_empty() {
|
||||
let snapshot = CgroupMemorySnapshot::default();
|
||||
assert_eq!(snapshot.current_bytes, None);
|
||||
assert_eq!(snapshot.limit_bytes, None);
|
||||
assert_eq!(snapshot.anon_bytes, None);
|
||||
assert_eq!(snapshot.file_bytes, None);
|
||||
assert_eq!(snapshot.active_file_bytes, None);
|
||||
assert_eq!(snapshot.inactive_file_bytes, None);
|
||||
}
|
||||
}
|
||||
@@ -122,6 +122,10 @@ fn record_active_http_requests(delta: i64) {
|
||||
gauge!(METRIC_HTTP_SERVER_ACTIVE_REQUESTS).set(next as f64);
|
||||
}
|
||||
|
||||
pub(crate) fn active_http_requests() -> u64 {
|
||||
ACTIVE_HTTP_REQUESTS.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
pub async fn start_http_server(
|
||||
config: &config::Config,
|
||||
readiness: Arc<GlobalReadiness>,
|
||||
|
||||
@@ -39,6 +39,7 @@ pub use service_state::ShutdownSignal;
|
||||
pub use service_state::wait_for_shutdown;
|
||||
|
||||
// Items only used within the library crate (admin handlers, server/http.rs, etc.).
|
||||
pub(crate) use http::active_http_requests;
|
||||
pub(crate) use module_switch::{
|
||||
ModuleSwitchSnapshot, ModuleSwitchSource, PersistedModuleSwitches, current_module_switch_snapshot,
|
||||
refresh_persisted_module_switches_from_store, save_persisted_module_switches_to_store, validate_module_switch_update,
|
||||
|
||||
+10
-3
@@ -52,6 +52,10 @@ if [ -z "${RUSTFS_UNSAFE_BYPASS_DISK_CHECK+x}" ] && [ -z "${MINIO_CI+x}" ]; then
|
||||
export RUSTFS_UNSAFE_BYPASS_DISK_CHECK=true
|
||||
fi
|
||||
|
||||
if [ -z "${RUSTFS_ALLOCATOR_RECLAIM_ENABLED+x}" ]; then
|
||||
export RUSTFS_ALLOCATOR_RECLAIM_ENABLED=true
|
||||
fi
|
||||
|
||||
export RUSTFS_VOLUMES="./target/volume/test{1...4}"
|
||||
# export RUSTFS_VOLUMES="./target/volume/test"
|
||||
export RUSTFS_ADDRESS=":9000"
|
||||
@@ -71,14 +75,14 @@ export RUSTFS_OBS_ENDPOINT=http://localhost:4318 # OpenTelemetry Collector addre
|
||||
#export RUSTFS_OBS_LOG_ENDPOINT=http://loki:3100/otlp/v1/logs # OpenTelemetry Collector logs address http://loki:3100/otlp/v1/logs
|
||||
#export OTEL_EXPORTER_OTLP_LOGS_ENDPOINT=http://loki:3100/otlp/v1/logs
|
||||
export RUSTFS_OBS_PROFILING_ENDPOINT=http://localhost:4040 # OpenTelemetry Collector profiling address
|
||||
export RUSTFS_OBS_USE_STDOUT=true # Whether to use standard output
|
||||
export RUSTFS_OBS_USE_STDOUT=false # Whether to use standard output
|
||||
export RUSTFS_OBS_SAMPLE_RATIO=2.0 # Sample ratio, between 0.0-1.0, 0.0 means no sampling, 1.0 means full sampling
|
||||
export RUSTFS_OBS_METER_INTERVAL=1 # Sampling interval in seconds
|
||||
export RUSTFS_OBS_SERVICE_NAME=rustfs # Service name
|
||||
export RUSTFS_OBS_SERVICE_VERSION=0.1.0 # Service version
|
||||
export RUSTFS_OBS_ENVIRONMENT=production # Environment name development, staging, production
|
||||
export RUSTFS_OBS_LOGGER_LEVEL=info # Log level, supports trace, debug, info, warn, error
|
||||
export RUSTFS_OBS_LOG_STDOUT_ENABLED=true # Whether to enable local stdout logging
|
||||
export RUSTFS_OBS_LOG_STDOUT_ENABLED=false # Whether to enable local stdout logging
|
||||
export RUSTFS_OBS_LOG_DIRECTORY="$current_dir/deploy/logs" # Log directory
|
||||
export RUSTFS_OBS_LOG_ROTATION_TIME="minutely" # Log rotation time unit, can be "minutely", "hourly", "daily"
|
||||
export RUSTFS_OBS_LOG_KEEP_FILES=10 # Number of log files to keep
|
||||
@@ -527,4 +531,7 @@ fi
|
||||
#cargo run --profile release --bin rustfs
|
||||
|
||||
# To run in debug mode, use the following line
|
||||
cargo run --bin rustfs
|
||||
#cargo run --bin rustfs
|
||||
|
||||
# Default local run mode: release
|
||||
cargo run --profile release --bin rustfs
|
||||
|
||||
Executable
+257
@@ -0,0 +1,257 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# Issue 2573 acceptance runner
|
||||
# Runs the key workload profiles discussed in docs/tasks/issue-2573/05-benchmark-and-acceptance.md
|
||||
# and samples the RustFS process RSS during load and cooldown.
|
||||
|
||||
WARP_BIN="${WARP_BIN:-warp}"
|
||||
HOST="${HOST:-http://127.0.0.1:9000}"
|
||||
ACCESS_KEY="${ACCESS_KEY:-rustfsadmin}"
|
||||
SECRET_KEY="${SECRET_KEY:-rustfsadmin}"
|
||||
BUCKET="${BUCKET:-rustfs-issue-2573}"
|
||||
REGION="${REGION:-us-east-1}"
|
||||
CONCURRENCY="${CONCURRENCY:-30}"
|
||||
DURATION="${DURATION:-60s}"
|
||||
COOLDOWN_SECS="${COOLDOWN_SECS:-180}"
|
||||
SAMPLE_SECS="${SAMPLE_SECS:-1}"
|
||||
RUSTFS_PID="${RUSTFS_PID:-}"
|
||||
OUT_DIR="${OUT_DIR:-target/bench/issue-2573-acceptance-$(date +%Y%m%d-%H%M%S)}"
|
||||
INSECURE="${INSECURE:-false}"
|
||||
|
||||
usage() {
|
||||
cat <<'USAGE'
|
||||
Usage:
|
||||
scripts/run_issue_2573_acceptance.sh [options]
|
||||
|
||||
Options:
|
||||
--warp-bin <path> warp binary (default: warp)
|
||||
--host <url> S3 endpoint; accepts either URL or host:port (default: http://127.0.0.1:9000)
|
||||
--access-key <ak> access key (default: rustfsadmin)
|
||||
--secret-key <sk> secret key (default: rustfsadmin)
|
||||
--bucket <name> bucket name (default: rustfs-issue-2573)
|
||||
--region <name> region (default: us-east-1)
|
||||
--concurrency <n> warp concurrency (default: 30)
|
||||
--duration <dur> warp duration per profile (default: 60s)
|
||||
--cooldown-secs <n> cooldown sampling after each profile (default: 180)
|
||||
--sample-secs <n> RSS sample interval seconds (default: 1)
|
||||
--pid <pid> rustfs process pid (optional; auto-detect if omitted)
|
||||
--out-dir <dir> output directory
|
||||
--insecure pass --insecure to warp
|
||||
-h, --help show help
|
||||
|
||||
Profiles executed:
|
||||
1. 4KiB mixed
|
||||
2. 11MiB mixed
|
||||
3. 11MiB delete
|
||||
USAGE
|
||||
}
|
||||
|
||||
require_cmd() {
|
||||
if ! command -v "$1" >/dev/null 2>&1; then
|
||||
echo "ERROR: command not found: $1" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
parse_args() {
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--warp-bin) WARP_BIN="$2"; shift 2 ;;
|
||||
--host) HOST="$2"; shift 2 ;;
|
||||
--access-key) ACCESS_KEY="$2"; shift 2 ;;
|
||||
--secret-key) SECRET_KEY="$2"; shift 2 ;;
|
||||
--bucket) BUCKET="$2"; shift 2 ;;
|
||||
--region) REGION="$2"; shift 2 ;;
|
||||
--concurrency) CONCURRENCY="$2"; shift 2 ;;
|
||||
--duration) DURATION="$2"; shift 2 ;;
|
||||
--cooldown-secs) COOLDOWN_SECS="$2"; shift 2 ;;
|
||||
--sample-secs) SAMPLE_SECS="$2"; shift 2 ;;
|
||||
--pid) RUSTFS_PID="$2"; shift 2 ;;
|
||||
--out-dir) OUT_DIR="$2"; shift 2 ;;
|
||||
--insecure) INSECURE=true; shift ;;
|
||||
-h|--help) usage; exit 0 ;;
|
||||
*)
|
||||
echo "ERROR: unknown arg: $1" >&2
|
||||
usage
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
}
|
||||
|
||||
resolve_pid() {
|
||||
if [[ -n "$RUSTFS_PID" ]]; then
|
||||
echo "$RUSTFS_PID"
|
||||
return
|
||||
fi
|
||||
|
||||
local pid
|
||||
pid="$(pgrep -n rustfs || true)"
|
||||
if [[ -z "$pid" ]]; then
|
||||
echo ""
|
||||
return
|
||||
fi
|
||||
echo "$pid"
|
||||
}
|
||||
|
||||
normalize_warp_host() {
|
||||
local raw="$1"
|
||||
# Strip scheme when a URL is provided.
|
||||
raw="${raw#http://}"
|
||||
raw="${raw#https://}"
|
||||
# Remove any path/query/fragment to satisfy warp's --host requirements.
|
||||
raw="${raw%%/*}"
|
||||
raw="${raw%%\?*}"
|
||||
raw="${raw%%\#*}"
|
||||
echo "$raw"
|
||||
}
|
||||
|
||||
sample_rss_loop() {
|
||||
local pid="$1"
|
||||
local out_file="$2"
|
||||
if [[ -z "$pid" ]]; then
|
||||
return 0
|
||||
fi
|
||||
local started_at
|
||||
started_at="$(date +%s)"
|
||||
|
||||
echo "timestamp,elapsed_seconds,rss_kib,vsz_kib" > "$out_file"
|
||||
while kill -0 "$pid" >/dev/null 2>&1; do
|
||||
local now elapsed sample
|
||||
now="$(date +%s)"
|
||||
elapsed="$((now - started_at))"
|
||||
sample="$(ps -o rss=,vsz= -p "$pid" | awk 'NF>=2 {print $1","$2}')"
|
||||
if [[ -n "$sample" ]]; then
|
||||
echo "$(date +%Y-%m-%dT%H:%M:%S),${elapsed},${sample}" >> "$out_file"
|
||||
fi
|
||||
sleep "$SAMPLE_SECS"
|
||||
done
|
||||
}
|
||||
|
||||
sample_rss_window() {
|
||||
local pid="$1"
|
||||
local seconds="$2"
|
||||
local out_file="$3"
|
||||
if [[ -z "$pid" ]]; then
|
||||
return 0
|
||||
fi
|
||||
local started_at deadline
|
||||
started_at="$(date +%s)"
|
||||
deadline="$((started_at + seconds))"
|
||||
|
||||
echo "timestamp,elapsed_seconds,rss_kib,vsz_kib" > "$out_file"
|
||||
while true; do
|
||||
local now elapsed sample
|
||||
now="$(date +%s)"
|
||||
if (( now > deadline )); then
|
||||
break
|
||||
fi
|
||||
elapsed="$((now - started_at))"
|
||||
if ! kill -0 "$pid" >/dev/null 2>&1; then
|
||||
break
|
||||
fi
|
||||
sample="$(ps -o rss=,vsz= -p "$pid" | awk 'NF>=2 {print $1","$2}')"
|
||||
if [[ -n "$sample" ]]; then
|
||||
echo "$(date +%Y-%m-%dT%H:%M:%S),${elapsed},${sample}" >> "$out_file"
|
||||
fi
|
||||
sleep "$SAMPLE_SECS"
|
||||
done
|
||||
}
|
||||
|
||||
run_profile() {
|
||||
local profile_name="$1"
|
||||
local mode="$2"
|
||||
local obj_size="$3"
|
||||
local pid="$4"
|
||||
local warp_host
|
||||
warp_host="$(normalize_warp_host "$HOST")"
|
||||
local benchdata="$OUT_DIR/${profile_name// /-}"
|
||||
local warp_log="$OUT_DIR/${profile_name// /-}.warp.log"
|
||||
local rss_during="$OUT_DIR/${profile_name// /-}.rss_during.csv"
|
||||
local rss_cooldown="$OUT_DIR/${profile_name// /-}.rss_cooldown.csv"
|
||||
|
||||
local -a cmd=(
|
||||
"$WARP_BIN" "$mode"
|
||||
"--host" "$warp_host"
|
||||
"--access-key" "$ACCESS_KEY"
|
||||
"--secret-key" "$SECRET_KEY"
|
||||
"--bucket" "$BUCKET"
|
||||
"--region" "$REGION"
|
||||
"--obj.size" "$obj_size"
|
||||
"--concurrent" "$CONCURRENCY"
|
||||
"--duration" "$DURATION"
|
||||
"--benchdata" "$benchdata"
|
||||
)
|
||||
if [[ "$INSECURE" == "true" ]]; then
|
||||
cmd+=("--insecure")
|
||||
fi
|
||||
|
||||
echo "==== Running profile: $profile_name ===="
|
||||
printf 'Command:'
|
||||
printf ' %q' "${cmd[@]}"
|
||||
printf '\n'
|
||||
|
||||
local sampler_pid=""
|
||||
if [[ -n "$pid" ]]; then
|
||||
sample_rss_loop "$pid" "$rss_during" &
|
||||
sampler_pid=$!
|
||||
else
|
||||
echo "WARN: rustfs pid unavailable; skipping RSS sampling for $profile_name" >&2
|
||||
fi
|
||||
if ! "${cmd[@]}" 2>&1 | tee "$warp_log"; then
|
||||
echo "ERROR: profile failed: $profile_name" >&2
|
||||
if [[ -n "$sampler_pid" ]]; then
|
||||
kill "$sampler_pid" >/dev/null 2>&1 || true
|
||||
wait "$sampler_pid" >/dev/null 2>&1 || true
|
||||
fi
|
||||
exit 1
|
||||
fi
|
||||
if [[ -n "$sampler_pid" ]]; then
|
||||
kill "$sampler_pid" >/dev/null 2>&1 || true
|
||||
wait "$sampler_pid" >/dev/null 2>&1 || true
|
||||
fi
|
||||
|
||||
echo "==== Cooldown sampling: $profile_name ($COOLDOWN_SECS s) ===="
|
||||
sample_rss_window "$pid" "$COOLDOWN_SECS" "$rss_cooldown"
|
||||
}
|
||||
|
||||
main() {
|
||||
parse_args "$@"
|
||||
require_cmd "$WARP_BIN"
|
||||
require_cmd awk
|
||||
require_cmd ps
|
||||
require_cmd pgrep
|
||||
require_cmd tee
|
||||
mkdir -p "$OUT_DIR"
|
||||
|
||||
local pid
|
||||
pid="$(resolve_pid)"
|
||||
local warp_host
|
||||
warp_host="$(normalize_warp_host "$HOST")"
|
||||
|
||||
echo "Output dir: $OUT_DIR"
|
||||
if [[ -n "$pid" ]]; then
|
||||
echo "RustFS pid: $pid"
|
||||
else
|
||||
echo "RustFS pid: auto-detect failed (continuing without RSS sampling)"
|
||||
fi
|
||||
echo "Host: $HOST"
|
||||
echo "Warp host: $warp_host"
|
||||
echo "Bucket: $BUCKET"
|
||||
echo "Profiles:"
|
||||
echo " - 4KiB mixed"
|
||||
echo " - 11MiB mixed"
|
||||
echo " - 11MiB delete"
|
||||
|
||||
run_profile "4KiB mixed" "mixed" "4KiB" "$pid"
|
||||
run_profile "11MiB mixed" "mixed" "11MiB" "$pid"
|
||||
run_profile "11MiB delete" "delete" "11MiB" "$pid"
|
||||
|
||||
echo
|
||||
echo "Acceptance run finished."
|
||||
echo "Artifacts:"
|
||||
find "$OUT_DIR" -maxdepth 1 -type f | sort
|
||||
}
|
||||
|
||||
main "$@"
|
||||
Reference in New Issue
Block a user