From 50d03ef02106ac88b553db29471f38991748b370 Mon Sep 17 00:00:00 2001 From: houseme Date: Mon, 27 Apr 2026 00:42:35 +0800 Subject: [PATCH] perf(memory): add reclaim signals and cache controls (#2689) --- .../grafana/dashboards/rustfs.json | 754 +++++++++++++++++- Cargo.lock | 121 ++- Cargo.toml | 2 +- crates/config/src/constants/runtime.rs | 19 + crates/ecstore/Cargo.toml | 1 + crates/ecstore/src/disk/local.rs | 289 ++++++- crates/ecstore/src/erasure_coding/encode.rs | 18 +- crates/heal/src/heal/manager.rs | 26 +- crates/heal/src/lib.rs | 19 + crates/io-metrics/src/lib.rs | 160 ++++ crates/scanner/src/lib.rs | 23 + crates/scanner/src/scanner.rs | 4 +- rustfs/Cargo.toml | 1 + rustfs/src/allocator_reclaim.rs | 243 ++++++ rustfs/src/app/object_usecase.rs | 78 +- rustfs/src/delete_tail_activity.rs | 115 +++ rustfs/src/lib.rs | 3 + rustfs/src/main.rs | 2 + rustfs/src/memory_observability.rs | 209 +++++ rustfs/src/server/http.rs | 4 + rustfs/src/server/mod.rs | 1 + scripts/run.sh | 13 +- scripts/run_issue_2573_acceptance.sh | 257 ++++++ 23 files changed, 2253 insertions(+), 109 deletions(-) create mode 100644 rustfs/src/allocator_reclaim.rs create mode 100644 rustfs/src/delete_tail_activity.rs create mode 100644 rustfs/src/memory_observability.rs create mode 100755 scripts/run_issue_2573_acceptance.sh diff --git a/.docker/observability/grafana/dashboards/rustfs.json b/.docker/observability/grafana/dashboards/rustfs.json index 18c9b1fa6..b12460dd7 100644 --- a/.docker/observability/grafana/dashboards/rustfs.json +++ b/.docker/observability/grafana/dashboards/rustfs.json @@ -2534,7 +2534,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (job) (rustfs_system_process_resident_memory_bytes{job=~\"$job\"})", + "expr": "sum by (job) (rustfs_system_process_resident_memory_bytes{job=~\"$job\"}) or sum by (job) (rustfs_memory_process_resident_bytes{job=~\"$job\"})", "legendFormat": "process rss - {{job}}", "range": true, "refId": "A" @@ -2545,8 +2545,8 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "rustfs_system_memory_used_bytes{job=~\"$job\"}", - "legendFormat": "system used - {{job}}", + "expr": "sum by (job) (rustfs_memory_process_virtual_bytes{job=~\"$job\"})", + "legendFormat": "process virtual - {{job}}", "range": true, "refId": "B" }, @@ -2556,13 +2556,755 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "rustfs_system_memory_used_perc{job=~\"$job\"}", - "legendFormat": "system used percent - {{job}}", + "expr": "sum by (job) (rustfs_memory_cgroup_anon_bytes{job=~\"$job\"})", + "legendFormat": "anon split - {{job}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (job) (rustfs_memory_cgroup_file_bytes{job=~\"$job\"})", + "legendFormat": "file split - {{job}}", + "range": true, + "refId": "D" + } + ], + "title": "Memory Split", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 80 + }, + "id": 501, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (job) (rustfs_delete_tail_activity_total_inflight_current{job=~\"$job\"})", + "legendFormat": "delete tail inflight - {{job}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (job) (rustfs_memory_allocator_reclaim_scanner_activity_current{job=~\"$job\"})", + "legendFormat": "scanner activity - {{job}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (job) (rustfs_memory_allocator_reclaim_heal_activity_current{job=~\"$job\"})", + "legendFormat": "heal activity - {{job}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (job) (rustfs_memory_allocator_reclaim_reclaimable_work_current{job=~\"$job\"})", + "legendFormat": "reclaimable work - {{job}}", + "range": true, + "refId": "D" + } + ], + "title": "Tail / Reclaim Activity", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 80 + }, + "id": 502, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (job) (rustfs_memory_allocator_reclaim_idle_streak{job=~\"$job\"})", + "legendFormat": "idle streak - {{job}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (job) (rate(rustfs_memory_allocator_reclaim_total{job=~\"$job\",result=\"ok\"}[5m]))", + "legendFormat": "reclaim ok rate - {{job}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (job,reason) (rate(rustfs_memory_allocator_reclaim_skipped_total{job=~\"$job\"}[5m]))", + "legendFormat": "reclaim skipped {{reason}} - {{job}}", "range": true, "refId": "C" } ], - "title": "Memory Usage", + "title": "Allocator Reclaim Health", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 87 + }, + "id": 503, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (job) (rustfs_memory_cgroup_current_bytes{job=~\"$job\"})", + "legendFormat": "cgroup current - {{job}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (job) (rustfs_memory_cgroup_limit_bytes{job=~\"$job\"})", + "legendFormat": "cgroup limit - {{job}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (job) (rustfs_memory_cgroup_active_file_bytes{job=~\"$job\"})", + "legendFormat": "active file - {{job}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (job) (rustfs_memory_cgroup_inactive_file_bytes{job=~\"$job\"})", + "legendFormat": "inactive file - {{job}}", + "range": true, + "refId": "D" + } + ], + "title": "Memory Cgroup Detail", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 87 + }, + "id": 504, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (job) (rustfs_ec_encode_inflight_bytes_current{job=~\"$job\"})", + "legendFormat": "ec inflight bytes - {{job}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (job) (rustfs_get_object_buffered_bytes_current{job=~\"$job\"})", + "legendFormat": "get buffered bytes - {{job}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (job) (rustfs_memory_allocator_reclaim_active_requests{job=~\"$job\"})", + "legendFormat": "allocator active requests - {{job}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (job) (rustfs_memory_allocator_reclaim_delete_tail_activity_current{job=~\"$job\"})", + "legendFormat": "allocator tail activity - {{job}}", + "range": true, + "refId": "D" + } + ], + "title": "Heap Amplification Signals", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 94 + }, + "id": 505, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (job,stage) (rustfs_delete_tail_activity_inflight_current{job=~\"$job\"})", + "legendFormat": "{{stage}} inflight - {{job}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (job,stage) (rate(rustfs_delete_tail_activity_started_total{job=~\"$job\"}[5m]))", + "legendFormat": "{{stage}} started rate - {{job}}", + "range": true, + "refId": "B" + } + ], + "title": "Delete Tail by Stage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 94 + }, + "id": 506, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (job,kind,result) (rate(rustfs_page_cache_reclaim_requests_total{job=~\"$job\"}[5m]))", + "legendFormat": "{{kind}} reclaim {{result}} rate - {{job}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (job,kind) (rate(rustfs_page_cache_reclaim_bytes_total{job=~\"$job\"}[5m]))", + "legendFormat": "{{kind}} reclaim bytes/s - {{job}}", + "range": true, + "refId": "B" + } + ], + "title": "Page Cache Reclaim", "type": "timeseries" }, { diff --git a/Cargo.lock b/Cargo.lock index fabb72abb..df7a68a54 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -40,18 +40,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b657e772794c6b04730ea897b66a058ccd866c16d1967da05eeeecec39043fe" dependencies = [ "crypto-common 0.2.1", - "inout 0.2.2", -] - -[[package]] -name = "aes" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" -dependencies = [ - "cfg-if", - "cipher 0.4.4", - "cpufeatures 0.2.17", + "inout", ] [[package]] @@ -60,7 +49,7 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "66bd29a732b644c0431c6140f370d097879203d79b80c94a6747ba0872adaef8" dependencies = [ - "cipher 0.5.1", + "cipher", "cpubits", "cpufeatures 0.3.0", ] @@ -72,8 +61,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e22c0c90bbe8d4f77c3ca9ddabe41a1f8382d6fc1f7cea89459d0f320371f972" dependencies = [ "aead", - "aes 0.9.0", - "cipher 0.5.1", + "aes", + "cipher", "ctr", "ghash", "subtle", @@ -203,7 +192,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -214,7 +203,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -1323,6 +1312,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cdd35008169921d80bc60d3d0ab416eecb028c4cd653352907921d95084790be" dependencies = [ "hybrid-array", + "zeroize", ] [[package]] @@ -1508,7 +1498,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" dependencies = [ "cfg-if", - "cipher 0.5.1", + "cipher", "cpufeatures 0.3.0", "rand_core 0.10.1", ] @@ -1521,7 +1511,7 @@ checksum = "1c9ed179664f12fd6f155f6dd632edf5f3806d48c228c67ff78366f2a0eb6b5e" dependencies = [ "aead", "chacha20", - "cipher 0.5.1", + "cipher", "poly1305", ] @@ -1576,16 +1566,6 @@ dependencies = [ "half", ] -[[package]] -name = "cipher" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" -dependencies = [ - "crypto-common 0.1.6", - "inout 0.1.4", -] - [[package]] name = "cipher" version = "0.5.1" @@ -1594,7 +1574,7 @@ checksum = "e34d8227fe1ba289043aeb13792056ff80fd6de1a9f49137a5f499de8e8c78ea" dependencies = [ "block-buffer 0.12.0", "crypto-common 0.2.1", - "inout 0.2.2", + "inout", ] [[package]] @@ -2045,9 +2025,9 @@ dependencies = [ [[package]] name = "crypto-common" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" dependencies = [ "generic-array", "typenum", @@ -2102,7 +2082,7 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "17469f8eb9bdbfad10f71f4cfddfd38b01143520c0e717d8796ccb4d44d44e42" dependencies = [ - "cipher 0.5.1", + "cipher", ] [[package]] @@ -2114,6 +2094,12 @@ dependencies = [ "cmov", ] +[[package]] +name = "cty" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b365fabc795046672053e29c954733ec3b05e4be654ab130fe8f1f94d7051f35" + [[package]] name = "curve25519-dalek" version = "4.1.3" @@ -3175,7 +3161,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer 0.10.4", "const-oid 0.9.6", - "crypto-common 0.1.6", + "crypto-common 0.1.7", "subtle", ] @@ -3189,6 +3175,7 @@ dependencies = [ "const-oid 0.10.2", "crypto-common 0.2.1", "ctutils", + "zeroize", ] [[package]] @@ -3209,7 +3196,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -3488,7 +3475,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -3777,9 +3764,9 @@ dependencies = [ [[package]] name = "generic-array" -version = "0.14.9" +version = "0.14.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ "typenum", "version_check", @@ -4691,15 +4678,6 @@ dependencies = [ "str_stack", ] -[[package]] -name = "inout" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" -dependencies = [ - "generic-array", -] - [[package]] name = "inout" version = "0.2.2" @@ -4768,7 +4746,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -4845,7 +4823,7 @@ dependencies = [ "portable-atomic", "portable-atomic-util", "serde_core", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -5161,6 +5139,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d1eacfa31c33ec25e873c136ba5669f00f9866d0688bea7be4d3f7e43067df6" dependencies = [ "cc", + "cty", ] [[package]] @@ -5814,7 +5793,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -6438,16 +6417,6 @@ dependencies = [ "once_cell", ] -[[package]] -name = "pbkdf2" -version = "0.12.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" -dependencies = [ - "digest 0.10.7", - "hmac 0.12.1", -] - [[package]] name = "pbkdf2" version = "0.13.0" @@ -7878,6 +7847,7 @@ dependencies = [ "jemalloc_pprof", "jiff", "libc", + "libmimalloc-sys", "libsystemd", "matchit 0.9.2", "md5", @@ -8066,7 +8036,7 @@ dependencies = [ "cfg-if", "chacha20poly1305", "jsonwebtoken", - "pbkdf2 0.13.0", + "pbkdf2", "rand 0.10.1", "serde_json", "sha2 0.11.0", @@ -8145,6 +8115,7 @@ dependencies = [ "rustfs-signer", "rustfs-utils", "rustfs-workers", + "rustix 1.1.4", "rustls", "s3s", "serde", @@ -8294,7 +8265,7 @@ dependencies = [ "indexmap 2.14.0", "kafka-protocol", "metrics", - "pbkdf2 0.13.0", + "pbkdf2", "rand 0.10.1", "rustls", "rustls-native-certs", @@ -8317,7 +8288,7 @@ dependencies = [ "hmac 0.13.0", "kafka-protocol", "metrics", - "pbkdf2 0.13.0", + "pbkdf2", "rand 0.10.1", "rustfs-kafka", "rustls", @@ -8917,7 +8888,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -8976,7 +8947,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -9647,7 +9618,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -9736,7 +9707,7 @@ dependencies = [ "cfg-if", "libc", "psm", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -9983,7 +9954,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix 1.1.4", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -10973,7 +10944,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -11644,24 +11615,24 @@ dependencies = [ [[package]] name = "zip" -version = "8.5.1" +version = "8.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcab981e19633ebcf0b001ddd37dd802996098bc1864f90b7c5d970ce76c1d59" +checksum = "2d04a6b5381502aa6087c94c669499eb1602eb9c5e8198e534de571f7154809b" dependencies = [ - "aes 0.8.4", + "aes", "bzip2", "constant_time_eq", "crc32fast", "deflate64", "flate2", "getrandom 0.4.2", - "hmac 0.12.1", + "hmac 0.13.0", "indexmap 2.14.0", "lzma-rust2", "memchr", - "pbkdf2 0.12.2", + "pbkdf2", "ppmd-rust", - "sha1 0.10.6", + "sha1 0.11.0", "time", "typed-path", "zeroize", diff --git a/Cargo.toml b/Cargo.toml index 1ed1fe89e..bc3809f42 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -280,7 +280,7 @@ walkdir = "2.5.0" wildmatch = { version = "2.6.1", features = ["serde"] } windows = { version = "0.62.2" } xxhash-rust = { version = "0.8.15", features = ["xxh64", "xxh3"] } -zip = "8.5.1" +zip = "8.6.0" zstd = "0.13.3" # Observability and Metrics diff --git a/crates/config/src/constants/runtime.rs b/crates/config/src/constants/runtime.rs index 06ffa16a9..e11a25c05 100644 --- a/crates/config/src/constants/runtime.rs +++ b/crates/config/src/constants/runtime.rs @@ -59,9 +59,28 @@ pub const DEFAULT_RUNTIME_DIAL9_ROTATION_COUNT: usize = 10; pub const DEFAULT_RUNTIME_DIAL9_SAMPLING_RATE: f64 = 1.0; // 100% sampling // Note: S3 bucket/prefix have no default; absence means upload is disabled (modeled as Option) +// Allocator reclaim configuration +pub const ENV_ALLOCATOR_RECLAIM_ENABLED: &str = "RUSTFS_ALLOCATOR_RECLAIM_ENABLED"; +pub const ENV_ALLOCATOR_RECLAIM_INTERVAL_SECS: &str = "RUSTFS_ALLOCATOR_RECLAIM_INTERVAL_SECS"; +pub const ENV_ALLOCATOR_RECLAIM_FORCE: &str = "RUSTFS_ALLOCATOR_RECLAIM_FORCE"; +pub const ENV_ALLOCATOR_RECLAIM_IDLE_INTERVALS: &str = "RUSTFS_ALLOCATOR_RECLAIM_IDLE_INTERVALS"; +pub const DEFAULT_ALLOCATOR_RECLAIM_ENABLED: bool = false; +pub const DEFAULT_ALLOCATOR_RECLAIM_INTERVAL_SECS: u64 = 30; +pub const DEFAULT_ALLOCATOR_RECLAIM_FORCE: bool = true; +pub const DEFAULT_ALLOCATOR_RECLAIM_IDLE_INTERVALS: u64 = 3; + +// File page-cache reclaim configuration +pub const ENV_OBJECT_FILE_CACHE_RECLAIM_WRITE_ENABLE: &str = "RUSTFS_OBJECT_FILE_CACHE_RECLAIM_WRITE_ENABLE"; +pub const ENV_OBJECT_FILE_CACHE_RECLAIM_READ_ENABLE: &str = "RUSTFS_OBJECT_FILE_CACHE_RECLAIM_READ_ENABLE"; +pub const ENV_OBJECT_FILE_CACHE_RECLAIM_THRESHOLD: &str = "RUSTFS_OBJECT_FILE_CACHE_RECLAIM_THRESHOLD"; +pub const DEFAULT_OBJECT_FILE_CACHE_RECLAIM_WRITE_ENABLE: bool = false; +pub const DEFAULT_OBJECT_FILE_CACHE_RECLAIM_READ_ENABLE: bool = false; +pub const DEFAULT_OBJECT_FILE_CACHE_RECLAIM_THRESHOLD: usize = 4 * 1024 * 1024; + /// Threshold for small object seek support in megabytes. /// /// When an object is smaller than this size, rustfs will provide seek support. /// /// Default is set to 10MB. +pub const ENV_OBJECT_SEEK_SUPPORT_THRESHOLD: &str = "RUSTFS_OBJECT_SEEK_SUPPORT_THRESHOLD"; pub const DEFAULT_OBJECT_SEEK_SUPPORT_THRESHOLD: usize = 10 * 1024 * 1024; diff --git a/crates/ecstore/Cargo.toml b/crates/ecstore/Cargo.toml index e8a6c68d5..0cdc9447a 100644 --- a/crates/ecstore/Cargo.toml +++ b/crates/ecstore/Cargo.toml @@ -100,6 +100,7 @@ pin-project-lite.workspace = true md-5.workspace = true memmap2 = { workspace = true } libc.workspace = true +rustix = { workspace = true } rustfs-madmin.workspace = true rustfs-workers.workspace = true reqwest = { workspace = true } diff --git a/crates/ecstore/src/disk/local.rs b/crates/ecstore/src/disk/local.rs index ed9714074..71cdb7449 100644 --- a/crates/ecstore/src/disk/local.rs +++ b/crates/ecstore/src/disk/local.rs @@ -31,6 +31,7 @@ use crate::disk::{ use crate::erasure_coding::bitrot_verify; use crate::global::{GLOBAL_IsErasureSD, GLOBAL_RootDiskThreshold}; use bytes::Bytes; +use metrics::counter; use parking_lot::RwLock as ParkingLotRwLock; use rustfs_filemeta::{ Cache, FileInfo, FileInfoOpts, FileMeta, MetaCacheEntry, MetacacheWriter, ObjectPartInfo, Opts, RawFileInfo, UpdateFn, @@ -79,6 +80,238 @@ pub enum InternalBuf<'a> { Owned(Bytes), } +struct FileCacheReclaimWriter { + inner: File, + reclaim_len: usize, + reclaim_on_shutdown: bool, + reclaimed: bool, +} + +struct FileCacheReclaimReader { + inner: File, + reclaim_offset: u64, + reclaim_len: usize, + reclaim_on_drop: bool, + reclaimed: bool, +} + +fn record_file_cache_reclaim_success(kind: &'static str, reclaim_len: usize, started: std::time::Instant) { + counter!("rustfs_page_cache_reclaim_requests_total", "kind" => kind.to_string(), "result" => "ok".to_string()).increment(1); + counter!("rustfs_page_cache_reclaim_bytes_total", "kind" => kind.to_string()).increment(reclaim_len as u64); + metrics::histogram!("rustfs_page_cache_reclaim_duration_seconds", "kind" => kind.to_string()) + .record(started.elapsed().as_secs_f64()); +} + +fn record_file_cache_reclaim_error(kind: &'static str) { + counter!("rustfs_page_cache_reclaim_requests_total", "kind" => kind.to_string(), "result" => "err".to_string()).increment(1); +} + +impl FileCacheReclaimReader { + fn new(inner: File, reclaim_offset: u64, reclaim_len: usize, reclaim_on_drop: bool) -> Self { + #[cfg(target_os = "macos")] + if reclaim_on_drop { + let _ = set_fd_nocache(&inner); + } + + Self { + inner, + reclaim_offset, + reclaim_len, + reclaim_on_drop, + reclaimed: false, + } + } + + #[cfg(target_os = "linux")] + fn reclaim_file_cache(&mut self) -> std::io::Result<()> { + use core::num::NonZeroU64; + use rustix::fs::{Advice, fadvise}; + + if !self.reclaim_on_drop || self.reclaimed || self.reclaim_len == 0 { + return Ok(()); + } + + let started = std::time::Instant::now(); + let reclaim_len = + NonZeroU64::new(self.reclaim_len as u64).expect("reclaim_len is guaranteed non-zero by the early return"); + fadvise(&self.inner, self.reclaim_offset, Some(reclaim_len), Advice::DontNeed).map_err(std::io::Error::from)?; + + self.reclaimed = true; + record_file_cache_reclaim_success("read", self.reclaim_len, started); + Ok(()) + } + + #[cfg(not(target_os = "linux"))] + fn reclaim_file_cache(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +#[cfg(target_os = "macos")] +#[allow(unsafe_code)] +fn set_fd_nocache(file: &File) -> std::io::Result<()> { + use std::os::fd::AsRawFd; + + // SAFETY: `fcntl` is called on a valid file descriptor owned by `file`. + let ret = unsafe { libc::fcntl(file.as_raw_fd(), libc::F_NOCACHE, 1) }; + if ret == -1 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) +} + +#[cfg(target_os = "macos")] +#[allow(unsafe_code)] +fn set_std_fd_nocache(file: &std::fs::File) -> std::io::Result<()> { + use std::os::fd::AsRawFd; + + // SAFETY: `fcntl` is called on a valid file descriptor owned by `file`. + let ret = unsafe { libc::fcntl(file.as_raw_fd(), libc::F_NOCACHE, 1) }; + if ret == -1 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) +} + +impl Drop for FileCacheReclaimReader { + fn drop(&mut self) { + if let Err(err) = self.reclaim_file_cache() { + record_file_cache_reclaim_error("read"); + debug!(error = ?err, reclaim_offset = self.reclaim_offset, reclaim_len = self.reclaim_len, "failed to reclaim file cache after read"); + } + } +} + +impl tokio::io::AsyncRead for FileCacheReclaimReader { + fn poll_read( + mut self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + buf: &mut tokio::io::ReadBuf<'_>, + ) -> std::task::Poll> { + std::pin::Pin::new(&mut self.inner).poll_read(cx, buf) + } +} + +impl FileCacheReclaimWriter { + fn new(inner: File, reclaim_len: usize, reclaim_on_shutdown: bool) -> Self { + #[cfg(target_os = "macos")] + if reclaim_on_shutdown { + let _ = set_fd_nocache(&inner); + } + + Self { + inner, + reclaim_len, + reclaim_on_shutdown, + reclaimed: false, + } + } + + #[cfg(target_os = "linux")] + fn reclaim_file_cache(&mut self) -> std::io::Result<()> { + use core::num::NonZeroU64; + use rustix::fs::{Advice, fadvise}; + + if !self.reclaim_on_shutdown || self.reclaimed || self.reclaim_len == 0 { + return Ok(()); + } + + let started = std::time::Instant::now(); + let reclaim_len = + NonZeroU64::new(self.reclaim_len as u64).expect("reclaim_len is guaranteed non-zero by the early return"); + fadvise(&self.inner, 0, Some(reclaim_len), Advice::DontNeed).map_err(std::io::Error::from)?; + + self.reclaimed = true; + record_file_cache_reclaim_success("write", self.reclaim_len, started); + Ok(()) + } + + #[cfg(not(target_os = "linux"))] + fn reclaim_file_cache(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +impl AsyncWrite for FileCacheReclaimWriter { + fn poll_write( + mut self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + buf: &[u8], + ) -> std::task::Poll> { + std::pin::Pin::new(&mut self.inner).poll_write(cx, buf) + } + + fn poll_flush(mut self: std::pin::Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> std::task::Poll> { + std::pin::Pin::new(&mut self.inner).poll_flush(cx) + } + + fn poll_shutdown( + mut self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + match std::pin::Pin::new(&mut self.inner).poll_shutdown(cx) { + std::task::Poll::Ready(Ok(())) => { + if let Err(err) = self.reclaim_file_cache() { + record_file_cache_reclaim_error("write"); + debug!(error = ?err, reclaim_len = self.reclaim_len, "failed to reclaim file cache after write"); + } + std::task::Poll::Ready(Ok(())) + } + other => other, + } + } + + fn poll_write_vectored( + mut self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + bufs: &[std::io::IoSlice<'_>], + ) -> std::task::Poll> { + std::pin::Pin::new(&mut self.inner).poll_write_vectored(cx, bufs) + } + + fn is_write_vectored(&self) -> bool { + self.inner.is_write_vectored() + } +} + +fn should_reclaim_file_cache_after_write(file_size: i64) -> bool { + if file_size <= 0 { + return false; + } + + if !rustfs_utils::get_env_bool( + rustfs_config::ENV_OBJECT_FILE_CACHE_RECLAIM_WRITE_ENABLE, + rustfs_config::DEFAULT_OBJECT_FILE_CACHE_RECLAIM_WRITE_ENABLE, + ) { + return false; + } + + let threshold = rustfs_utils::get_env_usize( + rustfs_config::ENV_OBJECT_FILE_CACHE_RECLAIM_THRESHOLD, + rustfs_config::DEFAULT_OBJECT_FILE_CACHE_RECLAIM_THRESHOLD, + ); + file_size as usize >= threshold +} + +fn should_reclaim_file_cache_after_read(length: usize) -> bool { + if length == 0 { + return false; + } + + if !rustfs_utils::get_env_bool( + rustfs_config::ENV_OBJECT_FILE_CACHE_RECLAIM_READ_ENABLE, + rustfs_config::DEFAULT_OBJECT_FILE_CACHE_RECLAIM_READ_ENABLE, + ) { + return false; + } + + let threshold = rustfs_utils::get_env_usize( + rustfs_config::ENV_OBJECT_FILE_CACHE_RECLAIM_THRESHOLD, + rustfs_config::DEFAULT_OBJECT_FILE_CACHE_RECLAIM_THRESHOLD, + ); + length >= threshold +} + pub struct LocalDisk { pub root: PathBuf, pub format_path: PathBuf, @@ -1847,8 +2080,9 @@ impl DiskAPI for LocalDisk { let f = super::fs::open_file(&file_path, O_CREATE | O_WRONLY) .await .map_err(to_file_error)?; + let reclaim_on_shutdown = should_reclaim_file_cache_after_write(_file_size); - Ok(Box::new(f)) + Ok(Box::new(FileCacheReclaimWriter::new(f, _file_size.max(0) as usize, reclaim_on_shutdown))) // Ok(()) } @@ -1920,7 +2154,8 @@ impl DiskAPI for LocalDisk { f.seek(SeekFrom::Start(offset as u64)).await?; } - Ok(Box::new(f)) + let reclaim_on_drop = should_reclaim_file_cache_after_read(length); + Ok(Box::new(FileCacheReclaimReader::new(f, offset as u64, length, reclaim_on_drop))) } /// Zero-copy file read using memory mapping (Unix) or efficient read (non-Unix). @@ -1965,9 +2200,15 @@ impl DiskAPI for LocalDisk { use memmap2::MmapOptions; let file_path_clone = file_path.clone(); + let should_reclaim_after_read = should_reclaim_file_cache_after_read(length); let bytes = tokio::task::spawn_blocking(move || { let file = std::fs::File::open(&file_path_clone).map_err(DiskError::from)?; + #[cfg(target_os = "macos")] + if should_reclaim_after_read { + let _ = set_std_fd_nocache(&file); + } + // mmap offsets on Unix must be page-size aligned. Align the // mapping down to the nearest page boundary, then slice out the // originally requested logical range. @@ -1995,7 +2236,21 @@ impl DiskAPI for LocalDisk { let end = logical_offset .checked_add(length) .ok_or_else(|| DiskError::other("mmap slice length overflow"))?; - Ok::(Bytes::copy_from_slice(&mmap[logical_offset..end])) + let bytes = Bytes::copy_from_slice(&mmap[logical_offset..end]); + + #[cfg(target_os = "linux")] + if should_reclaim_after_read { + use core::num::NonZeroU64; + use rustix::fs::{Advice, fadvise}; + + let reclaim_len = + NonZeroU64::new(map_len as u64).ok_or_else(|| DiskError::other("mmap reclaim length overflow"))?; + fadvise(&file, aligned_offset, Some(reclaim_len), Advice::DontNeed) + .map_err(std::io::Error::from) + .map_err(DiskError::from)?; + } + + Ok::(bytes) }) .await .map_err(DiskError::from)??; @@ -3372,4 +3627,32 @@ mod test { assert_eq!(normalize_path_components("C:\\a\\..\\b"), PathBuf::from("C:\\b")); } } + + #[test] + fn should_reclaim_file_cache_after_write_respects_env_and_threshold() { + temp_env::with_var_unset(rustfs_config::ENV_OBJECT_FILE_CACHE_RECLAIM_WRITE_ENABLE, || { + assert!(!should_reclaim_file_cache_after_write(8 * 1024 * 1024)); + }); + + temp_env::with_var(rustfs_config::ENV_OBJECT_FILE_CACHE_RECLAIM_WRITE_ENABLE, Some("true"), || { + temp_env::with_var(rustfs_config::ENV_OBJECT_FILE_CACHE_RECLAIM_THRESHOLD, Some("4194304"), || { + assert!(should_reclaim_file_cache_after_write(8 * 1024 * 1024)); + assert!(!should_reclaim_file_cache_after_write(1024)); + }); + }); + } + + #[test] + fn should_reclaim_file_cache_after_read_respects_env_and_threshold() { + temp_env::with_var_unset(rustfs_config::ENV_OBJECT_FILE_CACHE_RECLAIM_READ_ENABLE, || { + assert!(!should_reclaim_file_cache_after_read(8 * 1024 * 1024)); + }); + + temp_env::with_var(rustfs_config::ENV_OBJECT_FILE_CACHE_RECLAIM_READ_ENABLE, Some("true"), || { + temp_env::with_var(rustfs_config::ENV_OBJECT_FILE_CACHE_RECLAIM_THRESHOLD, Some("4194304"), || { + assert!(should_reclaim_file_cache_after_read(8 * 1024 * 1024)); + assert!(!should_reclaim_file_cache_after_read(1024)); + }); + }); + } } diff --git a/crates/ecstore/src/erasure_coding/encode.rs b/crates/ecstore/src/erasure_coding/encode.rs index 8f9b029e2..aff98c934 100644 --- a/crates/ecstore/src/erasure_coding/encode.rs +++ b/crates/ecstore/src/erasure_coding/encode.rs @@ -27,7 +27,7 @@ use tokio::sync::mpsc; use tracing::error; const ENV_RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES: &str = "RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES"; -const DEFAULT_RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES: usize = 32 * 1024 * 1024; +const DEFAULT_RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES: usize = 8 * 1024 * 1024; const DEFAULT_RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BLOCKS: usize = 8; fn encode_channel_capacity(expanded_block_bytes: usize, max_inflight_bytes: usize) -> usize { @@ -40,6 +40,16 @@ fn encode_channel_capacity(expanded_block_bytes: usize, max_inflight_bytes: usiz .clamp(1, DEFAULT_RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BLOCKS) } +fn queued_block_bytes(block: &[Bytes]) -> usize { + block.iter().map(Bytes::len).sum() +} + +async fn drain_queued_inflight_bytes(rx: &mut mpsc::Receiver>) { + while let Some(block) = rx.recv().await { + rustfs_io_metrics::remove_ec_encode_inflight_bytes(queued_block_bytes(&block)); + } +} + pub(crate) struct MultiWriter<'a> { writers: &'a mut [Option], write_quorum: usize, @@ -217,7 +227,10 @@ impl Erasure { Ok(n) if n > 0 => { total += n; let res = self.encode_data(&buf[..n])?; + let queued_bytes = queued_block_bytes(&res); + rustfs_io_metrics::add_ec_encode_inflight_bytes(queued_bytes); if let Err(err) = tx.send(res).await { + rustfs_io_metrics::remove_ec_encode_inflight_bytes(queued_bytes); return Err(std::io::Error::other(format!("Failed to send encoded data : {err}"))); } } @@ -250,6 +263,8 @@ impl Erasure { if block.is_empty() { break; } + let queued_bytes = queued_block_bytes(&block); + rustfs_io_metrics::remove_ec_encode_inflight_bytes(queued_bytes); if let Err(err) = writers.write(block).await { write_err = Some(err); break; @@ -259,6 +274,7 @@ impl Erasure { if let Some(err) = write_err { task.abort(); let _ = task.await; + drain_queued_inflight_bytes(&mut rx).await; if let Err(shutdown_err) = writers.shutdown().await { error!("failed to shutdown erasure writers after write error: {:?}", shutdown_err); } diff --git a/crates/heal/src/heal/manager.rs b/crates/heal/src/heal/manager.rs index a36689e20..800c59ee9 100644 --- a/crates/heal/src/heal/manager.rs +++ b/crates/heal/src/heal/manager.rs @@ -237,6 +237,14 @@ impl PriorityHealQueue { } } +fn publish_active_heal_count(active_heals: &HashMap>) { + crate::set_heal_active_tasks(active_heals.len()); +} + +fn publish_heal_queue_length(queue: &PriorityHealQueue) { + crate::set_heal_queue_length(queue.len()); +} + /// Heal config #[derive(Debug, Clone)] pub struct HealConfig { @@ -420,6 +428,8 @@ impl HealManager { } } active_heals.clear(); + publish_active_heal_count(&active_heals); + crate::set_heal_queue_length(0); // update state let mut state = self.state.write().await; @@ -435,6 +445,7 @@ impl HealManager { let mut queue = self.heal_queue.lock().await; let queue_len = queue.len(); + publish_heal_queue_length(&queue); let queue_capacity = config.queue_size; if queue.contains_key(&request) { @@ -505,6 +516,7 @@ impl HealManager { let push_outcome = queue.push(request); debug_assert_eq!(push_outcome, QueuePushOutcome::Accepted); + publish_heal_queue_length(&queue); // Log queue statistics periodically (when adding high/urgent priority items) if matches!(priority, HealPriority::High | HealPriority::Urgent) { @@ -544,7 +556,9 @@ impl HealManager { /// Get task progress pub async fn get_active_tasks_count(&self) -> usize { - self.active_heals.lock().await.len() + let active_heals = self.active_heals.lock().await; + publish_active_heal_count(&active_heals); + active_heals.len() } pub async fn get_task_progress(&self, task_id: &str) -> Result { @@ -564,6 +578,7 @@ impl HealManager { if let Some(task) = active_heals.get(task_id) { task.cancel().await?; active_heals.remove(task_id); + publish_active_heal_count(&active_heals); info!("Cancelled heal task: {}", task_id); Ok(()) } else { @@ -581,12 +596,14 @@ impl HealManager { /// Get active task count pub async fn get_active_task_count(&self) -> usize { let active_heals = self.active_heals.lock().await; + publish_active_heal_count(&active_heals); active_heals.len() } /// Get queue length pub async fn get_queue_length(&self) -> usize { let queue = self.heal_queue.lock().await; + publish_heal_queue_length(&queue); queue.len() } @@ -731,6 +748,7 @@ impl HealManager { ); let mut queue = heal_queue.lock().await; if matches!(queue.push(req), QueuePushOutcome::Accepted) { + publish_heal_queue_length(&queue); let config = config.read().await; if config.event_driven_scheduler_enable { notify.notify_one(); @@ -757,6 +775,7 @@ impl HealManager { ) { let config = config.read().await; let mut active_heals_guard = active_heals.lock().await; + publish_active_heal_count(&active_heals_guard); // Check if new heal tasks can be started let active_count = active_heals_guard.len(); @@ -769,6 +788,7 @@ impl HealManager { let mut queue = heal_queue.lock().await; let queue_len = queue.len(); + publish_heal_queue_length(&queue); if queue_len == 0 { return; @@ -804,6 +824,7 @@ impl HealManager { let task = Arc::new(HealTask::from_request(request, storage.clone())); let task_id = task.id.clone(); active_heals_guard.insert(task_id.clone(), task.clone()); + publish_active_heal_count(&active_heals_guard); update_task_running_metric_for_task(&active_heals_guard, task.as_ref()); let active_heals_clone = active_heals.clone(); let statistics_clone = statistics.clone(); @@ -828,6 +849,7 @@ impl HealManager { } let mut active_heals_guard = active_heals_clone.lock().await; if let Some(completed_task) = active_heals_guard.remove(&task_id) { + publish_active_heal_count(&active_heals_guard); update_task_running_metric_for_task(&active_heals_guard, completed_task.as_ref()); // update statistics let mut stats = statistics_clone.write().await; @@ -853,6 +875,8 @@ impl HealManager { let mut stats = statistics.write().await; stats.total_tasks += tasks_started as u64; stats.update_running_tasks(active_heals_guard.len() as u64); + publish_active_heal_count(&active_heals_guard); + publish_heal_queue_length(&queue); // Log queue status if items remain if !queue.is_empty() { diff --git a/crates/heal/src/lib.rs b/crates/heal/src/lib.rs index 24ac0a8e9..5fcfebf8c 100644 --- a/crates/heal/src/lib.rs +++ b/crates/heal/src/lib.rs @@ -17,6 +17,7 @@ pub mod heal; pub use error::{Error, Result}; pub use heal::{HealManager, HealOptions, HealPriority, HealRequest, HealType, channel::HealChannelProcessor}; +use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, OnceLock}; use tokio_util::sync::CancellationToken; use tracing::{error, info}; @@ -55,6 +56,8 @@ static GLOBAL_HEAL_MANAGER: OnceLock> = OnceLock::new(); /// Global heal channel processor instance static GLOBAL_HEAL_CHANNEL_PROCESSOR: OnceLock>> = OnceLock::new(); +static GLOBAL_HEAL_ACTIVE_TASKS: AtomicU64 = AtomicU64::new(0); +static GLOBAL_HEAL_QUEUE_LENGTH: AtomicU64 = AtomicU64::new(0); /// Initialize and start heal manager with channel processor pub async fn init_heal_manager( @@ -107,3 +110,19 @@ pub fn get_heal_manager() -> Option<&'static Arc> { pub fn get_heal_channel_processor() -> Option<&'static Arc>> { GLOBAL_HEAL_CHANNEL_PROCESSOR.get() } + +pub fn current_heal_active_tasks() -> u64 { + GLOBAL_HEAL_ACTIVE_TASKS.load(Ordering::Relaxed) +} + +pub fn current_heal_queue_length() -> u64 { + GLOBAL_HEAL_QUEUE_LENGTH.load(Ordering::Relaxed) +} + +pub(crate) fn set_heal_active_tasks(count: usize) { + GLOBAL_HEAL_ACTIVE_TASKS.store(count as u64, Ordering::Relaxed); +} + +pub(crate) fn set_heal_queue_length(count: usize) { + GLOBAL_HEAL_QUEUE_LENGTH.store(count as u64, Ordering::Relaxed); +} diff --git a/crates/io-metrics/src/lib.rs b/crates/io-metrics/src/lib.rs index dca5c51e0..20412555a 100644 --- a/crates/io-metrics/src/lib.rs +++ b/crates/io-metrics/src/lib.rs @@ -49,6 +49,8 @@ #[macro_use] extern crate metrics; +use std::sync::atomic::{AtomicU64, Ordering}; + // Public modules pub mod adaptive_ttl; pub mod autotuner; @@ -137,6 +139,43 @@ pub use config::{ pub use collector::MetricsCollector; pub use performance::PerformanceMetrics; +static EC_ENCODE_INFLIGHT_BYTES: AtomicU64 = AtomicU64::new(0); +static GET_OBJECT_BUFFERED_BYTES: AtomicU64 = AtomicU64::new(0); + +fn saturating_sub_atomic(counter: &AtomicU64, bytes: u64) -> u64 { + let mut current = counter.load(Ordering::Relaxed); + loop { + let next = current.saturating_sub(bytes); + match counter.compare_exchange_weak(current, next, Ordering::Relaxed, Ordering::Relaxed) { + Ok(_) => return next, + Err(actual) => current = actual, + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum TrackedMemoryGauge { + GetObjectBufferedBytes, +} + +/// Drop-based guard for tracked in-memory payloads. +#[derive(Debug)] +pub struct MemoryGaugeGuard { + gauge: TrackedMemoryGauge, + bytes: u64, +} + +impl Drop for MemoryGaugeGuard { + fn drop(&mut self) { + match self.gauge { + TrackedMemoryGauge::GetObjectBufferedBytes => { + let next = saturating_sub_atomic(&GET_OBJECT_BUFFERED_BYTES, self.bytes); + gauge!("rustfs_get_object_buffered_bytes_current").set(next as f64); + } + } + } +} + /// Record GetObject request start. #[inline(always)] pub fn record_get_object_request_start(concurrent_requests: usize) { @@ -544,6 +583,85 @@ pub fn record_memory_usage(used_bytes: u64, total_bytes: u64) { } } +/// Record process-level memory split metrics. +#[inline(always)] +pub fn record_process_memory_split(resident_bytes: u64, virtual_bytes: u64) { + gauge!("rustfs_memory_process_resident_bytes").set(resident_bytes as f64); + gauge!("rustfs_memory_process_virtual_bytes").set(virtual_bytes as f64); +} + +/// Record cgroup memory split metrics when available. +#[inline(always)] +pub fn record_cgroup_memory_split( + current_bytes: Option, + limit_bytes: Option, + anon_bytes: Option, + file_bytes: Option, + active_file_bytes: Option, + inactive_file_bytes: Option, +) { + if let Some(current_bytes) = current_bytes { + gauge!("rustfs_memory_cgroup_current_bytes").set(current_bytes as f64); + } + if let Some(limit_bytes) = limit_bytes { + gauge!("rustfs_memory_cgroup_limit_bytes").set(limit_bytes as f64); + } + if let Some(anon_bytes) = anon_bytes { + gauge!("rustfs_memory_cgroup_anon_bytes").set(anon_bytes as f64); + } + if let Some(file_bytes) = file_bytes { + gauge!("rustfs_memory_cgroup_file_bytes").set(file_bytes as f64); + } + if let Some(active_file_bytes) = active_file_bytes { + gauge!("rustfs_memory_cgroup_active_file_bytes").set(active_file_bytes as f64); + } + if let Some(inactive_file_bytes) = inactive_file_bytes { + gauge!("rustfs_memory_cgroup_inactive_file_bytes").set(inactive_file_bytes as f64); + } +} + +/// Track encoded bytes currently queued between erasure encode and disk writers. +#[inline(always)] +pub fn add_ec_encode_inflight_bytes(bytes: usize) { + let next = EC_ENCODE_INFLIGHT_BYTES.fetch_add(bytes as u64, Ordering::Relaxed) + bytes as u64; + gauge!("rustfs_ec_encode_inflight_bytes_current").set(next as f64); +} + +/// Remove encoded bytes from the tracked erasure encode in-flight gauge. +#[inline(always)] +pub fn remove_ec_encode_inflight_bytes(bytes: usize) { + let next = saturating_sub_atomic(&EC_ENCODE_INFLIGHT_BYTES, bytes as u64); + gauge!("rustfs_ec_encode_inflight_bytes_current").set(next as f64); +} + +/// Return the current tracked EC encode in-flight bytes. +#[inline(always)] +pub fn current_ec_encode_inflight_bytes() -> u64 { + EC_ENCODE_INFLIGHT_BYTES.load(Ordering::Relaxed) +} + +/// Track whole-object buffering on the GET path. +#[inline(always)] +pub fn track_get_object_buffered_bytes(bytes: usize) -> Option { + if bytes == 0 { + return None; + } + + let next = GET_OBJECT_BUFFERED_BYTES.fetch_add(bytes as u64, Ordering::Relaxed) + bytes as u64; + gauge!("rustfs_get_object_buffered_bytes_current").set(next as f64); + + Some(MemoryGaugeGuard { + gauge: TrackedMemoryGauge::GetObjectBufferedBytes, + bytes: bytes as u64, + }) +} + +/// Return the current tracked GET whole-buffered bytes. +#[inline(always)] +pub fn current_get_object_buffered_bytes() -> u64 { + GET_OBJECT_BUFFERED_BYTES.load(Ordering::Relaxed) +} + /// Record CPU usage. /// /// # Arguments @@ -758,6 +876,48 @@ mod tests { record_memory_usage(2 * 1024 * 1024 * 1024, 8 * 1024 * 1024 * 1024); } + #[test] + fn test_record_process_memory_split() { + record_process_memory_split(1024, 2048); + record_process_memory_split(4096, 8192); + } + + #[test] + fn test_record_cgroup_memory_split() { + record_cgroup_memory_split(Some(1), Some(2), Some(3), Some(4), Some(5), Some(6)); + record_cgroup_memory_split(None, None, None, None, None, None); + } + + #[test] + fn test_ec_encode_inflight_bytes_tracking() { + EC_ENCODE_INFLIGHT_BYTES.store(0, Ordering::Relaxed); + add_ec_encode_inflight_bytes(1024); + add_ec_encode_inflight_bytes(2048); + remove_ec_encode_inflight_bytes(1024); + remove_ec_encode_inflight_bytes(2048); + remove_ec_encode_inflight_bytes(4096); + assert_eq!(current_ec_encode_inflight_bytes(), 0); + } + + #[test] + fn test_get_object_buffered_bytes_guard() { + GET_OBJECT_BUFFERED_BYTES.store(0, Ordering::Relaxed); + drop(track_get_object_buffered_bytes(1024)); + let guard = track_get_object_buffered_bytes(2048); + drop(guard); + assert_eq!(current_get_object_buffered_bytes(), 0); + } + + #[test] + fn test_get_object_buffered_bytes_guard_saturates_on_underflow() { + GET_OBJECT_BUFFERED_BYTES.store(1024, Ordering::Relaxed); + drop(MemoryGaugeGuard { + gauge: TrackedMemoryGauge::GetObjectBufferedBytes, + bytes: 2048, + }); + assert_eq!(current_get_object_buffered_bytes(), 0); + } + #[test] fn test_record_cpu_usage() { record_cpu_usage(25.5); diff --git a/crates/scanner/src/lib.rs b/crates/scanner/src/lib.rs index c18ca5790..c9e8a182d 100644 --- a/crates/scanner/src/lib.rs +++ b/crates/scanner/src/lib.rs @@ -32,3 +32,26 @@ pub use data_usage_define::*; pub use error::ScannerError; pub use scanner::init_data_scanner; pub use sleeper::{DynamicSleeper, SCANNER_IDLE_MODE, SCANNER_SLEEPER}; +use std::sync::atomic::{AtomicU64, Ordering}; + +static SCANNER_ACTIVE_WORK_UNITS: AtomicU64 = AtomicU64::new(0); + +pub fn current_scanner_activity() -> u64 { + SCANNER_ACTIVE_WORK_UNITS.load(Ordering::Relaxed) +} + +pub(crate) struct ScannerActivityGuard; + +impl ScannerActivityGuard { + pub(crate) fn new() -> Self { + SCANNER_ACTIVE_WORK_UNITS.fetch_add(1, Ordering::Relaxed); + Self + } +} + +impl Drop for ScannerActivityGuard { + fn drop(&mut self) { + let _ = SCANNER_ACTIVE_WORK_UNITS + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| Some(current.saturating_sub(1))); + } +} diff --git a/crates/scanner/src/scanner.rs b/crates/scanner/src/scanner.rs index a8a2ce161..bb518037a 100644 --- a/crates/scanner/src/scanner.rs +++ b/crates/scanner/src/scanner.rs @@ -18,7 +18,7 @@ use crate::data_usage_define::{BACKGROUND_HEAL_INFO_PATH, DATA_USAGE_BLOOM_NAME_ use crate::scanner_folder::data_usage_update_dir_cycles; use crate::scanner_io::ScannerIO; use crate::sleeper::SCANNER_SLEEPER; -use crate::{DataUsageInfo, ScannerError}; +use crate::{DataUsageInfo, ScannerActivityGuard, ScannerError}; use chrono::{DateTime, Utc}; use rustfs_common::heal_channel::HealScanMode; use rustfs_common::metrics::{CurrentCycle, Metric, Metrics, emit_scan_cycle_complete, global_metrics}; @@ -183,6 +183,7 @@ fn get_lock_acquire_timeout() -> Duration { #[instrument(skip_all)] async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc, cycle_info: &mut CurrentCycle) { + let _activity_guard = ScannerActivityGuard::new(); SCANNER_SLEEPER.refresh_from_env(); info!("Start run data scanner cycle"); cycle_info.current = cycle_info.next; @@ -324,6 +325,7 @@ pub async fn store_data_usage_in_backend( let mut attempts = 1u32; while let Some(data_usage_info) = receiver.recv().await { + let _activity_guard = ScannerActivityGuard::new(); if ctx.is_cancelled() { break; } diff --git a/rustfs/Cargo.toml b/rustfs/Cargo.toml index 7edf316a8..a34d7fe15 100644 --- a/rustfs/Cargo.toml +++ b/rustfs/Cargo.toml @@ -171,6 +171,7 @@ libsystemd.workspace = true [target.'cfg(not(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")))'.dependencies] mimalloc = { workspace = true } +libmimalloc-sys = { version = "0.1.47", features = ["extended"] } diff --git a/rustfs/src/allocator_reclaim.rs b/rustfs/src/allocator_reclaim.rs new file mode 100644 index 000000000..4ed083375 --- /dev/null +++ b/rustfs/src/allocator_reclaim.rs @@ -0,0 +1,243 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![allow(unsafe_code)] + +use metrics::{counter, gauge, histogram}; +use std::time::Duration; +use tokio_util::sync::CancellationToken; +use tracing::{debug, warn}; + +pub fn allocator_backend() -> &'static str { + #[cfg(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))] + { + "jemalloc" + } + + #[cfg(all( + not(target_os = "windows"), + not(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")) + ))] + { + "mimalloc" + } + + #[cfg(target_os = "windows")] + { + "mimalloc-windows" + } +} + +fn active_requests() -> u64 { + crate::server::active_http_requests() +} + +fn current_delete_tail_activity() -> u64 { + crate::delete_tail_activity::current_delete_tail_activity() +} + +fn current_scanner_activity() -> u64 { + rustfs_scanner::current_scanner_activity() +} + +fn current_heal_activity() -> u64 { + rustfs_heal::current_heal_active_tasks() + rustfs_heal::current_heal_queue_length() +} + +#[derive(Clone, Copy, Debug, Default)] +struct ReclaimableWorkSnapshot { + active_requests: u64, + delete_tail_activity: u64, + scanner_activity: u64, + heal_activity: u64, + ec_inflight_bytes: u64, + get_buffered_bytes: u64, +} + +impl ReclaimableWorkSnapshot { + fn active_signal_count(self) -> u64 { + u64::from(self.active_requests > 0) + + u64::from(self.delete_tail_activity > 0) + + u64::from(self.scanner_activity > 0) + + u64::from(self.heal_activity > 0) + + u64::from(self.ec_inflight_bytes > 0) + + u64::from(self.get_buffered_bytes > 0) + } +} + +fn reclaimable_work_snapshot() -> ReclaimableWorkSnapshot { + ReclaimableWorkSnapshot { + active_requests: active_requests(), + delete_tail_activity: current_delete_tail_activity(), + scanner_activity: current_scanner_activity(), + heal_activity: current_heal_activity(), + ec_inflight_bytes: rustfs_io_metrics::current_ec_encode_inflight_bytes(), + get_buffered_bytes: rustfs_io_metrics::current_get_object_buffered_bytes(), + } +} + +#[cfg(all( + not(target_os = "windows"), + not(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")) +))] +fn collect_allocator_memory(force: bool) -> Result<(), String> { + // SAFETY: `mi_collect` is provided by the active global allocator backend + // on this target family. It is explicitly intended to reclaim retained + // pages/segments and does not require additional invariants from the caller. + unsafe { + libmimalloc_sys::mi_collect(force); + } + Ok(()) +} + +#[cfg(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))] +fn collect_allocator_memory(_force: bool) -> Result<(), String> { + #[cfg(not(target_os = "macos"))] + let _ = tikv_jemalloc_ctl::background_thread::write(true); + tikv_jemalloc_ctl::epoch::advance().map_err(|err| err.to_string())?; + Ok(()) +} + +#[cfg(target_os = "windows")] +fn collect_allocator_memory(_force: bool) -> Result<(), String> { + Err("allocator reclaim is not supported on Windows".to_string()) +} + +fn run_allocator_reclaim(force: bool) { + let backend = allocator_backend(); + let start = std::time::Instant::now(); + + match collect_allocator_memory(force) { + Ok(()) => { + counter!("rustfs_memory_allocator_reclaim_total", "backend" => backend.to_string(), "result" => "ok".to_string()) + .increment(1); + histogram!( + "rustfs_memory_allocator_reclaim_duration_seconds", + "backend" => backend.to_string(), + "result" => "ok".to_string() + ) + .record(start.elapsed().as_secs_f64()); + } + Err(err) => { + counter!( + "rustfs_memory_allocator_reclaim_total", + "backend" => backend.to_string(), + "result" => "err".to_string() + ) + .increment(1); + warn!(backend, force, error = %err, "allocator reclaim failed"); + } + } +} + +pub fn init_allocator_reclaim(ctx: CancellationToken) { + let backend = allocator_backend(); + let enabled = rustfs_utils::get_env_bool( + rustfs_config::ENV_ALLOCATOR_RECLAIM_ENABLED, + rustfs_config::DEFAULT_ALLOCATOR_RECLAIM_ENABLED, + ); + gauge!("rustfs_memory_allocator_reclaim_enabled").set(if enabled { 1.0 } else { 0.0 }); + counter!("rustfs_memory_allocator_backend_info", "backend" => backend.to_string()).increment(1); + + if !enabled { + debug!("allocator reclaim loop disabled"); + return; + } + + let configured_force = + rustfs_utils::get_env_bool(rustfs_config::ENV_ALLOCATOR_RECLAIM_FORCE, rustfs_config::DEFAULT_ALLOCATOR_RECLAIM_FORCE); + let force = if backend == "jemalloc" && configured_force { + warn!( + backend, + env = rustfs_config::ENV_ALLOCATOR_RECLAIM_FORCE, + "allocator reclaim force mode is not supported on jemalloc backend; ignoring configured force flag" + ); + false + } else { + configured_force + }; + let idle_intervals = rustfs_utils::get_env_u64( + rustfs_config::ENV_ALLOCATOR_RECLAIM_IDLE_INTERVALS, + rustfs_config::DEFAULT_ALLOCATOR_RECLAIM_IDLE_INTERVALS, + ) + .max(1); + let interval = Duration::from_secs( + rustfs_utils::get_env_u64( + rustfs_config::ENV_ALLOCATOR_RECLAIM_INTERVAL_SECS, + rustfs_config::DEFAULT_ALLOCATOR_RECLAIM_INTERVAL_SECS, + ) + .max(1), + ); + + tokio::spawn(async move { + let mut ticker = tokio::time::interval(interval); + ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + let mut idle_streak = 0_u64; + + loop { + tokio::select! { + _ = ctx.cancelled() => { + debug!("allocator reclaim loop cancelled"); + break; + } + _ = ticker.tick() => { + let snapshot = reclaimable_work_snapshot(); + let active_signal_count = snapshot.active_signal_count(); + gauge!("rustfs_memory_allocator_reclaim_active_requests").set(snapshot.active_requests as f64); + gauge!("rustfs_memory_allocator_reclaim_delete_tail_activity_current").set(snapshot.delete_tail_activity as f64); + gauge!("rustfs_memory_allocator_reclaim_scanner_activity_current").set(snapshot.scanner_activity as f64); + gauge!("rustfs_memory_allocator_reclaim_heal_activity_current").set(snapshot.heal_activity as f64); + gauge!("rustfs_memory_allocator_reclaim_ec_inflight_bytes_current").set(snapshot.ec_inflight_bytes as f64); + gauge!("rustfs_memory_allocator_reclaim_get_buffered_bytes_current").set(snapshot.get_buffered_bytes as f64); + gauge!("rustfs_memory_allocator_reclaim_reclaimable_work_current").set(active_signal_count as f64); + if active_signal_count == 0 { + idle_streak = idle_streak.saturating_add(1); + gauge!("rustfs_memory_allocator_reclaim_idle_streak").set(idle_streak as f64); + } else { + idle_streak = 0; + gauge!("rustfs_memory_allocator_reclaim_idle_streak").set(0.0); + } + + if idle_streak >= idle_intervals { + run_allocator_reclaim(force); + idle_streak = 0; + gauge!("rustfs_memory_allocator_reclaim_idle_streak").set(0.0); + } else { + let reason = if active_signal_count > 0 { + "work_inflight" + } else { + "idle_window" + }; + counter!("rustfs_memory_allocator_reclaim_skipped_total", "reason" => reason.to_string()).increment(1); + } + } + } + } + }); +} + +#[cfg(test)] +mod tests { + use super::{allocator_backend, reclaimable_work_snapshot}; + + #[test] + fn allocator_backend_name_is_available() { + assert!(!allocator_backend().is_empty()); + } + + #[test] + fn reclaimable_work_snapshot_is_collectable() { + let _ = reclaimable_work_snapshot(); + } +} diff --git a/rustfs/src/app/object_usecase.rs b/rustfs/src/app/object_usecase.rs index 02d70e870..dee317396 100644 --- a/rustfs/src/app/object_usecase.rs +++ b/rustfs/src/app/object_usecase.rs @@ -16,6 +16,7 @@ use crate::app::context::{AppContext, default_notify_interface, get_global_app_context}; use crate::config::RustFSBufferConfig; +use crate::delete_tail_activity::{DeleteTailActivityGuard, DeleteTailStage}; use crate::error::ApiError; use crate::storage::access::{PostObjectRequestMarker, authorize_request, has_bypass_governance_header, req_info_mut}; use crate::storage::concurrency::{ @@ -36,7 +37,7 @@ use bytes::Bytes; use datafusion::arrow::{ csv::WriterBuilder as CsvWriterBuilder, json::WriterBuilder as JsonWriterBuilder, json::writer::JsonArray, }; -use futures::{StreamExt, stream}; +use futures::StreamExt; use http::{HeaderMap, HeaderValue, StatusCode}; use md5::Context as Md5Context; use metrics::{counter, histogram}; @@ -118,7 +119,7 @@ use std::collections::HashMap; use std::ops::Add; use std::path::Path; use std::str::FromStr; -use std::sync::{Arc, Mutex}; +use std::sync::{Arc, Mutex, OnceLock}; use std::time::Duration; use time::{OffsetDateTime, format_description::well_known::Rfc3339}; use tokio::io::{AsyncRead, ReadBuf}; @@ -213,6 +214,7 @@ async fn enqueue_transitioned_delete_cleanup(bucket: &str, object: &str, opts: & let Some(existing) = existing else { return; }; + let _activity_guard = DeleteTailActivityGuard::new(DeleteTailStage::Cleanup); let je = if opts.delete_prefix { rustfs_ecstore::bucket::lifecycle::tier_sweeper::transitioned_force_delete_journal_entry(&existing.transitioned_object) @@ -255,6 +257,38 @@ pin_project! { } } +pin_project! { + struct MemoryTrackedBytesStream { + bytes: Bytes, + emitted: bool, + _guard: Option, + } +} + +impl MemoryTrackedBytesStream { + fn new(bytes: Bytes, guard: Option) -> Self { + Self { + bytes, + emitted: false, + _guard: guard, + } + } +} + +impl futures::Stream for MemoryTrackedBytesStream { + type Item = std::io::Result; + + fn poll_next(self: std::pin::Pin<&mut Self>, _cx: &mut std::task::Context<'_>) -> std::task::Poll> { + let this = self.project(); + if *this.emitted { + return std::task::Poll::Ready(None); + } + + *this.emitted = true; + std::task::Poll::Ready(Some(Ok(this.bytes.clone()))) + } +} + impl ExtractArchiveEtagReader { fn new(inner: R, etag: Arc>>) -> Self { Self { @@ -348,6 +382,16 @@ fn should_use_zero_copy(size: i64, headers: &HeaderMap) -> bool { true } +fn object_seek_support_threshold() -> usize { + static OBJECT_SEEK_SUPPORT_THRESHOLD: OnceLock = OnceLock::new(); + *OBJECT_SEEK_SUPPORT_THRESHOLD.get_or_init(|| { + rustfs_utils::get_env_usize( + rustfs_config::ENV_OBJECT_SEEK_SUPPORT_THRESHOLD, + rustfs_config::DEFAULT_OBJECT_SEEK_SUPPORT_THRESHOLD, + ) + }) +} + #[cfg(test)] mod deadlock_request_guard_tests { use super::DeadlockRequestGuard; @@ -994,8 +1038,10 @@ impl DefaultObjectUsecase { } fn build_memory_blob(buf: Vec, response_content_length: i64, _optimal_buffer_size: usize) -> Option { + let guard = rustfs_io_metrics::track_get_object_buffered_bytes(buf.len()); + let bytes = Bytes::from(buf); Some(StreamingBlob::wrap(bytes_stream( - stream::once(async move { Ok::(Bytes::from(buf)) }), + MemoryTrackedBytesStream::new(bytes, guard), response_content_length as usize, ))) } @@ -1211,11 +1257,9 @@ impl DefaultObjectUsecase { let info = reader.object_info; - use rustfs_io_metrics::{record_memory_copy_saved, record_zero_copy_read}; + use rustfs_io_metrics::record_zero_copy_read; let read_duration = read_start.elapsed(); - let estimated_saved = (info.size * 2) as usize; record_zero_copy_read(info.size as usize, read_duration.as_secs_f64() * 1000.0); - record_memory_copy_saved(estimated_saved); manager.record_disk_operation(info.size as u64, read_duration, true).await; @@ -1483,7 +1527,7 @@ impl DefaultObjectUsecase { R: AsyncRead + Send + Sync + Unpin + 'static, { if encryption_applied { - let seekable_object_size_threshold = rustfs_config::DEFAULT_OBJECT_SEEK_SUPPORT_THRESHOLD; + let seekable_object_size_threshold = object_seek_support_threshold(); let should_buffer_encrypted_object = response_content_length > 0 && response_content_length <= seekable_object_size_threshold as i64 && part_number.is_none() @@ -1514,7 +1558,7 @@ impl DefaultObjectUsecase { return Ok(Self::build_reader_blob(final_stream, response_content_length, optimal_buffer_size)); } - let seekable_object_size_threshold = rustfs_config::DEFAULT_OBJECT_SEEK_SUPPORT_THRESHOLD; + let seekable_object_size_threshold = object_seek_support_threshold(); let should_provide_seek_support = response_content_length > 0 && response_content_length <= seekable_object_size_threshold as i64 && part_number.is_none() @@ -2114,9 +2158,9 @@ impl DefaultObjectUsecase { let request_id = req .extensions - .get::() + .get::() .map(|ctx| ctx.request_id.clone()) - .unwrap_or_else(|| crate::storage::request_context::RequestContext::fallback().request_id); + .unwrap_or_else(|| request_context::RequestContext::fallback().request_id); let bootstrap = Self::init_get_object_bootstrap(&req.input.bucket, &req.input.key, &request_id)?; let timeout_config = bootstrap.timeout_config; let wrapper = bootstrap.wrapper; @@ -3083,6 +3127,7 @@ impl DefaultObjectUsecase { && (dobj.delete_marker_replication_status() == ReplicationStatusType::Pending || dobj.version_purge_status() == VersionPurgeStatusType::Pending) { + let _activity_guard = DeleteTailActivityGuard::new(DeleteTailStage::Replication); let mut dobj = dobj.clone(); if is_dir_object(dobj.object_name.as_str()) && dobj.version_id.is_none() { dobj.version_id = Some(Uuid::nil()); @@ -3104,11 +3149,9 @@ impl DefaultObjectUsecase { .as_ref() .map(|context| context.notify()) .unwrap_or_else(default_notify_interface); - let request_context = req - .extensions - .get::() - .cloned(); + let request_context = req.extensions.get::().cloned(); spawn_background_with_context(request_context, async move { + let _activity_guard = DeleteTailActivityGuard::new(DeleteTailStage::Notify); for res in delete_results { if let Some(dobj) = res.delete_object { let event_name = if dobj.delete_marker { @@ -3298,6 +3341,7 @@ impl DefaultObjectUsecase { let deleted_replication_info = existing_object_info .as_ref() .filter(|_| should_use_existing_delete_replication_info(&opts)); + let _delete_tail_guard = DeleteTailActivityGuard::new(DeleteTailStage::Tail); let deleted_object_source = deleted_replication_info.unwrap_or(&obj_info); let replication_state_source = delete_replication_state_source(&opts, existing_object_info.as_ref(), deleted_object_source); @@ -3311,6 +3355,7 @@ impl DefaultObjectUsecase { }; if schedule_delete_replication { + let _activity_guard = DeleteTailActivityGuard::new(DeleteTailStage::Replication); let mut deleted_object = DeletedObjectReplicationInfo { delete_object: rustfs_ecstore::store_api::DeletedObject { delete_marker: deleted_object_source.delete_marker && !deleted_delete_marker_version, @@ -4321,10 +4366,7 @@ impl DefaultObjectUsecase { }; let notify = notify.clone(); - let request_context = req - .extensions - .get::() - .cloned(); + let request_context = req.extensions.get::().cloned(); spawn_background_with_context(request_context, async move { notify.notify(event_args).await; }); diff --git a/rustfs/src/delete_tail_activity.rs b/rustfs/src/delete_tail_activity.rs new file mode 100644 index 000000000..08c099bf2 --- /dev/null +++ b/rustfs/src/delete_tail_activity.rs @@ -0,0 +1,115 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use metrics::{counter, gauge, histogram}; +use std::sync::atomic::{AtomicU64, Ordering}; + +static DELETE_TAIL_TOTAL: AtomicU64 = AtomicU64::new(0); +static DELETE_CLEANUP_TOTAL: AtomicU64 = AtomicU64::new(0); +static DELETE_REPLICATION_TOTAL: AtomicU64 = AtomicU64::new(0); +static DELETE_NOTIFY_TOTAL: AtomicU64 = AtomicU64::new(0); + +#[derive(Clone, Copy, Debug)] +pub enum DeleteTailStage { + Tail, + Cleanup, + Replication, + Notify, +} + +impl DeleteTailStage { + const fn as_str(self) -> &'static str { + match self { + Self::Tail => "tail", + Self::Cleanup => "cleanup", + Self::Replication => "replication", + Self::Notify => "notify", + } + } +} + +fn stage_counter(stage: DeleteTailStage) -> &'static AtomicU64 { + match stage { + DeleteTailStage::Tail => &DELETE_TAIL_TOTAL, + DeleteTailStage::Cleanup => &DELETE_CLEANUP_TOTAL, + DeleteTailStage::Replication => &DELETE_REPLICATION_TOTAL, + DeleteTailStage::Notify => &DELETE_NOTIFY_TOTAL, + } +} + +#[derive(Debug)] +pub struct DeleteTailActivityGuard { + stage: DeleteTailStage, + started_at: std::time::Instant, +} + +impl DeleteTailActivityGuard { + pub fn new(stage: DeleteTailStage) -> Self { + let total = stage_counter(stage).fetch_add(1, Ordering::Relaxed) + 1; + gauge!( + "rustfs_delete_tail_activity_inflight_current", + "stage" => stage.as_str().to_string() + ) + .set(total as f64); + gauge!("rustfs_delete_tail_activity_total_inflight_current").set(current_delete_tail_activity() as f64); + counter!( + "rustfs_delete_tail_activity_started_total", + "stage" => stage.as_str().to_string() + ) + .increment(1); + Self { + stage, + started_at: std::time::Instant::now(), + } + } +} + +impl Drop for DeleteTailActivityGuard { + fn drop(&mut self) { + let previous = stage_counter(self.stage).fetch_sub(1, Ordering::Relaxed); + let next = previous.saturating_sub(1); + gauge!( + "rustfs_delete_tail_activity_inflight_current", + "stage" => self.stage.as_str().to_string() + ) + .set(next as f64); + gauge!("rustfs_delete_tail_activity_total_inflight_current").set(current_delete_tail_activity() as f64); + histogram!( + "rustfs_delete_tail_activity_duration_seconds", + "stage" => self.stage.as_str().to_string() + ) + .record(self.started_at.elapsed().as_secs_f64()); + } +} + +pub fn current_delete_tail_activity() -> u64 { + DELETE_TAIL_TOTAL.load(Ordering::Relaxed) + + DELETE_CLEANUP_TOTAL.load(Ordering::Relaxed) + + DELETE_REPLICATION_TOTAL.load(Ordering::Relaxed) + + DELETE_NOTIFY_TOTAL.load(Ordering::Relaxed) +} + +#[cfg(test)] +mod tests { + use super::{DeleteTailActivityGuard, DeleteTailStage, current_delete_tail_activity}; + + #[test] + fn delete_tail_activity_guard_tracks_total_activity() { + let before = current_delete_tail_activity(); + let guard = DeleteTailActivityGuard::new(DeleteTailStage::Cleanup); + assert_eq!(current_delete_tail_activity(), before + 1); + drop(guard); + assert_eq!(current_delete_tail_activity(), before); + } +} diff --git a/rustfs/src/lib.rs b/rustfs/src/lib.rs index 44aef7b1c..007adbf36 100644 --- a/rustfs/src/lib.rs +++ b/rustfs/src/lib.rs @@ -51,15 +51,18 @@ //! tests, and then shut it down. pub mod admin; +pub mod allocator_reclaim; pub mod app; pub mod auth; pub mod auth_keystone; pub mod capacity; pub mod config; +pub mod delete_tail_activity; pub mod embedded; pub mod error; pub mod init; pub mod license; +pub mod memory_observability; pub mod profiling; #[cfg(any(feature = "ftps", feature = "webdav"))] pub mod protocols; diff --git a/rustfs/src/main.rs b/rustfs/src/main.rs index aad598a7c..f8634abc2 100644 --- a/rustfs/src/main.rs +++ b/rustfs/src/main.rs @@ -560,10 +560,12 @@ async fn run(config: rustfs::config::Config) -> Result<()> { print_server_info(); init_update_check(); + rustfs::allocator_reclaim::init_allocator_reclaim(ctx.clone()); if rustfs_obs::observability_metric_enabled() { // Initialize metrics system init_metrics_runtime(ctx.clone()); + rustfs::memory_observability::init_memory_observability(ctx.clone()); // Initialize auto-tuner for performance optimization (optional) rustfs::init::init_auto_tuner(ctx.clone()).await; diff --git a/rustfs/src/memory_observability.rs b/rustfs/src/memory_observability.rs new file mode 100644 index 000000000..ffec18a77 --- /dev/null +++ b/rustfs/src/memory_observability.rs @@ -0,0 +1,209 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use rustfs_io_metrics::{ + record_cgroup_memory_split, record_cpu_usage, record_memory_usage, record_process_memory_split, + snapshot_process_resource_and_system, +}; +use std::collections::HashMap; +use std::path::Path; +use std::sync::{Mutex, OnceLock}; +use std::time::Duration; +use sysinfo::System; +use tokio_util::sync::CancellationToken; +use tracing::debug; + +static MEMORY_SYSTEM: OnceLock> = OnceLock::new(); + +const ENV_MEMORY_OBSERVABILITY_INTERVAL_SECS: &str = "RUSTFS_MEMORY_OBSERVABILITY_INTERVAL_SECS"; +const DEFAULT_MEMORY_OBSERVABILITY_INTERVAL_SECS: u64 = 15; + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +struct CgroupMemorySnapshot { + current_bytes: Option, + limit_bytes: Option, + anon_bytes: Option, + file_bytes: Option, + active_file_bytes: Option, + inactive_file_bytes: Option, +} + +fn memory_system() -> &'static Mutex { + MEMORY_SYSTEM.get_or_init(|| Mutex::new(System::new())) +} + +fn refresh_total_memory() -> u64 { + let mut system = memory_system().lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + system.refresh_memory(); + system.total_memory() +} + +fn read_optional_u64(path: &Path) -> Option { + let content = std::fs::read_to_string(path).ok()?; + let trimmed = content.trim(); + if trimmed.is_empty() || trimmed == "max" { + return None; + } + trimmed.parse::().ok() +} + +fn parse_kv_stats(content: &str) -> HashMap { + content + .lines() + .filter_map(|line| { + let mut parts = line.split_whitespace(); + let key = parts.next()?; + let value = parts.next()?.parse::().ok()?; + Some((key.to_string(), value)) + }) + .collect() +} + +fn read_cgroup_v2() -> Option { + let root = Path::new("/sys/fs/cgroup"); + let stat_path = root.join("memory.stat"); + if !stat_path.exists() { + return None; + } + + let stats = parse_kv_stats(&std::fs::read_to_string(&stat_path).ok()?); + Some(CgroupMemorySnapshot { + current_bytes: read_optional_u64(&root.join("memory.current")), + limit_bytes: read_optional_u64(&root.join("memory.max")), + anon_bytes: stats.get("anon").copied(), + file_bytes: stats.get("file").copied(), + active_file_bytes: stats.get("active_file").copied(), + inactive_file_bytes: stats.get("inactive_file").copied(), + }) +} + +fn read_cgroup_v1() -> Option { + let root = Path::new("/sys/fs/cgroup/memory"); + let stat_path = root.join("memory.stat"); + if !stat_path.exists() { + return None; + } + + let stats = parse_kv_stats(&std::fs::read_to_string(&stat_path).ok()?); + Some(CgroupMemorySnapshot { + current_bytes: read_optional_u64(&root.join("memory.usage_in_bytes")), + limit_bytes: read_optional_u64(&root.join("memory.limit_in_bytes")), + anon_bytes: stats.get("total_rss").copied().or_else(|| stats.get("rss").copied()), + file_bytes: stats.get("total_cache").copied().or_else(|| stats.get("cache").copied()), + active_file_bytes: stats + .get("total_active_file") + .copied() + .or_else(|| stats.get("active_file").copied()), + inactive_file_bytes: stats + .get("total_inactive_file") + .copied() + .or_else(|| stats.get("inactive_file").copied()), + }) +} + +fn read_cgroup_memory_snapshot() -> Option { + read_cgroup_v2().or_else(read_cgroup_v1) +} + +async fn record_memory_snapshot() { + match tokio::task::spawn_blocking(|| { + let (resource, process) = snapshot_process_resource_and_system(); + let total_memory = refresh_total_memory(); + let cgroup = read_cgroup_memory_snapshot(); + (resource, process, total_memory, cgroup) + }) + .await + { + Ok((resource, process, total_memory, cgroup)) => { + record_memory_usage(process.resident_memory_bytes, total_memory); + record_cpu_usage(resource.cpu_percent); + record_process_memory_split(process.resident_memory_bytes, process.virtual_memory_bytes); + + if let Some(cgroup) = cgroup { + record_cgroup_memory_split( + cgroup.current_bytes, + cgroup.limit_bytes, + cgroup.anon_bytes, + cgroup.file_bytes, + cgroup.active_file_bytes, + cgroup.inactive_file_bytes, + ); + } + } + Err(err) => { + debug!(error = ?err, "memory observability sampler task failed"); + } + } +} + +pub fn init_memory_observability(ctx: CancellationToken) { + let interval_secs = + rustfs_utils::get_env_u64(ENV_MEMORY_OBSERVABILITY_INTERVAL_SECS, DEFAULT_MEMORY_OBSERVABILITY_INTERVAL_SECS); + let interval = Duration::from_secs(interval_secs.max(1)); + + tokio::spawn(async move { + let mut ticker = tokio::time::interval(interval); + ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + + loop { + tokio::select! { + _ = ctx.cancelled() => { + debug!("memory observability sampler cancelled"); + break; + } + _ = ticker.tick() => { + record_memory_snapshot().await; + } + } + } + }); +} + +#[cfg(test)] +mod tests { + use super::{CgroupMemorySnapshot, parse_kv_stats, read_optional_u64}; + use std::fs; + use std::path::PathBuf; + + #[test] + fn parse_kv_stats_extracts_numeric_pairs() { + let parsed = parse_kv_stats("anon 12\nfile 34\nactive_file 56\n"); + assert_eq!(parsed.get("anon").copied(), Some(12)); + assert_eq!(parsed.get("file").copied(), Some(34)); + assert_eq!(parsed.get("active_file").copied(), Some(56)); + } + + #[test] + fn read_optional_u64_parses_numeric_and_max_values() { + let tempdir = tempfile::tempdir().expect("tempdir"); + let value_path: PathBuf = tempdir.path().join("value"); + let max_path: PathBuf = tempdir.path().join("max"); + fs::write(&value_path, "123\n").expect("write numeric"); + fs::write(&max_path, "max\n").expect("write max"); + + assert_eq!(read_optional_u64(&value_path), Some(123)); + assert_eq!(read_optional_u64(&max_path), None); + } + + #[test] + fn cgroup_memory_snapshot_defaults_are_empty() { + let snapshot = CgroupMemorySnapshot::default(); + assert_eq!(snapshot.current_bytes, None); + assert_eq!(snapshot.limit_bytes, None); + assert_eq!(snapshot.anon_bytes, None); + assert_eq!(snapshot.file_bytes, None); + assert_eq!(snapshot.active_file_bytes, None); + assert_eq!(snapshot.inactive_file_bytes, None); + } +} diff --git a/rustfs/src/server/http.rs b/rustfs/src/server/http.rs index 6a54998cb..89ce6278c 100644 --- a/rustfs/src/server/http.rs +++ b/rustfs/src/server/http.rs @@ -122,6 +122,10 @@ fn record_active_http_requests(delta: i64) { gauge!(METRIC_HTTP_SERVER_ACTIVE_REQUESTS).set(next as f64); } +pub(crate) fn active_http_requests() -> u64 { + ACTIVE_HTTP_REQUESTS.load(Ordering::Relaxed) +} + pub async fn start_http_server( config: &config::Config, readiness: Arc, diff --git a/rustfs/src/server/mod.rs b/rustfs/src/server/mod.rs index 680bd2cae..dec4d90bc 100644 --- a/rustfs/src/server/mod.rs +++ b/rustfs/src/server/mod.rs @@ -39,6 +39,7 @@ pub use service_state::ShutdownSignal; pub use service_state::wait_for_shutdown; // Items only used within the library crate (admin handlers, server/http.rs, etc.). +pub(crate) use http::active_http_requests; pub(crate) use module_switch::{ ModuleSwitchSnapshot, ModuleSwitchSource, PersistedModuleSwitches, current_module_switch_snapshot, refresh_persisted_module_switches_from_store, save_persisted_module_switches_to_store, validate_module_switch_update, diff --git a/scripts/run.sh b/scripts/run.sh index de7435ae0..bb57cf98d 100755 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -52,6 +52,10 @@ if [ -z "${RUSTFS_UNSAFE_BYPASS_DISK_CHECK+x}" ] && [ -z "${MINIO_CI+x}" ]; then export RUSTFS_UNSAFE_BYPASS_DISK_CHECK=true fi +if [ -z "${RUSTFS_ALLOCATOR_RECLAIM_ENABLED+x}" ]; then + export RUSTFS_ALLOCATOR_RECLAIM_ENABLED=true +fi + export RUSTFS_VOLUMES="./target/volume/test{1...4}" # export RUSTFS_VOLUMES="./target/volume/test" export RUSTFS_ADDRESS=":9000" @@ -71,14 +75,14 @@ export RUSTFS_OBS_ENDPOINT=http://localhost:4318 # OpenTelemetry Collector addre #export RUSTFS_OBS_LOG_ENDPOINT=http://loki:3100/otlp/v1/logs # OpenTelemetry Collector logs address http://loki:3100/otlp/v1/logs #export OTEL_EXPORTER_OTLP_LOGS_ENDPOINT=http://loki:3100/otlp/v1/logs export RUSTFS_OBS_PROFILING_ENDPOINT=http://localhost:4040 # OpenTelemetry Collector profiling address -export RUSTFS_OBS_USE_STDOUT=true # Whether to use standard output +export RUSTFS_OBS_USE_STDOUT=false # Whether to use standard output export RUSTFS_OBS_SAMPLE_RATIO=2.0 # Sample ratio, between 0.0-1.0, 0.0 means no sampling, 1.0 means full sampling export RUSTFS_OBS_METER_INTERVAL=1 # Sampling interval in seconds export RUSTFS_OBS_SERVICE_NAME=rustfs # Service name export RUSTFS_OBS_SERVICE_VERSION=0.1.0 # Service version export RUSTFS_OBS_ENVIRONMENT=production # Environment name development, staging, production export RUSTFS_OBS_LOGGER_LEVEL=info # Log level, supports trace, debug, info, warn, error -export RUSTFS_OBS_LOG_STDOUT_ENABLED=true # Whether to enable local stdout logging +export RUSTFS_OBS_LOG_STDOUT_ENABLED=false # Whether to enable local stdout logging export RUSTFS_OBS_LOG_DIRECTORY="$current_dir/deploy/logs" # Log directory export RUSTFS_OBS_LOG_ROTATION_TIME="minutely" # Log rotation time unit, can be "minutely", "hourly", "daily" export RUSTFS_OBS_LOG_KEEP_FILES=10 # Number of log files to keep @@ -527,4 +531,7 @@ fi #cargo run --profile release --bin rustfs # To run in debug mode, use the following line -cargo run --bin rustfs +#cargo run --bin rustfs + +# Default local run mode: release +cargo run --profile release --bin rustfs diff --git a/scripts/run_issue_2573_acceptance.sh b/scripts/run_issue_2573_acceptance.sh new file mode 100755 index 000000000..931debdb5 --- /dev/null +++ b/scripts/run_issue_2573_acceptance.sh @@ -0,0 +1,257 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Issue 2573 acceptance runner +# Runs the key workload profiles discussed in docs/tasks/issue-2573/05-benchmark-and-acceptance.md +# and samples the RustFS process RSS during load and cooldown. + +WARP_BIN="${WARP_BIN:-warp}" +HOST="${HOST:-http://127.0.0.1:9000}" +ACCESS_KEY="${ACCESS_KEY:-rustfsadmin}" +SECRET_KEY="${SECRET_KEY:-rustfsadmin}" +BUCKET="${BUCKET:-rustfs-issue-2573}" +REGION="${REGION:-us-east-1}" +CONCURRENCY="${CONCURRENCY:-30}" +DURATION="${DURATION:-60s}" +COOLDOWN_SECS="${COOLDOWN_SECS:-180}" +SAMPLE_SECS="${SAMPLE_SECS:-1}" +RUSTFS_PID="${RUSTFS_PID:-}" +OUT_DIR="${OUT_DIR:-target/bench/issue-2573-acceptance-$(date +%Y%m%d-%H%M%S)}" +INSECURE="${INSECURE:-false}" + +usage() { + cat <<'USAGE' +Usage: + scripts/run_issue_2573_acceptance.sh [options] + +Options: + --warp-bin warp binary (default: warp) + --host S3 endpoint; accepts either URL or host:port (default: http://127.0.0.1:9000) + --access-key access key (default: rustfsadmin) + --secret-key secret key (default: rustfsadmin) + --bucket bucket name (default: rustfs-issue-2573) + --region region (default: us-east-1) + --concurrency warp concurrency (default: 30) + --duration warp duration per profile (default: 60s) + --cooldown-secs cooldown sampling after each profile (default: 180) + --sample-secs RSS sample interval seconds (default: 1) + --pid rustfs process pid (optional; auto-detect if omitted) + --out-dir output directory + --insecure pass --insecure to warp + -h, --help show help + +Profiles executed: + 1. 4KiB mixed + 2. 11MiB mixed + 3. 11MiB delete +USAGE +} + +require_cmd() { + if ! command -v "$1" >/dev/null 2>&1; then + echo "ERROR: command not found: $1" >&2 + exit 1 + fi +} + +parse_args() { + while [[ $# -gt 0 ]]; do + case "$1" in + --warp-bin) WARP_BIN="$2"; shift 2 ;; + --host) HOST="$2"; shift 2 ;; + --access-key) ACCESS_KEY="$2"; shift 2 ;; + --secret-key) SECRET_KEY="$2"; shift 2 ;; + --bucket) BUCKET="$2"; shift 2 ;; + --region) REGION="$2"; shift 2 ;; + --concurrency) CONCURRENCY="$2"; shift 2 ;; + --duration) DURATION="$2"; shift 2 ;; + --cooldown-secs) COOLDOWN_SECS="$2"; shift 2 ;; + --sample-secs) SAMPLE_SECS="$2"; shift 2 ;; + --pid) RUSTFS_PID="$2"; shift 2 ;; + --out-dir) OUT_DIR="$2"; shift 2 ;; + --insecure) INSECURE=true; shift ;; + -h|--help) usage; exit 0 ;; + *) + echo "ERROR: unknown arg: $1" >&2 + usage + exit 1 + ;; + esac + done +} + +resolve_pid() { + if [[ -n "$RUSTFS_PID" ]]; then + echo "$RUSTFS_PID" + return + fi + + local pid + pid="$(pgrep -n rustfs || true)" + if [[ -z "$pid" ]]; then + echo "" + return + fi + echo "$pid" +} + +normalize_warp_host() { + local raw="$1" + # Strip scheme when a URL is provided. + raw="${raw#http://}" + raw="${raw#https://}" + # Remove any path/query/fragment to satisfy warp's --host requirements. + raw="${raw%%/*}" + raw="${raw%%\?*}" + raw="${raw%%\#*}" + echo "$raw" +} + +sample_rss_loop() { + local pid="$1" + local out_file="$2" + if [[ -z "$pid" ]]; then + return 0 + fi + local started_at + started_at="$(date +%s)" + + echo "timestamp,elapsed_seconds,rss_kib,vsz_kib" > "$out_file" + while kill -0 "$pid" >/dev/null 2>&1; do + local now elapsed sample + now="$(date +%s)" + elapsed="$((now - started_at))" + sample="$(ps -o rss=,vsz= -p "$pid" | awk 'NF>=2 {print $1","$2}')" + if [[ -n "$sample" ]]; then + echo "$(date +%Y-%m-%dT%H:%M:%S),${elapsed},${sample}" >> "$out_file" + fi + sleep "$SAMPLE_SECS" + done +} + +sample_rss_window() { + local pid="$1" + local seconds="$2" + local out_file="$3" + if [[ -z "$pid" ]]; then + return 0 + fi + local started_at deadline + started_at="$(date +%s)" + deadline="$((started_at + seconds))" + + echo "timestamp,elapsed_seconds,rss_kib,vsz_kib" > "$out_file" + while true; do + local now elapsed sample + now="$(date +%s)" + if (( now > deadline )); then + break + fi + elapsed="$((now - started_at))" + if ! kill -0 "$pid" >/dev/null 2>&1; then + break + fi + sample="$(ps -o rss=,vsz= -p "$pid" | awk 'NF>=2 {print $1","$2}')" + if [[ -n "$sample" ]]; then + echo "$(date +%Y-%m-%dT%H:%M:%S),${elapsed},${sample}" >> "$out_file" + fi + sleep "$SAMPLE_SECS" + done +} + +run_profile() { + local profile_name="$1" + local mode="$2" + local obj_size="$3" + local pid="$4" + local warp_host + warp_host="$(normalize_warp_host "$HOST")" + local benchdata="$OUT_DIR/${profile_name// /-}" + local warp_log="$OUT_DIR/${profile_name// /-}.warp.log" + local rss_during="$OUT_DIR/${profile_name// /-}.rss_during.csv" + local rss_cooldown="$OUT_DIR/${profile_name// /-}.rss_cooldown.csv" + + local -a cmd=( + "$WARP_BIN" "$mode" + "--host" "$warp_host" + "--access-key" "$ACCESS_KEY" + "--secret-key" "$SECRET_KEY" + "--bucket" "$BUCKET" + "--region" "$REGION" + "--obj.size" "$obj_size" + "--concurrent" "$CONCURRENCY" + "--duration" "$DURATION" + "--benchdata" "$benchdata" + ) + if [[ "$INSECURE" == "true" ]]; then + cmd+=("--insecure") + fi + + echo "==== Running profile: $profile_name ====" + printf 'Command:' + printf ' %q' "${cmd[@]}" + printf '\n' + + local sampler_pid="" + if [[ -n "$pid" ]]; then + sample_rss_loop "$pid" "$rss_during" & + sampler_pid=$! + else + echo "WARN: rustfs pid unavailable; skipping RSS sampling for $profile_name" >&2 + fi + if ! "${cmd[@]}" 2>&1 | tee "$warp_log"; then + echo "ERROR: profile failed: $profile_name" >&2 + if [[ -n "$sampler_pid" ]]; then + kill "$sampler_pid" >/dev/null 2>&1 || true + wait "$sampler_pid" >/dev/null 2>&1 || true + fi + exit 1 + fi + if [[ -n "$sampler_pid" ]]; then + kill "$sampler_pid" >/dev/null 2>&1 || true + wait "$sampler_pid" >/dev/null 2>&1 || true + fi + + echo "==== Cooldown sampling: $profile_name ($COOLDOWN_SECS s) ====" + sample_rss_window "$pid" "$COOLDOWN_SECS" "$rss_cooldown" +} + +main() { + parse_args "$@" + require_cmd "$WARP_BIN" + require_cmd awk + require_cmd ps + require_cmd pgrep + require_cmd tee + mkdir -p "$OUT_DIR" + + local pid + pid="$(resolve_pid)" + local warp_host + warp_host="$(normalize_warp_host "$HOST")" + + echo "Output dir: $OUT_DIR" + if [[ -n "$pid" ]]; then + echo "RustFS pid: $pid" + else + echo "RustFS pid: auto-detect failed (continuing without RSS sampling)" + fi + echo "Host: $HOST" + echo "Warp host: $warp_host" + echo "Bucket: $BUCKET" + echo "Profiles:" + echo " - 4KiB mixed" + echo " - 11MiB mixed" + echo " - 11MiB delete" + + run_profile "4KiB mixed" "mixed" "4KiB" "$pid" + run_profile "11MiB mixed" "mixed" "11MiB" "$pid" + run_profile "11MiB delete" "delete" "11MiB" "$pid" + + echo + echo "Acceptance run finished." + echo "Artifacts:" + find "$OUT_DIR" -maxdepth 1 -type f | sort +} + +main "$@"