diff --git a/.docker/observability/grafana/dashboards/grafana-get-performance-attribution.json b/.docker/observability/grafana/dashboards/grafana-get-performance-attribution.json new file mode 100644 index 000000000..288e4c3ec --- /dev/null +++ b/.docker/observability/grafana/dashboards/grafana-get-performance-attribution.json @@ -0,0 +1,716 @@ +{ + "annotations": { + "list": [] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 10, + "pointSize": 5, + "lineWidth": 1, + "showPoints": "auto", + "spanNulls": false, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum by (strategy, mode) (rate(rustfs_io_get_object_reader_setup_strategy_total[$__rate_interval])) or vector(0)", + "legendFormat": "{{strategy}} / {{mode}}", + "refId": "A" + } + ], + "title": "GET Reader Setup Strategy Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 10, + "pointSize": 5, + "lineWidth": 1, + "showPoints": "auto", + "spanNulls": false, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 2, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum by (path, strategy, size_bucket) (rate(rustfs_io_get_object_reader_setup_strategy_by_size_total[$__rate_interval])) or vector(0)", + "legendFormat": "{{path}} / {{strategy}} / {{size_bucket}}", + "refId": "A" + } + ], + "title": "GET Reader Setup Strategy by Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unit": "none", + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 10, + "pointSize": 5, + "lineWidth": 1, + "showPoints": "auto", + "spanNulls": false, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 3, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum by (strategy, mode) (rate(rustfs_io_get_object_reader_setup_scheduled_sum[$__rate_interval])) / clamp_min(sum by (strategy, mode) (rate(rustfs_io_get_object_reader_setup_scheduled_count[$__rate_interval])), 1) or vector(0)", + "legendFormat": "scheduled {{strategy}} / {{mode}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum by (strategy, mode) (rate(rustfs_io_get_object_reader_setup_attempted_sum[$__rate_interval])) / clamp_min(sum by (strategy, mode) (rate(rustfs_io_get_object_reader_setup_attempted_count[$__rate_interval])), 1) or vector(0)", + "legendFormat": "attempted {{strategy}} / {{mode}}", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum by (strategy, mode) (rate(rustfs_io_get_object_reader_setup_ready_sum[$__rate_interval])) / clamp_min(sum by (strategy, mode) (rate(rustfs_io_get_object_reader_setup_ready_count[$__rate_interval])), 1) or vector(0)", + "legendFormat": "ready {{strategy}} / {{mode}}", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum by (strategy, mode) (rate(rustfs_io_get_object_reader_setup_deferred_sum[$__rate_interval])) / clamp_min(sum by (strategy, mode) (rate(rustfs_io_get_object_reader_setup_deferred_count[$__rate_interval])), 1) or vector(0)", + "legendFormat": "deferred {{strategy}} / {{mode}}", + "refId": "D" + } + ], + "title": "GET Reader Setup Fanout Average", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unit": "none", + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 10, + "pointSize": 5, + "lineWidth": 1, + "showPoints": "auto", + "spanNulls": false, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 4, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum by (path, strategy, size_bucket) (rate(rustfs_io_get_object_reader_setup_scheduled_by_size_sum[$__rate_interval])) / clamp_min(sum by (path, strategy, size_bucket) (rate(rustfs_io_get_object_reader_setup_scheduled_by_size_count[$__rate_interval])), 1) or vector(0)", + "legendFormat": "scheduled {{path}} / {{strategy}} / {{size_bucket}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum by (path, strategy, size_bucket) (rate(rustfs_io_get_object_reader_setup_ready_by_size_sum[$__rate_interval])) / clamp_min(sum by (path, strategy, size_bucket) (rate(rustfs_io_get_object_reader_setup_ready_by_size_count[$__rate_interval])), 1) or vector(0)", + "legendFormat": "ready {{path}} / {{strategy}} / {{size_bucket}}", + "refId": "B" + } + ], + "title": "GET Reader Setup Fanout by Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 10, + "pointSize": 5, + "lineWidth": 1, + "showPoints": "auto", + "spanNulls": false, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.95, sum by (path, stage, size_bucket, le) (rate(rustfs_io_get_object_stage_duration_seconds_by_size_bucket{stage=~\"reader_setup|reader_setup_schedule|reader_setup_wait_quorum|reader_setup_drop_pending|reader_task_file_open|reader_task_reader_construction|reader_task_bitrot_reader_init\"}[$__rate_interval]))) or vector(0)", + "legendFormat": "p95 {{path}} / {{stage}} / {{size_bucket}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.99, sum by (path, stage, size_bucket, le) (rate(rustfs_io_get_object_stage_duration_seconds_by_size_bucket{stage=~\"reader_setup|reader_setup_schedule|reader_setup_wait_quorum|reader_setup_drop_pending|reader_task_file_open|reader_task_reader_construction|reader_task_bitrot_reader_init\"}[$__rate_interval]))) or vector(0)", + "legendFormat": "p99 {{path}} / {{stage}} / {{size_bucket}}", + "refId": "B" + } + ], + "title": "GET Hot Stage Latency by Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 10, + "pointSize": 5, + "lineWidth": 1, + "showPoints": "auto", + "spanNulls": false, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 6, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.95, sum by (path, stage, le) (rate(rustfs_io_get_object_stage_duration_seconds_bucket{stage=~\"request_context|first_byte|response_handoff|full_body|body_build|output_poll|output_lock_wait|reader_open_mmap_copy_success|reader_open_mmap_copy_fallback|reader_open_stream|reader_stream_first_read|reader_mmap_blocking_wait|reader_mmap_blocking_task|reader_mmap_file_open|reader_mmap_map|reader_mmap_copy_buffer|reader_mmap_direct_read_copy\"}[$__rate_interval]))) or vector(0)", + "legendFormat": "p95 {{path}} / {{stage}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.99, sum by (path, stage, le) (rate(rustfs_io_get_object_stage_duration_seconds_bucket{stage=~\"request_context|first_byte|response_handoff|full_body|body_build|output_poll|output_lock_wait|reader_open_mmap_copy_success|reader_open_mmap_copy_fallback|reader_open_stream|reader_stream_first_read|reader_mmap_blocking_wait|reader_mmap_blocking_task|reader_mmap_file_open|reader_mmap_map|reader_mmap_copy_buffer|reader_mmap_direct_read_copy\"}[$__rate_interval]))) or vector(0)", + "legendFormat": "p99 {{path}} / {{stage}}", + "refId": "B" + } + ], + "title": "GET Reader, Handler and Body Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 10, + "pointSize": 5, + "lineWidth": 1, + "showPoints": "auto", + "spanNulls": false, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 7, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum by (subpath) (rate(rustfs_io_get_object_direct_memory_subpath_total[$__rate_interval])) or vector(0)", + "legendFormat": "{{subpath}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum by (subpath, size_bucket) (rate(rustfs_io_get_object_direct_memory_subpath_by_size_total[$__rate_interval])) or vector(0)", + "legendFormat": "{{subpath}} / {{size_bucket}}", + "refId": "B" + } + ], + "title": "GET Direct-Memory Subpath Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 10, + "pointSize": 5, + "lineWidth": 1, + "showPoints": "auto", + "spanNulls": false, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 8, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum by (path, stage, kind) (rate(rustfs_io_get_object_mmap_page_faults_total[$__rate_interval])) or vector(0)", + "legendFormat": "mmap {{path}} / {{stage}} / {{kind}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum by (path, stage, kind) (rate(rustfs_io_get_object_direct_read_page_faults_total[$__rate_interval])) or vector(0)", + "legendFormat": "direct-read {{path}} / {{stage}} / {{kind}}", + "refId": "B" + } + ], + "title": "GET mmap vs Direct-Read Page Fault Rate", + "type": "timeseries" + } + ], + "refresh": "30s", + "schemaVersion": 38, + "style": "dark", + "tags": [ + "rustfs", + "get", + "performance", + "attribution" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "", + "title": "RustFS GET Performance Attribution", + "uid": "rustfs-get-performance-attribution", + "version": 1, + "weekStart": "" +} diff --git a/.docker/observability/grafana/dashboards/grafana-put-performance-attribution.json b/.docker/observability/grafana/dashboards/grafana-put-performance-attribution.json new file mode 100644 index 000000000..02c7e9d6c --- /dev/null +++ b/.docker/observability/grafana/dashboards/grafana-put-performance-attribution.json @@ -0,0 +1,534 @@ +{ + "annotations": { + "list": [] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 10, + "pointSize": 5, + "lineWidth": 1, + "showPoints": "auto", + "spanNulls": false, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum by (path, eager_status, size_bucket, buffer_bucket, large_concurrency_tuning) (rate(rustfs_s3_put_object_diagnostics_total[$__rate_interval])) or vector(0)", + "legendFormat": "{{path}} / {{eager_status}} / {{size_bucket}} / {{buffer_bucket}} / large={{large_concurrency_tuning}}", + "refId": "A" + } + ], + "title": "PUT Diagnostics Decision Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 10, + "pointSize": 5, + "lineWidth": 1, + "showPoints": "auto", + "spanNulls": false, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 2, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum by (path) (rate(rustfs_s3_put_object_path_total[$__rate_interval])) or vector(0)", + "legendFormat": "{{path}}", + "refId": "A" + } + ], + "title": "PUT Path Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unit": "ms", + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 10, + "pointSize": 5, + "lineWidth": 1, + "showPoints": "auto", + "spanNulls": false, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 3, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.95, sum by (stage, le) (rate(rustfs_s3_put_object_stage_duration_ms_bucket{stage=~\"ingress_prepare|set_disk_writer_setup|set_disk_encode|set_disk_rename|set_disk_old_data_cleanup|multipart_ingress_prepare|multipart_set_disk_writer_setup|multipart_set_disk_encode|multipart_complete_tail\"}[$__rate_interval]))) or vector(0)", + "legendFormat": "p95 {{stage}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.99, sum by (stage, le) (rate(rustfs_s3_put_object_stage_duration_ms_bucket{stage=~\"ingress_prepare|set_disk_writer_setup|set_disk_encode|set_disk_rename|set_disk_old_data_cleanup|multipart_ingress_prepare|multipart_set_disk_writer_setup|multipart_set_disk_encode|multipart_complete_tail\"}[$__rate_interval]))) or vector(0)", + "legendFormat": "p99 {{stage}}", + "refId": "B" + } + ], + "title": "PUT Hot Stage Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 10, + "pointSize": 5, + "lineWidth": 1, + "showPoints": "auto", + "spanNulls": false, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 4, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.50, sum by (path, size_bucket, le) (rate(rustfs_s3_put_object_selected_buffer_size_bytes_bucket[$__rate_interval]))) or vector(0)", + "legendFormat": "p50 {{path}} / {{size_bucket}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.95, sum by (path, size_bucket, le) (rate(rustfs_s3_put_object_selected_buffer_size_bytes_bucket[$__rate_interval]))) or vector(0)", + "legendFormat": "p95 {{path}} / {{size_bucket}}", + "refId": "B" + } + ], + "title": "PUT Selected Buffer Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "min": 0, + "max": 1, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 15, + "pointSize": 5, + "lineWidth": 1, + "showPoints": "auto", + "spanNulls": false, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(rustfs_s3_put_object_zero_copy_eligible_total[$__rate_interval])) / clamp_min(sum(rate(rustfs_s3_put_object_total[$__rate_interval])), 1) or vector(0)", + "legendFormat": "eligible / total", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(rustfs_s3_put_object_zero_copy_enabled_total[$__rate_interval])) / clamp_min(sum(rate(rustfs_s3_put_object_total[$__rate_interval])), 1) or vector(0)", + "legendFormat": "enabled / total", + "refId": "B" + } + ], + "title": "PUT Zero-Copy Eligibility Ratio", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 10, + "pointSize": 5, + "lineWidth": 1, + "showPoints": "auto", + "spanNulls": false, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 6, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(rustfs_s3_put_object_total[$__rate_interval])) or vector(0)", + "legendFormat": "PUT total", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum by (stage) (rate(rustfs_system_storage_erasure_write_quorum_failures_total[$__rate_interval])) or vector(0)", + "legendFormat": "write quorum failure {{stage}}", + "refId": "B" + } + ], + "title": "PUT Throughput and Quorum Failure Signals", + "type": "timeseries" + } + ], + "refresh": "30s", + "schemaVersion": 38, + "style": "dark", + "tags": [ + "rustfs", + "put", + "performance", + "attribution" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "", + "title": "RustFS PUT Performance Attribution", + "uid": "rustfs-put-performance-attribution", + "version": 1, + "weekStart": "" +} diff --git a/Cargo.lock b/Cargo.lock index 20f1879cc..cec0095d2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2034,9 +2034,9 @@ dependencies = [ [[package]] name = "console" -version = "0.16.3" +version = "0.16.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d64e8af5551369d19cf50138de61f1c42074ab970f74e99be916646777f8fc87" +checksum = "4fe5f465a4f6fee88fad41b85d990f84c835335e85b5d9e6e63e0d06d28cba7c" dependencies = [ "encode_unicode", "libc", @@ -5826,9 +5826,9 @@ dependencies = [ [[package]] name = "libredox" -version = "0.1.17" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f02ab6bace2054fb888a3c16f990117b579d14a3088e472d63c6011fa185c9d3" +checksum = "c943259e342f1e06ff2da7a83eabdfe7f92ce10262688dbf1895ff0b3e6e4652" dependencies = [ "libc", ] @@ -11683,9 +11683,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.52" +version = "0.3.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e48db7b415311b615f910b3dcaa4557bcd4bf1982379c95c223fd8c2a20e210" +checksum = "18dfaaeddcb932337b5e7866ee7d0ce9b76d2fd092997146f187ec09b4558a50" dependencies = [ "deranged", "js-sys", @@ -13082,9 +13082,9 @@ dependencies = [ [[package]] name = "xxhash-rust" -version = "0.8.15" +version = "0.8.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3" +checksum = "4d93c89cdc2d3a63c3ec48ffe926931bdc069eafa8e4402fe6d8f790c9d1e576" [[package]] name = "yansi" @@ -13248,9 +13248,9 @@ dependencies = [ [[package]] name = "zlib-rs" -version = "0.6.4" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "977347db8caa080403f6b6b7c1cda9479a8e869316f7e13a59b19076a40f94e3" +checksum = "5431d5661c32445236631278f27946e444ddafe4684cac70b185272d4f9c52d5" [[package]] name = "zmij" diff --git a/Cargo.toml b/Cargo.toml index b4338390c..00bcc8d1a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -204,7 +204,7 @@ zeroize = { version = "1.9.0", features = ["derive"] } chrono = { version = "0.4.45", features = ["serde"] } humantime = "2.3.0" jiff = { version = "0.2.31", features = ["serde"] } -time = { version = "0.3.52", features = ["std", "parsing", "formatting", "macros", "serde"] } +time = { version = "0.3.53", features = ["std", "parsing", "formatting", "macros", "serde"] } # Database deadpool-postgres = { version = "0.14", features = ["rt_tokio_1"] } @@ -307,7 +307,7 @@ vaultrs = { version = "0.8.0" } walkdir = "2.5.0" wildmatch = { version = "2.6.1", features = ["serde"] } windows = { version = "0.62.2" } -xxhash-rust = { version = "0.8.15", features = ["xxh64", "xxh3"] } +xxhash-rust = { version = "0.8.16", features = ["xxh64", "xxh3"] } zip = "8.6.0" zstd = "0.13.3" diff --git a/crates/ecstore/src/erasure/coding/decode.rs b/crates/ecstore/src/erasure/coding/decode.rs index 1cd993c81..d327ebcb9 100644 --- a/crates/ecstore/src/erasure/coding/decode.rs +++ b/crates/ecstore/src/erasure/coding/decode.rs @@ -50,6 +50,10 @@ fn get_shard_locality_preference_enabled() -> bool { ) } +pub(crate) fn should_collect_shard_read_costs() -> bool { + rustfs_io_metrics::get_stage_metrics_enabled() || get_shard_locality_preference_enabled() +} + /// Number of stripes to prefetch in the legacy decode path. /// When > 1, stripe reads are batched to overlap disk I/O with decode. /// Default: 1 (no prefetch, current behavior). diff --git a/crates/ecstore/src/set_disk/mod.rs b/crates/ecstore/src/set_disk/mod.rs index 5811e8e40..d438570bd 100644 --- a/crates/ecstore/src/set_disk/mod.rs +++ b/crates/ecstore/src/set_disk/mod.rs @@ -347,6 +347,8 @@ const DEFAULT_RUSTFS_GET_CODEC_STREAMING_ENABLE: bool = false; // Disabled until const ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE: &str = "RUSTFS_GET_CODEC_STREAMING_MIN_SIZE"; const DEFAULT_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE: usize = MI_B; +const ENV_RUSTFS_GET_CODEC_STREAMING_RUSTFS_MIN_SIZE: &str = "RUSTFS_GET_CODEC_STREAMING_RUSTFS_MIN_SIZE"; +const DEFAULT_RUSTFS_GET_CODEC_STREAMING_RUSTFS_MIN_SIZE: usize = MI_B; const ENV_RUSTFS_GET_CODEC_STREAMING_ENGINE: &str = "RUSTFS_GET_CODEC_STREAMING_ENGINE"; const DEFAULT_RUSTFS_GET_CODEC_STREAMING_ENGINE: &str = GET_CODEC_STREAMING_ENGINE_LEGACY; @@ -366,6 +368,11 @@ const DEFAULT_RUSTFS_GET_CODEC_STREAMING_MULTIPART_ENABLE: bool = false; const ENV_RUSTFS_GET_CODEC_STREAMING_MULTIPART_MAX_PARTS: &str = "RUSTFS_GET_CODEC_STREAMING_MULTIPART_MAX_PARTS"; const DEFAULT_RUSTFS_GET_CODEC_STREAMING_MULTIPART_MAX_PARTS: usize = 256; +const ENV_RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_ENABLE: &str = "RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_ENABLE"; +const DEFAULT_RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_ENABLE: bool = false; +const ENV_RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_MAX_SIZE: &str = "RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_MAX_SIZE"; +const DEFAULT_RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_MAX_SIZE: usize = 512 * 1024; + const ENV_RUSTFS_GET_SMALL_OBJECT_DIRECT_MEMORY: &str = "RUSTFS_GET_SMALL_OBJECT_DIRECT_MEMORY"; const DEFAULT_RUSTFS_GET_SMALL_OBJECT_DIRECT_MEMORY: bool = false; const ENV_RUSTFS_GET_SMALL_OBJECT_DIRECT_MEMORY_THRESHOLD: &str = "RUSTFS_GET_SMALL_OBJECT_DIRECT_MEMORY_THRESHOLD"; @@ -648,7 +655,57 @@ pub fn should_use_metadata_early_stop(bucket: &str, object: &str) -> bool { } fn get_codec_streaming_min_size() -> usize { - rustfs_utils::get_env_usize(ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE, DEFAULT_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE) + if std::env::var_os(ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE).is_some() { + return rustfs_utils::get_env_usize(ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE, DEFAULT_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE); + } + + match get_codec_streaming_engine() { + GetCodecStreamingEngine::Rustfs => rustfs_utils::get_env_usize( + ENV_RUSTFS_GET_CODEC_STREAMING_RUSTFS_MIN_SIZE, + DEFAULT_RUSTFS_GET_CODEC_STREAMING_RUSTFS_MIN_SIZE, + ), + GetCodecStreamingEngine::Legacy => DEFAULT_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE, + } +} + +fn is_get_codec_streaming_data_blocks_first_enabled() -> bool { + #[cfg(test)] + { + rustfs_utils::get_env_bool( + ENV_RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_ENABLE, + DEFAULT_RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_ENABLE, + ) + } + #[cfg(not(test))] + { + static CACHED: OnceLock = OnceLock::new(); + *CACHED.get_or_init(|| { + rustfs_utils::get_env_bool( + ENV_RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_ENABLE, + DEFAULT_RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_ENABLE, + ) + }) + } +} + +fn get_codec_streaming_data_blocks_first_max_size() -> usize { + #[cfg(test)] + { + rustfs_utils::get_env_usize( + ENV_RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_MAX_SIZE, + DEFAULT_RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_MAX_SIZE, + ) + } + #[cfg(not(test))] + { + static CACHED: OnceLock = OnceLock::new(); + *CACHED.get_or_init(|| { + rustfs_utils::get_env_usize( + ENV_RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_MAX_SIZE, + DEFAULT_RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_MAX_SIZE, + ) + }) + } } fn get_object_metadata_cache_max_entries() -> usize { @@ -840,6 +897,7 @@ impl GetCodecStreamingObjectClass { struct GetCodecStreamingGate { object_class: GetCodecStreamingObjectClass, decision: GetCodecStreamingDecision, + prefer_data_blocks_first_reader_setup: bool, } fn record_get_codec_streaming_gate_decision( @@ -946,6 +1004,24 @@ fn is_get_small_object_direct_memory_eligible( ) } +fn should_prefer_codec_streaming_data_blocks_first_reader_setup( + object_class: GetCodecStreamingObjectClass, + object_size: i64, +) -> bool { + if !is_get_codec_streaming_data_blocks_first_enabled() + || object_class != GetCodecStreamingObjectClass::PlainSinglePart + || object_size <= 0 + { + return false; + } + + let Ok(object_size) = usize::try_from(object_size) else { + return false; + }; + let max_size = get_codec_streaming_data_blocks_first_max_size(); + max_size > 0 && object_size <= max_size +} + fn get_codec_streaming_reader_gate( bucket: &str, object: &str, @@ -960,42 +1036,49 @@ fn get_codec_streaming_reader_gate( return GetCodecStreamingGate { object_class, decision: GetCodecStreamingDecision::Fallback(GetCodecStreamingFallbackReason::Disabled), + prefer_data_blocks_first_reader_setup: false, }; } if !get_codec_streaming_rollout().is_opted_in() { return GetCodecStreamingGate { object_class, decision: GetCodecStreamingDecision::Fallback(GetCodecStreamingFallbackReason::RolloutNotOptedIn), + prefer_data_blocks_first_reader_setup: false, }; } if !should_use_codec_streaming(bucket, object) { return GetCodecStreamingGate { object_class, decision: GetCodecStreamingDecision::Fallback(GetCodecStreamingFallbackReason::RolloutPctNotSelected), + prefer_data_blocks_first_reader_setup: false, }; } if !is_get_codec_streaming_body_compat_confirmed() { return GetCodecStreamingGate { object_class, decision: GetCodecStreamingDecision::Fallback(GetCodecStreamingFallbackReason::BodyCompatibilityUnconfirmed), + prefer_data_blocks_first_reader_setup: false, }; } if !is_get_codec_streaming_header_compat_confirmed() { return GetCodecStreamingGate { object_class, decision: GetCodecStreamingDecision::Fallback(GetCodecStreamingFallbackReason::HeaderCompatibilityUnconfirmed), + prefer_data_blocks_first_reader_setup: false, }; } if object_class == GetCodecStreamingObjectClass::Range { return GetCodecStreamingGate { object_class, decision: GetCodecStreamingDecision::Fallback(GetCodecStreamingFallbackReason::Range), + prefer_data_blocks_first_reader_setup: false, }; } if !lock_optimization_enabled { return GetCodecStreamingGate { object_class, decision: GetCodecStreamingDecision::Fallback(GetCodecStreamingFallbackReason::LockOptimizationDisabled), + prefer_data_blocks_first_reader_setup: false, }; } @@ -1003,30 +1086,35 @@ fn get_codec_streaming_reader_gate( return GetCodecStreamingGate { object_class, decision: GetCodecStreamingDecision::Fallback(GetCodecStreamingFallbackReason::InvalidMinSize), + prefer_data_blocks_first_reader_setup: false, }; }; if object_info.size < min_size { return GetCodecStreamingGate { object_class, decision: GetCodecStreamingDecision::Fallback(GetCodecStreamingFallbackReason::BelowMinSize), + prefer_data_blocks_first_reader_setup: false, }; } if object_class == GetCodecStreamingObjectClass::Encrypted { return GetCodecStreamingGate { object_class, decision: GetCodecStreamingDecision::Fallback(GetCodecStreamingFallbackReason::Encrypted), + prefer_data_blocks_first_reader_setup: false, }; } if object_class == GetCodecStreamingObjectClass::Compressed { return GetCodecStreamingGate { object_class, decision: GetCodecStreamingDecision::Fallback(GetCodecStreamingFallbackReason::Compressed), + prefer_data_blocks_first_reader_setup: false, }; } if object_class == GetCodecStreamingObjectClass::Remote { return GetCodecStreamingGate { object_class, decision: GetCodecStreamingDecision::Fallback(GetCodecStreamingFallbackReason::Remote), + prefer_data_blocks_first_reader_setup: false, }; } if object_class == GetCodecStreamingObjectClass::Multipart { @@ -1034,12 +1122,14 @@ fn get_codec_streaming_reader_gate( return GetCodecStreamingGate { object_class, decision: GetCodecStreamingDecision::Fallback(GetCodecStreamingFallbackReason::Multipart), + prefer_data_blocks_first_reader_setup: false, }; } if fi.parts.len() > get_codec_streaming_multipart_max_parts() { return GetCodecStreamingGate { object_class, decision: GetCodecStreamingDecision::Fallback(GetCodecStreamingFallbackReason::MultipartPartLimit), + prefer_data_blocks_first_reader_setup: false, }; } } @@ -1047,6 +1137,10 @@ fn get_codec_streaming_reader_gate( GetCodecStreamingGate { object_class, decision: GetCodecStreamingDecision::Use, + prefer_data_blocks_first_reader_setup: should_prefer_codec_streaming_data_blocks_first_reader_setup( + object_class, + object_info.size, + ), } } @@ -1694,6 +1788,14 @@ fn classify_multipart_part_write_path(object_size: i64, block_size: usize) -> Sm } } +fn known_put_object_storage_size(data_size: i64) -> i64 { + if data_size >= 0 { + data_size + } else { + HashReader::SIZE_PRESERVE_LAYER + } +} + #[allow(clippy::too_many_arguments)] async fn build_inline_bitrot_readers( files: &[FileInfo], @@ -2274,6 +2376,9 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks { self.set_index, self.pool_index, opts.skip_verify_bitrot, + object_class.as_str(), + size_bucket, + codec_streaming_gate.prefer_data_blocks_first_reader_setup, ) .await? { @@ -2450,10 +2555,11 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks { let result: Result = async { let erasure = coding::Erasure::new(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size); + let put_object_size = known_put_object_storage_size(data.size()); let is_inline_buffer = - runtime_sources::storage_class_should_inline(erasure.shard_file_size(data.size()), opts.versioned); + runtime_sources::storage_class_should_inline(erasure.shard_file_size(put_object_size), opts.versioned); - let shard_file_size = erasure.shard_file_size(data.size()); + let shard_file_size = erasure.shard_file_size(put_object_size); let shard_size = erasure.shard_size(); let writer_setup_stage_start = Instant::now(); let writer_futs: Vec<_> = shuffle_disks @@ -2531,7 +2637,7 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks { HashReader::from_stream(Cursor::new(Vec::new()), 0, 0, None, None, false)?, ); - let write_path = classify_put_write_path(is_inline_buffer, data.size(), fi.erasure.block_size); + let write_path = classify_put_write_path(is_inline_buffer, put_object_size, fi.erasure.block_size); rustfs_io_metrics::record_put_object_path(write_path.metric_label()); let encode_stage_start = Instant::now(); @@ -9894,12 +10000,33 @@ mod tests { classify_put_write_path(false, 32 * 1024 * 1024, 1024 * 1024), SmallWritePath::Pipeline )); + assert!(matches!( + classify_put_write_path(false, 31 * 1024 * 1024, 1024 * 1024), + SmallWritePath::Pipeline + )); assert!(matches!( classify_put_write_path(true, 64 * 1024 * 1024, 1024 * 1024), SmallWritePath::Pipeline )); } + #[test] + fn put_object_classification_uses_only_known_storage_size() { + assert_eq!(known_put_object_storage_size(42), 42); + assert_eq!( + known_put_object_storage_size(HashReader::SIZE_PRESERVE_LAYER), + HashReader::SIZE_PRESERVE_LAYER + ); + assert!(matches!( + classify_put_write_path(false, known_put_object_storage_size(HashReader::SIZE_PRESERVE_LAYER), 1024 * 1024), + SmallWritePath::Pipeline + )); + assert!(matches!( + classify_put_write_path(false, known_put_object_storage_size(1024 * 1024), 1024 * 1024), + SmallWritePath::SingleBlockNonInline + )); + } + #[test] fn put_object_part_fast_path_selection_matches_single_block_non_inline_rules() { assert!(should_use_single_block_non_inline_fast_path(false, 4096, 4096)); diff --git a/crates/ecstore/src/set_disk/read.rs b/crates/ecstore/src/set_disk/read.rs index 628e4c2be..77f205d34 100644 --- a/crates/ecstore/src/set_disk/read.rs +++ b/crates/ecstore/src/set_disk/read.rs @@ -843,6 +843,13 @@ struct BitrotReaderSetup { deferred_count: usize, } +#[derive(Clone, Copy)] +struct BitrotReaderSetupAttribution { + path: &'static str, + object_class: &'static str, + size_bucket: &'static str, +} + #[derive(Clone, Copy)] enum BitrotReaderSetupMode { ReadQuorum, @@ -883,7 +890,8 @@ fn get_bitrot_reader_setup_strategy(mode: BitrotReaderSetupMode, prefer_data_blo BitrotReaderSetupStrategy::DataBlocksFirst } BitrotReaderSetupMode::VerifyReconstruction - if rustfs_utils::get_env_bool(ENV_RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_READER_SETUP, false) => + if prefer_data_blocks_first + || rustfs_utils::get_env_bool(ENV_RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_READER_SETUP, false) => { BitrotReaderSetupStrategy::DataBlocksFirst } @@ -955,6 +963,14 @@ impl BitrotReaderSetup { } } + fn scheduling_target(&self, data_shards: usize, parity_shards: usize, mode: BitrotReaderSetupMode) -> usize { + match mode { + BitrotReaderSetupMode::ReadQuorum => data_shards, + BitrotReaderSetupMode::VerifyReconstruction if !self.data_shards_attempted(data_shards) => data_shards, + BitrotReaderSetupMode::VerifyReconstruction => self.reconstruction_verification_target(data_shards, parity_shards), + } + } + fn deferred_shards(&self) -> usize { self.deferred_count } @@ -1105,16 +1121,49 @@ fn record_bitrot_reader_setup_fanout( strategy: BitrotReaderSetupStrategy, mode: BitrotReaderSetupMode, setup: &BitrotReaderSetup, + attribution: Option, ) { - rustfs_io_metrics::record_get_object_reader_setup_fanout( - strategy.as_str(), - mode.as_str(), - setup.scheduled_shards(), - setup.attempted_shards(), - setup.available_shards(), - setup.completed_failed_shards(), - setup.deferred_shards(), - ); + let strategy = strategy.as_str(); + let mode = mode.as_str(); + let scheduled = setup.scheduled_shards(); + let attempted = setup.attempted_shards(); + let ready = setup.available_shards(); + let failed = setup.completed_failed_shards(); + let deferred = setup.deferred_shards(); + rustfs_io_metrics::record_get_object_reader_setup_fanout(strategy, mode, scheduled, attempted, ready, failed, deferred); + if let Some(attribution) = attribution { + rustfs_io_metrics::record_get_object_reader_setup_fanout_by_size( + attribution.path, + strategy, + mode, + attribution.object_class, + attribution.size_bucket, + scheduled, + attempted, + ready, + failed, + deferred, + ); + } +} + +fn record_bitrot_reader_setup_strategy( + strategy: BitrotReaderSetupStrategy, + mode: BitrotReaderSetupMode, + attribution: Option, +) { + let strategy = strategy.as_str(); + let mode = mode.as_str(); + rustfs_io_metrics::record_get_object_reader_setup_strategy(strategy, mode); + if let Some(attribution) = attribution { + rustfs_io_metrics::record_get_object_reader_setup_strategy_by_size( + attribution.path, + strategy, + mode, + attribution.object_class, + attribution.size_bucket, + ); + } } #[allow(clippy::too_many_arguments)] @@ -1134,13 +1183,14 @@ async fn create_bitrot_readers_until_quorum_all_shards( parity_shards: usize, mode: BitrotReaderSetupMode, stage_metrics: Option, + attribution: Option, ) -> BitrotReaderSetup { let strategy = BitrotReaderSetupStrategy::AllShards; let mut setup = BitrotReaderSetup::new(disks.len()); let mut reader_tasks = FuturesUnordered::new(); let stage_metrics = stage_metrics.filter(|_| rustfs_io_metrics::get_stage_metrics_enabled()); - rustfs_io_metrics::record_get_object_reader_setup_strategy(strategy.as_str(), mode.as_str()); + record_bitrot_reader_setup_strategy(strategy, mode, attribution); let schedule_stage_start = stage_metrics.map(|_| Instant::now()); for (idx, disk_op) in disks.iter().enumerate() { @@ -1207,7 +1257,7 @@ async fn create_bitrot_readers_until_quorum_all_shards( if let Some(stage_metrics) = stage_metrics { record_get_stage_duration_if_enabled(stage_metrics.path, GET_STAGE_READER_SETUP_DROP_PENDING, drop_pending_stage_start); } - record_bitrot_reader_setup_fanout(strategy, mode, &setup); + record_bitrot_reader_setup_fanout(strategy, mode, &setup, attribution); setup } @@ -1229,6 +1279,7 @@ async fn create_bitrot_readers_until_quorum( parity_shards: usize, mode: BitrotReaderSetupMode, stage_metrics: Option, + attribution: Option, ) -> BitrotReaderSetup { create_bitrot_readers_until_quorum_with_preference( files, @@ -1247,6 +1298,7 @@ async fn create_bitrot_readers_until_quorum( mode, false, stage_metrics, + attribution, ) .await } @@ -1269,6 +1321,7 @@ async fn create_bitrot_readers_until_quorum_with_preference( mode: BitrotReaderSetupMode, prefer_data_blocks_first: bool, stage_metrics: Option, + attribution: Option, ) -> BitrotReaderSetup { let strategy = get_bitrot_reader_setup_strategy(mode, prefer_data_blocks_first); if strategy == BitrotReaderSetupStrategy::AllShards { @@ -1288,6 +1341,7 @@ async fn create_bitrot_readers_until_quorum_with_preference( parity_shards, mode, stage_metrics, + attribution, ) .await; } @@ -1297,10 +1351,11 @@ async fn create_bitrot_readers_until_quorum_with_preference( let total_shards = disks.len(); let stage_metrics = stage_metrics.filter(|_| rustfs_io_metrics::get_stage_metrics_enabled()); - rustfs_io_metrics::record_get_object_reader_setup_strategy(strategy.as_str(), mode.as_str()); + record_bitrot_reader_setup_strategy(strategy, mode, attribution); let schedule_stage_start = stage_metrics.map(|_| Instant::now()); - for idx in 0..data_shards.min(total_shards) { + let initial_target = setup.setup_target(data_shards, parity_shards, mode); + for idx in 0..initial_target.min(data_shards).min(total_shards) { schedule_bitrot_reader_task( &mut reader_tasks, &mut setup, @@ -1319,26 +1374,6 @@ async fn create_bitrot_readers_until_quorum_with_preference( stage_metrics, ); } - - if data_shards < total_shards { - schedule_bitrot_reader_task( - &mut reader_tasks, - &mut setup, - data_shards, - files, - disks, - bucket, - object, - part_number, - read_offset, - read_length, - shard_size, - checksum_algo.clone(), - skip_verify_bitrot, - use_mmap_read, - stage_metrics, - ); - } if let Some(stage_metrics) = stage_metrics { record_get_stage_duration_if_enabled(stage_metrics.path, GET_STAGE_READER_SETUP_SCHEDULE, schedule_stage_start); } @@ -1351,7 +1386,7 @@ async fn create_bitrot_readers_until_quorum_with_preference( break; } - let target = setup.setup_target(data_shards, parity_shards, mode); + let target = setup.scheduling_target(data_shards, parity_shards, mode); while setup.available_shards().saturating_add(setup.pending_scheduled_shards()) < target { let Some(next_idx) = next_unscheduled_reader_index(&setup, total_shards, data_shards) else { break; @@ -1401,7 +1436,7 @@ async fn create_bitrot_readers_until_quorum_with_preference( if let Some(stage_metrics) = stage_metrics { record_get_stage_duration_if_enabled(stage_metrics.path, GET_STAGE_READER_SETUP_DROP_PENDING, drop_pending_stage_start); } - record_bitrot_reader_setup_fanout(strategy, mode, &setup); + record_bitrot_reader_setup_fanout(strategy, mode, &setup, attribution); setup } @@ -1475,7 +1510,7 @@ async fn create_data_block_bitrot_readers( // The direct-memory path only consumes the data shard readers. If one of // them is missing, the caller falls back to the regular GET path. - record_bitrot_reader_setup_fanout(strategy, BitrotReaderSetupMode::ReadQuorum, &setup); + record_bitrot_reader_setup_fanout(strategy, BitrotReaderSetupMode::ReadQuorum, &setup, None); setup } @@ -2765,10 +2800,12 @@ impl SetDisks { let use_mmap_read = object_mmap_read_enabled(); let reader_setup_stage_start = Instant::now(); - let read_costs = disks - .iter() - .map(|disk| shard_read_cost_for_disk(disk.as_ref())) - .collect::>(); + let read_costs = coding::decode::should_collect_shard_read_costs().then(|| { + disks + .iter() + .map(|disk| shard_read_cost_for_disk(disk.as_ref())) + .collect::>() + }); let reader_setup = create_bitrot_readers_until_quorum_with_preference( &files, &disks, @@ -2786,6 +2823,11 @@ impl SetDisks { BitrotReaderSetupMode::ReadQuorum, prefer_data_blocks_first_reader_setup, None, + Some(BitrotReaderSetupAttribution { + path: metrics_path, + object_class: metrics_object_class, + size_bucket: metrics_size_bucket, + }), ) .await; let reader_setup_elapsed = reader_setup_stage_start.elapsed(); @@ -2926,9 +2968,13 @@ impl SetDisks { let decode_stage_start = Instant::now(); let unattempted_data_shards = !reader_setup.data_shards_attempted(erasure.data_shards); let readers = reader_setup.readers; - let (written, err) = erasure - .decode_with_read_costs(writer, readers, part_offset, part_length, part_size, read_costs) - .await; + let (written, err) = if let Some(read_costs) = read_costs { + erasure + .decode_with_read_costs(writer, readers, part_offset, part_length, part_size, read_costs) + .await + } else { + erasure.decode(writer, readers, part_offset, part_length, part_size).await + }; let decode_elapsed = decode_stage_start.elapsed(); rustfs_io_metrics::record_get_object_decode_duration(decode_elapsed.as_secs_f64()); rustfs_io_metrics::record_get_object_stage_duration_by_size( @@ -3056,6 +3102,9 @@ impl SetDisks { _set_index: usize, _pool_index: usize, skip_verify_bitrot: bool, + metrics_object_class: &'static str, + metrics_size_bucket: &'static str, + prefer_data_blocks_first_reader_setup: bool, ) -> Result { let (disks, files) = Self::shuffle_disks_and_parts_metadata_by_index(disks, files, fi); @@ -3082,6 +3131,9 @@ impl SetDisks { part_length, part.size, skip_verify_bitrot, + metrics_object_class, + metrics_size_bucket, + prefer_data_blocks_first_reader_setup, ) .await; } @@ -3121,6 +3173,9 @@ impl SetDisks { part.size, part.size, skip_verify_bitrot, + metrics_object_class, + metrics_size_bucket, + false, ) .await? { @@ -3149,6 +3204,9 @@ impl SetDisks { part_length: usize, part_size: usize, skip_verify_bitrot: bool, + metrics_object_class: &'static str, + metrics_size_bucket: &'static str, + prefer_data_blocks_first_reader_setup: bool, ) -> Result { if part_length > part_size { return Err(Error::other("codec streaming reader part length exceeds part size")); @@ -3173,11 +3231,13 @@ impl SetDisks { bitrot_reader_init_stage: GET_STAGE_READER_TASK_BITROT_READER_INIT, }); let reader_setup_stage_start = get_stage_timer_if_enabled(stage_metrics_enabled); - let read_costs = disks - .iter() - .map(|disk| shard_read_cost_for_disk(disk.as_ref())) - .collect::>(); - let reader_setup = create_bitrot_readers_until_quorum( + let read_costs = coding::decode::should_collect_shard_read_costs().then(|| { + disks + .iter() + .map(|disk| shard_read_cost_for_disk(disk.as_ref())) + .collect::>() + }); + let reader_setup = create_bitrot_readers_until_quorum_with_preference( files, disks, bucket, @@ -3192,7 +3252,13 @@ impl SetDisks { erasure.data_shards, erasure.parity_shards, BitrotReaderSetupMode::VerifyReconstruction, + prefer_data_blocks_first_reader_setup, reader_stage_metrics, + Some(BitrotReaderSetupAttribution { + path: metrics_path, + object_class: metrics_object_class, + size_bucket: metrics_size_bucket, + }), ) .await; record_get_stage_duration_if_enabled(metrics_path, GET_STAGE_READER_SETUP, reader_setup_stage_start); @@ -3214,14 +3280,24 @@ impl SetDisks { } let readers = reader_setup.readers; - let source = coding::decode::ParallelReader::new_with_metrics_path_read_costs_and_reconstruction_verification( - readers, - erasure.clone(), - part_offset, - part_size, - Some(metrics_path), - read_costs, - ); + let source = if let Some(read_costs) = read_costs { + coding::decode::ParallelReader::new_with_metrics_path_read_costs_and_reconstruction_verification( + readers, + erasure.clone(), + part_offset, + part_size, + Some(metrics_path), + read_costs, + ) + } else { + coding::decode::ParallelReader::new_with_metrics_path_and_reconstruction_verification( + readers, + erasure.clone(), + part_offset, + part_size, + Some(metrics_path), + ) + }; let engine = build_get_codec_streaming_decode_engine(erasure.clone())?; let reader = coding::decode_reader::ErasureDecodeReader::new_with_metrics_path(source, engine, part_length, metrics_path)?; @@ -3735,7 +3811,8 @@ mod metadata_cache_tests { #[cfg(test)] mod tests { use super::*; - use std::io::Cursor; + use crate::erasure::coding::BitrotWriter; + use std::io::{Cursor, ErrorKind}; use std::sync::{ Arc, atomic::{AtomicUsize, Ordering}, @@ -4556,6 +4633,7 @@ mod tests { parity_shards, mode, None, + None, ) .await }, @@ -4590,10 +4668,110 @@ mod tests { mode, prefer_data_blocks_first, None, + None, ) .await } + fn encoded_reader_setup_fileinfo(data: Option>) -> FileInfo { + let mut fi = FileInfo::new("object", 2, 2); + fi.volume = "bucket".to_string(); + fi.name = "object".to_string(); + fi.size = data + .as_ref() + .map_or(0, |data| i64::try_from(data.len()).expect("test data length should fit i64")); + fi.data = data.map(Bytes::from); + fi + } + + async fn setup_codec_data_blocks_first_encoded_bitrot_readers( + erasure: &coding::Erasure, + data: &[u8], + missing_indexes: &[usize], + bitrot_corrupt_indexes: &[usize], + inconsistent_source_indexes: &[usize], + hash_algo: HashAlgorithm, + ) -> BitrotReaderSetup { + let shard_size = erasure.shard_size(); + let encoded_shards = erasure.encode_data(data).expect("test stripe should encode"); + let mut files = Vec::with_capacity(encoded_shards.len()); + for (index, shard) in encoded_shards.into_iter().enumerate() { + if missing_indexes.contains(&index) { + files.push(encoded_reader_setup_fileinfo(None)); + continue; + } + + let mut shard = shard.to_vec(); + if inconsistent_source_indexes.contains(&index) + && let Some(byte) = shard.first_mut() + { + *byte ^= 0x80; + } + + let mut writer = BitrotWriter::new(Cursor::new(Vec::new()), shard_size, hash_algo.clone()); + writer.write(&shard).await.expect("test shard should write with bitrot hash"); + let mut encoded = writer.into_inner().into_inner(); + if bitrot_corrupt_indexes.contains(&index) { + let data_offset = hash_algo.size(); + let byte = encoded + .get_mut(data_offset) + .expect("encoded test shard should contain payload bytes"); + *byte ^= 0x80; + } + + files.push(encoded_reader_setup_fileinfo(Some(encoded))); + } + let disks = vec![None; files.len()]; + + temp_env::async_with_vars([(ENV_RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_READER_SETUP, Some("true"))], async { + create_bitrot_readers_until_quorum( + &files, + &disks, + "bucket", + "object", + 1, + 0, + shard_size, + shard_size, + hash_algo, + false, + false, + erasure.data_shards, + erasure.parity_shards, + BitrotReaderSetupMode::VerifyReconstruction, + None, + None, + ) + .await + }) + .await + } + + async fn decode_codec_data_blocks_first_setup( + erasure: coding::Erasure, + data: &[u8], + readers: Vec>, + ) -> std::io::Result> { + let source = coding::decode::ParallelReader::new_with_metrics_path_and_reconstruction_verification( + readers, + erasure.clone(), + 0, + data.len(), + Some(GET_OBJECT_PATH_CODEC_STREAMING_RUSTFS_ENGINE), + ); + let engine = CodecStreamingDecodeEngine::rustfs(&erasure).expect("rustfs codec engine should be created"); + let mut reader = coding::decode_reader::ErasureDecodeReader::new_with_metrics_path( + source, + engine, + data.len(), + GET_OBJECT_PATH_CODEC_STREAMING_RUSTFS_ENGINE, + ) + .expect("codec streaming reader should be constructed"); + let mut decoded = Vec::new(); + reader.read_to_end(&mut decoded).await?; + Ok(decoded) + } + #[tokio::test] async fn bitrot_reader_setup_stops_at_read_quorum() { let setup = setup_inline_bitrot_readers( @@ -4655,7 +4833,7 @@ mod tests { assert!(setup.has_setup_quorum(2, 2, BitrotReaderSetupMode::ReadQuorum)); assert_eq!(setup.available_shards(), 2); - assert!(setup.scheduled_shards() < 4); + assert_eq!(setup.scheduled_shards(), 2); assert_eq!(setup.readers.iter().filter(|reader| reader.is_some()).count(), 4); let fallback_index = setup @@ -4689,7 +4867,25 @@ mod tests { assert!(setup.has_setup_quorum(2, 2, BitrotReaderSetupMode::ReadQuorum)); assert_eq!(setup.available_shards(), 2); - assert!(setup.scheduled_shards() < 4); + assert_eq!(setup.scheduled_shards(), 2); + assert_eq!(setup.deferred_shards(), 2); + } + + #[tokio::test] + async fn bitrot_reader_setup_preference_can_apply_to_verify_mode() { + let setup = setup_inline_bitrot_readers_with_preference( + vec![Some(b"aaaa"), Some(b"bbbb"), Some(b"cccc"), Some(b"dddd")], + 2, + 2, + BitrotReaderSetupMode::VerifyReconstruction, + true, + ) + .await; + + assert!(setup.has_setup_quorum(2, 2, BitrotReaderSetupMode::VerifyReconstruction)); + assert_eq!(setup.available_data_shards(2), 2); + assert_eq!(setup.scheduled_shards(), 2); + assert_eq!(setup.attempted_shards(), 2); assert_eq!(setup.deferred_shards(), 2); } @@ -4707,6 +4903,7 @@ mod tests { assert!(setup.has_setup_quorum(2, 2, BitrotReaderSetupMode::ReadQuorum)); assert_eq!(setup.available_shards(), 2); assert_eq!(setup.completed_failed_shards(), 1); + assert_eq!(setup.scheduled_shards(), 3); assert!(setup.ready.iter().skip(2).any(|ready| *ready)); } @@ -4739,7 +4936,7 @@ mod tests { assert!(setup.has_setup_quorum(2, 2, BitrotReaderSetupMode::VerifyReconstruction)); assert_eq!(setup.available_data_shards(2), 2); - assert!(setup.scheduled_shards() < 4); + assert_eq!(setup.scheduled_shards(), 2); } #[tokio::test] @@ -4758,6 +4955,7 @@ mod tests { assert_eq!(setup.available_data_shards(2), 1); assert!(setup.data_shards_attempted(2)); assert_eq!(setup.completed_failed_shards(), 1); + assert_eq!(setup.scheduled_shards(), 4); } #[tokio::test] @@ -4806,6 +5004,69 @@ mod tests { assert_eq!(setup.completed_failed_shards(), 1); } + #[tokio::test] + async fn codec_data_blocks_first_recovers_corrupt_data_with_deferred_parity() { + let erasure = coding::Erasure::new(2, 2, 64); + let data = (0..64u8).collect::>(); + let setup = + setup_codec_data_blocks_first_encoded_bitrot_readers(&erasure, &data, &[], &[0], &[], HashAlgorithm::HighwayHash256) + .await; + + assert!(setup.has_setup_quorum(2, 2, BitrotReaderSetupMode::VerifyReconstruction)); + assert_eq!(setup.scheduled_shards(), 2); + assert_eq!(setup.attempted_shards(), 2); + assert_eq!(setup.available_shards(), 2); + assert_eq!(setup.deferred_shards(), 2); + + let decoded = decode_codec_data_blocks_first_setup(erasure, &data, setup.readers) + .await + .expect("deferred parity should recover the corrupt data shard"); + + assert_eq!(decoded, data); + } + + #[tokio::test] + async fn codec_data_blocks_first_reconstructs_missing_data_shard() { + let erasure = coding::Erasure::new(2, 2, 64); + let data = (0..64u8).rev().collect::>(); + let setup = + setup_codec_data_blocks_first_encoded_bitrot_readers(&erasure, &data, &[0], &[], &[], HashAlgorithm::HighwayHash256) + .await; + + assert!(setup.has_setup_quorum(2, 2, BitrotReaderSetupMode::VerifyReconstruction)); + assert_eq!(setup.scheduled_shards(), 4); + assert_eq!(setup.available_shards(), 3); + assert_eq!(setup.available_data_shards(2), 1); + assert_eq!(setup.completed_failed_shards(), 1); + + let decoded = decode_codec_data_blocks_first_setup(erasure, &data, setup.readers) + .await + .expect("parity shards should reconstruct the missing data shard"); + + assert_eq!(decoded, data); + } + + #[tokio::test] + async fn codec_data_blocks_first_rejects_inconsistent_deferred_reconstruction_source() { + let erasure = coding::Erasure::new(2, 2, 64); + let data = (0..64u8).map(|value| value.wrapping_mul(3)).collect::>(); + let setup = + setup_codec_data_blocks_first_encoded_bitrot_readers(&erasure, &data, &[], &[0], &[2], HashAlgorithm::HighwayHash256) + .await; + + assert!(setup.has_setup_quorum(2, 2, BitrotReaderSetupMode::VerifyReconstruction)); + assert_eq!(setup.scheduled_shards(), 2); + assert_eq!(setup.attempted_shards(), 2); + assert_eq!(setup.deferred_shards(), 2); + + let err = decode_codec_data_blocks_first_setup(erasure, &data, setup.readers) + .await + .expect_err("inconsistent deferred parity source must fail reconstruction verification"); + + assert_eq!(err.kind(), ErrorKind::InvalidData); + assert!(err.to_string().contains("inconsistent read source shards")); + } + #[test] fn codec_streaming_reader_gate_is_conservative() { temp_env::with_vars( @@ -4897,6 +5158,165 @@ mod tests { }); } + #[test] + fn rustfs_codec_streaming_uses_conservative_default_min_size() { + temp_env::with_vars( + [ + (ENV_RUSTFS_GET_CODEC_STREAMING_ENABLE, Some("true")), + (ENV_RUSTFS_GET_CODEC_STREAMING_ENGINE, Some(GET_CODEC_STREAMING_ENGINE_RUSTFS)), + (ENV_RUSTFS_GET_CODEC_STREAMING_ROLLOUT, Some("benchmark")), + (ENV_RUSTFS_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED, Some("true")), + (ENV_RUSTFS_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED, Some("true")), + (ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE, None::<&str>), + (ENV_RUSTFS_GET_CODEC_STREAMING_RUSTFS_MIN_SIZE, None::<&str>), + ], + || { + let below_threshold_fi = codec_streaming_test_fileinfo(512 * 1024, 1); + let below_threshold_object_info = codec_streaming_test_object_info(&below_threshold_fi); + assert_eq!( + codec_streaming_reader_gate_for_test(&None, &below_threshold_object_info, &below_threshold_fi, true).decision, + GetCodecStreamingDecision::Fallback(GetCodecStreamingFallbackReason::BelowMinSize) + ); + + let threshold_fi = codec_streaming_test_fileinfo(1_048_576, 1); + let threshold_object_info = codec_streaming_test_object_info(&threshold_fi); + assert_eq!( + codec_streaming_reader_gate_for_test(&None, &threshold_object_info, &threshold_fi, true).decision, + GetCodecStreamingDecision::Use + ); + }, + ); + } + + #[test] + fn rustfs_codec_streaming_min_size_override_can_lower_threshold() { + temp_env::with_vars( + [ + (ENV_RUSTFS_GET_CODEC_STREAMING_ENABLE, Some("true")), + (ENV_RUSTFS_GET_CODEC_STREAMING_ENGINE, Some(GET_CODEC_STREAMING_ENGINE_RUSTFS)), + (ENV_RUSTFS_GET_CODEC_STREAMING_ROLLOUT, Some("benchmark")), + (ENV_RUSTFS_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED, Some("true")), + (ENV_RUSTFS_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED, Some("true")), + (ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE, None::<&str>), + (ENV_RUSTFS_GET_CODEC_STREAMING_RUSTFS_MIN_SIZE, Some("524288")), + ], + || { + let fi = codec_streaming_test_fileinfo(512 * 1024, 1); + let object_info = codec_streaming_test_object_info(&fi); + assert_eq!( + codec_streaming_reader_gate_for_test(&None, &object_info, &fi, true).decision, + GetCodecStreamingDecision::Use + ); + }, + ); + } + + #[test] + fn codec_streaming_data_blocks_first_gate_defaults_to_off() { + temp_env::with_vars( + [ + (ENV_RUSTFS_GET_CODEC_STREAMING_ENABLE, Some("true")), + (ENV_RUSTFS_GET_CODEC_STREAMING_ENGINE, Some(GET_CODEC_STREAMING_ENGINE_RUSTFS)), + (ENV_RUSTFS_GET_CODEC_STREAMING_ROLLOUT, Some("benchmark")), + (ENV_RUSTFS_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED, Some("true")), + (ENV_RUSTFS_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED, Some("true")), + (ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE, Some("1")), + (ENV_RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_ENABLE, None::<&str>), + ], + || { + let fi = codec_streaming_test_fileinfo(512 * 1024, 1); + let object_info = codec_streaming_test_object_info(&fi); + let gate = codec_streaming_reader_gate_for_test(&None, &object_info, &fi, true); + + assert_eq!(gate.decision, GetCodecStreamingDecision::Use); + assert!(!gate.prefer_data_blocks_first_reader_setup); + }, + ); + } + + #[test] + fn codec_streaming_data_blocks_first_gate_allows_small_plain_single_part() { + temp_env::with_vars( + [ + (ENV_RUSTFS_GET_CODEC_STREAMING_ENABLE, Some("true")), + (ENV_RUSTFS_GET_CODEC_STREAMING_ENGINE, Some(GET_CODEC_STREAMING_ENGINE_RUSTFS)), + (ENV_RUSTFS_GET_CODEC_STREAMING_ROLLOUT, Some("benchmark")), + (ENV_RUSTFS_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED, Some("true")), + (ENV_RUSTFS_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED, Some("true")), + (ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE, Some("1")), + (ENV_RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_ENABLE, Some("true")), + (ENV_RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_MAX_SIZE, Some("524288")), + ], + || { + let fi = codec_streaming_test_fileinfo(512 * 1024, 1); + let object_info = codec_streaming_test_object_info(&fi); + let gate = codec_streaming_reader_gate_for_test(&None, &object_info, &fi, true); + + assert_eq!(gate.decision, GetCodecStreamingDecision::Use); + assert!(gate.prefer_data_blocks_first_reader_setup); + }, + ); + } + + #[test] + fn codec_streaming_data_blocks_first_gate_rejects_large_and_non_plain_objects() { + temp_env::with_vars( + [ + (ENV_RUSTFS_GET_CODEC_STREAMING_ENABLE, Some("true")), + (ENV_RUSTFS_GET_CODEC_STREAMING_ENGINE, Some(GET_CODEC_STREAMING_ENGINE_RUSTFS)), + (ENV_RUSTFS_GET_CODEC_STREAMING_ROLLOUT, Some("benchmark")), + (ENV_RUSTFS_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED, Some("true")), + (ENV_RUSTFS_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED, Some("true")), + (ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE, Some("1")), + (ENV_RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_ENABLE, Some("true")), + (ENV_RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_MAX_SIZE, Some("524288")), + ], + || { + let large_fi = codec_streaming_test_fileinfo(768 * 1024, 1); + let large_object_info = codec_streaming_test_object_info(&large_fi); + let large_gate = codec_streaming_reader_gate_for_test(&None, &large_object_info, &large_fi, true); + assert_eq!(large_gate.decision, GetCodecStreamingDecision::Use); + assert!(!large_gate.prefer_data_blocks_first_reader_setup); + + let multipart_fi = codec_streaming_test_fileinfo(512 * 1024, 2); + let multipart_object_info = codec_streaming_test_object_info(&multipart_fi); + let multipart_gate = codec_streaming_reader_gate_for_test(&None, &multipart_object_info, &multipart_fi, true); + assert!(!multipart_gate.prefer_data_blocks_first_reader_setup); + + let range = Some(HTTPRangeSpec { + is_suffix_length: false, + start: 0, + end: 1, + }); + let range_gate = codec_streaming_reader_gate_for_test(&range, &large_object_info, &large_fi, true); + assert!(!range_gate.prefer_data_blocks_first_reader_setup); + }, + ); + } + + #[test] + fn generic_codec_streaming_min_size_override_remains_authoritative() { + temp_env::with_vars( + [ + (ENV_RUSTFS_GET_CODEC_STREAMING_ENABLE, Some("true")), + (ENV_RUSTFS_GET_CODEC_STREAMING_ENGINE, Some(GET_CODEC_STREAMING_ENGINE_RUSTFS)), + (ENV_RUSTFS_GET_CODEC_STREAMING_ROLLOUT, Some("benchmark")), + (ENV_RUSTFS_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED, Some("true")), + (ENV_RUSTFS_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED, Some("true")), + (ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE, Some("1")), + (ENV_RUSTFS_GET_CODEC_STREAMING_RUSTFS_MIN_SIZE, Some("786432")), + ], + || { + let fi = codec_streaming_test_fileinfo(1024, 1); + let object_info = codec_streaming_test_object_info(&fi); + assert_eq!( + codec_streaming_reader_gate_for_test(&None, &object_info, &fi, true).decision, + GetCodecStreamingDecision::Use + ); + }, + ); + } + #[test] fn codec_streaming_engine_env_is_ignored_when_streaming_is_disabled() { temp_env::with_vars( diff --git a/crates/io-metrics/src/lib.rs b/crates/io-metrics/src/lib.rs index b6fd3a9db..f9af5d945 100644 --- a/crates/io-metrics/src/lib.rs +++ b/crates/io-metrics/src/lib.rs @@ -190,6 +190,9 @@ pub const GET_OBJECT_SIZE_BUCKET_LE_4_KIB: &str = "le_4kib"; pub const GET_OBJECT_SIZE_BUCKET_LE_16_KIB: &str = "le_16kib"; pub const GET_OBJECT_SIZE_BUCKET_LE_64_KIB: &str = "le_64kib"; pub const GET_OBJECT_SIZE_BUCKET_LE_128_KIB: &str = "le_128kib"; +pub const GET_OBJECT_SIZE_BUCKET_LE_192_KIB: &str = "le_192kib"; +pub const GET_OBJECT_SIZE_BUCKET_LE_256_KIB: &str = "le_256kib"; +pub const GET_OBJECT_SIZE_BUCKET_LE_512_KIB: &str = "le_512kib"; pub const GET_OBJECT_SIZE_BUCKET_LE_1_MIB: &str = "le_1mib"; pub const GET_OBJECT_SIZE_BUCKET_GT_1_MIB: &str = "gt_1mib"; @@ -201,7 +204,10 @@ pub const fn get_object_size_bucket(size_bytes: i64) -> &'static str { 4_097..=16_384 => GET_OBJECT_SIZE_BUCKET_LE_16_KIB, 16_385..=65_536 => GET_OBJECT_SIZE_BUCKET_LE_64_KIB, 65_537..=131_072 => GET_OBJECT_SIZE_BUCKET_LE_128_KIB, - 131_073..=1_048_576 => GET_OBJECT_SIZE_BUCKET_LE_1_MIB, + 131_073..=196_608 => GET_OBJECT_SIZE_BUCKET_LE_192_KIB, + 196_609..=262_144 => GET_OBJECT_SIZE_BUCKET_LE_256_KIB, + 262_145..=524_288 => GET_OBJECT_SIZE_BUCKET_LE_512_KIB, + 524_289..=1_048_576 => GET_OBJECT_SIZE_BUCKET_LE_1_MIB, _ => GET_OBJECT_SIZE_BUCKET_GT_1_MIB, } } @@ -1124,6 +1130,29 @@ pub fn record_get_object_reader_setup_strategy(strategy: &'static str, mode: &'s .increment(1); } +/// Record the bitrot reader setup scheduling strategy with bounded GET attribution labels. +#[inline(always)] +pub fn record_get_object_reader_setup_strategy_by_size( + path: &'static str, + strategy: &'static str, + mode: &'static str, + object_class: &'static str, + size_bucket: &'static str, +) { + if !get_stage_metrics_enabled() { + return; + } + counter!( + "rustfs_io_get_object_reader_setup_strategy_by_size_total", + "path" => path, + "strategy" => strategy, + "mode" => mode, + "object_class" => object_class, + "size_bucket" => size_bucket + ) + .increment(1); +} + /// Record the final bitrot reader setup fanout shape for a GET read. #[inline(always)] pub fn record_get_object_reader_setup_fanout( @@ -1170,6 +1199,71 @@ pub fn record_get_object_reader_setup_fanout( .record(shard_read_fanout_to_f64(deferred)); } +/// Record the final bitrot reader setup fanout shape with bounded GET attribution labels. +#[inline(always)] +#[allow(clippy::too_many_arguments)] +pub fn record_get_object_reader_setup_fanout_by_size( + path: &'static str, + strategy: &'static str, + mode: &'static str, + object_class: &'static str, + size_bucket: &'static str, + scheduled: usize, + attempted: usize, + ready: usize, + failed: usize, + deferred: usize, +) { + if !get_stage_metrics_enabled() { + return; + } + histogram!( + "rustfs_io_get_object_reader_setup_scheduled_by_size", + "path" => path, + "strategy" => strategy, + "mode" => mode, + "object_class" => object_class, + "size_bucket" => size_bucket + ) + .record(shard_read_fanout_to_f64(scheduled)); + histogram!( + "rustfs_io_get_object_reader_setup_attempted_by_size", + "path" => path, + "strategy" => strategy, + "mode" => mode, + "object_class" => object_class, + "size_bucket" => size_bucket + ) + .record(shard_read_fanout_to_f64(attempted)); + histogram!( + "rustfs_io_get_object_reader_setup_ready_by_size", + "path" => path, + "strategy" => strategy, + "mode" => mode, + "object_class" => object_class, + "size_bucket" => size_bucket + ) + .record(shard_read_fanout_to_f64(ready)); + histogram!( + "rustfs_io_get_object_reader_setup_failed_by_size", + "path" => path, + "strategy" => strategy, + "mode" => mode, + "object_class" => object_class, + "size_bucket" => size_bucket + ) + .record(shard_read_fanout_to_f64(failed)); + histogram!( + "rustfs_io_get_object_reader_setup_deferred_by_size", + "path" => path, + "strategy" => strategy, + "mode" => mode, + "object_class" => object_class, + "size_bucket" => size_bucket + ) + .record(shard_read_fanout_to_f64(deferred)); +} + /// Record GetObject metadata resolution duration. #[inline(always)] pub fn record_get_object_metadata_phase_duration(duration_secs: f64) { @@ -1458,6 +1552,72 @@ pub fn record_put_object_path(path: &'static str) { counter!("rustfs_s3_put_object_path_total", "path" => path).increment(1); } +#[inline(always)] +fn put_object_size_bucket(size_bytes: i64) -> &'static str { + const MI_B: i64 = 1024 * 1024; + + match size_bytes { + i64::MIN..=0 => "unknown", + 1..=MI_B => "le_1mib", + _ if size_bytes <= 10 * MI_B => "le_10mib", + _ if size_bytes <= 16 * MI_B => "le_16mib", + _ if size_bytes <= 32 * MI_B => "le_32mib", + _ if size_bytes <= 64 * MI_B => "le_64mib", + _ => "gt_64mib", + } +} + +#[inline(always)] +fn put_object_buffer_bucket(buffer_size: usize) -> &'static str { + const KI_B: usize = 1024; + const MI_B: usize = 1024 * 1024; + + match buffer_size { + 0..=65536 => "le_64kib", + _ if buffer_size <= 128 * KI_B => "le_128kib", + _ if buffer_size <= 256 * KI_B => "le_256kib", + _ if buffer_size <= 512 * KI_B => "le_512kib", + _ if buffer_size <= MI_B => "le_1mib", + _ => "gt_1mib", + } +} + +#[inline(always)] +fn bool_label(value: bool) -> &'static str { + if value { "true" } else { "false" } +} + +#[inline(always)] +pub fn record_put_object_diagnostics( + path: &'static str, + eager_status: &'static str, + size_bytes: i64, + buffer_size: usize, + large_concurrency_tuning: bool, +) { + if !put_stage_metrics_enabled() { + return; + } + + let size_bucket = put_object_size_bucket(size_bytes); + let buffer_bucket = put_object_buffer_bucket(buffer_size); + counter!( + "rustfs_s3_put_object_diagnostics_total", + "path" => path, + "eager_status" => eager_status, + "size_bucket" => size_bucket, + "buffer_bucket" => buffer_bucket, + "large_concurrency_tuning" => bool_label(large_concurrency_tuning), + ) + .increment(1); + histogram!( + "rustfs_s3_put_object_selected_buffer_size_bytes", + "path" => path, + "size_bucket" => size_bucket, + ) + .record(buffer_size as f64); +} + #[inline(always)] pub fn record_put_object_stage_duration(stage: &'static str, duration_ms: f64) { if !put_stage_metrics_enabled() { @@ -1935,7 +2095,26 @@ mod tests { record_get_object_shard_read_observation("codec_streaming", 0, "data", "local", "success", "none", 1024, 0.004, 0.001); record_get_object_shard_read_cost_summary("codec_streaming", 3, 1, 2, 0, 4, 4, 4, true); record_get_object_reader_setup_strategy("data_blocks_first", "read_quorum"); + record_get_object_reader_setup_strategy_by_size( + "codec_streaming", + "data_blocks_first", + "read_quorum", + "plain_single_part", + "le_1mib", + ); record_get_object_reader_setup_fanout("data_blocks_first", "read_quorum", 3, 2, 2, 0, 2); + record_get_object_reader_setup_fanout_by_size( + "codec_streaming", + "data_blocks_first", + "read_quorum", + "plain_single_part", + "le_1mib", + 3, + 2, + 2, + 0, + 2, + ); assert!(0.005_f64.is_sign_positive()); } @@ -1977,9 +2156,22 @@ mod tests { record_put_object_path("write_inline"); record_put_object_stage_duration("ingress_prepare", 12.5); record_put_object_stage_duration("set_disk_encode", 8.0); + record_put_object_diagnostics("zero_copy_eager", "eligible", 32 * 1024 * 1024, 256 * 1024, true); + assert!(put_stage_metrics_enabled()); set_put_stage_metrics_enabled(false); } + #[test] + fn test_put_object_diagnostic_buckets() { + assert_eq!(put_object_size_bucket(0), "unknown"); + assert_eq!(put_object_size_bucket(10 * 1024 * 1024), "le_10mib"); + assert_eq!(put_object_size_bucket(32 * 1024 * 1024), "le_32mib"); + assert_eq!(put_object_size_bucket(32 * 1024 * 1024 + 1), "le_64mib"); + assert_eq!(put_object_buffer_bucket(64 * 1024), "le_64kib"); + assert_eq!(put_object_buffer_bucket(256 * 1024), "le_256kib"); + assert_eq!(put_object_buffer_bucket(2 * 1024 * 1024), "gt_1mib"); + } + #[test] fn test_put_stage_metrics_disabled_by_default() { let _guard = METRICS_FLAG_LOCK.lock().unwrap_or_else(|e| e.into_inner()); @@ -2098,7 +2290,13 @@ mod tests { assert_eq!(get_object_size_bucket((16 * 1024) + 1), GET_OBJECT_SIZE_BUCKET_LE_64_KIB); assert_eq!(get_object_size_bucket(100 * 1024), GET_OBJECT_SIZE_BUCKET_LE_128_KIB); assert_eq!(get_object_size_bucket(128 * 1024), GET_OBJECT_SIZE_BUCKET_LE_128_KIB); - assert_eq!(get_object_size_bucket((128 * 1024) + 1), GET_OBJECT_SIZE_BUCKET_LE_1_MIB); + assert_eq!(get_object_size_bucket((128 * 1024) + 1), GET_OBJECT_SIZE_BUCKET_LE_192_KIB); + assert_eq!(get_object_size_bucket(192 * 1024), GET_OBJECT_SIZE_BUCKET_LE_192_KIB); + assert_eq!(get_object_size_bucket((192 * 1024) + 1), GET_OBJECT_SIZE_BUCKET_LE_256_KIB); + assert_eq!(get_object_size_bucket(256 * 1024), GET_OBJECT_SIZE_BUCKET_LE_256_KIB); + assert_eq!(get_object_size_bucket((256 * 1024) + 1), GET_OBJECT_SIZE_BUCKET_LE_512_KIB); + assert_eq!(get_object_size_bucket(512 * 1024), GET_OBJECT_SIZE_BUCKET_LE_512_KIB); + assert_eq!(get_object_size_bucket((512 * 1024) + 1), GET_OBJECT_SIZE_BUCKET_LE_1_MIB); assert_eq!(get_object_size_bucket(1024 * 1024), GET_OBJECT_SIZE_BUCKET_LE_1_MIB); assert_eq!(get_object_size_bucket((1024 * 1024) + 1), GET_OBJECT_SIZE_BUCKET_GT_1_MIB); } diff --git a/rustfs/src/app/object_usecase.rs b/rustfs/src/app/object_usecase.rs index 9de95333f..73b4e4837 100644 --- a/rustfs/src/app/object_usecase.rs +++ b/rustfs/src/app/object_usecase.rs @@ -156,6 +156,17 @@ use s3s::stream::{ByteStream, RemainingLength}; use s3s::{S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error}; const DEFAULT_PUT_LARGE_CONCURRENCY_TUNING_MIN_SIZE_BYTES: i64 = 32 * 1024 * 1024; +const ENV_ZERO_COPY_EAGER_PUT_MAX_SIZE_BYTES: &str = "RUSTFS_ZERO_COPY_EAGER_PUT_MAX_SIZE_BYTES"; +const DEFAULT_ZERO_COPY_EAGER_PUT_MAX_SIZE_BYTES: usize = 16 * 1024 * 1024; +const PUT_EAGER_STATUS_ELIGIBLE: &str = "eligible"; +const PUT_EAGER_STATUS_EXTRACT: &str = "extract"; +const PUT_EAGER_STATUS_COMPRESSED: &str = "compressed"; +const PUT_EAGER_STATUS_ENCRYPTED: &str = "encrypted"; +const PUT_EAGER_STATUS_INVALID_SIZE: &str = "invalid_size"; +const PUT_EAGER_STATUS_ABOVE_EAGER_MAX: &str = "above_eager_max"; +const PUT_EAGER_STATUS_ZERO_COPY_INELIGIBLE: &str = "zero_copy_ineligible"; +const PUT_EAGER_STATUS_AWS_CHUNKED_MISSING_DECODED_LENGTH: &str = "aws_chunked_missing_decoded_length"; +static CACHED_ZERO_COPY_EAGER_PUT_MAX_SIZE_BYTES: std::sync::OnceLock = std::sync::OnceLock::new(); use std::collections::HashMap; use std::ops::Add; use std::path::Path; @@ -1006,6 +1017,7 @@ fn should_use_zero_copy(size: i64, headers: &HeaderMap) -> bool { true } +#[cfg(test)] fn should_use_zero_copy_eager_put_path( size: i64, headers: &HeaderMap, @@ -1013,25 +1025,68 @@ fn should_use_zero_copy_eager_put_path( should_compress: bool, is_extract: bool, ) -> bool { - const ZERO_COPY_EAGER_PUT_MAX_SIZE: i64 = 32 * 1024 * 1024; + zero_copy_eager_put_path_status(size, headers, server_side_encryption_requested, should_compress, is_extract) + == PUT_EAGER_STATUS_ELIGIBLE +} - if is_extract || should_compress || server_side_encryption_requested { - return false; +fn zero_copy_eager_put_path_status( + size: i64, + headers: &HeaderMap, + server_side_encryption_requested: bool, + should_compress: bool, + is_extract: bool, +) -> &'static str { + zero_copy_eager_put_path_status_with_max_size( + size, + headers, + server_side_encryption_requested, + should_compress, + is_extract, + zero_copy_eager_put_max_size_bytes(), + ) +} + +fn zero_copy_eager_put_path_status_with_max_size( + size: i64, + headers: &HeaderMap, + server_side_encryption_requested: bool, + should_compress: bool, + is_extract: bool, + max_size: i64, +) -> &'static str { + if is_extract { + return PUT_EAGER_STATUS_EXTRACT; + } + if should_compress { + return PUT_EAGER_STATUS_COMPRESSED; + } + if server_side_encryption_requested { + return PUT_EAGER_STATUS_ENCRYPTED; } - if size <= 0 || size > ZERO_COPY_EAGER_PUT_MAX_SIZE { - return false; + if size <= 0 { + return PUT_EAGER_STATUS_INVALID_SIZE; + } + if size > max_size { + return PUT_EAGER_STATUS_ABOVE_EAGER_MAX; } if !should_use_zero_copy(size, headers) { - return false; + return PUT_EAGER_STATUS_ZERO_COPY_INELIGIBLE; } if request_uses_aws_chunked(headers) && decoded_content_length_from_headers(headers).ok().flatten().is_none() { - return false; + return PUT_EAGER_STATUS_AWS_CHUNKED_MISSING_DECODED_LENGTH; } - true + PUT_EAGER_STATUS_ELIGIBLE +} + +fn zero_copy_eager_put_max_size_bytes() -> i64 { + let configured = *CACHED_ZERO_COPY_EAGER_PUT_MAX_SIZE_BYTES.get_or_init(|| { + rustfs_utils::get_env_usize(ENV_ZERO_COPY_EAGER_PUT_MAX_SIZE_BYTES, DEFAULT_ZERO_COPY_EAGER_PUT_MAX_SIZE_BYTES) + }); + i64::try_from(configured).unwrap_or(i64::MAX) } fn has_put_sse_request_headers(headers: &HeaderMap) -> bool { @@ -2853,7 +2908,8 @@ impl DefaultObjectUsecase { // Buffer sizes range from 32KB to 4MB depending on file size and configured workload profile. // Concurrency-aware adjustment reduces buffer size under high PUT concurrency to lower memory pressure. let base_buffer_size = get_buffer_size_opt_in(size); - let buffer_size = if Self::should_use_large_put_concurrency_tuning(size) { + let use_large_put_concurrency_tuning = Self::should_use_large_put_concurrency_tuning(size); + let buffer_size = if use_large_put_concurrency_tuning { get_put_concurrency_aware_buffer_size(size, base_buffer_size) } else { base_buffer_size @@ -2872,8 +2928,9 @@ impl DefaultObjectUsecase { let use_small_eager_put_path = should_use_small_eager_put_path(size, &req.headers, server_side_encryption_requested, should_compress, false); - let use_zero_copy_eager_put_path = - should_use_zero_copy_eager_put_path(size, &req.headers, server_side_encryption_requested, should_compress, false); + let zero_copy_eager_put_path_status = + zero_copy_eager_put_path_status(size, &req.headers, server_side_encryption_requested, should_compress, false); + let use_zero_copy_eager_put_path = zero_copy_eager_put_path_status == PUT_EAGER_STATUS_ELIGIBLE; if use_zero_copy_eager_put_path { counter!(buffered_write::ATTEMPTS_TOTAL).increment(1); histogram!(buffered_write::ATTEMPT_SIZE_BYTES).record(size as f64); @@ -2887,6 +2944,13 @@ impl DefaultObjectUsecase { } else { "streaming" }; + rustfs_io_metrics::record_put_object_diagnostics( + put_path, + zero_copy_eager_put_path_status, + size, + buffer_size, + use_large_put_concurrency_tuning, + ); let store = get_validated_store(&bucket).await?; @@ -6618,8 +6682,31 @@ mod tests { let headers = HeaderMap::new(); assert!(should_use_zero_copy_eager_put_path(2 * 1024 * 1024, &headers, false, false, false)); - assert!(should_use_zero_copy_eager_put_path(32 * 1024 * 1024, &headers, false, false, false)); - assert!(!should_use_zero_copy_eager_put_path(32 * 1024 * 1024 + 1, &headers, false, false, false)); + assert!(should_use_zero_copy_eager_put_path(16 * 1024 * 1024, &headers, false, false, false)); + assert!(!should_use_zero_copy_eager_put_path(16 * 1024 * 1024 + 1, &headers, false, false, false)); + assert_eq!( + zero_copy_eager_put_path_status(16 * 1024 * 1024, &headers, false, false, false), + PUT_EAGER_STATUS_ELIGIBLE + ); + assert_eq!( + zero_copy_eager_put_path_status(16 * 1024 * 1024 + 1, &headers, false, false, false), + PUT_EAGER_STATUS_ABOVE_EAGER_MAX + ); + } + + #[test] + fn zero_copy_eager_put_path_status_honors_configured_cap() { + let headers = HeaderMap::new(); + let max_size = 64 * 1024 * 1024; + + assert_eq!( + zero_copy_eager_put_path_status_with_max_size(33 * 1024 * 1024, &headers, false, false, false, max_size), + PUT_EAGER_STATUS_ELIGIBLE + ); + assert_eq!( + zero_copy_eager_put_path_status_with_max_size(65 * 1024 * 1024, &headers, false, false, false, max_size), + PUT_EAGER_STATUS_ABOVE_EAGER_MAX + ); } #[test] @@ -6629,6 +6716,18 @@ mod tests { assert!(!should_use_zero_copy_eager_put_path(2 * 1024 * 1024, &headers, true, false, false)); assert!(!should_use_zero_copy_eager_put_path(2 * 1024 * 1024, &headers, false, true, false)); assert!(!should_use_zero_copy_eager_put_path(2 * 1024 * 1024, &headers, false, false, true)); + assert_eq!( + zero_copy_eager_put_path_status(2 * 1024 * 1024, &headers, true, false, false), + PUT_EAGER_STATUS_ENCRYPTED + ); + assert_eq!( + zero_copy_eager_put_path_status(2 * 1024 * 1024, &headers, false, true, false), + PUT_EAGER_STATUS_COMPRESSED + ); + assert_eq!( + zero_copy_eager_put_path_status(2 * 1024 * 1024, &headers, false, false, true), + PUT_EAGER_STATUS_EXTRACT + ); } #[tokio::test] diff --git a/scripts/analyze_put_service_metrics_deltas.py b/scripts/analyze_put_service_metrics_deltas.py new file mode 100644 index 000000000..0fe48f7b0 --- /dev/null +++ b/scripts/analyze_put_service_metrics_deltas.py @@ -0,0 +1,322 @@ +#!/usr/bin/env python3 +"""Summarize per-round PUT service metric deltas from before/after snapshots.""" + +from __future__ import annotations + +import argparse +import csv +import re +from collections import defaultdict +from pathlib import Path + + +PROM_LINE_RE = re.compile( + r"^([a-zA-Z_:][a-zA-Z0-9_:]*)(?:\{(.*)\})?\s+([-+]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][-+]?\d+)?)" +) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--capture-csv", required=True, help="aggregate_service_metrics_captures.csv path") + parser.add_argument("--delta-csv", required=True, help="Output generic metric delta CSV") + parser.add_argument("--path-summary-csv", required=True, help="Output PUT path delta summary CSV") + parser.add_argument("--stage-summary-csv", required=True, help="Output PUT stage duration summary CSV") + return parser.parse_args() + + +def split_label_items(labels: str) -> list[str]: + items: list[str] = [] + start = 0 + in_quotes = False + escaped = False + for index, char in enumerate(labels): + if escaped: + escaped = False + continue + if char == "\\": + escaped = True + continue + if char == '"': + in_quotes = not in_quotes + continue + if char == "," and not in_quotes: + items.append(labels[start:index]) + start = index + 1 + items.append(labels[start:]) + return [item.strip() for item in items if item.strip()] + + +def parse_labels(labels: str) -> dict[str, str]: + parsed: dict[str, str] = {} + for item in split_label_items(labels): + key, sep, value = item.partition("=") + if not sep: + continue + value = value.strip() + if value.startswith('"') and value.endswith('"') and len(value) >= 2: + value = value[1:-1] + value = value.replace(r"\"", '"').replace(r"\\", "\\") + parsed[key.strip()] = value + return parsed + + +def canonical_labels(labels: dict[str, str]) -> str: + return ",".join(f'{key}="{escape_label_value(labels[key])}"' for key in sorted(labels)) + + +def escape_label_value(value: str) -> str: + return value.replace("\\", r"\\").replace('"', r"\"").replace("\n", r"\n") + + +def parse_prom(path: Path) -> dict[tuple[str, str], float]: + metrics: dict[tuple[str, str], float] = {} + with path.open("r", encoding="utf-8") as file: + for line in file: + line = line.strip() + if not line or line.startswith("#"): + continue + match = PROM_LINE_RE.match(line) + if not match: + continue + metric, raw_labels, raw_value = match.groups() + labels = canonical_labels(parse_labels(raw_labels or "")) + try: + value = float(raw_value) + except ValueError: + continue + metrics[(metric, labels)] = value + return metrics + + +def classify_metric(metric: str) -> str: + if metric == "rustfs_s3_put_object_path_total": + return "path_total" + if metric == "rustfs_s3_put_object_diagnostics_total": + return "diagnostics_total" + if metric.endswith("_bucket"): + return "histogram_bucket" + if metric.endswith("_count"): + return "histogram_count" + if metric.endswith("_sum"): + return "histogram_sum" + if metric.endswith("_total"): + return "counter_total" + return "gauge_delta" + + +def is_monotonic_metric(metric: str) -> bool: + return ( + metric.endswith("_total") + or metric.endswith("_bucket") + or metric.endswith("_count") + or metric.endswith("_sum") + ) + + +def group_capture_rows(capture_csv: Path) -> dict[tuple[str, str, str, str, str, str], dict[str, dict[str, str]]]: + groups: dict[tuple[str, str, str, str, str, str], dict[str, dict[str, str]]] = defaultdict(dict) + with capture_csv.open("r", encoding="utf-8", newline="") as file: + for row in csv.DictReader(file): + phase = row.get("phase", "") + if phase not in {"before", "after"}: + continue + key = ( + row.get("concurrency", ""), + row.get("run_dir", ""), + row.get("size", ""), + row.get("tool", ""), + row.get("round", ""), + row.get("attempt", ""), + ) + groups[key][phase] = row + return groups + + +def label_value(labels: str, name: str) -> str: + return parse_labels(labels).get(name, "") + + +def write_outputs(args: argparse.Namespace) -> None: + capture_csv = Path(args.capture_csv) + delta_csv = Path(args.delta_csv) + path_summary_csv = Path(args.path_summary_csv) + stage_summary_csv = Path(args.stage_summary_csv) + for output in (delta_csv, path_summary_csv, stage_summary_csv): + output.parent.mkdir(parents=True, exist_ok=True) + + groups = group_capture_rows(capture_csv) + + delta_fields = [ + "concurrency", + "run_dir", + "size", + "tool", + "round", + "attempt", + "source", + "before_status", + "after_status", + "metric", + "labels", + "before", + "after", + "delta", + "classification", + ] + path_fields = [ + "concurrency", + "run_dir", + "size", + "tool", + "round", + "attempt", + "path", + "delta", + "before_status", + "after_status", + ] + stage_fields = [ + "concurrency", + "run_dir", + "size", + "tool", + "round", + "attempt", + "stage", + "count_delta", + "sum_delta", + "avg_ms", + "before_status", + "after_status", + ] + + with ( + delta_csv.open("w", encoding="utf-8", newline="") as delta_file, + path_summary_csv.open("w", encoding="utf-8", newline="") as path_file, + stage_summary_csv.open("w", encoding="utf-8", newline="") as stage_file, + ): + delta_writer = csv.DictWriter(delta_file, fieldnames=delta_fields) + path_writer = csv.DictWriter(path_file, fieldnames=path_fields) + stage_writer = csv.DictWriter(stage_file, fieldnames=stage_fields) + delta_writer.writeheader() + path_writer.writeheader() + stage_writer.writeheader() + + for key in sorted(groups): + phases = groups[key] + before_row = phases.get("before") + after_row = phases.get("after") + if not before_row or not after_row: + continue + + before_path = Path(before_row.get("snapshot_file", "")) + after_path = Path(after_row.get("snapshot_file", "")) + if before_row.get("status") != "ok" or after_row.get("status") != "ok": + continue + if not before_path.is_file() or not after_path.is_file(): + continue + + before = parse_prom(before_path) + after = parse_prom(after_path) + concurrency, run_dir, size, tool, round_no, attempt = key + source = after_row.get("source", before_row.get("source", "")) + before_status = before_row.get("status", "") + after_status = after_row.get("status", "") + + stage_counts: dict[str, float] = defaultdict(float) + stage_sums: dict[str, float] = defaultdict(float) + + for metric_key in sorted(after): + metric, labels = metric_key + before_value = before.get(metric_key, 0.0) + after_value = after.get(metric_key, 0.0) + delta = after_value - before_value + if is_monotonic_metric(metric) and delta < 0: + continue + if delta == 0: + continue + + classification = classify_metric(metric) + delta_writer.writerow( + { + "concurrency": concurrency, + "run_dir": run_dir, + "size": size, + "tool": tool, + "round": round_no, + "attempt": attempt, + "source": source, + "before_status": before_status, + "after_status": after_status, + "metric": metric, + "labels": labels, + "before": f"{before_value:.12g}", + "after": f"{after_value:.12g}", + "delta": f"{delta:.12g}", + "classification": classification, + } + ) + + if metric == "rustfs_s3_put_object_path_total": + path_writer.writerow( + { + "concurrency": concurrency, + "run_dir": run_dir, + "size": size, + "tool": tool, + "round": round_no, + "attempt": attempt, + "path": label_value(labels, "path"), + "delta": f"{delta:.12g}", + "before_status": before_status, + "after_status": after_status, + } + ) + + if metric.endswith("_stage_duration_ms_count"): + stage = label_value(labels, "stage") + if stage: + stage_counts[stage] += delta + elif metric.endswith("_stage_duration_ms_sum"): + stage = label_value(labels, "stage") + if stage: + stage_sums[stage] += delta + elif metric.endswith("_stage_duration_seconds_count"): + stage = label_value(labels, "stage") + if stage: + stage_counts[stage] += delta + elif metric.endswith("_stage_duration_seconds_sum"): + stage = label_value(labels, "stage") + if stage: + stage_sums[stage] += delta * 1000.0 + + for stage in sorted(set(stage_counts) | set(stage_sums)): + count_delta = stage_counts.get(stage, 0.0) + if count_delta <= 0: + continue + sum_delta = stage_sums.get(stage, 0.0) + avg_ms = sum_delta / count_delta if count_delta else 0.0 + stage_writer.writerow( + { + "concurrency": concurrency, + "run_dir": run_dir, + "size": size, + "tool": tool, + "round": round_no, + "attempt": attempt, + "stage": stage, + "count_delta": f"{count_delta:.12g}", + "sum_delta": f"{sum_delta:.12g}", + "avg_ms": f"{avg_ms:.6f}", + "before_status": before_status, + "after_status": after_status, + } + ) + + +def main() -> None: + write_outputs(parse_args()) + + +if __name__ == "__main__": + main() diff --git a/scripts/run_get_codec_streaming_smoke.sh b/scripts/run_get_codec_streaming_smoke.sh index bd3d13c6f..dd4009f9b 100755 --- a/scripts/run_get_codec_streaming_smoke.sh +++ b/scripts/run_get_codec_streaming_smoke.sh @@ -35,6 +35,8 @@ PROFILE_ORDER="normal" CODEC_ENGINES="legacy" CODEC_MAX_INFLIGHT=1 CODEC_READER_SETUP="all_shards" +CODEC_DATA_BLOCKS_FIRST_GATE="off" +CODEC_DATA_BLOCKS_FIRST_MAX_SIZE="" CODEC_MULTIPART="off" CODEC_MULTIPART_MAX_PARTS=256 METADATA_EARLY_STOP="off" @@ -62,7 +64,9 @@ OUT_DIR="" RUSTFS_BIN="${PROJECT_ROOT}/target/release/rustfs" WARP_BIN="warp" PYTHON_BIN="python3" -CODEC_MIN_SIZE=1 +CODEC_MIN_SIZE="" +DEFAULT_CODEC_MIN_SIZE=1048576 +DEFAULT_RUSTFS_CODEC_MIN_SIZE=1048576 RUST_LOG="warn" HEALTH_TIMEOUT_SECS=60 COMPAT_OBJECT_KEY="__rustfs_get_v2_pr24_compat/object.bin" @@ -102,6 +106,12 @@ Core options: --codec-reader-setup RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_READER_SETUP for codec verify reader setup (default: all_shards) + --codec-data-blocks-first-gate + Enable size-gated data-blocks-first reader setup for + codec-rustfs plain single-part GETs (default: off) + --codec-data-blocks-first-max-size + RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_MAX_SIZE + for the gated path (default: server default) --codec-multipart Enable multipart codec streaming opt-in for codec profiles (default: off) --codec-multipart-max-parts @@ -172,7 +182,8 @@ Binary/options: --rustfs-bin RustFS binary (default: target/release/rustfs) --warp-bin warp binary (default: warp) --python-bin Python binary for SigV4 compatibility probe (default: python3) - --codec-min-size RUSTFS_GET_CODEC_STREAMING_MIN_SIZE (default: 1) + --codec-min-size Override RUSTFS_GET_CODEC_STREAMING_MIN_SIZE + (default: unset, use server size-aware default) --compat-object-key Object key used by the compatibility probe --compat-object-size Object size used by the compatibility probe (default: 65536) --skip-compat-probe skip post-benchmark compatibility/fallback probes @@ -282,6 +293,8 @@ parse_args() { --shard-locality-preference) SHARD_LOCALITY_PREFERENCE="$2"; shift 2 ;; --codec-max-inflight) CODEC_MAX_INFLIGHT="$2"; shift 2 ;; --codec-reader-setup) CODEC_READER_SETUP="$2"; shift 2 ;; + --codec-data-blocks-first-gate) CODEC_DATA_BLOCKS_FIRST_GATE="$2"; shift 2 ;; + --codec-data-blocks-first-max-size) CODEC_DATA_BLOCKS_FIRST_MAX_SIZE="$2"; shift 2 ;; --codec-multipart) CODEC_MULTIPART="$2"; shift 2 ;; --codec-multipart-max-parts) CODEC_MULTIPART_MAX_PARTS="$2"; shift 2 ;; --handoff-attribution) OUTPUT_HANDOFF_ATTRIBUTION=true; shift ;; @@ -388,6 +401,13 @@ validate_args() { all_shards|data_blocks_first) ;; *) die "--codec-reader-setup must be all_shards or data_blocks_first" ;; esac + case "$CODEC_DATA_BLOCKS_FIRST_GATE" in + on|off) ;; + *) die "--codec-data-blocks-first-gate must be on or off" ;; + esac + if [[ -n "$CODEC_DATA_BLOCKS_FIRST_MAX_SIZE" ]]; then + validate_positive_int "$CODEC_DATA_BLOCKS_FIRST_MAX_SIZE" "--codec-data-blocks-first-max-size" + fi validate_positive_int "$CODEC_MULTIPART_MAX_PARTS" "--codec-multipart-max-parts" validate_positive_int "$ROUNDS" "--rounds" validate_positive_int "$RETRY_PER_ROUND" "--retry-per-round" @@ -399,11 +419,6 @@ validate_args() { per-round|prepare-once|existing-only) ;; *) die "--warp-object-lifecycle must be per-round, prepare-once, or existing-only" ;; esac - if [[ "$WARP_OBJECT_LIFECYCLE" != "per-round" ]]; then - local size_count - size_count="$(printf '%s\n' "$SIZES" | awk -F',' '{ count=0; for (i=1; i<=NF; i++) { gsub(/^[ \t]+|[ \t]+$/, "", $i); if ($i != "") count++ } print count }')" - [[ "$size_count" -eq 1 ]] || die "--warp-object-lifecycle=${WARP_OBJECT_LIFECYCLE} currently requires a single --sizes value to avoid mixing object sizes in one bucket" - fi if [[ -n "$GET_OBJECT_METADATA_CACHE_MAX_ENTRIES" ]]; then validate_positive_int "$GET_OBJECT_METADATA_CACHE_MAX_ENTRIES" "--metadata-cache-max-entries" fi @@ -422,7 +437,9 @@ validate_args() { *) die "--local-read-copy-method must be mmap_copy or direct_read_copy" ;; esac fi - validate_positive_int "$CODEC_MIN_SIZE" "--codec-min-size" + if [[ -n "$CODEC_MIN_SIZE" ]]; then + validate_positive_int "$CODEC_MIN_SIZE" "--codec-min-size" + fi validate_positive_int "$COMPAT_OBJECT_SIZE" "--compat-object-size" validate_positive_int "$RESOURCE_SAMPLE_INTERVAL_SECS" "--resource-sample-interval-secs" validate_positive_int "$HEALTH_TIMEOUT_SECS" "--health-timeout-secs" @@ -705,6 +722,8 @@ metadata_early_stop=${METADATA_EARLY_STOP} shard_locality_preference=${SHARD_LOCALITY_PREFERENCE} codec_max_inflight=${CODEC_MAX_INFLIGHT} codec_reader_setup=${CODEC_READER_SETUP} +codec_data_blocks_first_gate=${CODEC_DATA_BLOCKS_FIRST_GATE} +codec_data_blocks_first_max_size=${CODEC_DATA_BLOCKS_FIRST_MAX_SIZE:-server-default} codec_rollout_codec_profile=benchmark codec_body_compat_confirmed_codec_profile=true codec_header_compat_confirmed_codec_profile=true @@ -748,7 +767,7 @@ dry_run=${DRY_RUN} rustfs_bin=${RUSTFS_BIN} warp_bin=${WARP_BIN} python_bin=${PYTHON_BIN} -codec_min_size=${CODEC_MIN_SIZE} +codec_min_size=${CODEC_MIN_SIZE:-server-default} compat_object_key=${COMPAT_OBJECT_KEY} compat_object_size=${COMPAT_OBJECT_SIZE} command_line=${command_line% } @@ -819,6 +838,20 @@ profile_metrics_path() { esac } +compat_probe_codec_min_size() { + local profile="$1" + if [[ -n "$CODEC_MIN_SIZE" ]]; then + echo "$CODEC_MIN_SIZE" + return + fi + + case "$profile" in + codec-rustfs) echo "${RUSTFS_GET_CODEC_STREAMING_RUSTFS_MIN_SIZE:-$DEFAULT_RUSTFS_CODEC_MIN_SIZE}" ;; + codec-legacy) echo "$DEFAULT_CODEC_MIN_SIZE" ;; + *) echo "0" ;; + esac +} + profile_data_root() { local profile="$1" echo "${OUT_DIR}/${profile}/data" @@ -845,6 +878,36 @@ single_benchmark_size() { }' } +benchmark_sizes() { + printf '%s\n' "$SIZES" | awk -F',' '{ + for (i=1; i<=NF; i++) { + gsub(/^[ \t]+|[ \t]+$/, "", $i) + if ($i != "") { + print $i + } + } + }' +} + +benchmark_size_count() { + benchmark_sizes | awk 'END { print NR + 0 }' +} + +sanitize_bucket_suffix() { + printf '%s' "$1" | tr '[:upper:]' '[:lower:]' | tr -c 'a-z0-9-' '-' +} + +bucket_for_size() { + local size="$1" + local size_count + size_count="$(benchmark_size_count)" + if [[ "$WARP_OBJECT_LIFECYCLE" == "per-round" || "$size_count" -le 1 ]]; then + echo "$BUCKET" + return + fi + echo "${BUCKET}-$(sanitize_bucket_suffix "$size")" +} + write_manifest() { local profile="$1" local profile_dir="$2" @@ -895,10 +958,12 @@ RUSTFS_GET_METADATA_EARLY_STOP_ENABLE=$(bool_from_on_off "$METADATA_EARLY_STOP") RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE=$(bool_from_on_off "$SHARD_LOCALITY_PREFERENCE") RUSTFS_GET_CODEC_STREAMING_MAX_INFLIGHT=${CODEC_MAX_INFLIGHT} RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_READER_SETUP=$(codec_reader_setup_data_blocks_first) +RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_ENABLE=$(bool_from_on_off "$CODEC_DATA_BLOCKS_FIRST_GATE") +RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_MAX_SIZE=${CODEC_DATA_BLOCKS_FIRST_MAX_SIZE:-server-default} RUSTFS_GET_CODEC_STREAMING_MULTIPART_ENABLE=$(bool_from_on_off "$CODEC_MULTIPART") RUSTFS_GET_CODEC_STREAMING_MULTIPART_MAX_PARTS=${CODEC_MULTIPART_MAX_PARTS} RUSTFS_GET_OUTPUT_HANDOFF_ATTRIBUTION_ENABLE=${OUTPUT_HANDOFF_ATTRIBUTION} -RUSTFS_GET_CODEC_STREAMING_MIN_SIZE=${CODEC_MIN_SIZE} +RUSTFS_GET_CODEC_STREAMING_MIN_SIZE=${CODEC_MIN_SIZE:-server-default} RUSTFS_GET_OBJECT_METADATA_CACHE_MAX_ENTRIES=${GET_OBJECT_METADATA_CACHE_MAX_ENTRIES} RUSTFS_GET_SMALL_OBJECT_DIRECT_MEMORY=${GET_SMALL_OBJECT_DIRECT_MEMORY} RUSTFS_GET_SMALL_OBJECT_DIRECT_MEMORY_THRESHOLD=${GET_SMALL_OBJECT_DIRECT_MEMORY_THRESHOLD} @@ -1027,11 +1092,22 @@ start_server() { export RUSTFS_GET_CODEC_STREAMING_MAX_INFLIGHT="$CODEC_MAX_INFLIGHT" RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_READER_SETUP="$(codec_reader_setup_data_blocks_first)" export RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_READER_SETUP + RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_ENABLE="$(bool_from_on_off "$CODEC_DATA_BLOCKS_FIRST_GATE")" + export RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_ENABLE + if [[ -n "$CODEC_DATA_BLOCKS_FIRST_MAX_SIZE" ]]; then + export RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_MAX_SIZE="$CODEC_DATA_BLOCKS_FIRST_MAX_SIZE" + else + unset RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_MAX_SIZE + fi RUSTFS_GET_CODEC_STREAMING_MULTIPART_ENABLE="$(bool_from_on_off "$CODEC_MULTIPART")" export RUSTFS_GET_CODEC_STREAMING_MULTIPART_ENABLE export RUSTFS_GET_CODEC_STREAMING_MULTIPART_MAX_PARTS="$CODEC_MULTIPART_MAX_PARTS" export RUSTFS_GET_OUTPUT_HANDOFF_ATTRIBUTION_ENABLE="$OUTPUT_HANDOFF_ATTRIBUTION" - export RUSTFS_GET_CODEC_STREAMING_MIN_SIZE="$CODEC_MIN_SIZE" + if [[ -n "$CODEC_MIN_SIZE" ]]; then + export RUSTFS_GET_CODEC_STREAMING_MIN_SIZE="$CODEC_MIN_SIZE" + else + unset RUSTFS_GET_CODEC_STREAMING_MIN_SIZE + fi if [[ -n "$GET_OBJECT_METADATA_CACHE_MAX_ENTRIES" ]]; then export RUSTFS_GET_OBJECT_METADATA_CACHE_MAX_ENTRIES="$GET_OBJECT_METADATA_CACHE_MAX_ENTRIES" fi @@ -1078,8 +1154,6 @@ prepare_warp_existing_objects() { local profile="$1" local profile_dir="${OUT_DIR}/${profile}" local prepare_dir="${profile_dir}/warp_prepare" - local size - size="$(single_benchmark_size)" if [[ "$WARP_OBJECT_LIFECYCLE" != "prepare-once" ]]; then return 0 @@ -1087,43 +1161,50 @@ prepare_warp_existing_objects() { mkdir -p "$prepare_dir" - local cmd=( - "$WARP_BIN" get - --host "$ADDRESS" - --access-key "$ACCESS_KEY" - --secret-key "$SECRET_KEY" - --bucket "$BUCKET" - --obj.size "$size" - --concurrent "$CONCURRENCY" - --duration "$WARP_PREPARE_DURATION" - --region "$REGION" - --noclear - ) - if [[ -n "$WARP_OBJECTS" ]]; then - cmd+=(--objects "$WARP_OBJECTS") - fi + local size bucket size_slug + while IFS= read -r size; do + [[ -n "$size" ]] || continue + bucket="$(bucket_for_size "$size")" + size_slug="$(sanitize_bucket_suffix "$size")" - { - printf 'profile=%s\n' "$profile" - printf 'mode=prepare-once\n' - printf 'size=%s\n' "$size" - printf 'duration=%s\n' "$WARP_PREPARE_DURATION" - printf 'bucket=%s\n' "$BUCKET" - printf 'command=' - printf '%q ' "${cmd[@]}" - printf '\n' - } >"${prepare_dir}/manifest.env" + local cmd=( + "$WARP_BIN" get + --host "$ADDRESS" + --access-key "$ACCESS_KEY" + --secret-key "$SECRET_KEY" + --bucket "$bucket" + --obj.size "$size" + --concurrent "$CONCURRENCY" + --duration "$WARP_PREPARE_DURATION" + --region "$REGION" + --noclear + ) + if [[ -n "$WARP_OBJECTS" ]]; then + cmd+=(--objects "$WARP_OBJECTS") + fi - if [[ "$DRY_RUN" == "true" ]]; then - log "[DRY-RUN] prepare existing warp objects profile=${profile} size=${size}" - printf '[DRY-RUN] ' >"${prepare_dir}/prepare.log" - printf '%q ' "${cmd[@]}" >>"${prepare_dir}/prepare.log" - printf '\n' >>"${prepare_dir}/prepare.log" - return 0 - fi + { + printf 'profile=%s\n' "$profile" + printf 'mode=prepare-once\n' + printf 'size=%s\n' "$size" + printf 'duration=%s\n' "$WARP_PREPARE_DURATION" + printf 'bucket=%s\n' "$bucket" + printf 'command=' + printf '%q ' "${cmd[@]}" + printf '\n' + } >"${prepare_dir}/manifest-${size_slug}.env" - log "Preparing existing warp objects profile=${profile} size=${size} lifecycle=${WARP_OBJECT_LIFECYCLE}..." - "${cmd[@]}" >"${prepare_dir}/prepare.log" 2>&1 + if [[ "$DRY_RUN" == "true" ]]; then + log "[DRY-RUN] prepare existing warp objects profile=${profile} size=${size} bucket=${bucket}" + printf '[DRY-RUN] ' >"${prepare_dir}/prepare-${size_slug}.log" + printf '%q ' "${cmd[@]}" >>"${prepare_dir}/prepare-${size_slug}.log" + printf '\n' >>"${prepare_dir}/prepare-${size_slug}.log" + continue + fi + + log "Preparing existing warp objects profile=${profile} size=${size} bucket=${bucket} lifecycle=${WARP_OBJECT_LIFECYCLE}..." + "${cmd[@]}" >"${prepare_dir}/prepare-${size_slug}.log" 2>&1 + done < <(benchmark_sizes) } run_bench() { @@ -1162,6 +1243,9 @@ run_bench() { lifecycle_args="${lifecycle_args} --objects ${WARP_OBJECTS}" fi cmd+=(--extra-args "$lifecycle_args") + if [[ "$(benchmark_size_count)" -gt 1 ]]; then + cmd+=(--bucket-size-suffix) + fi fi if [[ "$DIAGNOSTIC_METRICS" == "true" && -z "$DIAGNOSTIC_PROMETHEUS_QUERY_URL" ]]; then cmd+=( @@ -1268,7 +1352,9 @@ if payload.get("status") != "success": samples = [ sample for sample in payload.get("data", {}).get("result", []) - if not service_name or sample.get("metric", {}).get("service.name") == service_name + if not service_name + or sample.get("metric", {}).get("service.name") == service_name + or sample.get("metric", {}).get("service_name") == service_name ] lines = [line for sample in samples if (line := sample_to_line(sample))] if not lines: @@ -2508,7 +2594,7 @@ EOF return fi "$PYTHON_BIN" - "$(endpoint_url)" "$ACCESS_KEY" "$SECRET_KEY" "$REGION" "$BUCKET" \ - "$COMPAT_OBJECT_KEY" "$COMPAT_OBJECT_SIZE" "$CODEC_MIN_SIZE" "$compat_dir" "$profile" \ + "$COMPAT_OBJECT_KEY" "$COMPAT_OBJECT_SIZE" "$(compat_probe_codec_min_size "$profile")" "$compat_dir" "$profile" \ "$COMPRESSED_FALLBACK_PROBE" "$COMPRESSED_PROBE_EXTENSION" "$COMPRESSED_PROBE_MIME_TYPE" \ "$CODEC_MULTIPART" "$CODEC_MULTIPART_MAX_PARTS" <<'PY' import base64 @@ -3338,6 +3424,8 @@ for profile_dir in profile_dirs: "shard_locality_preference": manifest.get("RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE", ""), "codec_max_inflight": manifest.get("RUSTFS_GET_CODEC_STREAMING_MAX_INFLIGHT", ""), "codec_reader_setup": manifest.get("RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_READER_SETUP", ""), + "codec_data_blocks_first_gate": manifest.get("RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_ENABLE", ""), + "codec_data_blocks_first_max_size": manifest.get("RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_MAX_SIZE", ""), "local_read_copy_method": manifest.get("RUSTFS_OBJECT_MMAP_READ_METHOD", ""), "output_handoff_attribution": manifest.get("RUSTFS_GET_OUTPUT_HANDOFF_ATTRIBUTION_ENABLE", ""), "round_cooldown_secs": manifest.get("round_cooldown_secs", ""), @@ -3387,6 +3475,8 @@ fieldnames = [ "shard_locality_preference", "codec_max_inflight", "codec_reader_setup", + "codec_data_blocks_first_gate", + "codec_data_blocks_first_max_size", "local_read_copy_method", "output_handoff_attribution", "round_cooldown_secs", @@ -3423,6 +3513,8 @@ baseline_fields = [ "shard_locality_preference", "codec_max_inflight", "codec_reader_setup", + "codec_data_blocks_first_gate", + "codec_data_blocks_first_max_size", "local_read_copy_method", "output_handoff_attribution", "round_cooldown_secs", @@ -3467,6 +3559,8 @@ for row in rows: "shard_locality_preference": row["shard_locality_preference"], "codec_max_inflight": row["codec_max_inflight"], "codec_reader_setup": row["codec_reader_setup"], + "codec_data_blocks_first_gate": row["codec_data_blocks_first_gate"], + "codec_data_blocks_first_max_size": row["codec_data_blocks_first_max_size"], "local_read_copy_method": row["local_read_copy_method"], "output_handoff_attribution": row["output_handoff_attribution"], "round_cooldown_secs": row["round_cooldown_secs"], diff --git a/scripts/run_object_batch_bench_enhanced.sh b/scripts/run_object_batch_bench_enhanced.sh index 230cbe45d..e8a774fc1 100755 --- a/scripts/run_object_batch_bench_enhanced.sh +++ b/scripts/run_object_batch_bench_enhanced.sh @@ -14,12 +14,14 @@ ENDPOINT="" ACCESS_KEY="" SECRET_KEY="" BUCKET="rustfs-bench" +BUCKET_SIZE_SUFFIX=false REGION="us-east-1" CONCURRENCY=128 SIZES="$DEFAULT_SIZES" OUT_DIR="" INSECURE=false DRY_RUN=false +PYTHON_BIN="python3" # warp options WARP_BIN="warp" @@ -34,7 +36,7 @@ SAMPLES=20000 ROUNDS=3 RETRY_PER_ROUND=2 RETRY_SLEEP_SECS=2 -COOLDOWN_SECS=0 +COOLDOWN_SECS=20 BASELINE_CSV="" EXTRA_ARGS=() FAILED_FINAL_ROUNDS=0 @@ -44,7 +46,13 @@ SERVICE_METRICS_CAPTURE_ATTEMPTS=3 SERVICE_METRICS_CAPTURE_RETRY_SECS=1 SERVICE_METRICS_CONNECT_TIMEOUT_SECS=2 SERVICE_METRICS_MAX_TIME_SECS=15 -SERVICE_METRICS_FILTER_REGEX="rustfs_io_get_object_" +SERVICE_METRICS_SETTLE_SECS=0 +SERVICE_METRICS_FILTER_REGEX="" +SERVICE_METRICS_FILTER_REGEX_EXPLICIT=false +SERVICE_PROMETHEUS_QUERY_URL="" +SERVICE_PROMETHEUS_QUERY="" +SERVICE_PROMETHEUS_QUERY_EXPLICIT=false +SERVICE_METRICS_SERVICE_NAME="" usage() { cat <<'USAGE' @@ -60,12 +68,15 @@ Required: Core options: --bucket Bucket name (default: rustfs-bench) + --bucket-size-suffix Append a sanitized object size suffix to the + bucket for each size --region Region (default: us-east-1) --concurrency Concurrency for all sizes (default: 128) --sizes Comma-separated sizes (default: 1KiB..10MiB matrix) --out-dir Output directory (default: target/bench/object-batch-enhanced-) --insecure Allow insecure TLS --dry-run Print commands only, do not execute + --python-bin Python binary for Prometheus query capture (default: python3) Warp options: --warp-bin warp binary (default: warp) @@ -80,11 +91,17 @@ Enhanced options: --rounds Benchmark rounds per size (default: 3) --retry-per-round Retry count per failed round (default: 2) --retry-sleep-secs Sleep seconds between retries (default: 2) - --cooldown-secs Sleep seconds between rounds/sizes (default: 0) + --cooldown-secs Sleep seconds between rounds/sizes (default: 20) --round-cooldown-secs Compatibility alias for --cooldown-secs --baseline-csv Baseline median CSV to compare --extra-args Extra args appended to tool command, quoted as one string --service-metrics-url Optional Prometheus scrape URL captured before/after each round attempt + --service-prometheus-query-url + Optional Prometheus HTTP API /api/v1/query URL for OTLP-exported metrics + --service-prometheus-query PromQL used with --service-prometheus-query-url + (default: auto by tool/mode) + --service-metrics-service-name + Optional service.name/service_name label filter for Prometheus query results --service-metrics-dir Output directory for per-round service metric snapshots --service-metrics-attempts Capture attempts for each snapshot (default: 3) --service-metrics-retry-secs Sleep seconds between failed capture attempts (default: 1) @@ -92,9 +109,11 @@ Enhanced options: Curl connect timeout for each metrics capture (default: 2) --service-metrics-max-time-secs Curl max time for each metrics capture (default: 15) + --service-metrics-settle-secs + Sleep seconds before after-snapshot capture (default: 0) --service-metrics-filter-regex Regex for retained metrics lines after scrape - (default: rustfs_io_get_object_) + (default: auto by tool/mode) Output files: round_results.csv One row per round attempt (with retry trace) @@ -136,12 +155,14 @@ parse_args() { --access-key) ACCESS_KEY="$2"; shift 2 ;; --secret-key) SECRET_KEY="$2"; shift 2 ;; --bucket) BUCKET="$2"; shift 2 ;; + --bucket-size-suffix) BUCKET_SIZE_SUFFIX=true; shift ;; --region) REGION="$2"; shift 2 ;; --concurrency) CONCURRENCY="$2"; shift 2 ;; --sizes) SIZES="$2"; shift 2 ;; --out-dir) OUT_DIR="$2"; shift 2 ;; --insecure) INSECURE=true; shift ;; --dry-run) DRY_RUN=true; shift ;; + --python-bin) PYTHON_BIN="$2"; shift 2 ;; --warp-bin) WARP_BIN="$2"; shift 2 ;; --warp-mode) WARP_MODE="$2"; shift 2 ;; --duration) DURATION="$2"; shift 2 ;; @@ -154,12 +175,16 @@ parse_args() { --round-cooldown-secs) COOLDOWN_SECS="$2"; shift 2 ;; --baseline-csv) BASELINE_CSV="$2"; shift 2 ;; --service-metrics-url) SERVICE_METRICS_URL="$2"; shift 2 ;; + --service-prometheus-query-url) SERVICE_PROMETHEUS_QUERY_URL="$2"; shift 2 ;; + --service-prometheus-query) SERVICE_PROMETHEUS_QUERY="$2"; SERVICE_PROMETHEUS_QUERY_EXPLICIT=true; shift 2 ;; + --service-metrics-service-name) SERVICE_METRICS_SERVICE_NAME="$2"; shift 2 ;; --service-metrics-dir) SERVICE_METRICS_DIR="$2"; shift 2 ;; --service-metrics-attempts) SERVICE_METRICS_CAPTURE_ATTEMPTS="$2"; shift 2 ;; --service-metrics-retry-secs) SERVICE_METRICS_CAPTURE_RETRY_SECS="$2"; shift 2 ;; --service-metrics-connect-timeout-secs) SERVICE_METRICS_CONNECT_TIMEOUT_SECS="$2"; shift 2 ;; --service-metrics-max-time-secs) SERVICE_METRICS_MAX_TIME_SECS="$2"; shift 2 ;; - --service-metrics-filter-regex) SERVICE_METRICS_FILTER_REGEX="$2"; shift 2 ;; + --service-metrics-settle-secs) SERVICE_METRICS_SETTLE_SECS="$2"; shift 2 ;; + --service-metrics-filter-regex) SERVICE_METRICS_FILTER_REGEX="$2"; SERVICE_METRICS_FILTER_REGEX_EXPLICIT=true; shift 2 ;; --extra-args) # shellcheck disable=SC2206 EXTRA_ARGS=($2) @@ -205,6 +230,48 @@ cooldown_sleep() { fi } +default_service_metrics_filter_regex() { + if [[ "$TOOL" == "warp" ]]; then + case "$WARP_MODE" in + get) + echo "rustfs_io_get_object_|rustfs_s3_get_object_|rustfs_zero_copy_read|rustfs_mmap_|rustfs_get_object_" + return + ;; + put) + echo "rustfs_s3_put_object_|rustfs_io_put_object_|rustfs_zero_copy_write|rustfs_buffer_|rustfs_ec_|rustfs_io_bytespool_" + return + ;; + mixed) + echo "rustfs_s3_get_object_|rustfs_io_get_object_|rustfs_s3_put_object_|rustfs_io_put_object_|rustfs_zero_copy_|rustfs_buffer_|rustfs_ec_" + return + ;; + esac + fi + + echo "rustfs_s3_put_object_|rustfs_io_put_object_|rustfs_s3_get_object_|rustfs_io_get_object_" +} + +default_service_prometheus_query() { + if [[ "$TOOL" == "warp" ]]; then + case "$WARP_MODE" in + get) + echo '{__name__=~"rustfs_(io_get_object|s3_get_object|zero_copy_read|mmap|get_object)_.*"}' + return + ;; + put) + echo '{__name__=~"rustfs_(s3_put_object|io_put_object|zero_copy_write|buffer|ec|io_bytespool)_.*"}' + return + ;; + mixed) + echo '{__name__=~"rustfs_(s3_get_object|io_get_object|s3_put_object|io_put_object|zero_copy|buffer|ec)_.*"}' + return + ;; + esac + fi + + echo '{__name__=~"rustfs_(s3_put_object|io_put_object|s3_get_object|io_get_object)_.*"}' +} + validate_args() { if [[ "$TOOL" != "warp" && "$TOOL" != "s3bench" ]]; then echo "ERROR: --tool must be warp or s3bench" >&2 @@ -223,13 +290,21 @@ validate_args() { validate_nonnegative_int "$SERVICE_METRICS_CAPTURE_RETRY_SECS" "--service-metrics-retry-secs" validate_positive_int "$SERVICE_METRICS_CONNECT_TIMEOUT_SECS" "--service-metrics-connect-timeout-secs" validate_positive_int "$SERVICE_METRICS_MAX_TIME_SECS" "--service-metrics-max-time-secs" - if [[ -n "$SERVICE_METRICS_URL" && -z "$SERVICE_METRICS_DIR" ]]; then - echo "ERROR: --service-metrics-dir is required when --service-metrics-url is set" >&2 + validate_nonnegative_int "$SERVICE_METRICS_SETTLE_SECS" "--service-metrics-settle-secs" + if [[ -n "$SERVICE_METRICS_URL" && -n "$SERVICE_PROMETHEUS_QUERY_URL" ]]; then + echo "ERROR: --service-metrics-url and --service-prometheus-query-url are mutually exclusive" >&2 + exit 1 + fi + if [[ -n "$SERVICE_METRICS_URL" || -n "$SERVICE_PROMETHEUS_QUERY_URL" ]] && [[ -z "$SERVICE_METRICS_DIR" ]]; then + echo "ERROR: --service-metrics-dir is required when service metrics capture is enabled" >&2 exit 1 fi if [[ -n "$SERVICE_METRICS_URL" ]]; then require_cmd curl fi + if [[ -n "$SERVICE_PROMETHEUS_QUERY_URL" ]]; then + require_cmd "$PYTHON_BIN" + fi if [[ "$TOOL" == "s3bench" ]]; then validate_positive_int "$SAMPLES" "--samples" fi @@ -238,6 +313,10 @@ validate_args() { exit 1 fi if [[ "$TOOL" == "warp" ]]; then + if [[ "$WARP_MODE" != "get" && "$WARP_MODE" != "put" && "$WARP_MODE" != "mixed" ]]; then + echo "ERROR: --warp-mode must be get, put, or mixed" >&2 + exit 1 + fi local warp_host warp_host="$(normalize_warp_host "$ENDPOINT")" if [[ -z "$warp_host" ]]; then @@ -245,6 +324,89 @@ validate_args() { exit 1 fi fi + if [[ "$SERVICE_METRICS_FILTER_REGEX_EXPLICIT" != "true" && -z "$SERVICE_METRICS_FILTER_REGEX" ]]; then + SERVICE_METRICS_FILTER_REGEX="$(default_service_metrics_filter_regex)" + fi + if [[ "$SERVICE_PROMETHEUS_QUERY_EXPLICIT" != "true" && -z "$SERVICE_PROMETHEUS_QUERY" ]]; then + SERVICE_PROMETHEUS_QUERY="$(default_service_prometheus_query)" + fi +} + +git_value() { + local default_value="$1" + shift + git "$@" 2>/dev/null || echo "$default_value" +} + +git_dirty_state() { + if ! git rev-parse --is-inside-work-tree >/dev/null 2>&1; then + echo "unknown" + return + fi + if ! git diff --quiet --ignore-submodules --; then + echo "dirty" + return + fi + if ! git diff --cached --quiet --ignore-submodules --; then + echo "dirty" + return + fi + echo "clean" +} + +write_run_manifest() { + local manifest_file="$OUT_DIR/run_manifest.env" + local started_at_utc git_commit git_branch git_dirty rustc_version uname_s service_metrics_csv + started_at_utc="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + git_commit="$(git_value unknown rev-parse --short HEAD)" + git_branch="$(git_value unknown branch --show-current)" + git_dirty="$(git_dirty_state)" + rustc_version="$(rustc --version 2>/dev/null || echo unknown)" + uname_s="$(uname -a 2>/dev/null || echo unknown)" + service_metrics_csv="${SERVICE_METRICS_CSV:-}" + + cat >"$manifest_file" < "$ROUND_CSV" echo "size,tool,concurrency,successful_rounds,failed_rounds,median_throughput_bps,median_reqps,median_latency_ms,median_req_p90_ms,median_req_p99_ms" > "$MEDIAN_CSV" + if [[ -n "$SERVICE_METRICS_URL" || -n "$SERVICE_PROMETHEUS_QUERY_URL" ]]; then + echo "size,tool,round,attempt,phase,source,status,capture_attempt,raw_bytes,snapshot_bytes,status_file,snapshot_file,filter_regex,prometheus_query" > "$SERVICE_METRICS_CSV" + fi + write_run_manifest } trim() { echo "$1" | awk '{$1=$1;print}' } +sanitize_bucket_suffix() { + printf '%s' "$1" | tr '[:upper:]' '[:lower:]' | tr -c 'a-z0-9-' '-' +} + +bucket_for_size() { + local size="$1" + if [[ "$BUCKET_SIZE_SUFFIX" != "true" ]]; then + echo "$BUCKET" + return + fi + echo "${BUCKET}-$(sanitize_bucket_suffix "$size")" +} + to_bps() { local human="$1" local number unit factor @@ -400,17 +580,119 @@ filter_service_metrics_snapshot() { fi } +settle_before_after_metrics_capture() { + if (( SERVICE_METRICS_SETTLE_SECS <= 0 )); then + return 0 + fi + if [[ "$DRY_RUN" == "true" ]]; then + echo "[DRY-RUN] sleep ${SERVICE_METRICS_SETTLE_SECS}" + else + sleep "$SERVICE_METRICS_SETTLE_SECS" + fi +} + +capture_prometheus_query_snapshot() { + local phase="$1" + local snapshot_file="$2" + local status_file="$3" + local capture_attempt="$4" + + "$PYTHON_BIN" - "$SERVICE_PROMETHEUS_QUERY_URL" "$SERVICE_PROMETHEUS_QUERY" "$phase" "$snapshot_file" "$status_file" "$SERVICE_METRICS_SERVICE_NAME" "$SERVICE_METRICS_SETTLE_SECS" "$SERVICE_METRICS_MAX_TIME_SECS" "$SERVICE_METRICS_FILTER_REGEX" "$capture_attempt" <<'PY' +import json +import pathlib +import sys +import urllib.parse +import urllib.request + +query_url, query, phase, snapshot_raw, status_raw, service_name, settle_secs, timeout_secs_raw, filter_regex, capture_attempt = sys.argv[1:] +snapshot_path = pathlib.Path(snapshot_raw) +status_path = pathlib.Path(status_raw) +timeout_secs = float(timeout_secs_raw) + + +def write_status(status): + status_path.write_text( + "\n".join( + [ + f"phase={phase}", + f"status={status}", + f"url={query_url}", + f"query={query}", + f"service_name={service_name}", + f"settle_secs={settle_secs}", + f"max_time_secs={timeout_secs_raw}", + f"filter_regex={filter_regex}", + f"capture_attempt={capture_attempt}", + "", + ] + ), + encoding="utf-8", + ) + + +def escape_label(value): + return value.replace("\\", "\\\\").replace("\n", "\\n").replace('"', '\\"') + + +def sample_to_line(sample): + metric = dict(sample.get("metric", {})) + name = metric.pop("__name__", "") + if not name: + return None + labels = ",".join(f'{key}="{escape_label(value)}"' for key, value in sorted(metric.items())) + value = sample.get("value", [None, None])[1] + if value is None: + return None + if labels: + return f"{name}{{{labels}}} {value}" + return f"{name} {value}" + + +try: + separator = "&" if "?" in query_url else "?" + url = f"{query_url}{separator}{urllib.parse.urlencode({'query': query})}" + request = urllib.request.Request(url, headers={"Accept": "application/json"}) + with urllib.request.urlopen(request, timeout=timeout_secs) as response: + payload = json.loads(response.read().decode("utf-8")) +except Exception as err: # noqa: BLE001 - shell harness reports failures in status files. + snapshot_path.write_text(f"# prometheus_query_failed error={err}\n", encoding="utf-8") + write_status("query_failed") + raise SystemExit(0) + +if payload.get("status") != "success": + snapshot_path.write_text(f"# prometheus_query_failed payload_status={payload.get('status')}\n", encoding="utf-8") + write_status("query_failed") + raise SystemExit(0) + +samples = [ + sample + for sample in payload.get("data", {}).get("result", []) + if not service_name + or sample.get("metric", {}).get("service.name") == service_name + or sample.get("metric", {}).get("service_name") == service_name +] +lines = [line for sample in samples if (line := sample_to_line(sample))] +if not lines: + snapshot_path.write_text(f"# no_matching_metrics service_name={service_name}\n", encoding="utf-8") + write_status("no_matching_metrics") + raise SystemExit(0) + +snapshot_path.write_text("\n".join(lines) + "\n", encoding="utf-8") +write_status("ok") +PY +} + capture_round_service_metrics() { local size="$1" local round="$2" local attempt="$3" local phase="$4" - if [[ -z "$SERVICE_METRICS_URL" ]]; then + if [[ -z "$SERVICE_METRICS_URL" && -z "$SERVICE_PROMETHEUS_QUERY_URL" ]]; then return 0 fi - local token snapshot_file status_file tmp_file filtered_file capture_attempt raw_bytes snapshot_bytes + local token snapshot_file status_file tmp_file filtered_file capture_attempt raw_bytes snapshot_bytes capture_status token="$(metric_snapshot_token "$size" "$round" "$attempt")" snapshot_file="${SERVICE_METRICS_DIR}/${token}_${phase}.prom" status_file="${SERVICE_METRICS_DIR}/${token}_${phase}.status" @@ -426,8 +708,54 @@ attempt=${attempt} phase=${phase} status=not_run_dry_run url=${SERVICE_METRICS_URL} +prometheus_query_url=${SERVICE_PROMETHEUS_QUERY_URL} filter_regex=${SERVICE_METRICS_FILTER_REGEX} +prometheus_query=${SERVICE_PROMETHEUS_QUERY} +settle_secs=${SERVICE_METRICS_SETTLE_SECS} EOF + echo "$size,$TOOL,$round,$attempt,$phase,dry_run,not_run_dry_run,0,0,0,$status_file,$snapshot_file,$SERVICE_METRICS_FILTER_REGEX,$SERVICE_PROMETHEUS_QUERY" >> "$SERVICE_METRICS_CSV" + return 0 + fi + + if [[ -n "$SERVICE_PROMETHEUS_QUERY_URL" ]]; then + for ((capture_attempt=1; capture_attempt<=SERVICE_METRICS_CAPTURE_ATTEMPTS; capture_attempt++)); do + capture_prometheus_query_snapshot "$phase" "$tmp_file" "$status_file" "$capture_attempt" + capture_status="$(awk -F= '$1=="status" {print $2; exit}' "$status_file")" + if [[ "${capture_status:-unknown}" == "ok" && -s "$tmp_file" ]]; then + raw_bytes="$(wc -c <"$tmp_file" | tr -d '[:space:]')" + filter_service_metrics_snapshot "$tmp_file" "$filtered_file" + mv "$filtered_file" "$snapshot_file" + snapshot_bytes="$(wc -c <"$snapshot_file" | tr -d '[:space:]')" + { + echo "raw_bytes=${raw_bytes}" + echo "snapshot_bytes=${snapshot_bytes}" + } >>"$status_file" + echo "$size,$TOOL,$round,$attempt,$phase,prometheus_query,ok,$capture_attempt,$raw_bytes,$snapshot_bytes,$status_file,$snapshot_file,$SERVICE_METRICS_FILTER_REGEX,$SERVICE_PROMETHEUS_QUERY" >> "$SERVICE_METRICS_CSV" + return 0 + fi + + rm -f "$tmp_file" "$filtered_file" + if (( capture_attempt < SERVICE_METRICS_CAPTURE_ATTEMPTS && SERVICE_METRICS_CAPTURE_RETRY_SECS > 0 )); then + sleep "$SERVICE_METRICS_CAPTURE_RETRY_SECS" + fi + done + + : >"$snapshot_file" + cat >"$status_file" <> "$SERVICE_METRICS_CSV" + echo "WARN: failed to capture service metrics size=${size} round=${round} attempt=${attempt} phase=${phase}" >&2 return 0 fi @@ -454,7 +782,9 @@ max_time_secs=${SERVICE_METRICS_MAX_TIME_SECS} filter_regex=${SERVICE_METRICS_FILTER_REGEX} raw_bytes=${raw_bytes} snapshot_bytes=${snapshot_bytes} +settle_secs=${SERVICE_METRICS_SETTLE_SECS} EOF + echo "$size,$TOOL,$round,$attempt,$phase,prometheus_text,ok,$capture_attempt,$raw_bytes,$snapshot_bytes,$status_file,$snapshot_file,$SERVICE_METRICS_FILTER_REGEX,$SERVICE_PROMETHEUS_QUERY" >> "$SERVICE_METRICS_CSV" return 0 fi fi @@ -477,7 +807,9 @@ url=${SERVICE_METRICS_URL} connect_timeout_secs=${SERVICE_METRICS_CONNECT_TIMEOUT_SECS} max_time_secs=${SERVICE_METRICS_MAX_TIME_SECS} filter_regex=${SERVICE_METRICS_FILTER_REGEX} +settle_secs=${SERVICE_METRICS_SETTLE_SECS} EOF + echo "$size,$TOOL,$round,$attempt,$phase,prometheus_text,capture_failed,$SERVICE_METRICS_CAPTURE_ATTEMPTS,N/A,N/A,$status_file,$snapshot_file,$SERVICE_METRICS_FILTER_REGEX,$SERVICE_PROMETHEUS_QUERY" >> "$SERVICE_METRICS_CSV" echo "WARN: failed to capture service metrics size=${size} round=${round} attempt=${attempt} phase=${phase}" >&2 } @@ -515,14 +847,15 @@ run_one_attempt() { started_at_utc="$(date -u +%Y-%m-%dT%H:%M:%SZ)" if [[ "$TOOL" == "warp" ]]; then - local warp_host + local warp_host bucket warp_host="$(normalize_warp_host "$ENDPOINT")" + bucket="$(bucket_for_size "$size")" local cmd=( "$WARP_BIN" "$WARP_MODE" "--host" "$warp_host" "--access-key" "$ACCESS_KEY" "--secret-key" "$SECRET_KEY" - "--bucket" "$BUCKET" + "--bucket" "$bucket" "--obj.size" "$size" "--concurrent" "$CONCURRENCY" "--duration" "$DURATION" @@ -548,11 +881,13 @@ run_one_attempt() { fi fi else + local bucket + bucket="$(bucket_for_size "$size")" local cmd=( "$S3BENCH_BIN" "-accessKey=$ACCESS_KEY" "-secretKey=$SECRET_KEY" - "-bucket=$BUCKET" + "-bucket=$bucket" "-endpoint=$ENDPOINT" "-region=$REGION" "-numClients=$CONCURRENCY" @@ -580,6 +915,7 @@ run_one_attempt() { fi fi finished_at_utc="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + settle_before_after_metrics_capture capture_round_service_metrics "$size" "$round" "$attempt" after local metrics throughput_human reqps latency_human throughput_bps latency_ms req_p90_human req_p90_ms req_p99_human req_p99_ms @@ -733,6 +1069,16 @@ main() { echo "Rounds: $ROUNDS" echo "Retry per round: $RETRY_PER_ROUND" echo "Cooldown secs: $COOLDOWN_SECS" + if [[ -n "$SERVICE_METRICS_URL" || -n "$SERVICE_PROMETHEUS_QUERY_URL" ]]; then + echo "Service metrics filter: $SERVICE_METRICS_FILTER_REGEX" + if [[ -n "$SERVICE_PROMETHEUS_QUERY_URL" ]]; then + echo "Service Prometheus query URL: $SERVICE_PROMETHEUS_QUERY_URL" + echo "Service Prometheus query: $SERVICE_PROMETHEUS_QUERY" + if [[ -n "$SERVICE_METRICS_SERVICE_NAME" ]]; then + echo "Service metrics service name: $SERVICE_METRICS_SERVICE_NAME" + fi + fi + fi IFS=',' read -r -a size_arr <<< "$SIZES" local total_sizes="${#size_arr[@]}" diff --git a/scripts/run_put_large_stage_breakdown.sh b/scripts/run_put_large_stage_breakdown.sh index 20db1c2ae..bd477740a 100755 --- a/scripts/run_put_large_stage_breakdown.sh +++ b/scripts/run_put_large_stage_breakdown.sh @@ -11,6 +11,7 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" ENHANCED_BENCH_SCRIPT="${PROJECT_ROOT}/scripts/run_object_batch_bench_enhanced.sh" +SERVICE_METRICS_DELTA_SCRIPT="${PROJECT_ROOT}/scripts/analyze_put_service_metrics_deltas.py" TOOL="warp" ENDPOINT="" @@ -24,13 +25,24 @@ DURATION="120s" ROUNDS=3 RETRY_PER_ROUND=1 RETRY_SLEEP_SECS=2 -COOLDOWN_SECS=0 +COOLDOWN_SECS=20 WARP_BIN="${WARP_BIN:-warp}" OUT_DIR="" BASELINE_ROOT="" EXTRA_ARGS="" INSECURE=false DRY_RUN=false +PYTHON_BIN="python3" +SERVICE_METRICS_URL="" +SERVICE_PROMETHEUS_QUERY_URL="" +SERVICE_PROMETHEUS_QUERY="" +SERVICE_METRICS_SERVICE_NAME="" +SERVICE_METRICS_FILTER_REGEX="" +SERVICE_METRICS_CAPTURE_ATTEMPTS=3 +SERVICE_METRICS_CAPTURE_RETRY_SECS=1 +SERVICE_METRICS_CONNECT_TIMEOUT_SECS=2 +SERVICE_METRICS_MAX_TIME_SECS=15 +SERVICE_METRICS_SETTLE_SECS=0 TOPOLOGY_NODES="" TOPOLOGY_DISKS_PER_NODE="" @@ -63,13 +75,29 @@ Core options: --rounds Rounds per size (default: 3) --retry-per-round Retries per failed round (default: 1) --retry-sleep-secs Sleep between retries (default: 2) - --cooldown-secs Sleep between rounds/sizes (default: 0) + --cooldown-secs Sleep between rounds/sizes/concurrency runs (default: 20) + --round-cooldown-secs Compatibility alias for --cooldown-secs --out-dir Output root (default: target/bench/put-large-stage-breakdown-) --baseline-root Existing root from a previous run of this script --extra-args "" Extra args passed to run_object_batch_bench_enhanced.sh --warp-bin warp binary (default: warp) --insecure Pass --insecure to warp --dry-run Print commands only + --python-bin Python binary for Prometheus query capture (default: python3) + --service-metrics-url Plain Prometheus text scrape URL + --service-prometheus-query-url + Prometheus HTTP API /api/v1/query URL for OTLP-exported metrics + --service-prometheus-query PromQL for --service-prometheus-query-url + --service-metrics-service-name + Optional service.name/service_name filter for query results + --service-metrics-filter-regex + Regex for retained plain text metrics lines + --service-metrics-attempts Direct scrape attempts per snapshot (default: 3) + --service-metrics-retry-secs Sleep between direct scrape attempts (default: 1) + --service-metrics-connect-timeout-secs + Curl connect timeout for direct scrape (default: 2) + --service-metrics-max-time-secs Curl max time for direct scrape (default: 15) + --service-metrics-settle-secs Sleep before after-snapshot capture (default: 0) Topology metadata (optional but recommended): --nodes @@ -117,6 +145,16 @@ arg_value() { printf '%s\n' "$value" } +arg_value_allow_option() { + local flag="$1" + local value="${2:-}" + if [[ -z "$value" ]]; then + echo "ERROR: missing value for $flag" >&2 + exit 1 + fi + printf '%s\n' "$value" +} + parse_args() { while [[ $# -gt 0 ]]; do case "$1" in @@ -131,11 +169,22 @@ parse_args() { --rounds) ROUNDS="$(arg_value "$1" "${2:-}")"; shift 2 ;; --retry-per-round) RETRY_PER_ROUND="$(arg_value "$1" "${2:-}")"; shift 2 ;; --retry-sleep-secs) RETRY_SLEEP_SECS="$(arg_value "$1" "${2:-}")"; shift 2 ;; - --cooldown-secs) COOLDOWN_SECS="$(arg_value "$1" "${2:-}")"; shift 2 ;; + --cooldown-secs|--round-cooldown-secs) COOLDOWN_SECS="$(arg_value "$1" "${2:-}")"; shift 2 ;; --out-dir) OUT_DIR="$(arg_value "$1" "${2:-}")"; shift 2 ;; --baseline-root) BASELINE_ROOT="$(arg_value "$1" "${2:-}")"; shift 2 ;; - --extra-args) EXTRA_ARGS="$(arg_value "$1" "${2:-}")"; shift 2 ;; + --extra-args) EXTRA_ARGS="$(arg_value_allow_option "$1" "${2:-}")"; shift 2 ;; --warp-bin) WARP_BIN="$(arg_value "$1" "${2:-}")"; shift 2 ;; + --python-bin) PYTHON_BIN="$(arg_value "$1" "${2:-}")"; shift 2 ;; + --service-metrics-url) SERVICE_METRICS_URL="$(arg_value "$1" "${2:-}")"; shift 2 ;; + --service-prometheus-query-url) SERVICE_PROMETHEUS_QUERY_URL="$(arg_value "$1" "${2:-}")"; shift 2 ;; + --service-prometheus-query) SERVICE_PROMETHEUS_QUERY="$(arg_value "$1" "${2:-}")"; shift 2 ;; + --service-metrics-service-name) SERVICE_METRICS_SERVICE_NAME="$(arg_value "$1" "${2:-}")"; shift 2 ;; + --service-metrics-filter-regex) SERVICE_METRICS_FILTER_REGEX="$(arg_value "$1" "${2:-}")"; shift 2 ;; + --service-metrics-attempts) SERVICE_METRICS_CAPTURE_ATTEMPTS="$(arg_value "$1" "${2:-}")"; shift 2 ;; + --service-metrics-retry-secs) SERVICE_METRICS_CAPTURE_RETRY_SECS="$(arg_value "$1" "${2:-}")"; shift 2 ;; + --service-metrics-connect-timeout-secs) SERVICE_METRICS_CONNECT_TIMEOUT_SECS="$(arg_value "$1" "${2:-}")"; shift 2 ;; + --service-metrics-max-time-secs) SERVICE_METRICS_MAX_TIME_SECS="$(arg_value "$1" "${2:-}")"; shift 2 ;; + --service-metrics-settle-secs) SERVICE_METRICS_SETTLE_SECS="$(arg_value "$1" "${2:-}")"; shift 2 ;; --nodes) TOPOLOGY_NODES="$(arg_value "$1" "${2:-}")"; shift 2 ;; --disks-per-node) TOPOLOGY_DISKS_PER_NODE="$(arg_value "$1" "${2:-}")"; shift 2 ;; --total-disks) TOPOLOGY_TOTAL_DISKS="$(arg_value "$1" "${2:-}")"; shift 2 ;; @@ -195,6 +244,14 @@ validate_args() { echo "ERROR: --baseline-root does not exist: $BASELINE_ROOT" >&2 exit 1 fi + if [[ -n "$SERVICE_METRICS_URL" && -n "$SERVICE_PROMETHEUS_QUERY_URL" ]]; then + echo "ERROR: --service-metrics-url and --service-prometheus-query-url are mutually exclusive" >&2 + exit 1 + fi + if ! is_positive_int "$SERVICE_METRICS_CAPTURE_ATTEMPTS" || ! is_nonnegative_int "$SERVICE_METRICS_CAPTURE_RETRY_SECS" || ! is_positive_int "$SERVICE_METRICS_CONNECT_TIMEOUT_SECS" || ! is_positive_int "$SERVICE_METRICS_MAX_TIME_SECS" || ! is_nonnegative_int "$SERVICE_METRICS_SETTLE_SECS"; then + echo "ERROR: service metrics attempt/timeout/settle options must be valid integers" >&2 + exit 1 + fi } setup_output() { @@ -204,11 +261,16 @@ setup_output() { RUNS_DIR="${OUT_DIR}/runs" AGG_MEDIAN_CSV="${OUT_DIR}/aggregate_median_summary.csv" AGG_COMPARE_CSV="${OUT_DIR}/aggregate_baseline_compare.csv" + AGG_SERVICE_METRICS_CSV="${OUT_DIR}/aggregate_service_metrics_captures.csv" + AGG_SERVICE_METRICS_DELTAS_CSV="${OUT_DIR}/aggregate_service_metrics_deltas.csv" + AGG_SERVICE_METRICS_PATH_SUMMARY_CSV="${OUT_DIR}/aggregate_service_metrics_path_summary.csv" + AGG_SERVICE_METRICS_STAGE_SUMMARY_CSV="${OUT_DIR}/aggregate_service_metrics_stage_summary.csv" RUN_MATRIX_CSV="${OUT_DIR}/run_matrix.csv" mkdir -p "$RUNS_DIR" echo "concurrency,size,tool,successful_rounds,failed_rounds,median_throughput_bps,median_reqps,median_latency_ms,bucket,run_dir" > "$AGG_MEDIAN_CSV" echo "concurrency,size,tool,new_median_reqps,baseline_median_reqps,delta_reqps_pct,new_median_latency_ms,baseline_median_latency_ms,delta_latency_pct,new_median_throughput_bps,baseline_median_throughput_bps,delta_throughput_pct,run_dir" > "$AGG_COMPARE_CSV" + echo "concurrency,run_dir,size,tool,round,attempt,phase,source,status,capture_attempt,raw_bytes,snapshot_bytes,status_file,snapshot_file,filter_regex,prometheus_query" > "$AGG_SERVICE_METRICS_CSV" echo "concurrency,bucket,run_dir,baseline_csv,status" > "$RUN_MATRIX_CSV" } @@ -253,6 +315,17 @@ write_manifest() { echo "dry_run=$(join_bool "$DRY_RUN")" echo "baseline_root=${BASELINE_ROOT:-N/A}" echo "extra_args_present=$([[ -n "$EXTRA_ARGS" ]] && echo true || echo false)" + echo "python_bin=${PYTHON_BIN}" + echo "service_metrics_url=${SERVICE_METRICS_URL:-N/A}" + echo "service_prometheus_query_url=${SERVICE_PROMETHEUS_QUERY_URL:-N/A}" + echo "service_prometheus_query=${SERVICE_PROMETHEUS_QUERY:-N/A}" + echo "service_metrics_service_name=${SERVICE_METRICS_SERVICE_NAME:-N/A}" + echo "service_metrics_filter_regex=${SERVICE_METRICS_FILTER_REGEX:-N/A}" + echo "service_metrics_capture_attempts=${SERVICE_METRICS_CAPTURE_ATTEMPTS}" + echo "service_metrics_retry_secs=${SERVICE_METRICS_CAPTURE_RETRY_SECS}" + echo "service_metrics_connect_timeout_secs=${SERVICE_METRICS_CONNECT_TIMEOUT_SECS}" + echo "service_metrics_max_time_secs=${SERVICE_METRICS_MAX_TIME_SECS}" + echo "service_metrics_settle_secs=${SERVICE_METRICS_SETTLE_SECS}" echo "workload_label=${WORKLOAD_LABEL}" echo "nodes=${TOPOLOGY_NODES:-N/A}" echo "disks_per_node=${TOPOLOGY_DISKS_PER_NODE:-N/A}" @@ -275,11 +348,16 @@ Top-level artifacts: - run_matrix.csv: one row per concurrency run, including bucket and baseline linkage - aggregate_median_summary.csv: merged median_summary rows from every concurrency run - aggregate_baseline_compare.csv: merged baseline_compare rows when a matching baseline exists +- aggregate_service_metrics_captures.csv: merged service metrics capture status from every concurrency run +- aggregate_service_metrics_deltas.csv: per-round before/after service metric deltas +- aggregate_service_metrics_path_summary.csv: per-round PUT path deltas +- aggregate_service_metrics_stage_summary.csv: per-round PUT stage duration averages from deltas - runs/cXX/: direct output directory of scripts/run_object_batch_bench_enhanced.sh Per-concurrency directory: - round_results.csv - median_summary.csv +- service_metrics_captures.csv (when service metrics capture is enabled) - baseline_compare.csv (only when a baseline CSV is supplied) - logs/ EOF @@ -338,12 +416,35 @@ append_aggregate_rows() { local run_dir="$3" local median_csv="$4" local compare_csv="$5" + local service_metrics_csv="${run_dir}/service_metrics_captures.csv" awk -F',' -v c="$concurrency" -v b="$bucket" -v rd="$run_dir" 'NR>1 {print c "," $1 "," $2 "," $4 "," $5 "," $6 "," $7 "," $8 "," b "," rd}' "$median_csv" >> "$AGG_MEDIAN_CSV" if [[ -f "$compare_csv" ]]; then awk -F',' -v c="$concurrency" -v rd="$run_dir" 'NR>1 {print c "," $1 "," $2 "," $4 "," $5 "," $6 "," $7 "," $8 "," $9 "," $10 "," $11 "," $12 "," rd}' "$compare_csv" >> "$AGG_COMPARE_CSV" fi + if [[ -f "$service_metrics_csv" ]]; then + awk -F',' -v c="$concurrency" -v rd="$run_dir" 'NR>1 {print c "," rd "," $0}' "$service_metrics_csv" >> "$AGG_SERVICE_METRICS_CSV" + fi +} + +analyze_service_metric_deltas() { + if [[ ! -s "$AGG_SERVICE_METRICS_CSV" ]]; then + return 0 + fi + if [[ "$DRY_RUN" == "true" ]]; then + return 0 + fi + if [[ ! -f "$SERVICE_METRICS_DELTA_SCRIPT" ]]; then + echo "ERROR: missing service metrics delta script: $SERVICE_METRICS_DELTA_SCRIPT" >&2 + exit 1 + fi + + "$PYTHON_BIN" "$SERVICE_METRICS_DELTA_SCRIPT" \ + --capture-csv "$AGG_SERVICE_METRICS_CSV" \ + --delta-csv "$AGG_SERVICE_METRICS_DELTAS_CSV" \ + --path-summary-csv "$AGG_SERVICE_METRICS_PATH_SUMMARY_CSV" \ + --stage-summary-csv "$AGG_SERVICE_METRICS_STAGE_SUMMARY_CSV" } run_concurrency() { @@ -383,6 +484,32 @@ run_concurrency() { if [[ -n "$EXTRA_ARGS" ]]; then cmd+=(--extra-args "$EXTRA_ARGS") fi + if [[ -n "$SERVICE_METRICS_URL" || -n "$SERVICE_PROMETHEUS_QUERY_URL" ]]; then + cmd+=(--service-metrics-dir "${run_dir}/service-metrics") + fi + if [[ -n "$SERVICE_METRICS_URL" ]]; then + cmd+=(--service-metrics-url "$SERVICE_METRICS_URL") + fi + if [[ -n "$SERVICE_PROMETHEUS_QUERY_URL" ]]; then + cmd+=(--service-prometheus-query-url "$SERVICE_PROMETHEUS_QUERY_URL") + fi + if [[ -n "$SERVICE_PROMETHEUS_QUERY" ]]; then + cmd+=(--service-prometheus-query "$SERVICE_PROMETHEUS_QUERY") + fi + if [[ -n "$SERVICE_METRICS_SERVICE_NAME" ]]; then + cmd+=(--service-metrics-service-name "$SERVICE_METRICS_SERVICE_NAME") + fi + if [[ -n "$SERVICE_METRICS_FILTER_REGEX" ]]; then + cmd+=(--service-metrics-filter-regex "$SERVICE_METRICS_FILTER_REGEX") + fi + cmd+=( + --python-bin "$PYTHON_BIN" + --service-metrics-attempts "$SERVICE_METRICS_CAPTURE_ATTEMPTS" + --service-metrics-retry-secs "$SERVICE_METRICS_CAPTURE_RETRY_SECS" + --service-metrics-connect-timeout-secs "$SERVICE_METRICS_CONNECT_TIMEOUT_SECS" + --service-metrics-max-time-secs "$SERVICE_METRICS_MAX_TIME_SECS" + --service-metrics-settle-secs "$SERVICE_METRICS_SETTLE_SECS" + ) if [[ "$DRY_RUN" == "true" ]]; then cmd+=(--dry-run) fi @@ -433,6 +560,10 @@ main() { echo "Duration: $DURATION" echo "Rounds: $ROUNDS" echo "Cooldown secs: $COOLDOWN_SECS" + if [[ -n "$SERVICE_METRICS_URL" || -n "$SERVICE_PROMETHEUS_QUERY_URL" ]]; then + echo "Service metrics capture: enabled" + echo "Service metrics source: $([[ -n "$SERVICE_PROMETHEUS_QUERY_URL" ]] && echo prometheus_query || echo prometheus_text)" + fi local conc_count conc_index conc_count="$(csv_to_lines "$CONCURRENCIES" | awk 'END{print NR+0}')" @@ -446,12 +577,22 @@ main() { fi done < <(csv_to_lines "$CONCURRENCIES") + analyze_service_metric_deltas + echo echo "Stage-breakdown run finished." echo "Artifacts written to: $OUT_DIR" echo "Top-level summaries:" echo " - $RUN_MATRIX_CSV" echo " - $AGG_MEDIAN_CSV" + if [[ -s "$AGG_SERVICE_METRICS_CSV" ]]; then + echo " - $AGG_SERVICE_METRICS_CSV" + fi + if [[ -s "$AGG_SERVICE_METRICS_DELTAS_CSV" ]]; then + echo " - $AGG_SERVICE_METRICS_DELTAS_CSV" + echo " - $AGG_SERVICE_METRICS_PATH_SUMMARY_CSV" + echo " - $AGG_SERVICE_METRICS_STAGE_SUMMARY_CSV" + fi if [[ -s "$AGG_COMPARE_CSV" ]]; then echo " - $AGG_COMPARE_CSV" fi