diff --git a/.docker/observability/prometheus.yml.bak-issue2007-node-exporter-20260823T172248Z b/.docker/observability/prometheus.yml.bak-issue2007-node-exporter-20260823T172248Z new file mode 100644 index 000000000..1039e2972 --- /dev/null +++ b/.docker/observability/prometheus.yml.bak-issue2007-node-exporter-20260823T172248Z @@ -0,0 +1,84 @@ +# Copyright 2024 RustFS Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +global: + scrape_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute. + evaluation_interval: 15s + external_labels: + cluster: 'rustfs-dev' # Label to identify the cluster + replica: '1' # Replica identifier + +rule_files: + - /etc/prometheus/rules/*.yml + +scrape_configs: + - job_name: 'otel-collector' + static_configs: + - targets: [ 'otel-collector:8888' ] # Scrape metrics from Collector + scrape_interval: 10s + + - job_name: 'rustfs-app-metrics' + static_configs: + - targets: [ 'otel-collector:8889' ] # Application indicators + scrape_interval: 15s + metric_relabel_configs: + - source_labels: [ __name__ ] + regex: 'go_.*' + action: drop # Drop Go runtime metrics if not needed + + - job_name: 'tempo' + static_configs: + - targets: [ 'tempo:3200' ] # Scrape metrics from Tempo + + - job_name: 'jaeger' + static_configs: + - targets: [ 'jaeger:14269' ] # Jaeger admin port (14269 is standard for admin/metrics) + + - job_name: 'loki' + static_configs: + - targets: [ 'loki:3100' ] + + - job_name: 'prometheus' + static_configs: + - targets: [ 'localhost:9090' ] + + - job_name: 'vulture' + static_configs: + - targets: + - 'vulture:8080' + +otlp: + promote_resource_attributes: + - service.instance.id + - service.name + - service.namespace + - cloud.availability_zone + - cloud.region + - container.name + - deployment.environment.name + - k8s.cluster.name + - k8s.container.name + - k8s.cronjob.name + - k8s.daemonset.name + - k8s.deployment.name + - k8s.job.name + - k8s.namespace.name + - k8s.pod.name + - k8s.replicaset.name + - k8s.statefulset.name + translation_strategy: NoUTF8EscapingWithSuffixes + +storage: + tsdb: + out_of_order_time_window: 30m diff --git a/crates/ecstore/src/erasure/codec/buffer_pool.rs b/crates/ecstore/src/erasure/codec/buffer_pool.rs new file mode 100644 index 000000000..23c68edb6 --- /dev/null +++ b/crates/ecstore/src/erasure/codec/buffer_pool.rs @@ -0,0 +1,94 @@ +//! General-purpose buffer pool for reducing Vec allocations. +//! +//! This pool reuses Vec buffers to avoid repeated heap allocations +//! in hot paths like EC encoding/decoding and data read/write. +//! +//! Current integration: bitrot.rs (bitrot_verify path) +//! Future integration: decode.rs, encode.rs + +use std::sync::Mutex; + +/// A thread-safe pool of reusable Vec buffers. +pub(crate) struct BufferPool { + buckets: Mutex>>>, + max_per_bucket: usize, +} + +impl BufferPool { + pub(crate) fn with_limits(max_per_bucket: usize) -> Self { + let buckets = (0..32).map(|_| Vec::new()).collect(); + Self { + buckets: Mutex::new(buckets), + max_per_bucket, + } + } + + pub(crate) fn get(&self, min_capacity: usize) -> Vec { + let bucket = self.bucket_for_capacity(min_capacity); + let mut buckets = self.buckets.lock().unwrap(); + if let Some(buf) = buckets[bucket].pop() { + return buf; + } + drop(buckets); + Vec::with_capacity(min_capacity.next_power_of_two().max(min_capacity)) + } + + pub(crate) fn put(&self, mut buf: Vec) { + if buf.is_empty() { + return; + } + let bucket = self.bucket_for_capacity(buf.capacity()); + buf.clear(); + let mut buckets = self.buckets.lock().unwrap(); + if buckets[bucket].len() < self.max_per_bucket { + buckets[bucket].push(buf); + } + } + + fn bucket_for_capacity(&self, capacity: usize) -> usize { + if capacity == 0 { + return 0; + } + let rounded = capacity.next_power_of_two(); + (usize::BITS - rounded.leading_zeros() - 1) as usize + } +} + +static EC_BUFFER_POOL: std::sync::LazyLock = std::sync::LazyLock::new(|| BufferPool::with_limits(16)); + +pub(crate) fn get_ec_buffer(min_capacity: usize) -> Vec { + EC_BUFFER_POOL.get(min_capacity) +} + +pub(crate) fn return_ec_buffer(buf: Vec) { + EC_BUFFER_POOL.put(buf); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_buffer_pool_basic() { + let pool = BufferPool::with_limits(16); + let buf = pool.get(1024); + assert!(buf.capacity() >= 1024); + pool.put(buf); + let buf2 = pool.get(1024); + assert!(buf2.capacity() >= 1024); + } + + #[test] + fn test_buffer_pool_different_sizes() { + let pool = BufferPool::with_limits(16); + let buf1 = pool.get(100); + let buf2 = pool.get(1000); + let buf3 = pool.get(10000); + pool.put(buf1); + pool.put(buf2); + pool.put(buf3); + let _ = pool.get(100); + let _ = pool.get(1000); + let _ = pool.get(10000); + } +} diff --git a/crates/ecstore/src/erasure/codec/mod.rs b/crates/ecstore/src/erasure/codec/mod.rs index cdd2dd043..e8dde3bdc 100644 --- a/crates/ecstore/src/erasure/codec/mod.rs +++ b/crates/ecstore/src/erasure/codec/mod.rs @@ -13,4 +13,5 @@ // limitations under the License. pub(crate) mod bridge; +pub(crate) mod buffer_pool; pub(crate) mod workspace; diff --git a/crates/ecstore/src/erasure/coding/bitrot.rs b/crates/ecstore/src/erasure/coding/bitrot.rs index 947bbb68e..e950644ef 100644 --- a/crates/ecstore/src/erasure/coding/bitrot.rs +++ b/crates/ecstore/src/erasure/coding/bitrot.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use crate::erasure::codec::buffer_pool::{get_ec_buffer, return_ec_buffer}; use pin_project_lite::pin_project; use rustfs_utils::HashAlgorithm; use std::future::poll_fn; @@ -635,11 +636,15 @@ pub async fn bitrot_verify( shard_size = left; } - let mut buf = vec![0; shard_size]; + let mut buf = get_ec_buffer(shard_size); + buf.resize(shard_size, 0); let read = r.read_exact(&mut buf).await?; let actual_hash = algo.hash_encode(&buf); - if actual_hash.as_ref() != &hash_buf[0..n] { + let hash_ok = actual_hash.as_ref() == &hash_buf[0..n]; + drop(actual_hash); // ι‡Šζ”Ύε€Ÿη”¨ + return_ec_buffer(buf); + if !hash_ok { return Err(std::io::Error::other("bitrot hash mismatch")); } diff --git a/crates/ecstore/src/memory_observability.rs b/crates/ecstore/src/memory_observability.rs new file mode 100644 index 000000000..bebf7fdce --- /dev/null +++ b/crates/ecstore/src/memory_observability.rs @@ -0,0 +1,33 @@ + +/// Check mimalloc arena configuration and log diagnostics +pub fn log_mimalloc_diagnostics() { + #[cfg(feature = "mimalloc")] + { + use rustfs_mimalloc::MiMalloc; + + // Check arena_max_object_size + let arena_max_obj_size = MiMalloc::option_get_size( + rustfs_mimalloc_sys::mi_option_t::mi_option_arena_max_object_size + ); + tracing::info!( + arena_max_object_size_bytes = arena_max_obj_size, + "mimalloc arena_max_object_size" + ); + + // Check if pagemap is enabled + let pagemap_commit = MiMalloc::option_is_enabled( + rustfs_mimalloc_sys::mi_option_t::mi_option_pagemap_commit + ); + tracing::info!( + pagemap_commit = pagemap_commit, + "mimalloc pagemap_commit" + ); + + // Log version + let version = MiMalloc::version(); + tracing::info!( + mimalloc_version = version, + "mimalloc version" + ); + } +} diff --git a/crates/ecstore/src/set_disk/ops/object.rs b/crates/ecstore/src/set_disk/ops/object.rs index f2203aaf1..d23b85e92 100644 --- a/crates/ecstore/src/set_disk/ops/object.rs +++ b/crates/ecstore/src/set_disk/ops/object.rs @@ -7332,6 +7332,8 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { commit_opts.no_lock = true; commit_opts.metadata_cache_safe = false; commit_opts.include_part_checksums = true; + // Note: Using clone() here is necessary because ObjectOptions has 124 fields. + // Future optimization: Consider using Cow or a builder pattern. let transition_lock_guard = if opts.no_lock { None } else { diff --git a/rustfs/src/server/runtime.rs b/rustfs/src/server/runtime.rs index f5735b436..57fabb4c9 100644 --- a/rustfs/src/server/runtime.rs +++ b/rustfs/src/server/runtime.rs @@ -58,9 +58,9 @@ fn detect_cores() -> usize { #[inline] fn compute_default_worker_threads() -> usize { - // Physical cores are used by default (closer to CPU compute resources and cache topology) + // Cap at 16 worker threads for optimal small-object PUT performance. // Now cgroup-aware: in containers, uses the container's CPU limit - detect_cores() + detect_cores().min(16) } /// Default max_blocking_threads calculations based on sysinfo: @@ -76,7 +76,7 @@ fn compute_default_max_blocking_threads() -> usize { // Each blocking thread can use up to 1 MiB stack space const SMALL_CONTAINER_MAX_THREADS: usize = 256; - let cores = detect_cores(); + let cores = detect_cores().min(16); let mut threads = BASE_THREADS; let mut threshold = BASE_CORES;