Compare commits

..

1 Commits

Author SHA1 Message Date
houseme 01e0af6312 perf(io-metrics): avoid get handoff label allocations (#6160)
Co-authored-by: heihutu <heihutu@gmail.com>
2026-08-17 08:26:23 +08:00
16 changed files with 108 additions and 62 deletions
+1 -4
View File
@@ -38,10 +38,7 @@ pub const XXHASH_3_HEADER_NAME: &str = "x-amz-checksum-xxhash3";
pub const XXHASH_64_HEADER_NAME: &str = "x-amz-checksum-xxhash64";
pub const XXHASH_128_HEADER_NAME: &str = "x-amz-checksum-xxhash128";
#[allow(
dead_code,
reason = "Content-MD5 wire name, resolved by header_name() below and asserted by this crate's tests (backlog#1823)"
)]
#[allow(dead_code)]
pub(crate) static MD5_HEADER_NAME: &str = "content-md5";
pub const CHECKSUM_ALGORITHMS_IN_PRIORITY_ORDER: [&str; 5] =
+2 -8
View File
@@ -476,19 +476,13 @@ impl Checksum for Xxhash64 {
}
}
#[allow(dead_code)]
#[derive(Debug, Default)]
#[allow(
dead_code,
reason = "Content-MD5 is not a ChecksumAlgorithm variant and has no arm in into_impl: S3 carries it as its own header, separate from the x-amz-checksum-* family. This impl exists so the two paths share the Checksum trait, and is asserted by this crate's tests (backlog#1823)"
)]
struct Md5 {
hasher: md5::Md5,
}
#[allow(
dead_code,
reason = "Content-MD5 is not a ChecksumAlgorithm variant and has no arm in into_impl: S3 carries it as its own header, separate from the x-amz-checksum-* family. This impl exists so the two paths share the Checksum trait, and is asserted by this crate's tests (backlog#1823)"
)]
#[allow(dead_code)]
impl Md5 {
fn update(&mut self, bytes: &[u8]) {
use md5::Digest;
+33 -30
View File
@@ -487,22 +487,21 @@ pub fn record_get_object_completion(total_duration_secs: f64, response_size_byte
/// Record the streaming strategy chosen for a GetObject response body.
#[inline(always)]
pub fn record_get_object_stream_strategy(strategy: &str, buffer_size_bytes: usize, response_size_bytes: i64) {
pub fn record_get_object_stream_strategy(strategy: &'static str, buffer_size_bytes: usize, response_size_bytes: i64) {
if !get_stage_metrics_enabled() {
return;
}
counter!("rustfs_io_get_object_stream_strategy_total", "strategy" => strategy.to_string()).increment(1);
histogram!("rustfs_io_get_object_stream_buffer_size_bytes", "strategy" => strategy.to_string())
.record(usize_to_f64(buffer_size_bytes));
histogram!("rustfs_io_get_object_stream_response_size_bytes", "strategy" => strategy.to_string())
counter!("rustfs_io_get_object_stream_strategy_total", "strategy" => strategy).increment(1);
histogram!("rustfs_io_get_object_stream_buffer_size_bytes", "strategy" => strategy).record(usize_to_f64(buffer_size_bytes));
histogram!("rustfs_io_get_object_stream_response_size_bytes", "strategy" => strategy)
.record(i64_non_negative_to_f64(response_size_bytes));
}
/// Record the response-body handoff shape from a GetObject reader into the S3 streaming body.
#[inline(always)]
pub fn record_get_object_response_handoff(
strategy: &str,
buffer_source: &str,
strategy: &'static str,
buffer_source: &'static str,
buffer_size_bytes: usize,
response_size_bytes: i64,
duration_secs: f64,
@@ -512,26 +511,26 @@ pub fn record_get_object_response_handoff(
}
counter!(
"rustfs_io_get_object_response_handoff_total",
"strategy" => strategy.to_string(),
"buffer_source" => buffer_source.to_string()
"strategy" => strategy,
"buffer_source" => buffer_source
)
.increment(1);
histogram!(
"rustfs_io_get_object_response_handoff_buffer_size_bytes",
"strategy" => strategy.to_string(),
"buffer_source" => buffer_source.to_string()
"strategy" => strategy,
"buffer_source" => buffer_source
)
.record(usize_to_f64(buffer_size_bytes));
histogram!(
"rustfs_io_get_object_response_handoff_response_size_bytes",
"strategy" => strategy.to_string(),
"buffer_source" => buffer_source.to_string()
"strategy" => strategy,
"buffer_source" => buffer_source
)
.record(i64_non_negative_to_f64(response_size_bytes));
histogram!(
"rustfs_io_get_object_response_handoff_duration_seconds",
"strategy" => strategy.to_string(),
"buffer_source" => buffer_source.to_string()
"strategy" => strategy,
"buffer_source" => buffer_source
)
.record(duration_secs);
record_get_object_response_handoff_duration("s3_handler", duration_secs);
@@ -539,14 +538,18 @@ pub fn record_get_object_response_handoff(
/// Record ReaderStream capacity chosen for GetObject handoff.
#[inline(always)]
pub fn record_get_object_reader_stream_buffer_size(strategy: &str, buffer_source: &str, buffer_size_bytes: usize) {
pub fn record_get_object_reader_stream_buffer_size(
strategy: &'static str,
buffer_source: &'static str,
buffer_size_bytes: usize,
) {
if !get_stage_metrics_enabled() {
return;
}
histogram!(
"rustfs_io_get_object_reader_stream_buffer_size_bytes",
"strategy" => strategy.to_string(),
"buffer_source" => buffer_source.to_string()
"strategy" => strategy,
"buffer_source" => buffer_source
)
.record(usize_to_f64(buffer_size_bytes));
}
@@ -554,8 +557,8 @@ pub fn record_get_object_reader_stream_buffer_size(strategy: &str, buffer_source
/// Record ReaderStream poll outcomes for GetObject handoff attribution.
#[inline(always)]
pub fn record_get_object_reader_stream_poll(
strategy: &str,
buffer_source: &str,
strategy: &'static str,
buffer_source: &'static str,
outcome: &'static str,
remaining_before: usize,
bytes: usize,
@@ -567,36 +570,36 @@ pub fn record_get_object_reader_stream_poll(
let bytes = u64::try_from(bytes).unwrap_or(u64::MAX);
counter!(
"rustfs_io_get_object_reader_stream_poll_total",
"strategy" => strategy.to_string(),
"buffer_source" => buffer_source.to_string(),
"strategy" => strategy,
"buffer_source" => buffer_source,
"outcome" => outcome
)
.increment(1);
counter!(
"rustfs_io_get_object_reader_stream_poll_bytes_total",
"strategy" => strategy.to_string(),
"buffer_source" => buffer_source.to_string(),
"strategy" => strategy,
"buffer_source" => buffer_source,
"outcome" => outcome
)
.increment(bytes);
histogram!(
"rustfs_io_get_object_reader_stream_poll_remaining_bytes",
"strategy" => strategy.to_string(),
"buffer_source" => buffer_source.to_string(),
"strategy" => strategy,
"buffer_source" => buffer_source,
"outcome" => outcome
)
.record(usize_to_f64(remaining_before));
histogram!(
"rustfs_io_get_object_reader_stream_poll_bytes",
"strategy" => strategy.to_string(),
"buffer_source" => buffer_source.to_string(),
"strategy" => strategy,
"buffer_source" => buffer_source,
"outcome" => outcome
)
.record(usize_to_f64(bytes as usize));
histogram!(
"rustfs_io_get_object_reader_stream_poll_duration_seconds",
"strategy" => strategy.to_string(),
"buffer_source" => buffer_source.to_string(),
"strategy" => strategy,
"buffer_source" => buffer_source,
"outcome" => outcome
)
.record(duration_secs);
+1 -4
View File
@@ -31,10 +31,7 @@ pub struct KeystoneClient {
admin_password: Option<String>,
admin_project: Option<String>,
admin_domain: String,
#[allow(
dead_code,
reason = "TLS verification flag parsed from config; the reqwest client is built before it is consulted, so nothing reads it back (backlog#1823)"
)]
#[allow(dead_code)]
verify_ssl: bool,
/// Request timeout applied to the underlying HTTP client.
timeout: std::time::Duration,
+1 -4
View File
@@ -20,10 +20,7 @@ use tracing::{debug, info};
/// Maps Keystone identities to RustFS concepts
pub struct KeystoneIdentityMapper {
#[allow(
dead_code,
reason = "keeps the Keystone client alive for the mapper's lifetime; the mapping paths do not call through it yet (backlog#1823)"
)]
#[allow(dead_code)]
client: Arc<KeystoneClient>,
role_policy_map: HashMap<String, String>,
enable_tenant_prefix: bool,
+1 -4
View File
@@ -40,10 +40,7 @@ impl RuleEvents for RuleView {
#[derive(Debug)]
struct CompiledRules {
// Keep RulesMap (can be used later if you want to make more complex judgments during the snapshot reading phase)
#[allow(
dead_code,
reason = "speculative retention: the comment above keeps it for richer snapshot-time judgements that no code performs yet (backlog#1823)"
)]
#[allow(dead_code)]
rules_map: RulesMap,
// for RulesContainer::iter_rules
rule_views: Vec<RuleView>,
+3
View File
@@ -187,6 +187,7 @@ impl RulesMap {
/// # Parameters
/// * `event_name` - The EventName from which to remove the rule.
/// * `pattern` - The pattern of the rule to be removed.
#[allow(dead_code)]
pub fn remove_rule(&mut self, event_name: &EventName, pattern: &str) {
let mut remove_event = false;
@@ -208,6 +209,7 @@ impl RulesMap {
///
/// # Parameters
/// * `event_names` - A slice of EventNames to be removed.
#[allow(dead_code)]
pub fn remove_rules(&mut self, event_names: &[EventName]) {
for event_name in event_names {
self.map.remove(event_name);
@@ -221,6 +223,7 @@ impl RulesMap {
/// * `event_name` - The EventName to update.
/// * `pattern` - The pattern of the rule to be updated.
/// * `target_id` - The TargetID to be added.
#[allow(dead_code)]
pub fn update_rule(&mut self, event_name: EventName, pattern: String, target_id: TargetID) {
self.map.entry(event_name).or_default().add(pattern, target_id);
self.total_events_mask |= event_name.mask(); // Update only the relevant bitmask
+6
View File
@@ -18,6 +18,12 @@ use rustfs_targets::arn::TargetID;
/// TargetIDSet - A collection representation of TargetID.
pub type TargetIdSet = HashSet<TargetID>;
/// Provides a Go-like method for TargetIdSet (can be implemented as trait if needed)
#[allow(dead_code)]
pub(crate) fn new_target_id_set(target_ids: Vec<TargetID>) -> TargetIdSet {
target_ids.into_iter().collect()
}
// HashSet has built-in clone, union, difference and other operations.
// But the Go version of the method returns a new Set, and the HashSet method is usually iterator or modify itself.
// If you need to exactly match Go's API style, you can add wrapper functions.
+4
View File
@@ -219,6 +219,10 @@ impl PartialEq for Functions {
}
}
#[derive(Clone, Serialize, Deserialize)]
#[allow(dead_code)]
pub struct Value;
#[cfg(test)]
mod tests {
use crate::policy::Functions;
+2 -4
View File
@@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#[allow(dead_code)]
pub fn is_simple_match<P, N>(pattern: P, name: N) -> bool
where
P: AsRef<str>,
@@ -28,10 +29,7 @@ where
inner_match(pattern, name, false)
}
#[allow(
dead_code,
reason = "prefix-matcher asserted by this file's tests; no production caller yet (backlog#1823)"
)]
#[allow(dead_code)]
pub fn is_match_as_pattern_prefix<P, N>(pattern: P, text: N) -> bool
where
P: AsRef<str>,
@@ -27,10 +27,6 @@ use crate::CloudMetadataFetcher;
#[derive(Debug, Clone)]
pub struct AwsMetadataFetcher {
client: Client,
#[allow(
dead_code,
reason = "IMDS endpoint retained beside the client it configures; requests build their own URLs (backlog#1823)"
)]
metadata_endpoint: String,
}
@@ -50,6 +46,55 @@ impl AwsMetadataFetcher {
metadata_endpoint: "http://169.254.169.254".to_string(),
}
}
/// Retrieves an IMDSv2 token for secure metadata access.
#[allow(dead_code)]
async fn get_metadata_token(&self) -> Result<String, AppError> {
let url = format!("{}/latest/api/token", self.metadata_endpoint);
match self
.client
.put(&url)
.header("X-aws-ec2-metadata-token-ttl-seconds", "21600")
.send()
.await
{
Ok(response) => {
if response.status().is_success() {
let token = response
.text()
.await
.map_err(|e| AppError::cloud(format!("Failed to read IMDSv2 token: {}", e)))?;
Ok(token)
} else {
debug!(
event = "trusted_proxies.cloud_metadata",
component = "trusted_proxies",
subsystem = "aws_metadata",
provider = "aws",
operation = "imdsv2_token",
result = "http_error",
status = %response.status(),
"trusted proxy cloud metadata request failed"
);
Err(AppError::cloud("Failed to obtain IMDSv2 token"))
}
}
Err(e) => {
debug!(
event = "trusted_proxies.cloud_metadata",
component = "trusted_proxies",
subsystem = "aws_metadata",
provider = "aws",
operation = "imdsv2_token",
result = "request_failed",
error = %e,
"trusted proxy cloud metadata request failed"
);
Err(AppError::cloud(format!("IMDSv2 request failed: {}", e)))
}
}
}
}
#[async_trait]
+1
View File
@@ -68,6 +68,7 @@ pub fn is_env_set(key: &str) -> bool {
}
/// Returns a list of all proxy-related environment variables and their current values.
#[allow(dead_code)]
pub fn get_all_proxy_env_vars() -> Vec<(String, String)> {
let vars = [
ENV_TRUSTED_PROXY_ENABLED,
+1
View File
@@ -68,6 +68,7 @@ pub async fn read_full_or_eof<R: AsyncRead + Send + Sync + Unpin>(
/// Read exactly buf.len() bytes into buf, or return an error if EOF is reached before any bytes are read.
/// Like Go's io.ReadFull.
#[allow(dead_code)]
pub async fn read_full<R: AsyncRead + Send + Sync + Unpin>(reader: R, buf: &mut [u8]) -> std::io::Result<usize> {
match read_full_or_eof(reader, buf).await? {
Some(n) => Ok(n),
+1
View File
@@ -431,6 +431,7 @@ pub fn parse_and_resolve_address(addr_str: &str) -> std::io::Result<SocketAddr>
Ok(resolved_addr)
}
#[allow(dead_code)]
pub fn bytes_stream<S, E>(stream: S, content_length: usize) -> impl Stream<Item = Result<Bytes, E>> + Send + 'static
where
S: Stream<Item = Result<Bytes, E>> + Send + 'static,
+1
View File
@@ -16,6 +16,7 @@
///
/// The table follows Linux `include/uapi/linux/magic.h`; filesystem magic
/// values without a stable Linux uapi source stay `UNKNOWN`.
#[allow(dead_code)]
pub(crate) fn get_fs_type(fs_type: u64) -> &'static str {
// Magic numbers for various filesystems.
match fs_type {
+1
View File
@@ -70,6 +70,7 @@ pub fn is_dir_object(object: &str) -> bool {
///
/// If the object name ends with `GLOBAL_DIR_SUFFIX`, it is replaced with a slash.
/// Otherwise, the name is returned as is.
#[allow(dead_code)]
pub fn decode_dir_object(object: &str) -> String {
if has_suffix(object, GLOBAL_DIR_SUFFIX) {
format!("{}{}", object.trim_end_matches(GLOBAL_DIR_SUFFIX), SLASH_SEPARATOR)