gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-06 05:17:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

perf(observability): avoid cgroup path allocation (#5711)

Browse Source 
* perf(observability): avoid cgroup path allocation

Co-Authored-By: heihutu <heihutu@gmail.com>

* test(e2e): satisfy regression test clippy

Co-Authored-By: heihutu <heihutu@gmail.com>

---------

Co-authored-by: heihutu <heihutu@gmail.com>
This commit is contained in:
houseme
2026-08-05 01:47:07 +08:00
committed by GitHub
parent 15c2bade5f
commit 3dabac4a09
9 changed files with 50 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/e2e_test/src/bucket_stats_regression_test.rs
+14 -15
View File
@@ -32,7 +32,6 @@
#[cfg(test)]
mod tests {
use crate::common::{RustFSTestEnvironment, awscurl_get, init_logging};
use aws_sdk_s3::Client;
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
use rustfs_data_usage::DataUsageInfo;
@@ -92,13 +91,13 @@ mod tests {
for attempt in 0..18 {
sleep(Duration::from_secs(5)).await;
if let Ok(usage) = get_data_usage(&env).await {
if let Some(bucket_usage) = usage.buckets_usage.get(bucket) {
info!(" attempt {attempt}: objectsCount = {}", bucket_usage.objects_count);
if bucket_usage.objects_count >= 10 {
found_nonzero = true;
break;
}
if let Ok(usage) = get_data_usage(&env).await
&& let Some(bucket_usage) = usage.buckets_usage.get(bucket)
{
info!(" attempt {attempt}: objectsCount = {}", bucket_usage.objects_count);
if bucket_usage.objects_count >= 10 {
found_nonzero = true;
break;
}
}
}
@@ -160,13 +159,13 @@ mod tests {
for attempt in 0..18 {
sleep(Duration::from_secs(5)).await;
if let Ok(usage) = get_data_usage(&env).await {
if let Some(bucket_usage) = usage.buckets_usage.get(bucket) {
info!(" attempt {attempt}: objectsCount = {}", bucket_usage.objects_count);
if bucket_usage.objects_count == 0 {
found_zero = true;
break;
}
if let Ok(usage) = get_data_usage(&env).await
&& let Some(bucket_usage) = usage.buckets_usage.get(bucket)
{
info!(" attempt {attempt}: objectsCount = {}", bucket_usage.objects_count);
if bucket_usage.objects_count == 0 {
found_zero = true;
break;
}
}
}
crates/e2e_test/src/delete_regression_test.rs
+12 -5
View File
@@ -31,7 +31,6 @@
#[cfg(test)]
mod tests {
use crate::common::{RustFSTestEnvironment, init_logging};
use aws_sdk_s3::Client;
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{BucketVersioningStatus, Delete, ObjectIdentifier, VersioningConfiguration};
use serial_test::serial;
@@ -85,8 +84,13 @@ mod tests {
.await
.expect("list objects before delete");
let keys: Vec<_> = list.contents().iter().map(|o| o.key().unwrap_or("")).collect();
assert!(keys.contains(&"to-delete.txt"), "RT-05 FAIL: object not in LIST before delete");
assert!(
list.contents()
.iter()
.map(|o| o.key().unwrap_or(""))
.any(|key| key == "to-delete.txt"),
"RT-05 FAIL: object not in LIST before delete"
);
// DELETE
client
@@ -105,9 +109,12 @@ mod tests {
.await
.expect("list objects after delete");
let keys: Vec<_> = list.contents().iter().map(|o| o.key().unwrap_or("")).collect();
assert!(
!keys.contains(&"to-delete.txt"),
!list
.contents()
.iter()
.map(|o| o.key().unwrap_or(""))
.any(|key| key == "to-delete.txt"),
"RT-05 FAIL: deleted object still in LIST (regression rustfs#5375)"
);
crates/e2e_test/src/distributed_startup_regression_test.rs
+2 -2
View File
@@ -34,8 +34,8 @@ mod tests {
use aws_sdk_s3::primitives::ByteStream;
use serial_test::serial;
use std::error::Error;
use tokio::time::{Duration, sleep, timeout};
use tracing::{info, warn};
use tokio::time::{Duration, sleep};
use tracing::info;
type TestResult = Result<(), Box<dyn Error + Send + Sync>>;
crates/e2e_test/src/lifecycle_regression_test.rs
+1 -2
View File
@@ -38,8 +38,7 @@ mod tests {
};
use serial_test::serial;
use std::error::Error;
use tokio::time::{Duration, sleep, timeout};
use tracing::{info, warn};
use tracing::info;
type TestResult = Result<(), Box<dyn Error + Send + Sync>>;
crates/e2e_test/src/listing_regression_test.rs
-1
View File
@@ -30,7 +30,6 @@
#[cfg(test)]
mod tests {
use crate::common::{RustFSTestEnvironment, init_logging};
use aws_sdk_s3::Client;
use aws_sdk_s3::primitives::ByteStream;
use serial_test::serial;
use std::collections::HashSet;
crates/e2e_test/src/notification_startup_regression_test.rs
-1
View File
@@ -31,7 +31,6 @@
#[cfg(test)]
mod tests {
use crate::common::{RustFSTestEnvironment, init_logging};
use aws_sdk_s3::Client;
use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
use serial_test::serial;
use std::error::Error;
crates/e2e_test/src/tier_transition_regression_test.rs
+1 -4
View File
@@ -29,13 +29,10 @@
#[cfg(test)]
mod tests {
use crate::common::{RustFSTestEnvironment, admin_ok, init_logging};
use aws_sdk_s3::Client;
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
use serde_json::Value;
use serial_test::serial;
use std::error::Error;
use tracing::{info, warn};
use tracing::info;
type TestResult = Result<(), Box<dyn Error + Send + Sync>>;
rustfs/src/admin/service/federated_identity.rs
+8 -7
View File
@@ -430,12 +430,16 @@ mod tests {
#[test]
fn issued_credentials_and_replication_item_use_minio_parent_shape() {
let transaction = transaction();
let expected_parent = transaction
.authorization
.oidc_virtual_parent()
.expect("test authorization should have a virtual parent");
let secret = "federated-session-test-signing-secret";
let selected_policy_names = vec!["readonly".to_string()];
let credentials =
issue_credentials(&transaction, &selected_policy_names, Some(secret)).expect("credential issuance should succeed");
assert_eq!(credentials.parent_user, "TwyekekG2eMes0qk9Tgh7KXEitwGi1z2W1f2KccrXGA");
assert_eq!(credentials.parent_user, expected_parent);
assert_eq!(credentials.groups, Some(vec!["devs".to_string()]));
assert_eq!(credentials.status, "on");
assert!(!credentials.access_key.is_empty());
@@ -468,11 +472,8 @@ mod tests {
("preferred_username".to_string(), serde_json::json!("user")),
("groups".to_string(), serde_json::json!(["devs"])),
("roles".to_string(), serde_json::json!(["admin", "reader"])),
("parent".to_string(), serde_json::json!("TwyekekG2eMes0qk9Tgh7KXEitwGi1z2W1f2KccrXGA")),
(
OIDC_VIRTUAL_PARENT_CLAIM.to_string(),
serde_json::json!("TwyekekG2eMes0qk9Tgh7KXEitwGi1z2W1f2KccrXGA"),
),
("parent".to_string(), serde_json::json!(expected_parent)),
(OIDC_VIRTUAL_PARENT_CLAIM.to_string(), serde_json::json!(expected_parent)),
("policy".to_string(), serde_json::json!("readonly")),
])
);
@@ -499,7 +500,7 @@ mod tests {
"accessKey": "<access-key>",
"secretKey": "<secret-key>",
"sessionToken": "<session-token>",
"parentUser": "TwyekekG2eMes0qk9Tgh7KXEitwGi1z2W1f2KccrXGA",
"parentUser": expected_parent,
"parentPolicyMapping": OIDC_STS_REQUIRES_VIRTUAL_PARENT_RECEIVER_POLICY,
"apiVersion": SITE_REPL_API_VERSION,
},
rustfs/src/memory_observability.rs
+12 -18
View File
@@ -33,6 +33,12 @@ static MEMORY_SYSTEM: OnceLock<Mutex<System>> = OnceLock::new();
const ENV_MEMORY_OBSERVABILITY_INTERVAL_SECS: &str = "RUSTFS_MEMORY_OBSERVABILITY_INTERVAL_SECS";
const DEFAULT_MEMORY_OBSERVABILITY_INTERVAL_SECS: u64 = 15;
const MEMORY_OBSERVABILITY_SERVICE_NAME: &str = "memory_observability";
const CGROUP_V2_MEMORY_STAT_PATH: &str = "/sys/fs/cgroup/memory.stat";
const CGROUP_V2_MEMORY_CURRENT_PATH: &str = "/sys/fs/cgroup/memory.current";
const CGROUP_V2_MEMORY_MAX_PATH: &str = "/sys/fs/cgroup/memory.max";
const CGROUP_V1_MEMORY_STAT_PATH: &str = "/sys/fs/cgroup/memory/memory.stat";
const CGROUP_V1_MEMORY_USAGE_PATH: &str = "/sys/fs/cgroup/memory/memory.usage_in_bytes";
const CGROUP_V1_MEMORY_LIMIT_PATH: &str = "/sys/fs/cgroup/memory/memory.limit_in_bytes";
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
@@ -198,16 +204,10 @@ fn parse_cgroup_memory_stat(content: &str) -> CgroupMemoryStatFields {
}
fn read_cgroup_v2() -> Option<CgroupMemorySnapshot> {
let root = Path::new("/sys/fs/cgroup");
let stat_path = root.join("memory.stat");
if !stat_path.exists() {
return None;
}
let stats = parse_cgroup_memory_stat(&std::fs::read_to_string(&stat_path).ok()?);
let stats = parse_cgroup_memory_stat(&std::fs::read_to_string(CGROUP_V2_MEMORY_STAT_PATH).ok()?);
Some(CgroupMemorySnapshot {
current_bytes: read_optional_u64(&root.join("memory.current")),
limit_bytes: read_optional_u64(&root.join("memory.max")),
current_bytes: read_optional_u64(Path::new(CGROUP_V2_MEMORY_CURRENT_PATH)),
limit_bytes: read_optional_u64(Path::new(CGROUP_V2_MEMORY_MAX_PATH)),
anon_bytes: stats.anon,
file_bytes: stats.file,
active_file_bytes: stats.active_file,
@@ -216,16 +216,10 @@ fn read_cgroup_v2() -> Option<CgroupMemorySnapshot> {
}
fn read_cgroup_v1() -> Option<CgroupMemorySnapshot> {
let root = Path::new("/sys/fs/cgroup/memory");
let stat_path = root.join("memory.stat");
if !stat_path.exists() {
return None;
}
let stats = parse_cgroup_memory_stat(&std::fs::read_to_string(&stat_path).ok()?);
let stats = parse_cgroup_memory_stat(&std::fs::read_to_string(CGROUP_V1_MEMORY_STAT_PATH).ok()?);
Some(CgroupMemorySnapshot {
current_bytes: read_optional_u64(&root.join("memory.usage_in_bytes")),
limit_bytes: read_optional_u64(&root.join("memory.limit_in_bytes")),
current_bytes: read_optional_u64(Path::new(CGROUP_V1_MEMORY_USAGE_PATH)),
limit_bytes: read_optional_u64(Path::new(CGROUP_V1_MEMORY_LIMIT_PATH)),
anon_bytes: stats.total_rss.or(stats.rss),
file_bytes: stats.total_cache.or(stats.cache),
active_file_bytes: stats.total_active_file.or(stats.active_file),