mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-22 20:36:38 +00:00
Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| a79b806fb4 |
@@ -1,2 +1,2 @@
|
||||
sha256-darwin=9f767b37ed8b1c82da62ea441462d75487785c8086e56f08fb6f6cd89c6e2e52
|
||||
sha256-linux=fbdaf42b220958d4b1e8880e0f8b5a7992d38e21051bb60596dd4538424757d6
|
||||
sha256-darwin=f832043fcca8c0b616c5d820a3a652da7544298ef5812a8668a3a9a3e4607b8b
|
||||
sha256-linux=49d32f52307be4e6ca9351c79c339a106ff96432b0e9a2eba92daf8111dbef57
|
||||
|
||||
Generated
+27
-22
@@ -1858,9 +1858,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.4.4"
|
||||
version = "1.4.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0ad534f4357a5264cce5019c989cf66a4f0dc4e0d1b1d15f8aacec0ff7360273"
|
||||
checksum = "509591b7bcd67f4ef775afad7662703b4935daaa6ec0e5605cfb1090b32a2b6d"
|
||||
dependencies = [
|
||||
"find-msvc-tools",
|
||||
"jobserver",
|
||||
@@ -2522,6 +2522,12 @@ dependencies = [
|
||||
"subtle",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cty"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b365fabc795046672053e29c954733ec3b05e4be654ab130fe8f1f94d7051f35"
|
||||
|
||||
[[package]]
|
||||
name = "curve25519-dalek"
|
||||
version = "4.1.3"
|
||||
@@ -5982,6 +5988,15 @@ version = "0.2.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981"
|
||||
|
||||
[[package]]
|
||||
name = "libmimalloc-sys"
|
||||
version = "0.1.49"
|
||||
source = "git+https://github.com/xonatius/mimalloc_rust.git?rev=6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11#6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"cty",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "libredox"
|
||||
version = "0.1.20"
|
||||
@@ -6382,6 +6397,14 @@ dependencies = [
|
||||
"synstructure 0.13.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "mimalloc"
|
||||
version = "0.1.52"
|
||||
source = "git+https://github.com/xonatius/mimalloc_rust.git?rev=6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11#6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11"
|
||||
dependencies = [
|
||||
"libmimalloc-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "mime"
|
||||
version = "0.3.17"
|
||||
@@ -9139,11 +9162,13 @@ dependencies = [
|
||||
"insta",
|
||||
"jiff",
|
||||
"libc",
|
||||
"libmimalloc-sys",
|
||||
"libsystemd",
|
||||
"matchit 0.9.2",
|
||||
"md-5 0.11.0",
|
||||
"metrics",
|
||||
"metrics-util",
|
||||
"mimalloc",
|
||||
"mime_guess",
|
||||
"opentelemetry",
|
||||
"opentelemetry_sdk",
|
||||
@@ -9179,8 +9204,6 @@ dependencies = [
|
||||
"rustfs-lock",
|
||||
"rustfs-log-analyzer",
|
||||
"rustfs-madmin",
|
||||
"rustfs-mimalloc",
|
||||
"rustfs-mimalloc-sys",
|
||||
"rustfs-notify",
|
||||
"rustfs-object-capacity",
|
||||
"rustfs-object-data-cache",
|
||||
@@ -9852,24 +9875,6 @@ dependencies = [
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-mimalloc"
|
||||
version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a406f4aa07084301d485beec873af6dccc8e3f8762da244743df92038b1db1a6"
|
||||
dependencies = [
|
||||
"rustfs-mimalloc-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-mimalloc-sys"
|
||||
version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c3051b819175f58445d4c369a72f0ab88149f3885ba8bea2aff3be01f53fe7cd"
|
||||
dependencies = [
|
||||
"cc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-notify"
|
||||
version = "1.0.0-rc.3"
|
||||
|
||||
+2
-2
@@ -350,8 +350,8 @@ russh-sftp = "2.4.0"
|
||||
dav-server = "0.11.0"
|
||||
|
||||
# Performance Analysis and Memory Profiling
|
||||
rustfs-mimalloc = { version = "0.5.0" }
|
||||
rustfs-mimalloc-sys = { version = "0.5.0" }
|
||||
mimalloc = { version = "0.1.52", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11" }
|
||||
libmimalloc-sys = { version = "0.1.49", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11", features = ["extended"] }
|
||||
hotpath = { version = "0.23.3", default-features = false }
|
||||
# Snapshot testing for output format regression detection
|
||||
insta = { version = "1.48" }
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
//! E2E tests for group management (fixes #2028).
|
||||
|
||||
use crate::common::{RustFSTestEnvironment, admin_request, awscurl_delete, awscurl_get, awscurl_put, init_logging};
|
||||
use crate::common::{RustFSTestEnvironment, admin_ok, admin_request, init_logging};
|
||||
use aws_sdk_s3::config::{Credentials, Region};
|
||||
use aws_sdk_s3::{Client, Config};
|
||||
use tracing::info;
|
||||
@@ -83,7 +83,6 @@ async fn update_group_members_rejects_invalid_new_group_names() -> Result<(), Bo
|
||||
|
||||
/// Test that deleting a group with members fails, and deleting an empty group succeeds.
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[ignore = "requires awscurl and spawns a real RustFS server"]
|
||||
async fn test_delete_group_requires_empty_membership() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
@@ -91,29 +90,49 @@ async fn test_delete_group_requires_empty_membership() -> Result<(), Box<dyn std
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
// 1. Create a user
|
||||
let add_user_url = format!("{}/rustfs/admin/v3/add-user?accessKey=testuser1", env.url);
|
||||
let user_body = serde_json::json!({
|
||||
"secretKey": "testuser1secret",
|
||||
"status": "enabled"
|
||||
});
|
||||
awscurl_put(&add_user_url, &user_body.to_string(), &env.access_key, &env.secret_key).await?;
|
||||
admin_ok(
|
||||
&env,
|
||||
http::Method::PUT,
|
||||
"/rustfs/admin/v3/add-user?accessKey=testuser1",
|
||||
Some(user_body.to_string()),
|
||||
)
|
||||
.await?;
|
||||
info!("Created testuser1");
|
||||
|
||||
// 2. Create a group with testuser1 as a member
|
||||
let update_members_url = format!("{}/rustfs/admin/v3/update-group-members", env.url);
|
||||
let add_member_body = serde_json::json!({
|
||||
"group": "testgroup",
|
||||
"members": ["testuser1"],
|
||||
"isRemove": false,
|
||||
"groupStatus": "enabled"
|
||||
});
|
||||
awscurl_put(&update_members_url, &add_member_body.to_string(), &env.access_key, &env.secret_key).await?;
|
||||
admin_ok(
|
||||
&env,
|
||||
http::Method::PUT,
|
||||
"/rustfs/admin/v3/update-group-members",
|
||||
Some(add_member_body.to_string()),
|
||||
)
|
||||
.await?;
|
||||
info!("Added testuser1 to testgroup");
|
||||
|
||||
// 3. Attempt to delete the group while it still has members — should fail
|
||||
let delete_group_url = format!("{}/rustfs/admin/v3/group/testgroup", env.url);
|
||||
let delete_result = awscurl_delete(&delete_group_url, &env.access_key, &env.secret_key).await;
|
||||
assert!(delete_result.is_err(), "deleting a non-empty group should fail");
|
||||
let (delete_status, delete_body) = admin_request(
|
||||
&env.url,
|
||||
http::Method::DELETE,
|
||||
"/rustfs/admin/v3/group/testgroup",
|
||||
None,
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
)
|
||||
.await?;
|
||||
assert!(
|
||||
!delete_status.is_success(),
|
||||
"deleting a non-empty group should fail, got {delete_status}: {delete_body}"
|
||||
);
|
||||
info!("Delete of non-empty group correctly rejected");
|
||||
|
||||
// 4. Remove the member from the group
|
||||
@@ -123,17 +142,33 @@ async fn test_delete_group_requires_empty_membership() -> Result<(), Box<dyn std
|
||||
"isRemove": true,
|
||||
"groupStatus": "enabled"
|
||||
});
|
||||
awscurl_put(&update_members_url, &remove_member_body.to_string(), &env.access_key, &env.secret_key).await?;
|
||||
admin_ok(
|
||||
&env,
|
||||
http::Method::PUT,
|
||||
"/rustfs/admin/v3/update-group-members",
|
||||
Some(remove_member_body.to_string()),
|
||||
)
|
||||
.await?;
|
||||
info!("Removed testuser1 from testgroup");
|
||||
|
||||
// 5. Delete the now-empty group — should succeed
|
||||
awscurl_delete(&delete_group_url, &env.access_key, &env.secret_key).await?;
|
||||
admin_ok(&env, http::Method::DELETE, "/rustfs/admin/v3/group/testgroup", None).await?;
|
||||
info!("Deleted empty testgroup successfully");
|
||||
|
||||
// 6. Verify the group no longer exists
|
||||
let get_group_url = format!("{}/rustfs/admin/v3/group?group=testgroup", env.url);
|
||||
let get_result = awscurl_get(&get_group_url, &env.access_key, &env.secret_key).await;
|
||||
assert!(get_result.is_err(), "group should no longer exist after deletion");
|
||||
let (get_status, get_body) = admin_request(
|
||||
&env.url,
|
||||
http::Method::GET,
|
||||
"/rustfs/admin/v3/group?group=testgroup",
|
||||
None,
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
)
|
||||
.await?;
|
||||
assert!(
|
||||
!get_status.is_success(),
|
||||
"group should no longer exist after deletion, got {get_status}: {get_body}"
|
||||
);
|
||||
info!("Confirmed testgroup no longer exists");
|
||||
|
||||
Ok(())
|
||||
@@ -142,7 +177,6 @@ async fn test_delete_group_requires_empty_membership() -> Result<(), Box<dyn std
|
||||
/// Test that a user with only group membership (no explicit user policy) gets group policies
|
||||
/// and can perform actions allowed by the group (regression test for #2028.1).
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[ignore = "requires awscurl and spawns a real RustFS server"]
|
||||
async fn test_user_with_only_group_gets_group_policies() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
@@ -163,36 +197,53 @@ async fn test_user_with_only_group_gets_group_policies() -> Result<(), Box<dyn s
|
||||
"Resource": ["*"]
|
||||
}]
|
||||
});
|
||||
let add_policy_url = format!("{}/rustfs/admin/v3/add-canned-policy?name={}", env.url, policy_name);
|
||||
awscurl_put(&add_policy_url, &policy_doc.to_string(), &env.access_key, &env.secret_key).await?;
|
||||
admin_ok(
|
||||
&env,
|
||||
http::Method::PUT,
|
||||
&format!("/rustfs/admin/v3/add-canned-policy?name={policy_name}"),
|
||||
Some(policy_doc.to_string()),
|
||||
)
|
||||
.await?;
|
||||
info!("Created canned policy {}", policy_name);
|
||||
|
||||
// 2. Create user with no explicit policy
|
||||
let add_user_url = format!("{}/rustfs/admin/v3/add-user?accessKey={}", env.url, user_name);
|
||||
let user_body = serde_json::json!({
|
||||
"secretKey": user_secret,
|
||||
"status": "enabled"
|
||||
});
|
||||
awscurl_put(&add_user_url, &user_body.to_string(), &env.access_key, &env.secret_key).await?;
|
||||
admin_ok(
|
||||
&env,
|
||||
http::Method::PUT,
|
||||
&format!("/rustfs/admin/v3/add-user?accessKey={user_name}"),
|
||||
Some(user_body.to_string()),
|
||||
)
|
||||
.await?;
|
||||
info!("Created user {} with no explicit policy", user_name);
|
||||
|
||||
// 3. Add user to group (creates group with this member; user_group_memberships must be updated)
|
||||
let update_members_url = format!("{}/rustfs/admin/v3/update-group-members", env.url);
|
||||
let add_member_body = serde_json::json!({
|
||||
"group": group_name,
|
||||
"members": [user_name],
|
||||
"isRemove": false,
|
||||
"groupStatus": "enabled"
|
||||
});
|
||||
awscurl_put(&update_members_url, &add_member_body.to_string(), &env.access_key, &env.secret_key).await?;
|
||||
admin_ok(
|
||||
&env,
|
||||
http::Method::PUT,
|
||||
"/rustfs/admin/v3/update-group-members",
|
||||
Some(add_member_body.to_string()),
|
||||
)
|
||||
.await?;
|
||||
info!("Added {} to group {}", user_name, group_name);
|
||||
|
||||
// 4. Attach policy to group
|
||||
let set_policy_url = format!(
|
||||
"{}/rustfs/admin/v3/set-user-or-group-policy?policyName={}&userOrGroup={}&isGroup=true",
|
||||
env.url, policy_name, group_name
|
||||
);
|
||||
awscurl_put(&set_policy_url, "", &env.access_key, &env.secret_key).await?;
|
||||
admin_ok(
|
||||
&env,
|
||||
http::Method::PUT,
|
||||
&format!("/rustfs/admin/v3/set-user-or-group-policy?policyName={policy_name}&userOrGroup={group_name}&isGroup=true"),
|
||||
Some(String::new()),
|
||||
)
|
||||
.await?;
|
||||
info!("Attached policy {} to group {}", policy_name, group_name);
|
||||
|
||||
// 5. User with only group (no user policy) should be able to list buckets
|
||||
@@ -209,7 +260,6 @@ async fn test_user_with_only_group_gets_group_policies() -> Result<(), Box<dyn s
|
||||
/// Test that after deleting a user who was the only member of a group, the group can be deleted
|
||||
/// (regression test for #2028.2: delete group uses backend membership, not stale cache).
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[ignore = "requires awscurl and spawns a real RustFS server"]
|
||||
async fn test_delete_group_after_deleting_user() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
@@ -221,33 +271,47 @@ async fn test_delete_group_after_deleting_user() -> Result<(), Box<dyn std::erro
|
||||
let group_name = "soledeletegroup";
|
||||
|
||||
// 1. Create user
|
||||
let add_user_url = format!("{}/rustfs/admin/v3/add-user?accessKey={}", env.url, user_name);
|
||||
let user_body = serde_json::json!({
|
||||
"secretKey": user_secret,
|
||||
"status": "enabled"
|
||||
});
|
||||
awscurl_put(&add_user_url, &user_body.to_string(), &env.access_key, &env.secret_key).await?;
|
||||
admin_ok(
|
||||
&env,
|
||||
http::Method::PUT,
|
||||
&format!("/rustfs/admin/v3/add-user?accessKey={user_name}"),
|
||||
Some(user_body.to_string()),
|
||||
)
|
||||
.await?;
|
||||
info!("Created user {}", user_name);
|
||||
|
||||
// 2. Add user to group
|
||||
let update_members_url = format!("{}/rustfs/admin/v3/update-group-members", env.url);
|
||||
let add_member_body = serde_json::json!({
|
||||
"group": group_name,
|
||||
"members": [user_name],
|
||||
"isRemove": false,
|
||||
"groupStatus": "enabled"
|
||||
});
|
||||
awscurl_put(&update_members_url, &add_member_body.to_string(), &env.access_key, &env.secret_key).await?;
|
||||
admin_ok(
|
||||
&env,
|
||||
http::Method::PUT,
|
||||
"/rustfs/admin/v3/update-group-members",
|
||||
Some(add_member_body.to_string()),
|
||||
)
|
||||
.await?;
|
||||
info!("Added {} to group {}", user_name, group_name);
|
||||
|
||||
// 3. Delete the user (backend and cache update so group membership becomes empty)
|
||||
let remove_user_url = format!("{}/rustfs/admin/v3/remove-user?accessKey={}", env.url, user_name);
|
||||
awscurl_delete(&remove_user_url, &env.access_key, &env.secret_key).await?;
|
||||
admin_ok(
|
||||
&env,
|
||||
http::Method::DELETE,
|
||||
&format!("/rustfs/admin/v3/remove-user?accessKey={user_name}"),
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
info!("Deleted user {}", user_name);
|
||||
|
||||
// 4. Deleting the group should succeed (backend has empty members; no stale cache)
|
||||
let delete_group_url = format!("{}/rustfs/admin/v3/group/{}", env.url, group_name);
|
||||
awscurl_delete(&delete_group_url, &env.access_key, &env.secret_key).await?;
|
||||
admin_ok(&env, http::Method::DELETE, &format!("/rustfs/admin/v3/group/{group_name}"), None).await?;
|
||||
info!("Deleted group {} after user was removed", group_name);
|
||||
|
||||
Ok(())
|
||||
|
||||
@@ -43,6 +43,9 @@ allow-git = [
|
||||
# RustFS fork carrying presigned expiry and constant-time authentication fixes.
|
||||
# owner: rustfs-maintainers review: 2026-10
|
||||
"https://github.com/rustfs/s3s.git",
|
||||
# MiMalloc fork pinned for hotpath allocation counting support.
|
||||
# owner: houseme review: 2026-10
|
||||
"https://github.com/xonatius/mimalloc_rust.git",
|
||||
]
|
||||
|
||||
[bans]
|
||||
|
||||
@@ -52,7 +52,7 @@
|
||||
| fault_proxy | 7 | |
|
||||
| get_codec_streaming_compat_test | 1 | |
|
||||
| get_stream_failure_observability_test | 1 | |
|
||||
| group_delete_test | 1 | |
|
||||
| group_delete_test | 4 | |
|
||||
| head_object_consistency_test | 1 | ✅ |
|
||||
| head_object_range_test | 1 | ✅ |
|
||||
| heal_erasure_disk_rebuild_test | 4 | 🌙 |
|
||||
@@ -99,4 +99,4 @@
|
||||
| tls_hot_reload_test | 1 | ✅ |
|
||||
| version_id_regression_test | 10 | ✅ |
|
||||
|
||||
**Total listed: 575 tests across 82 modules · PR smoke: 163 tests / 36 modules · merge/main full: 453 tests / 73 modules · nightly replication: 55 tests · nightly cluster faults: 28 tests / 7 modules · nightly protocols: 16 tests** · updated 2026-08-23.
|
||||
**Total listed: 578 tests across 82 modules · PR smoke: 163 tests / 36 modules · merge/main full: 456 tests / 73 modules · nightly replication: 55 tests · nightly cluster faults: 28 tests / 7 modules · nightly protocols: 16 tests** · updated 2026-08-23.
|
||||
|
||||
+2
-2
@@ -336,13 +336,13 @@ opentelemetry = { workspace = true }
|
||||
tracing-opentelemetry = { workspace = true }
|
||||
# Data structures
|
||||
hashbrown = { workspace = true, features = ["serde", "rayon"] }
|
||||
rustfs-mimalloc = { workspace = true }
|
||||
mimalloc = { workspace = true }
|
||||
|
||||
[target.'cfg(target_os = "linux")'.dependencies]
|
||||
libsystemd.workspace = true
|
||||
|
||||
[target.'cfg(not(target_os = "windows"))'.dependencies]
|
||||
rustfs-mimalloc-sys.workspace = true
|
||||
libmimalloc-sys.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
uuid = { workspace = true, features = ["v4", "fast-rng", "macro-diagnostics"] }
|
||||
|
||||
@@ -369,8 +369,14 @@ pub fn allocator_reclaim_controller_snapshot(ctx: &CancellationToken) -> Allocat
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "windows"))]
|
||||
#[allow(unsafe_code)]
|
||||
fn collect_allocator_memory(force: bool) -> Result<(), String> {
|
||||
rustfs_mimalloc::MiMalloc::collect(force);
|
||||
// SAFETY: `mi_collect` is provided by the active global allocator backend
|
||||
// on this target family. It is explicitly intended to reclaim retained
|
||||
// pages/segments and does not require additional invariants from the caller.
|
||||
unsafe {
|
||||
libmimalloc_sys::mi_collect(force);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
+8
-10
@@ -26,22 +26,22 @@ struct MiMallocAllocator;
|
||||
unsafe impl GlobalAlloc for MiMallocAllocator {
|
||||
unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
|
||||
// SAFETY: the caller upholds GlobalAlloc's contract for layout.
|
||||
unsafe { rustfs_mimalloc::MiMalloc.alloc(layout) }
|
||||
unsafe { mimalloc::MiMalloc.alloc(layout) }
|
||||
}
|
||||
|
||||
unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 {
|
||||
// SAFETY: the caller upholds GlobalAlloc's contract for layout.
|
||||
unsafe { rustfs_mimalloc::MiMalloc.alloc_zeroed(layout) }
|
||||
unsafe { mimalloc::MiMalloc.alloc_zeroed(layout) }
|
||||
}
|
||||
|
||||
unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) {
|
||||
// SAFETY: ptr and layout came from this allocator and are forwarded unchanged.
|
||||
unsafe { rustfs_mimalloc::MiMalloc.dealloc(ptr, layout) }
|
||||
unsafe { mimalloc::MiMalloc.dealloc(ptr, layout) }
|
||||
}
|
||||
|
||||
unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 {
|
||||
// SAFETY: ptr and layout came from this allocator and are forwarded unchanged.
|
||||
unsafe { rustfs_mimalloc::MiMalloc.realloc(ptr, layout, new_size) }
|
||||
unsafe { mimalloc::MiMalloc.realloc(ptr, layout, new_size) }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -51,7 +51,7 @@ static GLOBAL: hotpath::CountingAllocator<MiMallocAllocator> = hotpath::Counting
|
||||
|
||||
#[cfg(not(all(feature = "hotpath", feature = "hotpath-alloc")))]
|
||||
#[global_allocator]
|
||||
static GLOBAL: rustfs_mimalloc::MiMalloc = rustfs_mimalloc::MiMalloc;
|
||||
static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;
|
||||
|
||||
fn main() {
|
||||
let _hotpath_guard = hotpath::HotpathGuardBuilder::new("main").build();
|
||||
@@ -71,9 +71,8 @@ mod tests {
|
||||
allocation.extend_from_slice(&[7_u8; 64]);
|
||||
|
||||
assert_eq!(allocation.len(), 64);
|
||||
let heap = rustfs_mimalloc::heap::Heap::main();
|
||||
// SAFETY: the live Vec pointer is valid to inspect for heap ownership.
|
||||
assert!(unsafe { heap.contains(allocation.as_ptr()) });
|
||||
assert!(unsafe { libmimalloc_sys::mi_is_in_heap_region(allocation.as_ptr().cast()) });
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -86,13 +85,12 @@ mod tests {
|
||||
let layout = Layout::from_size_align(32, 8).expect("valid test allocation layout");
|
||||
let grown_layout = Layout::from_size_align(64, 8).expect("valid grown test allocation layout");
|
||||
let allocator = super::MiMallocAllocator;
|
||||
let heap = rustfs_mimalloc::heap::Heap::main();
|
||||
|
||||
// SAFETY: The pointer is checked for null before use and later released
|
||||
// through the same allocator with the corresponding layout.
|
||||
let ptr = unsafe { allocator.alloc_zeroed(layout) };
|
||||
assert!(!ptr.is_null());
|
||||
assert!(unsafe { heap.contains(ptr) });
|
||||
assert!(unsafe { libmimalloc_sys::mi_is_in_heap_region(ptr.cast()) });
|
||||
assert!(unsafe { std::slice::from_raw_parts(ptr, 32).iter().all(|byte| *byte == 0) });
|
||||
|
||||
// SAFETY: `ptr` was allocated by `allocator` with `layout`; on failure
|
||||
@@ -104,7 +102,7 @@ mod tests {
|
||||
panic!("mimalloc realloc failed in allocator smoke test");
|
||||
}
|
||||
|
||||
assert!(unsafe { heap.contains(grown_ptr) });
|
||||
assert!(unsafe { libmimalloc_sys::mi_is_in_heap_region(grown_ptr.cast()) });
|
||||
// SAFETY: `grown_ptr` was reallocated by `allocator` and is released
|
||||
// with the matching grown layout.
|
||||
unsafe { allocator.dealloc(grown_ptr, grown_layout) };
|
||||
|
||||
@@ -17,7 +17,10 @@ use rustfs_io_metrics::{
|
||||
record_cpu_usage, record_memory_usage, record_process_memory_split,
|
||||
};
|
||||
use serde::Serialize;
|
||||
#[cfg(any(test, not(target_os = "windows")))]
|
||||
use serde_json::Value;
|
||||
#[cfg(not(target_os = "windows"))]
|
||||
use std::ffi::CStr;
|
||||
use std::path::Path;
|
||||
use std::sync::{Arc, Mutex, OnceLock};
|
||||
use std::time::Duration;
|
||||
@@ -228,18 +231,7 @@ fn read_cgroup_memory_snapshot() -> Option<CgroupMemorySnapshot> {
|
||||
read_cgroup_v2().or_else(read_cgroup_v1)
|
||||
}
|
||||
|
||||
fn read_allocator_memory_snapshot() -> Option<AllocatorMemorySnapshot> {
|
||||
let json = rustfs_mimalloc::MiMalloc::stats_json();
|
||||
if json.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let observation = parse_mimalloc_stats_json(&json)?;
|
||||
Some(AllocatorMemorySnapshot {
|
||||
backend: crate::allocator_reclaim::allocator_backend(),
|
||||
observation,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(any(test, not(target_os = "windows")))]
|
||||
fn numeric_json_value(value: &Value) -> Option<u64> {
|
||||
match value {
|
||||
Value::Number(number) => number
|
||||
@@ -250,6 +242,7 @@ fn numeric_json_value(value: &Value) -> Option<u64> {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(test, not(target_os = "windows")))]
|
||||
fn numeric_json_field(value: &Value, field: &str) -> Option<u64> {
|
||||
match value {
|
||||
Value::Object(fields) => fields
|
||||
@@ -261,6 +254,7 @@ fn numeric_json_field(value: &Value, field: &str) -> Option<u64> {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(test, not(target_os = "windows")))]
|
||||
fn mimalloc_stat_field(value: &Value, metric: &str, field: &str) -> Option<u64> {
|
||||
match value {
|
||||
Value::Object(fields) => {
|
||||
@@ -277,10 +271,12 @@ fn mimalloc_stat_field(value: &Value, metric: &str, field: &str) -> Option<u64>
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(test, not(target_os = "windows")))]
|
||||
fn mimalloc_stat_current(value: &Value, metric: &str) -> Option<u64> {
|
||||
mimalloc_stat_field(value, metric, "current")
|
||||
}
|
||||
|
||||
#[cfg(any(test, not(target_os = "windows")))]
|
||||
fn mimalloc_stat_sum(value: &Value, metrics: &[&str], field: &str) -> Option<u64> {
|
||||
metrics
|
||||
.iter()
|
||||
@@ -289,6 +285,7 @@ fn mimalloc_stat_sum(value: &Value, metrics: &[&str], field: &str) -> Option<u64
|
||||
.filter(|value| *value > 0)
|
||||
}
|
||||
|
||||
#[cfg(any(test, not(target_os = "windows")))]
|
||||
fn parse_mimalloc_stats_json(stats_json: &str) -> Option<AllocatorMemoryObservation> {
|
||||
let value = serde_json::from_str::<Value>(stats_json).ok()?;
|
||||
let malloc_metrics = ["malloc_normal", "malloc_huge"];
|
||||
@@ -315,6 +312,33 @@ fn parse_mimalloc_stats_json(stats_json: &str) -> Option<AllocatorMemoryObservat
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "windows"))]
|
||||
#[allow(unsafe_code)]
|
||||
fn read_allocator_memory_snapshot() -> Option<AllocatorMemorySnapshot> {
|
||||
// SAFETY: `mi_stats_get_json` returns a null-terminated JSON buffer owned by
|
||||
// mimalloc when called with a null input buffer. The mimalloc API requires
|
||||
// freeing that buffer with `mi_free`; parsing finishes before the buffer is freed.
|
||||
let observation = unsafe {
|
||||
let stats_ptr = libmimalloc_sys::mi_stats_get_json(0, std::ptr::null_mut());
|
||||
if stats_ptr.is_null() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let observation = CStr::from_ptr(stats_ptr).to_str().ok().and_then(parse_mimalloc_stats_json);
|
||||
libmimalloc_sys::mi_free(stats_ptr.cast());
|
||||
observation?
|
||||
};
|
||||
Some(AllocatorMemorySnapshot {
|
||||
backend: crate::allocator_reclaim::allocator_backend(),
|
||||
observation,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
fn read_allocator_memory_snapshot() -> Option<AllocatorMemorySnapshot> {
|
||||
None
|
||||
}
|
||||
|
||||
fn configured_memory_observability_interval_secs() -> u64 {
|
||||
rustfs_utils::get_env_u64(ENV_MEMORY_OBSERVABILITY_INTERVAL_SECS, DEFAULT_MEMORY_OBSERVABILITY_INTERVAL_SECS).max(1)
|
||||
}
|
||||
@@ -542,13 +566,6 @@ mod tests {
|
||||
assert_eq!(parse_mimalloc_stats_json(r#"{ "allocator": "unknown" }"#), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn read_allocator_memory_snapshot_uses_mimalloc_stats_json() {
|
||||
let snapshot = super::read_allocator_memory_snapshot();
|
||||
#[cfg(not(target_os = "windows"))]
|
||||
assert!(snapshot.is_some(), "allocator snapshot should be available on non-Windows");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn memory_observability_snapshot_reports_disabled_when_metrics_are_disabled() {
|
||||
let snapshot = build_memory_observability_status_snapshot(false, 15, false);
|
||||
|
||||
Reference in New Issue
Block a user