gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-07-27 00:38:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

perf(iam): load IAM lists in fixed chunks to avoid O(n^2) startup load (backlog#806) (#4537)

Browse Source 
load_all fetched policies/users/user-policies by passing the FULL remaining
list to load_*_concurrent every iteration and then split_off(32), so each item
was re-fetched once per preceding chunk — O(n^2) redundant loads on startup for
>32 items. Replace the split_off loops with chunks(32) so each item is fetched
exactly once. Behavior-preserving (cache inserts were already idempotent).
This commit is contained in:
Zhengchao An
2026-07-09 02:01:36 +08:00
committed by GitHub
parent 47c1e730c7
commit 16a91c35ec
1 changed files with 17 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/iam/src/store/object.rs
+17 -74
View File
@@ -1130,39 +1130,22 @@ impl Store for ObjectStore {
let mut policy_docs_cache = CacheEntity::new(get_default_policyes()); let mut policy_docs_cache = CacheEntity::new(get_default_policyes());
if let Some(policies_list) = listed_config_items.get(POLICIES_LIST_KEY) { if let Some(policies_list) = listed_config_items.get(POLICIES_LIST_KEY) {
let mut policies_list = policies_list.clone(); // Load in fixed-size chunks so each policy is fetched exactly once.
// The previous split_off loop re-passed the full remaining list to
loop { // load_policy_doc_concurrent every iteration — an O(n^2) redundant
if policies_list.len() < 32 { // load (backlog#806).
let policy_docs = self.load_policy_doc_concurrent(&policies_list, LOAD_ALL_MODE).await?; for chunk in policies_list.chunks(32) {
let policy_docs = self.load_policy_doc_concurrent(chunk, LOAD_ALL_MODE).await?;
for (idx, p) in policy_docs.into_iter().enumerate() {
if p.policy.version.is_empty() {
continue;
}
let policy_name = rustfs_utils::path::dir(&policies_list[idx]);
debug!(policy = %policy_name, "IAM policy loaded");
policy_docs_cache.insert(policy_name, p);
}
break;
}
let policy_docs = self.load_policy_doc_concurrent(&policies_list, LOAD_ALL_MODE).await?;
for (idx, p) in policy_docs.into_iter().enumerate() { for (idx, p) in policy_docs.into_iter().enumerate() {
if p.policy.version.is_empty() { if p.policy.version.is_empty() {
continue; continue;
} }
let policy_name = rustfs_utils::path::dir(&policies_list[idx]); let policy_name = rustfs_utils::path::dir(&chunk[idx]);
debug!(policy = %policy_name, "IAM policy loaded"); debug!(policy = %policy_name, "IAM policy loaded");
policy_docs_cache.insert(policy_name, p); policy_docs_cache.insert(policy_name, p);
} }
policies_list = policies_list.split_off(32);
} }
} }
@@ -1170,41 +1153,20 @@ impl Store for ObjectStore {
// users // users
if let Some(item_name_list) = listed_config_items.get(USERS_LIST_KEY) { if let Some(item_name_list) = listed_config_items.get(USERS_LIST_KEY) {
let mut item_name_list = item_name_list.clone(); // Load in fixed-size chunks so each user is fetched exactly once
// (backlog#806 — the split_off loop was O(n^2)).
loop { for chunk in item_name_list.chunks(32) {
if item_name_list.len() < 32 { let items = self.load_user_concurrent(chunk, UserType::Reg, LOAD_ALL_MODE).await?;
let items = self
.load_user_concurrent(&item_name_list, UserType::Reg, LOAD_ALL_MODE)
.await?;
for (idx, p) in items.into_iter().enumerate() {
if p.credentials.access_key.is_empty() {
continue;
}
let name = rustfs_utils::path::dir(&item_name_list[idx]);
debug!(user = %name, "IAM regular user loaded");
user_items_cache.insert(name, p);
}
break;
}
let items = self
.load_user_concurrent(&item_name_list, UserType::Reg, LOAD_ALL_MODE)
.await?;
for (idx, p) in items.into_iter().enumerate() { for (idx, p) in items.into_iter().enumerate() {
if p.credentials.access_key.is_empty() { if p.credentials.access_key.is_empty() {
continue; continue;
} }
let name = rustfs_utils::path::dir(&item_name_list[idx]); let name = rustfs_utils::path::dir(&chunk[idx]);
debug!(user = %name, "IAM regular user loaded"); debug!(user = %name, "IAM regular user loaded");
user_items_cache.insert(name, p); user_items_cache.insert(name, p);
} }
item_name_list = item_name_list.split_off(32);
} }
} }
@@ -1227,30 +1189,13 @@ impl Store for ObjectStore {
// user policies // user policies
let mut user_policies_cache = None; let mut user_policies_cache = None;
if let Some(item_name_list) = listed_config_items.get(POLICY_DB_USERS_LIST_KEY) { if let Some(item_name_list) = listed_config_items.get(POLICY_DB_USERS_LIST_KEY) {
let mut item_name_list = item_name_list.clone();
let mut items_cache = CacheEntity::default(); let mut items_cache = CacheEntity::default();
loop { // Load in fixed-size chunks so each mapping is fetched exactly once
if item_name_list.len() < 32 { // (backlog#806 — the split_off loop was O(n^2)).
let items = self for chunk in item_name_list.chunks(32) {
.load_mapped_policy_concurrent(&item_name_list, UserType::Reg, false, LOAD_ALL_MODE)
.await?;
for (idx, p) in items.into_iter().enumerate() {
if p.policies.is_empty() {
continue;
}
let name = item_name_list[idx].trim_end_matches(".json").to_owned();
debug!(user = %name, "IAM user policy loaded");
items_cache.insert(name, p);
}
break;
}
let items = self let items = self
.load_mapped_policy_concurrent(&item_name_list, UserType::Reg, false, LOAD_ALL_MODE) .load_mapped_policy_concurrent(chunk, UserType::Reg, false, LOAD_ALL_MODE)
.await?; .await?;
for (idx, p) in items.into_iter().enumerate() { for (idx, p) in items.into_iter().enumerate() {
@@ -1258,12 +1203,10 @@ impl Store for ObjectStore {
continue; continue;
} }
let name = item_name_list[idx].trim_end_matches(".json").to_owned(); let name = chunk[idx].trim_end_matches(".json").to_owned();
debug!(user = %name, "IAM user policy loaded"); debug!(user = %name, "IAM user policy loaded");
items_cache.insert(name, p); items_cache.insert(name, p);
} }
item_name_list = item_name_list.split_off(32);
} }
user_policies_cache = Some(items_cache); user_policies_cache = Some(items_cache);