mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-26 08:18:18 +00:00
chore: add delete-objects lock batch diagnostics (#3218)
* chore(ecstore): add missing-shard first-scene diagnostics Log rename_data quorum context behind RUSTFS_ISSUE3031_DIAG_ENABLE so partial-disk success can be correlated with later missing shard reads. Also log put_object commit success and tmp cleanup boundaries to capture when successful quorum writes are followed by tmp_dir cleanup. * chore(ecstore): add delete-objects lock batch diagnostics Log DeleteObjects batch-lock request size, distributed lock quorum summary, and failed object keys behind RUSTFS_ISSUE3031_DIAG_ENABLE. Include a Chinese operator checklist under docs/ that explains which logs to capture and how to interpret DeleteObjects lock_batch timeout incidents. * chore: remove lock batch checklist from repo Keep docs/issue-658-deleteobjects-lock-batch-checklist-zh.md only in the local checkout and drop it from the PR history.
This commit is contained in:
@@ -1375,6 +1375,33 @@ impl SetDisks {
|
||||
}
|
||||
}
|
||||
|
||||
if issue3031_diag_enabled() {
|
||||
let succeeded_count = resolution_by_object
|
||||
.iter()
|
||||
.filter(|resolution| matches!(resolution, ObjectLockResolution::Succeeded))
|
||||
.count();
|
||||
let failed_count = resolution_by_object
|
||||
.iter()
|
||||
.filter(|resolution| matches!(resolution, ObjectLockResolution::Failed))
|
||||
.count();
|
||||
let pending_count = resolution_by_object
|
||||
.iter()
|
||||
.filter(|resolution| matches!(resolution, ObjectLockResolution::Pending))
|
||||
.count();
|
||||
warn!(
|
||||
target: "rustfs_ecstore::set_disk",
|
||||
request_count = requests.len(),
|
||||
locker_count = self.lockers.len(),
|
||||
write_quorum,
|
||||
succeeded_count,
|
||||
failed_count,
|
||||
pending_count,
|
||||
pending_clients,
|
||||
errors_by_object = ?errors_by_object,
|
||||
"issue3031_delete_objects_dist_batch_lock_summary"
|
||||
);
|
||||
}
|
||||
|
||||
if !pending.is_empty() {
|
||||
let cleanup_requests = requests.clone();
|
||||
let lockers = self.lockers.clone();
|
||||
@@ -1809,6 +1836,7 @@ impl ObjectOperations for SetDisks {
|
||||
batch = batch.add_write_lock(ObjectKey::new(bucket, dobj.object_name.clone()));
|
||||
}
|
||||
}
|
||||
let unique_lock_count = batch.requests.len();
|
||||
|
||||
let mut failed_map = HashMap::new();
|
||||
let mut _local_batch_guards: Vec<FastLockGuard> = Vec::with_capacity(batch.requests.len());
|
||||
@@ -1832,6 +1860,24 @@ impl ObjectOperations for SetDisks {
|
||||
}
|
||||
}
|
||||
|
||||
if issue3031_diag_enabled() {
|
||||
let failed_lock_count = failed_map.len();
|
||||
let locked_object_count = locked_objects.len();
|
||||
let dist_lock_id_count = dist_batch_lock_ids.iter().map(Vec::len).sum::<usize>();
|
||||
warn!(
|
||||
target: "rustfs_ecstore::set_disk",
|
||||
bucket = %bucket,
|
||||
requested_object_count = objects.len(),
|
||||
unique_lock_count,
|
||||
locked_object_count,
|
||||
failed_lock_count,
|
||||
dist_erasure,
|
||||
dist_lock_id_count,
|
||||
failed_objects = ?failed_map.keys().collect::<Vec<_>>(),
|
||||
"issue3031_delete_objects_lock_batch_context"
|
||||
);
|
||||
}
|
||||
|
||||
// Mark failures for objects that could not be locked
|
||||
for (i, dobj) in objects.iter().enumerate() {
|
||||
if let Some(err) = failed_map.get(&(bucket.to_string(), dobj.object_name.clone())) {
|
||||
|
||||
Reference in New Issue
Block a user