mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-27 15:37:02 +00:00
chore: add delete-objects lock batch diagnostics (#3218)
* chore(ecstore): add missing-shard first-scene diagnostics Log rename_data quorum context behind RUSTFS_ISSUE3031_DIAG_ENABLE so partial-disk success can be correlated with later missing shard reads. Also log put_object commit success and tmp cleanup boundaries to capture when successful quorum writes are followed by tmp_dir cleanup. * chore(ecstore): add delete-objects lock batch diagnostics Log DeleteObjects batch-lock request size, distributed lock quorum summary, and failed object keys behind RUSTFS_ISSUE3031_DIAG_ENABLE. Include a Chinese operator checklist under docs/ that explains which logs to capture and how to interpret DeleteObjects lock_batch timeout incidents. * chore: remove lock batch checklist from repo Keep docs/issue-658-deleteobjects-lock-batch-checklist-zh.md only in the local checkout and drop it from the PR history.
This commit is contained in:
@@ -1375,6 +1375,33 @@ impl SetDisks {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if issue3031_diag_enabled() {
|
||||||
|
let succeeded_count = resolution_by_object
|
||||||
|
.iter()
|
||||||
|
.filter(|resolution| matches!(resolution, ObjectLockResolution::Succeeded))
|
||||||
|
.count();
|
||||||
|
let failed_count = resolution_by_object
|
||||||
|
.iter()
|
||||||
|
.filter(|resolution| matches!(resolution, ObjectLockResolution::Failed))
|
||||||
|
.count();
|
||||||
|
let pending_count = resolution_by_object
|
||||||
|
.iter()
|
||||||
|
.filter(|resolution| matches!(resolution, ObjectLockResolution::Pending))
|
||||||
|
.count();
|
||||||
|
warn!(
|
||||||
|
target: "rustfs_ecstore::set_disk",
|
||||||
|
request_count = requests.len(),
|
||||||
|
locker_count = self.lockers.len(),
|
||||||
|
write_quorum,
|
||||||
|
succeeded_count,
|
||||||
|
failed_count,
|
||||||
|
pending_count,
|
||||||
|
pending_clients,
|
||||||
|
errors_by_object = ?errors_by_object,
|
||||||
|
"issue3031_delete_objects_dist_batch_lock_summary"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
if !pending.is_empty() {
|
if !pending.is_empty() {
|
||||||
let cleanup_requests = requests.clone();
|
let cleanup_requests = requests.clone();
|
||||||
let lockers = self.lockers.clone();
|
let lockers = self.lockers.clone();
|
||||||
@@ -1809,6 +1836,7 @@ impl ObjectOperations for SetDisks {
|
|||||||
batch = batch.add_write_lock(ObjectKey::new(bucket, dobj.object_name.clone()));
|
batch = batch.add_write_lock(ObjectKey::new(bucket, dobj.object_name.clone()));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
let unique_lock_count = batch.requests.len();
|
||||||
|
|
||||||
let mut failed_map = HashMap::new();
|
let mut failed_map = HashMap::new();
|
||||||
let mut _local_batch_guards: Vec<FastLockGuard> = Vec::with_capacity(batch.requests.len());
|
let mut _local_batch_guards: Vec<FastLockGuard> = Vec::with_capacity(batch.requests.len());
|
||||||
@@ -1832,6 +1860,24 @@ impl ObjectOperations for SetDisks {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if issue3031_diag_enabled() {
|
||||||
|
let failed_lock_count = failed_map.len();
|
||||||
|
let locked_object_count = locked_objects.len();
|
||||||
|
let dist_lock_id_count = dist_batch_lock_ids.iter().map(Vec::len).sum::<usize>();
|
||||||
|
warn!(
|
||||||
|
target: "rustfs_ecstore::set_disk",
|
||||||
|
bucket = %bucket,
|
||||||
|
requested_object_count = objects.len(),
|
||||||
|
unique_lock_count,
|
||||||
|
locked_object_count,
|
||||||
|
failed_lock_count,
|
||||||
|
dist_erasure,
|
||||||
|
dist_lock_id_count,
|
||||||
|
failed_objects = ?failed_map.keys().collect::<Vec<_>>(),
|
||||||
|
"issue3031_delete_objects_lock_batch_context"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
// Mark failures for objects that could not be locked
|
// Mark failures for objects that could not be locked
|
||||||
for (i, dobj) in objects.iter().enumerate() {
|
for (i, dobj) in objects.iter().enumerate() {
|
||||||
if let Some(err) = failed_map.get(&(bucket.to_string(), dobj.object_name.clone())) {
|
if let Some(err) = failed_map.get(&(bucket.to_string(), dobj.object_name.clone())) {
|
||||||
|
|||||||
Reference in New Issue
Block a user