chore: add delete-objects lock batch diagnostics (#3218)

* chore(ecstore): add missing-shard first-scene diagnostics

Log rename_data quorum context behind RUSTFS_ISSUE3031_DIAG_ENABLE so partial-disk success can be correlated with later missing shard reads.

Also log put_object commit success and tmp cleanup boundaries to capture when successful quorum writes are followed by tmp_dir cleanup.

* chore(ecstore): add delete-objects lock batch diagnostics

Log DeleteObjects batch-lock request size, distributed lock quorum summary, and failed object keys behind RUSTFS_ISSUE3031_DIAG_ENABLE.

Include a Chinese operator checklist under docs/ that explains which logs to capture and how to interpret DeleteObjects lock_batch timeout incidents.

* chore: remove lock batch checklist from repo

Keep docs/issue-658-deleteobjects-lock-batch-checklist-zh.md only in the local checkout and drop it from the PR history.
This commit is contained in:
houseme
2026-06-05 01:00:58 +08:00
committed by GitHub
parent 3bd89944c2
commit 0a74629c48
+46
View File
@@ -1375,6 +1375,33 @@ impl SetDisks {
}
}
if issue3031_diag_enabled() {
let succeeded_count = resolution_by_object
.iter()
.filter(|resolution| matches!(resolution, ObjectLockResolution::Succeeded))
.count();
let failed_count = resolution_by_object
.iter()
.filter(|resolution| matches!(resolution, ObjectLockResolution::Failed))
.count();
let pending_count = resolution_by_object
.iter()
.filter(|resolution| matches!(resolution, ObjectLockResolution::Pending))
.count();
warn!(
target: "rustfs_ecstore::set_disk",
request_count = requests.len(),
locker_count = self.lockers.len(),
write_quorum,
succeeded_count,
failed_count,
pending_count,
pending_clients,
errors_by_object = ?errors_by_object,
"issue3031_delete_objects_dist_batch_lock_summary"
);
}
if !pending.is_empty() {
let cleanup_requests = requests.clone();
let lockers = self.lockers.clone();
@@ -1809,6 +1836,7 @@ impl ObjectOperations for SetDisks {
batch = batch.add_write_lock(ObjectKey::new(bucket, dobj.object_name.clone()));
}
}
let unique_lock_count = batch.requests.len();
let mut failed_map = HashMap::new();
let mut _local_batch_guards: Vec<FastLockGuard> = Vec::with_capacity(batch.requests.len());
@@ -1832,6 +1860,24 @@ impl ObjectOperations for SetDisks {
}
}
if issue3031_diag_enabled() {
let failed_lock_count = failed_map.len();
let locked_object_count = locked_objects.len();
let dist_lock_id_count = dist_batch_lock_ids.iter().map(Vec::len).sum::<usize>();
warn!(
target: "rustfs_ecstore::set_disk",
bucket = %bucket,
requested_object_count = objects.len(),
unique_lock_count,
locked_object_count,
failed_lock_count,
dist_erasure,
dist_lock_id_count,
failed_objects = ?failed_map.keys().collect::<Vec<_>>(),
"issue3031_delete_objects_lock_batch_context"
);
}
// Mark failures for objects that could not be locked
for (i, dobj) in objects.iter().enumerate() {
if let Some(err) = failed_map.get(&(bucket.to_string(), dobj.object_name.clone())) {