gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-13 00:26:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: preserve request ids across async recovery logs (#3451)

Browse Source 
* feat(obs): promote request ids in structured logs

* refactor(tracing): propagate spans into request tasks

* test(ecstore): baseline recovery monitor log chains

* fix(replication): reduce startup resync log noise

* chore(docs): stop tracking local recovery baseline

* chore(obs): polish request id logging cleanup
This commit is contained in:
houseme
2026-06-14 23:16:04 +08:00
committed by GitHub
parent 9499391370
commit efaf07d323
21 changed files with 1194 additions and 360 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/set_disk.rs
+48 -45
View File
@@ -127,7 +127,7 @@ use tokio::{
};
use tokio_util::sync::CancellationToken;
use tracing::error;
use tracing::{debug, info, warn};
use tracing::{Instrument, debug, info, warn};
use uuid::Uuid;
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
@@ -1491,55 +1491,58 @@ impl SetDisks {
if !pending.is_empty() {
let cleanup_requests = requests.clone();
let lockers = self.lockers.clone();
let handle = tokio::spawn(async move {
let mut late_lock_ids_by_client = vec![Vec::new(); lockers.len()];
let mut pending = pending;
while let Some(join_result) = pending.join_next().await {
match join_result {
Ok((client_idx, Ok(responses))) => {
for (req_idx, request) in cleanup_requests.iter().enumerate() {
if let Some(response) = responses.get(req_idx)
&& response.success
{
let lock_id = response
.lock_info
.as_ref()
.map(|lock_info| lock_info.id.clone())
.unwrap_or_else(|| request.lock_id.clone());
if let Some(client_locks) = late_lock_ids_by_client.get_mut(client_idx) {
client_locks.push(lock_id);
let handle = tokio::spawn(
async move {
let mut late_lock_ids_by_client = vec![Vec::new(); lockers.len()];
let mut pending = pending;
while let Some(join_result) = pending.join_next().await {
match join_result {
Ok((client_idx, Ok(responses))) => {
for (req_idx, request) in cleanup_requests.iter().enumerate() {
if let Some(response) = responses.get(req_idx)
&& response.success
{
let lock_id = response
.lock_info
.as_ref()
.map(|lock_info| lock_info.id.clone())
.unwrap_or_else(|| request.lock_id.clone());
if let Some(client_locks) = late_lock_ids_by_client.get_mut(client_idx) {
client_locks.push(lock_id);
}
}
}
}
}
Ok((_client_idx, Err(err))) => {
tracing::warn!("late distributed delete lock batch request failed: {}", err);
}
Err(err) => {
tracing::warn!("late distributed delete lock batch task join failed: {}", err);
}
}
}
join_all(lockers.iter().cloned().enumerate().filter_map(|(client_idx, client)| {
let lock_ids = late_lock_ids_by_client.get(client_idx).cloned().unwrap_or_default();
if lock_ids.is_empty() {
None
} else {
Some(async move {
if let Err(err) = client.release_locks_batch(&lock_ids).await {
tracing::warn!(
client_idx,
lock_count = lock_ids.len(),
"failed to cleanup late distributed delete locks in batch: {}",
err
);
Ok((_client_idx, Err(err))) => {
tracing::warn!("late distributed delete lock batch request failed: {}", err);
}
})
Err(err) => {
tracing::warn!("late distributed delete lock batch task join failed: {}", err);
}
}
}
}))
.await;
});
join_all(lockers.iter().cloned().enumerate().filter_map(|(client_idx, client)| {
let lock_ids = late_lock_ids_by_client.get(client_idx).cloned().unwrap_or_default();
if lock_ids.is_empty() {
None
} else {
Some(async move {
if let Err(err) = client.release_locks_batch(&lock_ids).await {
tracing::warn!(
client_idx,
lock_count = lock_ids.len(),
"failed to cleanup late distributed delete locks in batch: {}",
err
);
}
})
}
}))
.await;
}
.instrument(tracing::Span::current()),
);
drop(handle);
}