gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-16 09:58:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ecstore): retry manual ILM job CAS updates (#6012)

Browse Source 
Co-authored-by: heihutu <heihutu@gmail.com>
This commit is contained in:
houseme
2026-08-13 02:40:10 +08:00
committed by GitHub
parent 59d8d93832
commit 398d2d87c8
7 changed files with 735 additions and 221 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rustfs/src/admin/handlers/ilm_transition.rs
+44 -36
View File
@@ -24,10 +24,10 @@ use crate::admin::storage_api::lifecycle::{
claim_manual_transition_scope_admission, delete_manual_transition_scope_admission_if_current,
delete_transition_candidate_for_operator, enqueue_transition_for_existing_objects_scoped,
finalize_missing_transition_transaction_for_operator, inspect_transition_transaction_for_operator,
load_manual_transition_job_record, load_manual_transition_job_record_with_etag, load_manual_transition_scope_admission,
manual_transition_job_lease_expired, manual_transition_queue_snapshot, manual_transition_scope_admission_lease_expired,
persist_manual_transition_job_progress, renew_manual_transition_job_lease, request_manual_transition_job_cancel,
save_manual_transition_job_record, save_manual_transition_job_record_if_current,
load_manual_transition_job_record, load_manual_transition_scope_admission, manual_transition_job_lease_expired,
manual_transition_queue_snapshot, manual_transition_scope_admission_lease_expired,
persist_manual_transition_job_progress_if_owned, renew_manual_transition_job_lease_if_owned,
request_manual_transition_job_cancel, save_manual_transition_job_record, update_manual_transition_job_record,
};
use crate::admin::storage_api::runtime::ECStore;
use crate::auth::{check_key_valid, get_session_token};
@@ -645,31 +645,15 @@ fn json_response<T: Serialize>(response: &T, status: StatusCode) -> S3Result<S3R
Ok(S3Response::with_headers((status, Body::from(body)), headers))
}
async fn update_manual_transition_job_record_cas(
async fn update_manual_transition_job_record_if_owned(
store: Arc<ECStore>,
job_id: Uuid,
mut update: impl FnMut(&mut ManualTransitionJobRecord),
expected_lease_id: Uuid,
mut update: impl FnMut(&mut ManualTransitionJobRecord) -> bool,
) -> S3Result<ManualTransitionJobRecord> {
for _ in 0..4 {
let (mut record, etag) = load_manual_transition_job_record_with_etag(store.clone(), job_id)
.await
.map_err(|err| map_manual_transition_job_load_error(err, job_id))?;
update(&mut record);
match save_manual_transition_job_record_if_current(store.clone(), &record, &etag).await {
Ok(()) => return Ok(record),
Err(StorageError::PreconditionFailed) => continue,
Err(err) => {
return Err(S3Error::with_message(
S3ErrorCode::InternalError,
format!("manual transition job store failed: {err}"),
));
}
}
}
Err(s3_error!(
OperationAborted,
"manual transition job record changed concurrently; retry the request"
))
update_manual_transition_job_record(store, job_id, Some(expected_lease_id), |record| update(record))
.await
.map_err(|err| map_manual_transition_job_load_error(err, job_id))
}
fn manual_transition_durable_cancel_check(store: Arc<ECStore>, job_id: Uuid) -> ManualTransitionCancelCheck {
@@ -707,11 +691,11 @@ fn manual_transition_durable_cancel_check(store: Arc<ECStore>, job_id: Uuid) ->
})
}
fn manual_transition_progress_sink(store: Arc<ECStore>, job_id: Uuid) -> ManualTransitionProgressSink {
fn manual_transition_progress_sink(store: Arc<ECStore>, job_id: Uuid, lease_id: Uuid) -> ManualTransitionProgressSink {
Arc::new(move |report| {
let store = store.clone();
Box::pin(async move {
persist_manual_transition_job_progress(store, job_id, &report, manual_transition_queue_snapshot())
persist_manual_transition_job_progress_if_owned(store, job_id, lease_id, &report, manual_transition_queue_snapshot())
.await
.map(|_| ())
})
@@ -741,9 +725,13 @@ fn release_manual_transition_admission(store: Arc<ECStore>, record: &ManualTrans
async fn finalize_manual_transition_job(
store: Arc<ECStore>,
job_id: Uuid,
lease_id: Uuid,
result: Result<ManualTransitionRunReport, StorageError>,
) -> Option<ManualTransitionJobRecord> {
let updated = update_manual_transition_job_record_cas(store.clone(), job_id, |record| {
let updated = update_manual_transition_job_record_if_owned(store.clone(), job_id, lease_id, |record| {
if record.is_terminal() {
return false;
}
let cancel_requested = record.cancel_requested;
match &result {
Ok(report) => {
@@ -763,10 +751,12 @@ async fn finalize_manual_transition_job(
}
}
}
true
})
.await;
match updated {
Ok(record) => Some(record),
Err(err) if err.code() == &S3ErrorCode::OperationAborted => None,
Err(err) => {
error!(
event = EVENT_ADMIN_ILM_TRANSITION_STATE,
@@ -786,6 +776,7 @@ async fn finalize_manual_transition_job(
fn spawn_manual_transition_job_heartbeat(
store: Arc<ECStore>,
job_id: Uuid,
lease_id: Uuid,
scan_cancel_token: CancellationToken,
shutdown_token: CancellationToken,
) {
@@ -795,7 +786,7 @@ fn spawn_manual_transition_job_heartbeat(
tokio::select! {
_ = shutdown_token.cancelled() => return,
_ = interval.tick() => {
match renew_manual_transition_job_lease(store.clone(), job_id, manual_transition_queue_snapshot()).await {
match renew_manual_transition_job_lease_if_owned(store.clone(), job_id, lease_id, manual_transition_queue_snapshot()).await {
Ok(record) if record.is_terminal() => {
remove_active_manual_transition_job(job_id);
scan_cancel_token.cancel();
@@ -803,6 +794,11 @@ fn spawn_manual_transition_job_heartbeat(
}
Ok(record) if record.cancel_requested => scan_cancel_token.cancel(),
Ok(_) => {}
Err(StorageError::PreconditionFailed) => {
remove_active_manual_transition_job(job_id);
scan_cancel_token.cancel();
return;
}
Err(err) => {
warn!(
event = EVENT_ADMIN_ILM_TRANSITION_STATE,
@@ -840,15 +836,23 @@ async fn start_manual_transition_job(
match claim_manual_transition_scope_admission(store.clone(), &ManualTransitionScopeAdmission::from_job(&record)).await {
Ok(ManualTransitionScopeAdmissionClaim::Claimed) => {}
Ok(ManualTransitionScopeAdmissionClaim::Conflict(active)) => {
let _ = update_manual_transition_job_record_cas(store.clone(), job_id, |record| {
let _ = update_manual_transition_job_record_if_owned(store.clone(), job_id, record.lease_id, |record| {
if record.is_terminal() {
return false;
}
record.fail("manual transition admission conflict");
true
})
.await;
return Ok(StartManualTransitionJobResult::Conflict(manual_transition_job_conflict_response(*active)));
}
Err(err) => {
let _ = update_manual_transition_job_record_cas(store.clone(), job_id, |record| {
let _ = update_manual_transition_job_record_if_owned(store.clone(), job_id, record.lease_id, |record| {
if record.is_terminal() {
return false;
}
record.fail(format!("manual transition admission failed: {err}"));
true
})
.await;
return Err(S3Error::with_message(
@@ -862,21 +866,22 @@ async fn start_manual_transition_job(
let heartbeat_shutdown_token = CancellationToken::new();
insert_active_manual_transition_job(job_id, scan_cancel_token.clone());
let mut run_options = options;
let lease_id = record.lease_id;
run_options.job_id = Some(job_id);
run_options.cancel_token = Some(scan_cancel_token.clone());
run_options.cancel_check = Some(manual_transition_durable_cancel_check(store.clone(), job_id));
run_options.progress_sink = Some(manual_transition_progress_sink(store.clone(), job_id));
run_options.progress_sink = Some(manual_transition_progress_sink(store.clone(), job_id, lease_id));
let run_store = store.clone();
let job_scan_cancel_token = scan_cancel_token.clone();
let job_heartbeat_shutdown_token = heartbeat_shutdown_token.clone();
spawn_manual_transition_job_heartbeat(store, job_id, scan_cancel_token, heartbeat_shutdown_token);
spawn_manual_transition_job_heartbeat(store, job_id, lease_id, scan_cancel_token, heartbeat_shutdown_token);
tokio::spawn(async move {
#[cfg(feature = "e2e-test-hooks")]
if std::env::var_os(E2E_MANUAL_TRANSITION_CANCEL_BARRIER_ENV).is_some() {
job_scan_cancel_token.cancelled().await;
}
let result = enqueue_transition_for_existing_objects_scoped(run_store.clone(), &bucket, run_options).await;
if let Some(final_record) = finalize_manual_transition_job(run_store.clone(), job_id, result).await
if let Some(final_record) = finalize_manual_transition_job(run_store.clone(), job_id, lease_id, result).await
&& final_record.is_terminal()
{
release_manual_transition_admission(run_store, &final_record);
@@ -988,9 +993,12 @@ impl Operation for ManualTransitionJobStatusHandler {
&& !manual_transition_scope_admission_lease_expired(&admission)
});
if !local_active && !leased_elsewhere && manual_transition_job_lease_expired(&record) {
record = update_manual_transition_job_record_cas(store.clone(), job_id, |record| {
record = update_manual_transition_job_record_if_owned(store.clone(), job_id, record.lease_id, |record| {
if record.state == ManualTransitionJobState::Running && manual_transition_job_lease_expired(record) {
record.mark_unknown_if_unowned();
true
} else {
false
}
})
.await?;
rustfs/src/admin/storage_api.rs
+4 -4
View File
@@ -197,10 +197,10 @@ pub(crate) mod lifecycle {
pub(crate) use super::ecstore_bucket::lifecycle::manual_transition_job::{
ManualTransitionJobRecord, ManualTransitionJobState, ManualTransitionScopeAdmission, ManualTransitionScopeAdmissionClaim,
claim_manual_transition_scope_admission, delete_manual_transition_scope_admission_if_current,
load_manual_transition_job_record, load_manual_transition_job_record_with_etag, load_manual_transition_scope_admission,
manual_transition_job_lease_expired, manual_transition_scope_admission_lease_expired,
persist_manual_transition_job_progress, renew_manual_transition_job_lease, request_manual_transition_job_cancel,
save_manual_transition_job_record, save_manual_transition_job_record_if_current,
load_manual_transition_job_record, load_manual_transition_scope_admission, manual_transition_job_lease_expired,
manual_transition_scope_admission_lease_expired, persist_manual_transition_job_progress_if_owned,
renew_manual_transition_job_lease_if_owned, request_manual_transition_job_cancel, save_manual_transition_job_record,
update_manual_transition_job_record,
};
pub(crate) type ManualTransitionCancelCheck =
super::ecstore_bucket::lifecycle::bucket_lifecycle_ops::ManualTransitionCancelCheck;