gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-08 22:33:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ilm): replace whole-copy-back restore lock with accept-path CAS (#4956)

Browse Source 
fix(ilm): serialize RestoreObject accepts with a CAS guard, not the whole copy-back

Implements the backlog#1304 decision: replace #4877's object write lock
held across the entire tier copy-back with an atomic compare-and-set on
the restore ongoing flag.

- ecstore: add ECStore::acquire_restore_accept_guard + RestoreAcceptGuard
  (opaque, purpose-scoped object write lock with an is_lock_lost fence);
  handle_restore_transitioned_object no longer locks the copy-back, so
  HEAD/get_object_info stay non-blocking during a restore and the inner
  put_object/complete_multipart_upload commit locks no longer self-deadlock.
- API: execute_restore_object holds the guard across the restore-status
  read-check-write (no_lock inside the scope), drops it before spawning
  the copy-back; SELECT-type restores keep the plain read-locked path;
  the copy-back pins the resolved version so a concurrent PUT cannot
  strand the flagged version at ongoing=true; concurrent restores are
  rejected with 409 RestoreAlreadyInProgress (was a retryable 500) and
  guard contention maps to 503 SlowDown.
- tests: drop the #4877 entry-blocking unit test (semantics deliberately
  reversed), pin accept-guard mutual exclusion, update the ilm-8 restore
  integration test to the final semantics, and add a concurrent
  double-POST test asserting exactly one acceptance and one tier GET.
This commit is contained in:
Zhengchao An
2026-07-17 17:56:45 +08:00
committed by GitHub
parent a9e3613cfd
commit 701c3eee5b
4 changed files with 246 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rustfs/src/app/object_usecase.rs
+56 -8
View File
@@ -6369,10 +6369,36 @@ impl DefaultObjectUsecase {
};
let version_id_str = version_id.clone().unwrap_or_default();
let opts = post_restore_opts(&version_id_str, &bucket, &object)
let mut opts = post_restore_opts(&version_id_str, &bucket, &object)
.await
.map_err(|_| S3Error::with_message(S3ErrorCode::Custom("ErrPostRestoreOpts".into()), "restore object failed."))?;
// SELECT-type restores skip both the ongoing check and the metadata
// write below, so the accept guard would protect nothing for them —
// they keep the plain (read-locked) accept path.
let is_select = rreq.type_.as_ref().is_some_and(|t| t.as_str() == "SELECT");
// Hold the restore-accept guard across the restore-status read, the
// ongoing/already-restored decision, and the metadata write below, so
// two concurrent (non-SELECT) POST ?restore cannot both observe
// ongoing=false and both start a copy-back (backlog#1304). Reads and
// writes inside this scope run with no_lock; the guard is dropped
// before the copy-back is spawned so it never blocks readers.
// Contention on the accept guard (e.g. a concurrent accept or an
// in-flight commit on the same object) is transient — answer 503
// SlowDown so SDK clients back off and retry instead of treating it
// as a hard failure.
let accept_guard = if is_select {
None
} else {
let guard = store
.acquire_restore_accept_guard(&bucket, &object)
.await
.map_err(|_| S3Error::with_message(S3ErrorCode::SlowDown, "restore object failed."))?;
opts.no_lock = true;
Some(guard)
};
let mut obj_info = store
.get_object_info(&bucket, &object, &opts)
.await
@@ -6394,11 +6420,13 @@ impl DefaultObjectUsecase {
));
}
// Check if restore is already in progress
if obj_info.restore_ongoing && (rreq.type_.as_ref().is_none_or(|t| t.as_str() != "SELECT")) {
// Check if restore is already in progress. AWS answers this with
// 409 RestoreAlreadyInProgress; a Custom code would serialize as a
// retryable 500 and make SDK clients retry the conflict (backlog#1304).
if obj_info.restore_ongoing && !is_select {
return Err(S3Error::with_message(
S3ErrorCode::Custom("ErrObjectRestoreAlreadyInProgress".into()),
"restore object failed.",
S3ErrorCode::RestoreAlreadyInProgress,
"Object restore is already in progress.",
));
}
@@ -6417,7 +6445,7 @@ impl DefaultObjectUsecase {
let event_object_info = obj_info.clone();
let obj_info_ = obj_info.clone();
if rreq.type_.as_ref().is_none_or(|t| t.as_str() != "SELECT") {
if !is_select {
obj_info.metadata_only = true;
metadata.insert(AMZ_RESTORE_EXPIRY_DAYS.to_string(), rreq.days.unwrap_or(1).to_string());
let request_date = OffsetDateTime::now_utc().format(&Rfc3339).map_err(|e| {
@@ -6445,6 +6473,14 @@ impl DefaultObjectUsecase {
}
obj_info.user_defined = Arc::new(metadata);
// Fence the compare-and-set write: if the accept guard was lost
// (lock-service degradation), another node may have concurrently
// accepted this restore — back off instead of committing a second
// ongoing flag and double-starting the copy-back.
if accept_guard.as_ref().is_some_and(|g| g.is_lock_lost()) {
return Err(S3Error::with_message(S3ErrorCode::SlowDown, "restore object failed."));
}
store
.clone()
.copy_object(
@@ -6455,11 +6491,14 @@ impl DefaultObjectUsecase {
&mut obj_info,
&ObjectOptions {
version_id: obj_info_.version_id.map(|v| v.to_string()),
// Inside the accept-guard critical section (see above).
no_lock: true,
..Default::default()
},
&ObjectOptions {
version_id: obj_info_.version_id.map(|v| v.to_string()),
mod_time: obj_info_.mod_time,
no_lock: true,
..Default::default()
},
)
@@ -6482,6 +6521,10 @@ impl DefaultObjectUsecase {
}
}
// The accept decision is committed; release the object write lock so
// the background copy-back and concurrent reads are never blocked on it.
drop(accept_guard);
// Handle output location for SELECT requests
if let Some(output_location) = &rreq.output_location
&& let Some(s3) = &output_location.s3
@@ -6493,12 +6536,17 @@ impl DefaultObjectUsecase {
}
}
// Spawn restoration task in the background
// Spawn restoration task in the background. Pin the copy-back to the
// version the accept resolved and flagged: with a versionless request
// on a versioned bucket, a PUT landing between the accept and the
// copy-back would otherwise re-resolve "latest" to the new version,
// fail (not transitioned), and strand the flagged version at
// ongoing=true forever (backlog#1304).
let store_clone = store.clone();
let bucket_clone = bucket.clone();
let object_clone = object.clone();
let rreq_clone = rreq.clone();
let version_id_clone = version_id.clone();
let version_id_clone = obj_info_.version_id.map(|v| v.to_string());
spawn_traced(async move {
let opts = ObjectOptions {