gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-13 08:36:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(readiness): gate on lock quorum health (#3100)

Browse Source 
* fix(readiness): gate on lock quorum health

Include distributed lock quorum in runtime readiness decisions and expose the signal in health responses.

Map namespace lock quorum failures to ServiceUnavailable instead of InternalError so clients can safely retry while keeping quorum enforcement unchanged.

* fix(ecstore): restore namespace lock error decoding

Add the missing from_u32 arm for NamespaceLockQuorumUnavailable and cover the numeric roundtrip in error tests.
This commit is contained in:
houseme
2026-05-28 01:12:52 +08:00
committed by GitHub
parent 4648de9e62
commit 5dfc1b5f07
9 changed files with 502 additions and 113 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/error.rs
+45
View File
@@ -160,6 +160,14 @@ pub enum StorageError {
NotModified,
#[error("Invalid range specified: {0}")]
InvalidRangeSpec(String),
#[error("Namespace lock quorum unavailable for {mode} lock on {bucket}/{object}: required {required}, achieved {achieved}")]
NamespaceLockQuorumUnavailable {
mode: &'static str,
bucket: String,
object: String,
required: usize,
achieved: usize,
},
// ── Generic ──────────────────────────────────────────────────────
#[error("Unexpected error")]
@@ -204,6 +212,7 @@ impl StorageError {
| StorageError::ErasureWriteQuorum
| StorageError::InsufficientReadQuorum(_, _)
| StorageError::InsufficientWriteQuorum(_, _)
| StorageError::NamespaceLockQuorumUnavailable { .. }
)
}
}
@@ -433,6 +442,19 @@ impl Clone for StorageError {
StorageError::NotModified => StorageError::NotModified,
StorageError::InvalidPartNumber(a) => StorageError::InvalidPartNumber(*a),
StorageError::InvalidRangeSpec(a) => StorageError::InvalidRangeSpec(a.clone()),
StorageError::NamespaceLockQuorumUnavailable {
mode,
bucket,
object,
required,
achieved,
} => StorageError::NamespaceLockQuorumUnavailable {
mode,
bucket: bucket.clone(),
object: object.clone(),
required: *required,
achieved: *achieved,
},
}
}
}
@@ -505,6 +527,7 @@ impl StorageError {
StorageError::InvalidRangeSpec(_) => 0x3D,
StorageError::NotModified => 0x3E,
StorageError::InvalidPartNumber(_) => 0x3F,
StorageError::NamespaceLockQuorumUnavailable { .. } => 0x42,
}
}
@@ -579,6 +602,13 @@ impl StorageError {
0x3D => Some(StorageError::InvalidRangeSpec(Default::default())),
0x3E => Some(StorageError::NotModified),
0x3F => Some(StorageError::InvalidPartNumber(Default::default())),
0x42 => Some(StorageError::NamespaceLockQuorumUnavailable {
mode: "write",
bucket: Default::default(),
object: Default::default(),
required: Default::default(),
achieved: Default::default(),
}),
_ => None,
}
}
@@ -984,6 +1014,17 @@ mod tests {
assert_eq!(StorageError::DecommissionAlreadyRunning.to_u32(), 0x30);
assert_eq!(StorageError::RebalanceAlreadyRunning.to_u32(), 0x40);
assert_eq!(StorageError::OperationCanceled.to_u32(), 0x41);
assert_eq!(
StorageError::NamespaceLockQuorumUnavailable {
mode: "write",
bucket: "bucket".into(),
object: "object".into(),
required: 3,
achieved: 2,
}
.to_u32(),
0x42
);
}
#[test]
@@ -998,6 +1039,10 @@ mod tests {
assert!(matches!(StorageError::from_u32(0x30), Some(StorageError::DecommissionAlreadyRunning)));
assert!(matches!(StorageError::from_u32(0x40), Some(StorageError::RebalanceAlreadyRunning)));
assert!(matches!(StorageError::from_u32(0x41), Some(StorageError::OperationCanceled)));
assert!(matches!(
StorageError::from_u32(0x42),
Some(StorageError::NamespaceLockQuorumUnavailable { .. })
));
// Test invalid code returns None
assert!(StorageError::from_u32(0xFF).is_none());
crates/ecstore/src/set_disk.rs
+42 -72
View File
@@ -622,12 +622,7 @@ impl ObjectIO for SetDisks {
.await?
.get_read_lock(get_lock_acquire_timeout())
.await
.map_err(|e| {
Error::other(format!(
"Failed to acquire read lock: {}",
self.format_lock_error_from_error(bucket, object, "read", &e)
))
})?;
.map_err(|e| self.map_namespace_lock_error(bucket, object, "read", e))?;
// Record lock acquisition for deadlock detection
let _lock_id = record_lock_acquire(bucket, object, "read");
@@ -756,12 +751,12 @@ impl ObjectIO for SetDisks {
if let Some(http_preconditions) = opts.http_preconditions.clone() {
if !opts.no_lock {
let ns_lock = self.new_ns_lock(bucket, object).await?;
object_lock_guard = Some(ns_lock.get_write_lock(get_lock_acquire_timeout()).await.map_err(|e| {
StorageError::other(format!(
"Failed to acquire write lock: {}",
self.format_lock_error_from_error(bucket, object, "write", &e)
))
})?);
object_lock_guard = Some(
ns_lock
.get_write_lock(get_lock_acquire_timeout())
.await
.map_err(|e| self.map_namespace_lock_error(bucket, object, "write", e))?,
);
}
if let Some(err) = self.check_write_precondition(bucket, object, opts).await {
@@ -975,12 +970,12 @@ impl ObjectIO for SetDisks {
if !opts.no_lock && object_lock_guard.is_none() {
let ns_lock = self.new_ns_lock(bucket, object).await?;
object_lock_guard = Some(ns_lock.get_write_lock(get_lock_acquire_timeout()).await.map_err(|e| {
StorageError::other(format!(
"Failed to acquire write lock: {}",
self.format_lock_error_from_error(bucket, object, "write", &e)
))
})?);
object_lock_guard = Some(
ns_lock
.get_write_lock(get_lock_acquire_timeout())
.await
.map_err(|e| self.map_namespace_lock_error(bucket, object, "write", e))?,
);
}
let (online_disks, _, op_old_dir) = Self::rename_data(
@@ -1398,12 +1393,7 @@ impl ObjectOperations for SetDisks {
.await?
.get_write_lock(get_lock_acquire_timeout())
.await
.map_err(|e| {
Error::other(format!(
"Failed to acquire write lock: {}",
self.format_lock_error_from_error(src_bucket, src_object, "write", &e)
))
})?;
.map_err(|e| self.map_namespace_lock_error(src_bucket, src_object, "write", e))?;
let disks = self.get_disks_internal().await;
@@ -1785,12 +1775,7 @@ impl ObjectOperations for SetDisks {
.await?
.get_write_lock(get_lock_acquire_timeout())
.await
.map_err(|e| {
Error::other(format!(
"Failed to acquire write lock: {}",
self.format_lock_error_from_error(bucket, object, "write", &e)
))
})?,
.map_err(|e| self.map_namespace_lock_error(bucket, object, "write", e))?,
)
} else {
None
@@ -1931,12 +1916,7 @@ impl ObjectOperations for SetDisks {
.await?
.get_read_lock(get_lock_acquire_timeout())
.await
.map_err(|e| {
Error::other(format!(
"Failed to acquire read lock: {}",
self.format_lock_error_from_error(bucket, object, "read", &e)
))
})?,
.map_err(|e| self.map_namespace_lock_error(bucket, object, "read", e))?,
)
} else {
None
@@ -1990,12 +1970,7 @@ impl ObjectOperations for SetDisks {
.await?
.get_write_lock(get_lock_acquire_timeout())
.await
.map_err(|e| {
Error::other(format!(
"Failed to acquire write lock: {}",
self.format_lock_error_from_error(bucket, object, "write", &e)
))
})?,
.map_err(|e| self.map_namespace_lock_error(bucket, object, "write", e))?,
)
} else {
None
@@ -2527,12 +2502,7 @@ impl SetDisks {
.await?
.get_write_lock(get_lock_acquire_timeout())
.await
.map_err(|e| {
Error::other(format!(
"Failed to acquire write lock: {}",
self.format_lock_error_from_error(bucket, object, "write", &e)
))
})?,
.map_err(|e| self.map_namespace_lock_error(bucket, object, "write", e))?,
)
} else {
None
@@ -3067,12 +3037,12 @@ impl MultipartOperations for SetDisks {
if let Some(http_preconditions) = opts.http_preconditions.clone() {
let object_lock_guard = if !opts.no_lock {
let ns_lock = self.new_ns_lock(bucket, object).await?;
Some(ns_lock.get_write_lock(get_lock_acquire_timeout()).await.map_err(|e| {
StorageError::other(format!(
"Failed to acquire write lock: {}",
self.format_lock_error_from_error(bucket, object, "write", &e)
))
})?)
Some(
ns_lock
.get_write_lock(get_lock_acquire_timeout())
.await
.map_err(|e| self.map_namespace_lock_error(bucket, object, "write", e))?,
)
} else {
None
};
@@ -3249,12 +3219,12 @@ impl MultipartOperations for SetDisks {
if let Some(http_preconditions) = opts.http_preconditions.clone() {
if !opts.no_lock {
let ns_lock = self.new_ns_lock(bucket, object).await?;
object_lock_guard = Some(ns_lock.get_write_lock(get_lock_acquire_timeout()).await.map_err(|e| {
StorageError::other(format!(
"Failed to acquire write lock: {}",
self.format_lock_error_from_error(bucket, object, "write", &e)
))
})?);
object_lock_guard = Some(
ns_lock
.get_write_lock(get_lock_acquire_timeout())
.await
.map_err(|e| self.map_namespace_lock_error(bucket, object, "write", e))?,
);
}
if let Some(err) = self.check_write_precondition(bucket, object, opts).await {
@@ -3599,12 +3569,12 @@ impl MultipartOperations for SetDisks {
if !opts.no_lock && object_lock_guard.is_none() {
let ns_lock = self.new_ns_lock(bucket, object).await?;
object_lock_guard = Some(ns_lock.get_write_lock(get_lock_acquire_timeout()).await.map_err(|e| {
StorageError::other(format!(
"Failed to acquire write lock: {}",
self.format_lock_error_from_error(bucket, object, "write", &e)
))
})?);
object_lock_guard = Some(
ns_lock
.get_write_lock(get_lock_acquire_timeout())
.await
.map_err(|e| self.map_namespace_lock_error(bucket, object, "write", e))?,
);
}
self.cleanup_multipart_path(&parts).await;
@@ -3685,12 +3655,12 @@ impl HealOperations for SetDisks {
) -> Result<(HealResultItem, Option<Error>)> {
let _write_lock_guard = if !opts.no_lock {
let ns_lock = self.new_ns_lock(bucket, object).await?;
Some(ns_lock.get_write_lock(get_lock_acquire_timeout()).await.map_err(|e| {
StorageError::other(format!(
"Failed to acquire write lock: {}",
self.format_lock_error_from_error(bucket, object, "write", &e)
))
})?)
Some(
ns_lock
.get_write_lock(get_lock_acquire_timeout())
.await
.map_err(|e| self.map_namespace_lock_error(bucket, object, "write", e))?,
)
} else {
None
};
crates/ecstore/src/set_disk/heal.rs
+7 -13
View File
@@ -39,12 +39,12 @@ impl SetDisks {
let write_lock_guard = if !opts.no_lock {
let ns_lock = self.new_ns_lock(bucket, object).await?;
Some(ns_lock.get_write_lock(get_lock_acquire_timeout()).await.map_err(|e| {
StorageError::other(format!(
"Failed to acquire write lock: {}",
self.format_lock_error_from_error(bucket, object, "write", &e)
))
})?)
Some(
ns_lock
.get_write_lock(get_lock_acquire_timeout())
.await
.map_err(|e| self.map_namespace_lock_error(bucket, object, "write", e))?,
)
} else {
None
};
@@ -713,13 +713,7 @@ impl SetDisks {
.await?
.get_write_lock(get_lock_acquire_timeout())
.await
.map_err(|e| {
let message = format!(
"Failed to acquire write lock: {}",
self.format_lock_error_from_error(bucket, object, "write", &e)
);
DiskError::other(message)
})?;
.map_err(|e| DiskError::other(self.map_namespace_lock_error(bucket, object, "write", e).to_string()))?;
self.heal_object_dir_locked(bucket, object, dry_run, remove).await
}
crates/ecstore/src/set_disk/lock.rs
+24
View File
@@ -50,6 +50,30 @@ impl SetDisks {
}
}
pub(super) fn map_namespace_lock_error(
&self,
bucket: &str,
object: &str,
mode: &'static str,
err: rustfs_lock::error::LockError,
) -> StorageError {
match err {
rustfs_lock::error::LockError::QuorumNotReached { required, achieved } => {
StorageError::NamespaceLockQuorumUnavailable {
mode,
bucket: bucket.to_string(),
object: object.to_string(),
required,
achieved,
}
}
other => StorageError::other(format!(
"Failed to acquire {mode} lock: {}",
self.format_lock_error_from_error(bucket, object, mode, &other)
)),
}
}
pub(super) async fn get_disks_internal(&self) -> Vec<Option<DiskStore>> {
let rl = self.disks.read().await;