fix(readiness): gate on lock quorum health (#3100)

* fix(readiness): gate on lock quorum health

Include distributed lock quorum in runtime readiness decisions and expose the signal in health responses.

Map namespace lock quorum failures to ServiceUnavailable instead of InternalError so clients can safely retry while keeping quorum enforcement unchanged.

* fix(ecstore): restore namespace lock error decoding

Add the missing from_u32 arm for NamespaceLockQuorumUnavailable and cover the numeric roundtrip in error tests.
This commit is contained in:
houseme
2026-05-28 01:12:52 +08:00
committed by GitHub
parent 4648de9e62
commit 5dfc1b5f07
9 changed files with 502 additions and 113 deletions
+45
View File
@@ -160,6 +160,14 @@ pub enum StorageError {
NotModified,
#[error("Invalid range specified: {0}")]
InvalidRangeSpec(String),
#[error("Namespace lock quorum unavailable for {mode} lock on {bucket}/{object}: required {required}, achieved {achieved}")]
NamespaceLockQuorumUnavailable {
mode: &'static str,
bucket: String,
object: String,
required: usize,
achieved: usize,
},
// ── Generic ──────────────────────────────────────────────────────
#[error("Unexpected error")]
@@ -204,6 +212,7 @@ impl StorageError {
| StorageError::ErasureWriteQuorum
| StorageError::InsufficientReadQuorum(_, _)
| StorageError::InsufficientWriteQuorum(_, _)
| StorageError::NamespaceLockQuorumUnavailable { .. }
)
}
}
@@ -433,6 +442,19 @@ impl Clone for StorageError {
StorageError::NotModified => StorageError::NotModified,
StorageError::InvalidPartNumber(a) => StorageError::InvalidPartNumber(*a),
StorageError::InvalidRangeSpec(a) => StorageError::InvalidRangeSpec(a.clone()),
StorageError::NamespaceLockQuorumUnavailable {
mode,
bucket,
object,
required,
achieved,
} => StorageError::NamespaceLockQuorumUnavailable {
mode,
bucket: bucket.clone(),
object: object.clone(),
required: *required,
achieved: *achieved,
},
}
}
}
@@ -505,6 +527,7 @@ impl StorageError {
StorageError::InvalidRangeSpec(_) => 0x3D,
StorageError::NotModified => 0x3E,
StorageError::InvalidPartNumber(_) => 0x3F,
StorageError::NamespaceLockQuorumUnavailable { .. } => 0x42,
}
}
@@ -579,6 +602,13 @@ impl StorageError {
0x3D => Some(StorageError::InvalidRangeSpec(Default::default())),
0x3E => Some(StorageError::NotModified),
0x3F => Some(StorageError::InvalidPartNumber(Default::default())),
0x42 => Some(StorageError::NamespaceLockQuorumUnavailable {
mode: "write",
bucket: Default::default(),
object: Default::default(),
required: Default::default(),
achieved: Default::default(),
}),
_ => None,
}
}
@@ -984,6 +1014,17 @@ mod tests {
assert_eq!(StorageError::DecommissionAlreadyRunning.to_u32(), 0x30);
assert_eq!(StorageError::RebalanceAlreadyRunning.to_u32(), 0x40);
assert_eq!(StorageError::OperationCanceled.to_u32(), 0x41);
assert_eq!(
StorageError::NamespaceLockQuorumUnavailable {
mode: "write",
bucket: "bucket".into(),
object: "object".into(),
required: 3,
achieved: 2,
}
.to_u32(),
0x42
);
}
#[test]
@@ -998,6 +1039,10 @@ mod tests {
assert!(matches!(StorageError::from_u32(0x30), Some(StorageError::DecommissionAlreadyRunning)));
assert!(matches!(StorageError::from_u32(0x40), Some(StorageError::RebalanceAlreadyRunning)));
assert!(matches!(StorageError::from_u32(0x41), Some(StorageError::OperationCanceled)));
assert!(matches!(
StorageError::from_u32(0x42),
Some(StorageError::NamespaceLockQuorumUnavailable { .. })
));
// Test invalid code returns None
assert!(StorageError::from_u32(0xFF).is_none());