gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-09-02 10:18:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

test(ecstore): add targeted refresh-loss harness (#6924)

Browse Source 
Co-authored-by: heihutu <heihutu@gmail.com>
This commit is contained in:
houseme
2026-08-31 08:45:04 +08:00
committed by GitHub
parent b6c3108e53
commit 602ed2cbcd
2 changed files with 154 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/store/object.rs
+21 -60
View File
@@ -3934,7 +3934,7 @@ mod tests {
};
use crate::core::pools::{PoolDecommissionInfo, PoolStatus};
use crate::core::sets::make_local_two_set_sets_with_ctx;
use crate::ecstore_validation_blackbox::{make_local_set_disks, make_local_set_disks_with_ctx};
use crate::ecstore_validation_blackbox::{RefreshLossLockClient, make_local_set_disks, make_local_set_disks_with_ctx};
use crate::layout::{
endpoints::{Endpoints, PoolEndpoints, SetupType},
format::FormatV3,
@@ -3949,7 +3949,7 @@ mod tests {
use bytes::Bytes;
use std::io::Cursor;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::atomic::{AtomicUsize, Ordering};
use tokio::io::AsyncReadExt;
struct WaitForLockLossReader {
@@ -3991,68 +3991,17 @@ mod tests {
calls: AtomicUsize,
}
#[derive(Debug)]
struct RefreshFailureLockClient {
inner: LocalClient,
fail_refresh: AtomicBool,
}
#[async_trait::async_trait]
impl rustfs_lock::LockClient for RefreshFailureLockClient {
async fn acquire_lock(&self, request: &rustfs_lock::LockRequest) -> rustfs_lock::Result<rustfs_lock::LockResponse> {
rustfs_lock::LockClient::acquire_lock(&self.inner, request).await
}
async fn release(&self, lock_id: &rustfs_lock::LockId) -> rustfs_lock::Result<bool> {
rustfs_lock::LockClient::release(&self.inner, lock_id).await
}
async fn refresh(&self, lock_id: &rustfs_lock::LockId) -> rustfs_lock::Result<bool> {
if self.fail_refresh.load(Ordering::Acquire) {
return Ok(false);
}
rustfs_lock::LockClient::refresh(&self.inner, lock_id).await
}
async fn force_release(&self, lock_id: &rustfs_lock::LockId) -> rustfs_lock::Result<bool> {
rustfs_lock::LockClient::force_release(&self.inner, lock_id).await
}
async fn check_status(&self, lock_id: &rustfs_lock::LockId) -> rustfs_lock::Result<Option<rustfs_lock::LockInfo>> {
rustfs_lock::LockClient::check_status(&self.inner, lock_id).await
}
async fn get_stats(&self) -> rustfs_lock::Result<rustfs_lock::LockStats> {
rustfs_lock::LockClient::get_stats(&self.inner).await
}
async fn close(&self) -> rustfs_lock::Result<()> {
rustfs_lock::LockClient::close(&self.inner).await
}
async fn is_online(&self) -> bool {
rustfs_lock::LockClient::is_online(&self.inner).await
}
async fn is_local(&self) -> bool {
rustfs_lock::LockClient::is_local(&self.inner).await
}
}
async fn refresh_failure_test_guard(
owner: &'static str,
) -> (
ObjectLockDiagGuard,
Arc<rustfs_lock::distributed_lock::LockLostSignal>,
Arc<RefreshFailureLockClient>,
Arc<RefreshLossLockClient>,
) {
let manager = Arc::new(rustfs_lock::GlobalLockManager::Enabled(Arc::new(
rustfs_lock::FastObjectLockManager::new(),
)));
let client = Arc::new(RefreshFailureLockClient {
inner: LocalClient::with_manager(manager),
fail_refresh: AtomicBool::new(false),
});
let client = Arc::new(RefreshLossLockClient::with_manager(manager));
let namespace_lock = rustfs_lock::NamespaceLock::with_clients_and_quorum(
owner.to_string(),
vec![Arc::clone(&client) as Arc<dyn rustfs_lock::LockClient>],
@@ -4087,7 +4036,7 @@ mod tests {
) -> (
Arc<SelectObjectSnapshotLease>,
Arc<rustfs_lock::distributed_lock::LockLostSignal>,
Arc<RefreshFailureLockClient>,
Arc<RefreshLossLockClient>,
) {
let (guard, signal, client) = refresh_failure_test_guard(owner).await;
(Arc::new(SelectObjectSnapshotLease::new(vec![guard])), signal, client)
@@ -4220,7 +4169,11 @@ mod tests {
let release_signal = Arc::clone(&signal);
let release_task = tokio::spawn(async move {
poll_started_rx.await.expect("reader poll should start");
release_client.fail_refresh.store(true, Ordering::Release);
release_client.reject_refreshes();
release_client
.wait_for_rejected_refresh(Duration::from_secs(5))
.await
.expect("refresh rejection should be observed");
tokio::time::timeout(Duration::from_secs(5), release_signal.notified())
.await
.expect("heartbeat should observe the rejected refresh");
@@ -4258,7 +4211,11 @@ mod tests {
#[tokio::test(flavor = "current_thread")]
async fn select_snapshot_reader_checks_guards_at_eof_before_monitor_runs() {
let (guard, signal, client) = refresh_failure_test_guard("select-snapshot-eof-fence").await;
client.fail_refresh.store(true, Ordering::Release);
client.reject_refreshes();
client
.wait_for_rejected_refresh(Duration::from_secs(5))
.await
.expect("refresh rejection should be observed");
tokio::time::timeout(Duration::from_secs(5), signal.notified())
.await
.expect("heartbeat should observe the rejected refresh");
@@ -4323,7 +4280,11 @@ mod tests {
second_started_rx
.await
.expect("second inner reader should reach Poll::Pending");
second_client.fail_refresh.store(true, Ordering::Release);
second_client.reject_refreshes();
second_client
.wait_for_rejected_refresh(Duration::from_secs(5))
.await
.expect("refresh rejection should be observed");
let (first_result, second_result) = tokio::join!(
tokio::time::timeout(Duration::from_secs(5), first_read_task),
tokio::time::timeout(Duration::from_secs(5), second_read_task),
@@ -4336,7 +4297,7 @@ mod tests {
assert_eq!(error.kind(), std::io::ErrorKind::Other);
}
assert!(!first_client.fail_refresh.load(Ordering::Acquire));
assert!(!first_client.refreshes_rejected());
assert!(!first_signal.is_lost());
assert!(second_signal.is_lost());
}