gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-09-03 10:48:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(scanner): rebuild missing usage floor after upgrade (#6624)

Browse Source 
* fix(scanner): rebuild missing usage floor after upgrade

* fix(scanner): preserve missing-floor reset across conflicts

---------

Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
Co-authored-by: overtrue <anzhengchao@gmail.com>
This commit is contained in:
Henry Guo
2026-08-26 12:30:53 +08:00
committed by GitHub
parent c7f201e6cb
commit 1590d9107b
4 changed files with 293 additions and 171 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/scanner/src/scanner.rs
+54 -67
View File
@@ -403,7 +403,7 @@ pub(super) fn data_usage_info_has_persisted_baseline_identity(info: &DataUsageIn
&& u64::try_from(info.buckets_usage.len()).ok() == Some(info.buckets_count) && u64::try_from(info.buckets_usage.len()).ok() == Some(info.buckets_count)
} }
pub(super) fn data_usage_info_is_pristine_bootstrap_pending(info: &DataUsageInfo) -> bool { pub(super) fn data_usage_info_is_bootstrap_pending(info: &DataUsageInfo) -> bool {
if info.last_update.is_none() || info.scanner_cycle.is_some() { if info.last_update.is_none() || info.scanner_cycle.is_some() {
return false; return false;
} }
@@ -681,27 +681,30 @@ async fn initial_scanner_startup_usage_state(storeapi: &Arc<ECStore>) -> (bool,
(persisted_usage_cache_is_cold_for_startup(storeapi).await, has_buckets) (persisted_usage_cache_is_cold_for_startup(storeapi).await, has_buckets)
} }
fn scanner_cycle_state_is_pristine( fn prepare_cycle_for_usage_floor_bootstrap(
cycle_info: &CurrentCycle, cycle_info: &mut CurrentCycle,
leader_epoch: u64, usage_floor: PersistedUsageFloor,
cycle_revision: &DataUsageCacheRevision, startup: PersistedUsageFloorStartup,
) -> bool { ) -> (bool, bool) {
cycle_info.next == 0 && leader_epoch == 0 && matches!(cycle_revision, DataUsageCacheRevision::Missing) match startup {
} PersistedUsageFloorStartup::Authoritative => (false, false),
PersistedUsageFloorStartup::Missing => {
fn scanner_may_bootstrap_missing_usage_floor( // Cycle progress without its corresponding usage floor cannot
cycle_info: &CurrentCycle, // prove namespace coverage. Restart from cycle zero while keeping
leader_epoch: u64, // the separately fenced leader epoch monotonic.
cycle_revision: &DataUsageCacheRevision, *cycle_info = CurrentCycle::default();
) -> bool { (true, true)
// The server becomes ready before the scanner starts, so a first bucket may }
// already exist. The bootstrap marker is non-authoritative; only prior PersistedUsageFloorStartup::BootstrapPending => {
// durable scanner progress must block its creation. // An unfenced marker may have been written before an upgrade's old
scanner_cycle_state_is_pristine(cycle_info, leader_epoch, cycle_revision) // cycle state was replaced. A fenced marker belongs to the current
} // scanner generation and may retain partial-cycle progress.
if usage_floor.leader_epoch == 0 {
fn scanner_may_resume_pristine_usage_bootstrap(cycle_info: &CurrentCycle) -> bool { *cycle_info = CurrentCycle::default();
cycle_info.next == 0 }
(true, usage_floor.leader_epoch == 0)
}
}
} }
pub async fn init_data_scanner(ctx: CancellationToken, storeapi: Arc<ECStore>) { pub async fn init_data_scanner(ctx: CancellationToken, storeapi: Arc<ECStore>) {
@@ -1220,7 +1223,7 @@ where
LockLost: Future<Output = ()>, LockLost: Future<Output = ()>,
{ {
let fence_ctx = ctx.child_token(); let fence_ctx = ctx.child_token();
let claim = claim_scanner_leadership(&fence_ctx, storeapi, cycle_info, cycle_revision, leader_epoch, false); let claim = claim_scanner_leadership(&fence_ctx, storeapi, cycle_info, cycle_revision, leader_epoch, false, false);
tokio::pin!(claim); tokio::pin!(claim);
tokio::pin!(lock_lost); tokio::pin!(lock_lost);
tokio::select! { tokio::select! {
@@ -2186,47 +2189,30 @@ async fn run_data_scanner_with_maintenance_state(
return Err(err); return Err(err);
} }
}; };
let may_bootstrap_missing_usage_floor = scanner_may_bootstrap_missing_usage_floor(&cycle_info, leader_epoch, &cycle_revision); let (usage_floor, usage_floor_startup) = match persisted_usage_floor_for_startup(storeapi.clone(), true).await {
let (usage_floor, usage_floor_startup) = Ok(result) => result,
match persisted_usage_floor_for_startup(storeapi.clone(), may_bootstrap_missing_usage_floor).await { Err(err) => {
Ok(result) => result, error!(
Err(err) => { target: "rustfs::scanner",
error!( event = EVENT_SCANNER_PERSIST_STATE,
target: "rustfs::scanner", component = LOG_COMPONENT_SCANNER,
event = EVENT_SCANNER_PERSIST_STATE, subsystem = LOG_SUBSYSTEM_RUNTIME,
component = LOG_COMPONENT_SCANNER, path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
subsystem = LOG_SUBSYSTEM_RUNTIME, state = "usage_floor_load_failed",
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(), error = %err,
state = "usage_floor_load_failed", "Scanner stopped because the persisted usage floor could not be loaded"
error = %err, );
"Scanner stopped because the persisted usage floor could not be loaded" global_metrics().set_cycle(None).await;
); return Ok(());
global_metrics().set_cycle(None).await; }
return Ok(()); };
} let (allow_usage_floor_bootstrap_pending, reset_usage_floor_bootstrap_cycle_on_conflict) =
}; prepare_cycle_for_usage_floor_bootstrap(&mut cycle_info, usage_floor, usage_floor_startup);
if usage_floor_startup == PersistedUsageFloorStartup::BootstrapPending
&& !scanner_may_resume_pristine_usage_bootstrap(&cycle_info)
{
error!(
target: "rustfs::scanner",
event = EVENT_SCANNER_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
state = "usage_floor_bootstrap_conflict",
next_cycle = cycle_info.next,
"Scanner stopped because a pristine usage bootstrap conflicts with persisted cycle progress"
);
global_metrics().set_cycle(None).await;
return Ok(());
}
apply_persisted_usage_floor(&mut cycle_info, &mut leader_epoch, usage_floor); apply_persisted_usage_floor(&mut cycle_info, &mut leader_epoch, usage_floor);
let allow_pristine_bootstrap_pending = match usage_floor_startup { match usage_floor_startup {
PersistedUsageFloorStartup::Authoritative => false, PersistedUsageFloorStartup::Authoritative | PersistedUsageFloorStartup::BootstrapPending => {}
PersistedUsageFloorStartup::BootstrapPending => true,
PersistedUsageFloorStartup::Missing => { PersistedUsageFloorStartup::Missing => {
if !may_bootstrap_missing_usage_floor || ctx.is_cancelled() || guard.is_lock_lost() { if ctx.is_cancelled() || guard.is_lock_lost() {
global_metrics().set_cycle(None).await; global_metrics().set_cycle(None).await;
return Ok(()); return Ok(());
} }
@@ -2234,12 +2220,12 @@ async fn run_data_scanner_with_maintenance_state(
let bootstrap_ctx = ctx.child_token(); let bootstrap_ctx = ctx.child_token();
match await_scanner_cycle_with_lock_fence( match await_scanner_cycle_with_lock_fence(
&bootstrap_ctx, &bootstrap_ctx,
initialize_pristine_usage_baseline(storeapi.clone()), initialize_usage_baseline_bootstrap(storeapi.clone()),
guard.lock_lost_notified(), guard.lock_lost_notified(),
) )
.await .await
{ {
Some(Ok(())) => true, Some(Ok(())) => {}
Some(Err(err)) => { Some(Err(err)) => {
error!( error!(
target: "rustfs::scanner", target: "rustfs::scanner",
@@ -2249,7 +2235,7 @@ async fn run_data_scanner_with_maintenance_state(
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(), path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
state = "usage_floor_bootstrap_failed", state = "usage_floor_bootstrap_failed",
error = %err, error = %err,
"Scanner stopped because the pristine usage bootstrap could not be initialized" "Scanner stopped because the usage baseline bootstrap could not be initialized"
); );
global_metrics().set_cycle(None).await; global_metrics().set_cycle(None).await;
return Ok(()); return Ok(());
@@ -2260,7 +2246,7 @@ async fn run_data_scanner_with_maintenance_state(
} }
} }
} }
}; }
if ctx.is_cancelled() || guard.is_lock_lost() { if ctx.is_cancelled() || guard.is_lock_lost() {
global_metrics().set_cycle(None).await; global_metrics().set_cycle(None).await;
@@ -2275,7 +2261,8 @@ async fn run_data_scanner_with_maintenance_state(
&mut cycle_info, &mut cycle_info,
&mut cycle_revision, &mut cycle_revision,
&mut leader_epoch, &mut leader_epoch,
allow_pristine_bootstrap_pending, allow_usage_floor_bootstrap_pending,
reset_usage_floor_bootstrap_cycle_on_conflict,
), ),
guard.lock_lost_notified(), guard.lock_lost_notified(),
) )
crates/scanner/src/scanner/cycle_state.rs
+9 -11
View File
@@ -1319,7 +1319,7 @@ pub(super) async fn persisted_usage_floor(
pub(super) async fn persisted_usage_floor_for_startup( pub(super) async fn persisted_usage_floor_for_startup(
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>, storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
allow_missing_for_pristine_startup: bool, allow_missing_for_bootstrap: bool,
) -> Result<(PersistedUsageFloor, PersistedUsageFloorStartup), ScannerError> { ) -> Result<(PersistedUsageFloor, PersistedUsageFloorStartup), ScannerError> {
let Some(read_epoch) = scanner_publication_epoch(storeapi.clone()).await else { let Some(read_epoch) = scanner_publication_epoch(storeapi.clone()).await else {
return Err(ScannerError::Other("scanner usage floor read is blocked by data movement".to_string())); return Err(ScannerError::Other("scanner usage floor read is blocked by data movement".to_string()));
@@ -1345,11 +1345,9 @@ pub(super) async fn persisted_usage_floor_for_startup(
let usage = serde_json::from_slice::<DataUsageInfo>(&data).map_err(|err| { let usage = serde_json::from_slice::<DataUsageInfo>(&data).map_err(|err| {
ScannerError::Other(format!("failed to decode scanner usage floor from {primary_path}: {err}")) ScannerError::Other(format!("failed to decode scanner usage floor from {primary_path}: {err}"))
})?; })?;
if data_usage_info_is_pristine_bootstrap_pending(&usage) && primary_path == DATA_USAGE_OBJ_NAME_PATH.as_str() { if data_usage_info_is_bootstrap_pending(&usage) && primary_path == DATA_USAGE_OBJ_NAME_PATH.as_str() {
if bootstrap_pending { if bootstrap_pending {
return Err(ScannerError::Other( return Err(ScannerError::Other("multiple scanner usage bootstrap markers were found".to_string()));
"multiple pristine scanner usage bootstrap markers were found".to_string(),
));
} }
bootstrap_pending = true; bootstrap_pending = true;
update_floor(&mut floor, &usage, primary_path)?; update_floor(&mut floor, &usage, primary_path)?;
@@ -1376,7 +1374,7 @@ pub(super) async fn persisted_usage_floor_for_startup(
Ok((Some(data), _)) => { Ok((Some(data), _)) => {
if bootstrap_pending { if bootstrap_pending {
return Err(ScannerError::Other( return Err(ScannerError::Other(
"pristine scanner usage bootstrap conflicts with a persisted backup".to_string(), "scanner usage bootstrap conflicts with a persisted backup".to_string(),
)); ));
} }
any_found = true; any_found = true;
@@ -1406,7 +1404,7 @@ pub(super) async fn persisted_usage_floor_for_startup(
if any_found { if any_found {
if bootstrap_pending { if bootstrap_pending {
return Err(ScannerError::Other( return Err(ScannerError::Other(
"pristine scanner usage bootstrap conflicts with an authoritative usage floor".to_string(), "scanner usage bootstrap conflicts with an authoritative usage floor".to_string(),
)); ));
} }
found_any = true; found_any = true;
@@ -1415,14 +1413,14 @@ pub(super) async fn persisted_usage_floor_for_startup(
} }
if !found_any && !bootstrap_pending { if !found_any && !bootstrap_pending {
if !allow_missing_for_pristine_startup { if !allow_missing_for_bootstrap {
return Err(ScannerError::Other( return Err(ScannerError::Other(
"persisted scanner usage floor has no authoritative baseline".to_string(), "persisted scanner usage floor has no authoritative baseline".to_string(),
)); ));
} }
let Some(publication_admission) = scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch).await else { let Some(publication_admission) = scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch).await else {
return Err(ScannerError::Other( return Err(ScannerError::Other(
"scanner usage floor changed before pristine state confirmation".to_string(), "scanner usage floor changed before missing-state confirmation".to_string(),
)); ));
}; };
for path in [ for path in [
@@ -1435,12 +1433,12 @@ pub(super) async fn persisted_usage_floor_for_startup(
Ok((None, _)) => {} Ok((None, _)) => {}
Ok((Some(_), _)) => { Ok((Some(_), _)) => {
return Err(ScannerError::Other(format!( return Err(ScannerError::Other(format!(
"scanner usage floor changed while confirming pristine state: {path} appeared" "scanner usage floor changed while confirming missing state: {path} appeared"
))); )));
} }
Err(err) => { Err(err) => {
return Err(ScannerError::Other(format!( return Err(ScannerError::Other(format!(
"failed to confirm pristine scanner usage floor at {path}: {err}" "failed to confirm missing scanner usage floor at {path}: {err}"
))); )));
} }
} }
crates/scanner/src/scanner/leadership.rs
+35 -32
View File
@@ -63,14 +63,12 @@ pub(super) async fn reconcile_scanner_leadership_claim(
pub(super) fn decode_usage_snapshot_for_epoch_fence( pub(super) fn decode_usage_snapshot_for_epoch_fence(
data: &[u8], data: &[u8],
path: &str, path: &str,
allow_pristine_bootstrap_pending: bool, allow_bootstrap_pending: bool,
) -> Result<DataUsageInfo, ScannerError> { ) -> Result<DataUsageInfo, ScannerError> {
let usage: DataUsageInfo = serde_json::from_slice(data) let usage: DataUsageInfo = serde_json::from_slice(data)
.map_err(|err| ScannerError::Other(format!("failed to decode scanner usage epoch fence from {path}: {err}")))?; .map_err(|err| ScannerError::Other(format!("failed to decode scanner usage epoch fence from {path}: {err}")))?;
if !data_usage_info_has_persisted_baseline_identity(&usage) if !data_usage_info_has_persisted_baseline_identity(&usage)
&& !(allow_pristine_bootstrap_pending && !(allow_bootstrap_pending && path == DATA_USAGE_OBJ_NAME_PATH.as_str() && data_usage_info_is_bootstrap_pending(&usage))
&& path == DATA_USAGE_OBJ_NAME_PATH.as_str()
&& data_usage_info_is_pristine_bootstrap_pending(&usage))
{ {
return Err(ScannerError::Other(format!( return Err(ScannerError::Other(format!(
"scanner usage epoch fence from {path} has no persisted baseline identity" "scanner usage epoch fence from {path} has no persisted baseline identity"
@@ -82,15 +80,11 @@ pub(super) fn decode_usage_snapshot_for_epoch_fence(
pub(super) async fn usage_snapshot_for_epoch_fence( pub(super) async fn usage_snapshot_for_epoch_fence(
storeapi: Arc<impl ScannerObjectIO>, storeapi: Arc<impl ScannerObjectIO>,
primary: Option<&[u8]>, primary: Option<&[u8]>,
allow_pristine_bootstrap_pending: bool, allow_bootstrap_pending: bool,
) -> Result<Option<DataUsageInfo>, ScannerError> { ) -> Result<Option<DataUsageInfo>, ScannerError> {
if let Some(primary) = primary { if let Some(primary) = primary {
return decode_usage_snapshot_for_epoch_fence( return decode_usage_snapshot_for_epoch_fence(primary, DATA_USAGE_OBJ_NAME_PATH.as_str(), allow_bootstrap_pending)
primary, .map(Some);
DATA_USAGE_OBJ_NAME_PATH.as_str(),
allow_pristine_bootstrap_pending,
)
.map(Some);
} }
let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str()); let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str());
@@ -118,12 +112,12 @@ pub(super) async fn usage_snapshot_for_epoch_fence(
Ok(None) Ok(None)
} }
pub(super) async fn initialize_pristine_usage_baseline( pub(super) async fn initialize_usage_baseline_bootstrap(
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>, storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
) -> Result<(), ScannerError> { ) -> Result<(), ScannerError> {
let Some(expected_epoch) = scanner_publication_epoch(storeapi.clone()).await else { let Some(expected_epoch) = scanner_publication_epoch(storeapi.clone()).await else {
return Err(ScannerError::Other( return Err(ScannerError::Other(
"pristine scanner usage baseline initialization is blocked by data movement".to_string(), "scanner usage baseline bootstrap is blocked by data movement".to_string(),
)); ));
}; };
let baseline = DataUsageInfo { let baseline = DataUsageInfo {
@@ -133,7 +127,7 @@ pub(super) async fn initialize_pristine_usage_baseline(
..Default::default() ..Default::default()
}; };
let data = serde_json::to_vec(&baseline) let data = serde_json::to_vec(&baseline)
.map_err(|err| ScannerError::Other(format!("failed to encode pristine scanner usage baseline: {err}")))?; .map_err(|err| ScannerError::Other(format!("failed to encode scanner usage baseline bootstrap: {err}")))?;
let save_result = save_config_with_publication_admission_for_epoch( let save_result = save_config_with_publication_admission_for_epoch(
storeapi.clone(), storeapi.clone(),
DATA_USAGE_OBJ_NAME_PATH.as_str(), DATA_USAGE_OBJ_NAME_PATH.as_str(),
@@ -153,14 +147,14 @@ pub(super) async fn initialize_pristine_usage_baseline(
let (persisted, revision) = read_config_with_revision(storeapi, DATA_USAGE_OBJ_NAME_PATH.as_str()) let (persisted, revision) = read_config_with_revision(storeapi, DATA_USAGE_OBJ_NAME_PATH.as_str())
.await .await
.map_err(|err| ScannerError::Other(format!("failed to reconcile pristine scanner usage bootstrap: {err}")))?; .map_err(|err| ScannerError::Other(format!("failed to reconcile scanner usage bootstrap: {err}")))?;
if persisted.as_deref() == Some(data.as_slice()) && matches!(revision, DataUsageCacheRevision::Etag(_)) { if persisted.as_deref() == Some(data.as_slice()) && matches!(revision, DataUsageCacheRevision::Etag(_)) {
return Ok(()); return Ok(());
} }
Err(ScannerError::Other(match save_result { Err(ScannerError::Other(match save_result {
Ok(_) => "pristine scanner usage bootstrap returned no ETag and could not be confirmed".to_string(), Ok(_) => "scanner usage bootstrap returned no ETag and could not be confirmed".to_string(),
Err(err) => format!("failed to persist pristine scanner usage bootstrap: {err}"), Err(err) => format!("failed to persist scanner usage bootstrap: {err}"),
})) }))
} }
@@ -169,7 +163,7 @@ pub(super) async fn fence_scanner_usage_epoch_with_expected_epoch(
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>, storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
claimed_epoch: u64, claimed_epoch: u64,
expected_publication_epoch: Option<u64>, expected_publication_epoch: Option<u64>,
allow_pristine_bootstrap_pending: bool, allow_bootstrap_pending: bool,
) -> Result<(), ScannerError> { ) -> Result<(), ScannerError> {
for retry in 0..=SCANNER_PERSIST_CAS_RETRIES { for retry in 0..=SCANNER_PERSIST_CAS_RETRIES {
if ctx.is_cancelled() { if ctx.is_cancelled() {
@@ -193,7 +187,7 @@ pub(super) async fn fence_scanner_usage_epoch_with_expected_epoch(
.await .await
.map_err(|err| ScannerError::Other(format!("failed to read scanner usage epoch fence: {err}")))?; .map_err(|err| ScannerError::Other(format!("failed to read scanner usage epoch fence: {err}")))?;
let Some(mut usage) = let Some(mut usage) =
usage_snapshot_for_epoch_fence(storeapi.clone(), primary.as_deref(), allow_pristine_bootstrap_pending).await? usage_snapshot_for_epoch_fence(storeapi.clone(), primary.as_deref(), allow_bootstrap_pending).await?
else { else {
let Some(_publication_admission) = scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch).await else { let Some(_publication_admission) = scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch).await else {
if retry < SCANNER_PERSIST_CAS_RETRIES { if retry < SCANNER_PERSIST_CAS_RETRIES {
@@ -243,11 +237,8 @@ pub(super) async fn fence_scanner_usage_epoch_with_expected_epoch(
.await .await
.map_err(|err| ScannerError::Other(format!("failed to reconcile scanner usage epoch fence: {err}")))?; .map_err(|err| ScannerError::Other(format!("failed to reconcile scanner usage epoch fence: {err}")))?;
if let Some(persisted) = persisted { if let Some(persisted) = persisted {
let persisted = decode_usage_snapshot_for_epoch_fence( let persisted =
&persisted, decode_usage_snapshot_for_epoch_fence(&persisted, DATA_USAGE_OBJ_NAME_PATH.as_str(), allow_bootstrap_pending)?;
DATA_USAGE_OBJ_NAME_PATH.as_str(),
allow_pristine_bootstrap_pending,
)?;
match persisted.scanner_epoch { match persisted.scanner_epoch {
Some(epoch) if epoch == claimed_epoch => return Ok(()), Some(epoch) if epoch == claimed_epoch => return Ok(()),
Some(epoch) if epoch > claimed_epoch => { Some(epoch) if epoch > claimed_epoch => {
@@ -277,14 +268,14 @@ pub(super) async fn complete_scanner_leadership_claim(
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>, storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
claimed_epoch: u64, claimed_epoch: u64,
expected_publication_epoch: Option<u64>, expected_publication_epoch: Option<u64>,
allow_pristine_bootstrap_pending: bool, allow_bootstrap_pending: bool,
) -> bool { ) -> bool {
if let Err(err) = fence_scanner_usage_epoch_with_expected_epoch( if let Err(err) = fence_scanner_usage_epoch_with_expected_epoch(
ctx, ctx,
storeapi, storeapi,
claimed_epoch, claimed_epoch,
expected_publication_epoch, expected_publication_epoch,
allow_pristine_bootstrap_pending, allow_bootstrap_pending,
) )
.await .await
{ {
@@ -310,7 +301,8 @@ pub(super) async fn claim_scanner_leadership(
cycle_info: &mut CurrentCycle, cycle_info: &mut CurrentCycle,
revision: &mut DataUsageCacheRevision, revision: &mut DataUsageCacheRevision,
persisted_epoch: &mut u64, persisted_epoch: &mut u64,
allow_pristine_bootstrap_pending: bool, allow_bootstrap_pending: bool,
reset_bootstrap_cycle_on_conflict: bool,
) -> bool { ) -> bool {
for retry in 0..=SCANNER_PERSIST_CAS_RETRIES { for retry in 0..=SCANNER_PERSIST_CAS_RETRIES {
if ctx.is_cancelled() { if ctx.is_cancelled() {
@@ -365,7 +357,7 @@ pub(super) async fn claim_scanner_leadership(
return false; return false;
} }
}; };
match usage_snapshot_for_epoch_fence(storeapi.clone(), usage_primary.as_deref(), allow_pristine_bootstrap_pending).await { match usage_snapshot_for_epoch_fence(storeapi.clone(), usage_primary.as_deref(), allow_bootstrap_pending).await {
Ok(Some(_)) => {} Ok(Some(_)) => {}
Ok(None) => { Ok(None) => {
warn!( warn!(
@@ -413,7 +405,7 @@ pub(super) async fn claim_scanner_leadership(
storeapi, storeapi,
claimed_epoch, claimed_epoch,
Some(read_epoch), Some(read_epoch),
allow_pristine_bootstrap_pending, allow_bootstrap_pending,
) )
.await; .await;
} }
@@ -435,11 +427,16 @@ pub(super) async fn claim_scanner_leadership(
storeapi, storeapi,
claimed_epoch, claimed_epoch,
Some(read_epoch), Some(read_epoch),
allow_pristine_bootstrap_pending, allow_bootstrap_pending,
) )
.await; .await;
} }
Ok(ScannerLeadershipClaimReconcile::Changed) if retry < SCANNER_PERSIST_CAS_RETRIES => continue, Ok(ScannerLeadershipClaimReconcile::Changed) if retry < SCANNER_PERSIST_CAS_RETRIES => {
if reset_bootstrap_cycle_on_conflict {
*cycle_info = CurrentCycle::default();
}
continue;
}
Ok(ScannerLeadershipClaimReconcile::Changed | ScannerLeadershipClaimReconcile::Unchanged) => { Ok(ScannerLeadershipClaimReconcile::Changed | ScannerLeadershipClaimReconcile::Unchanged) => {
error!( error!(
target: "rustfs::scanner", target: "rustfs::scanner",
@@ -486,18 +483,24 @@ pub(super) async fn claim_scanner_leadership(
storeapi, storeapi,
claimed_epoch, claimed_epoch,
Some(read_epoch), Some(read_epoch),
allow_pristine_bootstrap_pending, allow_bootstrap_pending,
) )
.await; .await;
} }
Ok(ScannerLeadershipClaimReconcile::Changed) Ok(ScannerLeadershipClaimReconcile::Changed)
if retry < SCANNER_PERSIST_CAS_RETRIES && !ctx.is_cancelled() => if retry < SCANNER_PERSIST_CAS_RETRIES && !ctx.is_cancelled() =>
{ {
if reset_bootstrap_cycle_on_conflict {
*cycle_info = CurrentCycle::default();
}
continue; continue;
} }
Ok(ScannerLeadershipClaimReconcile::Unchanged) Ok(ScannerLeadershipClaimReconcile::Unchanged)
if precondition_failed && retry < SCANNER_PERSIST_CAS_RETRIES && !ctx.is_cancelled() => if precondition_failed && retry < SCANNER_PERSIST_CAS_RETRIES && !ctx.is_cancelled() =>
{ {
if reset_bootstrap_cycle_on_conflict {
*cycle_info = CurrentCycle::default();
}
continue; continue;
} }
Ok(ScannerLeadershipClaimReconcile::Changed | ScannerLeadershipClaimReconcile::Unchanged) => { Ok(ScannerLeadershipClaimReconcile::Changed | ScannerLeadershipClaimReconcile::Unchanged) => {
crates/scanner/src/scanner/tests.rs
+195 -61
View File
@@ -2111,11 +2111,11 @@ async fn scanner_usage_floor_fails_closed_on_corrupt_or_exhausted_usage_state()
} }
#[tokio::test] #[tokio::test]
async fn scanner_usage_floor_allows_only_explicit_pristine_bootstrap() { async fn scanner_usage_floor_allows_only_explicit_missing_state_bootstrap() {
let store = Arc::new(MemoryConfigStore::default()); let store = Arc::new(MemoryConfigStore::default());
let (floor, state) = persisted_usage_floor_for_startup(store.clone(), true) let (floor, state) = persisted_usage_floor_for_startup(store.clone(), true)
.await .await
.expect("a verified pristine startup should use the empty floor"); .expect("a verified missing state should use the empty floor");
assert_eq!(floor, PersistedUsageFloor::default()); assert_eq!(floor, PersistedUsageFloor::default());
assert_eq!(state, PersistedUsageFloorStartup::Missing); assert_eq!(state, PersistedUsageFloorStartup::Missing);
assert!(persisted_usage_floor_for_startup(store.clone(), false).await.is_err()); assert!(persisted_usage_floor_for_startup(store.clone(), false).await.is_err());
@@ -2126,7 +2126,7 @@ async fn scanner_usage_floor_allows_only_explicit_pristine_bootstrap() {
); );
assert!( assert!(
persisted_usage_floor_for_startup(store, true).await.is_err(), persisted_usage_floor_for_startup(store, true).await.is_err(),
"pristine bootstrap must not hide corrupt persisted state" "usage bootstrap must not hide corrupt persisted state"
); );
} }
@@ -2156,16 +2156,16 @@ async fn scanner_usage_floor_fails_closed_on_zero_byte_usage_objects() {
let err = persisted_usage_floor_for_startup(appearing, true) let err = persisted_usage_floor_for_startup(appearing, true)
.await .await
.expect_err("an empty usage object appearing during confirmation must prevent pristine bootstrap"); .expect_err("an empty usage object appearing during confirmation must prevent usage bootstrap");
assert!( assert!(
err.to_string().contains("changed while confirming pristine state"), err.to_string().contains("changed while confirming missing state"),
"unexpected confirmation error for {path}: {err}" "unexpected confirmation error for {path}: {err}"
); );
} }
} }
#[tokio::test] #[tokio::test]
async fn scanner_usage_floor_requires_publication_admission_for_pristine_bootstrap() { async fn scanner_usage_floor_requires_publication_admission_for_bootstrap() {
let store = Arc::new(MemoryConfigStore::default()); let store = Arc::new(MemoryConfigStore::default());
store.publication_admission_blocked.store(true, Ordering::Release); store.publication_admission_blocked.store(true, Ordering::Release);
@@ -2173,18 +2173,18 @@ async fn scanner_usage_floor_requires_publication_admission_for_pristine_bootstr
} }
#[tokio::test] #[tokio::test]
async fn scanner_usage_floor_fails_closed_when_usage_appears_during_pristine_confirmation() { async fn scanner_usage_floor_fails_closed_when_usage_appears_during_missing_confirmation() {
let store = Arc::new(MemoryConfigStore::default()); let store = Arc::new(MemoryConfigStore::default());
insert_usage_after_first_legacy_backup_read(store.as_ref()).await; insert_usage_after_first_legacy_backup_read(store.as_ref()).await;
let err = persisted_usage_floor_for_startup(store, true) let err = persisted_usage_floor_for_startup(store, true)
.await .await
.expect_err("an appearing usage snapshot must prevent pristine bootstrap"); .expect_err("an appearing usage snapshot must prevent usage bootstrap");
assert!(err.to_string().contains("changed while confirming pristine state")); assert!(err.to_string().contains("changed while confirming missing state"));
} }
#[tokio::test] #[tokio::test]
async fn scanner_usage_floor_rejects_publication_change_during_pristine_confirmation() { async fn scanner_usage_floor_rejects_publication_change_during_missing_confirmation() {
let store = Arc::new(MemoryConfigStore::default()); let store = Arc::new(MemoryConfigStore::default());
store.block_publication_after_admissions.store(2, Ordering::Release); store.block_publication_after_admissions.store(2, Ordering::Release);
@@ -2192,42 +2192,139 @@ async fn scanner_usage_floor_rejects_publication_change_during_pristine_confirma
} }
#[test] #[test]
fn scanner_pristine_cycle_state_requires_no_durable_progress() { fn missing_usage_floor_discards_unfenced_cycle_progress() {
let cycle = CurrentCycle::default(); let mut cycle = CurrentCycle {
assert!(scanner_cycle_state_is_pristine(&cycle, 0, &DataUsageCacheRevision::Missing)); current: 11,
assert!(scanner_may_resume_pristine_usage_bootstrap(&cycle)); next: 12,
assert!(!scanner_cycle_state_is_pristine( cycle_completed: vec![Utc::now()],
&CurrentCycle { started: Utc::now(),
next: 1, };
..Default::default()
}, assert_eq!(
0, prepare_cycle_for_usage_floor_bootstrap(&mut cycle, PersistedUsageFloor::default(), PersistedUsageFloorStartup::Missing,),
&DataUsageCacheRevision::Missing (true, true)
)); );
assert!(!scanner_may_resume_pristine_usage_bootstrap(&CurrentCycle { assert_eq!(cycle.next, 0);
next: 1, assert_eq!(cycle.current, 0);
assert!(cycle.cycle_completed.is_empty());
cycle.next = 12;
assert_eq!(
prepare_cycle_for_usage_floor_bootstrap(
&mut cycle,
PersistedUsageFloor::default(),
PersistedUsageFloorStartup::BootstrapPending,
),
(true, true)
);
assert_eq!(cycle.next, 0);
}
#[test]
fn fenced_usage_bootstrap_retains_partial_cycle_progress() {
let mut cycle = CurrentCycle {
next: 12,
..Default::default() ..Default::default()
})); };
assert!(!scanner_cycle_state_is_pristine(&cycle, 1, &DataUsageCacheRevision::Missing));
assert!(!scanner_cycle_state_is_pristine( assert_eq!(
&cycle, prepare_cycle_for_usage_floor_bootstrap(
0, &mut cycle,
&DataUsageCacheRevision::Etag("etag".to_string()) PersistedUsageFloor {
)); next_cycle: 0,
leader_epoch: 7,
},
PersistedUsageFloorStartup::BootstrapPending,
),
(true, false)
);
assert_eq!(cycle.next, 12);
assert_eq!(
prepare_cycle_for_usage_floor_bootstrap(
&mut cycle,
PersistedUsageFloor {
next_cycle: 13,
leader_epoch: 7,
},
PersistedUsageFloorStartup::Authoritative,
),
(false, false)
);
assert_eq!(cycle.next, 12);
} }
#[tokio::test] #[tokio::test]
async fn scanner_pristine_bootstrap_allows_first_bucket_to_win_startup() { async fn missing_usage_floor_rebuilds_persisted_cycle_before_leadership_claim() {
let store = Arc::new(MemoryConfigStore::default());
let state_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str());
let stale_cycle = CurrentCycle {
next: 12,
..Default::default()
};
store.objects.lock().await.insert(
state_key.clone(),
encode_scanner_cycle_state(&stale_cycle, 4).expect("stale cycle state should encode"),
);
store.revisions.lock().await.insert(state_key, 7);
let ScannerCycleStateStartup::Ready {
cycle: mut cycle_info,
leader_epoch: mut persisted_epoch,
revision: mut cycle_revision,
} = load_scanner_cycle_state_for_startup(store.clone()).await
else {
panic!("valid persisted cycle state should load");
};
let (usage_floor, startup) = persisted_usage_floor_for_startup(store.clone(), true)
.await
.expect("stably missing usage floor should admit a bootstrap marker");
assert_eq!(startup, PersistedUsageFloorStartup::Missing);
let (allow_bootstrap_pending, reset_bootstrap_cycle_on_conflict) =
prepare_cycle_for_usage_floor_bootstrap(&mut cycle_info, usage_floor, startup);
apply_persisted_usage_floor(&mut cycle_info, &mut persisted_epoch, usage_floor);
initialize_usage_baseline_bootstrap(store.clone())
.await
.expect("missing usage floor should publish a pending marker");
assert!(
claim_scanner_leadership(
&CancellationToken::new(),
store.clone(),
&mut cycle_info,
&mut cycle_revision,
&mut persisted_epoch,
allow_bootstrap_pending,
reset_bootstrap_cycle_on_conflict,
)
.await
);
let persisted_cycle = read_config(store.clone(), &DATA_USAGE_BLOOM_NAME_PATH)
.await
.expect("rebuilt cycle state should be persisted");
let (persisted_cycle, persisted_cycle_epoch) =
decode_scanner_cycle_state(&persisted_cycle).expect("rebuilt cycle state should decode");
assert_eq!(persisted_cycle.next, 0);
assert_eq!(persisted_cycle_epoch, 5);
let pending = read_config(store, DATA_USAGE_OBJ_NAME_PATH.as_str())
.await
.expect("fenced bootstrap marker should remain persisted");
let pending = serde_json::from_slice::<DataUsageInfo>(&pending).expect("bootstrap marker should decode");
assert!(data_usage_info_is_bootstrap_pending(&pending));
assert_eq!(pending.scanner_epoch, Some(5));
assert!(!data_usage_info_has_persisted_baseline_identity(&pending));
}
#[tokio::test]
async fn scanner_usage_bootstrap_allows_first_bucket_to_win_startup() {
let (_temp_dir, store) = setup_scanner_cycle_store_with_usage_baseline(false).await; let (_temp_dir, store) = setup_scanner_cycle_store_with_usage_baseline(false).await;
let cycle = CurrentCycle::default();
let revision = DataUsageCacheRevision::Missing;
store store
.make_bucket("first-user-bucket", &crate::storage_api::scan::MakeBucketOptions::default()) .make_bucket("first-user-bucket", &crate::storage_api::scan::MakeBucketOptions::default())
.await .await
.expect("test bucket should be created"); .expect("test bucket should be created");
assert!(scanner_may_bootstrap_missing_usage_floor(&cycle, 0, &revision));
assert_eq!( assert_eq!(
persisted_usage_floor_for_startup(store.clone(), true) persisted_usage_floor_for_startup(store.clone(), true)
.await .await
@@ -2235,14 +2332,14 @@ async fn scanner_pristine_bootstrap_allows_first_bucket_to_win_startup() {
.1, .1,
PersistedUsageFloorStartup::Missing PersistedUsageFloorStartup::Missing
); );
initialize_pristine_usage_baseline(store.clone()) initialize_usage_baseline_bootstrap(store.clone())
.await .await
.expect("first startup should persist its pending marker"); .expect("first startup should persist its pending marker");
let pending = read_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()) let pending = read_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
.await .await
.expect("pending marker should be stored"); .expect("pending marker should be stored");
let pending = serde_json::from_slice::<DataUsageInfo>(&pending).expect("pending marker should decode"); let pending = serde_json::from_slice::<DataUsageInfo>(&pending).expect("pending marker should decode");
assert!(data_usage_info_is_pristine_bootstrap_pending(&pending)); assert!(data_usage_info_is_bootstrap_pending(&pending));
assert!(!data_usage_info_has_persisted_baseline_identity(&pending)); assert!(!data_usage_info_has_persisted_baseline_identity(&pending));
assert_eq!( assert_eq!(
persisted_usage_floor_for_startup(store.clone(), false) persisted_usage_floor_for_startup(store.clone(), false)
@@ -2553,7 +2650,7 @@ async fn test_leadership_claim_preserves_usage_epoch_floor_across_old_epoch_conf
); );
let mut persisted_epoch = 8; let mut persisted_epoch = 8;
assert!(claim_scanner_leadership(&ctx, store.clone(), &mut cycle, &mut revision, &mut persisted_epoch, false).await); assert!(claim_scanner_leadership(&ctx, store.clone(), &mut cycle, &mut revision, &mut persisted_epoch, false, false).await);
let state = read_config(store.clone(), &DATA_USAGE_BLOOM_NAME_PATH) let state = read_config(store.clone(), &DATA_USAGE_BLOOM_NAME_PATH)
.await .await
@@ -2565,6 +2662,42 @@ async fn test_leadership_claim_preserves_usage_epoch_floor_across_old_epoch_conf
assert_eq!(store.put_counts.lock().await.get(&key), Some(&3)); assert_eq!(store.put_counts.lock().await.get(&key), Some(&3));
} }
#[tokio::test]
async fn unfenced_usage_bootstrap_discards_old_epoch_conflict_progress() {
let store = Arc::new(MemoryConfigStore::default());
let ctx = CancellationToken::new();
let mut revision = DataUsageCacheRevision::Missing;
let mut cycle = CurrentCycle {
next: 12,
..Default::default()
};
assert!(persist_scanner_cycle_state(&ctx, store.clone(), &mut cycle, &mut revision, 1).await);
initialize_usage_baseline_bootstrap(store.clone())
.await
.expect("missing usage floor should publish a pending marker");
cycle = CurrentCycle::default();
let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str());
let stale_cycle = CurrentCycle {
next: 14,
..Default::default()
};
store.interleaving_puts.lock().await.insert(
key,
(2, encode_scanner_cycle_state(&stale_cycle, 1).expect("stale cycle state should encode")),
);
let mut persisted_epoch = 1;
assert!(claim_scanner_leadership(&ctx, store.clone(), &mut cycle, &mut revision, &mut persisted_epoch, true, true).await);
let state = read_config(store, &DATA_USAGE_BLOOM_NAME_PATH)
.await
.expect("rebuilt leadership claim should remain persisted");
let (claimed_cycle, claimed_epoch) = decode_scanner_cycle_state(&state).expect("claimed cycle state should decode");
assert_eq!(claimed_cycle.next, 0);
assert_eq!(claimed_epoch, 2);
}
#[tokio::test] #[tokio::test]
async fn test_leadership_claim_rejects_terminal_epoch() { async fn test_leadership_claim_rejects_terminal_epoch() {
let store = Arc::new(MemoryConfigStore::default()); let store = Arc::new(MemoryConfigStore::default());
@@ -2576,7 +2709,7 @@ async fn test_leadership_claim_rejects_terminal_epoch() {
}; };
let mut persisted_epoch = u64::MAX - 1; let mut persisted_epoch = u64::MAX - 1;
assert!(!claim_scanner_leadership(&ctx, store.clone(), &mut cycle, &mut revision, &mut persisted_epoch, false).await); assert!(!claim_scanner_leadership(&ctx, store.clone(), &mut cycle, &mut revision, &mut persisted_epoch, false, false).await);
assert_eq!(persisted_epoch, u64::MAX - 1); assert_eq!(persisted_epoch, u64::MAX - 1);
assert!(read_config(store, &DATA_USAGE_BLOOM_NAME_PATH).await.is_err()); assert!(read_config(store, &DATA_USAGE_BLOOM_NAME_PATH).await.is_err());
} }
@@ -2592,64 +2725,64 @@ async fn scanner_defers_leadership_when_usage_snapshots_are_stably_absent() {
}; };
let mut persisted_epoch = 0; let mut persisted_epoch = 0;
assert!(!claim_scanner_leadership(&ctx, store.clone(), &mut cycle, &mut revision, &mut persisted_epoch, false).await); assert!(!claim_scanner_leadership(&ctx, store.clone(), &mut cycle, &mut revision, &mut persisted_epoch, false, false).await);
assert!(read_config(store.clone(), &DATA_USAGE_BLOOM_NAME_PATH).await.is_err()); assert!(read_config(store.clone(), &DATA_USAGE_BLOOM_NAME_PATH).await.is_err());
assert!(read_config(store, DATA_USAGE_OBJ_NAME_PATH.as_str()).await.is_err()); assert!(read_config(store, DATA_USAGE_OBJ_NAME_PATH.as_str()).await.is_err());
} }
#[tokio::test] #[tokio::test]
async fn pristine_usage_bootstrap_pending_unblocks_first_leadership_claim() { async fn usage_bootstrap_pending_unblocks_first_leadership_claim() {
let store = Arc::new(MemoryConfigStore::default()); let store = Arc::new(MemoryConfigStore::default());
initialize_pristine_usage_baseline(store.clone()) initialize_usage_baseline_bootstrap(store.clone())
.await .await
.expect("verified pristine startup should publish its pending marker"); .expect("verified missing state should publish its pending marker");
let ctx = CancellationToken::new(); let ctx = CancellationToken::new();
let mut revision = DataUsageCacheRevision::Missing; let mut revision = DataUsageCacheRevision::Missing;
let mut cycle = CurrentCycle::default(); let mut cycle = CurrentCycle::default();
let mut persisted_epoch = 0; let mut persisted_epoch = 0;
assert!(!claim_scanner_leadership(&ctx, store.clone(), &mut cycle, &mut revision, &mut persisted_epoch, false,).await); assert!(!claim_scanner_leadership(&ctx, store.clone(), &mut cycle, &mut revision, &mut persisted_epoch, false, false,).await);
assert!(read_config(store.clone(), &DATA_USAGE_BLOOM_NAME_PATH).await.is_err()); assert!(read_config(store.clone(), &DATA_USAGE_BLOOM_NAME_PATH).await.is_err());
assert!(claim_scanner_leadership(&ctx, store.clone(), &mut cycle, &mut revision, &mut persisted_epoch, true).await); assert!(claim_scanner_leadership(&ctx, store.clone(), &mut cycle, &mut revision, &mut persisted_epoch, true, true).await);
let usage = read_config(store, DATA_USAGE_OBJ_NAME_PATH.as_str()) let usage = read_config(store, DATA_USAGE_OBJ_NAME_PATH.as_str())
.await .await
.expect("leadership claim should fence the pristine baseline"); .expect("leadership claim should fence the bootstrap marker");
let usage = serde_json::from_slice::<DataUsageInfo>(&usage).expect("pristine bootstrap marker should remain valid"); let usage = serde_json::from_slice::<DataUsageInfo>(&usage).expect("bootstrap marker should remain valid");
assert!(data_usage_info_is_pristine_bootstrap_pending(&usage)); assert!(data_usage_info_is_bootstrap_pending(&usage));
assert!(!data_usage_info_has_persisted_baseline_identity(&usage)); assert!(!data_usage_info_has_persisted_baseline_identity(&usage));
assert_eq!(usage.scanner_epoch, Some(1)); assert_eq!(usage.scanner_epoch, Some(1));
} }
#[tokio::test] #[tokio::test]
async fn existing_pristine_usage_bootstrap_is_resumed_after_restart() { async fn existing_usage_bootstrap_is_resumed_after_restart() {
let store = Arc::new(MemoryConfigStore::default()); let store = Arc::new(MemoryConfigStore::default());
initialize_pristine_usage_baseline(store.clone()) initialize_usage_baseline_bootstrap(store.clone())
.await .await
.expect("verified pristine startup should publish its pending marker"); .expect("verified missing state should publish its pending marker");
let (floor, state) = persisted_usage_floor_for_startup(store.clone(), false) let (floor, state) = persisted_usage_floor_for_startup(store.clone(), false)
.await .await
.expect("restart should recognize the pending pristine bootstrap"); .expect("restart should recognize the pending usage bootstrap");
assert_eq!(floor, PersistedUsageFloor::default()); assert_eq!(floor, PersistedUsageFloor::default());
assert_eq!(state, PersistedUsageFloorStartup::BootstrapPending); assert_eq!(state, PersistedUsageFloorStartup::BootstrapPending);
assert!(persisted_usage_floor(store).await.is_err()); assert!(persisted_usage_floor(store).await.is_err());
} }
#[tokio::test] #[tokio::test]
async fn pristine_usage_bootstrap_reconciles_post_commit_error() { async fn usage_bootstrap_reconciles_post_commit_error() {
let store = Arc::new(MemoryConfigStore::default()); let store = Arc::new(MemoryConfigStore::default());
let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()); let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str());
store.error_after_commit_put_number.lock().await.insert(key, 1); store.error_after_commit_put_number.lock().await.insert(key, 1);
initialize_pristine_usage_baseline(store.clone()) initialize_usage_baseline_bootstrap(store.clone())
.await .await
.expect("a committed pending marker should reconcile after a lost response"); .expect("a committed pending marker should reconcile after a lost response");
let usage = read_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()) let usage = read_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
.await .await
.expect("the reconciled pending marker should remain"); .expect("the reconciled pending marker should remain");
let usage = serde_json::from_slice::<DataUsageInfo>(&usage).expect("pending marker should decode"); let usage = serde_json::from_slice::<DataUsageInfo>(&usage).expect("pending marker should decode");
assert!(data_usage_info_is_pristine_bootstrap_pending(&usage)); assert!(data_usage_info_is_bootstrap_pending(&usage));
assert!(!data_usage_info_has_persisted_baseline_identity(&usage)); assert!(!data_usage_info_has_persisted_baseline_identity(&usage));
assert_eq!( assert_eq!(
persisted_usage_floor_for_startup(store.clone(), false) persisted_usage_floor_for_startup(store.clone(), false)
@@ -2662,7 +2795,7 @@ async fn pristine_usage_bootstrap_reconciles_post_commit_error() {
} }
#[tokio::test] #[tokio::test]
async fn pristine_usage_bootstrap_does_not_overwrite_concurrent_replacement() { async fn usage_bootstrap_does_not_overwrite_concurrent_replacement() {
let store = Arc::new(MemoryConfigStore::default()); let store = Arc::new(MemoryConfigStore::default());
let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()); let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str());
let replacement = serde_json::to_vec(&complete_usage_with_bucket_count(None, 1)).expect("replacement should encode"); let replacement = serde_json::to_vec(&complete_usage_with_bucket_count(None, 1)).expect("replacement should encode");
@@ -2672,7 +2805,7 @@ async fn pristine_usage_bootstrap_does_not_overwrite_concurrent_replacement() {
.await .await
.insert(key, (1, replacement.clone())); .insert(key, (1, replacement.clone()));
initialize_pristine_usage_baseline(store.clone()) initialize_usage_baseline_bootstrap(store.clone())
.await .await
.expect("the bootstrap write completed before the replacement"); .expect("the bootstrap write completed before the replacement");
assert_eq!( assert_eq!(
@@ -2698,7 +2831,7 @@ async fn leadership_claim_defers_on_corrupt_usage_baseline_without_bloom_write()
}; };
let mut persisted_epoch = 0; let mut persisted_epoch = 0;
assert!(!claim_scanner_leadership(&ctx, store.clone(), &mut cycle, &mut revision, &mut persisted_epoch, false).await); assert!(!claim_scanner_leadership(&ctx, store.clone(), &mut cycle, &mut revision, &mut persisted_epoch, false, false).await);
assert!(read_config(store, &DATA_USAGE_BLOOM_NAME_PATH).await.is_err()); assert!(read_config(store, &DATA_USAGE_BLOOM_NAME_PATH).await.is_err());
} }
@@ -2718,7 +2851,7 @@ async fn leadership_claim_defers_on_unidentified_usage_baseline_without_bloom_wr
}; };
let mut persisted_epoch = 0; let mut persisted_epoch = 0;
assert!(!claim_scanner_leadership(&ctx, store.clone(), &mut cycle, &mut revision, &mut persisted_epoch, false).await); assert!(!claim_scanner_leadership(&ctx, store.clone(), &mut cycle, &mut revision, &mut persisted_epoch, false, false).await);
assert!(read_config(store, &DATA_USAGE_BLOOM_NAME_PATH).await.is_err()); assert!(read_config(store, &DATA_USAGE_BLOOM_NAME_PATH).await.is_err());
} }
@@ -2740,7 +2873,7 @@ async fn test_leadership_claim_confirms_commit_after_returned_error() {
let mut persisted_epoch = 0; let mut persisted_epoch = 0;
seed_usage_snapshot_for_leadership_claim(&store).await; seed_usage_snapshot_for_leadership_claim(&store).await;
assert!(claim_scanner_leadership(&ctx, store.clone(), &mut cycle, &mut revision, &mut persisted_epoch, false).await); assert!(claim_scanner_leadership(&ctx, store.clone(), &mut cycle, &mut revision, &mut persisted_epoch, false, false).await);
let state = read_config(store.clone(), &DATA_USAGE_BLOOM_NAME_PATH) let state = read_config(store.clone(), &DATA_USAGE_BLOOM_NAME_PATH)
.await .await
@@ -2796,7 +2929,7 @@ async fn test_leadership_claim_usage_fence_rejects_old_inflight_writer() {
..Default::default() ..Default::default()
}; };
let mut persisted_epoch = 4; let mut persisted_epoch = 4;
assert!(claim_scanner_leadership(&ctx, store.clone(), &mut cycle, &mut revision, &mut persisted_epoch, false).await); assert!(claim_scanner_leadership(&ctx, store.clone(), &mut cycle, &mut revision, &mut persisted_epoch, false, false).await);
let (fenced_data, fenced_revision) = read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()) let (fenced_data, fenced_revision) = read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
.await .await
@@ -2862,6 +2995,7 @@ async fn cycle_budget_lease_takeover_rejects_old_generation() {
&mut replacement_revision, &mut replacement_revision,
&mut replacement_epoch, &mut replacement_epoch,
false, false,
false,
) )
.await .await
); );