gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-09-02 10:18:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(scanner): fence system metadata publication (#6444)

Browse Source 
* feat(scanner): fence usage publication during data movement

* fix(scanner): detect movement refresh state changes

* fix(scanner): fence publication during data movement

* fix(scanner): close movement epoch publication races

* fix(scanner): fence movement-sensitive publication paths

* fix(scanner): fence cache and heal recovery paths

* fix(scanner): carry publication epoch through scan cycle

* fix(scanner): recheck remote cache epoch after save

* fix(scanner): recheck local cache epoch before publish

* fix(scanner): fence data usage writers and baseline

* fix(scanner): expose decommission activity to publication fence

* fix(scanner): release publication gate before reads

* fix(scanner): complete publication fence integration

* fix(scanner): avoid empty usage baseline publication

* chore(scanner): gate test-only helpers

* fix: use decommission canceler in reload test

---------

Co-authored-by: houseme <housemecn@gmail.com>
This commit is contained in:
cxymds
2026-08-23 17:28:21 +08:00
committed by GitHub
parent 9cda615519
commit e196a134cc
26 changed files with 2465 additions and 370 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/scanner/src/scanner/cycle_state.rs
+293 -53
View File
@@ -162,6 +162,8 @@ pub(crate) enum ScannerCycleStateStartup {
enum CycleRecoveryMarkerReadError {
#[error("cycle recovery marker backend read failed: {0}")]
Backend(#[source] EcstoreError),
#[error("cycle recovery marker publication is blocked by data movement")]
PublicationBlocked,
#[error("invalid cycle recovery marker: {0}")]
Invalid(&'static str),
#[error("cycle recovery marker revision changed while publishing")]
@@ -326,12 +328,13 @@ fn cycle_state_generation_and_epoch(buf: &[u8]) -> (u64, u64) {
}
async fn persist_cycle_recovery_marker(
storeapi: Arc<impl ScannerObjectIO>,
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
primary_revision: &DataUsageCacheRevision,
generation: u64,
leader_epoch: u64,
classification: &'static str,
reason: &'static str,
expected_epoch: u64,
) -> Result<ScannerCycleRecoveryMarker, CycleRecoveryMarkerReadError> {
let now = unix_now_secs();
let (existing, existing_revision) = match read_cycle_recovery_marker_bytes(storeapi.clone()).await {
@@ -370,6 +373,9 @@ async fn persist_cycle_recovery_marker(
state: "blocked".to_string(),
};
let bytes = serde_json::to_vec(&marker).map_err(|_| CycleRecoveryMarkerReadError::Invalid("marker serialization failed"))?;
let Some(_publication_admission) = scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch).await else {
return Err(CycleRecoveryMarkerReadError::PublicationBlocked);
};
let save_result = save_config_with_preconditions(
storeapi.clone(),
DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(),
@@ -478,7 +484,7 @@ async fn read_cycle_recovery_marker_revision(
}
async fn quarantine_invalid_cycle_state(
storeapi: Arc<impl ScannerObjectIO>,
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
revision: &DataUsageCacheRevision,
buf: &[u8],
) -> ScannerCycleStateStartup {
@@ -488,7 +494,7 @@ async fn quarantine_invalid_cycle_state(
}
async fn quarantine_invalid_cycle_state_with_reason(
storeapi: Arc<impl ScannerObjectIO>,
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
revision: &DataUsageCacheRevision,
generation: u64,
leader_epoch: u64,
@@ -515,7 +521,19 @@ async fn quarantine_invalid_cycle_state_with_reason(
reason: Some(reason.to_string()),
};
set_scanner_cycle_recovery_status(base_status);
match persist_cycle_recovery_marker(storeapi, revision, generation, leader_epoch, classification, reason).await {
let Some(expected_epoch) = scanner_publication_epoch(storeapi.clone()).await else {
set_scanner_cycle_recovery_status(recovery_status(
"transient",
Some("cycle recovery marker publication is blocked by data movement"),
true,
));
return ScannerCycleStateStartup::Transient(ScannerError::Other(
"cycle recovery marker publication is blocked by data movement".to_string(),
));
};
match persist_cycle_recovery_marker(storeapi, revision, generation, leader_epoch, classification, reason, expected_epoch)
.await
{
Ok(marker) => set_scanner_cycle_recovery_status(recovery_status_from_marker(&marker, "blocked")),
Err(CycleRecoveryMarkerReadError::Backend(_)) => {
// Keep the poison object untouched and retry marker creation with the
@@ -524,6 +542,16 @@ async fn quarantine_invalid_cycle_state_with_reason(
"failed to persist scanner cycle recovery marker".to_string(),
));
}
Err(CycleRecoveryMarkerReadError::PublicationBlocked) => {
set_scanner_cycle_recovery_status(recovery_status(
"transient",
Some("cycle recovery marker publication is blocked by data movement"),
true,
));
return ScannerCycleStateStartup::Transient(ScannerError::Other(
"cycle recovery marker publication is blocked by data movement".to_string(),
));
}
Err(CycleRecoveryMarkerReadError::Conflict) => {
set_scanner_cycle_recovery_status(recovery_status(
"transient",
@@ -546,16 +574,18 @@ async fn mark_cycle_recovery_cleanup_pending(
storeapi: Arc<ECStore>,
mut marker: ScannerCycleRecoveryMarker,
marker_revision: &DataUsageCacheRevision,
expected_epoch: u64,
) -> Result<(ScannerCycleRecoveryMarker, DataUsageCacheRevision), ScannerError> {
marker.state = "cleanup-pending".to_string();
marker.last_attempt_at_unix_secs = unix_now_secs();
let bytes = serde_json::to_vec(&marker)
.map_err(|err| ScannerError::Other(format!("failed to encode cycle recovery marker: {err}")))?;
let info = save_config_with_preconditions(
let info = save_config_with_publication_admission_for_epoch(
storeapi.clone(),
DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(),
bytes,
marker_revision.preconditions(),
expected_epoch,
)
.await
.map_err(|err| ScannerError::Other(format!("failed to mark cycle recovery cleanup pending: {err}")))?;
@@ -567,7 +597,9 @@ async fn mark_cycle_recovery_cleanup_pending(
Ok((marker, revision))
}
pub(crate) async fn load_scanner_cycle_state_for_startup(storeapi: Arc<impl ScannerObjectIO>) -> ScannerCycleStateStartup {
pub(crate) async fn load_scanner_cycle_state_for_startup(
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
) -> ScannerCycleStateStartup {
let marker = match read_cycle_recovery_marker_bytes(storeapi.clone()).await {
Ok((None, _)) => None,
Ok((Some(data), marker_revision)) => match serde_json::from_slice::<ScannerCycleRecoveryMarker>(&data) {
@@ -594,6 +626,16 @@ pub(crate) async fn load_scanner_cycle_state_for_startup(storeapi: Arc<impl Scan
"failed to read scanner cycle recovery marker: {err}"
)));
}
Err(CycleRecoveryMarkerReadError::PublicationBlocked) => {
set_scanner_cycle_recovery_status(recovery_status(
"transient",
Some("cycle recovery marker publication is blocked by data movement"),
true,
));
return ScannerCycleStateStartup::Transient(ScannerError::Other(
"cycle recovery marker publication is blocked by data movement".to_string(),
));
}
Err(CycleRecoveryMarkerReadError::Invalid(reason)) => {
set_scanner_cycle_recovery_status(recovery_status("recovery-required", Some(reason), false));
return ScannerCycleStateStartup::Blocked;
@@ -750,6 +792,10 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
return Err(ScannerError::Other("scanner leader lock was lost before recovery reset".to_string()));
}
let Some(reset_epoch) = scanner_publication_epoch(storeapi.clone()).await else {
return Err(ScannerError::Other("scanner recovery reset is blocked by data movement".to_string()));
};
let (marker_data, marker_revision, marker_body_invalid) = match read_cycle_recovery_marker_bytes(storeapi.clone()).await {
Ok((marker_data, marker_revision)) => (marker_data, marker_revision, false),
Err(CycleRecoveryMarkerReadError::Invalid(_)) => {
@@ -835,7 +881,7 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
};
if let Some((primary_cycle, primary_epoch)) = primary_state {
let (cleanup_marker, cleanup_marker_revision) =
mark_cycle_recovery_cleanup_pending(storeapi.clone(), marker.clone(), &marker_revision).await?;
mark_cycle_recovery_cleanup_pending(storeapi.clone(), marker.clone(), &marker_revision, reset_epoch).await?;
set_scanner_cycle_recovery_status(recovery_status_from_marker(&cleanup_marker, "cleanup-pending"));
let usage_floor = persisted_usage_floor(storeapi.clone()).await?;
let fence_epoch = primary_epoch
@@ -855,14 +901,21 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
"preserved scanner cycle state exceeds the bounded object size".to_string(),
));
}
let preserved_info = save_config_with_preconditions(
let preserved_info = save_config_with_publication_admission_for_epoch(
storeapi.clone(),
DATA_USAGE_BLOOM_NAME_PATH.as_str(),
preserved_data,
primary_revision.preconditions(),
reset_epoch,
)
.await
.map_err(|err| ScannerError::Other(format!("failed to fence preserved scanner cycle state: {err}")))?;
.map_err(|err| {
if scanner_publication_epoch_changed(&err) {
ScannerError::Other("scanner recovery reset deferred by a movement epoch change".to_string())
} else {
ScannerError::Other(format!("failed to fence preserved scanner cycle state: {err}"))
}
})?;
let preserved_revision = preserved_info
.etag
.filter(|etag| !etag.is_empty())
@@ -873,7 +926,7 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
"scanner leader lock was lost after fencing newer cycle state".to_string(),
));
}
fence_scanner_usage_epoch(&ctx, storeapi.clone(), fence_epoch)
fence_scanner_usage_epoch_with_expected_epoch(&ctx, storeapi.clone(), fence_epoch, Some(reset_epoch))
.await
.map_err(|err| ScannerError::Other(format!("failed to fence preserved scanner usage epoch: {err}")))?;
if guard.is_lock_lost() {
@@ -889,20 +942,27 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
"scanner cycle state changed before recovery marker cleanup".to_string(),
));
}
storeapi
.delete_config_object(
RUSTFS_META_BUCKET,
DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(),
ScannerObjectOptions {
// This is one exact metadata object. Prefix-delete mode
// bypasses HTTP preconditions in the ECStore path.
delete_prefix: false,
http_preconditions: Some(cleanup_marker_revision.preconditions()),
..Default::default()
},
)
.await
.map_err(|err| ScannerError::Other(format!("failed to clear stale cycle recovery marker: {err}")))?;
delete_config_with_publication_admission_for_epoch(
storeapi.clone(),
RUSTFS_META_BUCKET,
DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(),
ScannerObjectOptions {
// This is one exact metadata object. Prefix-delete mode
// bypasses HTTP preconditions in the ECStore path.
delete_prefix: false,
http_preconditions: Some(cleanup_marker_revision.preconditions()),
..Default::default()
},
reset_epoch,
)
.await
.map_err(|err| {
if scanner_publication_epoch_changed(&err) {
ScannerError::Other("scanner recovery reset deferred by a movement epoch change".to_string())
} else {
ScannerError::Other(format!("failed to clear stale cycle recovery marker: {err}"))
}
})?;
set_scanner_cycle_recovery_status(recovery_status("healthy", None, false));
super::notify_scanner_cycle_recovery_wake();
return Ok(());
@@ -946,16 +1006,23 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
let (marker, marker_revision) = if marker.state == "cleanup-pending" {
(marker, marker_revision)
} else {
mark_cycle_recovery_cleanup_pending(storeapi.clone(), marker, &marker_revision).await?
mark_cycle_recovery_cleanup_pending(storeapi.clone(), marker, &marker_revision, reset_epoch).await?
};
let rebuilt_info = save_config_with_preconditions(
let rebuilt_info = save_config_with_publication_admission_for_epoch(
storeapi.clone(),
DATA_USAGE_BLOOM_NAME_PATH.as_str(),
data,
primary_revision.preconditions(),
reset_epoch,
)
.await
.map_err(|err| ScannerError::Other(format!("failed to persist rebuilt scanner cycle state: {err}")))?;
.map_err(|err| {
if scanner_publication_epoch_changed(&err) {
ScannerError::Other("scanner recovery reset deferred by a movement epoch change".to_string())
} else {
ScannerError::Other(format!("failed to persist rebuilt scanner cycle state: {err}"))
}
})?;
let rebuilt_revision = rebuilt_info
.etag
.filter(|etag| !etag.is_empty())
@@ -965,7 +1032,8 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
"scanner leader lock was lost after rebuilding cycle state".to_string(),
));
}
if let Err(err) = fence_scanner_usage_epoch(&ctx, storeapi.clone(), leader_epoch).await {
if let Err(err) = fence_scanner_usage_epoch_with_expected_epoch(&ctx, storeapi.clone(), leader_epoch, Some(reset_epoch)).await
{
set_scanner_cycle_recovery_status(ScannerCycleRecoveryStatus {
path: DATA_USAGE_BLOOM_NAME_PATH.clone(),
quarantine_path: Some(DATA_USAGE_BLOOM_RECOVERY_PATH.clone()),
@@ -1034,20 +1102,40 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
));
}
if let Err(err) = storeapi
.delete_config_object(
RUSTFS_META_BUCKET,
DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(),
ScannerObjectOptions {
// This is one exact metadata object. Prefix-delete mode
// bypasses HTTP preconditions in the ECStore path.
delete_prefix: false,
http_preconditions: Some(marker_revision.preconditions()),
..Default::default()
},
)
.await
if let Err(err) = delete_config_with_publication_admission_for_epoch(
storeapi.clone(),
RUSTFS_META_BUCKET,
DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(),
ScannerObjectOptions {
// This is one exact metadata object. Prefix-delete mode
// bypasses HTTP preconditions in the ECStore path.
delete_prefix: false,
http_preconditions: Some(marker_revision.preconditions()),
..Default::default()
},
reset_epoch,
)
.await
{
if scanner_publication_epoch_changed(&err) {
set_scanner_cycle_recovery_status(ScannerCycleRecoveryStatus {
path: DATA_USAGE_BLOOM_NAME_PATH.clone(),
quarantine_path: Some(DATA_USAGE_BLOOM_RECOVERY_PATH.clone()),
state: "cleanup-pending".to_string(),
classification: Some(marker.classification.clone()),
primary_revision: Some(rebuilt_revision.clone()),
generation: Some(next),
leader_epoch: Some(leader_epoch),
retry_count: marker.retry_count,
max_retries: MAX_SCANNER_CYCLE_RECOVERY_RETRIES,
retryable: true,
reason: Some("movement epoch changed before recovery marker cleanup".to_string()),
..Default::default()
});
return Err(ScannerError::Other(
"scanner recovery reset deferred by a movement epoch change".to_string(),
));
}
set_scanner_cycle_recovery_status(ScannerCycleRecoveryStatus {
path: DATA_USAGE_BLOOM_NAME_PATH.clone(),
quarantine_path: Some(DATA_USAGE_BLOOM_RECOVERY_PATH.clone()),
@@ -1209,8 +1297,14 @@ pub(super) fn advance_scanner_cycle(cycle_info: &mut CurrentCycle) -> Result<(),
Ok(())
}
pub(super) async fn persisted_usage_floor(storeapi: Arc<impl ScannerObjectIO>) -> Result<PersistedUsageFloor, ScannerError> {
pub(super) async fn persisted_usage_floor(
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
) -> Result<PersistedUsageFloor, ScannerError> {
let Some(read_epoch) = scanner_publication_epoch(storeapi.clone()).await else {
return Err(ScannerError::Other("scanner usage floor read is blocked by data movement".to_string()));
};
let mut floor = PersistedUsageFloor::default();
let mut found_any = false;
let update_floor = |floor: &mut PersistedUsageFloor, usage: &DataUsageInfo, path: &str| -> Result<(), ScannerError> {
floor.leader_epoch = floor.leader_epoch.max(usage.scanner_epoch.unwrap_or_default());
if let Some(completed_cycle) = usage.scanner_cycle {
@@ -1229,6 +1323,11 @@ pub(super) async fn persisted_usage_floor(storeapi: Arc<impl ScannerObjectIO>) -
let usage = serde_json::from_slice::<DataUsageInfo>(&data).map_err(|err| {
ScannerError::Other(format!("failed to decode scanner usage floor from {primary_path}: {err}"))
})?;
if !data_usage_info_has_persisted_baseline_identity(&usage) {
return Err(ScannerError::Other(format!(
"scanner usage floor from {primary_path} has no persisted baseline identity"
)));
}
let epoch = usage.scanner_epoch.unwrap_or_default();
update_floor(&mut floor, &usage, primary_path)?;
Some(epoch)
@@ -1247,6 +1346,11 @@ pub(super) async fn persisted_usage_floor(storeapi: Arc<impl ScannerObjectIO>) -
let usage = serde_json::from_slice::<DataUsageInfo>(&data).map_err(|err| {
ScannerError::Other(format!("failed to decode scanner usage floor from {backup_path}: {err}"))
})?;
if !data_usage_info_has_persisted_baseline_identity(&usage) {
return Err(ScannerError::Other(format!(
"scanner usage floor from {backup_path} has no persisted baseline identity"
)));
}
let backup_epoch = usage.scanner_epoch.unwrap_or_default();
// A backup write from an older leader may complete after the
// primary epoch has been fenced. It must not advance the startup
@@ -1263,9 +1367,21 @@ pub(super) async fn persisted_usage_floor(storeapi: Arc<impl ScannerObjectIO>) -
}
}
if any_found {
found_any = true;
break;
}
}
if !found_any {
return Err(ScannerError::Other(
"persisted scanner usage floor has no authoritative baseline".to_string(),
));
}
let Some(_publication_admission) = scanner_publication_admission_for_epoch(storeapi, read_epoch).await else {
return Err(ScannerError::Other(
"scanner usage floor changed while its epoch proof was being confirmed".to_string(),
));
};
Ok(floor)
}
@@ -1274,12 +1390,30 @@ pub(super) fn apply_persisted_usage_floor(cycle_info: &mut CurrentCycle, leader_
*leader_epoch = (*leader_epoch).max(floor.leader_epoch);
}
#[derive(Clone, Copy)]
pub(super) struct ScannerCycleFloorOptions {
pub(super) required_cycle: u64,
pub(super) expected_publication_epoch: Option<u64>,
}
#[cfg(test)]
pub(super) async fn persist_scanner_cycle_state(
ctx: &CancellationToken,
storeapi: Arc<impl ScannerObjectIO>,
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
cycle_info: &mut CurrentCycle,
revision: &mut DataUsageCacheRevision,
leader_epoch: u64,
) -> bool {
persist_scanner_cycle_state_for_epoch(ctx, storeapi, cycle_info, revision, leader_epoch, None).await
}
pub(super) async fn persist_scanner_cycle_state_for_epoch(
ctx: &CancellationToken,
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
cycle_info: &mut CurrentCycle,
revision: &mut DataUsageCacheRevision,
leader_epoch: u64,
expected_publication_epoch: Option<u64>,
) -> bool {
let buf = match encode_scanner_cycle_state(cycle_info, leader_epoch) {
Ok(buf) => buf,
@@ -1315,9 +1449,29 @@ pub(super) async fn persist_scanner_cycle_state(
#[cfg(test)]
notify_scanner_cycle_state_persist_test_hook(leader_epoch);
match save_config_with_preconditions(storeapi.clone(), &DATA_USAGE_BLOOM_NAME_PATH, buf.clone(), revision.preconditions())
.await
{
let Some(read_epoch) = scanner_publication_epoch(storeapi.clone()).await else {
return false;
};
if expected_publication_epoch.is_some_and(|expected| expected != read_epoch) {
return false;
}
let save_result = {
let Some(_publication_admission) = scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch).await else {
error!(
target: "rustfs::scanner",
event = EVENT_SCANNER_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
path = %&*DATA_USAGE_BLOOM_NAME_PATH,
state = "publication_admission_unavailable",
"Scanner state persistence skipped without movement admission"
);
return false;
};
save_config_with_preconditions(storeapi.clone(), &DATA_USAGE_BLOOM_NAME_PATH, buf.clone(), revision.preconditions())
.await
};
match save_result {
Ok(object_info) => {
let Some(etag) = object_info.etag.filter(|etag| !etag.is_empty()) else {
error!(
@@ -1345,6 +1499,13 @@ pub(super) async fn persist_scanner_cycle_state(
);
return false;
}
if let Some(expected_epoch) = expected_publication_epoch
&& scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch)
.await
.is_none()
{
return false;
}
debug!(
target: "rustfs::scanner",
event = EVENT_SCANNER_PERSIST_STATE,
@@ -1436,6 +1597,13 @@ pub(super) async fn persist_scanner_cycle_state(
}
if persisted_cycle.next >= cycle_info.next {
if let Some(expected_epoch) = expected_publication_epoch
&& scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch)
.await
.is_none()
{
return false;
}
*cycle_info = persisted_cycle;
debug!(
target: "rustfs::scanner",
@@ -1496,19 +1664,33 @@ pub(super) async fn persist_scanner_cycle_state(
false
}
#[cfg(test)]
pub(super) async fn finalize_partial_scan_cycle(
ctx: &CancellationToken,
storeapi: Arc<impl ScannerObjectIO>,
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
cycle_info: &mut CurrentCycle,
revision: &mut DataUsageCacheRevision,
leader_epoch: u64,
cycle_metrics_guard: &mut ScannerCycleMetricsGuard,
) -> bool {
finalize_partial_scan_cycle_for_epoch(ctx, storeapi, cycle_info, revision, leader_epoch, cycle_metrics_guard, None).await
}
pub(super) async fn finalize_partial_scan_cycle_for_epoch(
ctx: &CancellationToken,
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
cycle_info: &mut CurrentCycle,
revision: &mut DataUsageCacheRevision,
leader_epoch: u64,
cycle_metrics_guard: &mut ScannerCycleMetricsGuard,
expected_publication_epoch: Option<u64>,
) -> bool {
// A budget-limited cycle is deliberate pacing, not a failure. The cycle counter
// must still advance (and persist) because per-bucket next_cycle is stamped from
// it and compacted folders are only rescanned when their hash matches
// next_cycle % DATA_USAGE_UPDATE_DIR_CYCLES; a pinned counter starves lifecycle
// expiry and usage refresh on every folder outside the stuck window.
let previous_cycle_info = cycle_info.clone();
if let Err(err) = advance_scanner_cycle(cycle_info) {
error!(
target: "rustfs::scanner",
@@ -1524,28 +1706,69 @@ pub(super) async fn finalize_partial_scan_cycle(
}
cycle_info.current = 0;
global_metrics().clear_current_scan_mode();
let persisted = persist_scanner_cycle_state(ctx, storeapi, cycle_info, revision, leader_epoch).await;
let persisted = persist_scanner_cycle_state_for_epoch(
ctx,
storeapi.clone(),
cycle_info,
revision,
leader_epoch,
expected_publication_epoch,
)
.await;
if !persisted
&& let Some(expected_epoch) = expected_publication_epoch
&& scanner_publication_admission_for_epoch(storeapi, expected_epoch)
.await
.is_none()
{
*cycle_info = previous_cycle_info;
}
cycle_metrics_guard.finish(cycle_info.clone()).await;
persisted
}
#[cfg(test)]
pub(super) async fn persist_required_scanner_cycle_floor(
ctx: &CancellationToken,
storeapi: Arc<impl ScannerObjectIO>,
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
cycle_info: &mut CurrentCycle,
revision: &mut DataUsageCacheRevision,
leader_epoch: u64,
required_cycle: u64,
cycle_metrics_guard: &mut ScannerCycleMetricsGuard,
) -> bool {
if required_cycle <= cycle_info.current || required_cycle == u64::MAX {
persist_required_scanner_cycle_floor_for_epoch(
ctx,
storeapi,
cycle_info,
revision,
leader_epoch,
cycle_metrics_guard,
ScannerCycleFloorOptions {
required_cycle,
expected_publication_epoch: None,
},
)
.await
}
pub(super) async fn persist_required_scanner_cycle_floor_for_epoch(
ctx: &CancellationToken,
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
cycle_info: &mut CurrentCycle,
revision: &mut DataUsageCacheRevision,
leader_epoch: u64,
cycle_metrics_guard: &mut ScannerCycleMetricsGuard,
options: ScannerCycleFloorOptions,
) -> bool {
if options.required_cycle <= cycle_info.current || options.required_cycle == u64::MAX {
error!(
target: "rustfs::scanner",
event = EVENT_SCANNER_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
current_cycle = cycle_info.current,
required_cycle,
required_cycle = options.required_cycle,
state = "invalid_cache_cycle_floor",
"Scanner cache cycle floor is invalid"
);
@@ -1553,10 +1776,27 @@ pub(super) async fn persist_required_scanner_cycle_floor(
return false;
}
cycle_info.next = cycle_info.next.max(required_cycle);
let previous_cycle_info = cycle_info.clone();
cycle_info.next = cycle_info.next.max(options.required_cycle);
cycle_info.current = 0;
global_metrics().clear_current_scan_mode();
let persisted = persist_scanner_cycle_state(ctx, storeapi, cycle_info, revision, leader_epoch).await;
let persisted = persist_scanner_cycle_state_for_epoch(
ctx,
storeapi.clone(),
cycle_info,
revision,
leader_epoch,
options.expected_publication_epoch,
)
.await;
if !persisted
&& let Some(expected_epoch) = options.expected_publication_epoch
&& scanner_publication_admission_for_epoch(storeapi, expected_epoch)
.await
.is_none()
{
*cycle_info = previous_cycle_info;
}
cycle_metrics_guard.finish(cycle_info.clone()).await;
persisted
}
crates/scanner/src/scanner/heal_info.rs
+107 -18
View File
@@ -26,31 +26,90 @@ pub struct BackgroundHealInfo {
pub current_scan_mode: HealScanMode,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub(super) enum BackgroundHealInfoReadStatus {
ErasureSd,
Loaded,
Missing,
Blocked,
Transient,
Failed,
}
pub(super) struct BackgroundHealInfoRead {
pub(super) info: BackgroundHealInfo,
pub(super) expected_epoch: Option<u64>,
pub(super) status: BackgroundHealInfoReadStatus,
}
pub(super) fn classify_background_heal_read_error(error: &EcstoreError) -> BackgroundHealInfoReadStatus {
if matches!(error, EcstoreError::ConfigNotFound) {
BackgroundHealInfoReadStatus::Missing
} else {
BackgroundHealInfoReadStatus::Transient
}
}
pub(super) fn decode_background_heal_info(data: &[u8]) -> Result<BackgroundHealInfo, serde_json::Error> {
serde_json::from_slice(data)
}
/// Read background healing information from storage
pub async fn read_background_heal_info(storeapi: Arc<ECStore>) -> BackgroundHealInfo {
read_background_heal_info_with_epoch(storeapi).await.info
}
/// Read background healing information together with the movement epoch that
/// fenced the read. The epoch must be reused by the matching cycle update so a
/// missing-object default cannot be committed across a movement transition.
pub(super) async fn read_background_heal_info_with_epoch(storeapi: Arc<ECStore>) -> BackgroundHealInfoRead {
// Skip for ErasureSD setup
if scanner_is_erasure_sd().await {
return BackgroundHealInfo::default();
return BackgroundHealInfoRead {
info: BackgroundHealInfo::default(),
expected_epoch: None,
status: BackgroundHealInfoReadStatus::ErasureSd,
};
}
let expected_epoch = scanner_publication_epoch(storeapi.clone()).await;
if expected_epoch.is_none() {
return BackgroundHealInfoRead {
info: BackgroundHealInfo::default(),
expected_epoch,
status: BackgroundHealInfoReadStatus::Blocked,
};
}
// Get last healing information
match read_config(storeapi, &BACKGROUND_HEAL_INFO_PATH).await {
Ok(buf) => serde_json::from_slice::<BackgroundHealInfo>(&buf).unwrap_or_else(|e| {
error!(
target: "rustfs::scanner",
event = EVENT_SCANNER_BACKGROUND_HEAL_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_BACKGROUND_HEAL,
path = %&*BACKGROUND_HEAL_INFO_PATH,
state = "decode_failed",
error = %e,
"Scanner background heal decode failed"
);
BackgroundHealInfo::default()
}),
Ok(buf) => match decode_background_heal_info(&buf) {
Ok(info) => BackgroundHealInfoRead {
info,
expected_epoch,
status: BackgroundHealInfoReadStatus::Loaded,
},
Err(e) => {
error!(
target: "rustfs::scanner",
event = EVENT_SCANNER_BACKGROUND_HEAL_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_BACKGROUND_HEAL,
path = %&*BACKGROUND_HEAL_INFO_PATH,
state = "decode_failed",
error = %e,
"Scanner background heal decode failed"
);
BackgroundHealInfoRead {
info: BackgroundHealInfo::default(),
expected_epoch,
status: BackgroundHealInfoReadStatus::Failed,
}
}
},
Err(e) => {
// Only log if it's not a ConfigNotFound error
if e != EcstoreError::ConfigNotFound {
let status = classify_background_heal_read_error(&e);
if status == BackgroundHealInfoReadStatus::Transient {
warn!(
target: "rustfs::scanner",
event = EVENT_SCANNER_BACKGROUND_HEAL_STATE,
@@ -62,7 +121,11 @@ pub async fn read_background_heal_info(storeapi: Arc<ECStore>) -> BackgroundHeal
"Scanner background heal read failed"
);
}
BackgroundHealInfo::default()
BackgroundHealInfoRead {
info: BackgroundHealInfo::default(),
expected_epoch,
status,
}
}
}
}
@@ -70,6 +133,14 @@ pub async fn read_background_heal_info(storeapi: Arc<ECStore>) -> BackgroundHeal
/// Save background healing information to storage
#[instrument(skip(storeapi))]
pub async fn save_background_heal_info(storeapi: Arc<ECStore>, info: BackgroundHealInfo) {
save_background_heal_info_for_epoch(storeapi, info, None).await;
}
pub(super) async fn save_background_heal_info_for_epoch(
storeapi: Arc<ECStore>,
info: BackgroundHealInfo,
expected_epoch: Option<u64>,
) {
// Skip for ErasureSD setup
if scanner_is_erasure_sd().await {
return;
@@ -93,7 +164,25 @@ pub async fn save_background_heal_info(storeapi: Arc<ECStore>, info: BackgroundH
}
};
// Save configuration
// Save configuration only after storage-owned movement admission. The
// read path may return an in-memory default for a missing object, but a
// movement transition must not let that default become durable state.
let publication_admission = match expected_epoch {
Some(expected_epoch) => scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch).await,
None => storeapi.scanner_data_usage_publication_admission().await,
};
let Some(_publication_admission) = publication_admission else {
warn!(
target: "rustfs::scanner",
event = EVENT_SCANNER_BACKGROUND_HEAL_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_BACKGROUND_HEAL,
path = %&*BACKGROUND_HEAL_INFO_PATH,
state = "publication_admission_unavailable",
"Scanner background heal save skipped without movement admission"
);
return;
};
if let Err(e) = save_config(storeapi, &BACKGROUND_HEAL_INFO_PATH, data).await {
warn!(
target: "rustfs::scanner",
crates/scanner/src/scanner/leadership.rs
+119 -20
View File
@@ -61,16 +61,22 @@ pub(super) async fn reconcile_scanner_leadership_claim(
}
pub(super) fn decode_usage_snapshot_for_epoch_fence(data: &[u8], path: &str) -> Result<DataUsageInfo, ScannerError> {
serde_json::from_slice(data)
.map_err(|err| ScannerError::Other(format!("failed to decode scanner usage epoch fence from {path}: {err}")))
let usage: DataUsageInfo = serde_json::from_slice(data)
.map_err(|err| ScannerError::Other(format!("failed to decode scanner usage epoch fence from {path}: {err}")))?;
if !data_usage_info_has_persisted_baseline_identity(&usage) {
return Err(ScannerError::Other(format!(
"scanner usage epoch fence from {path} has no persisted baseline identity"
)));
}
Ok(usage)
}
pub(super) async fn usage_snapshot_for_epoch_fence(
storeapi: Arc<impl ScannerObjectIO>,
primary: Option<&[u8]>,
) -> Result<DataUsageInfo, ScannerError> {
) -> Result<Option<DataUsageInfo>, ScannerError> {
if let Some(primary) = primary {
return decode_usage_snapshot_for_epoch_fence(primary, DATA_USAGE_OBJ_NAME_PATH.as_str());
return decode_usage_snapshot_for_epoch_fence(primary, DATA_USAGE_OBJ_NAME_PATH.as_str()).map(Some);
}
let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str());
@@ -78,7 +84,7 @@ pub(super) async fn usage_snapshot_for_epoch_fence(
.await
.map_err(|err| ScannerError::Other(format!("failed to read scanner usage epoch fence backup: {err}")))?;
if let Some(backup) = backup.as_deref() {
return decode_usage_snapshot_for_epoch_fence(backup, &backup_path);
return decode_usage_snapshot_for_epoch_fence(backup, &backup_path).map(Some);
}
for path in [
@@ -89,26 +95,53 @@ pub(super) async fn usage_snapshot_for_epoch_fence(
.await
.map_err(|err| ScannerError::Other(format!("failed to read legacy scanner usage epoch fence: {err}")))?;
if let Some(legacy) = legacy.as_deref() {
return decode_usage_snapshot_for_epoch_fence(legacy, &path);
return decode_usage_snapshot_for_epoch_fence(legacy, &path).map(Some);
}
}
Ok(DataUsageInfo::default())
// A missing usage snapshot is an uninitialized state, not an empty
// snapshot. Leadership fencing may proceed without creating a plausible
// default; the first authoritative scanner publication will create it.
Ok(None)
}
pub(super) async fn fence_scanner_usage_epoch(
pub(super) async fn fence_scanner_usage_epoch_with_expected_epoch(
ctx: &CancellationToken,
storeapi: Arc<impl ScannerObjectIO>,
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
claimed_epoch: u64,
expected_publication_epoch: Option<u64>,
) -> Result<(), ScannerError> {
for retry in 0..=SCANNER_PERSIST_CAS_RETRIES {
if ctx.is_cancelled() {
return Err(ScannerError::Other("scanner leadership was cancelled before usage fencing".to_string()));
}
let Some(read_epoch) = scanner_publication_epoch(storeapi.clone()).await else {
return Err(ScannerError::Other(
"scanner usage epoch fence publication is blocked by data movement".to_string(),
));
};
if expected_publication_epoch.is_some_and(|expected| expected != read_epoch) {
if retry < SCANNER_PERSIST_CAS_RETRIES {
continue;
}
return Err(ScannerError::Other(
"scanner usage epoch fence changed while recovery reset was in progress".to_string(),
));
}
let (primary, revision) = read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
.await
.map_err(|err| ScannerError::Other(format!("failed to read scanner usage epoch fence: {err}")))?;
let mut usage = usage_snapshot_for_epoch_fence(storeapi.clone(), primary.as_deref()).await?;
let Some(mut usage) = usage_snapshot_for_epoch_fence(storeapi.clone(), primary.as_deref()).await? else {
let Some(_publication_admission) = scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch).await else {
if retry < SCANNER_PERSIST_CAS_RETRIES {
continue;
}
return Err(ScannerError::Other(
"scanner usage epoch fence changed while confirming a missing usage baseline".to_string(),
));
};
return Err(ScannerError::Other("authoritative scanner usage baseline is missing".to_string()));
};
match usage.scanner_epoch {
Some(epoch) if epoch > claimed_epoch => {
return Err(ScannerError::Other(format!(
@@ -122,9 +155,18 @@ pub(super) async fn fence_scanner_usage_epoch(
let data = serde_json::to_vec(&usage)
.map_err(|err| ScannerError::Other(format!("failed to encode scanner usage epoch fence: {err}")))?;
let save_result =
let save_result = {
let Some(_publication_admission) = scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch).await else {
if retry < SCANNER_PERSIST_CAS_RETRIES {
continue;
}
return Err(ScannerError::Other(
"scanner usage epoch fence changed while preparing its conditional write".to_string(),
));
};
save_config_with_preconditions(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), data, revision.preconditions())
.await;
.await
};
if save_result
.as_ref()
.ok()
@@ -165,10 +207,13 @@ pub(super) async fn fence_scanner_usage_epoch(
pub(super) async fn complete_scanner_leadership_claim(
ctx: &CancellationToken,
storeapi: Arc<impl ScannerObjectIO>,
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
claimed_epoch: u64,
expected_publication_epoch: Option<u64>,
) -> bool {
if let Err(err) = fence_scanner_usage_epoch(ctx, storeapi, claimed_epoch).await {
if let Err(err) =
fence_scanner_usage_epoch_with_expected_epoch(ctx, storeapi, claimed_epoch, expected_publication_epoch).await
{
error!(
target: "rustfs::scanner",
event = EVENT_SCANNER_PERSIST_STATE,
@@ -187,7 +232,7 @@ pub(super) async fn complete_scanner_leadership_claim(
pub(super) async fn claim_scanner_leadership(
ctx: &CancellationToken,
storeapi: Arc<impl ScannerObjectIO>,
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
cycle_info: &mut CurrentCycle,
revision: &mut DataUsageCacheRevision,
persisted_epoch: &mut u64,
@@ -226,15 +271,69 @@ pub(super) async fn claim_scanner_leadership(
};
let previous_revision = revision.clone();
let save_result =
let Some(read_epoch) = scanner_publication_epoch(storeapi.clone()).await else {
return false;
};
let (usage_primary, _) = match read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()).await {
Ok(result) => result,
Err(err) => {
error!(
target: "rustfs::scanner",
event = EVENT_SCANNER_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
state = "leader_usage_baseline_read_failed",
error = %err,
"Scanner leadership claim deferred because the usage baseline could not be read"
);
return false;
}
};
match usage_snapshot_for_epoch_fence(storeapi.clone(), usage_primary.as_deref()).await {
Ok(Some(_)) => {}
Ok(None) => {
warn!(
target: "rustfs::scanner",
event = EVENT_SCANNER_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
state = "leader_usage_baseline_missing",
"Scanner leadership claim deferred until a usage baseline is published"
);
return false;
}
Err(err) => {
error!(
target: "rustfs::scanner",
event = EVENT_SCANNER_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
state = "leader_usage_baseline_invalid",
error = %err,
"Scanner leadership claim deferred because the usage baseline is invalid"
);
return false;
}
}
let save_result = {
let Some(_publication_admission) = scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch).await else {
if retry < SCANNER_PERSIST_CAS_RETRIES {
continue;
}
return false;
};
save_config_with_preconditions(storeapi.clone(), &DATA_USAGE_BLOOM_NAME_PATH, data.clone(), revision.preconditions())
.await;
.await
};
match save_result {
Ok(object_info) => {
if let Some(etag) = object_info.etag.filter(|etag| !etag.is_empty()) {
*revision = DataUsageCacheRevision::Etag(etag);
*persisted_epoch = claimed_epoch;
return complete_scanner_leadership_claim(ctx, storeapi, claimed_epoch).await;
return complete_scanner_leadership_claim(ctx, storeapi, claimed_epoch, Some(read_epoch)).await;
}
match reconcile_scanner_leadership_claim(
@@ -249,7 +348,7 @@ pub(super) async fn claim_scanner_leadership(
.await
{
Ok(ScannerLeadershipClaimReconcile::Durable) => {
return complete_scanner_leadership_claim(ctx, storeapi, claimed_epoch).await;
return complete_scanner_leadership_claim(ctx, storeapi, claimed_epoch, Some(read_epoch)).await;
}
Ok(ScannerLeadershipClaimReconcile::Changed) if retry < SCANNER_PERSIST_CAS_RETRIES => continue,
Ok(ScannerLeadershipClaimReconcile::Changed | ScannerLeadershipClaimReconcile::Unchanged) => {
@@ -293,7 +392,7 @@ pub(super) async fn claim_scanner_leadership(
.await
{
Ok(ScannerLeadershipClaimReconcile::Durable) => {
return complete_scanner_leadership_claim(ctx, storeapi, claimed_epoch).await;
return complete_scanner_leadership_claim(ctx, storeapi, claimed_epoch, Some(read_epoch)).await;
}
Ok(ScannerLeadershipClaimReconcile::Changed)
if retry < SCANNER_PERSIST_CAS_RETRIES && !ctx.is_cancelled() =>
crates/scanner/src/scanner/tests.rs
+185 -28
View File
@@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use super::heal_info::{classify_background_heal_read_error, decode_background_heal_info};
use super::*;
use crate::EcstoreResult;
use crate::{
@@ -22,6 +23,7 @@ use crate::{
};
use std::collections::{HashMap, HashSet};
use std::io::Cursor;
use std::sync::atomic::{AtomicBool, Ordering};
use std::task::Poll;
use temp_env::{with_var, with_var_unset};
use tokio::io::AsyncReadExt;
@@ -343,6 +345,7 @@ struct MemoryConfigStore {
cancel_after_successful_puts: Mutex<HashMap<String, (usize, CancellationToken)>>,
replace_after_successful_puts: Mutex<HashMap<String, (usize, Vec<u8>)>>,
put_counts: Mutex<HashMap<String, usize>>,
publication_admission_blocked: AtomicBool,
}
fn memory_config_key(bucket: &str, object: &str) -> String {
@@ -1350,7 +1353,7 @@ async fn full_rescan_reset_preserves_valid_primary_when_marker_is_malformed() {
let old_usage = DataUsageInfo {
scanner_epoch: Some(7),
scanner_cycle: Some(41),
..Default::default()
..complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0)
};
let old_usage_data = serde_json::to_vec(&old_usage).expect("usage snapshot should encode");
save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), old_usage_data.clone())
@@ -1424,7 +1427,7 @@ async fn full_rescan_reset_resumes_cleanup_pending_preserved_primary() {
let usage = DataUsageInfo {
scanner_epoch: Some(7),
scanner_cycle: Some(41),
..Default::default()
..complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0)
};
save_config(
store.clone(),
@@ -1695,7 +1698,7 @@ async fn full_rescan_reset_rejects_usage_floor_that_would_be_terminal() {
DATA_USAGE_OBJ_NAME_PATH.as_str(),
serde_json::to_vec(&DataUsageInfo {
scanner_epoch: Some(u64::MAX - 1),
..Default::default()
..complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0)
})
.expect("usage floor should encode"),
)
@@ -1847,14 +1850,12 @@ async fn scanner_startup_uses_primary_and_backup_usage_floor() {
let store = Arc::new(MemoryConfigStore::default());
let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str());
for (path, epoch, cycle) in [(DATA_USAGE_OBJ_NAME_PATH.as_str(), 8, 100), (backup_path.as_str(), 11, 103)] {
let mut usage = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0);
usage.scanner_epoch = Some(epoch);
usage.scanner_cycle = Some(cycle);
store.objects.lock().await.insert(
memory_config_key(RUSTFS_META_BUCKET, path),
serde_json::to_vec(&DataUsageInfo {
scanner_epoch: Some(epoch),
scanner_cycle: Some(cycle),
..Default::default()
})
.expect("usage snapshot should encode"),
serde_json::to_vec(&usage).expect("usage snapshot should encode"),
);
}
@@ -1879,14 +1880,12 @@ async fn scanner_usage_floor_ignores_older_backup_after_primary_epoch_fence() {
let store = Arc::new(MemoryConfigStore::default());
let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str());
for (path, epoch, cycle) in [(DATA_USAGE_OBJ_NAME_PATH.as_str(), 8, 100), (backup_path.as_str(), 7, 10_000)] {
let mut usage = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0);
usage.scanner_epoch = Some(epoch);
usage.scanner_cycle = Some(cycle);
store.objects.lock().await.insert(
memory_config_key(RUSTFS_META_BUCKET, path),
serde_json::to_vec(&DataUsageInfo {
scanner_epoch: Some(epoch),
scanner_cycle: Some(cycle),
..Default::default()
})
.expect("usage snapshot should encode"),
serde_json::to_vec(&usage).expect("usage snapshot should encode"),
);
}
@@ -1916,6 +1915,23 @@ fn scanner_startup_treats_incomplete_usage_snapshot_as_cold() {
}));
}
#[test]
fn scanner_baseline_identity_requires_complete_or_strict_legacy_shape() {
assert!(!data_usage_info_has_persisted_baseline_identity(&DataUsageInfo {
scanner_epoch: Some(3),
scanner_cycle: Some(7),
..Default::default()
}));
let mut legacy = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0);
legacy.usage_snapshot_complete = false;
legacy.scanner_cycle = Some(7);
assert!(data_usage_info_has_persisted_baseline_identity(&legacy));
legacy.scanner_epoch = Some(3);
assert!(!data_usage_info_has_persisted_baseline_identity(&legacy));
}
#[test]
fn scanner_startup_prompts_only_for_a_newer_valid_observation() {
let authoritative = DataUsageInfo {
@@ -1948,12 +1964,9 @@ fn scanner_startup_prompts_only_for_a_newer_valid_observation() {
#[tokio::test]
async fn scanner_startup_prefers_v2_over_legacy_usage() {
let store = Arc::new(MemoryConfigStore::default());
let legacy = DataUsageInfo {
scanner_epoch: Some(19),
scanner_cycle: Some(41),
last_update: Some(std::time::SystemTime::now()),
..Default::default()
};
let mut legacy = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0);
legacy.scanner_epoch = Some(19);
legacy.scanner_cycle = Some(41);
let legacy_data = serde_json::to_vec(&legacy).expect("legacy usage snapshot should encode");
store.objects.lock().await.insert(
memory_config_key(RUSTFS_META_BUCKET, LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str()),
@@ -1976,13 +1989,9 @@ async fn scanner_startup_prefers_v2_over_legacy_usage() {
}
);
let authoritative = DataUsageInfo {
scanner_epoch: Some(23),
scanner_cycle: Some(51),
last_update: Some(std::time::SystemTime::now()),
usage_snapshot_complete: true,
..Default::default()
};
let mut authoritative = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0);
authoritative.scanner_epoch = Some(23);
authoritative.scanner_cycle = Some(51);
let authoritative_data = serde_json::to_vec(&authoritative).expect("v2 usage snapshot should encode");
store.objects.lock().await.insert(
memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()),
@@ -2024,6 +2033,8 @@ async fn scanner_startup_prefers_v2_over_legacy_usage() {
#[tokio::test]
async fn scanner_usage_floor_fails_closed_on_corrupt_or_exhausted_usage_state() {
let store = Arc::new(MemoryConfigStore::default());
assert!(persisted_usage_floor(store.clone()).await.is_err());
store.objects.lock().await.insert(
memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()),
b"not-json".to_vec(),
@@ -2087,6 +2098,39 @@ async fn scanner_usage_backup_uses_durable_cycle_cadence_across_tasks() {
}
}
#[tokio::test]
async fn scanner_backup_sync_distinguishes_movement_from_missing_or_corrupt_primary() {
let store = Arc::new(MemoryConfigStore::default());
let primary_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str());
let primary = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0);
store.objects.lock().await.insert(
primary_key.clone(),
serde_json::to_vec(&primary).expect("primary usage snapshot should encode"),
);
store.revisions.lock().await.insert(primary_key.clone(), 1);
store.publication_admission_blocked.store(true, Ordering::Release);
let movement_error = sync_data_usage_backup_from_primary(&CancellationToken::new(), store.clone())
.await
.expect_err("movement admission loss should fail backup synchronization");
assert!(scanner_publication_epoch_changed(&movement_error));
store.publication_admission_blocked.store(false, Ordering::Release);
store.objects.lock().await.remove(&primary_key);
store.revisions.lock().await.remove(&primary_key);
assert!(matches!(
sync_data_usage_backup_from_primary(&CancellationToken::new(), store.clone()).await,
Err(EcstoreError::ConfigNotFound)
));
store.objects.lock().await.insert(primary_key.clone(), b"not-json".to_vec());
store.revisions.lock().await.insert(primary_key, 1);
let corrupt_error = sync_data_usage_backup_from_primary(&CancellationToken::new(), store)
.await
.expect_err("corrupt primary should fail backup synchronization");
assert!(!scanner_publication_epoch_changed(&corrupt_error));
}
#[async_trait::async_trait]
impl crate::ScannerConfigObjectDelete for MemoryConfigStore {
async fn delete_config_object(&self, bucket: &str, object: &str, opts: ObjectOptions) -> EcstoreResult<ObjectInfo> {
@@ -2110,6 +2154,10 @@ impl crate::ScannerConfigObjectDelete for MemoryConfigStore {
revisions.remove(&key);
Ok(ObjectInfo::default())
}
async fn scanner_data_usage_publication_admission(&self) -> Option<crate::ScannerDataUsagePublicationAdmission> {
(!self.publication_admission_blocked.load(Ordering::Acquire)).then(crate::ScannerDataUsagePublicationAdmission::unfenced)
}
}
#[test]
@@ -2270,6 +2318,7 @@ async fn test_leadership_claim_preserves_usage_epoch_floor_across_old_epoch_conf
started: Utc::now(),
};
assert!(persist_scanner_cycle_state(&ctx, store.clone(), &mut cycle, &mut revision, 1).await);
seed_usage_snapshot_for_leadership_claim(&store).await;
let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str());
let old_epoch_commit = CurrentCycle {
@@ -2313,6 +2362,61 @@ async fn test_leadership_claim_rejects_terminal_epoch() {
assert!(read_config(store, &DATA_USAGE_BLOOM_NAME_PATH).await.is_err());
}
#[tokio::test]
async fn leadership_claim_defers_without_usage_baseline_before_bloom_write() {
let store = Arc::new(MemoryConfigStore::default());
let ctx = CancellationToken::new();
let mut revision = DataUsageCacheRevision::Missing;
let mut cycle = CurrentCycle {
next: 12,
..Default::default()
};
let mut persisted_epoch = 0;
assert!(!claim_scanner_leadership(&ctx, store.clone(), &mut cycle, &mut revision, &mut persisted_epoch,).await);
assert!(read_config(store.clone(), &DATA_USAGE_BLOOM_NAME_PATH).await.is_err());
assert!(read_config(store, DATA_USAGE_OBJ_NAME_PATH.as_str()).await.is_err());
}
#[tokio::test]
async fn leadership_claim_defers_on_corrupt_usage_baseline_without_bloom_write() {
let store = Arc::new(MemoryConfigStore::default());
let usage_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str());
store.objects.lock().await.insert(usage_key.clone(), b"not-json".to_vec());
store.revisions.lock().await.insert(usage_key, 1);
let ctx = CancellationToken::new();
let mut revision = DataUsageCacheRevision::Missing;
let mut cycle = CurrentCycle {
next: 12,
..Default::default()
};
let mut persisted_epoch = 0;
assert!(!claim_scanner_leadership(&ctx, store.clone(), &mut cycle, &mut revision, &mut persisted_epoch,).await);
assert!(read_config(store, &DATA_USAGE_BLOOM_NAME_PATH).await.is_err());
}
#[tokio::test]
async fn leadership_claim_defers_on_unidentified_usage_baseline_without_bloom_write() {
let store = Arc::new(MemoryConfigStore::default());
let usage_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str());
let data = serde_json::to_vec(&DataUsageInfo::default()).expect("default usage should encode");
store.objects.lock().await.insert(usage_key.clone(), data);
store.revisions.lock().await.insert(usage_key, 1);
let ctx = CancellationToken::new();
let mut revision = DataUsageCacheRevision::Missing;
let mut cycle = CurrentCycle {
next: 12,
..Default::default()
};
let mut persisted_epoch = 0;
assert!(!claim_scanner_leadership(&ctx, store.clone(), &mut cycle, &mut revision, &mut persisted_epoch,).await);
assert!(read_config(store, &DATA_USAGE_BLOOM_NAME_PATH).await.is_err());
}
#[tokio::test]
async fn test_leadership_claim_confirms_commit_after_returned_error() {
let store = Arc::new(MemoryConfigStore::default());
@@ -2329,6 +2433,7 @@ async fn test_leadership_claim_confirms_commit_after_returned_error() {
started: Utc::now(),
};
let mut persisted_epoch = 0;
seed_usage_snapshot_for_leadership_claim(&store).await;
assert!(claim_scanner_leadership(&ctx, store.clone(), &mut cycle, &mut revision, &mut persisted_epoch).await);
@@ -2373,6 +2478,7 @@ async fn test_leadership_claim_usage_fence_rejects_old_inflight_writer() {
);
old_usage.buckets_count = 1;
old_usage.calculate_totals();
old_usage.usage_snapshot_complete = true;
let old_data = serde_json::to_vec(&old_usage).expect("old usage snapshot should encode");
store.objects.lock().await.insert(usage_key.clone(), old_data.clone());
store.revisions.lock().await.insert(usage_key, 1);
@@ -2419,6 +2525,7 @@ async fn cycle_budget_lease_takeover_rejects_old_generation() {
started: Utc::now(),
};
assert!(persist_scanner_cycle_state(&ctx, store.clone(), &mut cycle, &mut revision, 1).await);
seed_usage_snapshot_for_leadership_claim(&store).await;
let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str());
store
@@ -2590,6 +2697,35 @@ async fn test_usage_save_route_barrier_prevents_missing_snapshot_creation() {
}
}
#[tokio::test]
async fn test_observational_usage_defers_when_authoritative_baseline_is_missing() {
let store = Arc::new(MemoryConfigStore::default());
let (sender, receiver) = mpsc::channel(1);
let mut observation = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)), 1);
observation.usage_snapshot_converged = Some(false);
sender.send(observation).await.expect("observation should enqueue");
drop(sender);
let outcome = store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe(
CancellationToken::new(),
store.clone(),
receiver,
None,
None,
|| async { false },
)
.await;
assert_eq!(outcome, DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement));
assert!(
!store
.objects
.lock()
.await
.contains_key(&memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str()))
);
}
#[tokio::test]
async fn test_usage_route_barrier_precedes_durable_reconciliation() {
let store = Arc::new(MemoryConfigStore::default());
@@ -3425,6 +3561,14 @@ fn complete_usage_with_bucket_count(last_update: Option<std::time::SystemTime>,
info
}
async fn seed_usage_snapshot_for_leadership_claim(store: &Arc<MemoryConfigStore>) {
let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str());
let data = serde_json::to_vec(&complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0))
.expect("leadership usage baseline should encode");
store.objects.lock().await.insert(key.clone(), data);
store.revisions.lock().await.insert(key, 1);
}
fn usage_with_last_update(last_update: Option<std::time::SystemTime>) -> DataUsageInfo {
complete_usage_with_bucket_count(last_update, 0)
}
@@ -5129,6 +5273,19 @@ fn test_background_heal_info_for_scan_start_marks_deep_active() {
assert_eq!(info.bitrot_start_time, Some(now));
}
#[test]
fn background_heal_read_failures_never_become_initializable_defaults() {
assert_eq!(
classify_background_heal_read_error(&EcstoreError::ConfigNotFound),
BackgroundHealInfoReadStatus::Missing
);
assert_eq!(
classify_background_heal_read_error(&EcstoreError::SlowDown),
BackgroundHealInfoReadStatus::Transient
);
assert!(decode_background_heal_info(b"not-json").is_err());
}
#[test]
fn test_background_heal_info_for_scan_start_keeps_deep_window_start() {
with_var_unset(ENV_SCANNER_BITROT_CYCLE_SECS, || {
crates/scanner/src/scanner/usage_store.rs
+221 -50
View File
@@ -112,11 +112,39 @@ pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_basel
}
pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe<F, Fut>(
ctx: CancellationToken,
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
receiver: mpsc::Receiver<DataUsageInfo>,
leader_epoch: Option<u64>,
initial_baseline: Option<DataUsagePersistBaseline>,
route_probe: F,
) -> DataUsagePersistOutcome
where
F: Fn() -> Fut + Send + Sync,
Fut: Future<Output = bool> + Send,
{
store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe_for_publication_epoch(
ctx,
storeapi,
receiver,
leader_epoch,
initial_baseline,
None,
route_probe,
)
.await
}
pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe_for_publication_epoch<
F,
Fut,
>(
ctx: CancellationToken,
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
mut receiver: mpsc::Receiver<DataUsageInfo>,
leader_epoch: Option<u64>,
initial_baseline: Option<DataUsagePersistBaseline>,
expected_publication_epoch: Option<u64>,
route_probe: F,
) -> DataUsagePersistOutcome
where
@@ -134,6 +162,14 @@ where
if let Some(leader_epoch) = leader_epoch {
data_usage_info.scanner_epoch = Some(leader_epoch);
}
if let Some(expected_epoch) = expected_publication_epoch
&& scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch)
.await
.is_none()
{
outcome = DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
break 'updates;
}
let observational = data_usage_info.usage_snapshot_converged == Some(false);
let target_path = if observational {
DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str()
@@ -154,7 +190,28 @@ where
break;
}
let mut publication_epoch = expected_publication_epoch;
if observational && data_usage_info.usage_snapshot_authoritative_baseline.is_none() {
let read_epoch = match expected_publication_epoch {
Some(expected_epoch) => {
if scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch)
.await
.is_none()
{
outcome = DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
break 'updates;
}
expected_epoch
}
None => {
let Some(read_epoch) = scanner_publication_epoch(storeapi.clone()).await else {
outcome = DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
break 'updates;
};
read_epoch
}
};
publication_epoch = Some(read_epoch);
let authoritative_data = match next_baseline.as_ref() {
Some(baseline) => baseline.data.clone(),
None => match read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()).await {
@@ -175,25 +232,48 @@ where
}
},
};
let authoritative = match authoritative_data.as_deref() {
Some(data) => match serde_json::from_slice::<DataUsageInfo>(data) {
Ok(info) => info,
Err(err) => {
error!(
target: "rustfs::scanner",
event = EVENT_SCANNER_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
state = "observed_baseline_decode_failed",
error = %err,
"Scanner refused to publish an observation from an invalid authoritative baseline"
);
outcome = DataUsagePersistOutcome::Failed;
continue;
}
},
None => DataUsageInfo::default(),
let Some(authoritative_data) = authoritative_data else {
warn!(
target: "rustfs::scanner",
event = EVENT_SCANNER_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
state = "observed_baseline_missing",
"Scanner deferred observational publication until an authoritative usage baseline exists"
);
outcome = DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
break 'updates;
};
let authoritative = match serde_json::from_slice::<DataUsageInfo>(&authoritative_data) {
Ok(info) if data_usage_info_has_persisted_baseline_identity(&info) => info,
Ok(_) => {
error!(
target: "rustfs::scanner",
event = EVENT_SCANNER_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
state = "observed_baseline_identity_missing",
"Scanner refused to publish an observation without authoritative baseline identity"
);
outcome = DataUsagePersistOutcome::Failed;
continue;
}
Err(err) => {
error!(
target: "rustfs::scanner",
event = EVENT_SCANNER_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
state = "observed_baseline_decode_failed",
error = %err,
"Scanner refused to publish an observation from an invalid authoritative baseline"
);
outcome = DataUsagePersistOutcome::Failed;
continue;
}
};
data_usage_info.usage_snapshot_authoritative_baseline = Some(authoritative.snapshot_identity());
}
@@ -240,6 +320,26 @@ where
break 'updates;
}
let publication_epoch_for_save = match expected_publication_epoch {
Some(expected_epoch) => {
if scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch)
.await
.is_none()
{
break DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
}
expected_epoch
}
None => match publication_epoch.take() {
Some(epoch) => epoch,
None => {
let Some(epoch) = scanner_publication_epoch(storeapi.clone()).await else {
break DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
};
epoch
}
},
};
let baseline = if !observational && cas_retry == 0 {
next_baseline.take()
} else {
@@ -329,14 +429,22 @@ where
}
let done_save = Metrics::time(Metric::SaveUsage);
let save_result = save_config_shared_with_preconditions(
storeapi.clone(),
target_path,
data.clone(),
sha256hex.clone(),
revision.preconditions(),
)
.await;
let save_result = {
let Some(_publication_admission) =
scanner_publication_admission_for_epoch(storeapi.clone(), publication_epoch_for_save).await
else {
done_save();
break DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
};
save_config_shared_with_preconditions(
storeapi.clone(),
target_path,
data.clone(),
sha256hex.clone(),
revision.preconditions(),
)
.await
};
done_save();
match save_result {
@@ -427,7 +535,16 @@ where
if observational {
invalidate_admin_data_usage_snapshot_cache().await;
} else {
cleanup_observed_data_usage_snapshot(storeapi.clone(), &data_usage_info).await;
let cleanup_ok = cleanup_observed_data_usage_snapshot_for_epoch(
storeapi.clone(),
&data_usage_info,
expected_publication_epoch,
)
.await;
if expected_publication_epoch.is_some() && !cleanup_ok {
outcome = DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
break 'updates;
}
invalidate_data_usage_snapshot_cache().await;
replace_bucket_usage_memory_from_info(&data_usage_info).await;
}
@@ -438,7 +555,16 @@ where
if observational {
invalidate_admin_data_usage_snapshot_cache().await;
} else {
cleanup_observed_data_usage_snapshot(storeapi.clone(), &data_usage_info).await;
let cleanup_ok = cleanup_observed_data_usage_snapshot_for_epoch(
storeapi.clone(),
&data_usage_info,
expected_publication_epoch,
)
.await;
if expected_publication_epoch.is_some() && !cleanup_ok {
outcome = DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
break 'updates;
}
invalidate_data_usage_snapshot_cache().await;
}
global_metrics().record_scanner_usage_save_result(ScannerUsageSaveResult::Success);
@@ -460,7 +586,16 @@ where
if observational {
invalidate_admin_data_usage_snapshot_cache().await;
} else {
cleanup_observed_data_usage_snapshot(storeapi.clone(), &data_usage_info).await;
let cleanup_ok = cleanup_observed_data_usage_snapshot_for_epoch(
storeapi.clone(),
&data_usage_info,
expected_publication_epoch,
)
.await;
if expected_publication_epoch.is_some() && !cleanup_ok {
outcome = DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
break 'updates;
}
invalidate_data_usage_snapshot_cache().await;
replace_bucket_usage_memory_from_info(&data_usage_info).await;
}
@@ -471,7 +606,10 @@ where
if backup_due {
let done_save = Metrics::time(Metric::SaveUsage);
if let Err(e) = sync_data_usage_backup_from_primary(&ctx, storeapi.clone()).await {
let backup_result =
sync_data_usage_backup_from_primary_for_epoch(&ctx, storeapi.clone(), expected_publication_epoch).await;
done_save();
if let Err(e) = backup_result {
warn!(
target: "rustfs::scanner",
event = EVENT_SCANNER_PERSIST_STATE,
@@ -482,22 +620,50 @@ where
error = %e,
"Scanner data usage backup save failed"
);
if scanner_publication_epoch_changed(&e) {
outcome = DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
break 'updates;
}
outcome = DataUsagePersistOutcome::Failed;
break 'updates;
}
done_save();
}
}
outcome
}
pub(super) async fn cleanup_observed_data_usage_snapshot(
pub(super) async fn cleanup_observed_data_usage_snapshot_for_epoch(
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
authoritative: &DataUsageInfo,
) {
expected_publication_epoch: Option<u64>,
) -> bool {
let read_epoch = match expected_publication_epoch {
Some(expected_epoch) => {
if scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch)
.await
.is_none()
{
return false;
}
expected_epoch
}
None => match scanner_publication_epoch(storeapi.clone()).await {
Some(read_epoch) => read_epoch,
None => return false,
},
};
if expected_publication_epoch.is_some()
&& scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch)
.await
.is_none()
{
return false;
}
let (observed_data, revision) =
match read_config_with_revision(storeapi.clone(), DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str()).await {
Ok((Some(data), revision)) => (data, revision),
Ok((None, _)) => return,
Ok((None, _)) => return true,
Err(err) => {
error!(
target: "rustfs::scanner",
@@ -509,7 +675,7 @@ pub(super) async fn cleanup_observed_data_usage_snapshot(
error = %err,
"Scanner could not inspect observational data usage snapshot before authoritative cleanup"
);
return;
return true;
}
};
let observed = match serde_json::from_slice::<DataUsageInfo>(&observed_data) {
@@ -525,25 +691,26 @@ pub(super) async fn cleanup_observed_data_usage_snapshot(
error = %err,
"Scanner refused to remove an invalid observational data usage snapshot after authoritative save"
);
return;
return true;
}
};
if observed_data_usage_is_newer(&observed, authoritative) {
return;
return true;
}
let result = storeapi
.delete_config_object(
RUSTFS_META_BUCKET,
DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(),
ScannerObjectOptions {
delete_prefix: true,
delete_prefix_object: true,
http_preconditions: Some(revision.preconditions()),
..Default::default()
},
)
.await;
let result = delete_config_with_publication_admission_for_epoch(
storeapi,
RUSTFS_META_BUCKET,
DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(),
ScannerObjectOptions {
delete_prefix: true,
delete_prefix_object: true,
http_preconditions: Some(revision.preconditions()),
..Default::default()
},
read_epoch,
)
.await;
match result {
Ok(_)
@@ -564,6 +731,10 @@ pub(super) async fn cleanup_observed_data_usage_snapshot(
error = %err,
"Scanner could not remove stale observational data usage snapshot after authoritative save"
);
if scanner_publication_epoch_changed(&err) {
return false;
}
}
}
true
}