mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-24 21:26:28 +00:00
fix(scanner): fence system metadata publication (#6444)
* feat(scanner): fence usage publication during data movement * fix(scanner): detect movement refresh state changes * fix(scanner): fence publication during data movement * fix(scanner): close movement epoch publication races * fix(scanner): fence movement-sensitive publication paths * fix(scanner): fence cache and heal recovery paths * fix(scanner): carry publication epoch through scan cycle * fix(scanner): recheck remote cache epoch after save * fix(scanner): recheck local cache epoch before publish * fix(scanner): fence data usage writers and baseline * fix(scanner): expose decommission activity to publication fence * fix(scanner): release publication gate before reads * fix(scanner): complete publication fence integration * fix(scanner): avoid empty usage baseline publication * chore(scanner): gate test-only helpers * fix: use decommission canceler in reload test --------- Co-authored-by: houseme <housemecn@gmail.com>
This commit is contained in:
@@ -36,13 +36,13 @@ use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf};
|
||||
use tokio::time::{Duration, Instant, sleep, timeout};
|
||||
use tracing::{debug, warn};
|
||||
|
||||
use crate::ScannerObjectIO;
|
||||
use crate::storage_api::owner::HTTPPreconditions;
|
||||
use crate::{
|
||||
BUCKET_META_PREFIX, EcstoreError as Error, EcstoreResult as StorageResult, RUSTFS_META_BUCKET, ReplicationConfig,
|
||||
ScannerObjectInfo as ObjectInfo, ScannerObjectOptions as ObjectOptions, StorageError, TRANSITION_COMPLETE, save_config,
|
||||
save_config_with_preconditions, storageclass,
|
||||
SCANNER_PUBLICATION_EPOCH_CHANGED, ScannerObjectInfo as ObjectInfo, ScannerObjectOptions as ObjectOptions, StorageError,
|
||||
TRANSITION_COMPLETE, save_config, save_config_with_preconditions, scanner_publication_admission_for_epoch, storageclass,
|
||||
};
|
||||
use crate::{ScannerConfigObjectDelete, ScannerObjectIO};
|
||||
|
||||
// Data usage constants
|
||||
pub const DATA_USAGE_ROOT: &str = SLASH_SEPARATOR;
|
||||
@@ -119,9 +119,13 @@ pub(crate) async fn read_config_with_revision<S: ScannerObjectIO>(
|
||||
.ok_or_else(|| StorageError::other(format!("scanner config object {path} has no ETag")))?;
|
||||
Ok((Some(reader.read_all().await?), revision))
|
||||
}
|
||||
Err(Error::FileNotFound | Error::VolumeNotFound | Error::ObjectNotFound(_, _) | Error::BucketNotFound(_)) => {
|
||||
Ok((None, DataUsageCacheRevision::Missing))
|
||||
}
|
||||
Err(
|
||||
Error::ConfigNotFound
|
||||
| Error::FileNotFound
|
||||
| Error::VolumeNotFound
|
||||
| Error::ObjectNotFound(_, _)
|
||||
| Error::BucketNotFound(_),
|
||||
) => Ok((None, DataUsageCacheRevision::Missing)),
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
}
|
||||
@@ -147,9 +151,13 @@ pub(crate) async fn read_config_revision<S: ScannerObjectIO>(store: Arc<S>, path
|
||||
.filter(|etag| !etag.is_empty())
|
||||
.map(DataUsageCacheRevision::Etag)
|
||||
.ok_or_else(|| StorageError::other(format!("scanner config object {path} has no ETag"))),
|
||||
Err(Error::FileNotFound | Error::VolumeNotFound | Error::ObjectNotFound(_, _) | Error::BucketNotFound(_)) => {
|
||||
Ok(DataUsageCacheRevision::Missing)
|
||||
}
|
||||
Err(
|
||||
Error::ConfigNotFound
|
||||
| Error::FileNotFound
|
||||
| Error::VolumeNotFound
|
||||
| Error::ObjectNotFound(_, _)
|
||||
| Error::BucketNotFound(_),
|
||||
) => Ok(DataUsageCacheRevision::Missing),
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,6 +15,42 @@
|
||||
use super::*;
|
||||
use std::sync::Arc;
|
||||
|
||||
#[derive(Debug)]
|
||||
struct CachePublicationAdmissionUnavailable;
|
||||
|
||||
impl std::fmt::Display for CachePublicationAdmissionUnavailable {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
formatter.write_str("scanner cache publication admission is unavailable")
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for CachePublicationAdmissionUnavailable {}
|
||||
|
||||
fn cache_publication_admission_unavailable() -> Error {
|
||||
Error::Io(std::io::Error::other(CachePublicationAdmissionUnavailable))
|
||||
}
|
||||
|
||||
fn cache_publication_epoch_changed() -> Error {
|
||||
Error::other(SCANNER_PUBLICATION_EPOCH_CHANGED)
|
||||
}
|
||||
|
||||
fn is_cache_publication_admission_unavailable(error: &StorageError) -> bool {
|
||||
matches!(
|
||||
error,
|
||||
StorageError::Io(io_error)
|
||||
if io_error
|
||||
.get_ref()
|
||||
.is_some_and(|source| source.downcast_ref::<CachePublicationAdmissionUnavailable>().is_some())
|
||||
)
|
||||
}
|
||||
|
||||
fn is_cache_publication_epoch_changed(error: &StorageError) -> bool {
|
||||
matches!(
|
||||
error,
|
||||
StorageError::Io(io_error) if io_error.to_string() == SCANNER_PUBLICATION_EPOCH_CHANGED
|
||||
)
|
||||
}
|
||||
|
||||
pub(super) enum DataUsageCacheLoadAttempt {
|
||||
Loaded {
|
||||
cache: Box<DataUsageCache>,
|
||||
@@ -368,6 +404,12 @@ impl DataUsageCache {
|
||||
fn should_retry_save_error(err: &StorageError) -> bool {
|
||||
// Usage-cache files are best-effort scanner checkpoints. Retrying namespace
|
||||
// lock failures immediately only adds more lock traffic to the same hot object.
|
||||
if is_cache_publication_admission_unavailable(err) {
|
||||
return false;
|
||||
}
|
||||
if is_cache_publication_epoch_changed(err) {
|
||||
return false;
|
||||
}
|
||||
!matches!(
|
||||
err,
|
||||
StorageError::Lock(_)
|
||||
@@ -423,13 +465,14 @@ impl DataUsageCache {
|
||||
Err(last_err.unwrap_or_else(|| StorageError::other("Failed to save data usage cache".to_string())))
|
||||
}
|
||||
|
||||
async fn save_path_with_retry<S: ScannerObjectIO>(
|
||||
async fn save_path_with_retry<S: ScannerObjectIO + ScannerConfigObjectDelete>(
|
||||
store: Arc<S>,
|
||||
path: &str,
|
||||
buf: &[u8],
|
||||
timeout_duration: Duration,
|
||||
max_retries: u32,
|
||||
revision: Option<DataUsageCacheRevision>,
|
||||
expected_epoch: Option<u64>,
|
||||
) -> StorageResult<()> {
|
||||
Self::ensure_cache_save_metrics_registered();
|
||||
let path_type = Self::cache_path_type(path);
|
||||
@@ -441,6 +484,17 @@ impl DataUsageCache {
|
||||
let buf_clone = buf.to_vec();
|
||||
let revision = revision.clone();
|
||||
async move {
|
||||
let publication_admission = match expected_epoch {
|
||||
Some(expected_epoch) => scanner_publication_admission_for_epoch(store_clone.clone(), expected_epoch).await,
|
||||
None => store_clone.scanner_data_usage_publication_admission().await,
|
||||
};
|
||||
let Some(_publication_admission) = publication_admission else {
|
||||
return Err(if expected_epoch.is_some() {
|
||||
cache_publication_epoch_changed()
|
||||
} else {
|
||||
cache_publication_admission_unavailable()
|
||||
});
|
||||
};
|
||||
if let Some(revision) = revision {
|
||||
save_config_with_preconditions(store_clone, &path_clone, buf_clone, revision.preconditions()).await?;
|
||||
} else {
|
||||
@@ -454,6 +508,14 @@ impl DataUsageCache {
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
// An epoch-specific admission failure is authoritative: reconciling
|
||||
// identical bytes cannot prove that this snapshot belongs to the
|
||||
// captured movement epoch. Do not turn that fence failure into a
|
||||
// successful stale publication.
|
||||
if is_cache_publication_epoch_changed(&save_err) {
|
||||
return Err(save_err);
|
||||
}
|
||||
|
||||
for attempt in 0..=max_retries {
|
||||
let reconcile = timeout(timeout_duration, async {
|
||||
let mut reader = store
|
||||
@@ -486,24 +548,36 @@ impl DataUsageCache {
|
||||
Err(save_err)
|
||||
}
|
||||
|
||||
pub async fn save<S: ScannerObjectIO>(&self, store: Arc<S>, name: &str) -> StorageResult<()> {
|
||||
self.save_inner(store, name, None).await
|
||||
pub async fn save<S: ScannerObjectIO + ScannerConfigObjectDelete>(&self, store: Arc<S>, name: &str) -> StorageResult<()> {
|
||||
self.save_inner(store, name, None, None).await
|
||||
}
|
||||
|
||||
pub(crate) async fn save_with_revisions<S: ScannerObjectIO>(
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn save_with_revisions<S: ScannerObjectIO + ScannerConfigObjectDelete>(
|
||||
&self,
|
||||
store: Arc<S>,
|
||||
name: &str,
|
||||
revisions: &DataUsageCacheRevisions,
|
||||
) -> StorageResult<()> {
|
||||
self.save_inner(store, name, Some(revisions)).await
|
||||
self.save_inner(store, name, Some(revisions), None).await
|
||||
}
|
||||
|
||||
async fn save_inner<S: ScannerObjectIO>(
|
||||
pub(crate) async fn save_with_revisions_for_epoch<S: ScannerObjectIO + ScannerConfigObjectDelete>(
|
||||
&self,
|
||||
store: Arc<S>,
|
||||
name: &str,
|
||||
revisions: &DataUsageCacheRevisions,
|
||||
expected_epoch: u64,
|
||||
) -> StorageResult<()> {
|
||||
self.save_inner(store, name, Some(revisions), Some(expected_epoch)).await
|
||||
}
|
||||
|
||||
async fn save_inner<S: ScannerObjectIO + ScannerConfigObjectDelete>(
|
||||
&self,
|
||||
store: Arc<S>,
|
||||
name: &str,
|
||||
revisions: Option<&DataUsageCacheRevisions>,
|
||||
expected_epoch: Option<u64>,
|
||||
) -> StorageResult<()> {
|
||||
let mut buf = Vec::new();
|
||||
self.serialize(&mut rmp_serde::Serializer::new(&mut buf))?;
|
||||
@@ -517,6 +591,7 @@ impl DataUsageCache {
|
||||
timeout_duration,
|
||||
DATA_USAGE_CACHE_SAVE_RETRIES,
|
||||
revisions.map(|revisions| revisions.main.clone()),
|
||||
expected_epoch,
|
||||
)
|
||||
.await?;
|
||||
|
||||
@@ -534,6 +609,7 @@ impl DataUsageCache {
|
||||
backup_timeout_duration,
|
||||
DATA_USAGE_CACHE_BACKUP_SAVE_RETRIES,
|
||||
backup_revision,
|
||||
expected_epoch,
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -548,6 +624,9 @@ impl DataUsageCache {
|
||||
error = %e,
|
||||
"Scanner cache backup save failed"
|
||||
);
|
||||
if is_cache_publication_admission_unavailable(&e) || is_cache_publication_epoch_changed(&e) {
|
||||
return Err(e);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -198,6 +198,22 @@ impl ObjectIO for CacheReadStore {
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl crate::ScannerConfigObjectDelete for CacheReadStore {
|
||||
async fn delete_config_object(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
_object: &str,
|
||||
_opts: crate::ScannerObjectOptions,
|
||||
) -> crate::EcstoreResult<crate::ScannerObjectInfo> {
|
||||
Err(crate::EcstoreError::NotImplemented)
|
||||
}
|
||||
|
||||
async fn scanner_data_usage_publication_admission(&self) -> Option<crate::ScannerDataUsagePublicationAdmission> {
|
||||
Some(crate::ScannerDataUsagePublicationAdmission::unfenced())
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl ObjectIO for AmbiguousCacheCommitStore {
|
||||
type Error = Error;
|
||||
@@ -243,6 +259,22 @@ impl ObjectIO for AmbiguousCacheCommitStore {
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl crate::ScannerConfigObjectDelete for AmbiguousCacheCommitStore {
|
||||
async fn delete_config_object(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
_object: &str,
|
||||
_opts: crate::ScannerObjectOptions,
|
||||
) -> crate::EcstoreResult<crate::ScannerObjectInfo> {
|
||||
Err(crate::EcstoreError::NotImplemented)
|
||||
}
|
||||
|
||||
async fn scanner_data_usage_publication_admission(&self) -> Option<crate::ScannerDataUsagePublicationAdmission> {
|
||||
Some(crate::ScannerDataUsagePublicationAdmission::unfenced())
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl ObjectIO for BackupFallbackStore {
|
||||
type Error = Error;
|
||||
@@ -314,6 +346,22 @@ impl ObjectIO for BackupFallbackStore {
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl crate::ScannerConfigObjectDelete for BackupFallbackStore {
|
||||
async fn delete_config_object(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
_object: &str,
|
||||
_opts: crate::ScannerObjectOptions,
|
||||
) -> crate::EcstoreResult<crate::ScannerObjectInfo> {
|
||||
Err(crate::EcstoreError::NotImplemented)
|
||||
}
|
||||
|
||||
async fn scanner_data_usage_publication_admission(&self) -> Option<crate::ScannerDataUsagePublicationAdmission> {
|
||||
Some(crate::ScannerDataUsagePublicationAdmission::unfenced())
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cache_revisions_map_to_compare_and_swap_preconditions() {
|
||||
let missing = DataUsageCacheRevision::Missing.preconditions();
|
||||
|
||||
@@ -513,6 +513,75 @@ where
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn save_config_with_publication_admission_for_epoch<S>(
|
||||
api: Arc<S>,
|
||||
file: &str,
|
||||
data: Vec<u8>,
|
||||
preconditions: HTTPPreconditions,
|
||||
expected_epoch: u64,
|
||||
) -> EcstoreResult<ScannerObjectInfo>
|
||||
where
|
||||
S: ScannerObjectIO + ScannerConfigObjectDelete,
|
||||
{
|
||||
let Some(_admission) = scanner_publication_admission_for_epoch(api.clone(), expected_epoch).await else {
|
||||
return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED));
|
||||
};
|
||||
save_config_with_preconditions(api, file, data, preconditions).await
|
||||
}
|
||||
|
||||
pub(crate) const SCANNER_PUBLICATION_EPOCH_CHANGED: &str = "scanner publication epoch changed before commit";
|
||||
|
||||
pub(crate) fn scanner_publication_epoch_changed(error: &EcstoreError) -> bool {
|
||||
matches!(
|
||||
error,
|
||||
EcstoreError::Io(io_error) if io_error.to_string() == SCANNER_PUBLICATION_EPOCH_CHANGED
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) async fn delete_config_with_publication_admission_for_epoch<S>(
|
||||
api: Arc<S>,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
opts: ScannerObjectOptions,
|
||||
expected_epoch: u64,
|
||||
) -> EcstoreResult<ScannerObjectInfo>
|
||||
where
|
||||
S: ScannerObjectIO + ScannerConfigObjectDelete,
|
||||
{
|
||||
let Some(_admission) = scanner_publication_admission_for_epoch(api.clone(), expected_epoch).await else {
|
||||
return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED));
|
||||
};
|
||||
api.delete_config_object(bucket, object, opts).await
|
||||
}
|
||||
|
||||
/// Capture the storage-owned publication epoch without retaining the read
|
||||
/// guard across a potentially slow metadata read. Callers must compare this
|
||||
/// token with a fresh admission immediately before their conditional write.
|
||||
pub(crate) async fn scanner_publication_epoch<S>(api: Arc<S>) -> Option<u64>
|
||||
where
|
||||
S: ScannerConfigObjectDelete,
|
||||
{
|
||||
let admission = api.scanner_data_usage_publication_admission().await?;
|
||||
Some(admission.epoch())
|
||||
}
|
||||
|
||||
/// Re-admit a publication only when the storage-owned movement epoch is still
|
||||
/// the one observed before the caller's metadata read. The returned guard
|
||||
/// remains held through the caller's short conditional commit.
|
||||
pub(crate) async fn scanner_publication_admission_for_epoch<S>(
|
||||
api: Arc<S>,
|
||||
expected_epoch: u64,
|
||||
) -> Option<ScannerDataUsagePublicationAdmission>
|
||||
where
|
||||
S: ScannerConfigObjectDelete,
|
||||
{
|
||||
let admission = api.scanner_data_usage_publication_admission().await?;
|
||||
if admission.epoch() != expected_epoch {
|
||||
return None;
|
||||
}
|
||||
Some(admission)
|
||||
}
|
||||
|
||||
pub(crate) async fn save_config_shared_with_preconditions<S>(
|
||||
api: Arc<S>,
|
||||
file: &str,
|
||||
@@ -587,6 +656,39 @@ pub trait ScannerConfigObjectDelete: Send + Sync + std::fmt::Debug + 'static {
|
||||
object: &str,
|
||||
opts: ScannerObjectOptions,
|
||||
) -> EcstoreResult<ScannerObjectInfo>;
|
||||
|
||||
/// Acquire storage-owned admission for one short data-usage publication
|
||||
/// commit. Implementations without a storage-owned movement owner fail
|
||||
/// closed; test fixtures opt into the explicit unfenced helper.
|
||||
async fn scanner_data_usage_publication_admission(&self) -> Option<ScannerDataUsagePublicationAdmission> {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pub struct ScannerDataUsagePublicationAdmission {
|
||||
epoch: u64,
|
||||
_read_guard: Option<tokio::sync::OwnedRwLockReadGuard<()>>,
|
||||
}
|
||||
|
||||
impl ScannerDataUsagePublicationAdmission {
|
||||
#[cfg(test)]
|
||||
pub(crate) fn unfenced() -> Self {
|
||||
Self {
|
||||
epoch: 0,
|
||||
_read_guard: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn fenced(read_guard: tokio::sync::OwnedRwLockReadGuard<()>, epoch: u64) -> Self {
|
||||
Self {
|
||||
epoch,
|
||||
_read_guard: Some(read_guard),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn epoch(&self) -> u64 {
|
||||
self.epoch
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
@@ -599,6 +701,28 @@ impl ScannerConfigObjectDelete for ECStore {
|
||||
) -> EcstoreResult<ScannerObjectInfo> {
|
||||
ObjectOperations::delete_object(self, bucket, object, opts).await
|
||||
}
|
||||
|
||||
async fn scanner_data_usage_publication_admission(&self) -> Option<ScannerDataUsagePublicationAdmission> {
|
||||
let (read_guard, epoch) = self.scanner_data_usage_publication_admission_guard().await?;
|
||||
Some(ScannerDataUsagePublicationAdmission::fenced(read_guard, epoch))
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl ScannerConfigObjectDelete for SetDisks {
|
||||
async fn delete_config_object(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
opts: ScannerObjectOptions,
|
||||
) -> EcstoreResult<ScannerObjectInfo> {
|
||||
ObjectOperations::delete_object(self, bucket, object, opts).await
|
||||
}
|
||||
|
||||
async fn scanner_data_usage_publication_admission(&self) -> Option<ScannerDataUsagePublicationAdmission> {
|
||||
let (read_guard, epoch) = self.scanner_data_usage_publication_admission_guard().await?;
|
||||
Some(ScannerDataUsagePublicationAdmission::fenced(read_guard, epoch))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -23,6 +23,7 @@ use crate::storage_api::owner::NS_SCANNER_PROTOCOL_VERSION;
|
||||
use crate::{
|
||||
DATA_USAGE_CACHE_NAME, DataUsageCache, DataUsageCachePrepareOutcome, DataUsageCacheSource, DataUsageEntryInfo,
|
||||
DataUsageScanPlanDigest, Disk, ScannerError, StorageError, resolve_scanner_object_store_handle,
|
||||
scanner_publication_admission_for_epoch, scanner_publication_epoch,
|
||||
};
|
||||
use hmac::{Hmac, KeyInit, Mac};
|
||||
use rustfs_common::heal_channel::HealScanMode;
|
||||
@@ -686,6 +687,9 @@ async fn scan_and_persist_local_bucket(
|
||||
source.pool_index, source.set_index
|
||||
))
|
||||
})?;
|
||||
let expected_publication_epoch = scanner_publication_epoch(set.clone()).await.ok_or_else(|| {
|
||||
RemoteScannerServerError::worker("remote namespace scanner cache publication is blocked by data movement")
|
||||
})?;
|
||||
let cache_name = path_join_buf(&[&bucket, DATA_USAGE_CACHE_NAME]);
|
||||
let guard = acquire_scanner_cache_locks(set.as_ref(), &cache_name, source)
|
||||
.await
|
||||
@@ -708,6 +712,14 @@ async fn scan_and_persist_local_bucket(
|
||||
"remote namespace scanner cache lock was lost before reusing the current snapshot",
|
||||
));
|
||||
}
|
||||
if scanner_publication_admission_for_epoch(set.clone(), expected_publication_epoch)
|
||||
.await
|
||||
.is_none()
|
||||
{
|
||||
return Err(RemoteScannerServerError::retry_bucket(
|
||||
"remote namespace scanner cache publication epoch changed before reusing the current snapshot",
|
||||
));
|
||||
}
|
||||
return Ok(RemoteScannerFrameResult::Complete(Box::new(RemoteScannerComplete {
|
||||
source,
|
||||
scan_plan_digest,
|
||||
@@ -796,9 +808,22 @@ async fn scan_and_persist_local_bucket(
|
||||
.await
|
||||
.map_err(|err| RemoteScannerServerError::worker(format!("remote namespace scanner leader fence changed: {err}")))?;
|
||||
let done_save = Metrics::time(Metric::SaveUsage);
|
||||
let save_result = cache.save_with_revisions(set, &cache_name, &revisions).await;
|
||||
// Each physical main/backup PUT must still prove the epoch captured before
|
||||
// the scan. A movement transition that starts and ends during the scan
|
||||
// therefore cannot admit the stale cache under the new epoch.
|
||||
let save_result = cache
|
||||
.save_with_revisions_for_epoch(set.clone(), &cache_name, &revisions, expected_publication_epoch)
|
||||
.await;
|
||||
done_save();
|
||||
save_result.map_err(|err| RemoteScannerServerError::worker(format!("remote namespace scanner cache save failed: {err}")))?;
|
||||
if scanner_publication_admission_for_epoch(set, expected_publication_epoch)
|
||||
.await
|
||||
.is_none()
|
||||
{
|
||||
return Err(RemoteScannerServerError::retry_bucket(
|
||||
"remote namespace scanner cache publication epoch changed after persistence",
|
||||
));
|
||||
}
|
||||
validate_remote_scanner_request_fence_with_store(next_cycle, leader_epoch, store)
|
||||
.await
|
||||
.map_err(|err| RemoteScannerServerError::worker(format!("remote namespace scanner leader fence changed: {err}")))?;
|
||||
|
||||
+214
-33
@@ -18,6 +18,7 @@ use std::future::Future;
|
||||
use std::sync::Mutex as StdMutex;
|
||||
use std::sync::{Arc, LazyLock, RwLock};
|
||||
|
||||
use self::heal_info::{BackgroundHealInfoReadStatus, read_background_heal_info_with_epoch, save_background_heal_info_for_epoch};
|
||||
use crate::data_usage_define::{
|
||||
BACKGROUND_HEAL_INFO_PATH, DATA_USAGE_BLOOM_NAME_PATH, DATA_USAGE_OBJ_NAME_PATH, DATA_USAGE_OBSERVED_OBJ_NAME_PATH,
|
||||
DataUsageCache, DataUsageCacheRevision, LEGACY_DATA_USAGE_OBJ_NAME_PATH, read_config_revision, read_config_with_revision,
|
||||
@@ -30,8 +31,9 @@ use crate::runtime_config::{
|
||||
use crate::scanner_budget::{ScannerCycleBudget, ScannerCycleBudgetConfig, ScannerCycleBudgetReason};
|
||||
use crate::scanner_folder::{data_usage_update_dir_cycles, heal_object_select_prob};
|
||||
use crate::scanner_io::{
|
||||
ScannerCycleDeferReason, ScannerCycleStatus, ScannerIOCycle, dirty_usage_bucket_notified, dirty_usage_buckets_pending,
|
||||
dirty_usage_generation, scanner_dirty_usage_state, scanner_maintenance_changed, scanner_maintenance_generation,
|
||||
ScannerCycleDeferReason, ScannerCycleResult, ScannerCycleStatus, ScannerIOCycle, dirty_usage_bucket_notified,
|
||||
dirty_usage_buckets_pending, dirty_usage_generation, scanner_dirty_usage_state, scanner_maintenance_changed,
|
||||
scanner_maintenance_generation,
|
||||
};
|
||||
use crate::sleeper::{SCANNER_SLEEPER, set_scanner_default_speed};
|
||||
use crate::{DataUsageInfo, ScannerActivityGuard, ScannerError, ScannerRuntimeGuard};
|
||||
@@ -66,10 +68,12 @@ use crate::storage_api::scan::{
|
||||
SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SCANNER_ACTIVITY_PROTOCOL_VERSION,
|
||||
};
|
||||
use crate::{
|
||||
ECStore, EcstoreError, RUSTFS_META_BUCKET, ScannerLifecycleConfigExt as _, ScannerReplicationConfigExt as _,
|
||||
get_lifecycle_config, get_replication_config, invalidate_admin_data_usage_snapshot_cache,
|
||||
invalidate_data_usage_snapshot_cache, read_config, replace_bucket_usage_memory_from_info, save_config,
|
||||
save_config_shared_with_preconditions, save_config_with_preconditions, scanner_is_erasure_sd,
|
||||
ECStore, EcstoreError, RUSTFS_META_BUCKET, SCANNER_PUBLICATION_EPOCH_CHANGED, ScannerLifecycleConfigExt as _,
|
||||
ScannerReplicationConfigExt as _, delete_config_with_publication_admission_for_epoch, get_lifecycle_config,
|
||||
get_replication_config, invalidate_admin_data_usage_snapshot_cache, invalidate_data_usage_snapshot_cache, read_config,
|
||||
replace_bucket_usage_memory_from_info, save_config, save_config_shared_with_preconditions, save_config_with_preconditions,
|
||||
save_config_with_publication_admission_for_epoch, scanner_is_erasure_sd, scanner_publication_admission_for_epoch,
|
||||
scanner_publication_epoch, scanner_publication_epoch_changed,
|
||||
};
|
||||
|
||||
const LOG_COMPONENT_SCANNER: &str = "scanner";
|
||||
@@ -346,6 +350,23 @@ fn data_usage_info_is_cold(info: &DataUsageInfo) -> bool {
|
||||
!info.is_complete_bucket_usage_snapshot()
|
||||
}
|
||||
|
||||
pub(super) fn data_usage_info_has_persisted_baseline_identity(info: &DataUsageInfo) -> bool {
|
||||
if info.is_complete_bucket_usage_snapshot() {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Pre-marker snapshots remain readable only when their legacy identity is
|
||||
// complete: a timestamp, a scanner cycle, and an exact bucket cardinality.
|
||||
// A current snapshot with only scanner_epoch/scanner_cycle (or an explicit
|
||||
// incomplete marker) is not evidence of a durable usage baseline.
|
||||
!info.usage_snapshot_complete
|
||||
&& info.scanner_epoch.is_none()
|
||||
&& info.usage_snapshot_converged != Some(false)
|
||||
&& info.last_update.is_some()
|
||||
&& info.scanner_cycle.is_some()
|
||||
&& u64::try_from(info.buckets_usage.len()).ok() == Some(info.buckets_count)
|
||||
}
|
||||
|
||||
fn usage_cache_needs_prompt_scan(authoritative: &DataUsageInfo, observed: Option<&DataUsageInfo>) -> bool {
|
||||
data_usage_info_is_cold(authoritative)
|
||||
|| observed.is_some_and(|observed| observed_data_usage_is_newer(observed, authoritative))
|
||||
@@ -381,9 +402,18 @@ fn data_usage_backup_due(data_usage_info: &DataUsageInfo) -> bool {
|
||||
.is_some_and(|cycle| cycle % DATA_USAGE_BACKUP_INTERVAL_CYCLES == 0)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
async fn sync_data_usage_backup_from_primary(
|
||||
ctx: &CancellationToken,
|
||||
storeapi: Arc<impl ScannerObjectIO>,
|
||||
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
|
||||
) -> Result<(), EcstoreError> {
|
||||
sync_data_usage_backup_from_primary_for_epoch(ctx, storeapi, None).await
|
||||
}
|
||||
|
||||
async fn sync_data_usage_backup_from_primary_for_epoch(
|
||||
ctx: &CancellationToken,
|
||||
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
|
||||
expected_publication_epoch: Option<u64>,
|
||||
) -> Result<(), EcstoreError> {
|
||||
let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str());
|
||||
for retry in 0..=SCANNER_PERSIST_CAS_RETRIES {
|
||||
@@ -391,26 +421,62 @@ async fn sync_data_usage_backup_from_primary(
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let read_epoch = match expected_publication_epoch {
|
||||
Some(expected_epoch) => {
|
||||
if scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch)
|
||||
.await
|
||||
.is_none()
|
||||
{
|
||||
return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED));
|
||||
}
|
||||
expected_epoch
|
||||
}
|
||||
None => scanner_publication_epoch(storeapi.clone())
|
||||
.await
|
||||
.ok_or_else(|| EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED))?,
|
||||
};
|
||||
let (primary, _) = read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()).await?;
|
||||
let primary = primary.ok_or_else(|| EcstoreError::other("authoritative data usage snapshot is missing"))?;
|
||||
serde_json::from_slice::<DataUsageInfo>(&primary)
|
||||
let primary = primary.ok_or(EcstoreError::ConfigNotFound)?;
|
||||
let primary_info = serde_json::from_slice::<DataUsageInfo>(&primary)
|
||||
.map_err(|err| EcstoreError::other(format!("authoritative data usage snapshot is invalid: {err}")))?;
|
||||
if !data_usage_info_has_persisted_baseline_identity(&primary_info) {
|
||||
return Err(EcstoreError::other(
|
||||
"authoritative data usage snapshot has no persisted baseline identity",
|
||||
));
|
||||
}
|
||||
let primary = Bytes::from(primary);
|
||||
|
||||
let (backup, revision) = read_config_with_revision(storeapi.clone(), &backup_path).await?;
|
||||
if backup.as_deref() == Some(primary.as_ref()) {
|
||||
return Ok(());
|
||||
if scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch)
|
||||
.await
|
||||
.is_some()
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
if retry < SCANNER_PERSIST_CAS_RETRIES {
|
||||
continue;
|
||||
}
|
||||
return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED));
|
||||
}
|
||||
|
||||
let sha256hex = Some(hex_simd::encode_to_string(Sha256::digest(&primary), hex_simd::AsciiCase::Lower));
|
||||
let save_result = save_config_shared_with_preconditions(
|
||||
storeapi.clone(),
|
||||
&backup_path,
|
||||
primary.clone(),
|
||||
sha256hex,
|
||||
revision.preconditions(),
|
||||
)
|
||||
.await;
|
||||
let save_result = {
|
||||
let Some(_publication_admission) = scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch).await else {
|
||||
if retry < SCANNER_PERSIST_CAS_RETRIES {
|
||||
continue;
|
||||
}
|
||||
return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED));
|
||||
};
|
||||
save_config_shared_with_preconditions(
|
||||
storeapi.clone(),
|
||||
&backup_path,
|
||||
primary.clone(),
|
||||
sha256hex,
|
||||
revision.preconditions(),
|
||||
)
|
||||
.await
|
||||
};
|
||||
|
||||
match save_result {
|
||||
Ok(_) => {}
|
||||
@@ -427,9 +493,20 @@ async fn sync_data_usage_backup_from_primary(
|
||||
}
|
||||
|
||||
let (current_primary, _) = read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()).await?;
|
||||
if current_primary.as_deref() == Some(primary.as_ref()) {
|
||||
if current_primary.as_deref() == Some(primary.as_ref())
|
||||
&& scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch)
|
||||
.await
|
||||
.is_some()
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
if expected_publication_epoch.is_some()
|
||||
&& scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch)
|
||||
.await
|
||||
.is_none()
|
||||
{
|
||||
return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED));
|
||||
}
|
||||
if retry < SCANNER_PERSIST_CAS_RETRIES {
|
||||
continue;
|
||||
}
|
||||
@@ -1052,7 +1129,7 @@ async fn fence_scanner_epoch_after_cycle_timeout<Store, LockLost>(
|
||||
lock_lost: LockLost,
|
||||
) -> bool
|
||||
where
|
||||
Store: ScannerObjectIO,
|
||||
Store: ScannerObjectIO + ScannerConfigObjectDelete,
|
||||
LockLost: Future<Output = ()>,
|
||||
{
|
||||
let fence_ctx = ctx.child_token();
|
||||
@@ -1089,7 +1166,7 @@ async fn handle_scanner_cycle_deadline<Store>(
|
||||
worker_stopped: bool,
|
||||
guard: &mut NamespaceLockGuard,
|
||||
) where
|
||||
Store: ScannerObjectIO,
|
||||
Store: ScannerObjectIO + ScannerConfigObjectDelete,
|
||||
{
|
||||
let fenced = fence_scanner_epoch_after_cycle_timeout(
|
||||
ctx,
|
||||
@@ -1182,7 +1259,33 @@ async fn run_data_scanner_cycle_with_budget(
|
||||
|
||||
let mut cycle_metrics_guard = ScannerCycleMetricsGuard::new(cycle_info.clone()).await;
|
||||
|
||||
let mut background_heal_info = read_background_heal_info(storeapi.clone()).await;
|
||||
// Refresh the storage-owned movement snapshot before reading background
|
||||
// heal state. A missing heal object yields an in-memory default; do not
|
||||
// let that default influence a cycle while publication is blocked.
|
||||
if storeapi.scanner_data_usage_publication_blocked().await {
|
||||
mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await;
|
||||
return ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
|
||||
}
|
||||
let background_heal_read = read_background_heal_info_with_epoch(storeapi.clone()).await;
|
||||
match background_heal_read.status {
|
||||
BackgroundHealInfoReadStatus::Blocked => {
|
||||
mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await;
|
||||
return ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
|
||||
}
|
||||
BackgroundHealInfoReadStatus::Transient => {
|
||||
mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await;
|
||||
return ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable);
|
||||
}
|
||||
BackgroundHealInfoReadStatus::Failed => {
|
||||
mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await;
|
||||
return ScannerCycleOutcome::Failed;
|
||||
}
|
||||
BackgroundHealInfoReadStatus::ErasureSd
|
||||
| BackgroundHealInfoReadStatus::Loaded
|
||||
| BackgroundHealInfoReadStatus::Missing => {}
|
||||
}
|
||||
let mut background_heal_info = background_heal_read.info;
|
||||
let background_heal_epoch = background_heal_read.expected_epoch;
|
||||
|
||||
let scan_mode = get_cycle_scan_mode(
|
||||
cycle_info.current,
|
||||
@@ -1209,11 +1312,23 @@ async fn run_data_scanner_cycle_with_budget(
|
||||
configured_bitrot_cycle,
|
||||
) {
|
||||
background_heal_info = new_heal_info.clone();
|
||||
save_background_heal_info(storeapi.clone(), new_heal_info).await;
|
||||
save_background_heal_info_for_epoch(storeapi.clone(), new_heal_info, background_heal_epoch).await;
|
||||
}
|
||||
|
||||
let cycle_start = std::time::Instant::now();
|
||||
let usage_persist_baseline = match read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()).await {
|
||||
// Baseline reads are part of the same publication proof as the eventual
|
||||
// scanner aggregate. Hold only the short storage-owned admission guard
|
||||
// across this metadata read; the full bucket scan runs after it is
|
||||
// released and carries the captured epoch forward.
|
||||
let Some((baseline_publication_guard, baseline_publication_epoch)) =
|
||||
storeapi.scanner_data_usage_publication_admission_guard().await
|
||||
else {
|
||||
mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await;
|
||||
return ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
|
||||
};
|
||||
let usage_persist_baseline_result = read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()).await;
|
||||
drop(baseline_publication_guard);
|
||||
let usage_persist_baseline = match usage_persist_baseline_result {
|
||||
Ok((data, revision)) => DataUsagePersistBaseline {
|
||||
data: data.map(Bytes::from),
|
||||
revision,
|
||||
@@ -1250,9 +1365,17 @@ async fn run_data_scanner_cycle_with_budget(
|
||||
)
|
||||
.await;
|
||||
let publication_defer_reason = match &scan_result {
|
||||
Ok(result)
|
||||
if result
|
||||
.publication_epoch()
|
||||
.is_some_and(|publication_epoch| publication_epoch != baseline_publication_epoch) =>
|
||||
{
|
||||
Some(ScannerCycleDeferReason::DataMovement)
|
||||
}
|
||||
Ok(result) => final_data_usage_publication_defer_reason(storeapi.as_ref(), result.status).await,
|
||||
Err(_) => Some(ScannerCycleDeferReason::ActivityBaselineUnavailable),
|
||||
};
|
||||
let publication_epoch = scan_result.as_ref().ok().and_then(ScannerCycleResult::publication_epoch);
|
||||
let budget_elapsed = cycle_budget.budget_elapsed() && !ctx.is_cancelled();
|
||||
let usage_persist_outcome = match publication_defer_reason {
|
||||
Some(reason) => {
|
||||
@@ -1267,12 +1390,13 @@ async fn run_data_scanner_cycle_with_budget(
|
||||
let ctx_clone = ctx.clone();
|
||||
let route_probe_store = storeapi.clone();
|
||||
let mut usage_persist_task = AbortOnDropHandle::new(tokio::spawn(async move {
|
||||
store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe(
|
||||
store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe_for_publication_epoch(
|
||||
ctx_clone,
|
||||
storeapi_clone,
|
||||
receiver,
|
||||
Some(leader_epoch),
|
||||
Some(usage_persist_baseline),
|
||||
publication_epoch,
|
||||
move || {
|
||||
let storeapi = route_probe_store.clone();
|
||||
async move { storeapi.scanner_data_usage_publication_blocked().await }
|
||||
@@ -1344,7 +1468,7 @@ async fn run_data_scanner_cycle_with_budget(
|
||||
if !ctx.is_cancelled()
|
||||
&& let Some(new_heal_info) = background_heal_info_for_scan_result(background_heal_info.clone(), scan_mode, false)
|
||||
{
|
||||
save_background_heal_info(storeapi.clone(), new_heal_info).await;
|
||||
save_background_heal_info_for_epoch(storeapi.clone(), new_heal_info, background_heal_epoch).await;
|
||||
}
|
||||
mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await;
|
||||
return ScannerCycleOutcome::Failed;
|
||||
@@ -1377,19 +1501,29 @@ async fn run_data_scanner_cycle_with_budget(
|
||||
"Scanner cycle is recovering to a newer durable cache generation"
|
||||
);
|
||||
emit_scan_cycle_partial_with_source(cycle_start.elapsed(), ScanCyclePartialReason::Unknown, None);
|
||||
let persisted = persist_required_scanner_cycle_floor(
|
||||
let persisted = persist_required_scanner_cycle_floor_for_epoch(
|
||||
ctx,
|
||||
storeapi.clone(),
|
||||
cycle_info,
|
||||
cycle_revision,
|
||||
leader_epoch,
|
||||
required_cycle,
|
||||
&mut cycle_metrics_guard,
|
||||
ScannerCycleFloorOptions {
|
||||
required_cycle,
|
||||
expected_publication_epoch: publication_epoch,
|
||||
},
|
||||
)
|
||||
.await;
|
||||
return if persisted {
|
||||
cycle_budget.mark_cycle_state_persisted();
|
||||
ScannerCycleOutcome::Partial
|
||||
} else if let Some(expected_epoch) = publication_epoch
|
||||
&& scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch)
|
||||
.await
|
||||
.is_none()
|
||||
{
|
||||
emit_scan_cycle_deferred(cycle_start.elapsed());
|
||||
ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement)
|
||||
} else {
|
||||
ScannerCycleOutcome::Failed
|
||||
};
|
||||
@@ -1446,18 +1580,26 @@ async fn run_data_scanner_cycle_with_budget(
|
||||
scan_cycle_partial_reason(budget_reason),
|
||||
scan_cycle_partial_source(budget_reason),
|
||||
);
|
||||
let persisted = finalize_partial_scan_cycle(
|
||||
let persisted = finalize_partial_scan_cycle_for_epoch(
|
||||
ctx,
|
||||
storeapi.clone(),
|
||||
cycle_info,
|
||||
cycle_revision,
|
||||
leader_epoch,
|
||||
&mut cycle_metrics_guard,
|
||||
publication_epoch,
|
||||
)
|
||||
.await;
|
||||
return if persisted {
|
||||
cycle_budget.mark_cycle_state_persisted();
|
||||
ScannerCycleOutcome::Partial
|
||||
} else if let Some(expected_epoch) = publication_epoch
|
||||
&& scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch)
|
||||
.await
|
||||
.is_none()
|
||||
{
|
||||
emit_scan_cycle_deferred(cycle_start.elapsed());
|
||||
ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement)
|
||||
} else {
|
||||
ScannerCycleOutcome::Failed
|
||||
};
|
||||
@@ -1531,18 +1673,26 @@ async fn run_data_scanner_cycle_with_budget(
|
||||
);
|
||||
}
|
||||
emit_scan_cycle_partial_with_source(cycle_start.elapsed(), ScanCyclePartialReason::Unknown, None);
|
||||
let persisted = finalize_partial_scan_cycle(
|
||||
let persisted = finalize_partial_scan_cycle_for_epoch(
|
||||
ctx,
|
||||
storeapi.clone(),
|
||||
cycle_info,
|
||||
cycle_revision,
|
||||
leader_epoch,
|
||||
&mut cycle_metrics_guard,
|
||||
publication_epoch,
|
||||
)
|
||||
.await;
|
||||
return if persisted {
|
||||
cycle_budget.mark_cycle_state_persisted();
|
||||
ScannerCycleOutcome::Partial
|
||||
} else if let Some(expected_epoch) = publication_epoch
|
||||
&& scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch)
|
||||
.await
|
||||
.is_none()
|
||||
{
|
||||
emit_scan_cycle_deferred(cycle_start.elapsed());
|
||||
ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement)
|
||||
} else {
|
||||
ScannerCycleOutcome::Failed
|
||||
};
|
||||
@@ -1572,13 +1722,14 @@ async fn run_data_scanner_cycle_with_budget(
|
||||
state = "superseded",
|
||||
"Scanner cycle usage snapshot was superseded by concurrent namespace activity"
|
||||
);
|
||||
if finalize_partial_scan_cycle(
|
||||
if finalize_partial_scan_cycle_for_epoch(
|
||||
ctx,
|
||||
storeapi.clone(),
|
||||
cycle_info,
|
||||
cycle_revision,
|
||||
leader_epoch,
|
||||
&mut cycle_metrics_guard,
|
||||
publication_epoch,
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -1586,11 +1737,29 @@ async fn run_data_scanner_cycle_with_budget(
|
||||
emit_scan_cycle_superseded(cycle_start.elapsed());
|
||||
return ScannerCycleOutcome::Superseded;
|
||||
}
|
||||
if let Some(expected_epoch) = publication_epoch
|
||||
&& scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch)
|
||||
.await
|
||||
.is_none()
|
||||
{
|
||||
emit_scan_cycle_deferred(cycle_start.elapsed());
|
||||
return ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
|
||||
}
|
||||
emit_scan_cycle_complete(false, cycle_start.elapsed());
|
||||
return ScannerCycleOutcome::Failed;
|
||||
}
|
||||
ScannerCycleOutcome::Completed | ScannerCycleOutcome::CompletedWithPendingMaintenance => {}
|
||||
}
|
||||
if let Some(expected_epoch) = publication_epoch
|
||||
&& scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch)
|
||||
.await
|
||||
.is_none()
|
||||
{
|
||||
emit_scan_cycle_deferred(cycle_start.elapsed());
|
||||
mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await;
|
||||
return ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
|
||||
}
|
||||
let previous_cycle_info = cycle_info.clone();
|
||||
if let Err(err) = advance_scanner_cycle(cycle_info) {
|
||||
error!(
|
||||
target: "rustfs::scanner",
|
||||
@@ -1610,7 +1779,19 @@ async fn run_data_scanner_cycle_with_budget(
|
||||
global_metrics().clear_current_scan_mode();
|
||||
|
||||
retain_recent_cycle_completions(&mut cycle_info.cycle_completed);
|
||||
if !persist_scanner_cycle_state(ctx, storeapi.clone(), cycle_info, cycle_revision, leader_epoch).await {
|
||||
if !persist_scanner_cycle_state_for_epoch(ctx, storeapi.clone(), cycle_info, cycle_revision, leader_epoch, publication_epoch)
|
||||
.await
|
||||
{
|
||||
if let Some(expected_epoch) = publication_epoch
|
||||
&& scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch)
|
||||
.await
|
||||
.is_none()
|
||||
{
|
||||
*cycle_info = previous_cycle_info;
|
||||
emit_scan_cycle_deferred(cycle_start.elapsed());
|
||||
mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await;
|
||||
return ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
|
||||
}
|
||||
cycle_metrics_guard.finish(cycle_info.clone()).await;
|
||||
emit_scan_cycle_complete(false, cycle_start.elapsed());
|
||||
return ScannerCycleOutcome::Failed;
|
||||
@@ -1620,7 +1801,7 @@ async fn run_data_scanner_cycle_with_budget(
|
||||
done_cycle();
|
||||
emit_scan_cycle_complete(true, cycle_start.elapsed());
|
||||
if let Some(new_heal_info) = background_heal_info_for_scan_result(background_heal_info.clone(), scan_mode, true) {
|
||||
save_background_heal_info(storeapi.clone(), new_heal_info).await;
|
||||
save_background_heal_info_for_epoch(storeapi.clone(), new_heal_info, background_heal_epoch).await;
|
||||
}
|
||||
|
||||
info!(
|
||||
|
||||
@@ -162,6 +162,8 @@ pub(crate) enum ScannerCycleStateStartup {
|
||||
enum CycleRecoveryMarkerReadError {
|
||||
#[error("cycle recovery marker backend read failed: {0}")]
|
||||
Backend(#[source] EcstoreError),
|
||||
#[error("cycle recovery marker publication is blocked by data movement")]
|
||||
PublicationBlocked,
|
||||
#[error("invalid cycle recovery marker: {0}")]
|
||||
Invalid(&'static str),
|
||||
#[error("cycle recovery marker revision changed while publishing")]
|
||||
@@ -326,12 +328,13 @@ fn cycle_state_generation_and_epoch(buf: &[u8]) -> (u64, u64) {
|
||||
}
|
||||
|
||||
async fn persist_cycle_recovery_marker(
|
||||
storeapi: Arc<impl ScannerObjectIO>,
|
||||
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
|
||||
primary_revision: &DataUsageCacheRevision,
|
||||
generation: u64,
|
||||
leader_epoch: u64,
|
||||
classification: &'static str,
|
||||
reason: &'static str,
|
||||
expected_epoch: u64,
|
||||
) -> Result<ScannerCycleRecoveryMarker, CycleRecoveryMarkerReadError> {
|
||||
let now = unix_now_secs();
|
||||
let (existing, existing_revision) = match read_cycle_recovery_marker_bytes(storeapi.clone()).await {
|
||||
@@ -370,6 +373,9 @@ async fn persist_cycle_recovery_marker(
|
||||
state: "blocked".to_string(),
|
||||
};
|
||||
let bytes = serde_json::to_vec(&marker).map_err(|_| CycleRecoveryMarkerReadError::Invalid("marker serialization failed"))?;
|
||||
let Some(_publication_admission) = scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch).await else {
|
||||
return Err(CycleRecoveryMarkerReadError::PublicationBlocked);
|
||||
};
|
||||
let save_result = save_config_with_preconditions(
|
||||
storeapi.clone(),
|
||||
DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(),
|
||||
@@ -478,7 +484,7 @@ async fn read_cycle_recovery_marker_revision(
|
||||
}
|
||||
|
||||
async fn quarantine_invalid_cycle_state(
|
||||
storeapi: Arc<impl ScannerObjectIO>,
|
||||
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
|
||||
revision: &DataUsageCacheRevision,
|
||||
buf: &[u8],
|
||||
) -> ScannerCycleStateStartup {
|
||||
@@ -488,7 +494,7 @@ async fn quarantine_invalid_cycle_state(
|
||||
}
|
||||
|
||||
async fn quarantine_invalid_cycle_state_with_reason(
|
||||
storeapi: Arc<impl ScannerObjectIO>,
|
||||
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
|
||||
revision: &DataUsageCacheRevision,
|
||||
generation: u64,
|
||||
leader_epoch: u64,
|
||||
@@ -515,7 +521,19 @@ async fn quarantine_invalid_cycle_state_with_reason(
|
||||
reason: Some(reason.to_string()),
|
||||
};
|
||||
set_scanner_cycle_recovery_status(base_status);
|
||||
match persist_cycle_recovery_marker(storeapi, revision, generation, leader_epoch, classification, reason).await {
|
||||
let Some(expected_epoch) = scanner_publication_epoch(storeapi.clone()).await else {
|
||||
set_scanner_cycle_recovery_status(recovery_status(
|
||||
"transient",
|
||||
Some("cycle recovery marker publication is blocked by data movement"),
|
||||
true,
|
||||
));
|
||||
return ScannerCycleStateStartup::Transient(ScannerError::Other(
|
||||
"cycle recovery marker publication is blocked by data movement".to_string(),
|
||||
));
|
||||
};
|
||||
match persist_cycle_recovery_marker(storeapi, revision, generation, leader_epoch, classification, reason, expected_epoch)
|
||||
.await
|
||||
{
|
||||
Ok(marker) => set_scanner_cycle_recovery_status(recovery_status_from_marker(&marker, "blocked")),
|
||||
Err(CycleRecoveryMarkerReadError::Backend(_)) => {
|
||||
// Keep the poison object untouched and retry marker creation with the
|
||||
@@ -524,6 +542,16 @@ async fn quarantine_invalid_cycle_state_with_reason(
|
||||
"failed to persist scanner cycle recovery marker".to_string(),
|
||||
));
|
||||
}
|
||||
Err(CycleRecoveryMarkerReadError::PublicationBlocked) => {
|
||||
set_scanner_cycle_recovery_status(recovery_status(
|
||||
"transient",
|
||||
Some("cycle recovery marker publication is blocked by data movement"),
|
||||
true,
|
||||
));
|
||||
return ScannerCycleStateStartup::Transient(ScannerError::Other(
|
||||
"cycle recovery marker publication is blocked by data movement".to_string(),
|
||||
));
|
||||
}
|
||||
Err(CycleRecoveryMarkerReadError::Conflict) => {
|
||||
set_scanner_cycle_recovery_status(recovery_status(
|
||||
"transient",
|
||||
@@ -546,16 +574,18 @@ async fn mark_cycle_recovery_cleanup_pending(
|
||||
storeapi: Arc<ECStore>,
|
||||
mut marker: ScannerCycleRecoveryMarker,
|
||||
marker_revision: &DataUsageCacheRevision,
|
||||
expected_epoch: u64,
|
||||
) -> Result<(ScannerCycleRecoveryMarker, DataUsageCacheRevision), ScannerError> {
|
||||
marker.state = "cleanup-pending".to_string();
|
||||
marker.last_attempt_at_unix_secs = unix_now_secs();
|
||||
let bytes = serde_json::to_vec(&marker)
|
||||
.map_err(|err| ScannerError::Other(format!("failed to encode cycle recovery marker: {err}")))?;
|
||||
let info = save_config_with_preconditions(
|
||||
let info = save_config_with_publication_admission_for_epoch(
|
||||
storeapi.clone(),
|
||||
DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(),
|
||||
bytes,
|
||||
marker_revision.preconditions(),
|
||||
expected_epoch,
|
||||
)
|
||||
.await
|
||||
.map_err(|err| ScannerError::Other(format!("failed to mark cycle recovery cleanup pending: {err}")))?;
|
||||
@@ -567,7 +597,9 @@ async fn mark_cycle_recovery_cleanup_pending(
|
||||
Ok((marker, revision))
|
||||
}
|
||||
|
||||
pub(crate) async fn load_scanner_cycle_state_for_startup(storeapi: Arc<impl ScannerObjectIO>) -> ScannerCycleStateStartup {
|
||||
pub(crate) async fn load_scanner_cycle_state_for_startup(
|
||||
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
|
||||
) -> ScannerCycleStateStartup {
|
||||
let marker = match read_cycle_recovery_marker_bytes(storeapi.clone()).await {
|
||||
Ok((None, _)) => None,
|
||||
Ok((Some(data), marker_revision)) => match serde_json::from_slice::<ScannerCycleRecoveryMarker>(&data) {
|
||||
@@ -594,6 +626,16 @@ pub(crate) async fn load_scanner_cycle_state_for_startup(storeapi: Arc<impl Scan
|
||||
"failed to read scanner cycle recovery marker: {err}"
|
||||
)));
|
||||
}
|
||||
Err(CycleRecoveryMarkerReadError::PublicationBlocked) => {
|
||||
set_scanner_cycle_recovery_status(recovery_status(
|
||||
"transient",
|
||||
Some("cycle recovery marker publication is blocked by data movement"),
|
||||
true,
|
||||
));
|
||||
return ScannerCycleStateStartup::Transient(ScannerError::Other(
|
||||
"cycle recovery marker publication is blocked by data movement".to_string(),
|
||||
));
|
||||
}
|
||||
Err(CycleRecoveryMarkerReadError::Invalid(reason)) => {
|
||||
set_scanner_cycle_recovery_status(recovery_status("recovery-required", Some(reason), false));
|
||||
return ScannerCycleStateStartup::Blocked;
|
||||
@@ -750,6 +792,10 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
|
||||
return Err(ScannerError::Other("scanner leader lock was lost before recovery reset".to_string()));
|
||||
}
|
||||
|
||||
let Some(reset_epoch) = scanner_publication_epoch(storeapi.clone()).await else {
|
||||
return Err(ScannerError::Other("scanner recovery reset is blocked by data movement".to_string()));
|
||||
};
|
||||
|
||||
let (marker_data, marker_revision, marker_body_invalid) = match read_cycle_recovery_marker_bytes(storeapi.clone()).await {
|
||||
Ok((marker_data, marker_revision)) => (marker_data, marker_revision, false),
|
||||
Err(CycleRecoveryMarkerReadError::Invalid(_)) => {
|
||||
@@ -835,7 +881,7 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
|
||||
};
|
||||
if let Some((primary_cycle, primary_epoch)) = primary_state {
|
||||
let (cleanup_marker, cleanup_marker_revision) =
|
||||
mark_cycle_recovery_cleanup_pending(storeapi.clone(), marker.clone(), &marker_revision).await?;
|
||||
mark_cycle_recovery_cleanup_pending(storeapi.clone(), marker.clone(), &marker_revision, reset_epoch).await?;
|
||||
set_scanner_cycle_recovery_status(recovery_status_from_marker(&cleanup_marker, "cleanup-pending"));
|
||||
let usage_floor = persisted_usage_floor(storeapi.clone()).await?;
|
||||
let fence_epoch = primary_epoch
|
||||
@@ -855,14 +901,21 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
|
||||
"preserved scanner cycle state exceeds the bounded object size".to_string(),
|
||||
));
|
||||
}
|
||||
let preserved_info = save_config_with_preconditions(
|
||||
let preserved_info = save_config_with_publication_admission_for_epoch(
|
||||
storeapi.clone(),
|
||||
DATA_USAGE_BLOOM_NAME_PATH.as_str(),
|
||||
preserved_data,
|
||||
primary_revision.preconditions(),
|
||||
reset_epoch,
|
||||
)
|
||||
.await
|
||||
.map_err(|err| ScannerError::Other(format!("failed to fence preserved scanner cycle state: {err}")))?;
|
||||
.map_err(|err| {
|
||||
if scanner_publication_epoch_changed(&err) {
|
||||
ScannerError::Other("scanner recovery reset deferred by a movement epoch change".to_string())
|
||||
} else {
|
||||
ScannerError::Other(format!("failed to fence preserved scanner cycle state: {err}"))
|
||||
}
|
||||
})?;
|
||||
let preserved_revision = preserved_info
|
||||
.etag
|
||||
.filter(|etag| !etag.is_empty())
|
||||
@@ -873,7 +926,7 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
|
||||
"scanner leader lock was lost after fencing newer cycle state".to_string(),
|
||||
));
|
||||
}
|
||||
fence_scanner_usage_epoch(&ctx, storeapi.clone(), fence_epoch)
|
||||
fence_scanner_usage_epoch_with_expected_epoch(&ctx, storeapi.clone(), fence_epoch, Some(reset_epoch))
|
||||
.await
|
||||
.map_err(|err| ScannerError::Other(format!("failed to fence preserved scanner usage epoch: {err}")))?;
|
||||
if guard.is_lock_lost() {
|
||||
@@ -889,20 +942,27 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
|
||||
"scanner cycle state changed before recovery marker cleanup".to_string(),
|
||||
));
|
||||
}
|
||||
storeapi
|
||||
.delete_config_object(
|
||||
RUSTFS_META_BUCKET,
|
||||
DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(),
|
||||
ScannerObjectOptions {
|
||||
// This is one exact metadata object. Prefix-delete mode
|
||||
// bypasses HTTP preconditions in the ECStore path.
|
||||
delete_prefix: false,
|
||||
http_preconditions: Some(cleanup_marker_revision.preconditions()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map_err(|err| ScannerError::Other(format!("failed to clear stale cycle recovery marker: {err}")))?;
|
||||
delete_config_with_publication_admission_for_epoch(
|
||||
storeapi.clone(),
|
||||
RUSTFS_META_BUCKET,
|
||||
DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(),
|
||||
ScannerObjectOptions {
|
||||
// This is one exact metadata object. Prefix-delete mode
|
||||
// bypasses HTTP preconditions in the ECStore path.
|
||||
delete_prefix: false,
|
||||
http_preconditions: Some(cleanup_marker_revision.preconditions()),
|
||||
..Default::default()
|
||||
},
|
||||
reset_epoch,
|
||||
)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
if scanner_publication_epoch_changed(&err) {
|
||||
ScannerError::Other("scanner recovery reset deferred by a movement epoch change".to_string())
|
||||
} else {
|
||||
ScannerError::Other(format!("failed to clear stale cycle recovery marker: {err}"))
|
||||
}
|
||||
})?;
|
||||
set_scanner_cycle_recovery_status(recovery_status("healthy", None, false));
|
||||
super::notify_scanner_cycle_recovery_wake();
|
||||
return Ok(());
|
||||
@@ -946,16 +1006,23 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
|
||||
let (marker, marker_revision) = if marker.state == "cleanup-pending" {
|
||||
(marker, marker_revision)
|
||||
} else {
|
||||
mark_cycle_recovery_cleanup_pending(storeapi.clone(), marker, &marker_revision).await?
|
||||
mark_cycle_recovery_cleanup_pending(storeapi.clone(), marker, &marker_revision, reset_epoch).await?
|
||||
};
|
||||
let rebuilt_info = save_config_with_preconditions(
|
||||
let rebuilt_info = save_config_with_publication_admission_for_epoch(
|
||||
storeapi.clone(),
|
||||
DATA_USAGE_BLOOM_NAME_PATH.as_str(),
|
||||
data,
|
||||
primary_revision.preconditions(),
|
||||
reset_epoch,
|
||||
)
|
||||
.await
|
||||
.map_err(|err| ScannerError::Other(format!("failed to persist rebuilt scanner cycle state: {err}")))?;
|
||||
.map_err(|err| {
|
||||
if scanner_publication_epoch_changed(&err) {
|
||||
ScannerError::Other("scanner recovery reset deferred by a movement epoch change".to_string())
|
||||
} else {
|
||||
ScannerError::Other(format!("failed to persist rebuilt scanner cycle state: {err}"))
|
||||
}
|
||||
})?;
|
||||
let rebuilt_revision = rebuilt_info
|
||||
.etag
|
||||
.filter(|etag| !etag.is_empty())
|
||||
@@ -965,7 +1032,8 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
|
||||
"scanner leader lock was lost after rebuilding cycle state".to_string(),
|
||||
));
|
||||
}
|
||||
if let Err(err) = fence_scanner_usage_epoch(&ctx, storeapi.clone(), leader_epoch).await {
|
||||
if let Err(err) = fence_scanner_usage_epoch_with_expected_epoch(&ctx, storeapi.clone(), leader_epoch, Some(reset_epoch)).await
|
||||
{
|
||||
set_scanner_cycle_recovery_status(ScannerCycleRecoveryStatus {
|
||||
path: DATA_USAGE_BLOOM_NAME_PATH.clone(),
|
||||
quarantine_path: Some(DATA_USAGE_BLOOM_RECOVERY_PATH.clone()),
|
||||
@@ -1034,20 +1102,40 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
|
||||
));
|
||||
}
|
||||
|
||||
if let Err(err) = storeapi
|
||||
.delete_config_object(
|
||||
RUSTFS_META_BUCKET,
|
||||
DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(),
|
||||
ScannerObjectOptions {
|
||||
// This is one exact metadata object. Prefix-delete mode
|
||||
// bypasses HTTP preconditions in the ECStore path.
|
||||
delete_prefix: false,
|
||||
http_preconditions: Some(marker_revision.preconditions()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
if let Err(err) = delete_config_with_publication_admission_for_epoch(
|
||||
storeapi.clone(),
|
||||
RUSTFS_META_BUCKET,
|
||||
DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(),
|
||||
ScannerObjectOptions {
|
||||
// This is one exact metadata object. Prefix-delete mode
|
||||
// bypasses HTTP preconditions in the ECStore path.
|
||||
delete_prefix: false,
|
||||
http_preconditions: Some(marker_revision.preconditions()),
|
||||
..Default::default()
|
||||
},
|
||||
reset_epoch,
|
||||
)
|
||||
.await
|
||||
{
|
||||
if scanner_publication_epoch_changed(&err) {
|
||||
set_scanner_cycle_recovery_status(ScannerCycleRecoveryStatus {
|
||||
path: DATA_USAGE_BLOOM_NAME_PATH.clone(),
|
||||
quarantine_path: Some(DATA_USAGE_BLOOM_RECOVERY_PATH.clone()),
|
||||
state: "cleanup-pending".to_string(),
|
||||
classification: Some(marker.classification.clone()),
|
||||
primary_revision: Some(rebuilt_revision.clone()),
|
||||
generation: Some(next),
|
||||
leader_epoch: Some(leader_epoch),
|
||||
retry_count: marker.retry_count,
|
||||
max_retries: MAX_SCANNER_CYCLE_RECOVERY_RETRIES,
|
||||
retryable: true,
|
||||
reason: Some("movement epoch changed before recovery marker cleanup".to_string()),
|
||||
..Default::default()
|
||||
});
|
||||
return Err(ScannerError::Other(
|
||||
"scanner recovery reset deferred by a movement epoch change".to_string(),
|
||||
));
|
||||
}
|
||||
set_scanner_cycle_recovery_status(ScannerCycleRecoveryStatus {
|
||||
path: DATA_USAGE_BLOOM_NAME_PATH.clone(),
|
||||
quarantine_path: Some(DATA_USAGE_BLOOM_RECOVERY_PATH.clone()),
|
||||
@@ -1209,8 +1297,14 @@ pub(super) fn advance_scanner_cycle(cycle_info: &mut CurrentCycle) -> Result<(),
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) async fn persisted_usage_floor(storeapi: Arc<impl ScannerObjectIO>) -> Result<PersistedUsageFloor, ScannerError> {
|
||||
pub(super) async fn persisted_usage_floor(
|
||||
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
|
||||
) -> Result<PersistedUsageFloor, ScannerError> {
|
||||
let Some(read_epoch) = scanner_publication_epoch(storeapi.clone()).await else {
|
||||
return Err(ScannerError::Other("scanner usage floor read is blocked by data movement".to_string()));
|
||||
};
|
||||
let mut floor = PersistedUsageFloor::default();
|
||||
let mut found_any = false;
|
||||
let update_floor = |floor: &mut PersistedUsageFloor, usage: &DataUsageInfo, path: &str| -> Result<(), ScannerError> {
|
||||
floor.leader_epoch = floor.leader_epoch.max(usage.scanner_epoch.unwrap_or_default());
|
||||
if let Some(completed_cycle) = usage.scanner_cycle {
|
||||
@@ -1229,6 +1323,11 @@ pub(super) async fn persisted_usage_floor(storeapi: Arc<impl ScannerObjectIO>) -
|
||||
let usage = serde_json::from_slice::<DataUsageInfo>(&data).map_err(|err| {
|
||||
ScannerError::Other(format!("failed to decode scanner usage floor from {primary_path}: {err}"))
|
||||
})?;
|
||||
if !data_usage_info_has_persisted_baseline_identity(&usage) {
|
||||
return Err(ScannerError::Other(format!(
|
||||
"scanner usage floor from {primary_path} has no persisted baseline identity"
|
||||
)));
|
||||
}
|
||||
let epoch = usage.scanner_epoch.unwrap_or_default();
|
||||
update_floor(&mut floor, &usage, primary_path)?;
|
||||
Some(epoch)
|
||||
@@ -1247,6 +1346,11 @@ pub(super) async fn persisted_usage_floor(storeapi: Arc<impl ScannerObjectIO>) -
|
||||
let usage = serde_json::from_slice::<DataUsageInfo>(&data).map_err(|err| {
|
||||
ScannerError::Other(format!("failed to decode scanner usage floor from {backup_path}: {err}"))
|
||||
})?;
|
||||
if !data_usage_info_has_persisted_baseline_identity(&usage) {
|
||||
return Err(ScannerError::Other(format!(
|
||||
"scanner usage floor from {backup_path} has no persisted baseline identity"
|
||||
)));
|
||||
}
|
||||
let backup_epoch = usage.scanner_epoch.unwrap_or_default();
|
||||
// A backup write from an older leader may complete after the
|
||||
// primary epoch has been fenced. It must not advance the startup
|
||||
@@ -1263,9 +1367,21 @@ pub(super) async fn persisted_usage_floor(storeapi: Arc<impl ScannerObjectIO>) -
|
||||
}
|
||||
}
|
||||
if any_found {
|
||||
found_any = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if !found_any {
|
||||
return Err(ScannerError::Other(
|
||||
"persisted scanner usage floor has no authoritative baseline".to_string(),
|
||||
));
|
||||
}
|
||||
let Some(_publication_admission) = scanner_publication_admission_for_epoch(storeapi, read_epoch).await else {
|
||||
return Err(ScannerError::Other(
|
||||
"scanner usage floor changed while its epoch proof was being confirmed".to_string(),
|
||||
));
|
||||
};
|
||||
Ok(floor)
|
||||
}
|
||||
|
||||
@@ -1274,12 +1390,30 @@ pub(super) fn apply_persisted_usage_floor(cycle_info: &mut CurrentCycle, leader_
|
||||
*leader_epoch = (*leader_epoch).max(floor.leader_epoch);
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
pub(super) struct ScannerCycleFloorOptions {
|
||||
pub(super) required_cycle: u64,
|
||||
pub(super) expected_publication_epoch: Option<u64>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) async fn persist_scanner_cycle_state(
|
||||
ctx: &CancellationToken,
|
||||
storeapi: Arc<impl ScannerObjectIO>,
|
||||
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
|
||||
cycle_info: &mut CurrentCycle,
|
||||
revision: &mut DataUsageCacheRevision,
|
||||
leader_epoch: u64,
|
||||
) -> bool {
|
||||
persist_scanner_cycle_state_for_epoch(ctx, storeapi, cycle_info, revision, leader_epoch, None).await
|
||||
}
|
||||
|
||||
pub(super) async fn persist_scanner_cycle_state_for_epoch(
|
||||
ctx: &CancellationToken,
|
||||
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
|
||||
cycle_info: &mut CurrentCycle,
|
||||
revision: &mut DataUsageCacheRevision,
|
||||
leader_epoch: u64,
|
||||
expected_publication_epoch: Option<u64>,
|
||||
) -> bool {
|
||||
let buf = match encode_scanner_cycle_state(cycle_info, leader_epoch) {
|
||||
Ok(buf) => buf,
|
||||
@@ -1315,9 +1449,29 @@ pub(super) async fn persist_scanner_cycle_state(
|
||||
|
||||
#[cfg(test)]
|
||||
notify_scanner_cycle_state_persist_test_hook(leader_epoch);
|
||||
match save_config_with_preconditions(storeapi.clone(), &DATA_USAGE_BLOOM_NAME_PATH, buf.clone(), revision.preconditions())
|
||||
.await
|
||||
{
|
||||
let Some(read_epoch) = scanner_publication_epoch(storeapi.clone()).await else {
|
||||
return false;
|
||||
};
|
||||
if expected_publication_epoch.is_some_and(|expected| expected != read_epoch) {
|
||||
return false;
|
||||
}
|
||||
let save_result = {
|
||||
let Some(_publication_admission) = scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch).await else {
|
||||
error!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_PERSIST_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
path = %&*DATA_USAGE_BLOOM_NAME_PATH,
|
||||
state = "publication_admission_unavailable",
|
||||
"Scanner state persistence skipped without movement admission"
|
||||
);
|
||||
return false;
|
||||
};
|
||||
save_config_with_preconditions(storeapi.clone(), &DATA_USAGE_BLOOM_NAME_PATH, buf.clone(), revision.preconditions())
|
||||
.await
|
||||
};
|
||||
match save_result {
|
||||
Ok(object_info) => {
|
||||
let Some(etag) = object_info.etag.filter(|etag| !etag.is_empty()) else {
|
||||
error!(
|
||||
@@ -1345,6 +1499,13 @@ pub(super) async fn persist_scanner_cycle_state(
|
||||
);
|
||||
return false;
|
||||
}
|
||||
if let Some(expected_epoch) = expected_publication_epoch
|
||||
&& scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch)
|
||||
.await
|
||||
.is_none()
|
||||
{
|
||||
return false;
|
||||
}
|
||||
debug!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_PERSIST_STATE,
|
||||
@@ -1436,6 +1597,13 @@ pub(super) async fn persist_scanner_cycle_state(
|
||||
}
|
||||
|
||||
if persisted_cycle.next >= cycle_info.next {
|
||||
if let Some(expected_epoch) = expected_publication_epoch
|
||||
&& scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch)
|
||||
.await
|
||||
.is_none()
|
||||
{
|
||||
return false;
|
||||
}
|
||||
*cycle_info = persisted_cycle;
|
||||
debug!(
|
||||
target: "rustfs::scanner",
|
||||
@@ -1496,19 +1664,33 @@ pub(super) async fn persist_scanner_cycle_state(
|
||||
false
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) async fn finalize_partial_scan_cycle(
|
||||
ctx: &CancellationToken,
|
||||
storeapi: Arc<impl ScannerObjectIO>,
|
||||
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
|
||||
cycle_info: &mut CurrentCycle,
|
||||
revision: &mut DataUsageCacheRevision,
|
||||
leader_epoch: u64,
|
||||
cycle_metrics_guard: &mut ScannerCycleMetricsGuard,
|
||||
) -> bool {
|
||||
finalize_partial_scan_cycle_for_epoch(ctx, storeapi, cycle_info, revision, leader_epoch, cycle_metrics_guard, None).await
|
||||
}
|
||||
|
||||
pub(super) async fn finalize_partial_scan_cycle_for_epoch(
|
||||
ctx: &CancellationToken,
|
||||
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
|
||||
cycle_info: &mut CurrentCycle,
|
||||
revision: &mut DataUsageCacheRevision,
|
||||
leader_epoch: u64,
|
||||
cycle_metrics_guard: &mut ScannerCycleMetricsGuard,
|
||||
expected_publication_epoch: Option<u64>,
|
||||
) -> bool {
|
||||
// A budget-limited cycle is deliberate pacing, not a failure. The cycle counter
|
||||
// must still advance (and persist) because per-bucket next_cycle is stamped from
|
||||
// it and compacted folders are only rescanned when their hash matches
|
||||
// next_cycle % DATA_USAGE_UPDATE_DIR_CYCLES; a pinned counter starves lifecycle
|
||||
// expiry and usage refresh on every folder outside the stuck window.
|
||||
let previous_cycle_info = cycle_info.clone();
|
||||
if let Err(err) = advance_scanner_cycle(cycle_info) {
|
||||
error!(
|
||||
target: "rustfs::scanner",
|
||||
@@ -1524,28 +1706,69 @@ pub(super) async fn finalize_partial_scan_cycle(
|
||||
}
|
||||
cycle_info.current = 0;
|
||||
global_metrics().clear_current_scan_mode();
|
||||
let persisted = persist_scanner_cycle_state(ctx, storeapi, cycle_info, revision, leader_epoch).await;
|
||||
let persisted = persist_scanner_cycle_state_for_epoch(
|
||||
ctx,
|
||||
storeapi.clone(),
|
||||
cycle_info,
|
||||
revision,
|
||||
leader_epoch,
|
||||
expected_publication_epoch,
|
||||
)
|
||||
.await;
|
||||
if !persisted
|
||||
&& let Some(expected_epoch) = expected_publication_epoch
|
||||
&& scanner_publication_admission_for_epoch(storeapi, expected_epoch)
|
||||
.await
|
||||
.is_none()
|
||||
{
|
||||
*cycle_info = previous_cycle_info;
|
||||
}
|
||||
cycle_metrics_guard.finish(cycle_info.clone()).await;
|
||||
persisted
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) async fn persist_required_scanner_cycle_floor(
|
||||
ctx: &CancellationToken,
|
||||
storeapi: Arc<impl ScannerObjectIO>,
|
||||
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
|
||||
cycle_info: &mut CurrentCycle,
|
||||
revision: &mut DataUsageCacheRevision,
|
||||
leader_epoch: u64,
|
||||
required_cycle: u64,
|
||||
cycle_metrics_guard: &mut ScannerCycleMetricsGuard,
|
||||
) -> bool {
|
||||
if required_cycle <= cycle_info.current || required_cycle == u64::MAX {
|
||||
persist_required_scanner_cycle_floor_for_epoch(
|
||||
ctx,
|
||||
storeapi,
|
||||
cycle_info,
|
||||
revision,
|
||||
leader_epoch,
|
||||
cycle_metrics_guard,
|
||||
ScannerCycleFloorOptions {
|
||||
required_cycle,
|
||||
expected_publication_epoch: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(super) async fn persist_required_scanner_cycle_floor_for_epoch(
|
||||
ctx: &CancellationToken,
|
||||
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
|
||||
cycle_info: &mut CurrentCycle,
|
||||
revision: &mut DataUsageCacheRevision,
|
||||
leader_epoch: u64,
|
||||
cycle_metrics_guard: &mut ScannerCycleMetricsGuard,
|
||||
options: ScannerCycleFloorOptions,
|
||||
) -> bool {
|
||||
if options.required_cycle <= cycle_info.current || options.required_cycle == u64::MAX {
|
||||
error!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_PERSIST_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
current_cycle = cycle_info.current,
|
||||
required_cycle,
|
||||
required_cycle = options.required_cycle,
|
||||
state = "invalid_cache_cycle_floor",
|
||||
"Scanner cache cycle floor is invalid"
|
||||
);
|
||||
@@ -1553,10 +1776,27 @@ pub(super) async fn persist_required_scanner_cycle_floor(
|
||||
return false;
|
||||
}
|
||||
|
||||
cycle_info.next = cycle_info.next.max(required_cycle);
|
||||
let previous_cycle_info = cycle_info.clone();
|
||||
cycle_info.next = cycle_info.next.max(options.required_cycle);
|
||||
cycle_info.current = 0;
|
||||
global_metrics().clear_current_scan_mode();
|
||||
let persisted = persist_scanner_cycle_state(ctx, storeapi, cycle_info, revision, leader_epoch).await;
|
||||
let persisted = persist_scanner_cycle_state_for_epoch(
|
||||
ctx,
|
||||
storeapi.clone(),
|
||||
cycle_info,
|
||||
revision,
|
||||
leader_epoch,
|
||||
options.expected_publication_epoch,
|
||||
)
|
||||
.await;
|
||||
if !persisted
|
||||
&& let Some(expected_epoch) = options.expected_publication_epoch
|
||||
&& scanner_publication_admission_for_epoch(storeapi, expected_epoch)
|
||||
.await
|
||||
.is_none()
|
||||
{
|
||||
*cycle_info = previous_cycle_info;
|
||||
}
|
||||
cycle_metrics_guard.finish(cycle_info.clone()).await;
|
||||
persisted
|
||||
}
|
||||
|
||||
@@ -26,31 +26,90 @@ pub struct BackgroundHealInfo {
|
||||
pub current_scan_mode: HealScanMode,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub(super) enum BackgroundHealInfoReadStatus {
|
||||
ErasureSd,
|
||||
Loaded,
|
||||
Missing,
|
||||
Blocked,
|
||||
Transient,
|
||||
Failed,
|
||||
}
|
||||
|
||||
pub(super) struct BackgroundHealInfoRead {
|
||||
pub(super) info: BackgroundHealInfo,
|
||||
pub(super) expected_epoch: Option<u64>,
|
||||
pub(super) status: BackgroundHealInfoReadStatus,
|
||||
}
|
||||
|
||||
pub(super) fn classify_background_heal_read_error(error: &EcstoreError) -> BackgroundHealInfoReadStatus {
|
||||
if matches!(error, EcstoreError::ConfigNotFound) {
|
||||
BackgroundHealInfoReadStatus::Missing
|
||||
} else {
|
||||
BackgroundHealInfoReadStatus::Transient
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn decode_background_heal_info(data: &[u8]) -> Result<BackgroundHealInfo, serde_json::Error> {
|
||||
serde_json::from_slice(data)
|
||||
}
|
||||
|
||||
/// Read background healing information from storage
|
||||
pub async fn read_background_heal_info(storeapi: Arc<ECStore>) -> BackgroundHealInfo {
|
||||
read_background_heal_info_with_epoch(storeapi).await.info
|
||||
}
|
||||
|
||||
/// Read background healing information together with the movement epoch that
|
||||
/// fenced the read. The epoch must be reused by the matching cycle update so a
|
||||
/// missing-object default cannot be committed across a movement transition.
|
||||
pub(super) async fn read_background_heal_info_with_epoch(storeapi: Arc<ECStore>) -> BackgroundHealInfoRead {
|
||||
// Skip for ErasureSD setup
|
||||
if scanner_is_erasure_sd().await {
|
||||
return BackgroundHealInfo::default();
|
||||
return BackgroundHealInfoRead {
|
||||
info: BackgroundHealInfo::default(),
|
||||
expected_epoch: None,
|
||||
status: BackgroundHealInfoReadStatus::ErasureSd,
|
||||
};
|
||||
}
|
||||
|
||||
let expected_epoch = scanner_publication_epoch(storeapi.clone()).await;
|
||||
if expected_epoch.is_none() {
|
||||
return BackgroundHealInfoRead {
|
||||
info: BackgroundHealInfo::default(),
|
||||
expected_epoch,
|
||||
status: BackgroundHealInfoReadStatus::Blocked,
|
||||
};
|
||||
}
|
||||
|
||||
// Get last healing information
|
||||
match read_config(storeapi, &BACKGROUND_HEAL_INFO_PATH).await {
|
||||
Ok(buf) => serde_json::from_slice::<BackgroundHealInfo>(&buf).unwrap_or_else(|e| {
|
||||
error!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_BACKGROUND_HEAL_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_BACKGROUND_HEAL,
|
||||
path = %&*BACKGROUND_HEAL_INFO_PATH,
|
||||
state = "decode_failed",
|
||||
error = %e,
|
||||
"Scanner background heal decode failed"
|
||||
);
|
||||
BackgroundHealInfo::default()
|
||||
}),
|
||||
Ok(buf) => match decode_background_heal_info(&buf) {
|
||||
Ok(info) => BackgroundHealInfoRead {
|
||||
info,
|
||||
expected_epoch,
|
||||
status: BackgroundHealInfoReadStatus::Loaded,
|
||||
},
|
||||
Err(e) => {
|
||||
error!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_BACKGROUND_HEAL_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_BACKGROUND_HEAL,
|
||||
path = %&*BACKGROUND_HEAL_INFO_PATH,
|
||||
state = "decode_failed",
|
||||
error = %e,
|
||||
"Scanner background heal decode failed"
|
||||
);
|
||||
BackgroundHealInfoRead {
|
||||
info: BackgroundHealInfo::default(),
|
||||
expected_epoch,
|
||||
status: BackgroundHealInfoReadStatus::Failed,
|
||||
}
|
||||
}
|
||||
},
|
||||
Err(e) => {
|
||||
// Only log if it's not a ConfigNotFound error
|
||||
if e != EcstoreError::ConfigNotFound {
|
||||
let status = classify_background_heal_read_error(&e);
|
||||
if status == BackgroundHealInfoReadStatus::Transient {
|
||||
warn!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_BACKGROUND_HEAL_STATE,
|
||||
@@ -62,7 +121,11 @@ pub async fn read_background_heal_info(storeapi: Arc<ECStore>) -> BackgroundHeal
|
||||
"Scanner background heal read failed"
|
||||
);
|
||||
}
|
||||
BackgroundHealInfo::default()
|
||||
BackgroundHealInfoRead {
|
||||
info: BackgroundHealInfo::default(),
|
||||
expected_epoch,
|
||||
status,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -70,6 +133,14 @@ pub async fn read_background_heal_info(storeapi: Arc<ECStore>) -> BackgroundHeal
|
||||
/// Save background healing information to storage
|
||||
#[instrument(skip(storeapi))]
|
||||
pub async fn save_background_heal_info(storeapi: Arc<ECStore>, info: BackgroundHealInfo) {
|
||||
save_background_heal_info_for_epoch(storeapi, info, None).await;
|
||||
}
|
||||
|
||||
pub(super) async fn save_background_heal_info_for_epoch(
|
||||
storeapi: Arc<ECStore>,
|
||||
info: BackgroundHealInfo,
|
||||
expected_epoch: Option<u64>,
|
||||
) {
|
||||
// Skip for ErasureSD setup
|
||||
if scanner_is_erasure_sd().await {
|
||||
return;
|
||||
@@ -93,7 +164,25 @@ pub async fn save_background_heal_info(storeapi: Arc<ECStore>, info: BackgroundH
|
||||
}
|
||||
};
|
||||
|
||||
// Save configuration
|
||||
// Save configuration only after storage-owned movement admission. The
|
||||
// read path may return an in-memory default for a missing object, but a
|
||||
// movement transition must not let that default become durable state.
|
||||
let publication_admission = match expected_epoch {
|
||||
Some(expected_epoch) => scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch).await,
|
||||
None => storeapi.scanner_data_usage_publication_admission().await,
|
||||
};
|
||||
let Some(_publication_admission) = publication_admission else {
|
||||
warn!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_BACKGROUND_HEAL_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_BACKGROUND_HEAL,
|
||||
path = %&*BACKGROUND_HEAL_INFO_PATH,
|
||||
state = "publication_admission_unavailable",
|
||||
"Scanner background heal save skipped without movement admission"
|
||||
);
|
||||
return;
|
||||
};
|
||||
if let Err(e) = save_config(storeapi, &BACKGROUND_HEAL_INFO_PATH, data).await {
|
||||
warn!(
|
||||
target: "rustfs::scanner",
|
||||
|
||||
@@ -61,16 +61,22 @@ pub(super) async fn reconcile_scanner_leadership_claim(
|
||||
}
|
||||
|
||||
pub(super) fn decode_usage_snapshot_for_epoch_fence(data: &[u8], path: &str) -> Result<DataUsageInfo, ScannerError> {
|
||||
serde_json::from_slice(data)
|
||||
.map_err(|err| ScannerError::Other(format!("failed to decode scanner usage epoch fence from {path}: {err}")))
|
||||
let usage: DataUsageInfo = serde_json::from_slice(data)
|
||||
.map_err(|err| ScannerError::Other(format!("failed to decode scanner usage epoch fence from {path}: {err}")))?;
|
||||
if !data_usage_info_has_persisted_baseline_identity(&usage) {
|
||||
return Err(ScannerError::Other(format!(
|
||||
"scanner usage epoch fence from {path} has no persisted baseline identity"
|
||||
)));
|
||||
}
|
||||
Ok(usage)
|
||||
}
|
||||
|
||||
pub(super) async fn usage_snapshot_for_epoch_fence(
|
||||
storeapi: Arc<impl ScannerObjectIO>,
|
||||
primary: Option<&[u8]>,
|
||||
) -> Result<DataUsageInfo, ScannerError> {
|
||||
) -> Result<Option<DataUsageInfo>, ScannerError> {
|
||||
if let Some(primary) = primary {
|
||||
return decode_usage_snapshot_for_epoch_fence(primary, DATA_USAGE_OBJ_NAME_PATH.as_str());
|
||||
return decode_usage_snapshot_for_epoch_fence(primary, DATA_USAGE_OBJ_NAME_PATH.as_str()).map(Some);
|
||||
}
|
||||
|
||||
let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str());
|
||||
@@ -78,7 +84,7 @@ pub(super) async fn usage_snapshot_for_epoch_fence(
|
||||
.await
|
||||
.map_err(|err| ScannerError::Other(format!("failed to read scanner usage epoch fence backup: {err}")))?;
|
||||
if let Some(backup) = backup.as_deref() {
|
||||
return decode_usage_snapshot_for_epoch_fence(backup, &backup_path);
|
||||
return decode_usage_snapshot_for_epoch_fence(backup, &backup_path).map(Some);
|
||||
}
|
||||
|
||||
for path in [
|
||||
@@ -89,26 +95,53 @@ pub(super) async fn usage_snapshot_for_epoch_fence(
|
||||
.await
|
||||
.map_err(|err| ScannerError::Other(format!("failed to read legacy scanner usage epoch fence: {err}")))?;
|
||||
if let Some(legacy) = legacy.as_deref() {
|
||||
return decode_usage_snapshot_for_epoch_fence(legacy, &path);
|
||||
return decode_usage_snapshot_for_epoch_fence(legacy, &path).map(Some);
|
||||
}
|
||||
}
|
||||
Ok(DataUsageInfo::default())
|
||||
// A missing usage snapshot is an uninitialized state, not an empty
|
||||
// snapshot. Leadership fencing may proceed without creating a plausible
|
||||
// default; the first authoritative scanner publication will create it.
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
pub(super) async fn fence_scanner_usage_epoch(
|
||||
pub(super) async fn fence_scanner_usage_epoch_with_expected_epoch(
|
||||
ctx: &CancellationToken,
|
||||
storeapi: Arc<impl ScannerObjectIO>,
|
||||
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
|
||||
claimed_epoch: u64,
|
||||
expected_publication_epoch: Option<u64>,
|
||||
) -> Result<(), ScannerError> {
|
||||
for retry in 0..=SCANNER_PERSIST_CAS_RETRIES {
|
||||
if ctx.is_cancelled() {
|
||||
return Err(ScannerError::Other("scanner leadership was cancelled before usage fencing".to_string()));
|
||||
}
|
||||
|
||||
let Some(read_epoch) = scanner_publication_epoch(storeapi.clone()).await else {
|
||||
return Err(ScannerError::Other(
|
||||
"scanner usage epoch fence publication is blocked by data movement".to_string(),
|
||||
));
|
||||
};
|
||||
if expected_publication_epoch.is_some_and(|expected| expected != read_epoch) {
|
||||
if retry < SCANNER_PERSIST_CAS_RETRIES {
|
||||
continue;
|
||||
}
|
||||
return Err(ScannerError::Other(
|
||||
"scanner usage epoch fence changed while recovery reset was in progress".to_string(),
|
||||
));
|
||||
}
|
||||
let (primary, revision) = read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
|
||||
.await
|
||||
.map_err(|err| ScannerError::Other(format!("failed to read scanner usage epoch fence: {err}")))?;
|
||||
let mut usage = usage_snapshot_for_epoch_fence(storeapi.clone(), primary.as_deref()).await?;
|
||||
let Some(mut usage) = usage_snapshot_for_epoch_fence(storeapi.clone(), primary.as_deref()).await? else {
|
||||
let Some(_publication_admission) = scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch).await else {
|
||||
if retry < SCANNER_PERSIST_CAS_RETRIES {
|
||||
continue;
|
||||
}
|
||||
return Err(ScannerError::Other(
|
||||
"scanner usage epoch fence changed while confirming a missing usage baseline".to_string(),
|
||||
));
|
||||
};
|
||||
return Err(ScannerError::Other("authoritative scanner usage baseline is missing".to_string()));
|
||||
};
|
||||
match usage.scanner_epoch {
|
||||
Some(epoch) if epoch > claimed_epoch => {
|
||||
return Err(ScannerError::Other(format!(
|
||||
@@ -122,9 +155,18 @@ pub(super) async fn fence_scanner_usage_epoch(
|
||||
let data = serde_json::to_vec(&usage)
|
||||
.map_err(|err| ScannerError::Other(format!("failed to encode scanner usage epoch fence: {err}")))?;
|
||||
|
||||
let save_result =
|
||||
let save_result = {
|
||||
let Some(_publication_admission) = scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch).await else {
|
||||
if retry < SCANNER_PERSIST_CAS_RETRIES {
|
||||
continue;
|
||||
}
|
||||
return Err(ScannerError::Other(
|
||||
"scanner usage epoch fence changed while preparing its conditional write".to_string(),
|
||||
));
|
||||
};
|
||||
save_config_with_preconditions(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), data, revision.preconditions())
|
||||
.await;
|
||||
.await
|
||||
};
|
||||
if save_result
|
||||
.as_ref()
|
||||
.ok()
|
||||
@@ -165,10 +207,13 @@ pub(super) async fn fence_scanner_usage_epoch(
|
||||
|
||||
pub(super) async fn complete_scanner_leadership_claim(
|
||||
ctx: &CancellationToken,
|
||||
storeapi: Arc<impl ScannerObjectIO>,
|
||||
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
|
||||
claimed_epoch: u64,
|
||||
expected_publication_epoch: Option<u64>,
|
||||
) -> bool {
|
||||
if let Err(err) = fence_scanner_usage_epoch(ctx, storeapi, claimed_epoch).await {
|
||||
if let Err(err) =
|
||||
fence_scanner_usage_epoch_with_expected_epoch(ctx, storeapi, claimed_epoch, expected_publication_epoch).await
|
||||
{
|
||||
error!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_PERSIST_STATE,
|
||||
@@ -187,7 +232,7 @@ pub(super) async fn complete_scanner_leadership_claim(
|
||||
|
||||
pub(super) async fn claim_scanner_leadership(
|
||||
ctx: &CancellationToken,
|
||||
storeapi: Arc<impl ScannerObjectIO>,
|
||||
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
|
||||
cycle_info: &mut CurrentCycle,
|
||||
revision: &mut DataUsageCacheRevision,
|
||||
persisted_epoch: &mut u64,
|
||||
@@ -226,15 +271,69 @@ pub(super) async fn claim_scanner_leadership(
|
||||
};
|
||||
let previous_revision = revision.clone();
|
||||
|
||||
let save_result =
|
||||
let Some(read_epoch) = scanner_publication_epoch(storeapi.clone()).await else {
|
||||
return false;
|
||||
};
|
||||
let (usage_primary, _) = match read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()).await {
|
||||
Ok(result) => result,
|
||||
Err(err) => {
|
||||
error!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_PERSIST_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
|
||||
state = "leader_usage_baseline_read_failed",
|
||||
error = %err,
|
||||
"Scanner leadership claim deferred because the usage baseline could not be read"
|
||||
);
|
||||
return false;
|
||||
}
|
||||
};
|
||||
match usage_snapshot_for_epoch_fence(storeapi.clone(), usage_primary.as_deref()).await {
|
||||
Ok(Some(_)) => {}
|
||||
Ok(None) => {
|
||||
warn!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_PERSIST_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
|
||||
state = "leader_usage_baseline_missing",
|
||||
"Scanner leadership claim deferred until a usage baseline is published"
|
||||
);
|
||||
return false;
|
||||
}
|
||||
Err(err) => {
|
||||
error!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_PERSIST_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
|
||||
state = "leader_usage_baseline_invalid",
|
||||
error = %err,
|
||||
"Scanner leadership claim deferred because the usage baseline is invalid"
|
||||
);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
let save_result = {
|
||||
let Some(_publication_admission) = scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch).await else {
|
||||
if retry < SCANNER_PERSIST_CAS_RETRIES {
|
||||
continue;
|
||||
}
|
||||
return false;
|
||||
};
|
||||
save_config_with_preconditions(storeapi.clone(), &DATA_USAGE_BLOOM_NAME_PATH, data.clone(), revision.preconditions())
|
||||
.await;
|
||||
.await
|
||||
};
|
||||
match save_result {
|
||||
Ok(object_info) => {
|
||||
if let Some(etag) = object_info.etag.filter(|etag| !etag.is_empty()) {
|
||||
*revision = DataUsageCacheRevision::Etag(etag);
|
||||
*persisted_epoch = claimed_epoch;
|
||||
return complete_scanner_leadership_claim(ctx, storeapi, claimed_epoch).await;
|
||||
return complete_scanner_leadership_claim(ctx, storeapi, claimed_epoch, Some(read_epoch)).await;
|
||||
}
|
||||
|
||||
match reconcile_scanner_leadership_claim(
|
||||
@@ -249,7 +348,7 @@ pub(super) async fn claim_scanner_leadership(
|
||||
.await
|
||||
{
|
||||
Ok(ScannerLeadershipClaimReconcile::Durable) => {
|
||||
return complete_scanner_leadership_claim(ctx, storeapi, claimed_epoch).await;
|
||||
return complete_scanner_leadership_claim(ctx, storeapi, claimed_epoch, Some(read_epoch)).await;
|
||||
}
|
||||
Ok(ScannerLeadershipClaimReconcile::Changed) if retry < SCANNER_PERSIST_CAS_RETRIES => continue,
|
||||
Ok(ScannerLeadershipClaimReconcile::Changed | ScannerLeadershipClaimReconcile::Unchanged) => {
|
||||
@@ -293,7 +392,7 @@ pub(super) async fn claim_scanner_leadership(
|
||||
.await
|
||||
{
|
||||
Ok(ScannerLeadershipClaimReconcile::Durable) => {
|
||||
return complete_scanner_leadership_claim(ctx, storeapi, claimed_epoch).await;
|
||||
return complete_scanner_leadership_claim(ctx, storeapi, claimed_epoch, Some(read_epoch)).await;
|
||||
}
|
||||
Ok(ScannerLeadershipClaimReconcile::Changed)
|
||||
if retry < SCANNER_PERSIST_CAS_RETRIES && !ctx.is_cancelled() =>
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::heal_info::{classify_background_heal_read_error, decode_background_heal_info};
|
||||
use super::*;
|
||||
use crate::EcstoreResult;
|
||||
use crate::{
|
||||
@@ -22,6 +23,7 @@ use crate::{
|
||||
};
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::io::Cursor;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::task::Poll;
|
||||
use temp_env::{with_var, with_var_unset};
|
||||
use tokio::io::AsyncReadExt;
|
||||
@@ -343,6 +345,7 @@ struct MemoryConfigStore {
|
||||
cancel_after_successful_puts: Mutex<HashMap<String, (usize, CancellationToken)>>,
|
||||
replace_after_successful_puts: Mutex<HashMap<String, (usize, Vec<u8>)>>,
|
||||
put_counts: Mutex<HashMap<String, usize>>,
|
||||
publication_admission_blocked: AtomicBool,
|
||||
}
|
||||
|
||||
fn memory_config_key(bucket: &str, object: &str) -> String {
|
||||
@@ -1350,7 +1353,7 @@ async fn full_rescan_reset_preserves_valid_primary_when_marker_is_malformed() {
|
||||
let old_usage = DataUsageInfo {
|
||||
scanner_epoch: Some(7),
|
||||
scanner_cycle: Some(41),
|
||||
..Default::default()
|
||||
..complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0)
|
||||
};
|
||||
let old_usage_data = serde_json::to_vec(&old_usage).expect("usage snapshot should encode");
|
||||
save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), old_usage_data.clone())
|
||||
@@ -1424,7 +1427,7 @@ async fn full_rescan_reset_resumes_cleanup_pending_preserved_primary() {
|
||||
let usage = DataUsageInfo {
|
||||
scanner_epoch: Some(7),
|
||||
scanner_cycle: Some(41),
|
||||
..Default::default()
|
||||
..complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0)
|
||||
};
|
||||
save_config(
|
||||
store.clone(),
|
||||
@@ -1695,7 +1698,7 @@ async fn full_rescan_reset_rejects_usage_floor_that_would_be_terminal() {
|
||||
DATA_USAGE_OBJ_NAME_PATH.as_str(),
|
||||
serde_json::to_vec(&DataUsageInfo {
|
||||
scanner_epoch: Some(u64::MAX - 1),
|
||||
..Default::default()
|
||||
..complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0)
|
||||
})
|
||||
.expect("usage floor should encode"),
|
||||
)
|
||||
@@ -1847,14 +1850,12 @@ async fn scanner_startup_uses_primary_and_backup_usage_floor() {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str());
|
||||
for (path, epoch, cycle) in [(DATA_USAGE_OBJ_NAME_PATH.as_str(), 8, 100), (backup_path.as_str(), 11, 103)] {
|
||||
let mut usage = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0);
|
||||
usage.scanner_epoch = Some(epoch);
|
||||
usage.scanner_cycle = Some(cycle);
|
||||
store.objects.lock().await.insert(
|
||||
memory_config_key(RUSTFS_META_BUCKET, path),
|
||||
serde_json::to_vec(&DataUsageInfo {
|
||||
scanner_epoch: Some(epoch),
|
||||
scanner_cycle: Some(cycle),
|
||||
..Default::default()
|
||||
})
|
||||
.expect("usage snapshot should encode"),
|
||||
serde_json::to_vec(&usage).expect("usage snapshot should encode"),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1879,14 +1880,12 @@ async fn scanner_usage_floor_ignores_older_backup_after_primary_epoch_fence() {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str());
|
||||
for (path, epoch, cycle) in [(DATA_USAGE_OBJ_NAME_PATH.as_str(), 8, 100), (backup_path.as_str(), 7, 10_000)] {
|
||||
let mut usage = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0);
|
||||
usage.scanner_epoch = Some(epoch);
|
||||
usage.scanner_cycle = Some(cycle);
|
||||
store.objects.lock().await.insert(
|
||||
memory_config_key(RUSTFS_META_BUCKET, path),
|
||||
serde_json::to_vec(&DataUsageInfo {
|
||||
scanner_epoch: Some(epoch),
|
||||
scanner_cycle: Some(cycle),
|
||||
..Default::default()
|
||||
})
|
||||
.expect("usage snapshot should encode"),
|
||||
serde_json::to_vec(&usage).expect("usage snapshot should encode"),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1916,6 +1915,23 @@ fn scanner_startup_treats_incomplete_usage_snapshot_as_cold() {
|
||||
}));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_baseline_identity_requires_complete_or_strict_legacy_shape() {
|
||||
assert!(!data_usage_info_has_persisted_baseline_identity(&DataUsageInfo {
|
||||
scanner_epoch: Some(3),
|
||||
scanner_cycle: Some(7),
|
||||
..Default::default()
|
||||
}));
|
||||
|
||||
let mut legacy = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0);
|
||||
legacy.usage_snapshot_complete = false;
|
||||
legacy.scanner_cycle = Some(7);
|
||||
assert!(data_usage_info_has_persisted_baseline_identity(&legacy));
|
||||
|
||||
legacy.scanner_epoch = Some(3);
|
||||
assert!(!data_usage_info_has_persisted_baseline_identity(&legacy));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_startup_prompts_only_for_a_newer_valid_observation() {
|
||||
let authoritative = DataUsageInfo {
|
||||
@@ -1948,12 +1964,9 @@ fn scanner_startup_prompts_only_for_a_newer_valid_observation() {
|
||||
#[tokio::test]
|
||||
async fn scanner_startup_prefers_v2_over_legacy_usage() {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
let legacy = DataUsageInfo {
|
||||
scanner_epoch: Some(19),
|
||||
scanner_cycle: Some(41),
|
||||
last_update: Some(std::time::SystemTime::now()),
|
||||
..Default::default()
|
||||
};
|
||||
let mut legacy = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0);
|
||||
legacy.scanner_epoch = Some(19);
|
||||
legacy.scanner_cycle = Some(41);
|
||||
let legacy_data = serde_json::to_vec(&legacy).expect("legacy usage snapshot should encode");
|
||||
store.objects.lock().await.insert(
|
||||
memory_config_key(RUSTFS_META_BUCKET, LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str()),
|
||||
@@ -1976,13 +1989,9 @@ async fn scanner_startup_prefers_v2_over_legacy_usage() {
|
||||
}
|
||||
);
|
||||
|
||||
let authoritative = DataUsageInfo {
|
||||
scanner_epoch: Some(23),
|
||||
scanner_cycle: Some(51),
|
||||
last_update: Some(std::time::SystemTime::now()),
|
||||
usage_snapshot_complete: true,
|
||||
..Default::default()
|
||||
};
|
||||
let mut authoritative = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0);
|
||||
authoritative.scanner_epoch = Some(23);
|
||||
authoritative.scanner_cycle = Some(51);
|
||||
let authoritative_data = serde_json::to_vec(&authoritative).expect("v2 usage snapshot should encode");
|
||||
store.objects.lock().await.insert(
|
||||
memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()),
|
||||
@@ -2024,6 +2033,8 @@ async fn scanner_startup_prefers_v2_over_legacy_usage() {
|
||||
#[tokio::test]
|
||||
async fn scanner_usage_floor_fails_closed_on_corrupt_or_exhausted_usage_state() {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
assert!(persisted_usage_floor(store.clone()).await.is_err());
|
||||
|
||||
store.objects.lock().await.insert(
|
||||
memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()),
|
||||
b"not-json".to_vec(),
|
||||
@@ -2087,6 +2098,39 @@ async fn scanner_usage_backup_uses_durable_cycle_cadence_across_tasks() {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_backup_sync_distinguishes_movement_from_missing_or_corrupt_primary() {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
let primary_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str());
|
||||
let primary = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0);
|
||||
store.objects.lock().await.insert(
|
||||
primary_key.clone(),
|
||||
serde_json::to_vec(&primary).expect("primary usage snapshot should encode"),
|
||||
);
|
||||
store.revisions.lock().await.insert(primary_key.clone(), 1);
|
||||
|
||||
store.publication_admission_blocked.store(true, Ordering::Release);
|
||||
let movement_error = sync_data_usage_backup_from_primary(&CancellationToken::new(), store.clone())
|
||||
.await
|
||||
.expect_err("movement admission loss should fail backup synchronization");
|
||||
assert!(scanner_publication_epoch_changed(&movement_error));
|
||||
|
||||
store.publication_admission_blocked.store(false, Ordering::Release);
|
||||
store.objects.lock().await.remove(&primary_key);
|
||||
store.revisions.lock().await.remove(&primary_key);
|
||||
assert!(matches!(
|
||||
sync_data_usage_backup_from_primary(&CancellationToken::new(), store.clone()).await,
|
||||
Err(EcstoreError::ConfigNotFound)
|
||||
));
|
||||
|
||||
store.objects.lock().await.insert(primary_key.clone(), b"not-json".to_vec());
|
||||
store.revisions.lock().await.insert(primary_key, 1);
|
||||
let corrupt_error = sync_data_usage_backup_from_primary(&CancellationToken::new(), store)
|
||||
.await
|
||||
.expect_err("corrupt primary should fail backup synchronization");
|
||||
assert!(!scanner_publication_epoch_changed(&corrupt_error));
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl crate::ScannerConfigObjectDelete for MemoryConfigStore {
|
||||
async fn delete_config_object(&self, bucket: &str, object: &str, opts: ObjectOptions) -> EcstoreResult<ObjectInfo> {
|
||||
@@ -2110,6 +2154,10 @@ impl crate::ScannerConfigObjectDelete for MemoryConfigStore {
|
||||
revisions.remove(&key);
|
||||
Ok(ObjectInfo::default())
|
||||
}
|
||||
|
||||
async fn scanner_data_usage_publication_admission(&self) -> Option<crate::ScannerDataUsagePublicationAdmission> {
|
||||
(!self.publication_admission_blocked.load(Ordering::Acquire)).then(crate::ScannerDataUsagePublicationAdmission::unfenced)
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -2270,6 +2318,7 @@ async fn test_leadership_claim_preserves_usage_epoch_floor_across_old_epoch_conf
|
||||
started: Utc::now(),
|
||||
};
|
||||
assert!(persist_scanner_cycle_state(&ctx, store.clone(), &mut cycle, &mut revision, 1).await);
|
||||
seed_usage_snapshot_for_leadership_claim(&store).await;
|
||||
|
||||
let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str());
|
||||
let old_epoch_commit = CurrentCycle {
|
||||
@@ -2313,6 +2362,61 @@ async fn test_leadership_claim_rejects_terminal_epoch() {
|
||||
assert!(read_config(store, &DATA_USAGE_BLOOM_NAME_PATH).await.is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn leadership_claim_defers_without_usage_baseline_before_bloom_write() {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
let ctx = CancellationToken::new();
|
||||
let mut revision = DataUsageCacheRevision::Missing;
|
||||
let mut cycle = CurrentCycle {
|
||||
next: 12,
|
||||
..Default::default()
|
||||
};
|
||||
let mut persisted_epoch = 0;
|
||||
|
||||
assert!(!claim_scanner_leadership(&ctx, store.clone(), &mut cycle, &mut revision, &mut persisted_epoch,).await);
|
||||
assert!(read_config(store.clone(), &DATA_USAGE_BLOOM_NAME_PATH).await.is_err());
|
||||
assert!(read_config(store, DATA_USAGE_OBJ_NAME_PATH.as_str()).await.is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn leadership_claim_defers_on_corrupt_usage_baseline_without_bloom_write() {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
let usage_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str());
|
||||
store.objects.lock().await.insert(usage_key.clone(), b"not-json".to_vec());
|
||||
store.revisions.lock().await.insert(usage_key, 1);
|
||||
|
||||
let ctx = CancellationToken::new();
|
||||
let mut revision = DataUsageCacheRevision::Missing;
|
||||
let mut cycle = CurrentCycle {
|
||||
next: 12,
|
||||
..Default::default()
|
||||
};
|
||||
let mut persisted_epoch = 0;
|
||||
|
||||
assert!(!claim_scanner_leadership(&ctx, store.clone(), &mut cycle, &mut revision, &mut persisted_epoch,).await);
|
||||
assert!(read_config(store, &DATA_USAGE_BLOOM_NAME_PATH).await.is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn leadership_claim_defers_on_unidentified_usage_baseline_without_bloom_write() {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
let usage_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str());
|
||||
let data = serde_json::to_vec(&DataUsageInfo::default()).expect("default usage should encode");
|
||||
store.objects.lock().await.insert(usage_key.clone(), data);
|
||||
store.revisions.lock().await.insert(usage_key, 1);
|
||||
|
||||
let ctx = CancellationToken::new();
|
||||
let mut revision = DataUsageCacheRevision::Missing;
|
||||
let mut cycle = CurrentCycle {
|
||||
next: 12,
|
||||
..Default::default()
|
||||
};
|
||||
let mut persisted_epoch = 0;
|
||||
|
||||
assert!(!claim_scanner_leadership(&ctx, store.clone(), &mut cycle, &mut revision, &mut persisted_epoch,).await);
|
||||
assert!(read_config(store, &DATA_USAGE_BLOOM_NAME_PATH).await.is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_leadership_claim_confirms_commit_after_returned_error() {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
@@ -2329,6 +2433,7 @@ async fn test_leadership_claim_confirms_commit_after_returned_error() {
|
||||
started: Utc::now(),
|
||||
};
|
||||
let mut persisted_epoch = 0;
|
||||
seed_usage_snapshot_for_leadership_claim(&store).await;
|
||||
|
||||
assert!(claim_scanner_leadership(&ctx, store.clone(), &mut cycle, &mut revision, &mut persisted_epoch).await);
|
||||
|
||||
@@ -2373,6 +2478,7 @@ async fn test_leadership_claim_usage_fence_rejects_old_inflight_writer() {
|
||||
);
|
||||
old_usage.buckets_count = 1;
|
||||
old_usage.calculate_totals();
|
||||
old_usage.usage_snapshot_complete = true;
|
||||
let old_data = serde_json::to_vec(&old_usage).expect("old usage snapshot should encode");
|
||||
store.objects.lock().await.insert(usage_key.clone(), old_data.clone());
|
||||
store.revisions.lock().await.insert(usage_key, 1);
|
||||
@@ -2419,6 +2525,7 @@ async fn cycle_budget_lease_takeover_rejects_old_generation() {
|
||||
started: Utc::now(),
|
||||
};
|
||||
assert!(persist_scanner_cycle_state(&ctx, store.clone(), &mut cycle, &mut revision, 1).await);
|
||||
seed_usage_snapshot_for_leadership_claim(&store).await;
|
||||
|
||||
let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str());
|
||||
store
|
||||
@@ -2590,6 +2697,35 @@ async fn test_usage_save_route_barrier_prevents_missing_snapshot_creation() {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_observational_usage_defers_when_authoritative_baseline_is_missing() {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
let (sender, receiver) = mpsc::channel(1);
|
||||
let mut observation = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)), 1);
|
||||
observation.usage_snapshot_converged = Some(false);
|
||||
sender.send(observation).await.expect("observation should enqueue");
|
||||
drop(sender);
|
||||
|
||||
let outcome = store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe(
|
||||
CancellationToken::new(),
|
||||
store.clone(),
|
||||
receiver,
|
||||
None,
|
||||
None,
|
||||
|| async { false },
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(outcome, DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement));
|
||||
assert!(
|
||||
!store
|
||||
.objects
|
||||
.lock()
|
||||
.await
|
||||
.contains_key(&memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str()))
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_usage_route_barrier_precedes_durable_reconciliation() {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
@@ -3425,6 +3561,14 @@ fn complete_usage_with_bucket_count(last_update: Option<std::time::SystemTime>,
|
||||
info
|
||||
}
|
||||
|
||||
async fn seed_usage_snapshot_for_leadership_claim(store: &Arc<MemoryConfigStore>) {
|
||||
let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str());
|
||||
let data = serde_json::to_vec(&complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0))
|
||||
.expect("leadership usage baseline should encode");
|
||||
store.objects.lock().await.insert(key.clone(), data);
|
||||
store.revisions.lock().await.insert(key, 1);
|
||||
}
|
||||
|
||||
fn usage_with_last_update(last_update: Option<std::time::SystemTime>) -> DataUsageInfo {
|
||||
complete_usage_with_bucket_count(last_update, 0)
|
||||
}
|
||||
@@ -5129,6 +5273,19 @@ fn test_background_heal_info_for_scan_start_marks_deep_active() {
|
||||
assert_eq!(info.bitrot_start_time, Some(now));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn background_heal_read_failures_never_become_initializable_defaults() {
|
||||
assert_eq!(
|
||||
classify_background_heal_read_error(&EcstoreError::ConfigNotFound),
|
||||
BackgroundHealInfoReadStatus::Missing
|
||||
);
|
||||
assert_eq!(
|
||||
classify_background_heal_read_error(&EcstoreError::SlowDown),
|
||||
BackgroundHealInfoReadStatus::Transient
|
||||
);
|
||||
assert!(decode_background_heal_info(b"not-json").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_background_heal_info_for_scan_start_keeps_deep_window_start() {
|
||||
with_var_unset(ENV_SCANNER_BITROT_CYCLE_SECS, || {
|
||||
|
||||
@@ -112,11 +112,39 @@ pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_basel
|
||||
}
|
||||
|
||||
pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe<F, Fut>(
|
||||
ctx: CancellationToken,
|
||||
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
|
||||
receiver: mpsc::Receiver<DataUsageInfo>,
|
||||
leader_epoch: Option<u64>,
|
||||
initial_baseline: Option<DataUsagePersistBaseline>,
|
||||
route_probe: F,
|
||||
) -> DataUsagePersistOutcome
|
||||
where
|
||||
F: Fn() -> Fut + Send + Sync,
|
||||
Fut: Future<Output = bool> + Send,
|
||||
{
|
||||
store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe_for_publication_epoch(
|
||||
ctx,
|
||||
storeapi,
|
||||
receiver,
|
||||
leader_epoch,
|
||||
initial_baseline,
|
||||
None,
|
||||
route_probe,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe_for_publication_epoch<
|
||||
F,
|
||||
Fut,
|
||||
>(
|
||||
ctx: CancellationToken,
|
||||
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
|
||||
mut receiver: mpsc::Receiver<DataUsageInfo>,
|
||||
leader_epoch: Option<u64>,
|
||||
initial_baseline: Option<DataUsagePersistBaseline>,
|
||||
expected_publication_epoch: Option<u64>,
|
||||
route_probe: F,
|
||||
) -> DataUsagePersistOutcome
|
||||
where
|
||||
@@ -134,6 +162,14 @@ where
|
||||
if let Some(leader_epoch) = leader_epoch {
|
||||
data_usage_info.scanner_epoch = Some(leader_epoch);
|
||||
}
|
||||
if let Some(expected_epoch) = expected_publication_epoch
|
||||
&& scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch)
|
||||
.await
|
||||
.is_none()
|
||||
{
|
||||
outcome = DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
|
||||
break 'updates;
|
||||
}
|
||||
let observational = data_usage_info.usage_snapshot_converged == Some(false);
|
||||
let target_path = if observational {
|
||||
DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str()
|
||||
@@ -154,7 +190,28 @@ where
|
||||
break;
|
||||
}
|
||||
|
||||
let mut publication_epoch = expected_publication_epoch;
|
||||
if observational && data_usage_info.usage_snapshot_authoritative_baseline.is_none() {
|
||||
let read_epoch = match expected_publication_epoch {
|
||||
Some(expected_epoch) => {
|
||||
if scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch)
|
||||
.await
|
||||
.is_none()
|
||||
{
|
||||
outcome = DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
|
||||
break 'updates;
|
||||
}
|
||||
expected_epoch
|
||||
}
|
||||
None => {
|
||||
let Some(read_epoch) = scanner_publication_epoch(storeapi.clone()).await else {
|
||||
outcome = DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
|
||||
break 'updates;
|
||||
};
|
||||
read_epoch
|
||||
}
|
||||
};
|
||||
publication_epoch = Some(read_epoch);
|
||||
let authoritative_data = match next_baseline.as_ref() {
|
||||
Some(baseline) => baseline.data.clone(),
|
||||
None => match read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()).await {
|
||||
@@ -175,25 +232,48 @@ where
|
||||
}
|
||||
},
|
||||
};
|
||||
let authoritative = match authoritative_data.as_deref() {
|
||||
Some(data) => match serde_json::from_slice::<DataUsageInfo>(data) {
|
||||
Ok(info) => info,
|
||||
Err(err) => {
|
||||
error!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_PERSIST_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
|
||||
state = "observed_baseline_decode_failed",
|
||||
error = %err,
|
||||
"Scanner refused to publish an observation from an invalid authoritative baseline"
|
||||
);
|
||||
outcome = DataUsagePersistOutcome::Failed;
|
||||
continue;
|
||||
}
|
||||
},
|
||||
None => DataUsageInfo::default(),
|
||||
let Some(authoritative_data) = authoritative_data else {
|
||||
warn!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_PERSIST_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
|
||||
state = "observed_baseline_missing",
|
||||
"Scanner deferred observational publication until an authoritative usage baseline exists"
|
||||
);
|
||||
outcome = DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
|
||||
break 'updates;
|
||||
};
|
||||
let authoritative = match serde_json::from_slice::<DataUsageInfo>(&authoritative_data) {
|
||||
Ok(info) if data_usage_info_has_persisted_baseline_identity(&info) => info,
|
||||
Ok(_) => {
|
||||
error!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_PERSIST_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
|
||||
state = "observed_baseline_identity_missing",
|
||||
"Scanner refused to publish an observation without authoritative baseline identity"
|
||||
);
|
||||
outcome = DataUsagePersistOutcome::Failed;
|
||||
continue;
|
||||
}
|
||||
Err(err) => {
|
||||
error!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_PERSIST_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
|
||||
state = "observed_baseline_decode_failed",
|
||||
error = %err,
|
||||
"Scanner refused to publish an observation from an invalid authoritative baseline"
|
||||
);
|
||||
outcome = DataUsagePersistOutcome::Failed;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
data_usage_info.usage_snapshot_authoritative_baseline = Some(authoritative.snapshot_identity());
|
||||
}
|
||||
@@ -240,6 +320,26 @@ where
|
||||
break 'updates;
|
||||
}
|
||||
|
||||
let publication_epoch_for_save = match expected_publication_epoch {
|
||||
Some(expected_epoch) => {
|
||||
if scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch)
|
||||
.await
|
||||
.is_none()
|
||||
{
|
||||
break DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
|
||||
}
|
||||
expected_epoch
|
||||
}
|
||||
None => match publication_epoch.take() {
|
||||
Some(epoch) => epoch,
|
||||
None => {
|
||||
let Some(epoch) = scanner_publication_epoch(storeapi.clone()).await else {
|
||||
break DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
|
||||
};
|
||||
epoch
|
||||
}
|
||||
},
|
||||
};
|
||||
let baseline = if !observational && cas_retry == 0 {
|
||||
next_baseline.take()
|
||||
} else {
|
||||
@@ -329,14 +429,22 @@ where
|
||||
}
|
||||
|
||||
let done_save = Metrics::time(Metric::SaveUsage);
|
||||
let save_result = save_config_shared_with_preconditions(
|
||||
storeapi.clone(),
|
||||
target_path,
|
||||
data.clone(),
|
||||
sha256hex.clone(),
|
||||
revision.preconditions(),
|
||||
)
|
||||
.await;
|
||||
let save_result = {
|
||||
let Some(_publication_admission) =
|
||||
scanner_publication_admission_for_epoch(storeapi.clone(), publication_epoch_for_save).await
|
||||
else {
|
||||
done_save();
|
||||
break DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
|
||||
};
|
||||
save_config_shared_with_preconditions(
|
||||
storeapi.clone(),
|
||||
target_path,
|
||||
data.clone(),
|
||||
sha256hex.clone(),
|
||||
revision.preconditions(),
|
||||
)
|
||||
.await
|
||||
};
|
||||
done_save();
|
||||
|
||||
match save_result {
|
||||
@@ -427,7 +535,16 @@ where
|
||||
if observational {
|
||||
invalidate_admin_data_usage_snapshot_cache().await;
|
||||
} else {
|
||||
cleanup_observed_data_usage_snapshot(storeapi.clone(), &data_usage_info).await;
|
||||
let cleanup_ok = cleanup_observed_data_usage_snapshot_for_epoch(
|
||||
storeapi.clone(),
|
||||
&data_usage_info,
|
||||
expected_publication_epoch,
|
||||
)
|
||||
.await;
|
||||
if expected_publication_epoch.is_some() && !cleanup_ok {
|
||||
outcome = DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
|
||||
break 'updates;
|
||||
}
|
||||
invalidate_data_usage_snapshot_cache().await;
|
||||
replace_bucket_usage_memory_from_info(&data_usage_info).await;
|
||||
}
|
||||
@@ -438,7 +555,16 @@ where
|
||||
if observational {
|
||||
invalidate_admin_data_usage_snapshot_cache().await;
|
||||
} else {
|
||||
cleanup_observed_data_usage_snapshot(storeapi.clone(), &data_usage_info).await;
|
||||
let cleanup_ok = cleanup_observed_data_usage_snapshot_for_epoch(
|
||||
storeapi.clone(),
|
||||
&data_usage_info,
|
||||
expected_publication_epoch,
|
||||
)
|
||||
.await;
|
||||
if expected_publication_epoch.is_some() && !cleanup_ok {
|
||||
outcome = DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
|
||||
break 'updates;
|
||||
}
|
||||
invalidate_data_usage_snapshot_cache().await;
|
||||
}
|
||||
global_metrics().record_scanner_usage_save_result(ScannerUsageSaveResult::Success);
|
||||
@@ -460,7 +586,16 @@ where
|
||||
if observational {
|
||||
invalidate_admin_data_usage_snapshot_cache().await;
|
||||
} else {
|
||||
cleanup_observed_data_usage_snapshot(storeapi.clone(), &data_usage_info).await;
|
||||
let cleanup_ok = cleanup_observed_data_usage_snapshot_for_epoch(
|
||||
storeapi.clone(),
|
||||
&data_usage_info,
|
||||
expected_publication_epoch,
|
||||
)
|
||||
.await;
|
||||
if expected_publication_epoch.is_some() && !cleanup_ok {
|
||||
outcome = DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
|
||||
break 'updates;
|
||||
}
|
||||
invalidate_data_usage_snapshot_cache().await;
|
||||
replace_bucket_usage_memory_from_info(&data_usage_info).await;
|
||||
}
|
||||
@@ -471,7 +606,10 @@ where
|
||||
|
||||
if backup_due {
|
||||
let done_save = Metrics::time(Metric::SaveUsage);
|
||||
if let Err(e) = sync_data_usage_backup_from_primary(&ctx, storeapi.clone()).await {
|
||||
let backup_result =
|
||||
sync_data_usage_backup_from_primary_for_epoch(&ctx, storeapi.clone(), expected_publication_epoch).await;
|
||||
done_save();
|
||||
if let Err(e) = backup_result {
|
||||
warn!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_PERSIST_STATE,
|
||||
@@ -482,22 +620,50 @@ where
|
||||
error = %e,
|
||||
"Scanner data usage backup save failed"
|
||||
);
|
||||
if scanner_publication_epoch_changed(&e) {
|
||||
outcome = DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
|
||||
break 'updates;
|
||||
}
|
||||
outcome = DataUsagePersistOutcome::Failed;
|
||||
break 'updates;
|
||||
}
|
||||
done_save();
|
||||
}
|
||||
}
|
||||
|
||||
outcome
|
||||
}
|
||||
|
||||
pub(super) async fn cleanup_observed_data_usage_snapshot(
|
||||
pub(super) async fn cleanup_observed_data_usage_snapshot_for_epoch(
|
||||
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
|
||||
authoritative: &DataUsageInfo,
|
||||
) {
|
||||
expected_publication_epoch: Option<u64>,
|
||||
) -> bool {
|
||||
let read_epoch = match expected_publication_epoch {
|
||||
Some(expected_epoch) => {
|
||||
if scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch)
|
||||
.await
|
||||
.is_none()
|
||||
{
|
||||
return false;
|
||||
}
|
||||
expected_epoch
|
||||
}
|
||||
None => match scanner_publication_epoch(storeapi.clone()).await {
|
||||
Some(read_epoch) => read_epoch,
|
||||
None => return false,
|
||||
},
|
||||
};
|
||||
if expected_publication_epoch.is_some()
|
||||
&& scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch)
|
||||
.await
|
||||
.is_none()
|
||||
{
|
||||
return false;
|
||||
}
|
||||
let (observed_data, revision) =
|
||||
match read_config_with_revision(storeapi.clone(), DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str()).await {
|
||||
Ok((Some(data), revision)) => (data, revision),
|
||||
Ok((None, _)) => return,
|
||||
Ok((None, _)) => return true,
|
||||
Err(err) => {
|
||||
error!(
|
||||
target: "rustfs::scanner",
|
||||
@@ -509,7 +675,7 @@ pub(super) async fn cleanup_observed_data_usage_snapshot(
|
||||
error = %err,
|
||||
"Scanner could not inspect observational data usage snapshot before authoritative cleanup"
|
||||
);
|
||||
return;
|
||||
return true;
|
||||
}
|
||||
};
|
||||
let observed = match serde_json::from_slice::<DataUsageInfo>(&observed_data) {
|
||||
@@ -525,25 +691,26 @@ pub(super) async fn cleanup_observed_data_usage_snapshot(
|
||||
error = %err,
|
||||
"Scanner refused to remove an invalid observational data usage snapshot after authoritative save"
|
||||
);
|
||||
return;
|
||||
return true;
|
||||
}
|
||||
};
|
||||
if observed_data_usage_is_newer(&observed, authoritative) {
|
||||
return;
|
||||
return true;
|
||||
}
|
||||
|
||||
let result = storeapi
|
||||
.delete_config_object(
|
||||
RUSTFS_META_BUCKET,
|
||||
DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(),
|
||||
ScannerObjectOptions {
|
||||
delete_prefix: true,
|
||||
delete_prefix_object: true,
|
||||
http_preconditions: Some(revision.preconditions()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
let result = delete_config_with_publication_admission_for_epoch(
|
||||
storeapi,
|
||||
RUSTFS_META_BUCKET,
|
||||
DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(),
|
||||
ScannerObjectOptions {
|
||||
delete_prefix: true,
|
||||
delete_prefix_object: true,
|
||||
http_preconditions: Some(revision.preconditions()),
|
||||
..Default::default()
|
||||
},
|
||||
read_epoch,
|
||||
)
|
||||
.await;
|
||||
|
||||
match result {
|
||||
Ok(_)
|
||||
@@ -564,6 +731,10 @@ pub(super) async fn cleanup_observed_data_usage_snapshot(
|
||||
error = %err,
|
||||
"Scanner could not remove stale observational data usage snapshot after authoritative save"
|
||||
);
|
||||
if scanner_publication_epoch_changed(&err) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
@@ -56,9 +56,10 @@ use crate::storage_api::scan::NamespaceLocking as _;
|
||||
use crate::storage_api::scanner_io::{BucketInfo, BucketOptions};
|
||||
use crate::{
|
||||
BucketTargetSys, BucketVersioningSys, Disk, DiskError, ECStore, EcstoreError as Error, EcstoreResult as Result,
|
||||
RUSTFS_META_BUCKET, ReplicationConfig, STORAGE_FORMAT_FILE, ScannerDiskExt as _, ScannerLifecycleConfigExt as _,
|
||||
ScannerReplicationConfigExt as _, ScannerVersioningConfigExt as _, SetDisks, StorageError, enqueue_runtime_free_version,
|
||||
get_lifecycle_config, get_object_lock_config, get_replication_config, runtime_tier_names, storageclass,
|
||||
RUSTFS_META_BUCKET, ReplicationConfig, STORAGE_FORMAT_FILE, ScannerConfigObjectDelete as _, ScannerDiskExt as _,
|
||||
ScannerLifecycleConfigExt as _, ScannerReplicationConfigExt as _, ScannerVersioningConfigExt as _, SetDisks, StorageError,
|
||||
enqueue_runtime_free_version, get_lifecycle_config, get_object_lock_config, get_replication_config, runtime_tier_names,
|
||||
scanner_publication_admission_for_epoch, scanner_publication_epoch, storageclass,
|
||||
};
|
||||
|
||||
pub(crate) const SCANNER_SKIP_FILE_ERROR: &str = "skip file";
|
||||
@@ -143,6 +144,10 @@ pub struct ScannerBucketScanPlan {
|
||||
all_buckets: Arc<Vec<BucketInfo>>,
|
||||
digest: DataUsageScanPlanDigest,
|
||||
leader_epoch: u64,
|
||||
/// Epoch captured once for the whole scanner cycle. `None` is retained
|
||||
/// for unfenced test implementations; production plans always carry the
|
||||
/// admission token captured before bucket enumeration.
|
||||
publication_epoch: Option<u64>,
|
||||
dirty_usage_buckets: Arc<DirtyUsageBuckets>,
|
||||
bucket_failures: ScannerBucketFailureState,
|
||||
pending_maintenance_work: Arc<AtomicBool>,
|
||||
@@ -578,6 +583,7 @@ fn scanner_activity_preflight(
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct ScannerCycleResult {
|
||||
pub(crate) status: ScannerCycleStatus,
|
||||
publication_epoch: Option<u64>,
|
||||
dirty_usage_clear: Option<DirtyUsageBuckets>,
|
||||
remote_dirty_usage_acknowledgements: Vec<crate::scanner::ScannerDirtyUsageAcknowledgement>,
|
||||
failed_dirty_usage: bool,
|
||||
@@ -589,6 +595,7 @@ impl ScannerCycleResult {
|
||||
pub(crate) fn new(status: ScannerCycleStatus, dirty_usage_clear: Option<DirtyUsageBuckets>) -> Self {
|
||||
Self {
|
||||
status,
|
||||
publication_epoch: None,
|
||||
dirty_usage_clear,
|
||||
remote_dirty_usage_acknowledgements: Vec::new(),
|
||||
failed_dirty_usage: false,
|
||||
@@ -597,6 +604,15 @@ impl ScannerCycleResult {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn with_publication_epoch(mut self, publication_epoch: Option<u64>) -> Self {
|
||||
self.publication_epoch = publication_epoch;
|
||||
self
|
||||
}
|
||||
|
||||
pub(crate) fn publication_epoch(&self) -> Option<u64> {
|
||||
self.publication_epoch
|
||||
}
|
||||
|
||||
fn with_failed_dirty_usage(mut self, failed_dirty_usage: bool) -> Self {
|
||||
self.failed_dirty_usage = failed_dirty_usage;
|
||||
self
|
||||
|
||||
@@ -395,6 +395,7 @@ pub(super) async fn persist_and_publish_cache_snapshot(
|
||||
updates: &mpsc::Sender<DataUsageCache>,
|
||||
mut cache_snapshot: DataUsageCache,
|
||||
cache_cycle_floor: &AtomicU64,
|
||||
expected_publication_epoch: u64,
|
||||
) -> Option<SystemTime> {
|
||||
let source = cache_snapshot.info.source?;
|
||||
let guard = match acquire_scanner_cache_locks(store.as_ref(), DATA_USAGE_CACHE_NAME, source).await {
|
||||
@@ -489,7 +490,7 @@ pub(super) async fn persist_and_publish_cache_snapshot(
|
||||
|
||||
let done_save = Metrics::time(Metric::SaveUsage);
|
||||
if let Err(e) = cache_snapshot
|
||||
.save_with_revisions(store, DATA_USAGE_CACHE_NAME, &revisions)
|
||||
.save_with_revisions_for_epoch(store.clone(), DATA_USAGE_CACHE_NAME, &revisions, expected_publication_epoch)
|
||||
.await
|
||||
{
|
||||
error!(
|
||||
@@ -519,6 +520,24 @@ pub(super) async fn persist_and_publish_cache_snapshot(
|
||||
);
|
||||
return None;
|
||||
}
|
||||
// The persisted-root fast path performs no PUT, so it also needs the
|
||||
// cycle token re-admission before forwarding the root to the aggregate.
|
||||
// This final check covers both the fast path and a successful save.
|
||||
if scanner_publication_admission_for_epoch(store.clone(), expected_publication_epoch)
|
||||
.await
|
||||
.is_none()
|
||||
{
|
||||
error!(
|
||||
target: "rustfs::scanner::io",
|
||||
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_IO,
|
||||
cache_name = DATA_USAGE_CACHE_NAME,
|
||||
state = "publication_epoch_changed_before_publish",
|
||||
"Scanner cache root publish skipped after movement epoch change"
|
||||
);
|
||||
return None;
|
||||
}
|
||||
drop(guard);
|
||||
let last_update = cache_snapshot.info.last_update;
|
||||
|
||||
|
||||
@@ -31,6 +31,7 @@ impl ScannerIOCache for SetDisks {
|
||||
all_buckets,
|
||||
digest: scan_plan_digest,
|
||||
leader_epoch,
|
||||
publication_epoch,
|
||||
dirty_usage_buckets,
|
||||
bucket_failures,
|
||||
pending_maintenance_work,
|
||||
@@ -40,6 +41,12 @@ impl ScannerIOCache for SetDisks {
|
||||
let set_label = self.set_index.to_string();
|
||||
|
||||
let source = DataUsageCacheSource::new(self.pool_index, self.set_index);
|
||||
let expected_publication_epoch = match publication_epoch {
|
||||
Some(epoch) => epoch,
|
||||
None => scanner_publication_epoch(self.clone())
|
||||
.await
|
||||
.ok_or_else(|| StorageError::other("scanner cache publication is blocked by data movement"))?,
|
||||
};
|
||||
let mut old_cache = DataUsageCache::default();
|
||||
if let Err(e) = old_cache.load(self.clone(), DATA_USAGE_CACHE_NAME).await {
|
||||
warn!(
|
||||
@@ -76,10 +83,16 @@ impl ScannerIOCache for SetDisks {
|
||||
cache.replace(&bucket.name, DATA_USAGE_ROOT, DataUsageEntry::default());
|
||||
}
|
||||
reset_disk_bucket_scan_gauges(&pool_label, &set_label);
|
||||
return persist_and_publish_cache_snapshot(self, &updates, cache, cache_cycle_floor.as_ref())
|
||||
.await
|
||||
.map(|_| ())
|
||||
.ok_or_else(|| StorageError::other("failed to persist empty scanner set scope"));
|
||||
return persist_and_publish_cache_snapshot(
|
||||
self,
|
||||
&updates,
|
||||
cache,
|
||||
cache_cycle_floor.as_ref(),
|
||||
expected_publication_epoch,
|
||||
)
|
||||
.await
|
||||
.map(|_| ())
|
||||
.ok_or_else(|| StorageError::other("failed to persist empty scanner set scope"));
|
||||
}
|
||||
|
||||
let (disks, healing) = self.get_online_disks_with_healing(false).await;
|
||||
@@ -414,6 +427,7 @@ impl ScannerIOCache for SetDisks {
|
||||
let pending_maintenance_work_clone = pending_maintenance_work.clone();
|
||||
let dirty_usage_buckets_clone = dirty_usage_buckets.clone();
|
||||
let cache_cycle_floor_clone = cache_cycle_floor.clone();
|
||||
let expected_publication_epoch_clone = expected_publication_epoch;
|
||||
let remote_server_epoch = match worker_mode {
|
||||
NamespaceScannerWorkerMode::RemoteV4(server_epoch) => Some(server_epoch),
|
||||
NamespaceScannerWorkerMode::Coordinator => None,
|
||||
@@ -753,6 +767,23 @@ impl ScannerIOCache for SetDisks {
|
||||
);
|
||||
continue;
|
||||
}
|
||||
if scanner_publication_admission_for_epoch(store_clone_clone.clone(), expected_publication_epoch)
|
||||
.await
|
||||
.is_none()
|
||||
{
|
||||
record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
|
||||
error!(
|
||||
target: "rustfs::scanner::io",
|
||||
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_IO,
|
||||
bucket = %bucket.name,
|
||||
cache_name = %cache_name,
|
||||
state = "publication_epoch_changed_before_reuse",
|
||||
"Current scanner bucket cache root publish skipped after movement epoch change"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
if let Err(e) =
|
||||
send_cache_root_entry(&bucket_result_tx_clone, *root, &cache, &pending_maintenance_work_clone)
|
||||
.await
|
||||
@@ -901,7 +932,12 @@ impl ScannerIOCache for SetDisks {
|
||||
{
|
||||
let done_save = Metrics::time(Metric::SaveUsage);
|
||||
if let Err(e) = cache
|
||||
.save_with_revisions(store_clone_clone.clone(), cache_name.as_str(), &revisions)
|
||||
.save_with_revisions_for_epoch(
|
||||
store_clone_clone.clone(),
|
||||
cache_name.as_str(),
|
||||
&revisions,
|
||||
expected_publication_epoch_clone,
|
||||
)
|
||||
.await
|
||||
{
|
||||
error!(
|
||||
@@ -958,7 +994,12 @@ impl ScannerIOCache for SetDisks {
|
||||
false
|
||||
} else {
|
||||
match partial_cache
|
||||
.save_with_revisions(store_clone_clone.clone(), cache_name.as_str(), &revisions)
|
||||
.save_with_revisions_for_epoch(
|
||||
store_clone_clone.clone(),
|
||||
cache_name.as_str(),
|
||||
&revisions,
|
||||
expected_publication_epoch_clone,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(()) => true,
|
||||
@@ -1029,7 +1070,12 @@ impl ScannerIOCache for SetDisks {
|
||||
|
||||
let done_save = Metrics::time(Metric::SaveUsage);
|
||||
if let Err(e) = cache
|
||||
.save_with_revisions(store_clone_clone.clone(), &cache_name, &revisions)
|
||||
.save_with_revisions_for_epoch(
|
||||
store_clone_clone.clone(),
|
||||
&cache_name,
|
||||
&revisions,
|
||||
expected_publication_epoch_clone,
|
||||
)
|
||||
.await
|
||||
{
|
||||
done_save();
|
||||
@@ -1064,6 +1110,24 @@ impl ScannerIOCache for SetDisks {
|
||||
continue;
|
||||
}
|
||||
|
||||
if scanner_publication_admission_for_epoch(store_clone_clone.clone(), expected_publication_epoch_clone)
|
||||
.await
|
||||
.is_none()
|
||||
{
|
||||
record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
|
||||
error!(
|
||||
target: "rustfs::scanner::io",
|
||||
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_IO,
|
||||
bucket = %bucket.name,
|
||||
cache_name = %cache_name,
|
||||
state = "publication_epoch_changed_after_save",
|
||||
"Scanner bucket cache root publish skipped after movement epoch change"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
debug!(
|
||||
target: "rustfs::scanner::io",
|
||||
event = EVENT_SCANNER_DATA_USAGE_STREAM,
|
||||
@@ -1159,7 +1223,14 @@ impl ScannerIOCache for SetDisks {
|
||||
cache.info.lkg_scan_plan_digest = None;
|
||||
cache.clone()
|
||||
};
|
||||
let _ = persist_and_publish_cache_snapshot(self.clone(), &updates, cache_snapshot, cache_cycle_floor.as_ref()).await;
|
||||
let _ = persist_and_publish_cache_snapshot(
|
||||
self.clone(),
|
||||
&updates,
|
||||
cache_snapshot,
|
||||
cache_cycle_floor.as_ref(),
|
||||
expected_publication_epoch,
|
||||
)
|
||||
.await;
|
||||
} else {
|
||||
let mut incomplete_scope = cache_mutex.lock().await.clone();
|
||||
incomplete_scope.info.name = DATA_USAGE_ROOT.to_string();
|
||||
|
||||
@@ -68,6 +68,19 @@ impl ScannerIOCycle for ECStore {
|
||||
));
|
||||
}
|
||||
|
||||
// Capture one storage-owned movement epoch for the entire cycle. Set
|
||||
// workers must not each observe a fresh epoch: a movement transition
|
||||
// between sets would otherwise allow a mixed-generation aggregate.
|
||||
let publication_epoch = match self.scanner_data_usage_publication_admission().await {
|
||||
Some(admission) => Some(admission.epoch()),
|
||||
None => {
|
||||
return Ok(ScannerCycleResult::new(
|
||||
ScannerCycleStatus::Deferred(ScannerCycleDeferReason::DataMovement),
|
||||
None,
|
||||
));
|
||||
}
|
||||
};
|
||||
|
||||
let distributed = self.setup_is_dist_erasure().await;
|
||||
let activity_before = match scanner_activity_preflight(crate::scanner::probe_scanner_activity(self, distributed).await) {
|
||||
ScannerActivityPreflight::Ready(snapshot) => snapshot,
|
||||
@@ -131,7 +144,9 @@ impl ScannerIOCycle for ECStore {
|
||||
if all_buckets.is_empty() {
|
||||
reset_set_scan_gauges();
|
||||
if !bucket_plan_complete {
|
||||
return Ok(ScannerCycleResult::new(ScannerCycleStatus::Incomplete, None));
|
||||
return Ok(
|
||||
ScannerCycleResult::new(ScannerCycleStatus::Incomplete, None).with_publication_epoch(publication_epoch)
|
||||
);
|
||||
}
|
||||
let activity_status = scanner_cycle_activity_status(self, distributed, &activity_before).await;
|
||||
let dirty_usage_status = dirty_usage_snapshot_status(&dirty_usage_snapshot);
|
||||
@@ -155,7 +170,7 @@ impl ScannerIOCycle for ECStore {
|
||||
)
|
||||
.await?
|
||||
{
|
||||
return Ok(ScannerCycleResult::new(status, None));
|
||||
return Ok(ScannerCycleResult::new(status, None).with_publication_epoch(publication_epoch));
|
||||
}
|
||||
let dirty_usage_clear =
|
||||
(status == ScannerCycleStatus::Complete).then(|| dirty_usage_snapshot.buckets.as_ref().clone());
|
||||
@@ -165,6 +180,7 @@ impl ScannerIOCycle for ECStore {
|
||||
Vec::new()
|
||||
};
|
||||
return Ok(ScannerCycleResult::new(status, dirty_usage_clear)
|
||||
.with_publication_epoch(publication_epoch)
|
||||
.with_remote_dirty_usage_acknowledgements(remote_dirty_usage_acknowledgements));
|
||||
}
|
||||
|
||||
@@ -180,7 +196,7 @@ impl ScannerIOCycle for ECStore {
|
||||
"Scanner set state update detected missing disk sets"
|
||||
);
|
||||
reset_set_scan_gauges();
|
||||
return Ok(ScannerCycleResult::new(ScannerCycleStatus::Incomplete, None));
|
||||
return Ok(ScannerCycleResult::new(ScannerCycleStatus::Incomplete, None).with_publication_epoch(publication_epoch));
|
||||
}
|
||||
|
||||
let set_scan_limit = scanner_budgeted_concurrency_limit(
|
||||
@@ -250,6 +266,7 @@ impl ScannerIOCycle for ECStore {
|
||||
all_buckets: Arc::clone(&all_buckets),
|
||||
digest: scan_plan_digest,
|
||||
leader_epoch,
|
||||
publication_epoch,
|
||||
dirty_usage_buckets: dirty_usage_snapshot.buckets.clone(),
|
||||
bucket_failures: bucket_failures.clone(),
|
||||
pending_maintenance_work: pending_maintenance_work.clone(),
|
||||
@@ -430,6 +447,7 @@ impl ScannerIOCycle for ECStore {
|
||||
Vec::new()
|
||||
};
|
||||
Ok(ScannerCycleResult::new(cycle_status, dirty_usage_clear)
|
||||
.with_publication_epoch(publication_epoch)
|
||||
.with_remote_dirty_usage_acknowledgements(remote_dirty_usage_acknowledgements)
|
||||
.with_failed_dirty_usage(!failed_buckets.is_empty())
|
||||
.with_pending_maintenance_work(pending_maintenance_work)
|
||||
|
||||
@@ -145,6 +145,50 @@ async fn scanner_cache_locks_allow_cross_source_workers() {
|
||||
assert!(!second.is_lock_lost());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_set_cache_admission_tracks_owner_snapshot_and_fails_closed() {
|
||||
let (_temp_dir, store) = setup_two_pool_scanner_store().await;
|
||||
let set = store.pools[0].disk_set[0].clone();
|
||||
|
||||
assert!(
|
||||
set.scanner_data_usage_publication_admission_guard().await.is_none(),
|
||||
"a set must not publish before the owner has refreshed its movement snapshot"
|
||||
);
|
||||
assert!(!store.scanner_data_usage_publication_blocked().await);
|
||||
assert!(
|
||||
set.scanner_data_usage_publication_admission_guard().await.is_some(),
|
||||
"an idle owner snapshot should admit the set cache"
|
||||
);
|
||||
|
||||
let mut pool_stats = vec![EcstoreRebalanceStats::default(); store.pools.len()];
|
||||
pool_stats[0] = EcstoreRebalanceStats {
|
||||
participating: true,
|
||||
info: EcstoreRebalanceInfo {
|
||||
start_time: Some(OffsetDateTime::now_utc()),
|
||||
status: EcstoreRebalStatus::Started,
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
*store.rebalance_meta.write().await = Some(EcstoreRebalanceMeta {
|
||||
id: Uuid::new_v4().to_string(),
|
||||
pool_stats,
|
||||
..Default::default()
|
||||
});
|
||||
assert!(store.scanner_data_usage_publication_blocked().await);
|
||||
assert!(
|
||||
set.scanner_data_usage_publication_admission_guard().await.is_none(),
|
||||
"active movement must keep set cache publication blocked"
|
||||
);
|
||||
|
||||
*store.rebalance_meta.write().await = None;
|
||||
assert!(!store.scanner_data_usage_publication_blocked().await);
|
||||
assert!(
|
||||
set.scanner_data_usage_publication_admission_guard().await.is_some(),
|
||||
"an idle owner refresh must make set cache publication live again"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_cycle_is_deferred_while_rebalance_is_active() {
|
||||
let (_temp_dir, store) = setup_two_pool_scanner_store().await;
|
||||
|
||||
Reference in New Issue
Block a user