gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-30 00:47:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(scanner): fence movement generation publication (#6461)

Browse Source 
* feat(scanner): add movement generation fencing

* fix(scanner): prioritize unverified cycle deferral

* feat(ecstore): add scanner publication lease fence

* feat(rpc): add scanner publication lease protocol

* feat(scanner): hold remote leases through usage publish

* test(scanner): cover publication lease fencing

* fix(scanner): fence remote leases across restart and delay

* feat(rpc): fence scanner publication rename writes

* fix(scanner): fence observed cleanup deletes

* fix(proto): qualify lease release test types

* fix(scanner): pin movement notifications

* fix(scanner): clean publication imports

* fix(ecstore): satisfy scanner fence clippy

* refactor(scanner): group wait and publication options

* fix(scanner): satisfy final lint and facade guards

* fix(rpc): resolve facade export conflicts

* fix(ci): remove unused decommission and healing facades

* fix(ci): cfg-gate test-only usage overlay import

* fix(scanner): wake on remote scanner restart
This commit is contained in:
cxymds
2026-08-24 14:17:35 +08:00
committed by GitHub
parent 1c5c28842a
commit eec0e0e056
30 changed files with 2926 additions and 230 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/scanner/src/scanner.rs
+159 -13
View File
@@ -63,6 +63,7 @@ use tokio_util::sync::CancellationToken;
use tokio_util::task::AbortOnDropHandle;
use tracing::{debug, error, info, instrument, warn};
use crate::storage_api::owner::SCANNER_PUBLICATION_LEASE_TTL_MS;
use crate::storage_api::scan::{
BucketOperations, BucketOptions, NamespaceLocking as _, SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION,
SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SCANNER_ACTIVITY_PROTOCOL_VERSION,
@@ -71,9 +72,9 @@ use crate::{
ECStore, EcstoreError, RUSTFS_META_BUCKET, SCANNER_PUBLICATION_EPOCH_CHANGED, ScannerLifecycleConfigExt as _,
ScannerReplicationConfigExt as _, delete_config_with_publication_admission_for_epoch, get_lifecycle_config,
get_replication_config, invalidate_admin_data_usage_snapshot_cache, invalidate_data_usage_snapshot_cache, read_config,
replace_bucket_usage_memory_from_info, save_config, save_config_shared_with_preconditions, save_config_with_preconditions,
save_config_with_publication_admission_for_epoch, scanner_is_erasure_sd, scanner_publication_admission_for_epoch,
scanner_publication_epoch, scanner_publication_epoch_changed,
replace_bucket_usage_memory_from_info, save_config, save_config_shared_with_preconditions_and_lease_fence,
save_config_with_preconditions, save_config_with_publication_admission_for_epoch, scanner_is_erasure_sd,
scanner_publication_admission_for_epoch, scanner_publication_epoch, scanner_publication_epoch_changed,
};
const LOG_COMPONENT_SCANNER: &str = "scanner";
@@ -125,6 +126,8 @@ const MAINTENANCE_FEATURE_INSPECTION_RETRY_MAX_INTERVAL: Duration = Duration::fr
const MAX_MAINTENANCE_FEATURE_INSPECTION_ATTEMPTS: usize = 2;
const SCANNER_PERSIST_CAS_RETRIES: usize = 2;
const DATA_USAGE_BACKUP_INTERVAL_CYCLES: u64 = 10;
const SCANNER_PUBLICATION_LEASE_FENCE_MAX_ENTRIES: usize = 256;
const SCANNER_PUBLICATION_LEASE_FENCE_MAX_BYTES: usize = 64 * 1024;
const SCANNER_CYCLE_STATE_MAGIC: &[u8; 8] = b"RSCYC001";
const SCANNER_CYCLE_STATE_HEADER_LEN: usize = 24;
#[cfg(test)]
@@ -137,6 +140,10 @@ static SCANNER_CYCLE_STATE_PERSIST_TEST_HOOK: LazyLock<StdMutex<Option<ScannerCy
static SCANNER_CYCLE_RECOVERY_WAKE: LazyLock<Notify> = LazyLock::new(Notify::new);
fn remote_publication_lease_fence_targets_are_required(target_count: usize, grants_present: bool, fence_present: bool) -> bool {
target_count > 0 && (!grants_present || !fence_present)
}
pub(super) fn notify_scanner_cycle_recovery_wake() {
SCANNER_CYCLE_RECOVERY_WAKE.notify_one();
}
@@ -407,19 +414,24 @@ async fn sync_data_usage_backup_from_primary(
ctx: &CancellationToken,
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
) -> Result<(), EcstoreError> {
sync_data_usage_backup_from_primary_for_epoch(ctx, storeapi, None).await
sync_data_usage_backup_from_primary_for_epoch_and_lease_and_fence(ctx, storeapi, None, None, None).await
}
async fn sync_data_usage_backup_from_primary_for_epoch(
async fn sync_data_usage_backup_from_primary_for_epoch_and_lease_and_fence(
ctx: &CancellationToken,
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
expected_publication_epoch: Option<u64>,
remote_lease_deadline: Option<std::time::Instant>,
scanner_publication_lease_fence: Option<&str>,
) -> Result<(), EcstoreError> {
let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str());
for retry in 0..=SCANNER_PERSIST_CAS_RETRIES {
if ctx.is_cancelled() {
return Ok(());
}
if remote_lease_deadline.is_some_and(|deadline| std::time::Instant::now() >= deadline) {
return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED));
}
let read_epoch = match expected_publication_epoch {
Some(expected_epoch) => {
@@ -446,6 +458,10 @@ async fn sync_data_usage_backup_from_primary_for_epoch(
}
let primary = Bytes::from(primary);
if remote_lease_deadline.is_some_and(|deadline| std::time::Instant::now() >= deadline) {
return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED));
}
let (backup, revision) = read_config_with_revision(storeapi.clone(), &backup_path).await?;
if backup.as_deref() == Some(primary.as_ref()) {
if scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch)
@@ -462,18 +478,22 @@ async fn sync_data_usage_backup_from_primary_for_epoch(
let sha256hex = Some(hex_simd::encode_to_string(Sha256::digest(&primary), hex_simd::AsciiCase::Lower));
let save_result = {
if remote_lease_deadline.is_some_and(|deadline| std::time::Instant::now() >= deadline) {
return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED));
}
let Some(_publication_admission) = scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch).await else {
if retry < SCANNER_PERSIST_CAS_RETRIES {
continue;
}
return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED));
};
save_config_shared_with_preconditions(
save_config_shared_with_preconditions_and_lease_fence(
storeapi.clone(),
&backup_path,
primary.clone(),
sha256hex,
revision.preconditions(),
scanner_publication_lease_fence,
)
.await
};
@@ -1377,8 +1397,89 @@ async fn run_data_scanner_cycle_with_budget(
};
let publication_deferred = publication_defer_reason.is_some();
let publication_epoch = scan_result.as_ref().ok().and_then(ScannerCycleResult::publication_epoch);
let remote_publication_lease_targets = if publication_defer_reason.is_none() {
scan_result
.as_ref()
.ok()
.map(|result| result.remote_publication_lease_targets().to_vec())
.unwrap_or_default()
} else {
Vec::new()
};
let mut remote_publication_leases = None;
let remote_lease_defer_reason = if remote_publication_lease_targets.is_empty() {
None
} else if usage_persist_timeout >= Duration::from_millis(SCANNER_PUBLICATION_LEASE_TTL_MS) {
// The lease is intentionally fixed-duration and has no renewal path.
// Refuse a persistence budget that could outlive it instead of
// allowing the peer to admit movement while a local PUT is in flight.
Some(ScannerCycleDeferReason::ActivityBaselineUnavailable)
} else if let Some(notification_system) = storeapi.notification_system() {
match notification_system
.acquire_scanner_publication_leases(remote_publication_lease_targets.clone())
.await
{
Ok(grants) => {
remote_publication_leases = Some((notification_system, grants));
None
}
Err(_) => Some(ScannerCycleDeferReason::ActivityBaselineUnavailable),
}
} else {
Some(ScannerCycleDeferReason::ActivityBaselineUnavailable)
};
let remote_lease_deadline = remote_publication_leases
.as_ref()
.and_then(|(_, grants)| grants.iter().map(|grant| grant.lease.expires_at).min());
// The transient fence is carried only to the SetDisks rename boundary;
// it is never inserted into FileInfo metadata. Keep the representation
// bounded and require one authenticated token per remote target so a
// partial grant can never silently fall back to an unfenced rename.
let remote_lease_fence = remote_publication_leases.as_ref().and_then(|(_, grants)| {
if grants.len() != remote_publication_lease_targets.len() || grants.len() > SCANNER_PUBLICATION_LEASE_FENCE_MAX_ENTRIES {
return None;
}
let mut fence = BTreeMap::new();
for grant in grants {
if grant.host.is_empty() || grant.host.len() > 1024 {
return None;
}
if fence.insert(grant.host.clone(), grant.lease.token.to_string()).is_some() {
return None;
}
}
if remote_publication_lease_targets
.iter()
.any(|(host, _, _)| !fence.contains_key(host))
{
return None;
}
serde_json::to_string(&fence)
.ok()
.filter(|encoded| encoded.len() <= SCANNER_PUBLICATION_LEASE_FENCE_MAX_BYTES)
});
let remote_lease_fence_defer_reason = (remote_publication_lease_fence_targets_are_required(
remote_publication_lease_targets.len(),
remote_publication_leases.is_some(),
remote_lease_fence.is_some(),
))
.then_some(ScannerCycleDeferReason::ActivityBaselineUnavailable);
let remote_lease_covers_persistence = remote_lease_deadline.is_none_or(|deadline| {
std::time::Instant::now()
.checked_add(usage_persist_timeout)
.is_some_and(|latest_finish| latest_finish < deadline)
});
let publication_defer_reason = publication_defer_reason
.or(remote_lease_defer_reason)
.or(remote_lease_fence_defer_reason);
let publication_defer_reason = (!remote_lease_covers_persistence)
.then_some(ScannerCycleDeferReason::ActivityBaselineUnavailable)
.or(publication_defer_reason);
let budget_elapsed = cycle_budget.budget_elapsed() && !ctx.is_cancelled();
let usage_persist_outcome = match publication_defer_reason {
let remote_lease_probe = remote_publication_leases
.as_ref()
.map(|(notification_system, grants)| (Arc::clone(notification_system), grants.clone()));
let mut usage_persist_outcome = match publication_defer_reason {
Some(reason) => {
drop(receiver);
DataUsagePersistOutcome::Deferred(reason)
@@ -1390,17 +1491,33 @@ async fn run_data_scanner_cycle_with_budget(
let storeapi_clone = storeapi.clone();
let ctx_clone = ctx.clone();
let route_probe_store = storeapi.clone();
let remote_lease_fence = remote_lease_fence.clone();
let mut usage_persist_task = AbortOnDropHandle::new(tokio::spawn(async move {
store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe_for_publication_epoch(
store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe_for_publication_epoch_and_lease_fence(
ctx_clone,
storeapi_clone,
receiver,
Some(leader_epoch),
Some(usage_persist_baseline),
publication_epoch,
ScannerPublicationFence::new(
publication_epoch,
remote_lease_deadline,
remote_lease_fence,
),
move || {
let storeapi = route_probe_store.clone();
async move { storeapi.scanner_data_usage_publication_blocked().await }
let remote_lease_probe = remote_lease_probe.clone();
async move {
if let Some((notification_system, grants)) = remote_lease_probe.as_ref()
&& notification_system.validate_scanner_publication_leases(grants).await.is_err()
{
// A remote restart or movement flip invalidates
// the token proof; usage_store interprets this
// as a publication barrier and performs no PUT.
return true;
}
storeapi.scanner_data_usage_publication_blocked().await
}
},
)
.await
@@ -1448,6 +1565,22 @@ async fn run_data_scanner_cycle_with_budget(
}
}
};
let lease_expired = remote_publication_leases
.as_ref()
.is_some_and(|(_, grants)| grants.iter().any(|grant| !grant.lease.is_valid()));
if let Some((notification_system, grants)) = remote_publication_leases.take() {
let release_result = notification_system.release_scanner_publication_leases(grants).await;
if lease_expired || release_result.is_err() {
// A lease that expired or could not be released is never treated
// as a successful authoritative publication. The peer may have
// admitted movement immediately after the lease ended.
usage_persist_outcome = if usage_persist_outcome == DataUsagePersistOutcome::Failed {
DataUsagePersistOutcome::Failed
} else {
DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable)
};
}
}
let unresolved_heal_work = global_metrics().current_scan_cycle_has_unresolved_heal_work();
let scan_cycle_result = match scan_result {
@@ -2199,7 +2332,16 @@ async fn run_data_scanner_with_maintenance_state(
);
let activity_poll_interval = backoff_enabled.then_some(runtime_config.cycle_interval.max(Duration::from_secs(1)));
let wake_reason = wait_for_next_scanner_cycle_with_activity(
let movement_generation_before_wait = storeapi.scanner_data_movement_generation();
let movement_changed = storeapi.scanner_data_movement_changed();
let movement_store = storeapi.clone();
let movement = ScannerMovementWaitContext {
movement_generation_seen: Some(movement_generation_before_wait),
movement_changed,
current_movement_generation: move || movement_store.scanner_data_movement_generation(),
is_lock_lost: || guard.is_lock_lost(),
};
let wake_reason = wait_for_next_scanner_cycle_with_activity_and_movement(
&ctx,
wait_plan.delay,
activity_poll_interval,
@@ -2211,7 +2353,7 @@ async fn run_data_scanner_with_maintenance_state(
runtime_config_generation_seen,
maintenance_generation_before_wait,
),
|| guard.is_lock_lost(),
movement,
|| probe_scanner_activity(storeapi.as_ref(), distributed),
)
.await;
@@ -2239,6 +2381,10 @@ async fn run_data_scanner_with_maintenance_state(
ScannerCycleWakeReason::ClusterMaintenance => {
clean_idle_backoff.reset();
}
ScannerCycleWakeReason::MovementGeneration => {
scanner_activity_seen = None;
clean_idle_backoff.reset();
}
ScannerCycleWakeReason::Timer
| ScannerCycleWakeReason::DirtyUsage
| ScannerCycleWakeReason::ClusterActivity
@@ -2612,7 +2758,7 @@ use usage_store::*;
pub use activity::scanner_topology_digest;
pub(crate) use activity::{
ScannerActivitySnapshot, ScannerDirtyUsageAcknowledgement, probe_scanner_activity, scanner_activity_allows_usage_publication,
scanner_activity_snapshot_digest, scanner_dirty_usage_acknowledgements,
scanner_activity_publication_lease_targets, scanner_activity_snapshot_digest, scanner_dirty_usage_acknowledgements,
};
pub(crate) use activity::{ScannerCycleOutcome, scanner_cycle_outcome_with_pending_maintenance};
#[cfg(test)]