feat(scanner): send scoped dirty usage acknowledgements (#7322)

This commit is contained in:
houseme
2026-09-07 05:28:50 +08:00
committed by GitHub
parent 1651541d38
commit f4049598e4
22 changed files with 728 additions and 109 deletions
+1
View File
@@ -100,6 +100,7 @@ pub use storage_api::ScannerReplicationConfig as ReplicationConfig;
pub use storage_api::scan::{
SCANNER_ACTIVITY_PROTOCOL_VERSION, SCANNER_ACTIVITY_V6_PROTOCOL_VERSION, SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES,
SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION, SCANNER_DIRTY_USAGE_SNAPSHOT_RPC_MAX_MESSAGE_SIZE,
SCANNER_SCOPED_DIRTY_USAGE_ACK_MAX_ENTRIES,
};
pub use workload_admission::set_scanner_workload_admission_snapshot_provider;
+5 -7
View File
@@ -2216,10 +2216,7 @@ where
false
} else if let Some(notification_system) = storeapi.scanner_notification_system() {
let acknowledgement_count = remote_dirty_usage_acknowledgements.len();
let acknowledgements = remote_dirty_usage_acknowledgements
.into_iter()
.map(|acknowledgement| (acknowledgement.host, acknowledgement.instance_id, acknowledgement.generation))
.collect();
let acknowledgements = remote_dirty_usage_acknowledgements.into_iter().map(Into::into).collect();
remote_dirty_usage_acknowledgement_pending(
cycle_info.current,
acknowledgement_count,
@@ -3578,9 +3575,10 @@ use usage_store::*;
pub use activity::scanner_topology_digest;
pub(crate) use activity::{
ScannerActivitySnapshot, ScannerDirtyUsageAcknowledgement, probe_scanner_activity, scanner_activity_allows_usage_publication,
scanner_activity_dirty_usage_state_for_host, scanner_activity_publication_lease_targets, scanner_activity_snapshot_digest,
scanner_activity_structural_digest, scanner_dirty_usage_acknowledgements,
ScannerActivitySnapshot, ScannerDirtyUsageAcknowledgement, ScannerDirtyUsageAcknowledgementKind, probe_scanner_activity,
scanner_activity_allows_usage_publication, scanner_activity_dirty_usage_state_for_host,
scanner_activity_publication_lease_targets, scanner_activity_snapshot_digest, scanner_activity_structural_digest,
scanner_dirty_usage_acknowledgements,
};
pub(crate) use activity::{ScannerCycleOutcome, scanner_cycle_outcome_with_pending_maintenance};
pub use backlog::{
+29 -2
View File
@@ -445,7 +445,34 @@ pub(crate) type ScannerActivitySnapshot = BTreeMap<String, ScannerNodeActivity>;
pub(crate) struct ScannerDirtyUsageAcknowledgement {
pub(crate) host: String,
pub(crate) instance_id: String,
pub(crate) generation: u64,
pub(crate) kind: ScannerDirtyUsageAcknowledgementKind,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub(crate) enum ScannerDirtyUsageAcknowledgementKind {
Generation(u64),
Scoped {
owner_id: String,
entries: Vec<crate::storage_api::EcstoreScannerScopedDirtyUsageAckEntry>,
},
}
impl From<ScannerDirtyUsageAcknowledgement> for crate::storage_api::EcstoreScannerDirtyUsageAcknowledgement {
fn from(acknowledgement: ScannerDirtyUsageAcknowledgement) -> Self {
match acknowledgement.kind {
ScannerDirtyUsageAcknowledgementKind::Generation(generation) => Self::Generation {
host: acknowledgement.host,
instance_id: acknowledgement.instance_id,
generation,
},
ScannerDirtyUsageAcknowledgementKind::Scoped { owner_id, entries } => Self::Scoped {
host: acknowledgement.host,
owner_id,
instance_id: acknowledgement.instance_id,
entries,
},
}
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
@@ -974,7 +1001,7 @@ pub(crate) fn scanner_dirty_usage_acknowledgements(snapshot: &ScannerActivitySna
.map(|(host, activity)| ScannerDirtyUsageAcknowledgement {
host: host.clone(),
instance_id: activity.instance_id.clone(),
generation: activity.dirty_usage_generation,
kind: ScannerDirtyUsageAcknowledgementKind::Generation(activity.dirty_usage_generation),
})
.collect()
}
+3 -3
View File
@@ -7433,7 +7433,7 @@ fn finalizing_a_saved_enum_without_proof_keeps_dirty_pending() {
let remote_acknowledgement = ScannerDirtyUsageAcknowledgement {
host: "node-2".to_string(),
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
generation: 11,
kind: ScannerDirtyUsageAcknowledgementKind::Generation(11),
};
let unsaved = crate::scanner_io::ScannerCycleResult::new(ScannerCycleStatus::Complete, Some(dirty_snapshot.clone()))
.with_remote_dirty_usage_acknowledgements(vec![remote_acknowledgement.clone()]);
@@ -8982,7 +8982,7 @@ fn post_lease_activity_proof_rejects_a_put_tail_that_finished_before_lease_acqui
ScannerDirtyUsageAcknowledgement {
host: "node-2".to_string(),
instance_id: "epoch-a".to_string(),
generation: 5,
kind: ScannerDirtyUsageAcknowledgementKind::Generation(5),
},
]);
let (outcome, _, acknowledgements) = finalize_scanner_cycle_result(
@@ -9142,7 +9142,7 @@ fn scanner_dirty_usage_acknowledgements_exclude_local_and_clean_nodes() {
vec![ScannerDirtyUsageAcknowledgement {
host: "node-3".to_string(),
instance_id: "epoch-dirty".to_string(),
generation: 11,
kind: ScannerDirtyUsageAcknowledgementKind::Generation(11),
}]
);
}
@@ -463,7 +463,7 @@ async fn scoped_ack_publication_rejects_builder_mutation_after_real_root_publish
"remote_ack_target" => scan.with_remote_dirty_usage_acknowledgements(vec![ScannerDirtyUsageAcknowledgement {
host: "proof-peer:9000".to_string(),
instance_id: crate::scanner_activity_epoch().to_string(),
generation: changed_generation,
kind: crate::scanner::ScannerDirtyUsageAcknowledgementKind::Generation(changed_generation),
}]),
"publication_epoch" => scan.with_publication_epoch(Some(changed_epoch)),
"remote_lease_targets" => scan.with_remote_publication_lease_targets(vec![(
+46 -5
View File
@@ -159,19 +159,26 @@ struct ScannerPeerDirtyUsageExpectation {
pending: bool,
}
fn verified_remote_dirty_usage_buckets(
#[derive(Debug, PartialEq, Eq)]
struct VerifiedRemoteDirtyUsage {
dirty_buckets: HashSet<String>,
acknowledgements: Vec<crate::scanner::ScannerDirtyUsageAcknowledgement>,
}
fn verified_remote_dirty_usage(
expected_peers: &HashMap<String, ScannerPeerDirtyUsageExpectation>,
peer_snapshots: Vec<(String, EcstoreScannerPeerDirtyUsageSnapshot)>,
) -> Option<HashSet<String>> {
) -> Option<VerifiedRemoteDirtyUsage> {
if expected_peers.is_empty() || peer_snapshots.len() != expected_peers.len() {
return None;
}
let mut received_peers = HashSet::with_capacity(peer_snapshots.len());
let mut dirty_buckets = HashSet::new();
let mut acknowledgements = Vec::new();
for (host, snapshot) in peer_snapshots {
let expected = expected_peers.get(&host)?;
if !received_peers.insert(host)
if !received_peers.insert(host.clone())
|| snapshot.instance_id != expected.instance_id
|| snapshot.generation != expected.generation
|| snapshot.generation == u64::MAX
@@ -182,10 +189,44 @@ fn verified_remote_dirty_usage_buckets(
{
return None;
}
dirty_buckets.extend(snapshot.buckets.into_keys());
let entries = snapshot
.buckets
.iter()
.map(|(bucket, state)| crate::storage_api::EcstoreScannerScopedDirtyUsageAckEntry {
bucket: bucket.clone(),
bucket_incarnation: state.bucket_incarnation,
generation: state.generation,
})
.collect::<Vec<_>>();
dirty_buckets.extend(snapshot.buckets.keys().cloned());
if !entries.is_empty() {
acknowledgements.push(crate::scanner::ScannerDirtyUsageAcknowledgement {
host,
instance_id: snapshot.instance_id,
kind: crate::scanner::ScannerDirtyUsageAcknowledgementKind::Scoped {
owner_id: snapshot.owner_id,
entries,
},
});
}
}
(received_peers.len() == expected_peers.len()).then_some(dirty_buckets)
(received_peers.len() == expected_peers.len()).then_some(VerifiedRemoteDirtyUsage {
dirty_buckets,
acknowledgements,
})
}
fn scanner_scoped_dirty_usage_ack_exceeds_cost_threshold(
acknowledgements: &[crate::scanner::ScannerDirtyUsageAcknowledgement],
) -> bool {
acknowledgements.iter().any(|acknowledgement| {
matches!(
&acknowledgement.kind,
crate::scanner::ScannerDirtyUsageAcknowledgementKind::Scoped { entries, .. }
if entries.len() > crate::SCANNER_SCOPED_DIRTY_USAGE_ACK_MAX_ENTRIES
)
})
}
fn complete_scanner_cache_snapshot_plan_digest(
+89 -18
View File
@@ -107,23 +107,32 @@ struct ScannerBucketScopeResolution<'a> {
requires_full_scan: bool,
}
struct ScannerBucketScopeResolutionResult {
scope: ScannerBucketScanScope,
remote_dirty_usage_acknowledgements: Vec<crate::scanner::ScannerDirtyUsageAcknowledgement>,
}
async fn resolve_scanner_bucket_scan_scope<S>(
store: &S,
distributed: bool,
resolution: ScannerBucketScopeResolution<'_>,
) -> ScannerBucketScanScope
) -> ScannerBucketScopeResolutionResult
where
S: ScannerStorage,
{
let default_result = |scope: ScannerBucketScanScope| ScannerBucketScopeResolutionResult {
scope,
remote_dirty_usage_acknowledgements: Vec::new(),
};
if resolution.requires_full_scan {
return ScannerBucketScanScope::default();
return default_result(ScannerBucketScanScope::default());
}
if !resolution.requested_scope.is_default()
|| !resolution.dirty_usage_snapshot.covers_all_pending
|| resolution.dirty_usage_snapshot.generation == u64::MAX
|| resolution.dirty_usage_snapshot.buckets.len() > crate::SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES
{
return resolution.requested_scope;
return default_result(resolution.requested_scope);
}
let mut dirty_buckets = resolution
@@ -134,10 +143,10 @@ where
.collect::<HashSet<_>>();
if distributed {
let Some(notification_system) = store.scanner_notification_system() else {
return resolution.requested_scope;
return default_result(resolution.requested_scope);
};
let Ok(peer_snapshots) = notification_system.scanner_dirty_usage_snapshots().await else {
return resolution.requested_scope;
return default_result(resolution.requested_scope);
};
let mut expected_peers = HashMap::new();
for (host, lease_instance_id, _) in crate::scanner::scanner_activity_publication_lease_targets(resolution.activity_before)
@@ -145,10 +154,10 @@ where
let Some((activity_instance_id, generation, pending)) =
crate::scanner::scanner_activity_dirty_usage_state_for_host(resolution.activity_before, &host)
else {
return resolution.requested_scope;
return default_result(resolution.requested_scope);
};
if activity_instance_id != lease_instance_id || expected_peers.contains_key(&host) {
return resolution.requested_scope;
return default_result(resolution.requested_scope);
}
expected_peers.insert(
host,
@@ -159,20 +168,77 @@ where
},
);
}
let Some(remote_dirty_buckets) = verified_remote_dirty_usage_buckets(&expected_peers, peer_snapshots) else {
return resolution.requested_scope;
let Some(remote_dirty_usage) = verified_remote_dirty_usage(&expected_peers, peer_snapshots) else {
return default_result(resolution.requested_scope);
};
dirty_buckets.extend(remote_dirty_usage.dirty_buckets);
let scope = scoped_scan_scope_from_dirty_buckets(
resolution.requested_scope,
dirty_buckets,
true,
resolution.all_buckets,
resolution.baseline_proof,
);
if scope.is_default() {
return default_result(scope);
}
let Some(selected_buckets) = scope.selected_buckets.as_ref() else {
return default_result(scope);
};
let mut scoped_acknowledgements = Vec::with_capacity(remote_dirty_usage.acknowledgements.len());
for acknowledgement in remote_dirty_usage.acknowledgements {
let crate::scanner::ScannerDirtyUsageAcknowledgement {
host,
instance_id,
kind: crate::scanner::ScannerDirtyUsageAcknowledgementKind::Scoped { owner_id, entries },
} = acknowledgement
else {
return default_result(scope);
};
let entries = entries
.into_iter()
.filter(|entry| selected_buckets.contains(&entry.bucket))
.collect::<Vec<_>>();
if !entries.is_empty() {
scoped_acknowledgements.push(crate::scanner::ScannerDirtyUsageAcknowledgement {
host,
instance_id,
kind: crate::scanner::ScannerDirtyUsageAcknowledgementKind::Scoped { owner_id, entries },
});
}
}
if super::scanner_scoped_dirty_usage_ack_exceeds_cost_threshold(&scoped_acknowledgements) {
return default_result(ScannerBucketScanScope::default());
}
if !scoped_acknowledgements.is_empty() {
let capability_acknowledgements = scoped_acknowledgements
.clone()
.into_iter()
.map(Into::into)
.collect::<Vec<crate::storage_api::EcstoreScannerDirtyUsageAcknowledgement>>();
if !matches!(
notification_system
.scanner_scoped_dirty_usage_capabilities(capability_acknowledgements)
.await,
Ok(true)
) {
return default_result(ScannerBucketScanScope::default());
}
}
return ScannerBucketScopeResolutionResult {
scope,
remote_dirty_usage_acknowledgements: scoped_acknowledgements,
};
dirty_buckets.extend(remote_dirty_buckets);
}
scoped_scan_scope_from_dirty_buckets(
default_result(scoped_scan_scope_from_dirty_buckets(
resolution.requested_scope,
dirty_buckets,
(!distributed).then_some(resolution.dirty_usage_snapshot.scopes.as_ref()),
true,
resolution.all_buckets,
resolution.baseline_proof,
)
))
}
pub(crate) async fn nsscanner_with_storage_status_scoped<S>(store: &S, request: ScannerCycleRequest) -> Result<ScannerCycleResult>
@@ -295,7 +361,7 @@ where
let bucket_coverage_digest = scanner_bucket_plan_digest(&all_buckets, activity_digest);
let execution_digest = scanner_bucket_work_digest(bucket_coverage_digest, scan_mode, requires_full_scan);
let dirty_usage_snapshot = Arc::new(snapshot_dirty_usage_buckets(&all_buckets, dirty_generation_before_bucket_list));
let scan_scope = resolve_scanner_bucket_scan_scope(
let scope_resolution = resolve_scanner_bucket_scan_scope(
store,
distributed,
ScannerBucketScopeResolution {
@@ -315,6 +381,8 @@ where
},
)
.await;
let remote_dirty_usage_acknowledgements = scope_resolution.remote_dirty_usage_acknowledgements;
let scan_scope = scope_resolution.scope;
#[cfg(test)]
if let Some(observer) = resolved_scope_observer {
let _ = observer.send(scan_scope.clone());
@@ -675,11 +743,14 @@ where
if cycle_status == ScannerCycleStatus::Complete {
complete_tier_registry_cycle(want_cycle, leader_epoch);
}
let remote_dirty_usage_acknowledgements = if cycle_status == ScannerCycleStatus::Complete {
crate::scanner::scanner_dirty_usage_acknowledgements(&activity_before)
} else {
Vec::new()
};
let remote_dirty_usage_acknowledgements =
if cycle_status == ScannerCycleStatus::Complete && !remote_dirty_usage_acknowledgements.is_empty() {
remote_dirty_usage_acknowledgements
} else if cycle_status == ScannerCycleStatus::Complete && scan_scope.is_default() {
crate::scanner::scanner_dirty_usage_acknowledgements(&activity_before)
} else {
Vec::new()
};
Ok(ScannerCycleResult::new(cycle_status, dirty_usage_clear)
.with_publication_epoch(publication_epoch)
.with_activity_digest(activity_digest)
+66 -4
View File
@@ -1657,6 +1657,7 @@ fn peer_dirty_usage_snapshot(
buckets: &[(&str, u64)],
) -> EcstoreScannerPeerDirtyUsageSnapshot {
EcstoreScannerPeerDirtyUsageSnapshot {
owner_id: uuid::Uuid::from_u128(0x11111111111111111111111111111111).to_string(),
instance_id: instance_id.to_string(),
generation,
pending_bucket_count: u64::try_from(buckets.len()).expect("test bucket count should fit"),
@@ -1664,7 +1665,15 @@ fn peer_dirty_usage_snapshot(
complete,
buckets: buckets
.iter()
.map(|(bucket, generation)| ((*bucket).to_string(), *generation))
.map(|(bucket, generation)| {
(
(*bucket).to_string(),
crate::storage_api::EcstoreScannerPeerDirtyUsageBucket {
bucket_incarnation: uuid::Uuid::from_u128(0x22222222222222222222222222222222),
generation: *generation,
},
)
})
.collect(),
}
}
@@ -1691,7 +1700,7 @@ fn verified_remote_dirty_usage_buckets_merges_only_complete_current_snapshots()
]);
assert_eq!(
verified_remote_dirty_usage_buckets(
verified_remote_dirty_usage(
&expected_peers,
vec![
(
@@ -1704,10 +1713,63 @@ fn verified_remote_dirty_usage_buckets_merges_only_complete_current_snapshots()
),
],
),
Some(HashSet::from(["photos".to_string(), "archive".to_string()]))
Some(VerifiedRemoteDirtyUsage {
dirty_buckets: HashSet::from(["photos".to_string(), "archive".to_string()]),
acknowledgements: vec![
crate::scanner::ScannerDirtyUsageAcknowledgement {
host: "node-a:9000".to_string(),
instance_id: "instance-a".to_string(),
kind: crate::scanner::ScannerDirtyUsageAcknowledgementKind::Scoped {
owner_id: uuid::Uuid::from_u128(0x11111111111111111111111111111111).to_string(),
entries: vec![crate::storage_api::EcstoreScannerScopedDirtyUsageAckEntry {
bucket: "photos".to_string(),
bucket_incarnation: uuid::Uuid::from_u128(0x22222222222222222222222222222222),
generation: 7,
}],
},
},
crate::scanner::ScannerDirtyUsageAcknowledgement {
host: "node-b:9000".to_string(),
instance_id: "instance-b".to_string(),
kind: crate::scanner::ScannerDirtyUsageAcknowledgementKind::Scoped {
owner_id: uuid::Uuid::from_u128(0x11111111111111111111111111111111).to_string(),
entries: vec![crate::storage_api::EcstoreScannerScopedDirtyUsageAckEntry {
bucket: "archive".to_string(),
bucket_incarnation: uuid::Uuid::from_u128(0x22222222222222222222222222222222),
generation: 3,
}],
},
},
],
})
);
}
#[test]
fn scanner_scoped_dirty_usage_ack_cost_threshold_is_single_protocol_batch() {
let acknowledgement = |entry_count: usize| crate::scanner::ScannerDirtyUsageAcknowledgement {
host: "node-a:9000".to_string(),
instance_id: "instance-a".to_string(),
kind: crate::scanner::ScannerDirtyUsageAcknowledgementKind::Scoped {
owner_id: uuid::Uuid::from_u128(0x11111111111111111111111111111111).to_string(),
entries: (0..entry_count)
.map(|index| crate::storage_api::EcstoreScannerScopedDirtyUsageAckEntry {
bucket: format!("bucket-{index:02}"),
bucket_incarnation: uuid::Uuid::from_u128(0x22222222222222222222222222222222),
generation: 7,
})
.collect(),
},
};
assert!(!scanner_scoped_dirty_usage_ack_exceeds_cost_threshold(&[acknowledgement(
crate::SCANNER_SCOPED_DIRTY_USAGE_ACK_MAX_ENTRIES
)]));
assert!(scanner_scoped_dirty_usage_ack_exceeds_cost_threshold(&[acknowledgement(
crate::SCANNER_SCOPED_DIRTY_USAGE_ACK_MAX_ENTRIES + 1
)]));
}
#[test]
fn verified_remote_dirty_usage_buckets_rejects_incomplete_or_stale_peer_state() {
let expected_peers = HashMap::from([(
@@ -1726,7 +1788,7 @@ fn verified_remote_dirty_usage_buckets_rejects_incomplete_or_stale_peer_state()
peer_dirty_usage_snapshot("instance-a", 7, true, &[]),
] {
assert!(
verified_remote_dirty_usage_buckets(&expected_peers, vec![("node-a:9000".to_string(), snapshot)]).is_none(),
verified_remote_dirty_usage(&expected_peers, vec![("node-a:9000".to_string(), snapshot)]).is_none(),
"incomplete, stale, mismatched, or empty pending peer state must fall back to a full scan"
);
}
+7 -1
View File
@@ -103,8 +103,13 @@ pub(crate) use rustfs_ecstore::api::rebalance::{
RebalStatus as EcstoreRebalStatus, RebalanceInfo as EcstoreRebalanceInfo, RebalanceMeta as EcstoreRebalanceMeta,
RebalanceStats as EcstoreRebalanceStats,
};
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::rpc::ScannerPeerDirtyUsageBucket as EcstoreScannerPeerDirtyUsageBucket;
pub(crate) use rustfs_ecstore::api::rpc::{
ScannerBucketListing as EcstoreScannerBucketListing, ScannerPeerDirtyUsageSnapshot as EcstoreScannerPeerDirtyUsageSnapshot,
ScannerBucketListing as EcstoreScannerBucketListing,
ScannerDirtyUsageAcknowledgement as EcstoreScannerDirtyUsageAcknowledgement,
ScannerPeerDirtyUsageSnapshot as EcstoreScannerPeerDirtyUsageSnapshot,
ScannerScopedDirtyUsageAckEntry as EcstoreScannerScopedDirtyUsageAckEntry,
};
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::runtime::InstanceContext as EcstoreInstanceContext;
@@ -315,6 +320,7 @@ pub(crate) mod scan {
pub use super::storage_contracts::{
SCANNER_ACTIVITY_PROTOCOL_VERSION, SCANNER_ACTIVITY_V6_PROTOCOL_VERSION, SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES,
SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION, SCANNER_DIRTY_USAGE_SNAPSHOT_RPC_MAX_MESSAGE_SIZE,
SCANNER_SCOPED_DIRTY_USAGE_ACK_MAX_ENTRIES,
};
}