mirror of
https://github.com/rustfs/rustfs.git
synced 2026-09-06 20:19:14 +00:00
Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 050ae3ed81 |
@@ -520,7 +520,8 @@ pub mod rpc {
|
||||
pub use crate::cluster::rpc::{
|
||||
AuthenticatedChannel, KMS_SIGNAL_SUBSYSTEM, LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS,
|
||||
PeerRestClient, PeerS3Client, S3PeerSys, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC,
|
||||
ScannerBucketListing, ScannerPeerActivity, ScannerPeerDirtyUsageSnapshot, ScannerPublicationLease, TONIC_RPC_PREFIX,
|
||||
ScannerBucketListing, ScannerDirtyUsageAcknowledgement, ScannerPeerActivity, ScannerPeerDirtyUsageBucket,
|
||||
ScannerPeerDirtyUsageSnapshot, ScannerPublicationLease, ScannerScopedDirtyUsageAckEntry, TONIC_RPC_PREFIX,
|
||||
TonicInterceptor, build_put_file_auth_trailer, check_and_record_signed_rpc_nonce, decode_heal_bucket_rpc_options,
|
||||
encode_heal_bucket_rpc_options, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
|
||||
gen_tonic_signature_interceptor, node_service_time_out_client, node_service_time_out_client_no_auth,
|
||||
|
||||
@@ -48,7 +48,8 @@ pub use internode_data_transport::build_internode_data_transport_from_env;
|
||||
pub(crate) use peer_rest_client::TierConfigReloadOutcome;
|
||||
pub use peer_rest_client::{
|
||||
KMS_SIGNAL_SUBSYSTEM, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, SERVICE_SIGNAL_REFRESH_CONFIG,
|
||||
SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerPeerActivity, ScannerPeerDirtyUsageSnapshot, ScannerPublicationLease,
|
||||
SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerDirtyUsageAcknowledgement, ScannerPeerActivity, ScannerPeerDirtyUsageBucket,
|
||||
ScannerPeerDirtyUsageSnapshot, ScannerPublicationLease, ScannerScopedDirtyUsageAckEntry,
|
||||
};
|
||||
pub(crate) use peer_s3_client::heal_bucket_local_on_disks;
|
||||
pub use peer_s3_client::{
|
||||
|
||||
@@ -49,10 +49,11 @@ use rustfs_protos::proto_gen::node_service::{
|
||||
LoadRebalanceMetaRequest, LoadServiceAccountRequest, LoadTransitionTierConfigRequest, LoadUserRequest,
|
||||
LocalStorageInfoRequest, Mss, ReloadPoolMetaRequest, ReloadSiteReplicationConfigRequest, ReplacementRecoveryStatusRequest,
|
||||
ScannerActivityRequest, ScannerActivityResponse, ScannerDirtyUsageSnapshotRequest, ScannerDirtyUsageSnapshotResponse,
|
||||
ScannerPublicationLeaseReleaseRequest, ScannerPublicationLeaseRequest, ScannerPublicationLeaseResponse, ServerInfoRequest,
|
||||
SignalServiceRequest, SignalServiceResponse, StartDecommissionRequest, StartProfilingRequest, StopRebalanceRequest,
|
||||
TierDailyStatsRequest, TierMutationAbortRequest, TierMutationCommitRequest, TierMutationControlResponse,
|
||||
TierMutationFailureClass, TierMutationPeerState, TierMutationPrepareRequest, node_service_client::NodeServiceClient,
|
||||
ScannerPublicationLeaseReleaseRequest, ScannerPublicationLeaseRequest, ScannerPublicationLeaseResponse,
|
||||
ScannerScopedDirtyUsageAckRequest, ScannerScopedDirtyUsageEntry, ServerInfoRequest, SignalServiceRequest,
|
||||
SignalServiceResponse, StartDecommissionRequest, StartProfilingRequest, StopRebalanceRequest, TierDailyStatsRequest,
|
||||
TierMutationAbortRequest, TierMutationCommitRequest, TierMutationControlResponse, TierMutationFailureClass,
|
||||
TierMutationPeerState, TierMutationPrepareRequest, node_service_client::NodeServiceClient,
|
||||
tier_mutation_control_service_client::TierMutationControlServiceClient,
|
||||
};
|
||||
pub use rustfs_protos::{PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS};
|
||||
@@ -92,6 +93,7 @@ const HEAL_CONTROL_PAYLOAD_MAX_SIZE: usize = 64 * 1024;
|
||||
const PEER_REST_RECOVERY_MAX_ATTEMPTS: u32 = 60;
|
||||
const PEER_REST_RECOVERY_MAX_BACKOFF: Duration = Duration::from_secs(30);
|
||||
const SCANNER_ACTIVITY_MAX_MESSAGE_SIZE: usize = 1024;
|
||||
const SCANNER_SCOPED_DIRTY_USAGE_STAGE_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
/// Reserve time for the acquire response's network/clock uncertainty. The
|
||||
/// server owns the real expiry; this local deadline is intentionally earlier
|
||||
/// so a coordinator never starts a bounded persistence operation at the edge
|
||||
@@ -192,12 +194,102 @@ pub struct ScannerPeerActivity {
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct ScannerPeerDirtyUsageSnapshot {
|
||||
pub owner_id: String,
|
||||
pub instance_id: String,
|
||||
pub generation: u64,
|
||||
pub pending_bucket_count: u64,
|
||||
pub protocol_version: u32,
|
||||
pub complete: bool,
|
||||
pub buckets: BTreeMap<String, u64>,
|
||||
pub buckets: BTreeMap<String, ScannerPeerDirtyUsageBucket>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct ScannerPeerDirtyUsageBucket {
|
||||
pub bucket_incarnation: Uuid,
|
||||
pub generation: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct ScannerScopedDirtyUsageAckEntry {
|
||||
pub bucket: String,
|
||||
pub bucket_incarnation: Uuid,
|
||||
pub generation: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub enum ScannerDirtyUsageAcknowledgement {
|
||||
Generation {
|
||||
host: String,
|
||||
instance_id: String,
|
||||
generation: u64,
|
||||
},
|
||||
Scoped {
|
||||
host: String,
|
||||
owner_id: String,
|
||||
instance_id: String,
|
||||
entries: Vec<ScannerScopedDirtyUsageAckEntry>,
|
||||
},
|
||||
}
|
||||
|
||||
fn scanner_scoped_dirty_usage_ack_payloads(
|
||||
owner_id: String,
|
||||
instance_id: String,
|
||||
probe_only: bool,
|
||||
entries: Vec<ScannerScopedDirtyUsageAckEntry>,
|
||||
) -> Result<Vec<ScannerScopedDirtyUsageAckRequest>> {
|
||||
use rustfs_protos::scoped_dirty_usage::*;
|
||||
|
||||
if entries.is_empty() {
|
||||
return Err(Error::other("scoped dirty usage acknowledgement entries must be nonempty"));
|
||||
}
|
||||
|
||||
let mut payloads = Vec::with_capacity(entries.len().div_ceil(SCOPED_DIRTY_USAGE_MAX_ENTRIES as usize));
|
||||
let mut batch = Vec::with_capacity(SCOPED_DIRTY_USAGE_MAX_ENTRIES as usize);
|
||||
for entry in entries {
|
||||
batch.push(ScannerScopedDirtyUsageEntry {
|
||||
bucket: entry.bucket,
|
||||
bucket_incarnation: entry.bucket_incarnation.as_bytes().to_vec().into(),
|
||||
generation: entry.generation,
|
||||
});
|
||||
if batch.len() == SCOPED_DIRTY_USAGE_MAX_ENTRIES as usize {
|
||||
payloads.push(scanner_scoped_dirty_usage_ack_payload(
|
||||
&owner_id,
|
||||
&instance_id,
|
||||
probe_only,
|
||||
std::mem::take(&mut batch),
|
||||
)?);
|
||||
}
|
||||
}
|
||||
if !batch.is_empty() {
|
||||
payloads.push(scanner_scoped_dirty_usage_ack_payload(&owner_id, &instance_id, probe_only, batch)?);
|
||||
}
|
||||
|
||||
Ok(payloads)
|
||||
}
|
||||
|
||||
fn scanner_scoped_dirty_usage_ack_payload(
|
||||
owner_id: &str,
|
||||
instance_id: &str,
|
||||
probe_only: bool,
|
||||
entries: Vec<ScannerScopedDirtyUsageEntry>,
|
||||
) -> Result<ScannerScopedDirtyUsageAckRequest> {
|
||||
use rustfs_protos::scoped_dirty_usage::*;
|
||||
|
||||
let payload = ScannerScopedDirtyUsageAckRequest {
|
||||
challenge: Uuid::new_v4().as_bytes().to_vec().into(),
|
||||
protocol_version: SCOPED_DIRTY_USAGE_PROTOCOL_VERSION,
|
||||
owner_id: owner_id.to_string(),
|
||||
instance_id: instance_id.to_string(),
|
||||
scope: SCOPED_DIRTY_USAGE_BUCKET_SCOPE,
|
||||
probe_only,
|
||||
entries,
|
||||
};
|
||||
canonical_scoped_dirty_usage_request(&payload).map_err(|err| Error::other(err.to_string()))?;
|
||||
Ok(payload)
|
||||
}
|
||||
|
||||
fn scanner_scoped_dirty_usage_ack_reconciled(activity: &ScannerPeerActivity, expected_instance_id: &str) -> bool {
|
||||
activity.instance_id == expected_instance_id && activity.dirty_usage_pending == Some(false)
|
||||
}
|
||||
|
||||
fn scanner_instance_id_is_valid(instance_id: &str) -> bool {
|
||||
@@ -351,6 +443,11 @@ fn decode_scanner_dirty_usage_snapshot_with_verifier(
|
||||
if !scanner_instance_id_is_valid(&response.instance_id) {
|
||||
return Err(Error::other("peer returned an invalid scanner dirty usage snapshot instance ID"));
|
||||
}
|
||||
let owner_id = Uuid::parse_str(&response.owner_id)
|
||||
.ok()
|
||||
.filter(|owner_id| !owner_id.is_nil())
|
||||
.map(|owner_id| owner_id.to_string())
|
||||
.ok_or_else(|| Error::other("peer returned an invalid scanner dirty usage snapshot owner"))?;
|
||||
if response.generation == u64::MAX {
|
||||
return Err(Error::other("peer scanner dirty usage snapshot exhausted its generation"));
|
||||
}
|
||||
@@ -386,9 +483,14 @@ fn decode_scanner_dirty_usage_snapshot_with_verifier(
|
||||
if bucket.generation == 0 || bucket.generation > response.generation {
|
||||
return Err(Error::other("peer scanner dirty usage snapshot contains an invalid bucket generation"));
|
||||
}
|
||||
Uuid::from_slice(bucket.bucket_incarnation.as_ref())
|
||||
.ok()
|
||||
.filter(|bucket_incarnation| !bucket_incarnation.is_nil())
|
||||
.ok_or_else(|| Error::other("peer scanner dirty usage snapshot contains an invalid bucket incarnation"))?;
|
||||
}
|
||||
|
||||
Ok(ScannerPeerDirtyUsageSnapshot {
|
||||
owner_id,
|
||||
instance_id: response.instance_id,
|
||||
generation: response.generation,
|
||||
pending_bucket_count: response.pending_bucket_count,
|
||||
@@ -397,7 +499,16 @@ fn decode_scanner_dirty_usage_snapshot_with_verifier(
|
||||
buckets: response
|
||||
.buckets
|
||||
.into_iter()
|
||||
.map(|bucket| (bucket.bucket, bucket.generation))
|
||||
.map(|bucket| {
|
||||
(
|
||||
bucket.bucket,
|
||||
ScannerPeerDirtyUsageBucket {
|
||||
bucket_incarnation: Uuid::from_slice(bucket.bucket_incarnation.as_ref())
|
||||
.expect("bucket incarnation was validated"),
|
||||
generation: bucket.generation,
|
||||
},
|
||||
)
|
||||
})
|
||||
.collect(),
|
||||
})
|
||||
}
|
||||
@@ -2077,19 +2188,10 @@ impl PeerRestClient {
|
||||
&self,
|
||||
owner_id: String,
|
||||
instance_id: String,
|
||||
entries: Vec<rustfs_protos::proto_gen::node_service::ScannerScopedDirtyUsageEntry>,
|
||||
entries: Vec<ScannerScopedDirtyUsageAckEntry>,
|
||||
) -> Result<bool> {
|
||||
use rustfs_protos::scoped_dirty_usage::*;
|
||||
let payload = rustfs_protos::proto_gen::node_service::ScannerScopedDirtyUsageAckRequest {
|
||||
challenge: Uuid::new_v4().as_bytes().to_vec().into(),
|
||||
protocol_version: SCOPED_DIRTY_USAGE_PROTOCOL_VERSION,
|
||||
owner_id,
|
||||
instance_id,
|
||||
scope: SCOPED_DIRTY_USAGE_BUCKET_SCOPE,
|
||||
probe_only: true,
|
||||
entries,
|
||||
};
|
||||
let canonical = canonical_scoped_dirty_usage_request(&payload).map_err(|err| Error::other(err.to_string()))?;
|
||||
let payloads = scanner_scoped_dirty_usage_ack_payloads(owner_id, instance_id, true, entries)?;
|
||||
self.finalize_result(
|
||||
async {
|
||||
let mut client = super::client::scanner_control_time_out_client(
|
||||
@@ -2097,26 +2199,106 @@ impl PeerRestClient {
|
||||
TonicInterceptor::Signature(gen_tonic_signature_interceptor()),
|
||||
)
|
||||
.await?;
|
||||
for payload in payloads {
|
||||
let canonical =
|
||||
canonical_scoped_dirty_usage_request(&payload).map_err(|err| Error::other(err.to_string()))?;
|
||||
let mut request = Request::new(payload.clone());
|
||||
set_tonic_canonical_body_digest(&mut request, &canonical)?;
|
||||
let response = client.scanner_scoped_dirty_usage_ack(request).await?.into_inner();
|
||||
let body = canonical_scoped_dirty_usage_response(&canonical, &response)
|
||||
.map_err(|_| Error::other("scoped dirty usage capability response is too large"))?;
|
||||
verify_tonic_rpc_response_proof(&body, response.response_proof.as_ref())?;
|
||||
if response.protocol_version != SCOPED_DIRTY_USAGE_PROTOCOL_VERSION
|
||||
|| response.owner_id != payload.owner_id
|
||||
|| response.instance_id != payload.instance_id
|
||||
|| response.max_entries != SCOPED_DIRTY_USAGE_MAX_ENTRIES
|
||||
|| response.max_request_bytes != SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES
|
||||
|| response.cleared != 0
|
||||
{
|
||||
return Err(Error::other("scoped dirty usage capability response does not match request"));
|
||||
}
|
||||
if !response.supported {
|
||||
return Ok(false);
|
||||
}
|
||||
}
|
||||
Ok(true)
|
||||
}
|
||||
.await,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn acknowledge_scanner_scoped_dirty_usage(
|
||||
&self,
|
||||
owner_id: String,
|
||||
instance_id: String,
|
||||
entries: Vec<ScannerScopedDirtyUsageAckEntry>,
|
||||
) -> Result<ScannerPeerActivity> {
|
||||
use rustfs_protos::scoped_dirty_usage::*;
|
||||
let payloads = scanner_scoped_dirty_usage_ack_payloads(owner_id, instance_id.clone(), false, entries)?;
|
||||
let ack_attempt = async {
|
||||
let mut client = super::client::scanner_control_time_out_client(
|
||||
&self.grid_host,
|
||||
TonicInterceptor::Signature(gen_tonic_signature_interceptor()),
|
||||
)
|
||||
.await?;
|
||||
for payload in payloads {
|
||||
let canonical = canonical_scoped_dirty_usage_request(&payload).map_err(|err| Error::other(err.to_string()))?;
|
||||
let mut request = Request::new(payload.clone());
|
||||
set_tonic_canonical_body_digest(&mut request, &canonical)?;
|
||||
let response = client.scanner_scoped_dirty_usage_ack(request).await?.into_inner();
|
||||
let body = canonical_scoped_dirty_usage_response(&canonical, &response)
|
||||
.map_err(|_| Error::other("scoped dirty usage capability response is too large"))?;
|
||||
.map_err(|_| Error::other("scoped dirty usage acknowledgement response is too large"))?;
|
||||
verify_tonic_rpc_response_proof(&body, response.response_proof.as_ref())?;
|
||||
if response.protocol_version != SCOPED_DIRTY_USAGE_PROTOCOL_VERSION
|
||||
|| response.owner_id != payload.owner_id
|
||||
|| response.instance_id != payload.instance_id
|
||||
|| response.max_entries != SCOPED_DIRTY_USAGE_MAX_ENTRIES
|
||||
|| response.max_request_bytes != SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES
|
||||
|| response.cleared != 0
|
||||
|| !response.supported
|
||||
{
|
||||
return Err(Error::other("scoped dirty usage capability response does not match request"));
|
||||
return Err(Error::other("scoped dirty usage acknowledgement response does not match request"));
|
||||
}
|
||||
Ok(response.supported)
|
||||
}
|
||||
.await,
|
||||
)
|
||||
.await
|
||||
Ok(())
|
||||
};
|
||||
let result = match timeout(SCANNER_SCOPED_DIRTY_USAGE_STAGE_TIMEOUT, ack_attempt).await {
|
||||
Ok(result) => self.finalize_result(result).await,
|
||||
Err(_) => {
|
||||
self.prepare_retry_with_timeout(SCANNER_SCOPED_DIRTY_USAGE_STAGE_TIMEOUT)
|
||||
.await;
|
||||
Err(Error::other("scoped dirty usage acknowledgement deadline elapsed"))
|
||||
}
|
||||
};
|
||||
|
||||
match result {
|
||||
Ok(()) => {
|
||||
let activity = self.scanner_scoped_dirty_usage_activity_confirmation().await?;
|
||||
if activity.instance_id == instance_id {
|
||||
Ok(activity)
|
||||
} else {
|
||||
Err(Error::other(
|
||||
"scoped dirty usage acknowledgement peer restarted before activity confirmation",
|
||||
))
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
if Self::is_network_like_error(&err) {
|
||||
self.prepare_retry_with_timeout(SCANNER_SCOPED_DIRTY_USAGE_STAGE_TIMEOUT)
|
||||
.await;
|
||||
}
|
||||
match self.scanner_scoped_dirty_usage_activity_confirmation().await {
|
||||
Ok(activity) if scanner_scoped_dirty_usage_ack_reconciled(&activity, &instance_id) => Ok(activity),
|
||||
_ => Err(err),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn scanner_scoped_dirty_usage_activity_confirmation(&self) -> Result<ScannerPeerActivity> {
|
||||
timeout(SCANNER_SCOPED_DIRTY_USAGE_STAGE_TIMEOUT, self.scanner_activity())
|
||||
.await
|
||||
.map_err(|_| Error::other("scoped dirty usage activity confirmation timed out"))?
|
||||
}
|
||||
|
||||
pub async fn acknowledge_scanner_dirty_usage(&self, instance_id: String, generation: u64) -> Result<ScannerPeerActivity> {
|
||||
@@ -2845,16 +3027,79 @@ mod tests {
|
||||
rustfs_protos::proto_gen::node_service::ScannerDirtyUsageBucket {
|
||||
bucket: "archive".to_string(),
|
||||
generation: 3,
|
||||
bucket_incarnation: Uuid::from_u128(0x11111111111111111111111111111111).as_bytes().to_vec().into(),
|
||||
},
|
||||
rustfs_protos::proto_gen::node_service::ScannerDirtyUsageBucket {
|
||||
bucket: "photos".to_string(),
|
||||
generation: 7,
|
||||
bucket_incarnation: Uuid::from_u128(0x22222222222222222222222222222222).as_bytes().to_vec().into(),
|
||||
},
|
||||
],
|
||||
response_proof: b"proof".to_vec().into(),
|
||||
owner_id: "33333333-3333-3333-3333-333333333333".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_scoped_dirty_usage_ack_payloads_split_at_protocol_limit() {
|
||||
use rustfs_protos::scoped_dirty_usage::{SCOPED_DIRTY_USAGE_MAX_ENTRIES, canonical_scoped_dirty_usage_request};
|
||||
|
||||
let entries = (0..=SCOPED_DIRTY_USAGE_MAX_ENTRIES)
|
||||
.map(|index| ScannerScopedDirtyUsageAckEntry {
|
||||
bucket: format!("bucket-{index:02}"),
|
||||
bucket_incarnation: Uuid::from_u128(0x11111111111111111111111111111111),
|
||||
generation: 9,
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
let payloads = scanner_scoped_dirty_usage_ack_payloads(
|
||||
"33333333-3333-3333-3333-333333333333".to_string(),
|
||||
"0123456789abcdef0123456789abcdef".to_string(),
|
||||
false,
|
||||
entries,
|
||||
)
|
||||
.expect("33 entries should split into valid scoped dirty usage requests");
|
||||
|
||||
assert_eq!(payloads.len(), 2);
|
||||
assert_eq!(payloads[0].entries.len(), SCOPED_DIRTY_USAGE_MAX_ENTRIES as usize);
|
||||
assert_eq!(payloads[1].entries.len(), 1);
|
||||
assert_eq!(payloads[0].entries.first().map(|entry| entry.bucket.as_str()), Some("bucket-00"));
|
||||
assert_eq!(payloads[0].entries.last().map(|entry| entry.bucket.as_str()), Some("bucket-31"));
|
||||
assert_eq!(payloads[1].entries.first().map(|entry| entry.bucket.as_str()), Some("bucket-32"));
|
||||
for payload in payloads {
|
||||
canonical_scoped_dirty_usage_request(&payload).expect("each split scoped ACK payload should be canonical");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_scoped_dirty_usage_ack_reconciliation_requires_same_clean_instance() {
|
||||
let activity = |instance_id: &str, pending| ScannerPeerActivity {
|
||||
instance_id: instance_id.to_string(),
|
||||
namespace_generation: 1,
|
||||
maintenance_generation: 1,
|
||||
protocol_version: SCANNER_ACTIVITY_PROTOCOL_VERSION,
|
||||
topology_digest: Some([1; 32]),
|
||||
data_movement_active: Some(false),
|
||||
dirty_usage_generation: Some(9),
|
||||
dirty_usage_pending: pending,
|
||||
movement_generation: Some(1),
|
||||
publication_blocked: Some(false),
|
||||
};
|
||||
|
||||
assert!(scanner_scoped_dirty_usage_ack_reconciled(
|
||||
&activity("0123456789abcdef0123456789abcdef", Some(false)),
|
||||
"0123456789abcdef0123456789abcdef"
|
||||
));
|
||||
assert!(!scanner_scoped_dirty_usage_ack_reconciled(
|
||||
&activity("0123456789abcdef0123456789abcdef", Some(true)),
|
||||
"0123456789abcdef0123456789abcdef"
|
||||
));
|
||||
assert!(!scanner_scoped_dirty_usage_ack_reconciled(
|
||||
&activity("fedcba9876543210fedcba9876543210", Some(false)),
|
||||
"0123456789abcdef0123456789abcdef"
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_dirty_usage_snapshot_requires_a_complete_authenticated_ordered_view() {
|
||||
let decoded = decode_test_scanner_dirty_usage_snapshot(test_scanner_dirty_usage_snapshot_response())
|
||||
@@ -2863,9 +3108,18 @@ mod tests {
|
||||
assert_eq!(decoded.generation, 7);
|
||||
assert_eq!(decoded.pending_bucket_count, 2);
|
||||
assert_eq!(decoded.protocol_version, SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION);
|
||||
assert_eq!(decoded.owner_id, "33333333-3333-3333-3333-333333333333");
|
||||
assert!(decoded.complete);
|
||||
assert_eq!(decoded.buckets.get("archive"), Some(&3));
|
||||
assert_eq!(decoded.buckets.get("photos"), Some(&7));
|
||||
assert_eq!(
|
||||
decoded.buckets.get("archive").map(|bucket| bucket.bucket_incarnation),
|
||||
Some(Uuid::from_u128(0x11111111111111111111111111111111))
|
||||
);
|
||||
assert_eq!(decoded.buckets.get("archive").map(|bucket| bucket.generation), Some(3));
|
||||
assert_eq!(
|
||||
decoded.buckets.get("photos").map(|bucket| bucket.bucket_incarnation),
|
||||
Some(Uuid::from_u128(0x22222222222222222222222222222222))
|
||||
);
|
||||
assert_eq!(decoded.buckets.get("photos").map(|bucket| bucket.generation), Some(7));
|
||||
|
||||
let overflow_count =
|
||||
u64::try_from(SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES + 1).expect("the test snapshot entry limit should fit in u64");
|
||||
@@ -2916,6 +3170,14 @@ mod tests {
|
||||
empty_bucket.buckets[0].bucket.clear();
|
||||
cases.push((empty_bucket, "empty bucket name"));
|
||||
|
||||
let mut invalid_owner = test_scanner_dirty_usage_snapshot_response();
|
||||
invalid_owner.owner_id.clear();
|
||||
cases.push((invalid_owner, "snapshot owner"));
|
||||
|
||||
let mut invalid_incarnation = test_scanner_dirty_usage_snapshot_response();
|
||||
invalid_incarnation.buckets[0].bucket_incarnation = Uuid::nil().as_bytes().to_vec().into();
|
||||
cases.push((invalid_incarnation, "bucket incarnation"));
|
||||
|
||||
let mut partial = test_scanner_dirty_usage_snapshot_response();
|
||||
partial.complete = false;
|
||||
cases.push((partial, "entry-limit overflow"));
|
||||
@@ -2928,6 +3190,7 @@ mod tests {
|
||||
.map(|index| rustfs_protos::proto_gen::node_service::ScannerDirtyUsageBucket {
|
||||
bucket: format!("bucket-{index:04}"),
|
||||
generation: 1,
|
||||
bucket_incarnation: Uuid::from_u128(0x11111111111111111111111111111111).as_bytes().to_vec().into(),
|
||||
})
|
||||
.collect(),
|
||||
..test_scanner_dirty_usage_snapshot_response()
|
||||
|
||||
@@ -14,7 +14,8 @@
|
||||
|
||||
use crate::bucket::lifecycle::tier_last_day_stats::DailyAllTierStats;
|
||||
use crate::cluster::rpc::{
|
||||
PeerRestClient, ScannerPeerActivity, ScannerPeerDirtyUsageSnapshot, ScannerPublicationLease, TierConfigReloadOutcome,
|
||||
PeerRestClient, ScannerDirtyUsageAcknowledgement, ScannerPeerActivity, ScannerPeerDirtyUsageSnapshot,
|
||||
ScannerPublicationLease, TierConfigReloadOutcome,
|
||||
};
|
||||
use crate::diagnostics::admin_server_info::get_commit_id;
|
||||
use crate::disk::DiskAPI;
|
||||
@@ -2552,11 +2553,70 @@ impl NotificationSys {
|
||||
Ok(snapshots)
|
||||
}
|
||||
|
||||
pub async fn acknowledge_scanner_dirty_usage(&self, acknowledgements: Vec<(String, String, u64)>) -> Result<bool> {
|
||||
pub async fn scanner_scoped_dirty_usage_capabilities(
|
||||
&self,
|
||||
acknowledgements: Vec<ScannerDirtyUsageAcknowledgement>,
|
||||
) -> Result<bool> {
|
||||
let mut by_host = HashMap::with_capacity(acknowledgements.len());
|
||||
for (host, instance_id, generation) in acknowledgements {
|
||||
if by_host.insert(host.clone(), (instance_id, generation)).is_some() {
|
||||
return Err(Error::other(format!("duplicate scanner dirty usage acknowledgement target: {host}")));
|
||||
for acknowledgement in acknowledgements {
|
||||
let host = match &acknowledgement {
|
||||
ScannerDirtyUsageAcknowledgement::Scoped { host, .. } => host.clone(),
|
||||
ScannerDirtyUsageAcknowledgement::Generation { .. } => {
|
||||
return Err(Error::other("scanner scoped dirty usage capability requires scoped acknowledgements"));
|
||||
}
|
||||
};
|
||||
if by_host.insert(host.clone(), acknowledgement).is_some() {
|
||||
return Err(Error::other("duplicate scanner dirty usage acknowledgement target"));
|
||||
}
|
||||
}
|
||||
|
||||
let clients = self
|
||||
.peer_clients
|
||||
.iter()
|
||||
.flatten()
|
||||
.map(|client| (client.grid_host.clone(), client.clone()))
|
||||
.collect::<HashMap<_, _>>();
|
||||
let mut futures = Vec::with_capacity(by_host.len());
|
||||
for (host, acknowledgement) in by_host {
|
||||
let Some(client) = clients.get(&host).cloned() else {
|
||||
return Err(Error::other("scanner scoped dirty usage capability failed: peer is not reachable"));
|
||||
};
|
||||
futures.push(async move {
|
||||
let ScannerDirtyUsageAcknowledgement::Scoped {
|
||||
owner_id,
|
||||
instance_id,
|
||||
entries,
|
||||
..
|
||||
} = acknowledgement
|
||||
else {
|
||||
unreachable!("scoped acknowledgement was validated before probing");
|
||||
};
|
||||
timeout(
|
||||
SCANNER_ACTIVITY_PROBE_TIMEOUT,
|
||||
client.scanner_scoped_dirty_usage_capability(owner_id, instance_id, entries),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| Error::other("scanner scoped dirty usage capability timed out"))?
|
||||
});
|
||||
}
|
||||
|
||||
for result in join_all(futures).await {
|
||||
if !result? {
|
||||
return Ok(false);
|
||||
}
|
||||
}
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
pub async fn acknowledge_scanner_dirty_usage(&self, acknowledgements: Vec<ScannerDirtyUsageAcknowledgement>) -> Result<bool> {
|
||||
let mut by_host = HashMap::with_capacity(acknowledgements.len());
|
||||
for acknowledgement in acknowledgements {
|
||||
let host = match &acknowledgement {
|
||||
ScannerDirtyUsageAcknowledgement::Generation { host, .. }
|
||||
| ScannerDirtyUsageAcknowledgement::Scoped { host, .. } => host.clone(),
|
||||
};
|
||||
if by_host.insert(host.clone(), acknowledgement).is_some() {
|
||||
return Err(Error::other("duplicate scanner dirty usage acknowledgement target"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2568,18 +2628,34 @@ impl NotificationSys {
|
||||
.collect::<HashMap<_, _>>();
|
||||
let mut failures = Vec::new();
|
||||
let mut futures = Vec::with_capacity(by_host.len());
|
||||
for (host, (instance_id, generation)) in by_host {
|
||||
for (host, acknowledgement) in by_host {
|
||||
let Some(client) = clients.get(&host).cloned() else {
|
||||
failures.push(format!("peer {host} scanner dirty usage acknowledgement failed: peer is not reachable"));
|
||||
continue;
|
||||
};
|
||||
futures.push(async move {
|
||||
let result = scanner_activity_with_timeout(
|
||||
SCANNER_ACTIVITY_PROBE_TIMEOUT,
|
||||
&host,
|
||||
client.acknowledge_scanner_dirty_usage(instance_id, generation),
|
||||
)
|
||||
.await;
|
||||
let result = match acknowledgement {
|
||||
ScannerDirtyUsageAcknowledgement::Generation {
|
||||
instance_id, generation, ..
|
||||
} => {
|
||||
scanner_activity_with_timeout(
|
||||
SCANNER_ACTIVITY_PROBE_TIMEOUT,
|
||||
&host,
|
||||
client.acknowledge_scanner_dirty_usage(instance_id, generation),
|
||||
)
|
||||
.await
|
||||
}
|
||||
ScannerDirtyUsageAcknowledgement::Scoped {
|
||||
owner_id,
|
||||
instance_id,
|
||||
entries,
|
||||
..
|
||||
} => {
|
||||
client
|
||||
.acknowledge_scanner_scoped_dirty_usage(owner_id, instance_id, entries)
|
||||
.await
|
||||
}
|
||||
};
|
||||
(host, result)
|
||||
});
|
||||
}
|
||||
@@ -4744,15 +4820,28 @@ mod tests {
|
||||
peer_topology_hosts: Vec::new(),
|
||||
};
|
||||
let missing = sys
|
||||
.acknowledge_scanner_dirty_usage(vec![("peer-1".to_string(), "0123456789abcdef0123456789abcdef".to_string(), 7)])
|
||||
.acknowledge_scanner_dirty_usage(vec![ScannerDirtyUsageAcknowledgement::Generation {
|
||||
host: "peer-1".to_string(),
|
||||
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
|
||||
generation: 7,
|
||||
}])
|
||||
.await
|
||||
.expect_err("a missing acknowledgement target must remain pending");
|
||||
assert!(missing.to_string().contains("peer is not reachable"));
|
||||
|
||||
let duplicate = sys
|
||||
.acknowledge_scanner_dirty_usage(vec![
|
||||
("peer-1".to_string(), "0123456789abcdef0123456789abcdef".to_string(), 7),
|
||||
("peer-1".to_string(), "0123456789abcdef0123456789abcdef".to_string(), 7),
|
||||
ScannerDirtyUsageAcknowledgement::Generation {
|
||||
host: "peer-1".to_string(),
|
||||
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
|
||||
generation: 7,
|
||||
},
|
||||
ScannerDirtyUsageAcknowledgement::Scoped {
|
||||
host: "peer-1".to_string(),
|
||||
owner_id: "11111111-1111-1111-1111-111111111111".to_string(),
|
||||
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
|
||||
entries: Vec::new(),
|
||||
},
|
||||
])
|
||||
.await
|
||||
.expect_err("duplicate acknowledgement targets must be rejected");
|
||||
|
||||
@@ -1257,6 +1257,8 @@ pub struct ScannerDirtyUsageBucket {
|
||||
pub bucket: ::prost::alloc::string::String,
|
||||
#[prost(uint64, tag = "2")]
|
||||
pub generation: u64,
|
||||
#[prost(bytes = "bytes", tag = "3")]
|
||||
pub bucket_incarnation: ::prost::bytes::Bytes,
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct ScannerDirtyUsageSnapshotRequest {
|
||||
@@ -1282,6 +1284,8 @@ pub struct ScannerDirtyUsageSnapshotResponse {
|
||||
pub buckets: ::prost::alloc::vec::Vec<ScannerDirtyUsageBucket>,
|
||||
#[prost(bytes = "bytes", tag = "7")]
|
||||
pub response_proof: ::prost::bytes::Bytes,
|
||||
#[prost(string, tag = "8")]
|
||||
pub owner_id: ::prost::alloc::string::String,
|
||||
}
|
||||
/// Receiver-only protocol. Producers must retain whole-cycle ACK until they
|
||||
/// have a durable per-bucket publication proof.
|
||||
|
||||
@@ -575,11 +575,13 @@ pub fn canonical_scanner_dirty_usage_snapshot_response_body(
|
||||
body.push_u64(response.generation);
|
||||
body.push_u64(response.pending_bucket_count);
|
||||
body.push_u32(response.protocol_version);
|
||||
body.push_str(&response.owner_id)?;
|
||||
body.push_bool(response.complete);
|
||||
body.push_count(response.buckets.len())?;
|
||||
for bucket in &response.buckets {
|
||||
body.push_str(&bucket.bucket)?;
|
||||
body.push_u64(bucket.generation);
|
||||
body.push_bytes(bucket.bucket_incarnation.as_ref())?;
|
||||
}
|
||||
Ok(body.finish())
|
||||
}
|
||||
@@ -1842,13 +1844,16 @@ mod scanner_activity_tests {
|
||||
ScannerDirtyUsageBucket {
|
||||
bucket: "archive".to_string(),
|
||||
generation: 3,
|
||||
bucket_incarnation: vec![1; 16].into(),
|
||||
},
|
||||
ScannerDirtyUsageBucket {
|
||||
bucket: "photos".to_string(),
|
||||
generation: 7,
|
||||
bucket_incarnation: vec![2; 16].into(),
|
||||
},
|
||||
],
|
||||
response_proof: vec![9; 32].into(),
|
||||
owner_id: "11111111-1111-1111-1111-111111111111".to_string(),
|
||||
};
|
||||
let baseline = canonical_scanner_dirty_usage_snapshot_response_body(&[1; 16], &response)
|
||||
.expect("scanner dirty usage snapshot response should encode");
|
||||
@@ -1865,6 +1870,9 @@ mod scanner_activity_tests {
|
||||
let mut protocol = response.clone();
|
||||
protocol.protocol_version = 2;
|
||||
variants.push(protocol);
|
||||
let mut owner = response.clone();
|
||||
owner.owner_id = "22222222-2222-2222-2222-222222222222".to_string();
|
||||
variants.push(owner);
|
||||
let mut complete = response.clone();
|
||||
complete.complete = false;
|
||||
variants.push(complete);
|
||||
@@ -1874,6 +1882,9 @@ mod scanner_activity_tests {
|
||||
let mut bucket_generation = response.clone();
|
||||
bucket_generation.buckets[0].generation = 4;
|
||||
variants.push(bucket_generation);
|
||||
let mut bucket_incarnation = response.clone();
|
||||
bucket_incarnation.buckets[0].bucket_incarnation = vec![3; 16].into();
|
||||
variants.push(bucket_incarnation);
|
||||
let mut bucket_order = response.clone();
|
||||
bucket_order.buckets.reverse();
|
||||
variants.push(bucket_order);
|
||||
|
||||
@@ -885,6 +885,7 @@ message ScannerActivityResponse {
|
||||
message ScannerDirtyUsageBucket {
|
||||
string bucket = 1;
|
||||
uint64 generation = 2;
|
||||
bytes bucket_incarnation = 3;
|
||||
}
|
||||
|
||||
message ScannerDirtyUsageSnapshotRequest {
|
||||
@@ -901,6 +902,7 @@ message ScannerDirtyUsageSnapshotResponse {
|
||||
bool complete = 5;
|
||||
repeated ScannerDirtyUsageBucket buckets = 6;
|
||||
bytes response_proof = 7;
|
||||
string owner_id = 8;
|
||||
}
|
||||
|
||||
// Receiver-only protocol. Producers must retain whole-cycle ACK until they
|
||||
|
||||
@@ -100,6 +100,7 @@ pub use storage_api::ScannerReplicationConfig as ReplicationConfig;
|
||||
pub use storage_api::scan::{
|
||||
SCANNER_ACTIVITY_PROTOCOL_VERSION, SCANNER_ACTIVITY_V6_PROTOCOL_VERSION, SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES,
|
||||
SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION, SCANNER_DIRTY_USAGE_SNAPSHOT_RPC_MAX_MESSAGE_SIZE,
|
||||
SCANNER_SCOPED_DIRTY_USAGE_ACK_MAX_ENTRIES,
|
||||
};
|
||||
pub use workload_admission::set_scanner_workload_admission_snapshot_provider;
|
||||
|
||||
|
||||
@@ -2216,10 +2216,7 @@ where
|
||||
false
|
||||
} else if let Some(notification_system) = storeapi.scanner_notification_system() {
|
||||
let acknowledgement_count = remote_dirty_usage_acknowledgements.len();
|
||||
let acknowledgements = remote_dirty_usage_acknowledgements
|
||||
.into_iter()
|
||||
.map(|acknowledgement| (acknowledgement.host, acknowledgement.instance_id, acknowledgement.generation))
|
||||
.collect();
|
||||
let acknowledgements = remote_dirty_usage_acknowledgements.into_iter().map(Into::into).collect();
|
||||
remote_dirty_usage_acknowledgement_pending(
|
||||
cycle_info.current,
|
||||
acknowledgement_count,
|
||||
@@ -3578,9 +3575,10 @@ use usage_store::*;
|
||||
|
||||
pub use activity::scanner_topology_digest;
|
||||
pub(crate) use activity::{
|
||||
ScannerActivitySnapshot, ScannerDirtyUsageAcknowledgement, probe_scanner_activity, scanner_activity_allows_usage_publication,
|
||||
scanner_activity_dirty_usage_state_for_host, scanner_activity_publication_lease_targets, scanner_activity_snapshot_digest,
|
||||
scanner_activity_structural_digest, scanner_dirty_usage_acknowledgements,
|
||||
ScannerActivitySnapshot, ScannerDirtyUsageAcknowledgement, ScannerDirtyUsageAcknowledgementKind, probe_scanner_activity,
|
||||
scanner_activity_allows_usage_publication, scanner_activity_dirty_usage_state_for_host,
|
||||
scanner_activity_publication_lease_targets, scanner_activity_snapshot_digest, scanner_activity_structural_digest,
|
||||
scanner_dirty_usage_acknowledgements,
|
||||
};
|
||||
pub(crate) use activity::{ScannerCycleOutcome, scanner_cycle_outcome_with_pending_maintenance};
|
||||
pub use backlog::{
|
||||
|
||||
@@ -445,7 +445,34 @@ pub(crate) type ScannerActivitySnapshot = BTreeMap<String, ScannerNodeActivity>;
|
||||
pub(crate) struct ScannerDirtyUsageAcknowledgement {
|
||||
pub(crate) host: String,
|
||||
pub(crate) instance_id: String,
|
||||
pub(crate) generation: u64,
|
||||
pub(crate) kind: ScannerDirtyUsageAcknowledgementKind,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum ScannerDirtyUsageAcknowledgementKind {
|
||||
Generation(u64),
|
||||
Scoped {
|
||||
owner_id: String,
|
||||
entries: Vec<crate::storage_api::EcstoreScannerScopedDirtyUsageAckEntry>,
|
||||
},
|
||||
}
|
||||
|
||||
impl From<ScannerDirtyUsageAcknowledgement> for crate::storage_api::EcstoreScannerDirtyUsageAcknowledgement {
|
||||
fn from(acknowledgement: ScannerDirtyUsageAcknowledgement) -> Self {
|
||||
match acknowledgement.kind {
|
||||
ScannerDirtyUsageAcknowledgementKind::Generation(generation) => Self::Generation {
|
||||
host: acknowledgement.host,
|
||||
instance_id: acknowledgement.instance_id,
|
||||
generation,
|
||||
},
|
||||
ScannerDirtyUsageAcknowledgementKind::Scoped { owner_id, entries } => Self::Scoped {
|
||||
host: acknowledgement.host,
|
||||
owner_id,
|
||||
instance_id: acknowledgement.instance_id,
|
||||
entries,
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
@@ -974,7 +1001,7 @@ pub(crate) fn scanner_dirty_usage_acknowledgements(snapshot: &ScannerActivitySna
|
||||
.map(|(host, activity)| ScannerDirtyUsageAcknowledgement {
|
||||
host: host.clone(),
|
||||
instance_id: activity.instance_id.clone(),
|
||||
generation: activity.dirty_usage_generation,
|
||||
kind: ScannerDirtyUsageAcknowledgementKind::Generation(activity.dirty_usage_generation),
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
@@ -7433,7 +7433,7 @@ fn finalizing_a_saved_enum_without_proof_keeps_dirty_pending() {
|
||||
let remote_acknowledgement = ScannerDirtyUsageAcknowledgement {
|
||||
host: "node-2".to_string(),
|
||||
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
|
||||
generation: 11,
|
||||
kind: ScannerDirtyUsageAcknowledgementKind::Generation(11),
|
||||
};
|
||||
let unsaved = crate::scanner_io::ScannerCycleResult::new(ScannerCycleStatus::Complete, Some(dirty_snapshot.clone()))
|
||||
.with_remote_dirty_usage_acknowledgements(vec![remote_acknowledgement.clone()]);
|
||||
@@ -8982,7 +8982,7 @@ fn post_lease_activity_proof_rejects_a_put_tail_that_finished_before_lease_acqui
|
||||
ScannerDirtyUsageAcknowledgement {
|
||||
host: "node-2".to_string(),
|
||||
instance_id: "epoch-a".to_string(),
|
||||
generation: 5,
|
||||
kind: ScannerDirtyUsageAcknowledgementKind::Generation(5),
|
||||
},
|
||||
]);
|
||||
let (outcome, _, acknowledgements) = finalize_scanner_cycle_result(
|
||||
@@ -9142,7 +9142,7 @@ fn scanner_dirty_usage_acknowledgements_exclude_local_and_clean_nodes() {
|
||||
vec![ScannerDirtyUsageAcknowledgement {
|
||||
host: "node-3".to_string(),
|
||||
instance_id: "epoch-dirty".to_string(),
|
||||
generation: 11,
|
||||
kind: ScannerDirtyUsageAcknowledgementKind::Generation(11),
|
||||
}]
|
||||
);
|
||||
}
|
||||
|
||||
@@ -463,7 +463,7 @@ async fn scoped_ack_publication_rejects_builder_mutation_after_real_root_publish
|
||||
"remote_ack_target" => scan.with_remote_dirty_usage_acknowledgements(vec![ScannerDirtyUsageAcknowledgement {
|
||||
host: "proof-peer:9000".to_string(),
|
||||
instance_id: crate::scanner_activity_epoch().to_string(),
|
||||
generation: changed_generation,
|
||||
kind: crate::scanner::ScannerDirtyUsageAcknowledgementKind::Generation(changed_generation),
|
||||
}]),
|
||||
"publication_epoch" => scan.with_publication_epoch(Some(changed_epoch)),
|
||||
"remote_lease_targets" => scan.with_remote_publication_lease_targets(vec![(
|
||||
|
||||
@@ -148,19 +148,26 @@ struct ScannerPeerDirtyUsageExpectation {
|
||||
pending: bool,
|
||||
}
|
||||
|
||||
fn verified_remote_dirty_usage_buckets(
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
struct VerifiedRemoteDirtyUsage {
|
||||
dirty_buckets: HashSet<String>,
|
||||
acknowledgements: Vec<crate::scanner::ScannerDirtyUsageAcknowledgement>,
|
||||
}
|
||||
|
||||
fn verified_remote_dirty_usage(
|
||||
expected_peers: &HashMap<String, ScannerPeerDirtyUsageExpectation>,
|
||||
peer_snapshots: Vec<(String, EcstoreScannerPeerDirtyUsageSnapshot)>,
|
||||
) -> Option<HashSet<String>> {
|
||||
) -> Option<VerifiedRemoteDirtyUsage> {
|
||||
if expected_peers.is_empty() || peer_snapshots.len() != expected_peers.len() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut received_peers = HashSet::with_capacity(peer_snapshots.len());
|
||||
let mut dirty_buckets = HashSet::new();
|
||||
let mut acknowledgements = Vec::new();
|
||||
for (host, snapshot) in peer_snapshots {
|
||||
let expected = expected_peers.get(&host)?;
|
||||
if !received_peers.insert(host)
|
||||
if !received_peers.insert(host.clone())
|
||||
|| snapshot.instance_id != expected.instance_id
|
||||
|| snapshot.generation != expected.generation
|
||||
|| snapshot.generation == u64::MAX
|
||||
@@ -171,10 +178,44 @@ fn verified_remote_dirty_usage_buckets(
|
||||
{
|
||||
return None;
|
||||
}
|
||||
dirty_buckets.extend(snapshot.buckets.into_keys());
|
||||
let entries = snapshot
|
||||
.buckets
|
||||
.iter()
|
||||
.map(|(bucket, state)| crate::storage_api::EcstoreScannerScopedDirtyUsageAckEntry {
|
||||
bucket: bucket.clone(),
|
||||
bucket_incarnation: state.bucket_incarnation,
|
||||
generation: state.generation,
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
dirty_buckets.extend(snapshot.buckets.keys().cloned());
|
||||
if !entries.is_empty() {
|
||||
acknowledgements.push(crate::scanner::ScannerDirtyUsageAcknowledgement {
|
||||
host,
|
||||
instance_id: snapshot.instance_id,
|
||||
kind: crate::scanner::ScannerDirtyUsageAcknowledgementKind::Scoped {
|
||||
owner_id: snapshot.owner_id,
|
||||
entries,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
(received_peers.len() == expected_peers.len()).then_some(dirty_buckets)
|
||||
(received_peers.len() == expected_peers.len()).then_some(VerifiedRemoteDirtyUsage {
|
||||
dirty_buckets,
|
||||
acknowledgements,
|
||||
})
|
||||
}
|
||||
|
||||
fn scanner_scoped_dirty_usage_ack_exceeds_cost_threshold(
|
||||
acknowledgements: &[crate::scanner::ScannerDirtyUsageAcknowledgement],
|
||||
) -> bool {
|
||||
acknowledgements.iter().any(|acknowledgement| {
|
||||
matches!(
|
||||
&acknowledgement.kind,
|
||||
crate::scanner::ScannerDirtyUsageAcknowledgementKind::Scoped { entries, .. }
|
||||
if entries.len() > crate::SCANNER_SCOPED_DIRTY_USAGE_ACK_MAX_ENTRIES
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn complete_scanner_cache_snapshot_plan_digest(
|
||||
|
||||
@@ -107,23 +107,32 @@ struct ScannerBucketScopeResolution<'a> {
|
||||
requires_full_scan: bool,
|
||||
}
|
||||
|
||||
struct ScannerBucketScopeResolutionResult {
|
||||
scope: ScannerBucketScanScope,
|
||||
remote_dirty_usage_acknowledgements: Vec<crate::scanner::ScannerDirtyUsageAcknowledgement>,
|
||||
}
|
||||
|
||||
async fn resolve_scanner_bucket_scan_scope<S>(
|
||||
store: &S,
|
||||
distributed: bool,
|
||||
resolution: ScannerBucketScopeResolution<'_>,
|
||||
) -> ScannerBucketScanScope
|
||||
) -> ScannerBucketScopeResolutionResult
|
||||
where
|
||||
S: ScannerStorage,
|
||||
{
|
||||
let default_result = |scope: ScannerBucketScanScope| ScannerBucketScopeResolutionResult {
|
||||
scope,
|
||||
remote_dirty_usage_acknowledgements: Vec::new(),
|
||||
};
|
||||
if resolution.requires_full_scan {
|
||||
return ScannerBucketScanScope::default();
|
||||
return default_result(ScannerBucketScanScope::default());
|
||||
}
|
||||
if !resolution.requested_scope.is_default()
|
||||
|| !resolution.dirty_usage_snapshot.covers_all_pending
|
||||
|| resolution.dirty_usage_snapshot.generation == u64::MAX
|
||||
|| resolution.dirty_usage_snapshot.buckets.len() > crate::SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES
|
||||
{
|
||||
return resolution.requested_scope;
|
||||
return default_result(resolution.requested_scope);
|
||||
}
|
||||
|
||||
let mut dirty_buckets = resolution
|
||||
@@ -134,10 +143,10 @@ where
|
||||
.collect::<HashSet<_>>();
|
||||
if distributed {
|
||||
let Some(notification_system) = store.scanner_notification_system() else {
|
||||
return resolution.requested_scope;
|
||||
return default_result(resolution.requested_scope);
|
||||
};
|
||||
let Ok(peer_snapshots) = notification_system.scanner_dirty_usage_snapshots().await else {
|
||||
return resolution.requested_scope;
|
||||
return default_result(resolution.requested_scope);
|
||||
};
|
||||
let mut expected_peers = HashMap::new();
|
||||
for (host, lease_instance_id, _) in crate::scanner::scanner_activity_publication_lease_targets(resolution.activity_before)
|
||||
@@ -145,10 +154,10 @@ where
|
||||
let Some((activity_instance_id, generation, pending)) =
|
||||
crate::scanner::scanner_activity_dirty_usage_state_for_host(resolution.activity_before, &host)
|
||||
else {
|
||||
return resolution.requested_scope;
|
||||
return default_result(resolution.requested_scope);
|
||||
};
|
||||
if activity_instance_id != lease_instance_id || expected_peers.contains_key(&host) {
|
||||
return resolution.requested_scope;
|
||||
return default_result(resolution.requested_scope);
|
||||
}
|
||||
expected_peers.insert(
|
||||
host,
|
||||
@@ -159,19 +168,76 @@ where
|
||||
},
|
||||
);
|
||||
}
|
||||
let Some(remote_dirty_buckets) = verified_remote_dirty_usage_buckets(&expected_peers, peer_snapshots) else {
|
||||
return resolution.requested_scope;
|
||||
let Some(remote_dirty_usage) = verified_remote_dirty_usage(&expected_peers, peer_snapshots) else {
|
||||
return default_result(resolution.requested_scope);
|
||||
};
|
||||
dirty_buckets.extend(remote_dirty_usage.dirty_buckets);
|
||||
let scope = scoped_scan_scope_from_dirty_buckets(
|
||||
resolution.requested_scope,
|
||||
dirty_buckets,
|
||||
true,
|
||||
resolution.all_buckets,
|
||||
resolution.baseline_proof,
|
||||
);
|
||||
if scope.is_default() {
|
||||
return default_result(scope);
|
||||
}
|
||||
let Some(selected_buckets) = scope.selected_buckets.as_ref() else {
|
||||
return default_result(scope);
|
||||
};
|
||||
let mut scoped_acknowledgements = Vec::with_capacity(remote_dirty_usage.acknowledgements.len());
|
||||
for acknowledgement in remote_dirty_usage.acknowledgements {
|
||||
let crate::scanner::ScannerDirtyUsageAcknowledgement {
|
||||
host,
|
||||
instance_id,
|
||||
kind: crate::scanner::ScannerDirtyUsageAcknowledgementKind::Scoped { owner_id, entries },
|
||||
} = acknowledgement
|
||||
else {
|
||||
return default_result(scope);
|
||||
};
|
||||
let entries = entries
|
||||
.into_iter()
|
||||
.filter(|entry| selected_buckets.contains(&entry.bucket))
|
||||
.collect::<Vec<_>>();
|
||||
if !entries.is_empty() {
|
||||
scoped_acknowledgements.push(crate::scanner::ScannerDirtyUsageAcknowledgement {
|
||||
host,
|
||||
instance_id,
|
||||
kind: crate::scanner::ScannerDirtyUsageAcknowledgementKind::Scoped { owner_id, entries },
|
||||
});
|
||||
}
|
||||
}
|
||||
if super::scanner_scoped_dirty_usage_ack_exceeds_cost_threshold(&scoped_acknowledgements) {
|
||||
return default_result(ScannerBucketScanScope::default());
|
||||
}
|
||||
if !scoped_acknowledgements.is_empty() {
|
||||
let capability_acknowledgements = scoped_acknowledgements
|
||||
.clone()
|
||||
.into_iter()
|
||||
.map(Into::into)
|
||||
.collect::<Vec<crate::storage_api::EcstoreScannerDirtyUsageAcknowledgement>>();
|
||||
if !matches!(
|
||||
notification_system
|
||||
.scanner_scoped_dirty_usage_capabilities(capability_acknowledgements)
|
||||
.await,
|
||||
Ok(true)
|
||||
) {
|
||||
return default_result(ScannerBucketScanScope::default());
|
||||
}
|
||||
}
|
||||
return ScannerBucketScopeResolutionResult {
|
||||
scope,
|
||||
remote_dirty_usage_acknowledgements: scoped_acknowledgements,
|
||||
};
|
||||
dirty_buckets.extend(remote_dirty_buckets);
|
||||
}
|
||||
|
||||
scoped_scan_scope_from_dirty_buckets(
|
||||
default_result(scoped_scan_scope_from_dirty_buckets(
|
||||
resolution.requested_scope,
|
||||
dirty_buckets,
|
||||
true,
|
||||
resolution.all_buckets,
|
||||
resolution.baseline_proof,
|
||||
)
|
||||
))
|
||||
}
|
||||
|
||||
pub(crate) async fn nsscanner_with_storage_status_scoped<S>(store: &S, request: ScannerCycleRequest) -> Result<ScannerCycleResult>
|
||||
@@ -294,7 +360,7 @@ where
|
||||
let bucket_coverage_digest = scanner_bucket_plan_digest(&all_buckets, activity_digest);
|
||||
let execution_digest = scanner_bucket_work_digest(bucket_coverage_digest, scan_mode, requires_full_scan);
|
||||
let dirty_usage_snapshot = Arc::new(snapshot_dirty_usage_buckets(&all_buckets, dirty_generation_before_bucket_list));
|
||||
let scan_scope = resolve_scanner_bucket_scan_scope(
|
||||
let scope_resolution = resolve_scanner_bucket_scan_scope(
|
||||
store,
|
||||
distributed,
|
||||
ScannerBucketScopeResolution {
|
||||
@@ -314,6 +380,8 @@ where
|
||||
},
|
||||
)
|
||||
.await;
|
||||
let remote_dirty_usage_acknowledgements = scope_resolution.remote_dirty_usage_acknowledgements;
|
||||
let scan_scope = scope_resolution.scope;
|
||||
#[cfg(test)]
|
||||
if let Some(observer) = resolved_scope_observer {
|
||||
let _ = observer.send(scan_scope.clone());
|
||||
@@ -674,11 +742,14 @@ where
|
||||
if cycle_status == ScannerCycleStatus::Complete {
|
||||
complete_tier_registry_cycle(want_cycle, leader_epoch);
|
||||
}
|
||||
let remote_dirty_usage_acknowledgements = if cycle_status == ScannerCycleStatus::Complete {
|
||||
crate::scanner::scanner_dirty_usage_acknowledgements(&activity_before)
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
let remote_dirty_usage_acknowledgements =
|
||||
if cycle_status == ScannerCycleStatus::Complete && !remote_dirty_usage_acknowledgements.is_empty() {
|
||||
remote_dirty_usage_acknowledgements
|
||||
} else if cycle_status == ScannerCycleStatus::Complete && scan_scope.is_default() {
|
||||
crate::scanner::scanner_dirty_usage_acknowledgements(&activity_before)
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
Ok(ScannerCycleResult::new(cycle_status, dirty_usage_clear)
|
||||
.with_publication_epoch(publication_epoch)
|
||||
.with_activity_digest(activity_digest)
|
||||
|
||||
@@ -1561,6 +1561,7 @@ fn peer_dirty_usage_snapshot(
|
||||
buckets: &[(&str, u64)],
|
||||
) -> EcstoreScannerPeerDirtyUsageSnapshot {
|
||||
EcstoreScannerPeerDirtyUsageSnapshot {
|
||||
owner_id: uuid::Uuid::from_u128(0x11111111111111111111111111111111).to_string(),
|
||||
instance_id: instance_id.to_string(),
|
||||
generation,
|
||||
pending_bucket_count: u64::try_from(buckets.len()).expect("test bucket count should fit"),
|
||||
@@ -1568,7 +1569,15 @@ fn peer_dirty_usage_snapshot(
|
||||
complete,
|
||||
buckets: buckets
|
||||
.iter()
|
||||
.map(|(bucket, generation)| ((*bucket).to_string(), *generation))
|
||||
.map(|(bucket, generation)| {
|
||||
(
|
||||
(*bucket).to_string(),
|
||||
crate::storage_api::EcstoreScannerPeerDirtyUsageBucket {
|
||||
bucket_incarnation: uuid::Uuid::from_u128(0x22222222222222222222222222222222),
|
||||
generation: *generation,
|
||||
},
|
||||
)
|
||||
})
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
@@ -1595,7 +1604,7 @@ fn verified_remote_dirty_usage_buckets_merges_only_complete_current_snapshots()
|
||||
]);
|
||||
|
||||
assert_eq!(
|
||||
verified_remote_dirty_usage_buckets(
|
||||
verified_remote_dirty_usage(
|
||||
&expected_peers,
|
||||
vec![
|
||||
(
|
||||
@@ -1608,10 +1617,63 @@ fn verified_remote_dirty_usage_buckets_merges_only_complete_current_snapshots()
|
||||
),
|
||||
],
|
||||
),
|
||||
Some(HashSet::from(["photos".to_string(), "archive".to_string()]))
|
||||
Some(VerifiedRemoteDirtyUsage {
|
||||
dirty_buckets: HashSet::from(["photos".to_string(), "archive".to_string()]),
|
||||
acknowledgements: vec![
|
||||
crate::scanner::ScannerDirtyUsageAcknowledgement {
|
||||
host: "node-a:9000".to_string(),
|
||||
instance_id: "instance-a".to_string(),
|
||||
kind: crate::scanner::ScannerDirtyUsageAcknowledgementKind::Scoped {
|
||||
owner_id: uuid::Uuid::from_u128(0x11111111111111111111111111111111).to_string(),
|
||||
entries: vec![crate::storage_api::EcstoreScannerScopedDirtyUsageAckEntry {
|
||||
bucket: "photos".to_string(),
|
||||
bucket_incarnation: uuid::Uuid::from_u128(0x22222222222222222222222222222222),
|
||||
generation: 7,
|
||||
}],
|
||||
},
|
||||
},
|
||||
crate::scanner::ScannerDirtyUsageAcknowledgement {
|
||||
host: "node-b:9000".to_string(),
|
||||
instance_id: "instance-b".to_string(),
|
||||
kind: crate::scanner::ScannerDirtyUsageAcknowledgementKind::Scoped {
|
||||
owner_id: uuid::Uuid::from_u128(0x11111111111111111111111111111111).to_string(),
|
||||
entries: vec![crate::storage_api::EcstoreScannerScopedDirtyUsageAckEntry {
|
||||
bucket: "archive".to_string(),
|
||||
bucket_incarnation: uuid::Uuid::from_u128(0x22222222222222222222222222222222),
|
||||
generation: 3,
|
||||
}],
|
||||
},
|
||||
},
|
||||
],
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_scoped_dirty_usage_ack_cost_threshold_is_single_protocol_batch() {
|
||||
let acknowledgement = |entry_count: usize| crate::scanner::ScannerDirtyUsageAcknowledgement {
|
||||
host: "node-a:9000".to_string(),
|
||||
instance_id: "instance-a".to_string(),
|
||||
kind: crate::scanner::ScannerDirtyUsageAcknowledgementKind::Scoped {
|
||||
owner_id: uuid::Uuid::from_u128(0x11111111111111111111111111111111).to_string(),
|
||||
entries: (0..entry_count)
|
||||
.map(|index| crate::storage_api::EcstoreScannerScopedDirtyUsageAckEntry {
|
||||
bucket: format!("bucket-{index:02}"),
|
||||
bucket_incarnation: uuid::Uuid::from_u128(0x22222222222222222222222222222222),
|
||||
generation: 7,
|
||||
})
|
||||
.collect(),
|
||||
},
|
||||
};
|
||||
|
||||
assert!(!scanner_scoped_dirty_usage_ack_exceeds_cost_threshold(&[acknowledgement(
|
||||
crate::SCANNER_SCOPED_DIRTY_USAGE_ACK_MAX_ENTRIES
|
||||
)]));
|
||||
assert!(scanner_scoped_dirty_usage_ack_exceeds_cost_threshold(&[acknowledgement(
|
||||
crate::SCANNER_SCOPED_DIRTY_USAGE_ACK_MAX_ENTRIES + 1
|
||||
)]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn verified_remote_dirty_usage_buckets_rejects_incomplete_or_stale_peer_state() {
|
||||
let expected_peers = HashMap::from([(
|
||||
@@ -1630,7 +1692,7 @@ fn verified_remote_dirty_usage_buckets_rejects_incomplete_or_stale_peer_state()
|
||||
peer_dirty_usage_snapshot("instance-a", 7, true, &[]),
|
||||
] {
|
||||
assert!(
|
||||
verified_remote_dirty_usage_buckets(&expected_peers, vec![("node-a:9000".to_string(), snapshot)]).is_none(),
|
||||
verified_remote_dirty_usage(&expected_peers, vec![("node-a:9000".to_string(), snapshot)]).is_none(),
|
||||
"incomplete, stale, mismatched, or empty pending peer state must fall back to a full scan"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -103,8 +103,13 @@ pub(crate) use rustfs_ecstore::api::rebalance::{
|
||||
RebalStatus as EcstoreRebalStatus, RebalanceInfo as EcstoreRebalanceInfo, RebalanceMeta as EcstoreRebalanceMeta,
|
||||
RebalanceStats as EcstoreRebalanceStats,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::rpc::ScannerPeerDirtyUsageBucket as EcstoreScannerPeerDirtyUsageBucket;
|
||||
pub(crate) use rustfs_ecstore::api::rpc::{
|
||||
ScannerBucketListing as EcstoreScannerBucketListing, ScannerPeerDirtyUsageSnapshot as EcstoreScannerPeerDirtyUsageSnapshot,
|
||||
ScannerBucketListing as EcstoreScannerBucketListing,
|
||||
ScannerDirtyUsageAcknowledgement as EcstoreScannerDirtyUsageAcknowledgement,
|
||||
ScannerPeerDirtyUsageSnapshot as EcstoreScannerPeerDirtyUsageSnapshot,
|
||||
ScannerScopedDirtyUsageAckEntry as EcstoreScannerScopedDirtyUsageAckEntry,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::runtime::InstanceContext as EcstoreInstanceContext;
|
||||
@@ -315,6 +320,7 @@ pub(crate) mod scan {
|
||||
pub use super::storage_contracts::{
|
||||
SCANNER_ACTIVITY_PROTOCOL_VERSION, SCANNER_ACTIVITY_V6_PROTOCOL_VERSION, SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES,
|
||||
SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION, SCANNER_DIRTY_USAGE_SNAPSHOT_RPC_MAX_MESSAGE_SIZE,
|
||||
SCANNER_SCOPED_DIRTY_USAGE_ACK_MAX_ENTRIES,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -56,8 +56,9 @@ pub const SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION: u32 = 5;
|
||||
/// state is not authenticated by that version.
|
||||
pub const SCANNER_ACTIVITY_V6_PROTOCOL_VERSION: u32 = 6;
|
||||
pub const SCANNER_ACTIVITY_PROTOCOL_VERSION: u32 = 7;
|
||||
pub const SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION: u32 = 1;
|
||||
pub const SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES: usize = 4096;
|
||||
pub const SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION: u32 = 2;
|
||||
pub const SCANNER_SCOPED_DIRTY_USAGE_ACK_MAX_ENTRIES: usize = 32;
|
||||
pub const SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES: usize = SCANNER_SCOPED_DIRTY_USAGE_ACK_MAX_ENTRIES;
|
||||
pub const SCANNER_DIRTY_USAGE_SNAPSHOT_RPC_MAX_MESSAGE_SIZE: usize = 512 * 1024;
|
||||
|
||||
#[derive(Debug, serde::Deserialize, serde::Serialize)]
|
||||
|
||||
@@ -2418,6 +2418,7 @@ fn get_default_tcp_keepalive() -> TcpKeepalive {
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::server::compress::RequestPathCategory;
|
||||
use crate::storage_api::server::http::ScannerScopedDirtyUsageAckEntry;
|
||||
use bytes::Bytes;
|
||||
use http::Request as HttpRequest;
|
||||
use http::{HeaderMap, StatusCode};
|
||||
@@ -3454,9 +3455,9 @@ mod tests {
|
||||
.scanner_scoped_dirty_usage_capability(
|
||||
"11111111-1111-1111-1111-111111111111".to_string(),
|
||||
"a".repeat(32),
|
||||
vec![rustfs_protos::proto_gen::node_service::ScannerScopedDirtyUsageEntry {
|
||||
bucket: "photos".into(),
|
||||
bucket_incarnation: vec![1; 16].into(),
|
||||
vec![ScannerScopedDirtyUsageAckEntry {
|
||||
bucket: "photos".to_string(),
|
||||
bucket_incarnation: uuid::Uuid::from_u128(0x11111111111111111111111111111111),
|
||||
generation: 8,
|
||||
}],
|
||||
)
|
||||
|
||||
@@ -303,25 +303,38 @@ fn scanner_activity_response(
|
||||
}
|
||||
}
|
||||
|
||||
fn scanner_dirty_usage_snapshot_response(
|
||||
async fn scanner_dirty_usage_snapshot_response(
|
||||
store: &ECStore,
|
||||
snapshot: rustfs_scanner::ScannerDirtyUsageSnapshot,
|
||||
) -> ScannerDirtyUsageSnapshotResponse {
|
||||
ScannerDirtyUsageSnapshotResponse {
|
||||
) -> Result<ScannerDirtyUsageSnapshotResponse, Status> {
|
||||
if store.id.is_nil() {
|
||||
return Err(Status::failed_precondition("scanner dirty usage snapshot owner is unavailable"));
|
||||
}
|
||||
let mut buckets = Vec::with_capacity(snapshot.buckets.len());
|
||||
for bucket in snapshot.buckets {
|
||||
let bucket_incarnation = store
|
||||
.bucket_incarnation_id_from_disk(&bucket.bucket)
|
||||
.await
|
||||
.map_err(|_| Status::failed_precondition("scanner dirty usage bucket incarnation is unavailable"))?;
|
||||
if bucket_incarnation.is_nil() {
|
||||
return Err(Status::failed_precondition("scanner dirty usage bucket incarnation is unavailable"));
|
||||
}
|
||||
buckets.push(ScannerDirtyUsageBucket {
|
||||
bucket: bucket.bucket,
|
||||
generation: bucket.generation,
|
||||
bucket_incarnation: bucket_incarnation.as_bytes().to_vec().into(),
|
||||
});
|
||||
}
|
||||
Ok(ScannerDirtyUsageSnapshotResponse {
|
||||
instance_id: rustfs_scanner::scanner_activity_epoch().to_string(),
|
||||
generation: snapshot.generation,
|
||||
pending_bucket_count: snapshot.pending_bucket_count,
|
||||
protocol_version: rustfs_scanner::SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION,
|
||||
complete: snapshot.complete,
|
||||
buckets: snapshot
|
||||
.buckets
|
||||
.into_iter()
|
||||
.map(|bucket| ScannerDirtyUsageBucket {
|
||||
bucket: bucket.bucket,
|
||||
generation: bucket.generation,
|
||||
})
|
||||
.collect(),
|
||||
buckets,
|
||||
response_proof: Bytes::new(),
|
||||
}
|
||||
owner_id: store.id.to_string(),
|
||||
})
|
||||
}
|
||||
|
||||
fn scanner_activity_response_v7(
|
||||
@@ -2219,11 +2232,14 @@ impl Node for NodeService {
|
||||
.as_ref()
|
||||
.try_into()
|
||||
.map_err(|_| Status::invalid_argument("scanner dirty usage snapshot challenge must be 16 bytes"))?;
|
||||
let snapshot = rustfs_scanner::scanner_dirty_usage_snapshot(rustfs_scanner::SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES);
|
||||
let store = self
|
||||
.resolve_object_store()
|
||||
.ok_or_else(|| Status::unavailable("storage layer is not initialized"))?;
|
||||
let snapshot = rustfs_scanner::scanner_dirty_usage_snapshot(rustfs_scanner::SCANNER_SCOPED_DIRTY_USAGE_ACK_MAX_ENTRIES);
|
||||
if snapshot.generation == u64::MAX {
|
||||
return Err(Status::resource_exhausted("scanner dirty usage generation is exhausted"));
|
||||
}
|
||||
let mut response = scanner_dirty_usage_snapshot_response(snapshot);
|
||||
let mut response = scanner_dirty_usage_snapshot_response(&store, snapshot).await?;
|
||||
let canonical = rustfs_protos::canonical_scanner_dirty_usage_snapshot_response_body(&challenge, &response)
|
||||
.map_err(|_| Status::internal("scanner dirty usage snapshot response is too large to authenticate"))?;
|
||||
response.response_proof = sign_tonic_rpc_response_proof(&canonical)
|
||||
@@ -6510,7 +6526,28 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn test_scanner_dirty_usage_snapshot_requires_body_bound_auth_and_signs_a_consistent_view() {
|
||||
let _ = rustfs_credentials::set_global_rpc_secret("scanner-dirty-usage-snapshot-test-secret".to_string());
|
||||
let service = create_test_node_service();
|
||||
let _ = rustfs_credentials::init_global_action_credentials(
|
||||
Some("TESTROOTACCESSKEY".to_string()),
|
||||
Some("TESTROOTSECRET123".to_string()),
|
||||
);
|
||||
let temp_dir = tempfile::tempdir().expect("scanner dirty usage snapshot RPC test directory");
|
||||
let env = rustfs_test_utils::TestECStoreEnv::builder()
|
||||
.base_dir(temp_dir.path())
|
||||
.build()
|
||||
.await;
|
||||
ObjectStore::new(Arc::clone(&env.ecstore))
|
||||
.save_iam_config(serde_json::json!({"version": 1}), format!("{}/format.json", *IAM_CONFIG_PREFIX))
|
||||
.await
|
||||
.expect("seed IAM format");
|
||||
let iam = rustfs_iam::build_iam_sys(Arc::clone(&env.ecstore))
|
||||
.await
|
||||
.expect("build isolated IAM");
|
||||
let context = Arc::new(crate::runtime_sources::AppContext::with_default_interfaces(
|
||||
Arc::clone(&env.ecstore),
|
||||
iam,
|
||||
Arc::new(KmsServiceManager::new()),
|
||||
));
|
||||
let service = make_server_for_context(Some(context));
|
||||
let unsigned = service
|
||||
.scanner_dirty_usage_snapshot(Request::new(ScannerDirtyUsageSnapshotRequest {
|
||||
challenge: vec![7; 16].into(),
|
||||
@@ -6564,6 +6601,7 @@ mod tests {
|
||||
.into_inner();
|
||||
assert_eq!(response.instance_id, rustfs_scanner::scanner_activity_epoch());
|
||||
assert_eq!(response.protocol_version, rustfs_scanner::SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION);
|
||||
assert!(Uuid::parse_str(&response.owner_id).is_ok_and(|owner_id| !owner_id.is_nil()));
|
||||
let bucket_count = u64::try_from(response.buckets.len()).expect("snapshot bucket count should fit in u64");
|
||||
assert_eq!(response.complete, response.pending_bucket_count == bucket_count);
|
||||
let canonical = rustfs_protos::canonical_scanner_dirty_usage_snapshot_response_body(&challenge, &response)
|
||||
|
||||
@@ -550,8 +550,8 @@ pub(crate) mod ecstore_rpc {
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::rpc::{
|
||||
build_put_file_auth_trailer, gen_signature_headers, gen_tonic_signature_headers, set_tonic_canonical_body_digest,
|
||||
verify_put_file_capability, verify_tonic_rpc_response_proof,
|
||||
ScannerScopedDirtyUsageAckEntry, build_put_file_auth_trailer, gen_signature_headers, gen_tonic_signature_headers,
|
||||
set_tonic_canonical_body_digest, verify_put_file_capability, verify_tonic_rpc_response_proof,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -694,6 +694,8 @@ pub(crate) type ServerContextSlot = crate::storage::runtime_sources::ServerConte
|
||||
pub(crate) type LocalPeerS3Client = ecstore_rpc::LocalPeerS3Client;
|
||||
#[cfg(test)]
|
||||
pub(crate) type PeerRestClient = ecstore_rpc::PeerRestClient;
|
||||
#[cfg(test)]
|
||||
pub(crate) type ScannerScopedDirtyUsageAckEntry = ecstore_rpc::ScannerScopedDirtyUsageAckEntry;
|
||||
pub(crate) type MetricType = ecstore_metrics::MetricType;
|
||||
pub(crate) type ObjectPartInfo = rustfs_filemeta::ObjectPartInfo;
|
||||
pub(crate) type ObjectLockBlockReason = ecstore_bucket::object_lock::objectlock_sys::ObjectLockBlockReason;
|
||||
|
||||
@@ -147,8 +147,8 @@ pub(crate) mod server {
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) use crate::storage::storage_api::{
|
||||
Endpoint, EndpointServerPools, Endpoints, PeerRestClient, PoolEndpoints, gen_signature_headers,
|
||||
gen_tonic_signature_headers,
|
||||
Endpoint, EndpointServerPools, Endpoints, PeerRestClient, PoolEndpoints, ScannerScopedDirtyUsageAckEntry,
|
||||
gen_signature_headers, gen_tonic_signature_headers,
|
||||
};
|
||||
|
||||
pub(crate) mod ecfs {
|
||||
|
||||
@@ -45,7 +45,7 @@
|
||||
1|crates/ecstore/src/object_api/types.rs
|
||||
3|crates/ecstore/src/runtime/sources.rs
|
||||
4|crates/ecstore/src/services/batch_processor.rs
|
||||
14|crates/ecstore/src/services/notification_sys.rs
|
||||
13|crates/ecstore/src/services/notification_sys.rs
|
||||
16|crates/ecstore/src/services/rebalance/control.rs
|
||||
1|crates/ecstore/src/services/rebalance/entry.rs
|
||||
8|crates/ecstore/src/services/rebalance/meta.rs
|
||||
|
||||
Reference in New Issue
Block a user