mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-24 21:26:28 +00:00
fix(scanner): fence movement generation publication (#6461)
* feat(scanner): add movement generation fencing * fix(scanner): prioritize unverified cycle deferral * feat(ecstore): add scanner publication lease fence * feat(rpc): add scanner publication lease protocol * feat(scanner): hold remote leases through usage publish * test(scanner): cover publication lease fencing * fix(scanner): fence remote leases across restart and delay * feat(rpc): fence scanner publication rename writes * fix(scanner): fence observed cleanup deletes * fix(proto): qualify lease release test types * fix(scanner): pin movement notifications * fix(scanner): clean publication imports * fix(ecstore): satisfy scanner fence clippy * refactor(scanner): group wait and publication options * fix(scanner): satisfy final lint and facade guards * fix(rpc): resolve facade export conflicts * fix(ci): remove unused decommission and healing facades * fix(ci): cfg-gate test-only usage overlay import * fix(scanner): wake on remote scanner restart
This commit is contained in:
@@ -415,9 +415,9 @@ pub mod notification {
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub use crate::services::notification_sys::rotate_cross_pool_fence_fleet_proof_for_test;
|
||||
pub use crate::services::notification_sys::{
|
||||
CrossPoolFenceFleetProofToken, NotificationPeerErr, NotificationSys, acquire_cross_pool_fence_fleet_proof,
|
||||
cross_pool_fence_fleet_proof_matches, get_global_notification_sys, new_global_notification_sys,
|
||||
start_remote_version_state_fleet_probe,
|
||||
CrossPoolFenceFleetProofToken, NotificationPeerErr, NotificationSys, ScannerPublicationLeaseGrant,
|
||||
acquire_cross_pool_fence_fleet_proof, cross_pool_fence_fleet_proof_matches, get_global_notification_sys,
|
||||
new_global_notification_sys, start_remote_version_state_fleet_probe,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -426,9 +426,10 @@ pub mod object {
|
||||
BLOCK_SIZE_V2, ERASURE_ALGORITHM, EncryptionResolutionError, EncryptionResolutionErrorKind, GetObjectBodyCacheHook,
|
||||
GetObjectBodyCacheHookLookup, GetObjectBodySource, GetObjectReader, NamespaceLockFence, ObjectEncryptionResolver,
|
||||
ObjectInfo, ObjectLockConfigSnapshot, ObjectMutationHook, ObjectOptions, PutObjReader, QuotaAdmission,
|
||||
RangedDecompressReader, ReadEncryptionMaterial, ReadEncryptionMode, ReadEncryptionRequest, StreamConsumer,
|
||||
get_object_body_cache_plaintext_len, lookup_get_object_body_cache_hook, register_get_object_body_cache_hook,
|
||||
register_object_mutation_hook, unregister_get_object_body_cache_hook, unregister_object_mutation_hook,
|
||||
RangedDecompressReader, ReadEncryptionMaterial, ReadEncryptionMode, ReadEncryptionRequest,
|
||||
SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY, StreamConsumer, get_object_body_cache_plaintext_len,
|
||||
lookup_get_object_body_cache_hook, register_get_object_body_cache_hook, register_object_mutation_hook,
|
||||
unregister_get_object_body_cache_hook, unregister_object_mutation_hook,
|
||||
};
|
||||
pub use crate::store::{
|
||||
PrepareSelectObjectSnapshotError, PreparedGetObjectReader, SelectObjectSnapshot, SelectObjectSnapshotReadError,
|
||||
@@ -461,16 +462,17 @@ pub mod rpc {
|
||||
pub use crate::cluster::rpc::{
|
||||
AuthenticatedChannel, KMS_SIGNAL_SUBSYSTEM, LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS,
|
||||
PeerRestClient, PeerS3Client, S3PeerSys, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC,
|
||||
ScannerBucketListing, ScannerPeerActivity, TONIC_RPC_PREFIX, TonicInterceptor, build_put_file_auth_trailer,
|
||||
check_and_record_signed_rpc_nonce, decode_heal_bucket_rpc_options, encode_heal_bucket_rpc_options, gen_signature_headers,
|
||||
gen_tonic_replay_scope_headers, gen_tonic_signature_headers, gen_tonic_signature_interceptor,
|
||||
node_service_time_out_client, node_service_time_out_client_no_auth, normalize_tonic_rpc_audience,
|
||||
set_tonic_canonical_body_digest, sign_ns_scanner_capability, sign_ns_scanner_capability_with_tier_registry_generation,
|
||||
sign_put_file_capability, sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers,
|
||||
tonic_rpc_auth_failure_reason, verify_ns_scanner_capability, verify_ns_scanner_capability_with_tier_registry_generation,
|
||||
verify_put_file_auth_trailer, verify_put_file_capability, verify_rpc_signature, verify_tonic_boot_epoch_response,
|
||||
verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof,
|
||||
verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
|
||||
ScannerBucketListing, ScannerPeerActivity, ScannerPublicationLease, TONIC_RPC_PREFIX, TonicInterceptor,
|
||||
build_put_file_auth_trailer, check_and_record_signed_rpc_nonce, decode_heal_bucket_rpc_options,
|
||||
encode_heal_bucket_rpc_options, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
|
||||
gen_tonic_signature_interceptor, node_service_time_out_client, node_service_time_out_client_no_auth,
|
||||
normalize_tonic_rpc_audience, set_tonic_canonical_body_digest, sign_ns_scanner_capability,
|
||||
sign_ns_scanner_capability_with_tier_registry_generation, sign_put_file_capability, sign_tonic_rpc_response_proof,
|
||||
tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason,
|
||||
verify_ns_scanner_capability, verify_ns_scanner_capability_with_tier_registry_generation, verify_put_file_auth_trailer,
|
||||
verify_put_file_capability, verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
|
||||
verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
|
||||
verify_tonic_rpc_signature_with_bootstrap,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -497,7 +499,7 @@ pub mod storage {
|
||||
pub use crate::core::pools::HealLifecycleExpiryContext;
|
||||
pub use crate::store::HealWalkVersion;
|
||||
pub use crate::store::{
|
||||
ECStore, all_local_disk, all_local_disk_path, find_local_disk_by_ref, init_local_disks,
|
||||
ECStore, SCANNER_PUBLICATION_LEASE_TTL_MS, all_local_disk, all_local_disk_path, find_local_disk_by_ref, init_local_disks,
|
||||
init_local_disks_with_instance_ctx, init_lock_clients, prewarm_local_disk_id_map,
|
||||
prewarm_local_disk_id_map_with_instance_ctx,
|
||||
};
|
||||
|
||||
@@ -2994,6 +2994,7 @@ mod tests {
|
||||
dst_volume: "bucket".to_string(),
|
||||
dst_path: "object".to_string(),
|
||||
file_info_bin: vec![0x81, 0xA1, 0x76, 0x01].into(),
|
||||
scanner_publication_lease_token: Vec::new().into(),
|
||||
};
|
||||
let body = rustfs_protos::canonical_rename_data_request_body(&message).expect("small request should encode");
|
||||
let mut request = tonic::Request::new(());
|
||||
|
||||
@@ -48,7 +48,7 @@ pub use internode_data_transport::build_internode_data_transport_from_env;
|
||||
pub(crate) use peer_rest_client::TierConfigReloadOutcome;
|
||||
pub use peer_rest_client::{
|
||||
KMS_SIGNAL_SUBSYSTEM, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, SERVICE_SIGNAL_REFRESH_CONFIG,
|
||||
SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerPeerActivity,
|
||||
SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerPeerActivity, ScannerPublicationLease,
|
||||
};
|
||||
pub(crate) use peer_s3_client::heal_bucket_local_on_disks;
|
||||
pub use peer_s3_client::{
|
||||
|
||||
@@ -20,6 +20,7 @@ use crate::cluster::rpc::{set_tonic_canonical_body_digest, set_tonic_mutation_bo
|
||||
use crate::error::{Error, Result};
|
||||
use crate::storage_api_contracts::internode::{
|
||||
SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION, SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SCANNER_ACTIVITY_PROTOCOL_VERSION,
|
||||
SCANNER_ACTIVITY_V6_PROTOCOL_VERSION,
|
||||
};
|
||||
use crate::{
|
||||
bucket::replication::BucketStats,
|
||||
@@ -45,8 +46,9 @@ use rustfs_protos::proto_gen::node_service::{
|
||||
HealControlRequest, LoadBucketMetadataRequest, LoadGroupRequest, LoadPolicyMappingRequest, LoadPolicyRequest,
|
||||
LoadRebalanceMetaRequest, LoadServiceAccountRequest, LoadTransitionTierConfigRequest, LoadUserRequest,
|
||||
LocalStorageInfoRequest, Mss, ReloadPoolMetaRequest, ReloadSiteReplicationConfigRequest, ReplacementRecoveryStatusRequest,
|
||||
ScannerActivityRequest, ScannerActivityResponse, ServerInfoRequest, SignalServiceRequest, SignalServiceResponse,
|
||||
StartDecommissionRequest, StartProfilingRequest, StopRebalanceRequest, TierMutationAbortRequest, TierMutationCommitRequest,
|
||||
ScannerActivityRequest, ScannerActivityResponse, ScannerPublicationLeaseReleaseRequest, ScannerPublicationLeaseRequest,
|
||||
ScannerPublicationLeaseResponse, ServerInfoRequest, SignalServiceRequest, SignalServiceResponse, StartDecommissionRequest,
|
||||
StartProfilingRequest, StopRebalanceRequest, TierMutationAbortRequest, TierMutationCommitRequest,
|
||||
TierMutationControlResponse, TierMutationPeerState, TierMutationPrepareRequest, node_service_client::NodeServiceClient,
|
||||
tier_mutation_control_service_client::TierMutationControlServiceClient,
|
||||
};
|
||||
@@ -84,6 +86,11 @@ const HEAL_CONTROL_PAYLOAD_MAX_SIZE: usize = 64 * 1024;
|
||||
const PEER_REST_RECOVERY_MAX_ATTEMPTS: u32 = 60;
|
||||
const PEER_REST_RECOVERY_MAX_BACKOFF: Duration = Duration::from_secs(30);
|
||||
const SCANNER_ACTIVITY_MAX_MESSAGE_SIZE: usize = 1024;
|
||||
/// Reserve time for the acquire response's network/clock uncertainty. The
|
||||
/// server owns the real expiry; this local deadline is intentionally earlier
|
||||
/// so a coordinator never starts a bounded persistence operation at the edge
|
||||
/// of a remote lease.
|
||||
const SCANNER_PUBLICATION_LEASE_SAFETY_MARGIN: Duration = Duration::from_secs(5);
|
||||
const REPLICATION_STATS_MAX_MESSAGE_SIZE: usize = 8 * 1024 * 1024;
|
||||
const BUCKET_METADATA_RELOAD_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
|
||||
@@ -150,6 +157,8 @@ pub struct ScannerPeerActivity {
|
||||
pub data_movement_active: Option<bool>,
|
||||
pub dirty_usage_generation: Option<u64>,
|
||||
pub dirty_usage_pending: Option<bool>,
|
||||
pub movement_generation: Option<u64>,
|
||||
pub publication_blocked: Option<bool>,
|
||||
}
|
||||
|
||||
fn decode_scanner_activity_with_verifier(
|
||||
@@ -166,7 +175,14 @@ fn decode_scanner_activity_with_verifier(
|
||||
{
|
||||
return Err(Error::other("peer returned an invalid scanner activity instance ID"));
|
||||
}
|
||||
let (topology_digest, data_movement_active, dirty_usage_generation, dirty_usage_pending) = match response.protocol_version {
|
||||
let (
|
||||
topology_digest,
|
||||
data_movement_active,
|
||||
dirty_usage_generation,
|
||||
dirty_usage_pending,
|
||||
movement_generation,
|
||||
publication_blocked,
|
||||
) = match response.protocol_version {
|
||||
// RUSTFS_COMPAT_TODO(ns-scanner-rpc-v3): legacy response fields are unauthenticated. Remove after protocol v0 peers are unsupported.
|
||||
SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION
|
||||
if response.topology_digest.is_empty()
|
||||
@@ -175,7 +191,7 @@ fn decode_scanner_activity_with_verifier(
|
||||
&& response.dirty_usage_generation == 0
|
||||
&& !response.dirty_usage_pending =>
|
||||
{
|
||||
(None, None, None, None)
|
||||
(None, None, None, None, None, None)
|
||||
}
|
||||
SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION => {
|
||||
return Err(Error::other("legacy scanner activity peer returned unexpected extended fields"));
|
||||
@@ -198,9 +214,11 @@ fn decode_scanner_activity_with_verifier(
|
||||
Some(response.data_movement_active),
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
}
|
||||
SCANNER_ACTIVITY_PROTOCOL_VERSION => {
|
||||
SCANNER_ACTIVITY_V6_PROTOCOL_VERSION => {
|
||||
if response.dirty_usage_pending && response.dirty_usage_generation == 0 {
|
||||
return Err(Error::other("scanner activity peer returned pending dirty usage without a generation"));
|
||||
}
|
||||
@@ -218,11 +236,42 @@ fn decode_scanner_activity_with_verifier(
|
||||
Some(response.data_movement_active),
|
||||
Some(response.dirty_usage_generation),
|
||||
Some(response.dirty_usage_pending),
|
||||
None,
|
||||
None,
|
||||
)
|
||||
}
|
||||
version => {
|
||||
return Err(Error::other(format!("peer returned unsupported scanner activity protocol {version}")));
|
||||
SCANNER_ACTIVITY_PROTOCOL_VERSION => {
|
||||
if response.dirty_usage_pending && response.dirty_usage_generation == 0 {
|
||||
return Err(Error::other("scanner activity peer returned pending dirty usage without a generation"));
|
||||
}
|
||||
let movement_generation = response
|
||||
.movement_generation
|
||||
.ok_or_else(|| Error::other("scanner activity peer omitted its movement generation"))?;
|
||||
let publication_blocked = response
|
||||
.publication_blocked
|
||||
.ok_or_else(|| Error::other("scanner activity peer omitted its publication blocked state"))?;
|
||||
if movement_generation == u64::MAX {
|
||||
return Err(Error::other("scanner activity peer exhausted its movement generation"));
|
||||
}
|
||||
let canonical = rustfs_protos::canonical_scanner_activity_v7_response_body(challenge, &response)
|
||||
.map_err(|_| Error::other("scanner activity peer response is too large to authenticate"))?;
|
||||
verify_proof(&canonical, &response.response_proof)?;
|
||||
(
|
||||
Some(
|
||||
response
|
||||
.topology_digest
|
||||
.as_ref()
|
||||
.try_into()
|
||||
.map_err(|_| Error::other("peer returned an invalid scanner topology digest"))?,
|
||||
),
|
||||
Some(response.data_movement_active),
|
||||
Some(response.dirty_usage_generation),
|
||||
Some(response.dirty_usage_pending),
|
||||
Some(movement_generation),
|
||||
Some(publication_blocked),
|
||||
)
|
||||
}
|
||||
version => return Err(Error::other(format!("peer returned unsupported scanner activity protocol {version}"))),
|
||||
};
|
||||
Ok(ScannerPeerActivity {
|
||||
instance_id: response.instance_id,
|
||||
@@ -233,6 +282,8 @@ fn decode_scanner_activity_with_verifier(
|
||||
data_movement_active,
|
||||
dirty_usage_generation,
|
||||
dirty_usage_pending,
|
||||
movement_generation,
|
||||
publication_blocked,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -243,6 +294,17 @@ fn decode_scanner_activity(response: ScannerActivityResponse, challenge: &[u8; 1
|
||||
})
|
||||
}
|
||||
|
||||
fn scanner_activity_protocol_unsupported(err: &Error) -> bool {
|
||||
matches!(
|
||||
err,
|
||||
Error::Io(io_err)
|
||||
if embedded_tonic_status(io_err).is_some_and(|status| {
|
||||
status.code() == tonic::Code::FailedPrecondition
|
||||
&& status.message().starts_with("unsupported scanner activity request protocol")
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
fn validate_heal_control_capability_proof(canonical_ack: &[u8], proof: &[u8]) -> Result<()> {
|
||||
verify_tonic_rpc_response_proof(canonical_ack, proof)
|
||||
.map_err(|_| Error::other("peer returned an invalid heal control capability proof"))
|
||||
@@ -285,6 +347,76 @@ pub struct PeerLiveEventsBatch {
|
||||
pub truncated: bool,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct ScannerPublicationLease {
|
||||
pub token: Uuid,
|
||||
pub movement_generation: u64,
|
||||
/// Stable storage owner identity. This is distinct from the activity
|
||||
/// session and is bound into both acquire and release proofs.
|
||||
pub owner_id: String,
|
||||
/// Process/session nonce observed by the final activity probe.
|
||||
pub session_id: String,
|
||||
pub expires_at: std::time::Instant,
|
||||
}
|
||||
|
||||
impl ScannerPublicationLease {
|
||||
pub fn is_valid(&self) -> bool {
|
||||
std::time::Instant::now() < self.expires_at
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_scanner_publication_lease_response_fields(
|
||||
response: &ScannerPublicationLeaseResponse,
|
||||
expected_session_id: &str,
|
||||
expected_generation: u64,
|
||||
) -> Result<(Uuid, String)> {
|
||||
if !response.success {
|
||||
return Err(Error::other(
|
||||
response
|
||||
.error
|
||||
.as_ref()
|
||||
.map(|error| error.error_info.clone())
|
||||
.unwrap_or_else(|| "peer rejected scanner publication lease".to_string()),
|
||||
));
|
||||
}
|
||||
if response.movement_generation != expected_generation {
|
||||
return Err(Error::other("peer returned a different scanner publication lease generation"));
|
||||
}
|
||||
if response.session_id != expected_session_id {
|
||||
return Err(Error::other("peer returned a different scanner publication lease session"));
|
||||
}
|
||||
let owner_id = Uuid::parse_str(&response.owner_id)
|
||||
.ok()
|
||||
.filter(|owner_id| !owner_id.is_nil())
|
||||
.map(|owner_id| owner_id.to_string())
|
||||
.ok_or_else(|| Error::other("peer returned an invalid scanner publication lease owner"))?;
|
||||
if response.lease_ttl_ms != crate::store::SCANNER_PUBLICATION_LEASE_TTL_MS {
|
||||
return Err(Error::other("peer returned an unsupported scanner publication lease TTL"));
|
||||
}
|
||||
let token = Uuid::from_slice(response.token.as_ref())
|
||||
.map_err(|_| Error::other("peer returned an invalid scanner publication lease token"))?;
|
||||
Ok((token, owner_id))
|
||||
}
|
||||
|
||||
fn scanner_publication_lease_deadline(
|
||||
request_started: std::time::Instant,
|
||||
response_received: std::time::Instant,
|
||||
lease_ttl_ms: u64,
|
||||
) -> Result<std::time::Instant> {
|
||||
let lease_window = Duration::from_millis(lease_ttl_ms)
|
||||
.checked_sub(SCANNER_PUBLICATION_LEASE_SAFETY_MARGIN)
|
||||
.ok_or_else(|| Error::other("scanner publication lease TTL is shorter than its safety margin"))?;
|
||||
let elapsed = response_received
|
||||
.checked_duration_since(request_started)
|
||||
.ok_or_else(|| Error::other("scanner publication lease response clock moved backwards"))?;
|
||||
if elapsed >= lease_window {
|
||||
return Err(Error::other("scanner publication lease response arrived after its safety window"));
|
||||
}
|
||||
request_started
|
||||
.checked_add(lease_window)
|
||||
.ok_or_else(|| Error::other("scanner publication lease deadline overflowed"))
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct PeerRestClient {
|
||||
pub host: XHost,
|
||||
@@ -1640,10 +1772,11 @@ impl PeerRestClient {
|
||||
.await
|
||||
}
|
||||
|
||||
async fn scanner_activity_request(
|
||||
async fn scanner_activity_request_with_protocol(
|
||||
&self,
|
||||
acknowledge_instance_id: String,
|
||||
acknowledge_dirty_usage_generation: u64,
|
||||
protocol_version: u32,
|
||||
) -> Result<ScannerPeerActivity> {
|
||||
self.finalize_result(
|
||||
async {
|
||||
@@ -1655,7 +1788,7 @@ impl PeerRestClient {
|
||||
.max_encoding_message_size(SCANNER_ACTIVITY_MAX_MESSAGE_SIZE);
|
||||
let mut request = Request::new(ScannerActivityRequest {
|
||||
challenge: challenge.as_bytes().to_vec().into(),
|
||||
protocol_version: SCANNER_ACTIVITY_PROTOCOL_VERSION,
|
||||
protocol_version,
|
||||
acknowledge_instance_id,
|
||||
acknowledge_dirty_usage_generation,
|
||||
});
|
||||
@@ -1671,11 +1804,168 @@ impl PeerRestClient {
|
||||
}
|
||||
|
||||
pub async fn scanner_activity(&self) -> Result<ScannerPeerActivity> {
|
||||
self.scanner_activity_request(String::new(), 0).await
|
||||
let result = self
|
||||
.scanner_activity_request_with_protocol(String::new(), 0, SCANNER_ACTIVITY_PROTOCOL_VERSION)
|
||||
.await;
|
||||
if result.as_ref().err().is_some_and(scanner_activity_protocol_unsupported) {
|
||||
// A v6 peer cannot parse the v7 marker. Its authenticated
|
||||
// response is still decoded as untrusted terminal state, so the
|
||||
// scanner will defer publication until every peer is v7.
|
||||
self.scanner_activity_request_with_protocol(String::new(), 0, SCANNER_ACTIVITY_V6_PROTOCOL_VERSION)
|
||||
.await
|
||||
} else {
|
||||
result
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn acknowledge_scanner_dirty_usage(&self, instance_id: String, generation: u64) -> Result<ScannerPeerActivity> {
|
||||
self.scanner_activity_request(instance_id, generation).await
|
||||
let result = self
|
||||
.scanner_activity_request_with_protocol(instance_id.clone(), generation, SCANNER_ACTIVITY_PROTOCOL_VERSION)
|
||||
.await;
|
||||
if result.as_ref().err().is_some_and(scanner_activity_protocol_unsupported) {
|
||||
self.scanner_activity_request_with_protocol(instance_id, generation, SCANNER_ACTIVITY_V6_PROTOCOL_VERSION)
|
||||
.await
|
||||
} else {
|
||||
result
|
||||
}
|
||||
}
|
||||
|
||||
/// Acquire a bounded, storage-owned read admission on the peer that
|
||||
/// produced the final activity generation. Older peers do not implement
|
||||
/// the lease form and are rejected rather than downgraded.
|
||||
pub async fn acquire_scanner_publication_lease(
|
||||
&self,
|
||||
expected_session_id: &str,
|
||||
expected_generation: u64,
|
||||
) -> Result<ScannerPublicationLease> {
|
||||
let request_started = std::time::Instant::now();
|
||||
self.finalize_result(
|
||||
async {
|
||||
let challenge = Uuid::new_v4();
|
||||
let mut client = self
|
||||
.get_client()
|
||||
.await?
|
||||
.max_decoding_message_size(SCANNER_ACTIVITY_MAX_MESSAGE_SIZE)
|
||||
.max_encoding_message_size(SCANNER_ACTIVITY_MAX_MESSAGE_SIZE);
|
||||
let mut request = Request::new(ScannerPublicationLeaseRequest {
|
||||
challenge: challenge.as_bytes().to_vec().into(),
|
||||
expected_movement_generation: expected_generation,
|
||||
ttl_ms: crate::store::SCANNER_PUBLICATION_LEASE_TTL_MS,
|
||||
expected_session_id: expected_session_id.to_string(),
|
||||
token: Bytes::new(),
|
||||
});
|
||||
let canonical = rustfs_protos::canonical_scanner_publication_lease_request_body(request.get_ref())
|
||||
.map_err(|_| Error::other("scanner publication lease request is too large to authenticate"))?;
|
||||
set_tonic_canonical_body_digest(&mut request, &canonical)?;
|
||||
let response = client.acquire_scanner_publication_lease(request).await?.into_inner();
|
||||
let response_body =
|
||||
rustfs_protos::canonical_scanner_publication_lease_response_body(challenge.as_bytes(), &response)
|
||||
.map_err(|_| Error::other("scanner publication lease response is too large to authenticate"))?;
|
||||
verify_tonic_rpc_response_proof(&response_body, &response.response_proof)
|
||||
.map_err(|_| Error::other("peer returned an invalid scanner publication lease proof"))?;
|
||||
let (token, owner_id) =
|
||||
validate_scanner_publication_lease_response_fields(&response, expected_session_id, expected_generation)?;
|
||||
Ok(ScannerPublicationLease {
|
||||
token,
|
||||
movement_generation: response.movement_generation,
|
||||
owner_id,
|
||||
session_id: response.session_id,
|
||||
expires_at: scanner_publication_lease_deadline(
|
||||
request_started,
|
||||
std::time::Instant::now(),
|
||||
response.lease_ttl_ms,
|
||||
)?,
|
||||
})
|
||||
}
|
||||
.await,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Revalidate the exact token immediately before the coordinator commits
|
||||
/// its final publication. The peer keeps the original movement read
|
||||
/// guard in its token table; a restart drops that table and changes the
|
||||
/// activity session, so this proof fails closed instead of accepting an
|
||||
/// ABA generation value.
|
||||
pub async fn validate_scanner_publication_lease(&self, lease: &ScannerPublicationLease) -> Result<()> {
|
||||
self.finalize_result(
|
||||
async {
|
||||
let challenge = Uuid::new_v4();
|
||||
let mut client = self
|
||||
.get_client()
|
||||
.await?
|
||||
.max_decoding_message_size(SCANNER_ACTIVITY_MAX_MESSAGE_SIZE)
|
||||
.max_encoding_message_size(SCANNER_ACTIVITY_MAX_MESSAGE_SIZE);
|
||||
let mut request = Request::new(ScannerPublicationLeaseRequest {
|
||||
challenge: challenge.as_bytes().to_vec().into(),
|
||||
expected_movement_generation: lease.movement_generation,
|
||||
ttl_ms: crate::store::SCANNER_PUBLICATION_LEASE_TTL_MS,
|
||||
expected_session_id: lease.session_id.clone(),
|
||||
token: lease.token.as_bytes().to_vec().into(),
|
||||
});
|
||||
let canonical = rustfs_protos::canonical_scanner_publication_lease_request_body(request.get_ref())
|
||||
.map_err(|_| Error::other("scanner publication lease validation request is too large to authenticate"))?;
|
||||
set_tonic_canonical_body_digest(&mut request, &canonical)?;
|
||||
let response = client.acquire_scanner_publication_lease(request).await?.into_inner();
|
||||
let response_body =
|
||||
rustfs_protos::canonical_scanner_publication_lease_response_body(challenge.as_bytes(), &response).map_err(
|
||||
|_| Error::other("scanner publication lease validation response is too large to authenticate"),
|
||||
)?;
|
||||
verify_tonic_rpc_response_proof(&response_body, &response.response_proof)
|
||||
.map_err(|_| Error::other("peer returned an invalid scanner publication lease validation proof"))?;
|
||||
let (token, owner_id) =
|
||||
validate_scanner_publication_lease_response_fields(&response, &lease.session_id, lease.movement_generation)?;
|
||||
if token != lease.token {
|
||||
return Err(Error::other("peer returned a different scanner publication lease token"));
|
||||
}
|
||||
if owner_id != lease.owner_id {
|
||||
return Err(Error::other("peer returned a different scanner publication lease owner"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
.await,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn release_scanner_publication_lease(&self, lease: &ScannerPublicationLease) -> Result<()> {
|
||||
self.finalize_result(
|
||||
async {
|
||||
let challenge = Uuid::new_v4();
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(ScannerPublicationLeaseReleaseRequest {
|
||||
challenge: challenge.as_bytes().to_vec().into(),
|
||||
token: lease.token.as_bytes().to_vec().into(),
|
||||
owner_id: lease.owner_id.clone(),
|
||||
session_id: lease.session_id.clone(),
|
||||
});
|
||||
let canonical = rustfs_protos::canonical_scanner_publication_lease_release_request_body(request.get_ref())
|
||||
.map_err(|_| Error::other("scanner publication lease release request is too large to authenticate"))?;
|
||||
set_tonic_canonical_body_digest(&mut request, &canonical)?;
|
||||
let request_body = request.get_ref().clone();
|
||||
let response = client.release_scanner_publication_lease(request).await?.into_inner();
|
||||
let response_body = rustfs_protos::canonical_scanner_publication_lease_release_response_body(
|
||||
challenge.as_bytes(),
|
||||
&request_body,
|
||||
&response,
|
||||
)
|
||||
.map_err(|_| Error::other("scanner publication lease release response is too large to authenticate"))?;
|
||||
verify_tonic_rpc_response_proof(&response_body, &response.response_proof)
|
||||
.map_err(|_| Error::other("peer returned an invalid scanner publication lease release proof"))?;
|
||||
if response.success {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(Error::other(
|
||||
response
|
||||
.error
|
||||
.map(|error| error.error_info)
|
||||
.unwrap_or_else(|| "peer rejected scanner publication lease release".to_string()),
|
||||
))
|
||||
}
|
||||
}
|
||||
.await,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn get_metacache_listing(&self) -> Result<()> {
|
||||
@@ -1991,6 +2281,52 @@ mod tests {
|
||||
use temp_env::async_with_vars;
|
||||
use tracing_subscriber::{Registry, fmt::MakeWriter, layer::SubscriberExt};
|
||||
|
||||
#[test]
|
||||
fn scanner_publication_lease_response_rejects_stale_generation_and_session() {
|
||||
let token = Uuid::new_v4();
|
||||
let response = ScannerPublicationLeaseResponse {
|
||||
success: true,
|
||||
token: token.as_bytes().to_vec().into(),
|
||||
movement_generation: 7,
|
||||
lease_ttl_ms: crate::store::SCANNER_PUBLICATION_LEASE_TTL_MS,
|
||||
error: None,
|
||||
response_proof: Bytes::new(),
|
||||
owner_id: Uuid::new_v4().to_string(),
|
||||
session_id: "session-a".to_string(),
|
||||
};
|
||||
|
||||
assert!(validate_scanner_publication_lease_response_fields(&response, "session-a", 7).is_ok());
|
||||
|
||||
let stale_generation = ScannerPublicationLeaseResponse {
|
||||
movement_generation: 6,
|
||||
..response.clone()
|
||||
};
|
||||
let error = validate_scanner_publication_lease_response_fields(&stale_generation, "session-a", 7)
|
||||
.expect_err("a response from an older movement generation must be rejected");
|
||||
assert!(error.to_string().contains("different scanner publication lease generation"));
|
||||
|
||||
let stale_session = ScannerPublicationLeaseResponse {
|
||||
session_id: "session-b".to_string(),
|
||||
..response
|
||||
};
|
||||
let error = validate_scanner_publication_lease_response_fields(&stale_session, "session-a", 7)
|
||||
.expect_err("a response from an older scanner session must be rejected");
|
||||
assert!(error.to_string().contains("different scanner publication lease session"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_publication_lease_deadline_accounts_for_delayed_rpc_response() {
|
||||
let started = std::time::Instant::now();
|
||||
let expected_deadline = started + Duration::from_secs(55);
|
||||
let deadline = scanner_publication_lease_deadline(started, started + Duration::from_secs(10), 60_000)
|
||||
.expect("a response inside the safety window should retain the original deadline");
|
||||
assert_eq!(deadline, expected_deadline);
|
||||
|
||||
let error = scanner_publication_lease_deadline(started, started + Duration::from_secs(55), 60_000)
|
||||
.expect_err("a response arriving at the safety boundary must fail closed");
|
||||
assert!(error.to_string().contains("after its safety window"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replication_stats_response_decodes_valid_empty_provider() {
|
||||
let mut stats = BucketStats::default();
|
||||
@@ -2229,6 +2565,8 @@ mod tests {
|
||||
response_proof: Vec::new().into(),
|
||||
dirty_usage_generation: 0,
|
||||
dirty_usage_pending: false,
|
||||
movement_generation: None,
|
||||
publication_blocked: None,
|
||||
})
|
||||
.expect("legacy peers should retain their activity generations during a rolling upgrade");
|
||||
assert_eq!(
|
||||
@@ -2242,6 +2580,8 @@ mod tests {
|
||||
data_movement_active: None,
|
||||
dirty_usage_generation: None,
|
||||
dirty_usage_pending: None,
|
||||
movement_generation: None,
|
||||
publication_blocked: None,
|
||||
}
|
||||
);
|
||||
|
||||
@@ -2255,6 +2595,8 @@ mod tests {
|
||||
response_proof: b"proof".to_vec().into(),
|
||||
dirty_usage_generation: 0,
|
||||
dirty_usage_pending: false,
|
||||
movement_generation: None,
|
||||
publication_blocked: None,
|
||||
})
|
||||
.expect("protocol v4 peers should remain observable during a rolling upgrade");
|
||||
assert_eq!(
|
||||
@@ -2268,9 +2610,29 @@ mod tests {
|
||||
data_movement_active: Some(true),
|
||||
dirty_usage_generation: None,
|
||||
dirty_usage_pending: None,
|
||||
movement_generation: None,
|
||||
publication_blocked: None,
|
||||
}
|
||||
);
|
||||
|
||||
let v6 = decode_test_scanner_activity(ScannerActivityResponse {
|
||||
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
|
||||
namespace_generation: 7,
|
||||
maintenance_generation: 3,
|
||||
protocol_version: SCANNER_ACTIVITY_V6_PROTOCOL_VERSION,
|
||||
topology_digest: vec![7; 32].into(),
|
||||
data_movement_active: true,
|
||||
response_proof: b"proof".to_vec().into(),
|
||||
dirty_usage_generation: 11,
|
||||
dirty_usage_pending: true,
|
||||
movement_generation: None,
|
||||
publication_blocked: None,
|
||||
})
|
||||
.expect("v6 peers should remain readable without a v7 publication proof");
|
||||
assert_eq!(v6.movement_generation, None);
|
||||
assert_eq!(v6.publication_blocked, None);
|
||||
assert_eq!(v6.dirty_usage_generation, Some(11));
|
||||
|
||||
let malformed_topology = ScannerActivityResponse {
|
||||
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
|
||||
namespace_generation: 7,
|
||||
@@ -2281,6 +2643,8 @@ mod tests {
|
||||
response_proof: b"proof".to_vec().into(),
|
||||
dirty_usage_generation: 11,
|
||||
dirty_usage_pending: true,
|
||||
movement_generation: Some(19),
|
||||
publication_blocked: Some(false),
|
||||
};
|
||||
assert!(
|
||||
decode_test_scanner_activity(malformed_topology)
|
||||
@@ -2299,6 +2663,8 @@ mod tests {
|
||||
response_proof: b"proof".to_vec().into(),
|
||||
dirty_usage_generation: 11,
|
||||
dirty_usage_pending: true,
|
||||
movement_generation: Some(19),
|
||||
publication_blocked: Some(false),
|
||||
};
|
||||
assert!(
|
||||
decode_test_scanner_activity(missing_instance)
|
||||
@@ -2317,6 +2683,8 @@ mod tests {
|
||||
response_proof: b"proof".to_vec().into(),
|
||||
dirty_usage_generation: 11,
|
||||
dirty_usage_pending: true,
|
||||
movement_generation: Some(19),
|
||||
publication_blocked: Some(false),
|
||||
};
|
||||
assert!(
|
||||
decode_test_scanner_activity(malformed_instance)
|
||||
@@ -2335,6 +2703,8 @@ mod tests {
|
||||
response_proof: b"proof".to_vec().into(),
|
||||
dirty_usage_generation: 11,
|
||||
dirty_usage_pending: true,
|
||||
movement_generation: Some(19),
|
||||
publication_blocked: Some(false),
|
||||
})
|
||||
.expect("complete activity responses should be accepted");
|
||||
assert_eq!(
|
||||
@@ -2348,9 +2718,31 @@ mod tests {
|
||||
data_movement_active: Some(true),
|
||||
dirty_usage_generation: Some(11),
|
||||
dirty_usage_pending: Some(true),
|
||||
movement_generation: Some(19),
|
||||
publication_blocked: Some(false),
|
||||
}
|
||||
);
|
||||
|
||||
let missing_movement_generation = ScannerActivityResponse {
|
||||
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
|
||||
namespace_generation: 7,
|
||||
maintenance_generation: 3,
|
||||
protocol_version: SCANNER_ACTIVITY_PROTOCOL_VERSION,
|
||||
topology_digest: vec![7; 32].into(),
|
||||
data_movement_active: false,
|
||||
response_proof: b"proof".to_vec().into(),
|
||||
dirty_usage_generation: 11,
|
||||
dirty_usage_pending: false,
|
||||
movement_generation: None,
|
||||
publication_blocked: Some(false),
|
||||
};
|
||||
assert!(
|
||||
decode_test_scanner_activity(missing_movement_generation)
|
||||
.expect_err("v7 activity must carry movement generation")
|
||||
.to_string()
|
||||
.contains("movement generation")
|
||||
);
|
||||
|
||||
let pending_without_generation = ScannerActivityResponse {
|
||||
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
|
||||
namespace_generation: 7,
|
||||
@@ -2361,6 +2753,8 @@ mod tests {
|
||||
response_proof: b"proof".to_vec().into(),
|
||||
dirty_usage_generation: 0,
|
||||
dirty_usage_pending: true,
|
||||
movement_generation: Some(19),
|
||||
publication_blocked: Some(false),
|
||||
};
|
||||
assert!(
|
||||
decode_test_scanner_activity(pending_without_generation)
|
||||
@@ -2379,6 +2773,8 @@ mod tests {
|
||||
response_proof: b"proof".to_vec().into(),
|
||||
dirty_usage_generation: 11,
|
||||
dirty_usage_pending: true,
|
||||
movement_generation: None,
|
||||
publication_blocked: None,
|
||||
};
|
||||
assert!(
|
||||
decode_test_scanner_activity(previous_with_dirty_usage)
|
||||
@@ -2397,6 +2793,8 @@ mod tests {
|
||||
response_proof: b"proof".to_vec().into(),
|
||||
dirty_usage_generation: 0,
|
||||
dirty_usage_pending: false,
|
||||
movement_generation: None,
|
||||
publication_blocked: None,
|
||||
};
|
||||
assert!(
|
||||
decode_test_scanner_activity(legacy_with_topology)
|
||||
@@ -2415,6 +2813,8 @@ mod tests {
|
||||
response_proof: b"proof".to_vec().into(),
|
||||
dirty_usage_generation: 11,
|
||||
dirty_usage_pending: true,
|
||||
movement_generation: None,
|
||||
publication_blocked: None,
|
||||
};
|
||||
assert!(
|
||||
decode_test_scanner_activity(unsupported_protocol)
|
||||
@@ -2433,6 +2833,8 @@ mod tests {
|
||||
response_proof: Vec::new().into(),
|
||||
dirty_usage_generation: 11,
|
||||
dirty_usage_pending: true,
|
||||
movement_generation: Some(19),
|
||||
publication_blocked: Some(false),
|
||||
};
|
||||
assert!(
|
||||
decode_test_scanner_activity(missing_proof)
|
||||
|
||||
@@ -1981,6 +1981,20 @@ impl RemoteDisk {
|
||||
fi: &FileInfo,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
) -> Result<RenameDataResp> {
|
||||
self.rename_data_borrowed_with_fence(src_volume, src_path, fi, dst_volume, dst_path, None)
|
||||
.await
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
pub(crate) async fn rename_data_borrowed_with_fence(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
src_path: &str,
|
||||
fi: &FileInfo,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
scanner_publication_lease_token: Option<Uuid>,
|
||||
) -> Result<RenameDataResp> {
|
||||
trace!(
|
||||
event = EVENT_REMOTE_DISK_RPC,
|
||||
@@ -2013,9 +2027,18 @@ impl RemoteDisk {
|
||||
dst_volume: dst_volume.to_string(),
|
||||
dst_path: dst_path.to_string(),
|
||||
file_info_bin: file_info_bin.into(),
|
||||
scanner_publication_lease_token: scanner_publication_lease_token
|
||||
.map(|token| token.as_bytes().to_vec().into())
|
||||
.unwrap_or_default(),
|
||||
});
|
||||
let canonical_body = rustfs_protos::canonical_rename_data_request_body(request.get_ref());
|
||||
attach_mutation_body_digest(&mut request, canonical_body, "rename_data")?;
|
||||
if scanner_publication_lease_token.is_some() {
|
||||
let canonical_body =
|
||||
canonical_body.map_err(|_| Error::other("rename_data request length cannot be represented"))?;
|
||||
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &canonical_body).map_err(Error::other)?;
|
||||
} else {
|
||||
attach_mutation_body_digest(&mut request, canonical_body, "rename_data")?;
|
||||
}
|
||||
|
||||
let response = client.rename_data(request).await?.into_inner();
|
||||
|
||||
@@ -2035,6 +2058,70 @@ impl RemoteDisk {
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Delete a path while binding the target-side operation to a scanner
|
||||
/// publication lease. The ordinary `DiskAPI::delete` path keeps the
|
||||
/// legacy digest/compatibility behavior by passing no token.
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
pub(crate) async fn delete_with_scanner_publication_lease(
|
||||
&self,
|
||||
volume: &str,
|
||||
path: &str,
|
||||
opt: DeleteOptions,
|
||||
scanner_publication_lease_token: Option<Uuid>,
|
||||
) -> Result<()> {
|
||||
trace!(
|
||||
event = EVENT_REMOTE_DISK_RPC,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
|
||||
endpoint = %self.endpoint,
|
||||
volume,
|
||||
path,
|
||||
recursive = opt.recursive,
|
||||
immediate = opt.immediate,
|
||||
fenced = scanner_publication_lease_token.is_some(),
|
||||
op = "delete",
|
||||
state = "started",
|
||||
"Remote disk RPC started"
|
||||
);
|
||||
|
||||
self.execute_with_timeout(
|
||||
|| async {
|
||||
let options = serde_json::to_string(&opt)?;
|
||||
let mut client = self
|
||||
.get_client()
|
||||
.await
|
||||
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
|
||||
let mut request = Request::new(DeleteRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
volume: volume.to_string(),
|
||||
path: path.to_string(),
|
||||
options,
|
||||
scanner_publication_lease_token: scanner_publication_lease_token
|
||||
.map(|token| token.as_bytes().to_vec().into())
|
||||
.unwrap_or_default(),
|
||||
});
|
||||
let canonical_body = rustfs_protos::canonical_delete_request_body(request.get_ref());
|
||||
if scanner_publication_lease_token.is_some() {
|
||||
let canonical_body =
|
||||
canonical_body.map_err(|_| Error::other("delete request length cannot be represented"))?;
|
||||
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &canonical_body).map_err(Error::other)?;
|
||||
} else {
|
||||
attach_mutation_body_digest(&mut request, canonical_body, "delete")?;
|
||||
}
|
||||
|
||||
let response = client.delete(request).await?.into_inner();
|
||||
|
||||
if !response.success {
|
||||
return Err(response.error.unwrap_or_default().into());
|
||||
}
|
||||
|
||||
Ok(())
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
@@ -3446,47 +3533,7 @@ impl DiskAPI for RemoteDisk {
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()> {
|
||||
trace!(
|
||||
event = EVENT_REMOTE_DISK_RPC,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
|
||||
endpoint = %self.endpoint,
|
||||
volume,
|
||||
path,
|
||||
recursive = opt.recursive,
|
||||
immediate = opt.immediate,
|
||||
op = "delete",
|
||||
state = "started",
|
||||
"Remote disk RPC started"
|
||||
);
|
||||
|
||||
self.execute_with_timeout(
|
||||
|| async {
|
||||
let options = serde_json::to_string(&opt)?;
|
||||
let mut client = self
|
||||
.get_client()
|
||||
.await
|
||||
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
|
||||
let mut request = Request::new(DeleteRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
volume: volume.to_string(),
|
||||
path: path.to_string(),
|
||||
options,
|
||||
});
|
||||
let canonical_body = rustfs_protos::canonical_delete_request_body(request.get_ref());
|
||||
attach_mutation_body_digest(&mut request, canonical_body, "delete")?;
|
||||
|
||||
let response = client.delete(request).await?.into_inner();
|
||||
|
||||
if !response.success {
|
||||
return Err(response.error.unwrap_or_default().into());
|
||||
}
|
||||
|
||||
Ok(())
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
self.delete_with_scanner_publication_lease(volume, path, opt, None).await
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
|
||||
@@ -677,6 +677,23 @@ impl DiskAPI for Disk {
|
||||
}
|
||||
|
||||
impl Disk {
|
||||
pub(crate) async fn delete_with_scanner_publication_lease(
|
||||
&self,
|
||||
volume: &str,
|
||||
path: &str,
|
||||
opts: DeleteOptions,
|
||||
scanner_publication_lease_token: Option<Uuid>,
|
||||
) -> Result<()> {
|
||||
match self {
|
||||
Disk::Local(local_disk) => local_disk.delete(volume, path, opts).await,
|
||||
Disk::Remote(remote_disk) => {
|
||||
remote_disk
|
||||
.delete_with_scanner_publication_lease(volume, path, opts, scanner_publication_lease_token)
|
||||
.await
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn rename_data_borrowed(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
@@ -684,6 +701,19 @@ impl Disk {
|
||||
fi: &FileInfo,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
) -> Result<RenameDataResp> {
|
||||
self.rename_data_borrowed_with_fence(src_volume, src_path, fi, dst_volume, dst_path, None)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn rename_data_borrowed_with_fence(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
src_path: &str,
|
||||
fi: &FileInfo,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
scanner_publication_lease_token: Option<Uuid>,
|
||||
) -> Result<RenameDataResp> {
|
||||
match self {
|
||||
Disk::Local(local_disk) => {
|
||||
@@ -693,7 +723,14 @@ impl Disk {
|
||||
}
|
||||
Disk::Remote(remote_disk) => {
|
||||
remote_disk
|
||||
.rename_data_borrowed(src_volume, src_path, fi, dst_volume, dst_path)
|
||||
.rename_data_borrowed_with_fence(
|
||||
src_volume,
|
||||
src_path,
|
||||
fi,
|
||||
dst_volume,
|
||||
dst_path,
|
||||
scanner_publication_lease_token,
|
||||
)
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
@@ -446,6 +446,11 @@ pub struct ObjectOptions {
|
||||
pub tier_delete_journal_api: Option<Arc<crate::store::ECStore>>,
|
||||
}
|
||||
|
||||
/// Transient scanner-only carrier for target-side publication lease tokens.
|
||||
/// SetDisks consumes and removes this key before constructing durable
|
||||
/// FileInfo metadata; it must never appear in an S3-visible object.
|
||||
pub const SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY: &str = "x-rustfs-internal-scanner-publication-lease-fence-v1";
|
||||
|
||||
impl ObjectOptions {
|
||||
pub fn set_quota_admission(&mut self, current_usage: u64, quota_limit: u64) -> bool {
|
||||
self.quota_admission = (current_usage <= quota_limit).then_some(QuotaAdmission {
|
||||
|
||||
@@ -56,7 +56,8 @@ use std::sync::{
|
||||
Arc, OnceLock,
|
||||
atomic::{AtomicBool, AtomicU8, AtomicU64, Ordering},
|
||||
};
|
||||
use tokio::sync::{OnceCell, RwLock};
|
||||
use tokio::sync::{Mutex, Notify, OnceCell, OwnedRwLockReadGuard, RwLock};
|
||||
use tokio::time::Instant;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use uuid::Uuid;
|
||||
|
||||
@@ -64,6 +65,19 @@ const SCANNER_PUBLICATION_STATE_UNKNOWN: u8 = 0;
|
||||
const SCANNER_PUBLICATION_STATE_ALLOWED: u8 = 1;
|
||||
const SCANNER_PUBLICATION_STATE_BLOCKED: u8 = 2;
|
||||
|
||||
pub(crate) const SCANNER_PUBLICATION_LEASE_MAX_ENTRIES: usize = 256;
|
||||
|
||||
/// A lease is deliberately short-lived. The coordinator treats expiry as a
|
||||
/// failed publication rather than silently continuing with a peer that may
|
||||
/// have started movement after the lease was abandoned.
|
||||
pub(crate) const SCANNER_PUBLICATION_LEASE_TTL: std::time::Duration = std::time::Duration::from_secs(60);
|
||||
|
||||
pub(crate) struct ScannerPublicationLeaseEntry {
|
||||
pub(crate) expires_at: Instant,
|
||||
pub(crate) movement_generation: u64,
|
||||
pub(crate) _operation_guard: OwnedRwLockReadGuard<()>,
|
||||
}
|
||||
|
||||
/// Runtime state owned by a single `ECStore` instance.
|
||||
///
|
||||
/// This is intentionally minimal in the first migration slice; subsequent
|
||||
@@ -171,6 +185,11 @@ pub struct InstanceContext {
|
||||
/// Readers are held across one publication commit; movement transitions
|
||||
/// take the writer at their durable state commit boundary.
|
||||
data_movement_operation_gate: Arc<RwLock<()>>,
|
||||
/// Remote scanner publication leases own a read guard until explicit
|
||||
/// release or bounded expiry. Keeping the guard in storage-owned state
|
||||
/// makes a remote movement transition wait on the same fence as a local
|
||||
/// scanner commit.
|
||||
scanner_publication_leases: Arc<Mutex<HashMap<Uuid, ScannerPublicationLeaseEntry>>>,
|
||||
/// Monotonic admission epoch paired with the operation gate. A
|
||||
/// publication admitted before a movement transition must never be
|
||||
/// mistaken for one admitted after the transition.
|
||||
@@ -180,6 +199,13 @@ pub struct InstanceContext {
|
||||
/// saturating counter prevents an unchanged `u64::MAX` value from being
|
||||
/// mistaken for a fresh epoch after overflow.
|
||||
data_movement_operation_epoch_exhausted: AtomicBool,
|
||||
/// Storage-owned generation for movement state changes. This is separate
|
||||
/// from the publication admission epoch so scanners can wait for a
|
||||
/// terminal/clear transition without treating the wake as a publication
|
||||
/// permit.
|
||||
data_movement_generation: AtomicU64,
|
||||
data_movement_generation_exhausted: AtomicBool,
|
||||
data_movement_generation_notify: Arc<Notify>,
|
||||
/// Last storage-owned movement snapshot observed under the operation
|
||||
/// gate. SetDisks cache writers fail closed until ECStore refreshes it.
|
||||
scanner_publication_state: AtomicU8,
|
||||
@@ -224,8 +250,12 @@ impl InstanceContext {
|
||||
bucket_metadata_sys: std::sync::Mutex::new(None),
|
||||
background_cancel_token: OnceLock::new(),
|
||||
data_movement_operation_gate: Arc::new(RwLock::new(())),
|
||||
scanner_publication_leases: Arc::new(Mutex::new(HashMap::new())),
|
||||
data_movement_operation_epoch: AtomicU64::new(0),
|
||||
data_movement_operation_epoch_exhausted: AtomicBool::new(false),
|
||||
data_movement_generation: AtomicU64::new(0),
|
||||
data_movement_generation_exhausted: AtomicBool::new(false),
|
||||
data_movement_generation_notify: Arc::new(Notify::new()),
|
||||
scanner_publication_state: AtomicU8::new(SCANNER_PUBLICATION_STATE_UNKNOWN),
|
||||
object_encryption_resolver: OnceLock::new(),
|
||||
tier_delete_journal_recovery_stores: std::sync::Mutex::new(HashSet::new()),
|
||||
@@ -249,6 +279,75 @@ impl InstanceContext {
|
||||
Arc::clone(&self.data_movement_operation_gate)
|
||||
}
|
||||
|
||||
pub(crate) async fn install_scanner_publication_lease(
|
||||
&self,
|
||||
token: Uuid,
|
||||
expires_at: Instant,
|
||||
movement_generation: u64,
|
||||
operation_guard: OwnedRwLockReadGuard<()>,
|
||||
) -> bool {
|
||||
let mut leases = self.scanner_publication_leases.lock().await;
|
||||
if leases.len() >= SCANNER_PUBLICATION_LEASE_MAX_ENTRIES {
|
||||
return false;
|
||||
}
|
||||
leases.insert(
|
||||
token,
|
||||
ScannerPublicationLeaseEntry {
|
||||
expires_at,
|
||||
movement_generation,
|
||||
_operation_guard: operation_guard,
|
||||
},
|
||||
);
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) async fn remove_scanner_publication_lease(&self, token: Uuid) -> bool {
|
||||
self.scanner_publication_leases.lock().await.remove(&token).is_some()
|
||||
}
|
||||
|
||||
/// Check a lease token while the caller holds the movement read guard.
|
||||
///
|
||||
/// The token table is deliberately process-owned and non-persistent: a
|
||||
/// restarted instance has no entries from the previous process, so an old
|
||||
/// coordinator proof cannot become valid again merely because the
|
||||
/// movement generation counter restarted at zero.
|
||||
pub(crate) async fn scanner_publication_lease_is_active(&self, token: Uuid) -> bool {
|
||||
let mut leases = self.scanner_publication_leases.lock().await;
|
||||
let now = Instant::now();
|
||||
let Some(expires_at) = leases.get(&token).map(|entry| entry.expires_at) else {
|
||||
return false;
|
||||
};
|
||||
if expires_at <= now {
|
||||
leases.remove(&token);
|
||||
return false;
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
/// Return the generation bound to a live lease. The lease entry owns the
|
||||
/// movement read guard, so a successful lookup remains valid for the
|
||||
/// caller's guard-protected operation; expiry is still fail-closed.
|
||||
pub(crate) async fn scanner_publication_lease_generation(&self, token: Uuid) -> Option<u64> {
|
||||
let mut leases = self.scanner_publication_leases.lock().await;
|
||||
let now = Instant::now();
|
||||
let (expires_at, movement_generation) = leases
|
||||
.get(&token)
|
||||
.map(|entry| (entry.expires_at, entry.movement_generation))?;
|
||||
if expires_at <= now {
|
||||
leases.remove(&token);
|
||||
return None;
|
||||
}
|
||||
Some(movement_generation)
|
||||
}
|
||||
|
||||
pub(crate) async fn expire_scanner_publication_lease(&self, token: Uuid, expires_at: Instant) {
|
||||
let mut leases = self.scanner_publication_leases.lock().await;
|
||||
let should_remove = leases.get(&token).is_some_and(|entry| entry.expires_at <= expires_at);
|
||||
if should_remove {
|
||||
leases.remove(&token);
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn data_movement_operation_epoch(&self) -> u64 {
|
||||
self.data_movement_operation_epoch.load(Ordering::Acquire)
|
||||
}
|
||||
@@ -257,8 +356,21 @@ impl InstanceContext {
|
||||
self.data_movement_operation_epoch_exhausted.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
pub(crate) fn data_movement_generation(&self) -> u64 {
|
||||
self.data_movement_generation.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
pub(crate) fn data_movement_generation_exhausted(&self) -> bool {
|
||||
self.data_movement_generation_exhausted.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
pub(crate) fn data_movement_generation_notify(&self) -> Arc<Notify> {
|
||||
Arc::clone(&self.data_movement_generation_notify)
|
||||
}
|
||||
|
||||
pub(crate) fn scanner_publication_state_allowed(&self) -> bool {
|
||||
!self.data_movement_operation_epoch_exhausted()
|
||||
&& !self.data_movement_generation_exhausted()
|
||||
&& self.scanner_publication_state.load(Ordering::Acquire) == SCANNER_PUBLICATION_STATE_ALLOWED
|
||||
}
|
||||
|
||||
@@ -276,6 +388,7 @@ impl InstanceContext {
|
||||
pub(crate) fn advance_data_movement_operation_epoch(&self) -> u64 {
|
||||
self.scanner_publication_state
|
||||
.store(SCANNER_PUBLICATION_STATE_UNKNOWN, Ordering::Release);
|
||||
let previous = self.data_movement_operation_epoch.load(Ordering::Acquire);
|
||||
let _ = self
|
||||
.data_movement_operation_epoch
|
||||
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |epoch| Some(epoch.saturating_add(1)));
|
||||
@@ -283,9 +396,42 @@ impl InstanceContext {
|
||||
if result == u64::MAX {
|
||||
self.data_movement_operation_epoch_exhausted.store(true, Ordering::Release);
|
||||
}
|
||||
if result != previous {
|
||||
let _ = self.advance_data_movement_generation();
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
/// Advance the movement generation after a durable movement transition.
|
||||
/// The generation is deliberately bounded: once it reaches `u64::MAX`,
|
||||
/// publication and generation-based waits fail closed rather than reusing
|
||||
/// an indistinguishable saturated value.
|
||||
pub(crate) fn advance_data_movement_generation(&self) -> Option<u64> {
|
||||
if self.data_movement_generation_exhausted.load(Ordering::Acquire) {
|
||||
return None;
|
||||
}
|
||||
let updated = self
|
||||
.data_movement_generation
|
||||
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |generation| generation.checked_add(1));
|
||||
match updated {
|
||||
Ok(previous) => {
|
||||
let Some(generation) = previous.checked_add(1) else {
|
||||
self.data_movement_generation_exhausted.store(true, Ordering::Release);
|
||||
return None;
|
||||
};
|
||||
if generation == u64::MAX {
|
||||
self.data_movement_generation_exhausted.store(true, Ordering::Release);
|
||||
}
|
||||
self.data_movement_generation_notify.notify_waiters();
|
||||
Some(generation)
|
||||
}
|
||||
Err(_) => {
|
||||
self.data_movement_generation_exhausted.store(true, Ordering::Release);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn set_data_movement_operation_epoch_for_test(&self, epoch: u64) {
|
||||
self.data_movement_operation_epoch.store(epoch, Ordering::Release);
|
||||
@@ -295,6 +441,13 @@ impl InstanceContext {
|
||||
.store(SCANNER_PUBLICATION_STATE_UNKNOWN, Ordering::Release);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn set_data_movement_generation_for_test(&self, generation: u64) {
|
||||
self.data_movement_generation.store(generation, Ordering::Release);
|
||||
self.data_movement_generation_exhausted
|
||||
.store(generation == u64::MAX, Ordering::Release);
|
||||
}
|
||||
|
||||
/// Install the application-owned object-encryption resolver once.
|
||||
pub fn set_object_encryption_resolver(
|
||||
&self,
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::cluster::rpc::{PeerRestClient, ScannerPeerActivity, TierConfigReloadOutcome};
|
||||
use crate::cluster::rpc::{PeerRestClient, ScannerPeerActivity, ScannerPublicationLease, TierConfigReloadOutcome};
|
||||
use crate::diagnostics::admin_server_info::get_commit_id;
|
||||
use crate::disk::DiskAPI;
|
||||
use crate::error::{Error, Result};
|
||||
@@ -53,6 +53,12 @@ const REMOTE_VERSION_STATE_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
const REMOTE_VERSION_STATE_PROOF_TTL: Duration = Duration::from_secs(30);
|
||||
const CROSS_POOL_FENCE_SUPPORTED_VERSION: u32 = 2;
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct ScannerPublicationLeaseGrant {
|
||||
pub host: String,
|
||||
pub lease: ScannerPublicationLease,
|
||||
}
|
||||
|
||||
/// Cached result from the last successful admin call to a peer.
|
||||
struct PeerAdminCache {
|
||||
last_storage_info: Option<StorageInfo>,
|
||||
@@ -1491,6 +1497,102 @@ impl NotificationSys {
|
||||
aggregate_scanner_dirty_usage_acknowledgement_results(join_all(futures).await, failures)
|
||||
}
|
||||
|
||||
/// Acquire remote publication leases in a deterministic host order. A
|
||||
/// missing/legacy peer is a hard publication deferral; already acquired
|
||||
/// leases are released before returning so a partial acquisition cannot
|
||||
/// pin movement on one peer.
|
||||
pub async fn acquire_scanner_publication_leases(
|
||||
&self,
|
||||
mut targets: Vec<(String, String, u64)>,
|
||||
) -> Result<Vec<ScannerPublicationLeaseGrant>> {
|
||||
targets.sort_by(|left, right| left.0.cmp(&right.0));
|
||||
for pair in targets.windows(2) {
|
||||
if pair[0].0 == pair[1].0 {
|
||||
return Err(Error::other(format!("duplicate scanner publication lease target: {}", pair[0].0)));
|
||||
}
|
||||
}
|
||||
|
||||
let mut grants = Vec::with_capacity(targets.len());
|
||||
for (host, session_id, generation) in targets {
|
||||
let Some(client) = self
|
||||
.peer_clients
|
||||
.iter()
|
||||
.flatten()
|
||||
.find(|client| client.grid_host == host)
|
||||
.cloned()
|
||||
else {
|
||||
let _ = self.release_scanner_publication_leases(grants).await;
|
||||
return Err(Error::other(format!("scanner publication lease peer {host} is unavailable")));
|
||||
};
|
||||
match client.acquire_scanner_publication_lease(&session_id, generation).await {
|
||||
Ok(lease) => grants.push(ScannerPublicationLeaseGrant { host, lease }),
|
||||
Err(err) => {
|
||||
let _ = self.release_scanner_publication_leases(grants).await;
|
||||
return Err(Error::other(format!("scanner publication lease acquisition failed: {err}")));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(grants)
|
||||
}
|
||||
|
||||
pub async fn release_scanner_publication_leases(&self, mut grants: Vec<ScannerPublicationLeaseGrant>) -> Result<()> {
|
||||
grants.sort_by(|left, right| right.host.cmp(&left.host));
|
||||
let mut failures = Vec::new();
|
||||
for grant in grants {
|
||||
let Some(client) = self
|
||||
.peer_clients
|
||||
.iter()
|
||||
.flatten()
|
||||
.find(|client| client.grid_host == grant.host)
|
||||
else {
|
||||
failures.push(format!("peer {} is unavailable", grant.host));
|
||||
continue;
|
||||
};
|
||||
if let Err(err) = client.release_scanner_publication_lease(&grant.lease).await {
|
||||
failures.push(format!("peer {} release failed: {err}", grant.host));
|
||||
}
|
||||
}
|
||||
if failures.is_empty() {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(Error::other(format!(
|
||||
"scanner publication lease release failures: {}",
|
||||
failures.join("; ")
|
||||
)))
|
||||
}
|
||||
}
|
||||
|
||||
/// Revalidate every remote lease in deterministic host order immediately
|
||||
/// before a final scanner metadata write. A peer restart removes its
|
||||
/// process-owned token table and changes its activity session, so an old
|
||||
/// generation cannot pass this proof even when the numeric generation is
|
||||
/// reused.
|
||||
pub async fn validate_scanner_publication_leases(&self, grants: &[ScannerPublicationLeaseGrant]) -> Result<()> {
|
||||
let mut grants = grants.to_vec();
|
||||
grants.sort_by(|left, right| left.host.cmp(&right.host));
|
||||
for pair in grants.windows(2) {
|
||||
if pair[0].host == pair[1].host {
|
||||
return Err(Error::other(format!("duplicate scanner publication lease target: {}", pair[0].host)));
|
||||
}
|
||||
}
|
||||
for grant in grants {
|
||||
let Some(client) = self
|
||||
.peer_clients
|
||||
.iter()
|
||||
.flatten()
|
||||
.find(|client| client.grid_host == grant.host)
|
||||
.cloned()
|
||||
else {
|
||||
return Err(Error::other(format!("scanner publication lease peer {} is unavailable", grant.host)));
|
||||
};
|
||||
client
|
||||
.validate_scanner_publication_lease(&grant.lease)
|
||||
.await
|
||||
.map_err(|err| Error::other(format!("scanner publication lease validation failed for {}: {err}", grant.host)))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn reload_site_replication_config(&self) -> Vec<NotificationPeerErr> {
|
||||
let mut futures = Vec::with_capacity(self.peer_clients.len());
|
||||
for client in self.peer_clients.iter() {
|
||||
@@ -2633,6 +2735,38 @@ mod tests {
|
||||
assert!(err.to_string().contains("no remote peers"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_publication_lease_release_reports_all_unavailable_peers() {
|
||||
let sys = NotificationSys {
|
||||
peer_clients: Vec::new(),
|
||||
all_peer_clients: Vec::new(),
|
||||
peer_topology_hosts: Vec::new(),
|
||||
peer_admin_caches: Vec::new(),
|
||||
tier_config_reload_workers: Default::default(),
|
||||
};
|
||||
let grants = ["peer-a", "peer-b"]
|
||||
.into_iter()
|
||||
.map(|host| ScannerPublicationLeaseGrant {
|
||||
host: host.to_string(),
|
||||
lease: ScannerPublicationLease {
|
||||
token: Uuid::new_v4(),
|
||||
movement_generation: 3,
|
||||
owner_id: Uuid::new_v4().to_string(),
|
||||
session_id: "session-a".to_string(),
|
||||
expires_at: Instant::now() + Duration::from_secs(30),
|
||||
},
|
||||
})
|
||||
.collect();
|
||||
|
||||
let error = sys
|
||||
.release_scanner_publication_leases(grants)
|
||||
.await
|
||||
.expect_err("an unavailable peer must not silently release a remote lease");
|
||||
let message = error.to_string();
|
||||
assert!(message.contains("peer-a"));
|
||||
assert!(message.contains("peer-b"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_activity_probe_rejects_an_incomplete_peer_topology() {
|
||||
let client = PeerRestClient::new(
|
||||
@@ -2755,6 +2889,8 @@ mod tests {
|
||||
data_movement_active: Some(false),
|
||||
dirty_usage_generation: Some(2),
|
||||
dirty_usage_pending,
|
||||
movement_generation: Some(1),
|
||||
publication_blocked: Some(false),
|
||||
};
|
||||
|
||||
let pending = aggregate_scanner_dirty_usage_acknowledgement_results(
|
||||
|
||||
@@ -3499,6 +3499,26 @@ pub(in crate::set_disk) struct RenameDataCommit {
|
||||
pub(in crate::set_disk) tail_drain: Option<tokio::task::JoinHandle<()>>,
|
||||
}
|
||||
|
||||
/// Options shared by the normal and early-ack rename fanouts. Keeping the
|
||||
/// quorum and optional scanner lease map together avoids widening either
|
||||
/// fanout helper's argument list while preserving the fence semantics.
|
||||
pub(in crate::set_disk) struct RenameDataFenceOptions<'a> {
|
||||
write_quorum: usize,
|
||||
scanner_publication_lease_tokens: Option<&'a HashMap<String, Uuid>>,
|
||||
}
|
||||
|
||||
impl<'a> RenameDataFenceOptions<'a> {
|
||||
pub(in crate::set_disk) fn new(
|
||||
write_quorum: usize,
|
||||
scanner_publication_lease_tokens: Option<&'a HashMap<String, Uuid>>,
|
||||
) -> Self {
|
||||
Self {
|
||||
write_quorum,
|
||||
scanner_publication_lease_tokens,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
type RenameDataLegacyTuple = (
|
||||
Vec<Option<DiskStore>>,
|
||||
@@ -3690,16 +3710,51 @@ impl SetDisks {
|
||||
.map(RenameDataCommit::into_legacy_tuple)
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip(disks, file_infos))]
|
||||
async fn rename_data_owned_early_ack(
|
||||
pub(in crate::set_disk) fn scanner_publication_lease_token_for_disk(
|
||||
disk: Option<&DiskStore>,
|
||||
scanner_publication_lease_tokens: Option<&HashMap<String, Uuid>>,
|
||||
) -> disk::error::Result<Option<Uuid>> {
|
||||
let Some(disk) = disk else {
|
||||
return Ok(None);
|
||||
};
|
||||
if disk.is_local() {
|
||||
return Ok(None);
|
||||
}
|
||||
let Some(tokens) = scanner_publication_lease_tokens else {
|
||||
return Ok(None);
|
||||
};
|
||||
let host = disk.endpoint().grid_host();
|
||||
tokens
|
||||
.get(&host)
|
||||
.copied()
|
||||
.ok_or_else(|| DiskError::other(format!("scanner publication lease token missing for remote disk host {host}")))
|
||||
.map(Some)
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn scanner_publication_lease_tokens_for_disks(
|
||||
disks: &[Option<DiskStore>],
|
||||
scanner_publication_lease_tokens: Option<&HashMap<String, Uuid>>,
|
||||
) -> disk::error::Result<Vec<Option<Uuid>>> {
|
||||
disks
|
||||
.iter()
|
||||
.map(|disk| Self::scanner_publication_lease_token_for_disk(disk.as_ref(), scanner_publication_lease_tokens))
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip(disks, file_infos, fence_options))]
|
||||
async fn rename_data_owned_early_ack_with_fence(
|
||||
disks: &[Option<DiskStore>],
|
||||
src_bucket: &str,
|
||||
src_object: &str,
|
||||
file_infos: Vec<FileInfo>,
|
||||
dst_bucket: &str,
|
||||
dst_object: &str,
|
||||
write_quorum: usize,
|
||||
fence_options: RenameDataFenceOptions<'_>,
|
||||
) -> disk::error::Result<RenameDataCommit> {
|
||||
let RenameDataFenceOptions {
|
||||
write_quorum,
|
||||
scanner_publication_lease_tokens,
|
||||
} = fence_options;
|
||||
if let Some(file_info) = disks
|
||||
.iter()
|
||||
.zip(file_infos.iter())
|
||||
@@ -3714,6 +3769,7 @@ impl SetDisks {
|
||||
|
||||
let disk_count = disks.len();
|
||||
let fanout_disks = disks.to_vec();
|
||||
let fanout_fence_tokens = Self::scanner_publication_lease_tokens_for_disks(disks, scanner_publication_lease_tokens)?;
|
||||
let coordinator_disks = fanout_disks.clone();
|
||||
let src_bucket = Arc::new(src_bucket.to_string());
|
||||
let src_object = Arc::new(src_object.to_string());
|
||||
@@ -3730,7 +3786,12 @@ impl SetDisks {
|
||||
let successful_rename_completion_rank =
|
||||
rustfs_io_metrics::put_stage_metrics_enabled().then(|| Arc::new(AtomicUsize::new(0)));
|
||||
let mut tasks = JoinSet::new();
|
||||
for (i, (disk, file_info)) in fanout_disks.into_iter().zip(file_infos.iter()).enumerate() {
|
||||
for (i, ((disk, file_info), scanner_publication_lease_token)) in fanout_disks
|
||||
.into_iter()
|
||||
.zip(file_infos.iter())
|
||||
.zip(fanout_fence_tokens.into_iter())
|
||||
.enumerate()
|
||||
{
|
||||
let src_bucket = fanout_src_bucket.clone();
|
||||
let src_object = fanout_src_object.clone();
|
||||
let dst_bucket = fanout_dst_bucket.clone();
|
||||
@@ -3767,7 +3828,14 @@ impl SetDisks {
|
||||
|
||||
let disk_wait_started = rustfs_io_metrics::put_stage_timer();
|
||||
let result = disk
|
||||
.rename_data_borrowed(&src_bucket, &src_object, file_info, &dst_bucket, &dst_object)
|
||||
.rename_data_borrowed_with_fence(
|
||||
&src_bucket,
|
||||
&src_object,
|
||||
file_info,
|
||||
&dst_bucket,
|
||||
&dst_object,
|
||||
scanner_publication_lease_token,
|
||||
)
|
||||
.await;
|
||||
if let Some(disk_wait_started) = disk_wait_started {
|
||||
let duration_ms = disk_wait_started.elapsed().as_secs_f64() * 1000.0;
|
||||
@@ -3967,19 +4035,45 @@ impl SetDisks {
|
||||
dst_bucket: &str,
|
||||
dst_object: &str,
|
||||
write_quorum: usize,
|
||||
) -> disk::error::Result<RenameDataCommit> {
|
||||
Self::rename_data_owned_with_fence(
|
||||
disks,
|
||||
src_bucket,
|
||||
src_object,
|
||||
file_infos,
|
||||
dst_bucket,
|
||||
dst_object,
|
||||
RenameDataFenceOptions::new(write_quorum, None),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip(disks, file_infos, fence_options))]
|
||||
pub(in crate::set_disk) async fn rename_data_owned_with_fence(
|
||||
disks: &[Option<DiskStore>],
|
||||
src_bucket: &str,
|
||||
src_object: &str,
|
||||
file_infos: Vec<FileInfo>,
|
||||
dst_bucket: &str,
|
||||
dst_object: &str,
|
||||
fence_options: RenameDataFenceOptions<'_>,
|
||||
) -> disk::error::Result<RenameDataCommit> {
|
||||
if put_rename_early_ack_enabled() {
|
||||
return Self::rename_data_owned_early_ack(
|
||||
return Self::rename_data_owned_early_ack_with_fence(
|
||||
disks,
|
||||
src_bucket,
|
||||
src_object,
|
||||
file_infos,
|
||||
dst_bucket,
|
||||
dst_object,
|
||||
write_quorum,
|
||||
fence_options,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
let RenameDataFenceOptions {
|
||||
write_quorum,
|
||||
scanner_publication_lease_tokens,
|
||||
} = fence_options;
|
||||
if let Some(file_info) = disks
|
||||
.iter()
|
||||
.zip(file_infos.iter())
|
||||
@@ -4005,6 +4099,7 @@ impl SetDisks {
|
||||
let disk_count = disks.len();
|
||||
let fanout_disks = disks.to_vec();
|
||||
let fanout_file_infos = file_infos;
|
||||
let fanout_fence_tokens = Self::scanner_publication_lease_tokens_for_disks(disks, scanner_publication_lease_tokens)?;
|
||||
let fanout_src_bucket = src_bucket.clone();
|
||||
let fanout_src_object = src_object.clone();
|
||||
let fanout_dst_bucket = dst_bucket.clone();
|
||||
@@ -4019,8 +4114,9 @@ impl SetDisks {
|
||||
let futures = fanout_disks
|
||||
.into_iter()
|
||||
.zip(fanout_file_infos.iter())
|
||||
.zip(fanout_fence_tokens.into_iter())
|
||||
.enumerate()
|
||||
.map(|(i, (disk, file_info))| {
|
||||
.map(|(i, ((disk, file_info), scanner_publication_lease_token))| {
|
||||
let src_bucket = fanout_src_bucket.clone();
|
||||
let src_object = fanout_src_object.clone();
|
||||
let dst_object = fanout_dst_object.clone();
|
||||
@@ -4060,7 +4156,14 @@ impl SetDisks {
|
||||
|
||||
let disk_wait_started = rustfs_io_metrics::put_stage_timer();
|
||||
let result = disk
|
||||
.rename_data_borrowed(&src_bucket, &src_object, file_info, &dst_bucket, &dst_object)
|
||||
.rename_data_borrowed_with_fence(
|
||||
&src_bucket,
|
||||
&src_object,
|
||||
file_info,
|
||||
&dst_bucket,
|
||||
&dst_object,
|
||||
scanner_publication_lease_token,
|
||||
)
|
||||
.await;
|
||||
if let Some(disk_wait_started) = disk_wait_started {
|
||||
let duration_ms = disk_wait_started.elapsed().as_secs_f64() * 1000.0;
|
||||
@@ -5426,18 +5529,32 @@ impl SetDisks {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(in crate::set_disk) async fn delete_prefix(&self, bucket: &str, prefix: &str) -> disk::error::Result<()> {
|
||||
self.delete_prefix_with_scanner_publication_lease(bucket, prefix, None).await
|
||||
}
|
||||
|
||||
/// Delete a prefix with an optional per-remote-disk scanner publication
|
||||
/// lease fence. The fence is transient and never reaches disk metadata;
|
||||
/// remote deletes without a matching token fail closed before the RPC.
|
||||
pub(in crate::set_disk) async fn delete_prefix_with_scanner_publication_lease(
|
||||
&self,
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
scanner_publication_lease_tokens: Option<&HashMap<String, Uuid>>,
|
||||
) -> disk::error::Result<()> {
|
||||
let disks = self.get_disks_internal().await;
|
||||
let write_quorum = disks.len() / 2 + 1;
|
||||
let fanout_fence_tokens = Self::scanner_publication_lease_tokens_for_disks(&disks, scanner_publication_lease_tokens)?;
|
||||
|
||||
let mut futures = Vec::with_capacity(disks.len());
|
||||
|
||||
for disk_op in disks.iter() {
|
||||
for (disk_op, scanner_publication_lease_token) in disks.iter().zip(fanout_fence_tokens) {
|
||||
let bucket = bucket.to_string();
|
||||
let prefix = prefix.to_string();
|
||||
futures.push(async move {
|
||||
if let Some(disk) = disk_op {
|
||||
disk.delete(
|
||||
disk.delete_with_scanner_publication_lease(
|
||||
&bucket,
|
||||
&prefix,
|
||||
DeleteOptions {
|
||||
@@ -5445,6 +5562,7 @@ impl SetDisks {
|
||||
immediate: true,
|
||||
..Default::default()
|
||||
},
|
||||
scanner_publication_lease_token,
|
||||
)
|
||||
.await
|
||||
} else {
|
||||
|
||||
@@ -49,8 +49,8 @@ use crate::data_usage::quota_object_size;
|
||||
use crate::diagnostics::get::GetObjectFailureReason;
|
||||
use crate::disk::{DataDirDeleteStatus, OldCurrentSize};
|
||||
use crate::error::is_err_invalid_upload_id;
|
||||
use crate::object_api::NamespaceLockFence;
|
||||
use crate::object_api::{GetObjectBodySource, get_object_body_cache_hook_suppressed};
|
||||
use crate::object_api::{NamespaceLockFence, SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY};
|
||||
use crate::services::notification_sys::RemoteVersionStateFleetProofToken;
|
||||
use crate::services::tier::tier::{TierConfigMgr, TierOperationLease};
|
||||
use crate::store::ECStore;
|
||||
@@ -63,6 +63,60 @@ use std::sync::OnceLock;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
const OLD_DATA_CLEANUP_RECEIPT_FILE: &str = ".rustfs-old-data-cleanup-receipt.json";
|
||||
const SCANNER_PUBLICATION_LEASE_FENCE_MAX_BYTES: usize = 64 * 1024;
|
||||
const SCANNER_PUBLICATION_LEASE_FENCE_MAX_ENTRIES: usize = 256;
|
||||
|
||||
fn take_scanner_publication_lease_tokens(user_defined: &mut HashMap<String, String>) -> Result<Option<HashMap<String, Uuid>>> {
|
||||
let Some(encoded) = user_defined.remove(SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY) else {
|
||||
return Ok(None);
|
||||
};
|
||||
if encoded.len() > SCANNER_PUBLICATION_LEASE_FENCE_MAX_BYTES {
|
||||
return Err(Error::other("scanner publication lease fence is too large"));
|
||||
}
|
||||
let encoded_tokens = serde_json::from_str::<HashMap<String, String>>(&encoded)
|
||||
.map_err(|err| Error::other(format!("invalid scanner publication lease fence: {err}")))?;
|
||||
if encoded_tokens.len() > SCANNER_PUBLICATION_LEASE_FENCE_MAX_ENTRIES {
|
||||
return Err(Error::other("scanner publication lease fence has too many entries"));
|
||||
}
|
||||
let mut tokens = HashMap::with_capacity(encoded_tokens.len());
|
||||
for (host, token) in encoded_tokens {
|
||||
if host.is_empty() || host.len() > 1024 {
|
||||
return Err(Error::other("invalid scanner publication lease fence host"));
|
||||
}
|
||||
let token = Uuid::parse_str(&token)
|
||||
.map_err(|err| Error::other(format!("invalid scanner publication lease fence token: {err}")))?;
|
||||
if token.is_nil() {
|
||||
return Err(Error::other("invalid scanner publication lease fence token"));
|
||||
}
|
||||
tokens.insert(host, token);
|
||||
}
|
||||
Ok(Some(tokens))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod scanner_publication_lease_fence_tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn scanner_publication_lease_fence_is_transient_and_validated() {
|
||||
let token = Uuid::new_v4();
|
||||
let mut metadata = HashMap::from([(
|
||||
SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY.to_string(),
|
||||
serde_json::json!({"http://node-a:9000": token.to_string()}).to_string(),
|
||||
)]);
|
||||
let parsed = take_scanner_publication_lease_tokens(&mut metadata)
|
||||
.expect("valid scanner publication lease fence should parse")
|
||||
.expect("scanner publication lease fence should be present");
|
||||
assert_eq!(parsed.get("http://node-a:9000"), Some(&token));
|
||||
assert!(!metadata.contains_key(SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY));
|
||||
|
||||
let mut malformed = HashMap::from([(
|
||||
SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY.to_string(),
|
||||
r#"{"http://node-a:9000":"not-a-uuid"}"#.to_string(),
|
||||
)]);
|
||||
assert!(take_scanner_publication_lease_tokens(&mut malformed).is_err());
|
||||
}
|
||||
}
|
||||
|
||||
struct PutObjectCommitCancellation {
|
||||
token: CancellationToken,
|
||||
@@ -2177,6 +2231,7 @@ impl SetDisks {
|
||||
user_defined.insert(key.clone(), value.clone());
|
||||
}
|
||||
}
|
||||
let scanner_publication_lease_tokens = take_scanner_publication_lease_tokens(&mut user_defined)?;
|
||||
if replication_lww_applicable(opts) {
|
||||
// Object Lock evaluation stamps category timestamps with this
|
||||
// receiver's clock. Pin them back to the source-authored times
|
||||
@@ -2896,6 +2951,7 @@ impl SetDisks {
|
||||
let commit_bucket_lifecycle_lock_fence = opts.bucket_lifecycle_lock_fence.clone();
|
||||
let commit_capacity_scope_token = opts.capacity_scope_token;
|
||||
let commit_replication_state = replication_state_to_filemeta(&opts.put_replication_state());
|
||||
let commit_scanner_publication_lease_tokens = scanner_publication_lease_tokens;
|
||||
tmp_cleanup_owned = true;
|
||||
|
||||
let commit = move |cancellation: Option<CancellationToken>| async move {
|
||||
@@ -3015,14 +3071,17 @@ impl SetDisks {
|
||||
}
|
||||
|
||||
Self::assign_rename_data_indexes(&mut parts_metadatas);
|
||||
let rename_result = SetDisks::rename_data_owned(
|
||||
let rename_result = SetDisks::rename_data_owned_with_fence(
|
||||
&commit_disks,
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
commit_tmp_dir.as_str(),
|
||||
parts_metadatas,
|
||||
&commit_bucket,
|
||||
&commit_object,
|
||||
write_quorum,
|
||||
crate::set_disk::core::io_primitives::RenameDataFenceOptions::new(
|
||||
write_quorum,
|
||||
commit_scanner_publication_lease_tokens.as_ref(),
|
||||
),
|
||||
)
|
||||
.await;
|
||||
if quota_mutation_fence {
|
||||
@@ -6418,6 +6477,10 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
|
||||
#[tracing::instrument(skip(self))]
|
||||
async fn delete_object(&self, bucket: &str, object: &str, mut opts: ObjectOptions) -> Result<ObjectInfo> {
|
||||
// Scanner cleanup carries the per-peer lease fence as transient
|
||||
// request metadata. Consume it before any delete-prefix fanout so it
|
||||
// cannot be persisted or treated as user metadata.
|
||||
let scanner_publication_lease_tokens = take_scanner_publication_lease_tokens(&mut opts.user_defined)?;
|
||||
let preserve_delete_replication_state = should_preserve_delete_replication_state(&opts);
|
||||
let delete_config_snapshot = if opts.delete_prefix || opts.transition.expire_restored || preserve_delete_replication_state
|
||||
{
|
||||
@@ -6544,7 +6607,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
self.validate_bucket_incarnation(bucket, expected_incarnation_id).await?;
|
||||
}
|
||||
ensure_delete_commit_locks_held(_lock_guard.as_ref(), bucket, object, &opts)?;
|
||||
self.delete_prefix(bucket, object)
|
||||
self.delete_prefix_with_scanner_publication_lease(bucket, object, scanner_publication_lease_tokens.as_ref())
|
||||
.await
|
||||
.map_err(|e| to_object_err(e.into(), vec![bucket, object]))?;
|
||||
|
||||
|
||||
@@ -32,7 +32,8 @@ pub(crate) mod internode {
|
||||
PUT_FILE_AUTH_TRAILER_MAGIC, PUT_FILE_AUTH_V1, PUT_FILE_CAPABILITY_CHALLENGE_QUERY, PUT_FILE_CAPABILITY_QUERY,
|
||||
PUT_FILE_CAPABILITY_VERSION, PUT_FILE_NONCE_QUERY, PUT_FILE_SERVER_EPOCH_QUERY, PutFileCapabilityResponse,
|
||||
SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION, SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SCANNER_ACTIVITY_PROTOCOL_VERSION,
|
||||
WALK_DIR_BODY_SHA256_QUERY, WALK_DIR_STREAM_COMPLETION_QUERY, WALK_DIR_STREAM_COMPLETION_V1,
|
||||
SCANNER_ACTIVITY_V6_PROTOCOL_VERSION, WALK_DIR_BODY_SHA256_QUERY, WALK_DIR_STREAM_COMPLETION_QUERY,
|
||||
WALK_DIR_STREAM_COMPLETION_V1,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -87,6 +87,8 @@ type ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
|
||||
type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>;
|
||||
type WalkOptions = StorageWalkOptions<fn(&FileInfo) -> bool>;
|
||||
|
||||
pub const SCANNER_PUBLICATION_LEASE_TTL_MS: u64 = 60_000;
|
||||
|
||||
/// Check if a directory contains any xl.meta files (indicating actual S3 objects)
|
||||
/// This is used to determine if a bucket is empty for deletion purposes.
|
||||
pub(crate) async fn has_xlmeta_files(path: &std::path::Path) -> std::io::Result<bool> {
|
||||
@@ -344,6 +346,33 @@ impl ECStore {
|
||||
decommission || rebalance
|
||||
}
|
||||
|
||||
/// Return the storage-owned movement state and generation as one
|
||||
/// authenticated activity snapshot. The read lock is acquired before
|
||||
/// the state locks (cancelers, pool metadata, then rebalance metadata),
|
||||
/// matching the transition writer order and preventing a terminal state
|
||||
/// from being reported with the preceding generation.
|
||||
pub async fn scanner_data_movement_activity(&self) -> (bool, bool, u64) {
|
||||
let operation_gate = self.ctx.data_movement_operation_gate();
|
||||
let _operation_guard = operation_gate.read_owned().await;
|
||||
let (active, blocked) = self.scanner_data_movement_snapshot_locked().await;
|
||||
let blocked =
|
||||
blocked || self.ctx.data_movement_operation_epoch_exhausted() || self.ctx.data_movement_generation_exhausted();
|
||||
self.ctx.set_scanner_publication_state(blocked);
|
||||
(active, blocked, self.ctx.data_movement_generation())
|
||||
}
|
||||
|
||||
pub fn scanner_data_movement_generation(&self) -> u64 {
|
||||
self.ctx.data_movement_generation()
|
||||
}
|
||||
|
||||
pub fn scanner_data_movement_generation_exhausted(&self) -> bool {
|
||||
self.ctx.data_movement_generation_exhausted()
|
||||
}
|
||||
|
||||
pub fn scanner_data_movement_changed(&self) -> std::sync::Arc<tokio::sync::Notify> {
|
||||
self.ctx.data_movement_generation_notify()
|
||||
}
|
||||
|
||||
/// Returns whether scanner metadata may still be hidden by a local
|
||||
/// data-movement state. Terminal failed/canceled decommission entries
|
||||
/// remain suspended until an operator clears or retries them, so they are
|
||||
@@ -355,10 +384,16 @@ impl ECStore {
|
||||
}
|
||||
|
||||
async fn scanner_data_usage_publication_snapshot_blocked(&self) -> bool {
|
||||
if self.ctx.data_movement_operation_epoch_exhausted() {
|
||||
if self.ctx.data_movement_operation_epoch_exhausted() || self.ctx.data_movement_generation_exhausted() {
|
||||
self.ctx.set_scanner_publication_state(true);
|
||||
return true;
|
||||
}
|
||||
let (_, blocked) = self.scanner_data_movement_snapshot_locked().await;
|
||||
self.ctx.set_scanner_publication_state(blocked);
|
||||
blocked
|
||||
}
|
||||
|
||||
async fn scanner_data_movement_snapshot_locked(&self) -> (bool, bool) {
|
||||
let decommission_cancelers = self.decommission_cancelers.read().await;
|
||||
let decommission_active = decommission_cancelers
|
||||
.iter()
|
||||
@@ -385,8 +420,7 @@ impl ECStore {
|
||||
.is_some_and(is_rebalance_conflicting_with_decommission);
|
||||
|
||||
let blocked = decommission_active || decommission_terminal || rebalance_active;
|
||||
self.ctx.set_scanner_publication_state(blocked);
|
||||
blocked
|
||||
(decommission_active || rebalance_active, blocked)
|
||||
}
|
||||
|
||||
/// Admit one short data-usage publication commit under the same
|
||||
@@ -407,7 +441,7 @@ impl ECStore {
|
||||
pub async fn scanner_data_usage_publication_admission_guard(&self) -> Option<(tokio::sync::OwnedRwLockReadGuard<()>, u64)> {
|
||||
let operation_gate = self.ctx.data_movement_operation_gate();
|
||||
let operation_guard = operation_gate.read_owned().await;
|
||||
if self.ctx.data_movement_operation_epoch_exhausted() {
|
||||
if self.ctx.data_movement_operation_epoch_exhausted() || self.ctx.data_movement_generation_exhausted() {
|
||||
return None;
|
||||
}
|
||||
if self.scanner_data_usage_publication_snapshot_blocked().await {
|
||||
@@ -425,6 +459,98 @@ impl ECStore {
|
||||
drop(operation_guard);
|
||||
Some(epoch)
|
||||
}
|
||||
|
||||
/// Acquire a storage-owned read admission for a coordinator's final
|
||||
/// scanner publication. The guard remains in the context's lease table,
|
||||
/// so a local movement writer cannot pass the peer while its authoritative
|
||||
/// PUT is in flight.
|
||||
pub async fn acquire_scanner_publication_lease(
|
||||
&self,
|
||||
expected_generation: u64,
|
||||
ttl: std::time::Duration,
|
||||
) -> Result<(Uuid, u64)> {
|
||||
if ttl != crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL {
|
||||
return Err(Error::other("scanner publication lease TTL is not supported"));
|
||||
}
|
||||
|
||||
let operation_gate = self.ctx.data_movement_operation_gate();
|
||||
let operation_guard = operation_gate.read_owned().await;
|
||||
if self.ctx.data_movement_generation_exhausted()
|
||||
|| self.ctx.data_movement_operation_epoch_exhausted()
|
||||
|| self.ctx.data_movement_generation() != expected_generation
|
||||
{
|
||||
return Err(Error::other("scanner publication lease generation is stale"));
|
||||
}
|
||||
if self.scanner_data_movement_snapshot_locked().await.1 {
|
||||
return Err(Error::other("scanner publication lease is blocked by data movement"));
|
||||
}
|
||||
|
||||
let token = Uuid::new_v4();
|
||||
let expires_at = tokio::time::Instant::now() + ttl;
|
||||
if !self
|
||||
.ctx
|
||||
.install_scanner_publication_lease(token, expires_at, expected_generation, operation_guard)
|
||||
.await
|
||||
{
|
||||
return Err(Error::other("scanner publication lease capacity is exhausted"));
|
||||
}
|
||||
|
||||
let context = Arc::clone(&self.ctx);
|
||||
tokio::spawn(async move {
|
||||
tokio::time::sleep_until(expires_at).await;
|
||||
context.expire_scanner_publication_lease(token, expires_at).await;
|
||||
});
|
||||
Ok((token, expected_generation))
|
||||
}
|
||||
|
||||
pub async fn release_scanner_publication_lease(&self, token: Uuid) -> bool {
|
||||
self.ctx.remove_scanner_publication_lease(token).await
|
||||
}
|
||||
|
||||
/// Revalidate a previously acquired remote publication lease immediately
|
||||
/// before the coordinator's final metadata write. The operation read
|
||||
/// guard makes the movement snapshot and token lookup one storage-owned
|
||||
/// admission; a restarted context has no old token and therefore fails
|
||||
/// closed even if its generation counter has returned to zero.
|
||||
pub async fn validate_scanner_publication_lease(&self, token: Uuid, expected_generation: u64) -> Result<()> {
|
||||
let _operation_guard = self.acquire_scanner_publication_lease_guard(token).await?;
|
||||
if self.ctx.data_movement_generation_exhausted()
|
||||
|| self.ctx.data_movement_operation_epoch_exhausted()
|
||||
|| self.ctx.data_movement_generation() != expected_generation
|
||||
{
|
||||
return Err(Error::other("scanner publication lease generation is stale"));
|
||||
}
|
||||
if self.scanner_data_movement_snapshot_locked().await.1 {
|
||||
return Err(Error::other("scanner publication lease is blocked by data movement"));
|
||||
}
|
||||
if !self.ctx.scanner_publication_lease_is_active(token).await {
|
||||
return Err(Error::other("scanner publication lease is unknown or expired"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Acquire the target-side read guard bound to a previously granted lease.
|
||||
/// The guard is returned to the RPC handler and must remain alive through
|
||||
/// the complete rename/write operation. A lease token is process-owned;
|
||||
/// restart, expiry, generation changes, or blocked movement all reject it
|
||||
/// before the target disk is touched.
|
||||
pub async fn acquire_scanner_publication_lease_guard(&self, token: Uuid) -> Result<tokio::sync::OwnedRwLockReadGuard<()>> {
|
||||
let operation_gate = self.ctx.data_movement_operation_gate();
|
||||
let operation_guard = operation_gate.read_owned().await;
|
||||
if self.ctx.data_movement_generation_exhausted() || self.ctx.data_movement_operation_epoch_exhausted() {
|
||||
return Err(Error::other("scanner publication lease generation is exhausted"));
|
||||
}
|
||||
if self.scanner_data_movement_snapshot_locked().await.1 {
|
||||
return Err(Error::other("scanner publication lease is blocked by data movement"));
|
||||
}
|
||||
let Some(lease_generation) = self.ctx.scanner_publication_lease_generation(token).await else {
|
||||
return Err(Error::other("scanner publication lease is unknown or expired"));
|
||||
};
|
||||
if lease_generation != self.ctx.data_movement_generation() {
|
||||
return Err(Error::other("scanner publication lease generation is stale"));
|
||||
}
|
||||
Ok(operation_guard)
|
||||
}
|
||||
}
|
||||
|
||||
// impl Clone for ECStore {
|
||||
@@ -1084,6 +1210,149 @@ mod tests {
|
||||
.await
|
||||
.expect("idle store should admit the next publication");
|
||||
assert_eq!(next_epoch, 1);
|
||||
assert_eq!(store.scanner_data_movement_generation(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn scanner_publication_lease_blocks_movement_writer_until_release_or_expiry() {
|
||||
let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
|
||||
let (token, generation) = store
|
||||
.acquire_scanner_publication_lease(0, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
|
||||
.await
|
||||
.expect("an idle store should grant a publication lease");
|
||||
assert_eq!(generation, 0);
|
||||
|
||||
let gate = store.ctx.data_movement_operation_gate();
|
||||
let (writer_started, writer_started_rx) = tokio::sync::oneshot::channel();
|
||||
let movement_writer = tokio::spawn(async move {
|
||||
let _ = writer_started.send(());
|
||||
gate.write_owned().await
|
||||
});
|
||||
writer_started_rx
|
||||
.await
|
||||
.expect("movement writer should reach the gate before waiting");
|
||||
assert!(
|
||||
!movement_writer.is_finished(),
|
||||
"a movement writer must wait while the remote lease owns the read guard"
|
||||
);
|
||||
|
||||
assert!(store.release_scanner_publication_lease(token).await);
|
||||
tokio::time::timeout(Duration::from_secs(1), movement_writer)
|
||||
.await
|
||||
.expect("movement writer should proceed after lease release")
|
||||
.expect("movement writer task should not panic");
|
||||
|
||||
let (expiring_token, _) = store
|
||||
.acquire_scanner_publication_lease(0, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
|
||||
.await
|
||||
.expect("the store should grant a second publication lease");
|
||||
let expiry_gate = store.ctx.data_movement_operation_gate();
|
||||
let (expiry_started, expiry_started_rx) = tokio::sync::oneshot::channel();
|
||||
let mut expiry_writer = tokio::spawn(async move {
|
||||
let _ = expiry_started.send(());
|
||||
expiry_gate.write_owned().await
|
||||
});
|
||||
expiry_started_rx
|
||||
.await
|
||||
.expect("expiry writer should reach the gate before waiting");
|
||||
assert!(!expiry_writer.is_finished());
|
||||
tokio::time::advance(crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL + Duration::from_millis(1)).await;
|
||||
tokio::task::yield_now().await;
|
||||
tokio::time::timeout(Duration::from_secs(1), &mut expiry_writer)
|
||||
.await
|
||||
.expect("movement writer should proceed after lease expiry")
|
||||
.expect("expiry writer task should not panic");
|
||||
assert!(!store.release_scanner_publication_lease(expiring_token).await);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_publication_lease_rejects_stale_generation_before_install() {
|
||||
let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
|
||||
let error = store
|
||||
.acquire_scanner_publication_lease(1, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
|
||||
.await
|
||||
.expect_err("a stale movement generation must not acquire a lease");
|
||||
assert!(error.to_string().contains("generation is stale"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_target_guard_keeps_movement_writer_fenced_after_lease_release() {
|
||||
let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
|
||||
let (token, _) = store
|
||||
.acquire_scanner_publication_lease(0, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
|
||||
.await
|
||||
.expect("an idle store should grant a publication lease");
|
||||
let target_guard = store
|
||||
.acquire_scanner_publication_lease_guard(token)
|
||||
.await
|
||||
.expect("the target-side rename should acquire its short guard");
|
||||
assert!(store.release_scanner_publication_lease(token).await);
|
||||
|
||||
let movement_gate = store.ctx.data_movement_operation_gate();
|
||||
let movement_writer = tokio::spawn(async move { movement_gate.write_owned().await });
|
||||
tokio::task::yield_now().await;
|
||||
assert!(!movement_writer.is_finished(), "the target guard must span the rename operation");
|
||||
drop(target_guard);
|
||||
tokio::time::timeout(std::time::Duration::from_secs(1), movement_writer)
|
||||
.await
|
||||
.expect("movement writer should proceed after the target rename guard is dropped")
|
||||
.expect("movement writer task should not panic");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_publication_lease_rejects_restart_aba_token() {
|
||||
let first_store = build_store_with_ctx(Arc::new(InstanceContext::new()));
|
||||
let (token, generation) = first_store
|
||||
.acquire_scanner_publication_lease(0, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
|
||||
.await
|
||||
.expect("the initial instance should grant a publication lease");
|
||||
first_store
|
||||
.validate_scanner_publication_lease(token, generation)
|
||||
.await
|
||||
.expect("the current instance should validate its own live token");
|
||||
first_store
|
||||
.acquire_scanner_publication_lease_guard(token)
|
||||
.await
|
||||
.expect("the current instance should admit the target-side rename");
|
||||
|
||||
// A restarted storage instance starts its local generation at zero,
|
||||
// but its process-owned lease table is empty. The old token must not
|
||||
// pass validation just because the numeric generation matches again.
|
||||
let restarted_store = build_store_with_ctx(Arc::new(InstanceContext::new()));
|
||||
let error = restarted_store
|
||||
.validate_scanner_publication_lease(token, generation)
|
||||
.await
|
||||
.expect_err("a token from a prior instance must fail closed after restart");
|
||||
assert!(error.to_string().contains("unknown or expired"));
|
||||
let error = restarted_store
|
||||
.acquire_scanner_publication_lease_guard(token)
|
||||
.await
|
||||
.expect_err("a restarted instance must reject the target-side rename token");
|
||||
assert!(error.to_string().contains("unknown or expired"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn movement_generation_notifies_waiters_and_fails_closed_at_maximum() {
|
||||
let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
|
||||
let notify = store.scanner_data_movement_changed();
|
||||
let notified = notify.notified();
|
||||
tokio::pin!(notified);
|
||||
notified.as_mut().enable();
|
||||
|
||||
assert_eq!(store.ctx.advance_data_movement_operation_epoch(), 1);
|
||||
tokio::time::timeout(std::time::Duration::from_secs(1), notified)
|
||||
.await
|
||||
.expect("movement transition should wake scanner waiters");
|
||||
assert_eq!(store.scanner_data_movement_generation(), 1);
|
||||
|
||||
store.ctx.set_data_movement_generation_for_test(u64::MAX - 1);
|
||||
assert_eq!(store.ctx.advance_data_movement_generation(), Some(u64::MAX));
|
||||
assert!(store.scanner_data_movement_generation_exhausted());
|
||||
assert_eq!(store.ctx.advance_data_movement_generation(), None);
|
||||
assert!(
|
||||
store.scanner_data_usage_publication_admission_guard().await.is_none(),
|
||||
"generation exhaustion must close publication admission"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -142,6 +142,9 @@ pub struct DeleteRequest {
|
||||
pub path: ::prost::alloc::string::String,
|
||||
#[prost(string, tag = "4")]
|
||||
pub options: ::prost::alloc::string::String,
|
||||
/// Optional scanner publication lease token.
|
||||
#[prost(bytes = "bytes", tag = "5")]
|
||||
pub scanner_publication_lease_token: ::prost::bytes::Bytes,
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct DeleteResponse {
|
||||
@@ -393,6 +396,8 @@ pub struct RenameDataRequest {
|
||||
pub dst_path: ::prost::alloc::string::String,
|
||||
#[prost(bytes = "bytes", tag = "7")]
|
||||
pub file_info_bin: ::prost::bytes::Bytes,
|
||||
#[prost(bytes = "bytes", tag = "8")]
|
||||
pub scanner_publication_lease_token: ::prost::bytes::Bytes,
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct RenameDataResponse {
|
||||
@@ -1213,6 +1218,62 @@ pub struct ScannerActivityResponse {
|
||||
pub dirty_usage_generation: u64,
|
||||
#[prost(bool, tag = "9")]
|
||||
pub dirty_usage_pending: bool,
|
||||
#[prost(uint64, optional, tag = "10")]
|
||||
pub movement_generation: ::core::option::Option<u64>,
|
||||
#[prost(bool, optional, tag = "11")]
|
||||
pub publication_blocked: ::core::option::Option<bool>,
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct ScannerPublicationLeaseRequest {
|
||||
#[prost(bytes = "bytes", tag = "1")]
|
||||
pub challenge: ::prost::bytes::Bytes,
|
||||
#[prost(uint64, tag = "2")]
|
||||
pub expected_movement_generation: u64,
|
||||
#[prost(uint64, tag = "3")]
|
||||
pub ttl_ms: u64,
|
||||
#[prost(string, tag = "4")]
|
||||
pub expected_session_id: ::prost::alloc::string::String,
|
||||
#[prost(bytes = "bytes", tag = "5")]
|
||||
pub token: ::prost::bytes::Bytes,
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct ScannerPublicationLeaseResponse {
|
||||
#[prost(bool, tag = "1")]
|
||||
pub success: bool,
|
||||
#[prost(bytes = "bytes", tag = "2")]
|
||||
pub token: ::prost::bytes::Bytes,
|
||||
#[prost(uint64, tag = "3")]
|
||||
pub movement_generation: u64,
|
||||
#[prost(uint64, tag = "4")]
|
||||
pub lease_ttl_ms: u64,
|
||||
#[prost(message, optional, tag = "5")]
|
||||
pub error: ::core::option::Option<Error>,
|
||||
#[prost(bytes = "bytes", tag = "6")]
|
||||
pub response_proof: ::prost::bytes::Bytes,
|
||||
#[prost(string, tag = "7")]
|
||||
pub owner_id: ::prost::alloc::string::String,
|
||||
#[prost(string, tag = "8")]
|
||||
pub session_id: ::prost::alloc::string::String,
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct ScannerPublicationLeaseReleaseRequest {
|
||||
#[prost(bytes = "bytes", tag = "1")]
|
||||
pub challenge: ::prost::bytes::Bytes,
|
||||
#[prost(bytes = "bytes", tag = "2")]
|
||||
pub token: ::prost::bytes::Bytes,
|
||||
#[prost(string, tag = "3")]
|
||||
pub owner_id: ::prost::alloc::string::String,
|
||||
#[prost(string, tag = "4")]
|
||||
pub session_id: ::prost::alloc::string::String,
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct ScannerPublicationLeaseReleaseResponse {
|
||||
#[prost(bool, tag = "1")]
|
||||
pub success: bool,
|
||||
#[prost(message, optional, tag = "2")]
|
||||
pub error: ::core::option::Option<Error>,
|
||||
#[prost(bytes = "bytes", tag = "3")]
|
||||
pub response_proof: ::prost::bytes::Bytes,
|
||||
}
|
||||
#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct BackgroundHealStatusRequest {
|
||||
@@ -2695,6 +2756,36 @@ pub mod node_service_client {
|
||||
.insert(GrpcMethod::new("node_service.NodeService", "ScannerActivity"));
|
||||
self.inner.unary(req, path, codec).await
|
||||
}
|
||||
pub async fn acquire_scanner_publication_lease(
|
||||
&mut self,
|
||||
request: impl tonic::IntoRequest<super::ScannerPublicationLeaseRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::ScannerPublicationLeaseResponse>, tonic::Status> {
|
||||
self.inner
|
||||
.ready()
|
||||
.await
|
||||
.map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?;
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/AcquireScannerPublicationLease");
|
||||
let mut req = request.into_request();
|
||||
req.extensions_mut()
|
||||
.insert(GrpcMethod::new("node_service.NodeService", "AcquireScannerPublicationLease"));
|
||||
self.inner.unary(req, path, codec).await
|
||||
}
|
||||
pub async fn release_scanner_publication_lease(
|
||||
&mut self,
|
||||
request: impl tonic::IntoRequest<super::ScannerPublicationLeaseReleaseRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::ScannerPublicationLeaseReleaseResponse>, tonic::Status> {
|
||||
self.inner
|
||||
.ready()
|
||||
.await
|
||||
.map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?;
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/ReleaseScannerPublicationLease");
|
||||
let mut req = request.into_request();
|
||||
req.extensions_mut()
|
||||
.insert(GrpcMethod::new("node_service.NodeService", "ReleaseScannerPublicationLease"));
|
||||
self.inner.unary(req, path, codec).await
|
||||
}
|
||||
pub async fn background_heal_status(
|
||||
&mut self,
|
||||
request: impl tonic::IntoRequest<super::BackgroundHealStatusRequest>,
|
||||
@@ -3208,6 +3299,18 @@ pub mod node_service_server {
|
||||
&self,
|
||||
request: tonic::Request<super::ScannerActivityRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::ScannerActivityResponse>, tonic::Status>;
|
||||
async fn acquire_scanner_publication_lease(
|
||||
&self,
|
||||
_request: tonic::Request<super::ScannerPublicationLeaseRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::ScannerPublicationLeaseResponse>, tonic::Status> {
|
||||
Err(tonic::Status::unimplemented("scanner publication leases are unsupported"))
|
||||
}
|
||||
async fn release_scanner_publication_lease(
|
||||
&self,
|
||||
_request: tonic::Request<super::ScannerPublicationLeaseReleaseRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::ScannerPublicationLeaseReleaseResponse>, tonic::Status> {
|
||||
Err(tonic::Status::unimplemented("scanner publication leases are unsupported"))
|
||||
}
|
||||
async fn background_heal_status(
|
||||
&self,
|
||||
request: tonic::Request<super::BackgroundHealStatusRequest>,
|
||||
@@ -5488,6 +5591,67 @@ pub mod node_service_server {
|
||||
};
|
||||
Box::pin(fut)
|
||||
}
|
||||
"/node_service.NodeService/AcquireScannerPublicationLease" => {
|
||||
#[allow(non_camel_case_types)]
|
||||
struct AcquireScannerPublicationLeaseSvc<T: NodeService>(pub Arc<T>);
|
||||
impl<T: NodeService> tonic::server::UnaryService<super::ScannerPublicationLeaseRequest> for AcquireScannerPublicationLeaseSvc<T> {
|
||||
type Response = super::ScannerPublicationLeaseResponse;
|
||||
type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
|
||||
fn call(&mut self, request: tonic::Request<super::ScannerPublicationLeaseRequest>) -> Self::Future {
|
||||
let inner = Arc::clone(&self.0);
|
||||
let fut = async move { <T as NodeService>::acquire_scanner_publication_lease(&inner, request).await };
|
||||
Box::pin(fut)
|
||||
}
|
||||
}
|
||||
let accept_compression_encodings = self.accept_compression_encodings;
|
||||
let send_compression_encodings = self.send_compression_encodings;
|
||||
let max_decoding_message_size = self.max_decoding_message_size;
|
||||
let max_encoding_message_size = self.max_encoding_message_size;
|
||||
let inner = self.inner.clone();
|
||||
let fut = async move {
|
||||
let method = AcquireScannerPublicationLeaseSvc(inner);
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let mut grpc = tonic::server::Grpc::new(codec)
|
||||
.apply_compression_config(accept_compression_encodings, send_compression_encodings)
|
||||
.apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size);
|
||||
let res = grpc.unary(method, req).await;
|
||||
Ok(res)
|
||||
};
|
||||
Box::pin(fut)
|
||||
}
|
||||
"/node_service.NodeService/ReleaseScannerPublicationLease" => {
|
||||
#[allow(non_camel_case_types)]
|
||||
struct ReleaseScannerPublicationLeaseSvc<T: NodeService>(pub Arc<T>);
|
||||
impl<T: NodeService> tonic::server::UnaryService<super::ScannerPublicationLeaseReleaseRequest>
|
||||
for ReleaseScannerPublicationLeaseSvc<T>
|
||||
{
|
||||
type Response = super::ScannerPublicationLeaseReleaseResponse;
|
||||
type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
|
||||
fn call(
|
||||
&mut self,
|
||||
request: tonic::Request<super::ScannerPublicationLeaseReleaseRequest>,
|
||||
) -> Self::Future {
|
||||
let inner = Arc::clone(&self.0);
|
||||
let fut = async move { <T as NodeService>::release_scanner_publication_lease(&inner, request).await };
|
||||
Box::pin(fut)
|
||||
}
|
||||
}
|
||||
let accept_compression_encodings = self.accept_compression_encodings;
|
||||
let send_compression_encodings = self.send_compression_encodings;
|
||||
let max_decoding_message_size = self.max_decoding_message_size;
|
||||
let max_encoding_message_size = self.max_encoding_message_size;
|
||||
let inner = self.inner.clone();
|
||||
let fut = async move {
|
||||
let method = ReleaseScannerPublicationLeaseSvc(inner);
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let mut grpc = tonic::server::Grpc::new(codec)
|
||||
.apply_compression_config(accept_compression_encodings, send_compression_encodings)
|
||||
.apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size);
|
||||
let res = grpc.unary(method, req).await;
|
||||
Ok(res)
|
||||
};
|
||||
Box::pin(fut)
|
||||
}
|
||||
"/node_service.NodeService/BackgroundHealStatus" => {
|
||||
#[allow(non_camel_case_types)]
|
||||
struct BackgroundHealStatusSvc<T: NodeService>(pub Arc<T>);
|
||||
|
||||
+261
-2
@@ -486,6 +486,160 @@ pub fn canonical_scanner_activity_response_body(
|
||||
Ok(body)
|
||||
}
|
||||
|
||||
/// Builds the protocol-v7 response body. The optional movement fields are
|
||||
/// presence-bound so a missing terminal-generation proof cannot authenticate
|
||||
/// as the value zero.
|
||||
pub fn canonical_scanner_activity_v7_response_body(
|
||||
challenge: &[u8],
|
||||
response: &proto_gen::node_service::ScannerActivityResponse,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
const DOMAIN: &[u8] = b"rustfs-scanner-activity-response-v3\0";
|
||||
|
||||
let instance_id = response.instance_id.as_bytes();
|
||||
let topology_digest = response.topology_digest.as_ref();
|
||||
let mut body = Vec::with_capacity(DOMAIN.len() + challenge.len() + instance_id.len() + topology_digest.len() + 4 + 8 * 8 + 4);
|
||||
body.extend_from_slice(DOMAIN);
|
||||
body.extend_from_slice(&u64::try_from(challenge.len())?.to_be_bytes());
|
||||
body.extend_from_slice(challenge);
|
||||
body.extend_from_slice(&u64::try_from(instance_id.len())?.to_be_bytes());
|
||||
body.extend_from_slice(instance_id);
|
||||
body.extend_from_slice(&response.namespace_generation.to_be_bytes());
|
||||
body.extend_from_slice(&response.maintenance_generation.to_be_bytes());
|
||||
body.extend_from_slice(&response.protocol_version.to_be_bytes());
|
||||
body.extend_from_slice(&u64::try_from(topology_digest.len())?.to_be_bytes());
|
||||
body.extend_from_slice(topology_digest);
|
||||
body.push(u8::from(response.data_movement_active));
|
||||
body.extend_from_slice(&response.dirty_usage_generation.to_be_bytes());
|
||||
body.push(u8::from(response.dirty_usage_pending));
|
||||
body.push(u8::from(response.movement_generation.is_some()));
|
||||
if let Some(generation) = response.movement_generation {
|
||||
body.extend_from_slice(&generation.to_be_bytes());
|
||||
}
|
||||
body.push(u8::from(response.publication_blocked.is_some()));
|
||||
if let Some(blocked) = response.publication_blocked {
|
||||
body.push(u8::from(blocked));
|
||||
}
|
||||
Ok(body)
|
||||
}
|
||||
|
||||
/// Builds the body authenticated by the short-lived remote scanner publication
|
||||
/// lease request. This is a separate domain from ScannerActivity so v6/v7
|
||||
/// observation proofs remain byte-for-byte compatible.
|
||||
pub fn canonical_scanner_publication_lease_request_body(
|
||||
request: &proto_gen::node_service::ScannerPublicationLeaseRequest,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
const DOMAIN: &[u8] = b"rustfs-scanner-publication-lease-request-v1\0";
|
||||
let challenge = request.challenge.as_ref();
|
||||
let session_id = request.expected_session_id.as_bytes();
|
||||
let mut body = Vec::with_capacity(DOMAIN.len() + challenge.len() + session_id.len() + 40);
|
||||
body.extend_from_slice(DOMAIN);
|
||||
body.extend_from_slice(&u64::try_from(challenge.len())?.to_be_bytes());
|
||||
body.extend_from_slice(challenge);
|
||||
body.extend_from_slice(&request.expected_movement_generation.to_be_bytes());
|
||||
body.extend_from_slice(&request.ttl_ms.to_be_bytes());
|
||||
body.extend_from_slice(&u64::try_from(session_id.len())?.to_be_bytes());
|
||||
body.extend_from_slice(session_id);
|
||||
// Empty keeps the original acquire body byte-for-byte compatible. A
|
||||
// non-empty token is the authenticated validation form used immediately
|
||||
// before a coordinator commits its final publication.
|
||||
let token = request.token.as_ref();
|
||||
if !token.is_empty() {
|
||||
body.extend_from_slice(&u64::try_from(token.len())?.to_be_bytes());
|
||||
body.extend_from_slice(token);
|
||||
}
|
||||
Ok(body)
|
||||
}
|
||||
|
||||
/// Builds the body authenticated by a remote scanner publication lease
|
||||
/// release request.
|
||||
pub fn canonical_scanner_publication_lease_release_request_body(
|
||||
request: &proto_gen::node_service::ScannerPublicationLeaseReleaseRequest,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
const DOMAIN: &[u8] = b"rustfs-scanner-publication-lease-release-request-v1\0";
|
||||
let challenge = request.challenge.as_ref();
|
||||
let token = request.token.as_ref();
|
||||
let mut body = Vec::with_capacity(DOMAIN.len() + challenge.len() + token.len() + 16);
|
||||
body.extend_from_slice(DOMAIN);
|
||||
body.extend_from_slice(&u64::try_from(challenge.len())?.to_be_bytes());
|
||||
body.extend_from_slice(challenge);
|
||||
body.extend_from_slice(&u64::try_from(token.len())?.to_be_bytes());
|
||||
body.extend_from_slice(token);
|
||||
let owner_id = request.owner_id.as_bytes();
|
||||
let session_id = request.session_id.as_bytes();
|
||||
body.extend_from_slice(&u64::try_from(owner_id.len())?.to_be_bytes());
|
||||
body.extend_from_slice(owner_id);
|
||||
body.extend_from_slice(&u64::try_from(session_id.len())?.to_be_bytes());
|
||||
body.extend_from_slice(session_id);
|
||||
Ok(body)
|
||||
}
|
||||
|
||||
pub fn canonical_scanner_publication_lease_response_body(
|
||||
challenge: &[u8],
|
||||
response: &proto_gen::node_service::ScannerPublicationLeaseResponse,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
const DOMAIN: &[u8] = b"rustfs-scanner-publication-lease-response-v1\0";
|
||||
let token = response.token.as_ref();
|
||||
let owner_id = response.owner_id.as_bytes();
|
||||
let session_id = response.session_id.as_bytes();
|
||||
let error_info = response.error.as_ref().map(|error| error.error_info.as_bytes());
|
||||
let error_code = response.error.as_ref().map_or(0, |error| error.code);
|
||||
let mut body = Vec::with_capacity(
|
||||
DOMAIN.len() + challenge.len() + token.len() + owner_id.len() + session_id.len() + error_info.map_or(0, |v| v.len()) + 72,
|
||||
);
|
||||
body.extend_from_slice(DOMAIN);
|
||||
body.extend_from_slice(&u64::try_from(challenge.len())?.to_be_bytes());
|
||||
body.extend_from_slice(challenge);
|
||||
body.push(u8::from(response.success));
|
||||
body.extend_from_slice(&u64::try_from(token.len())?.to_be_bytes());
|
||||
body.extend_from_slice(token);
|
||||
body.extend_from_slice(&response.movement_generation.to_be_bytes());
|
||||
body.extend_from_slice(&response.lease_ttl_ms.to_be_bytes());
|
||||
body.extend_from_slice(&u64::try_from(owner_id.len())?.to_be_bytes());
|
||||
body.extend_from_slice(owner_id);
|
||||
body.extend_from_slice(&u64::try_from(session_id.len())?.to_be_bytes());
|
||||
body.extend_from_slice(session_id);
|
||||
body.push(u8::from(response.error.is_some()));
|
||||
body.extend_from_slice(&error_code.to_be_bytes());
|
||||
body.extend_from_slice(&u64::try_from(error_info.map_or(0, |value| value.len()))?.to_be_bytes());
|
||||
if let Some(error_info) = error_info {
|
||||
body.extend_from_slice(error_info);
|
||||
}
|
||||
Ok(body)
|
||||
}
|
||||
|
||||
pub fn canonical_scanner_publication_lease_release_response_body(
|
||||
challenge: &[u8],
|
||||
request: &proto_gen::node_service::ScannerPublicationLeaseReleaseRequest,
|
||||
response: &proto_gen::node_service::ScannerPublicationLeaseReleaseResponse,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
const DOMAIN: &[u8] = b"rustfs-scanner-publication-lease-release-response-v1\0";
|
||||
let token = request.token.as_ref();
|
||||
let owner_id = request.owner_id.as_bytes();
|
||||
let session_id = request.session_id.as_bytes();
|
||||
let error_info = response.error.as_ref().map(|error| error.error_info.as_bytes());
|
||||
let error_code = response.error.as_ref().map_or(0, |error| error.code);
|
||||
let mut body = Vec::with_capacity(
|
||||
DOMAIN.len() + challenge.len() + token.len() + owner_id.len() + session_id.len() + error_info.map_or(0, |v| v.len()) + 56,
|
||||
);
|
||||
body.extend_from_slice(DOMAIN);
|
||||
body.extend_from_slice(&u64::try_from(challenge.len())?.to_be_bytes());
|
||||
body.extend_from_slice(challenge);
|
||||
body.extend_from_slice(&u64::try_from(token.len())?.to_be_bytes());
|
||||
body.extend_from_slice(token);
|
||||
body.extend_from_slice(&u64::try_from(owner_id.len())?.to_be_bytes());
|
||||
body.extend_from_slice(owner_id);
|
||||
body.extend_from_slice(&u64::try_from(session_id.len())?.to_be_bytes());
|
||||
body.extend_from_slice(session_id);
|
||||
body.push(u8::from(response.success));
|
||||
body.push(u8::from(response.error.is_some()));
|
||||
body.extend_from_slice(&error_code.to_be_bytes());
|
||||
body.extend_from_slice(&u64::try_from(error_info.map_or(0, |value| value.len()))?.to_be_bytes());
|
||||
if let Some(error_info) = error_info {
|
||||
body.extend_from_slice(error_info);
|
||||
}
|
||||
Ok(body)
|
||||
}
|
||||
|
||||
/// Length-prefixed, domain-separated byte builder for the disk-mutation canonical bodies below.
|
||||
/// Every variable-length field is u64-length-prefixed and every list u64-count-prefixed, so
|
||||
/// distinct field values can never collide into the same canonical bytes.
|
||||
@@ -758,6 +912,11 @@ pub fn canonical_rename_data_request_body(
|
||||
body.push_str(&request.dst_volume)?;
|
||||
body.push_str(&request.dst_path)?;
|
||||
body.push_bytes(&request.file_info_bin)?;
|
||||
// Keep legacy rename requests byte-for-byte compatible. The optional
|
||||
// token is included only for the scanner's target-side lease fence.
|
||||
if !request.scanner_publication_lease_token.is_empty() {
|
||||
body.push_bytes(&request.scanner_publication_lease_token)?;
|
||||
}
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
@@ -840,6 +999,9 @@ pub fn canonical_delete_request_body(
|
||||
body.push_str(&request.volume)?;
|
||||
body.push_str(&request.path)?;
|
||||
body.push_str(&request.options)?;
|
||||
if !request.scanner_publication_lease_token.is_empty() {
|
||||
body.push_bytes(&request.scanner_publication_lease_token)?;
|
||||
}
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
@@ -1002,6 +1164,7 @@ mod disk_mutation_canonical_tests {
|
||||
dst_volume: "dst-vol".into(),
|
||||
dst_path: "dst-path".into(),
|
||||
file_info_bin: vec![0x81, 0x01].into(),
|
||||
scanner_publication_lease_token: Vec::new().into(),
|
||||
};
|
||||
let mut bodies = vec![canonical_rename_data_request_body(&baseline).unwrap()];
|
||||
for mutate in [
|
||||
@@ -1013,6 +1176,7 @@ mod disk_mutation_canonical_tests {
|
||||
|r: &mut RenameDataRequest| r.dst_path = "dst-path2".into(),
|
||||
|r: &mut RenameDataRequest| r.file_info_bin = vec![0x81, 0x02].into(),
|
||||
|r: &mut RenameDataRequest| r.file_info_bin = Vec::new().into(),
|
||||
|r: &mut RenameDataRequest| r.scanner_publication_lease_token = vec![0x01; 16].into(),
|
||||
] {
|
||||
let mut request = baseline.clone();
|
||||
mutate(&mut request);
|
||||
@@ -1173,6 +1337,7 @@ mod disk_mutation_canonical_tests {
|
||||
volume: "v".into(),
|
||||
path: "p".into(),
|
||||
options: "{\"o\":1}".into(),
|
||||
scanner_publication_lease_token: Vec::new().into(),
|
||||
};
|
||||
let mut bodies = vec![canonical_delete_request_body(&delete).unwrap()];
|
||||
for mutate in [
|
||||
@@ -1180,6 +1345,7 @@ mod disk_mutation_canonical_tests {
|
||||
|r: &mut DeleteRequest| r.volume = "v2".into(),
|
||||
|r: &mut DeleteRequest| r.path = "p2".into(),
|
||||
|r: &mut DeleteRequest| r.options = "{\"recursive\":true}".into(),
|
||||
|r: &mut DeleteRequest| r.scanner_publication_lease_token = vec![0x01; 16].into(),
|
||||
] {
|
||||
let mut request = delete.clone();
|
||||
mutate(&mut request);
|
||||
@@ -1565,8 +1731,12 @@ mod non_disk_mutation_canonical_tests {
|
||||
mod scanner_activity_tests {
|
||||
use super::{
|
||||
canonical_scanner_activity_request_body, canonical_scanner_activity_response_body,
|
||||
canonical_scanner_activity_v4_response_body,
|
||||
proto_gen::node_service::{ScannerActivityRequest, ScannerActivityResponse},
|
||||
canonical_scanner_activity_v4_response_body, canonical_scanner_activity_v7_response_body,
|
||||
canonical_scanner_publication_lease_release_request_body, canonical_scanner_publication_lease_request_body,
|
||||
canonical_scanner_publication_lease_response_body,
|
||||
proto_gen::node_service::{
|
||||
ScannerActivityRequest, ScannerActivityResponse, ScannerPublicationLeaseRequest, ScannerPublicationLeaseResponse,
|
||||
},
|
||||
};
|
||||
|
||||
#[test]
|
||||
@@ -1617,6 +1787,8 @@ mod scanner_activity_tests {
|
||||
response_proof: Vec::new().into(),
|
||||
dirty_usage_generation: 11,
|
||||
dirty_usage_pending: true,
|
||||
movement_generation: Some(19),
|
||||
publication_blocked: Some(false),
|
||||
};
|
||||
let baseline =
|
||||
canonical_scanner_activity_response_body(&[1; 16], &response).expect("scanner activity response should encode");
|
||||
@@ -1667,6 +1839,25 @@ mod scanner_activity_tests {
|
||||
canonical_scanner_activity_response_body(&[2; 16], &response)
|
||||
.expect("scanner activity response with a different challenge should encode")
|
||||
);
|
||||
|
||||
let v7_baseline =
|
||||
canonical_scanner_activity_v7_response_body(&[1; 16], &response).expect("scanner activity v7 response should encode");
|
||||
for variant in [
|
||||
ScannerActivityResponse {
|
||||
movement_generation: Some(20),
|
||||
..response.clone()
|
||||
},
|
||||
ScannerActivityResponse {
|
||||
publication_blocked: Some(true),
|
||||
..response
|
||||
},
|
||||
] {
|
||||
assert_ne!(
|
||||
v7_baseline,
|
||||
canonical_scanner_activity_v7_response_body(&[1; 16], &variant)
|
||||
.expect("scanner activity v7 response variant should encode")
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1681,6 +1872,8 @@ mod scanner_activity_tests {
|
||||
response_proof: Vec::new().into(),
|
||||
dirty_usage_generation: 0,
|
||||
dirty_usage_pending: false,
|
||||
movement_generation: None,
|
||||
publication_blocked: None,
|
||||
};
|
||||
let baseline =
|
||||
canonical_scanner_activity_v4_response_body(&[1; 16], &response).expect("scanner activity v4 response should encode");
|
||||
@@ -1714,6 +1907,72 @@ mod scanner_activity_tests {
|
||||
.expect("scanner activity v4 response should ignore v5 fields")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_publication_lease_canonical_bodies_bind_session_owner_and_generation() {
|
||||
let request = ScannerPublicationLeaseRequest {
|
||||
challenge: vec![1; 16].into(),
|
||||
expected_movement_generation: 7,
|
||||
ttl_ms: 60_000,
|
||||
expected_session_id: "session-a".to_string(),
|
||||
token: Vec::new().into(),
|
||||
};
|
||||
let baseline = canonical_scanner_publication_lease_request_body(&request).unwrap();
|
||||
for variant in [
|
||||
ScannerPublicationLeaseRequest {
|
||||
expected_movement_generation: 8,
|
||||
..request.clone()
|
||||
},
|
||||
ScannerPublicationLeaseRequest {
|
||||
expected_session_id: "session-b".to_string(),
|
||||
..request.clone()
|
||||
},
|
||||
ScannerPublicationLeaseRequest {
|
||||
ttl_ms: 30_000,
|
||||
..request.clone()
|
||||
},
|
||||
ScannerPublicationLeaseRequest {
|
||||
token: vec![3; 16].into(),
|
||||
..request
|
||||
},
|
||||
] {
|
||||
assert_ne!(baseline, canonical_scanner_publication_lease_request_body(&variant).unwrap());
|
||||
}
|
||||
|
||||
let release_a = crate::proto_gen::node_service::ScannerPublicationLeaseReleaseRequest {
|
||||
challenge: vec![2; 16].into(),
|
||||
token: vec![3; 16].into(),
|
||||
owner_id: "owner-a".to_string(),
|
||||
session_id: "session-a".to_string(),
|
||||
};
|
||||
let release_b = crate::proto_gen::node_service::ScannerPublicationLeaseReleaseRequest {
|
||||
owner_id: "owner-b".to_string(),
|
||||
..release_a.clone()
|
||||
};
|
||||
assert_ne!(
|
||||
canonical_scanner_publication_lease_release_request_body(&release_a).unwrap(),
|
||||
canonical_scanner_publication_lease_release_request_body(&release_b).unwrap()
|
||||
);
|
||||
|
||||
let response = ScannerPublicationLeaseResponse {
|
||||
success: true,
|
||||
token: vec![4; 16].into(),
|
||||
movement_generation: 7,
|
||||
lease_ttl_ms: 60_000,
|
||||
error: None,
|
||||
response_proof: Vec::new().into(),
|
||||
owner_id: "owner-a".to_string(),
|
||||
session_id: "session-a".to_string(),
|
||||
};
|
||||
let response_changed = ScannerPublicationLeaseResponse {
|
||||
owner_id: "owner-b".to_string(),
|
||||
..response.clone()
|
||||
};
|
||||
assert_ne!(
|
||||
canonical_scanner_publication_lease_response_body(&[1; 16], &response).unwrap(),
|
||||
canonical_scanner_publication_lease_response_body(&[1; 16], &response_changed).unwrap()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -111,6 +111,9 @@ message DeleteRequest {
|
||||
string volume = 2;
|
||||
string path = 3;
|
||||
string options = 4;
|
||||
// Optional scanner publication lease token. When present, the target binds
|
||||
// the complete delete operation to its movement read admission.
|
||||
bytes scanner_publication_lease_token = 5;
|
||||
}
|
||||
|
||||
message DeleteResponse {
|
||||
@@ -281,6 +284,10 @@ message RenameDataRequest {
|
||||
string dst_volume = 5;
|
||||
string dst_path = 6;
|
||||
bytes file_info_bin = 7;
|
||||
// Optional target-side scanner publication lease. Empty preserves the
|
||||
// legacy rename request body; a non-empty token is checked at the target's
|
||||
// rename linearization point.
|
||||
bytes scanner_publication_lease_token = 8;
|
||||
}
|
||||
|
||||
message RenameDataResponse {
|
||||
@@ -844,6 +851,52 @@ message ScannerActivityResponse {
|
||||
bytes response_proof = 7;
|
||||
uint64 dirty_usage_generation = 8;
|
||||
bool dirty_usage_pending = 9;
|
||||
// v7 fields. They are optional so v6 peers can continue to decode the
|
||||
// response shape while newer readers fail closed when they are absent.
|
||||
optional uint64 movement_generation = 10;
|
||||
optional bool publication_blocked = 11;
|
||||
}
|
||||
|
||||
// A short-lived storage-owned read admission used only around a final
|
||||
// scanner metadata publication. It is intentionally separate from the
|
||||
// ScannerActivity observation wire so v6/v7 rolling compatibility remains
|
||||
// unchanged.
|
||||
message ScannerPublicationLeaseRequest {
|
||||
bytes challenge = 1;
|
||||
uint64 expected_movement_generation = 2;
|
||||
uint64 ttl_ms = 3;
|
||||
// The activity instance is a process session nonce. It is intentionally
|
||||
// separate from the storage-owned deployment identity returned by the
|
||||
// lease response so a restart cannot reuse an old session token.
|
||||
string expected_session_id = 4;
|
||||
// A non-empty token turns the acquire RPC into an in-place validation of an
|
||||
// existing lease. Keeping this on the existing RPC lets old peers reject
|
||||
// the proof without changing the v7 activity wire shape.
|
||||
bytes token = 5;
|
||||
}
|
||||
|
||||
message ScannerPublicationLeaseResponse {
|
||||
bool success = 1;
|
||||
bytes token = 2;
|
||||
uint64 movement_generation = 3;
|
||||
uint64 lease_ttl_ms = 4;
|
||||
optional Error error = 5;
|
||||
bytes response_proof = 6;
|
||||
string owner_id = 7;
|
||||
string session_id = 8;
|
||||
}
|
||||
|
||||
message ScannerPublicationLeaseReleaseRequest {
|
||||
bytes challenge = 1;
|
||||
bytes token = 2;
|
||||
string owner_id = 3;
|
||||
string session_id = 4;
|
||||
}
|
||||
|
||||
message ScannerPublicationLeaseReleaseResponse {
|
||||
bool success = 1;
|
||||
optional Error error = 2;
|
||||
bytes response_proof = 3;
|
||||
}
|
||||
|
||||
message BackgroundHealStatusRequest {
|
||||
@@ -1096,6 +1149,8 @@ service NodeService {
|
||||
// rpc CommitBinary() returns () {};
|
||||
rpc SignalService(SignalServiceRequest) returns (SignalServiceResponse) {}; // auth-policy: body-bound
|
||||
rpc ScannerActivity(ScannerActivityRequest) returns (ScannerActivityResponse) {}; // auth-policy: body-bound
|
||||
rpc AcquireScannerPublicationLease(ScannerPublicationLeaseRequest) returns (ScannerPublicationLeaseResponse) {}; // auth-policy: body-bound
|
||||
rpc ReleaseScannerPublicationLease(ScannerPublicationLeaseReleaseRequest) returns (ScannerPublicationLeaseReleaseResponse) {}; // auth-policy: body-bound
|
||||
rpc BackgroundHealStatus(BackgroundHealStatusRequest) returns (BackgroundHealStatusResponse) {}; // auth-policy: read-only
|
||||
rpc ReplacementRecoveryStatus(ReplacementRecoveryStatusRequest) returns (ReplacementRecoveryStatusResponse) {}; // auth-policy: read-only
|
||||
rpc GetMetacacheListing(GetMetacacheListingRequest) returns (GetMetacacheListingResponse) {}; // auth-policy: unimplemented
|
||||
|
||||
@@ -92,7 +92,7 @@ pub use scanner_io::{
|
||||
pub use sleeper::{DynamicSleeper, SCANNER_IDLE_MODE, SCANNER_SLEEPER};
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
pub use storage_api::ScannerReplicationConfig as ReplicationConfig;
|
||||
pub use storage_api::scan::SCANNER_ACTIVITY_PROTOCOL_VERSION;
|
||||
pub use storage_api::scan::{SCANNER_ACTIVITY_PROTOCOL_VERSION, SCANNER_ACTIVITY_V6_PROTOCOL_VERSION};
|
||||
|
||||
static SCANNER_ACTIVE_WORK_UNITS: AtomicU64 = AtomicU64::new(0);
|
||||
static SCANNER_RUNTIME_INSTANCES: AtomicU64 = AtomicU64::new(0);
|
||||
@@ -796,17 +796,25 @@ where
|
||||
Some(admission)
|
||||
}
|
||||
|
||||
pub(crate) async fn save_config_shared_with_preconditions<S>(
|
||||
pub(crate) async fn save_config_shared_with_preconditions_and_lease_fence<S>(
|
||||
api: Arc<S>,
|
||||
file: &str,
|
||||
data: Bytes,
|
||||
sha256hex: Option<String>,
|
||||
preconditions: HTTPPreconditions,
|
||||
scanner_publication_lease_fence: Option<&str>,
|
||||
) -> EcstoreResult<ScannerObjectInfo>
|
||||
where
|
||||
S: ScannerObjectIO,
|
||||
{
|
||||
let mut reader = ScannerPutObjReader::from_prehashed_bytes(data, sha256hex)?;
|
||||
let mut user_defined = HashMap::new();
|
||||
if let Some(fence) = scanner_publication_lease_fence {
|
||||
user_defined.insert(
|
||||
storage_api::owner::SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY.to_string(),
|
||||
fence.to_string(),
|
||||
);
|
||||
}
|
||||
api.put_object(
|
||||
RUSTFS_META_BUCKET,
|
||||
file,
|
||||
@@ -814,6 +822,7 @@ where
|
||||
&ScannerObjectOptions {
|
||||
max_parity: true,
|
||||
http_preconditions: Some(preconditions),
|
||||
user_defined,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
|
||||
+159
-13
@@ -63,6 +63,7 @@ use tokio_util::sync::CancellationToken;
|
||||
use tokio_util::task::AbortOnDropHandle;
|
||||
use tracing::{debug, error, info, instrument, warn};
|
||||
|
||||
use crate::storage_api::owner::SCANNER_PUBLICATION_LEASE_TTL_MS;
|
||||
use crate::storage_api::scan::{
|
||||
BucketOperations, BucketOptions, NamespaceLocking as _, SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION,
|
||||
SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SCANNER_ACTIVITY_PROTOCOL_VERSION,
|
||||
@@ -71,9 +72,9 @@ use crate::{
|
||||
ECStore, EcstoreError, RUSTFS_META_BUCKET, SCANNER_PUBLICATION_EPOCH_CHANGED, ScannerLifecycleConfigExt as _,
|
||||
ScannerReplicationConfigExt as _, delete_config_with_publication_admission_for_epoch, get_lifecycle_config,
|
||||
get_replication_config, invalidate_admin_data_usage_snapshot_cache, invalidate_data_usage_snapshot_cache, read_config,
|
||||
replace_bucket_usage_memory_from_info, save_config, save_config_shared_with_preconditions, save_config_with_preconditions,
|
||||
save_config_with_publication_admission_for_epoch, scanner_is_erasure_sd, scanner_publication_admission_for_epoch,
|
||||
scanner_publication_epoch, scanner_publication_epoch_changed,
|
||||
replace_bucket_usage_memory_from_info, save_config, save_config_shared_with_preconditions_and_lease_fence,
|
||||
save_config_with_preconditions, save_config_with_publication_admission_for_epoch, scanner_is_erasure_sd,
|
||||
scanner_publication_admission_for_epoch, scanner_publication_epoch, scanner_publication_epoch_changed,
|
||||
};
|
||||
|
||||
const LOG_COMPONENT_SCANNER: &str = "scanner";
|
||||
@@ -125,6 +126,8 @@ const MAINTENANCE_FEATURE_INSPECTION_RETRY_MAX_INTERVAL: Duration = Duration::fr
|
||||
const MAX_MAINTENANCE_FEATURE_INSPECTION_ATTEMPTS: usize = 2;
|
||||
const SCANNER_PERSIST_CAS_RETRIES: usize = 2;
|
||||
const DATA_USAGE_BACKUP_INTERVAL_CYCLES: u64 = 10;
|
||||
const SCANNER_PUBLICATION_LEASE_FENCE_MAX_ENTRIES: usize = 256;
|
||||
const SCANNER_PUBLICATION_LEASE_FENCE_MAX_BYTES: usize = 64 * 1024;
|
||||
const SCANNER_CYCLE_STATE_MAGIC: &[u8; 8] = b"RSCYC001";
|
||||
const SCANNER_CYCLE_STATE_HEADER_LEN: usize = 24;
|
||||
#[cfg(test)]
|
||||
@@ -137,6 +140,10 @@ static SCANNER_CYCLE_STATE_PERSIST_TEST_HOOK: LazyLock<StdMutex<Option<ScannerCy
|
||||
|
||||
static SCANNER_CYCLE_RECOVERY_WAKE: LazyLock<Notify> = LazyLock::new(Notify::new);
|
||||
|
||||
fn remote_publication_lease_fence_targets_are_required(target_count: usize, grants_present: bool, fence_present: bool) -> bool {
|
||||
target_count > 0 && (!grants_present || !fence_present)
|
||||
}
|
||||
|
||||
pub(super) fn notify_scanner_cycle_recovery_wake() {
|
||||
SCANNER_CYCLE_RECOVERY_WAKE.notify_one();
|
||||
}
|
||||
@@ -407,19 +414,24 @@ async fn sync_data_usage_backup_from_primary(
|
||||
ctx: &CancellationToken,
|
||||
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
|
||||
) -> Result<(), EcstoreError> {
|
||||
sync_data_usage_backup_from_primary_for_epoch(ctx, storeapi, None).await
|
||||
sync_data_usage_backup_from_primary_for_epoch_and_lease_and_fence(ctx, storeapi, None, None, None).await
|
||||
}
|
||||
|
||||
async fn sync_data_usage_backup_from_primary_for_epoch(
|
||||
async fn sync_data_usage_backup_from_primary_for_epoch_and_lease_and_fence(
|
||||
ctx: &CancellationToken,
|
||||
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
|
||||
expected_publication_epoch: Option<u64>,
|
||||
remote_lease_deadline: Option<std::time::Instant>,
|
||||
scanner_publication_lease_fence: Option<&str>,
|
||||
) -> Result<(), EcstoreError> {
|
||||
let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str());
|
||||
for retry in 0..=SCANNER_PERSIST_CAS_RETRIES {
|
||||
if ctx.is_cancelled() {
|
||||
return Ok(());
|
||||
}
|
||||
if remote_lease_deadline.is_some_and(|deadline| std::time::Instant::now() >= deadline) {
|
||||
return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED));
|
||||
}
|
||||
|
||||
let read_epoch = match expected_publication_epoch {
|
||||
Some(expected_epoch) => {
|
||||
@@ -446,6 +458,10 @@ async fn sync_data_usage_backup_from_primary_for_epoch(
|
||||
}
|
||||
let primary = Bytes::from(primary);
|
||||
|
||||
if remote_lease_deadline.is_some_and(|deadline| std::time::Instant::now() >= deadline) {
|
||||
return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED));
|
||||
}
|
||||
|
||||
let (backup, revision) = read_config_with_revision(storeapi.clone(), &backup_path).await?;
|
||||
if backup.as_deref() == Some(primary.as_ref()) {
|
||||
if scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch)
|
||||
@@ -462,18 +478,22 @@ async fn sync_data_usage_backup_from_primary_for_epoch(
|
||||
|
||||
let sha256hex = Some(hex_simd::encode_to_string(Sha256::digest(&primary), hex_simd::AsciiCase::Lower));
|
||||
let save_result = {
|
||||
if remote_lease_deadline.is_some_and(|deadline| std::time::Instant::now() >= deadline) {
|
||||
return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED));
|
||||
}
|
||||
let Some(_publication_admission) = scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch).await else {
|
||||
if retry < SCANNER_PERSIST_CAS_RETRIES {
|
||||
continue;
|
||||
}
|
||||
return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED));
|
||||
};
|
||||
save_config_shared_with_preconditions(
|
||||
save_config_shared_with_preconditions_and_lease_fence(
|
||||
storeapi.clone(),
|
||||
&backup_path,
|
||||
primary.clone(),
|
||||
sha256hex,
|
||||
revision.preconditions(),
|
||||
scanner_publication_lease_fence,
|
||||
)
|
||||
.await
|
||||
};
|
||||
@@ -1377,8 +1397,89 @@ async fn run_data_scanner_cycle_with_budget(
|
||||
};
|
||||
let publication_deferred = publication_defer_reason.is_some();
|
||||
let publication_epoch = scan_result.as_ref().ok().and_then(ScannerCycleResult::publication_epoch);
|
||||
let remote_publication_lease_targets = if publication_defer_reason.is_none() {
|
||||
scan_result
|
||||
.as_ref()
|
||||
.ok()
|
||||
.map(|result| result.remote_publication_lease_targets().to_vec())
|
||||
.unwrap_or_default()
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
let mut remote_publication_leases = None;
|
||||
let remote_lease_defer_reason = if remote_publication_lease_targets.is_empty() {
|
||||
None
|
||||
} else if usage_persist_timeout >= Duration::from_millis(SCANNER_PUBLICATION_LEASE_TTL_MS) {
|
||||
// The lease is intentionally fixed-duration and has no renewal path.
|
||||
// Refuse a persistence budget that could outlive it instead of
|
||||
// allowing the peer to admit movement while a local PUT is in flight.
|
||||
Some(ScannerCycleDeferReason::ActivityBaselineUnavailable)
|
||||
} else if let Some(notification_system) = storeapi.notification_system() {
|
||||
match notification_system
|
||||
.acquire_scanner_publication_leases(remote_publication_lease_targets.clone())
|
||||
.await
|
||||
{
|
||||
Ok(grants) => {
|
||||
remote_publication_leases = Some((notification_system, grants));
|
||||
None
|
||||
}
|
||||
Err(_) => Some(ScannerCycleDeferReason::ActivityBaselineUnavailable),
|
||||
}
|
||||
} else {
|
||||
Some(ScannerCycleDeferReason::ActivityBaselineUnavailable)
|
||||
};
|
||||
let remote_lease_deadline = remote_publication_leases
|
||||
.as_ref()
|
||||
.and_then(|(_, grants)| grants.iter().map(|grant| grant.lease.expires_at).min());
|
||||
// The transient fence is carried only to the SetDisks rename boundary;
|
||||
// it is never inserted into FileInfo metadata. Keep the representation
|
||||
// bounded and require one authenticated token per remote target so a
|
||||
// partial grant can never silently fall back to an unfenced rename.
|
||||
let remote_lease_fence = remote_publication_leases.as_ref().and_then(|(_, grants)| {
|
||||
if grants.len() != remote_publication_lease_targets.len() || grants.len() > SCANNER_PUBLICATION_LEASE_FENCE_MAX_ENTRIES {
|
||||
return None;
|
||||
}
|
||||
let mut fence = BTreeMap::new();
|
||||
for grant in grants {
|
||||
if grant.host.is_empty() || grant.host.len() > 1024 {
|
||||
return None;
|
||||
}
|
||||
if fence.insert(grant.host.clone(), grant.lease.token.to_string()).is_some() {
|
||||
return None;
|
||||
}
|
||||
}
|
||||
if remote_publication_lease_targets
|
||||
.iter()
|
||||
.any(|(host, _, _)| !fence.contains_key(host))
|
||||
{
|
||||
return None;
|
||||
}
|
||||
serde_json::to_string(&fence)
|
||||
.ok()
|
||||
.filter(|encoded| encoded.len() <= SCANNER_PUBLICATION_LEASE_FENCE_MAX_BYTES)
|
||||
});
|
||||
let remote_lease_fence_defer_reason = (remote_publication_lease_fence_targets_are_required(
|
||||
remote_publication_lease_targets.len(),
|
||||
remote_publication_leases.is_some(),
|
||||
remote_lease_fence.is_some(),
|
||||
))
|
||||
.then_some(ScannerCycleDeferReason::ActivityBaselineUnavailable);
|
||||
let remote_lease_covers_persistence = remote_lease_deadline.is_none_or(|deadline| {
|
||||
std::time::Instant::now()
|
||||
.checked_add(usage_persist_timeout)
|
||||
.is_some_and(|latest_finish| latest_finish < deadline)
|
||||
});
|
||||
let publication_defer_reason = publication_defer_reason
|
||||
.or(remote_lease_defer_reason)
|
||||
.or(remote_lease_fence_defer_reason);
|
||||
let publication_defer_reason = (!remote_lease_covers_persistence)
|
||||
.then_some(ScannerCycleDeferReason::ActivityBaselineUnavailable)
|
||||
.or(publication_defer_reason);
|
||||
let budget_elapsed = cycle_budget.budget_elapsed() && !ctx.is_cancelled();
|
||||
let usage_persist_outcome = match publication_defer_reason {
|
||||
let remote_lease_probe = remote_publication_leases
|
||||
.as_ref()
|
||||
.map(|(notification_system, grants)| (Arc::clone(notification_system), grants.clone()));
|
||||
let mut usage_persist_outcome = match publication_defer_reason {
|
||||
Some(reason) => {
|
||||
drop(receiver);
|
||||
DataUsagePersistOutcome::Deferred(reason)
|
||||
@@ -1390,17 +1491,33 @@ async fn run_data_scanner_cycle_with_budget(
|
||||
let storeapi_clone = storeapi.clone();
|
||||
let ctx_clone = ctx.clone();
|
||||
let route_probe_store = storeapi.clone();
|
||||
let remote_lease_fence = remote_lease_fence.clone();
|
||||
let mut usage_persist_task = AbortOnDropHandle::new(tokio::spawn(async move {
|
||||
store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe_for_publication_epoch(
|
||||
store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe_for_publication_epoch_and_lease_fence(
|
||||
ctx_clone,
|
||||
storeapi_clone,
|
||||
receiver,
|
||||
Some(leader_epoch),
|
||||
Some(usage_persist_baseline),
|
||||
publication_epoch,
|
||||
ScannerPublicationFence::new(
|
||||
publication_epoch,
|
||||
remote_lease_deadline,
|
||||
remote_lease_fence,
|
||||
),
|
||||
move || {
|
||||
let storeapi = route_probe_store.clone();
|
||||
async move { storeapi.scanner_data_usage_publication_blocked().await }
|
||||
let remote_lease_probe = remote_lease_probe.clone();
|
||||
async move {
|
||||
if let Some((notification_system, grants)) = remote_lease_probe.as_ref()
|
||||
&& notification_system.validate_scanner_publication_leases(grants).await.is_err()
|
||||
{
|
||||
// A remote restart or movement flip invalidates
|
||||
// the token proof; usage_store interprets this
|
||||
// as a publication barrier and performs no PUT.
|
||||
return true;
|
||||
}
|
||||
storeapi.scanner_data_usage_publication_blocked().await
|
||||
}
|
||||
},
|
||||
)
|
||||
.await
|
||||
@@ -1448,6 +1565,22 @@ async fn run_data_scanner_cycle_with_budget(
|
||||
}
|
||||
}
|
||||
};
|
||||
let lease_expired = remote_publication_leases
|
||||
.as_ref()
|
||||
.is_some_and(|(_, grants)| grants.iter().any(|grant| !grant.lease.is_valid()));
|
||||
if let Some((notification_system, grants)) = remote_publication_leases.take() {
|
||||
let release_result = notification_system.release_scanner_publication_leases(grants).await;
|
||||
if lease_expired || release_result.is_err() {
|
||||
// A lease that expired or could not be released is never treated
|
||||
// as a successful authoritative publication. The peer may have
|
||||
// admitted movement immediately after the lease ended.
|
||||
usage_persist_outcome = if usage_persist_outcome == DataUsagePersistOutcome::Failed {
|
||||
DataUsagePersistOutcome::Failed
|
||||
} else {
|
||||
DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable)
|
||||
};
|
||||
}
|
||||
}
|
||||
let unresolved_heal_work = global_metrics().current_scan_cycle_has_unresolved_heal_work();
|
||||
|
||||
let scan_cycle_result = match scan_result {
|
||||
@@ -2199,7 +2332,16 @@ async fn run_data_scanner_with_maintenance_state(
|
||||
);
|
||||
|
||||
let activity_poll_interval = backoff_enabled.then_some(runtime_config.cycle_interval.max(Duration::from_secs(1)));
|
||||
let wake_reason = wait_for_next_scanner_cycle_with_activity(
|
||||
let movement_generation_before_wait = storeapi.scanner_data_movement_generation();
|
||||
let movement_changed = storeapi.scanner_data_movement_changed();
|
||||
let movement_store = storeapi.clone();
|
||||
let movement = ScannerMovementWaitContext {
|
||||
movement_generation_seen: Some(movement_generation_before_wait),
|
||||
movement_changed,
|
||||
current_movement_generation: move || movement_store.scanner_data_movement_generation(),
|
||||
is_lock_lost: || guard.is_lock_lost(),
|
||||
};
|
||||
let wake_reason = wait_for_next_scanner_cycle_with_activity_and_movement(
|
||||
&ctx,
|
||||
wait_plan.delay,
|
||||
activity_poll_interval,
|
||||
@@ -2211,7 +2353,7 @@ async fn run_data_scanner_with_maintenance_state(
|
||||
runtime_config_generation_seen,
|
||||
maintenance_generation_before_wait,
|
||||
),
|
||||
|| guard.is_lock_lost(),
|
||||
movement,
|
||||
|| probe_scanner_activity(storeapi.as_ref(), distributed),
|
||||
)
|
||||
.await;
|
||||
@@ -2239,6 +2381,10 @@ async fn run_data_scanner_with_maintenance_state(
|
||||
ScannerCycleWakeReason::ClusterMaintenance => {
|
||||
clean_idle_backoff.reset();
|
||||
}
|
||||
ScannerCycleWakeReason::MovementGeneration => {
|
||||
scanner_activity_seen = None;
|
||||
clean_idle_backoff.reset();
|
||||
}
|
||||
ScannerCycleWakeReason::Timer
|
||||
| ScannerCycleWakeReason::DirtyUsage
|
||||
| ScannerCycleWakeReason::ClusterActivity
|
||||
@@ -2612,7 +2758,7 @@ use usage_store::*;
|
||||
pub use activity::scanner_topology_digest;
|
||||
pub(crate) use activity::{
|
||||
ScannerActivitySnapshot, ScannerDirtyUsageAcknowledgement, probe_scanner_activity, scanner_activity_allows_usage_publication,
|
||||
scanner_activity_snapshot_digest, scanner_dirty_usage_acknowledgements,
|
||||
scanner_activity_publication_lease_targets, scanner_activity_snapshot_digest, scanner_dirty_usage_acknowledgements,
|
||||
};
|
||||
pub(crate) use activity::{ScannerCycleOutcome, scanner_cycle_outcome_with_pending_maintenance};
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -13,11 +13,13 @@
|
||||
// limitations under the License.
|
||||
/// Cycle wake/backoff policy and scanner activity observation (probing, generations, topology digest).
|
||||
use super::*;
|
||||
use crate::storage_api::scan::SCANNER_ACTIVITY_V6_PROTOCOL_VERSION;
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(super) enum ScannerCycleWakeReason {
|
||||
Timer,
|
||||
DirtyUsage,
|
||||
MovementGeneration,
|
||||
ClusterActivity,
|
||||
ClusterMaintenance,
|
||||
ClusterActivityUnavailable,
|
||||
@@ -250,6 +252,17 @@ impl ScannerCycleObservedGenerations {
|
||||
}
|
||||
}
|
||||
|
||||
/// Movement state observed while a scanner waits for the next cycle.
|
||||
///
|
||||
/// Keeping the movement inputs together makes it harder for callers to pair a
|
||||
/// generation with the wrong notification or lock predicate.
|
||||
pub(super) struct ScannerMovementWaitContext<G, F> {
|
||||
pub(super) movement_generation_seen: Option<u64>,
|
||||
pub(super) movement_changed: Arc<Notify>,
|
||||
pub(super) current_movement_generation: G,
|
||||
pub(super) is_lock_lost: F,
|
||||
}
|
||||
|
||||
pub(super) const LOCAL_SCANNER_ACTIVITY_NODE: &str = "<local>";
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
@@ -262,6 +275,8 @@ pub(crate) struct ScannerNodeActivity {
|
||||
pub(super) data_movement_active: bool,
|
||||
pub(super) dirty_usage_generation: u64,
|
||||
pub(super) dirty_usage_pending: bool,
|
||||
pub(super) movement_generation: u64,
|
||||
pub(super) publication_blocked: bool,
|
||||
}
|
||||
|
||||
pub(crate) type ScannerActivitySnapshot = BTreeMap<String, ScannerNodeActivity>;
|
||||
@@ -278,6 +293,14 @@ pub(super) enum ScannerActivityObservation {
|
||||
NotRequired,
|
||||
Unchanged,
|
||||
Changed,
|
||||
/// A storage-owned movement generation changed. This wake must bypass the
|
||||
/// ordinary deferred cluster-activity backoff so publication can retry
|
||||
/// after a transition reaches its terminal state.
|
||||
MovementChanged,
|
||||
/// A remote scanner process restarted. Publication leases are bound to the
|
||||
/// process instance, so this must bypass deferred cluster-activity backoff
|
||||
/// even when the restarted peer reports otherwise ordinary activity.
|
||||
RemoteRestarted,
|
||||
MaintenanceChanged,
|
||||
Unverified,
|
||||
}
|
||||
@@ -373,6 +396,8 @@ pub(super) fn scanner_activity_observed_work(observation: ScannerActivityObserva
|
||||
matches!(
|
||||
observation,
|
||||
ScannerActivityObservation::Changed
|
||||
| ScannerActivityObservation::MovementChanged
|
||||
| ScannerActivityObservation::RemoteRestarted
|
||||
| ScannerActivityObservation::MaintenanceChanged
|
||||
| ScannerActivityObservation::Unverified
|
||||
)
|
||||
@@ -386,6 +411,7 @@ pub(super) fn scanner_activity_backoff_blocked_after_wake(currently_blocked: boo
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) async fn wait_for_next_scanner_cycle<F>(
|
||||
ctx: &CancellationToken,
|
||||
delay: Duration,
|
||||
@@ -396,6 +422,36 @@ pub(super) async fn wait_for_next_scanner_cycle<F>(
|
||||
) -> ScannerCycleWakeReason
|
||||
where
|
||||
F: Fn() -> bool,
|
||||
{
|
||||
let movement = ScannerMovementWaitContext {
|
||||
movement_generation_seen: None,
|
||||
movement_changed: Arc::new(Notify::new()),
|
||||
current_movement_generation: || 0,
|
||||
is_lock_lost,
|
||||
};
|
||||
wait_for_next_scanner_cycle_with_movement(
|
||||
ctx,
|
||||
delay,
|
||||
ScannerCycleObservedGenerations {
|
||||
dirty_usage: dirty_usage_generation_seen,
|
||||
runtime_config: runtime_config_generation,
|
||||
maintenance: maintenance_generation,
|
||||
defer_cluster_activity: false,
|
||||
},
|
||||
&movement,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(super) async fn wait_for_next_scanner_cycle_with_movement<G, F>(
|
||||
ctx: &CancellationToken,
|
||||
delay: Duration,
|
||||
generations: ScannerCycleObservedGenerations,
|
||||
movement: &ScannerMovementWaitContext<G, F>,
|
||||
) -> ScannerCycleWakeReason
|
||||
where
|
||||
F: Fn() -> bool,
|
||||
G: Fn() -> u64,
|
||||
{
|
||||
let sleep = tokio::time::sleep(delay);
|
||||
tokio::pin!(sleep);
|
||||
@@ -403,55 +459,86 @@ where
|
||||
tokio::pin!(lock_poll);
|
||||
|
||||
loop {
|
||||
if is_lock_lost() {
|
||||
if (movement.is_lock_lost)() {
|
||||
return ScannerCycleWakeReason::LeaderLockLost;
|
||||
}
|
||||
if scanner_runtime_config_generation() != runtime_config_generation {
|
||||
if scanner_runtime_config_generation() != generations.runtime_config {
|
||||
return ScannerCycleWakeReason::RuntimeConfig;
|
||||
}
|
||||
if scanner_maintenance_generation() != maintenance_generation {
|
||||
if scanner_maintenance_generation() != generations.maintenance {
|
||||
return ScannerCycleWakeReason::MaintenanceConfig;
|
||||
}
|
||||
if dirty_usage_generation_seen.is_some_and(|seen| dirty_usage_buckets_pending() && dirty_usage_generation() != seen) {
|
||||
if generations
|
||||
.dirty_usage
|
||||
.is_some_and(|seen| dirty_usage_buckets_pending() && dirty_usage_generation() != seen)
|
||||
{
|
||||
return ScannerCycleWakeReason::DirtyUsage;
|
||||
}
|
||||
if movement
|
||||
.movement_generation_seen
|
||||
.is_some_and(|seen| (movement.current_movement_generation)() != seen)
|
||||
{
|
||||
return ScannerCycleWakeReason::MovementGeneration;
|
||||
}
|
||||
|
||||
let movement_notification = movement.movement_changed.notified();
|
||||
tokio::pin!(movement_notification);
|
||||
movement_notification.as_mut().enable();
|
||||
// A transition may finish between the initial generation read and
|
||||
// registration with Notify. Re-check after `enable()` so that such a
|
||||
// transition cannot be lost when it used `notify_waiters()`.
|
||||
if movement
|
||||
.movement_generation_seen
|
||||
.is_some_and(|seen| (movement.current_movement_generation)() != seen)
|
||||
{
|
||||
return ScannerCycleWakeReason::MovementGeneration;
|
||||
}
|
||||
tokio::select! {
|
||||
_ = ctx.cancelled() => return ScannerCycleWakeReason::Cancelled,
|
||||
_ = &mut sleep => return ScannerCycleWakeReason::Timer,
|
||||
_ = &mut lock_poll => {
|
||||
if is_lock_lost() {
|
||||
if (movement.is_lock_lost)() {
|
||||
return ScannerCycleWakeReason::LeaderLockLost;
|
||||
}
|
||||
lock_poll.as_mut().reset(Instant::now() + SCANNER_LEADER_LOCK_POLL_INTERVAL);
|
||||
}
|
||||
_ = dirty_usage_bucket_notified() => {
|
||||
if scanner_runtime_config_generation() != runtime_config_generation {
|
||||
if scanner_runtime_config_generation() != generations.runtime_config {
|
||||
return ScannerCycleWakeReason::RuntimeConfig;
|
||||
}
|
||||
if scanner_maintenance_generation() != maintenance_generation {
|
||||
if scanner_maintenance_generation() != generations.maintenance {
|
||||
return ScannerCycleWakeReason::MaintenanceConfig;
|
||||
}
|
||||
if dirty_usage_generation_seen
|
||||
if generations
|
||||
.dirty_usage
|
||||
.is_some_and(|seen| dirty_usage_buckets_pending() && dirty_usage_generation() != seen)
|
||||
{
|
||||
return ScannerCycleWakeReason::DirtyUsage;
|
||||
}
|
||||
}
|
||||
_ = scanner_runtime_config_changed() => {
|
||||
if scanner_runtime_config_generation() != runtime_config_generation {
|
||||
if scanner_runtime_config_generation() != generations.runtime_config {
|
||||
return ScannerCycleWakeReason::RuntimeConfig;
|
||||
}
|
||||
}
|
||||
_ = scanner_maintenance_changed() => {
|
||||
if scanner_maintenance_generation() != maintenance_generation {
|
||||
if scanner_maintenance_generation() != generations.maintenance {
|
||||
return ScannerCycleWakeReason::MaintenanceConfig;
|
||||
}
|
||||
}
|
||||
_ = &mut movement_notification => {
|
||||
if movement
|
||||
.movement_generation_seen
|
||||
.is_some_and(|seen| (movement.current_movement_generation)() != seen)
|
||||
{
|
||||
return ScannerCycleWakeReason::MovementGeneration;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) async fn wait_for_next_scanner_cycle_with_activity<F, Probe, ProbeFuture>(
|
||||
ctx: &CancellationToken,
|
||||
delay: Duration,
|
||||
@@ -459,10 +546,43 @@ pub(super) async fn wait_for_next_scanner_cycle_with_activity<F, Probe, ProbeFut
|
||||
activity_seen: &mut Option<ScannerActivitySnapshot>,
|
||||
generations: ScannerCycleObservedGenerations,
|
||||
is_lock_lost: F,
|
||||
probe_activity: Probe,
|
||||
) -> ScannerCycleWakeReason
|
||||
where
|
||||
F: Fn() -> bool,
|
||||
Probe: FnMut() -> ProbeFuture,
|
||||
ProbeFuture: Future<Output = Result<ScannerActivitySnapshot, String>>,
|
||||
{
|
||||
let movement = ScannerMovementWaitContext {
|
||||
movement_generation_seen: None,
|
||||
movement_changed: Arc::new(Notify::new()),
|
||||
current_movement_generation: || 0,
|
||||
is_lock_lost,
|
||||
};
|
||||
wait_for_next_scanner_cycle_with_activity_and_movement(
|
||||
ctx,
|
||||
delay,
|
||||
activity_poll_interval,
|
||||
activity_seen,
|
||||
generations,
|
||||
movement,
|
||||
probe_activity,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(super) async fn wait_for_next_scanner_cycle_with_activity_and_movement<F, G, Probe, ProbeFuture>(
|
||||
ctx: &CancellationToken,
|
||||
delay: Duration,
|
||||
activity_poll_interval: Option<Duration>,
|
||||
activity_seen: &mut Option<ScannerActivitySnapshot>,
|
||||
generations: ScannerCycleObservedGenerations,
|
||||
movement: ScannerMovementWaitContext<G, F>,
|
||||
mut probe_activity: Probe,
|
||||
) -> ScannerCycleWakeReason
|
||||
where
|
||||
F: Fn() -> bool,
|
||||
G: Fn() -> u64,
|
||||
Probe: FnMut() -> ProbeFuture,
|
||||
ProbeFuture: Future<Output = Result<ScannerActivitySnapshot, String>>,
|
||||
{
|
||||
@@ -475,15 +595,7 @@ where
|
||||
let wait_slice = activity_poll_interval
|
||||
.map(|interval| interval.max(Duration::from_secs(1)).min(remaining))
|
||||
.unwrap_or(remaining);
|
||||
let wake_reason = wait_for_next_scanner_cycle(
|
||||
ctx,
|
||||
wait_slice,
|
||||
generations.dirty_usage,
|
||||
generations.runtime_config,
|
||||
generations.maintenance,
|
||||
&is_lock_lost,
|
||||
)
|
||||
.await;
|
||||
let wake_reason = wait_for_next_scanner_cycle_with_movement(ctx, wait_slice, generations, &movement).await;
|
||||
if wake_reason != ScannerCycleWakeReason::Timer || Instant::now() >= deadline {
|
||||
return wake_reason;
|
||||
}
|
||||
@@ -491,7 +603,7 @@ where
|
||||
let Some(_) = activity_poll_interval else {
|
||||
return ScannerCycleWakeReason::Timer;
|
||||
};
|
||||
if is_lock_lost() {
|
||||
if (movement.is_lock_lost)() {
|
||||
return ScannerCycleWakeReason::LeaderLockLost;
|
||||
}
|
||||
|
||||
@@ -500,7 +612,7 @@ where
|
||||
let lock_lost = async {
|
||||
loop {
|
||||
tokio::time::sleep(SCANNER_LEADER_LOCK_POLL_INTERVAL).await;
|
||||
if is_lock_lost() {
|
||||
if (movement.is_lock_lost)() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
@@ -519,6 +631,9 @@ where
|
||||
}
|
||||
match observation {
|
||||
ScannerActivityObservation::Unchanged | ScannerActivityObservation::NotRequired => {}
|
||||
ScannerActivityObservation::MovementChanged | ScannerActivityObservation::RemoteRestarted => {
|
||||
return ScannerCycleWakeReason::ClusterActivity;
|
||||
}
|
||||
ScannerActivityObservation::Changed if !generations.defer_cluster_activity => {
|
||||
return ScannerCycleWakeReason::ClusterActivity;
|
||||
}
|
||||
@@ -568,12 +683,23 @@ pub(super) fn compare_scanner_activity(
|
||||
let Some(previous_activity) = previous.get(host) else {
|
||||
continue;
|
||||
};
|
||||
if host != LOCAL_SCANNER_ACTIVITY_NODE && previous_activity.instance_id != current_activity.instance_id {
|
||||
return ScannerActivityObservation::RemoteRestarted;
|
||||
}
|
||||
if host != LOCAL_SCANNER_ACTIVITY_NODE
|
||||
&& previous_activity.instance_id == current_activity.instance_id
|
||||
&& previous_activity.maintenance_generation != current_activity.maintenance_generation
|
||||
{
|
||||
return ScannerActivityObservation::MaintenanceChanged;
|
||||
}
|
||||
|
||||
if previous_activity.instance_id == current_activity.instance_id
|
||||
&& (previous_activity.data_movement_active != current_activity.data_movement_active
|
||||
|| previous_activity.movement_generation != current_activity.movement_generation
|
||||
|| previous_activity.publication_blocked != current_activity.publication_blocked)
|
||||
{
|
||||
return ScannerActivityObservation::MovementChanged;
|
||||
}
|
||||
}
|
||||
|
||||
ScannerActivityObservation::Changed
|
||||
@@ -630,12 +756,28 @@ pub(crate) fn scanner_activity_snapshot_digest(snapshot: &ScannerActivitySnapsho
|
||||
hasher.update([u8::from(activity.data_movement_active)]);
|
||||
hasher.update(activity.dirty_usage_generation.to_be_bytes());
|
||||
hasher.update([u8::from(activity.dirty_usage_pending)]);
|
||||
hasher.update(activity.movement_generation.to_be_bytes());
|
||||
hasher.update([u8::from(activity.publication_blocked)]);
|
||||
}
|
||||
hasher.finalize().into()
|
||||
}
|
||||
|
||||
pub(crate) fn scanner_activity_allows_usage_publication(snapshot: &ScannerActivitySnapshot) -> bool {
|
||||
snapshot.values().all(|activity| !activity.data_movement_active)
|
||||
!snapshot.is_empty()
|
||||
&& snapshot.values().all(|activity| {
|
||||
activity.protocol_version == SCANNER_ACTIVITY_PROTOCOL_VERSION
|
||||
&& activity.movement_generation != u64::MAX
|
||||
&& !activity.data_movement_active
|
||||
&& !activity.publication_blocked
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn scanner_activity_publication_lease_targets(snapshot: &ScannerActivitySnapshot) -> Vec<(String, String, u64)> {
|
||||
snapshot
|
||||
.iter()
|
||||
.filter(|(host, _)| host.as_str() != LOCAL_SCANNER_ACTIVITY_NODE)
|
||||
.map(|(host, activity)| (host.clone(), activity.instance_id.clone(), activity.movement_generation))
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub(crate) fn scanner_dirty_usage_acknowledgements(snapshot: &ScannerActivitySnapshot) -> Vec<ScannerDirtyUsageAcknowledgement> {
|
||||
@@ -695,11 +837,15 @@ pub(super) fn record_scanner_activity_instance(
|
||||
|
||||
pub(crate) async fn probe_scanner_activity(storeapi: &ECStore, distributed: bool) -> Result<ScannerActivitySnapshot, String> {
|
||||
let topology_digest = scanner_topology_digest(storeapi);
|
||||
let data_movement_active = storeapi.scanner_data_movement_active().await;
|
||||
let (data_movement_active, publication_blocked, movement_generation) = storeapi.scanner_data_movement_activity().await;
|
||||
let namespace_generation = storeapi.scanner_namespace_mutation_generation();
|
||||
let maintenance_generation = scanner_maintenance_generation();
|
||||
let dirty_usage = scanner_dirty_usage_state();
|
||||
if namespace_generation == u64::MAX || maintenance_generation == u64::MAX || dirty_usage.generation == u64::MAX {
|
||||
if namespace_generation == u64::MAX
|
||||
|| maintenance_generation == u64::MAX
|
||||
|| dirty_usage.generation == u64::MAX
|
||||
|| movement_generation == u64::MAX
|
||||
{
|
||||
return Err("local scanner activity generation is exhausted".to_string());
|
||||
}
|
||||
let local_instance_id = crate::scanner_io::scanner_activity_epoch().to_string();
|
||||
@@ -715,6 +861,8 @@ pub(crate) async fn probe_scanner_activity(storeapi: &ECStore, distributed: bool
|
||||
data_movement_active,
|
||||
dirty_usage_generation: dirty_usage.generation,
|
||||
dirty_usage_pending: dirty_usage.pending,
|
||||
movement_generation,
|
||||
publication_blocked,
|
||||
},
|
||||
)]);
|
||||
if !distributed {
|
||||
@@ -732,39 +880,57 @@ pub(crate) async fn probe_scanner_activity(storeapi: &ECStore, distributed: bool
|
||||
if activity.namespace_generation == u64::MAX || activity.maintenance_generation == u64::MAX {
|
||||
return Err(format!("scanner activity peer {host} exhausted its activity generation"));
|
||||
}
|
||||
let (peer_topology_digest, peer_data_movement_active, peer_dirty_usage_generation, peer_dirty_usage_pending) =
|
||||
match activity.protocol_version {
|
||||
SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION => {
|
||||
return Err(format!("scanner activity peer {host} cannot verify data movement publication fencing"));
|
||||
}
|
||||
SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION => {
|
||||
return Err(format!(
|
||||
"scanner activity peer {host} cannot safely share scanner cache locks with protocol {}",
|
||||
SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION
|
||||
));
|
||||
}
|
||||
SCANNER_ACTIVITY_PROTOCOL_VERSION => (
|
||||
activity
|
||||
.topology_digest
|
||||
.ok_or_else(|| format!("scanner activity peer {host} omitted its storage topology"))?,
|
||||
activity
|
||||
.data_movement_active
|
||||
.ok_or_else(|| format!("scanner activity peer {host} omitted its data movement state"))?,
|
||||
activity
|
||||
.dirty_usage_generation
|
||||
.ok_or_else(|| format!("scanner activity peer {host} omitted its dirty usage generation"))?,
|
||||
activity
|
||||
.dirty_usage_pending
|
||||
.ok_or_else(|| format!("scanner activity peer {host} omitted its dirty usage state"))?,
|
||||
),
|
||||
version => {
|
||||
return Err(format!(
|
||||
"scanner activity peer {host} uses protocol {version}, expected {}",
|
||||
SCANNER_ACTIVITY_PROTOCOL_VERSION
|
||||
));
|
||||
}
|
||||
};
|
||||
if peer_dirty_usage_generation == u64::MAX {
|
||||
let (
|
||||
peer_topology_digest,
|
||||
peer_data_movement_active,
|
||||
peer_dirty_usage_generation,
|
||||
peer_dirty_usage_pending,
|
||||
peer_movement_generation,
|
||||
peer_publication_blocked,
|
||||
) = match activity.protocol_version {
|
||||
SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION => {
|
||||
return Err(format!("scanner activity peer {host} cannot verify data movement publication fencing"));
|
||||
}
|
||||
SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION => {
|
||||
return Err(format!(
|
||||
"scanner activity peer {host} cannot safely share scanner cache locks with protocol {}",
|
||||
SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION
|
||||
));
|
||||
}
|
||||
SCANNER_ACTIVITY_V6_PROTOCOL_VERSION => {
|
||||
return Err(format!(
|
||||
"scanner activity peer {host} cannot verify terminal movement state with protocol {}",
|
||||
SCANNER_ACTIVITY_V6_PROTOCOL_VERSION
|
||||
));
|
||||
}
|
||||
SCANNER_ACTIVITY_PROTOCOL_VERSION => (
|
||||
activity
|
||||
.topology_digest
|
||||
.ok_or_else(|| format!("scanner activity peer {host} omitted its storage topology"))?,
|
||||
activity
|
||||
.data_movement_active
|
||||
.ok_or_else(|| format!("scanner activity peer {host} omitted its data movement state"))?,
|
||||
activity
|
||||
.dirty_usage_generation
|
||||
.ok_or_else(|| format!("scanner activity peer {host} omitted its dirty usage generation"))?,
|
||||
activity
|
||||
.dirty_usage_pending
|
||||
.ok_or_else(|| format!("scanner activity peer {host} omitted its dirty usage state"))?,
|
||||
activity
|
||||
.movement_generation
|
||||
.ok_or_else(|| format!("scanner activity peer {host} omitted its movement generation"))?,
|
||||
activity
|
||||
.publication_blocked
|
||||
.ok_or_else(|| format!("scanner activity peer {host} omitted its publication blocked state"))?,
|
||||
),
|
||||
version => {
|
||||
return Err(format!(
|
||||
"scanner activity peer {host} uses protocol {version}, expected {}",
|
||||
SCANNER_ACTIVITY_PROTOCOL_VERSION
|
||||
));
|
||||
}
|
||||
};
|
||||
if peer_dirty_usage_generation == u64::MAX || peer_movement_generation == u64::MAX {
|
||||
return Err(format!("scanner activity peer {host} exhausted its dirty usage generation"));
|
||||
}
|
||||
if peer_topology_digest != topology_digest {
|
||||
@@ -783,6 +949,8 @@ pub(crate) async fn probe_scanner_activity(storeapi: &ECStore, distributed: bool
|
||||
data_movement_active: peer_data_movement_active,
|
||||
dirty_usage_generation: peer_dirty_usage_generation,
|
||||
dirty_usage_pending: peer_dirty_usage_pending,
|
||||
movement_generation: peer_movement_generation,
|
||||
publication_blocked: peer_publication_blocked,
|
||||
},
|
||||
)
|
||||
.is_some()
|
||||
|
||||
@@ -23,7 +23,7 @@ use crate::{
|
||||
};
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::io::Cursor;
|
||||
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
||||
use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
|
||||
use std::task::Poll;
|
||||
use temp_env::{with_var, with_var_unset};
|
||||
use tokio::io::AsyncReadExt;
|
||||
@@ -2888,6 +2888,47 @@ async fn test_usage_route_barrier_precedes_durable_reconciliation() {
|
||||
assert_eq!(store.put_counts.lock().await.get(&key), None);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn coordinator_does_not_put_after_remote_generation_flip() {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str());
|
||||
let (sender, receiver) = mpsc::channel(1);
|
||||
sender
|
||||
.send(complete_usage_with_bucket_count(
|
||||
Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)),
|
||||
1,
|
||||
))
|
||||
.await
|
||||
.expect("usage snapshot should enqueue");
|
||||
drop(sender);
|
||||
|
||||
let route_store = store.clone();
|
||||
let outcome = store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe_for_publication_epoch(
|
||||
CancellationToken::new(),
|
||||
store.clone(),
|
||||
receiver,
|
||||
None,
|
||||
Some(DataUsagePersistBaseline {
|
||||
data: None,
|
||||
revision: DataUsageCacheRevision::Missing,
|
||||
}),
|
||||
ScannerPublicationFence::new(Some(0), None, None),
|
||||
move || {
|
||||
let route_store = route_store.clone();
|
||||
async move {
|
||||
// Model the remote lease holder flipping its movement generation
|
||||
// after the activity probe but before the coordinator's PUT.
|
||||
route_store.publication_admission_blocked.store(true, Ordering::Release);
|
||||
false
|
||||
}
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(outcome, DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement));
|
||||
assert_eq!(store.put_counts.lock().await.get(&key), None);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_deferred_usage_save_keeps_last_real_save_metric() {
|
||||
let metrics = global_metrics();
|
||||
@@ -4974,6 +5015,41 @@ async fn test_wait_for_next_scanner_cycle_stops_after_leader_lock_loss() {
|
||||
assert_eq!(reason, ScannerCycleWakeReason::LeaderLockLost);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn movement_generation_wakes_deferred_wait_without_dirty_bucket() {
|
||||
let ctx = CancellationToken::new();
|
||||
let movement_generation = Arc::new(AtomicU64::new(7));
|
||||
let movement_changed = Arc::new(Notify::new());
|
||||
let next_generation = Arc::clone(&movement_generation);
|
||||
let next_changed = Arc::clone(&movement_changed);
|
||||
tokio::spawn(async move {
|
||||
tokio::task::yield_now().await;
|
||||
next_generation.store(8, Ordering::Release);
|
||||
next_changed.notify_waiters();
|
||||
});
|
||||
|
||||
let movement = ScannerMovementWaitContext {
|
||||
movement_generation_seen: Some(7),
|
||||
movement_changed,
|
||||
current_movement_generation: move || movement_generation.load(Ordering::Acquire),
|
||||
is_lock_lost: || false,
|
||||
};
|
||||
let reason = wait_for_next_scanner_cycle_with_movement(
|
||||
&ctx,
|
||||
Duration::from_secs(60),
|
||||
ScannerCycleObservedGenerations {
|
||||
dirty_usage: None,
|
||||
runtime_config: crate::runtime_config::scanner_runtime_config_generation(),
|
||||
maintenance: crate::scanner_io::scanner_maintenance_generation(),
|
||||
defer_cluster_activity: false,
|
||||
},
|
||||
&movement,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(reason, ScannerCycleWakeReason::MovementGeneration);
|
||||
}
|
||||
|
||||
fn scanner_node_activity(epoch: &str, namespace_generation: u64, maintenance_generation: u64) -> ScannerNodeActivity {
|
||||
ScannerNodeActivity {
|
||||
instance_id: epoch.to_string(),
|
||||
@@ -4984,6 +5060,8 @@ fn scanner_node_activity(epoch: &str, namespace_generation: u64, maintenance_gen
|
||||
data_movement_active: false,
|
||||
dirty_usage_generation: 5,
|
||||
dirty_usage_pending: false,
|
||||
movement_generation: 9,
|
||||
publication_blocked: false,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5024,6 +5102,7 @@ fn scanner_activity_snapshot_fences_data_movement() {
|
||||
let mut moving = idle.clone();
|
||||
moving.get_mut("node-2").expect("node should exist").data_movement_active = true;
|
||||
|
||||
assert!(!scanner_activity_allows_usage_publication(&BTreeMap::new()));
|
||||
assert!(scanner_activity_allows_usage_publication(&idle));
|
||||
assert!(!scanner_activity_allows_usage_publication(&moving));
|
||||
assert_ne!(scanner_activity_snapshot_digest(&idle), scanner_activity_snapshot_digest(&moving));
|
||||
@@ -5107,7 +5186,7 @@ fn scanner_activity_observation_requires_a_complete_baseline() {
|
||||
|
||||
let restarted = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-b", 8, 0))]);
|
||||
let (observation, error) = apply_scanner_activity_probe_result(&mut seen, Ok(restarted));
|
||||
assert_eq!(observation, ScannerActivityObservation::Changed);
|
||||
assert_eq!(observation, ScannerActivityObservation::RemoteRestarted);
|
||||
assert!(error.is_none());
|
||||
|
||||
let (observation, error) =
|
||||
@@ -5142,6 +5221,36 @@ fn remote_maintenance_change_is_distinct_from_namespace_activity() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_movement_generation_change_is_distinct_from_cluster_activity() {
|
||||
let previous = BTreeMap::from([("node-2".to_string(), scanner_node_activity("remote", 7, 3))]);
|
||||
let movement_changed = BTreeMap::from([(
|
||||
"node-2".to_string(),
|
||||
ScannerNodeActivity {
|
||||
movement_generation: 10,
|
||||
..scanner_node_activity("remote", 7, 3)
|
||||
},
|
||||
)]);
|
||||
|
||||
assert_eq!(
|
||||
compare_scanner_activity(&previous, &movement_changed),
|
||||
ScannerActivityObservation::MovementChanged
|
||||
);
|
||||
assert!(scanner_activity_observed_work(ScannerActivityObservation::MovementChanged));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_restart_is_distinct_from_deferred_cluster_activity() {
|
||||
let previous = BTreeMap::from([("node-2".to_string(), scanner_node_activity("remote-a", 7, 3))]);
|
||||
let restarted = BTreeMap::from([("node-2".to_string(), scanner_node_activity("remote-b", 7, 3))]);
|
||||
|
||||
assert_eq!(
|
||||
compare_scanner_activity(&previous, &restarted),
|
||||
ScannerActivityObservation::RemoteRestarted
|
||||
);
|
||||
assert!(scanner_activity_observed_work(ScannerActivityObservation::RemoteRestarted));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn local_maintenance_wakeup_releases_a_remote_maintenance_block() {
|
||||
let blocked = scanner_activity_backoff_blocked_after_wake(false, ScannerCycleWakeReason::ClusterMaintenance);
|
||||
@@ -5231,6 +5340,74 @@ async fn superseded_retry_wait_defers_dirty_cluster_activity_until_timer() {
|
||||
assert_eq!(seen, Some(changed));
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn superseded_retry_wait_wakes_for_remote_movement_generation() {
|
||||
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
||||
let ctx = CancellationToken::new();
|
||||
let mut seen = Some(BTreeMap::from([("node-2".to_string(), scanner_node_activity("remote", 7, 3))]));
|
||||
let changed = BTreeMap::from([(
|
||||
"node-2".to_string(),
|
||||
ScannerNodeActivity {
|
||||
movement_generation: 10,
|
||||
..scanner_node_activity("remote", 7, 3)
|
||||
},
|
||||
)]);
|
||||
|
||||
let reason = wait_for_next_scanner_cycle_with_activity(
|
||||
&ctx,
|
||||
Duration::from_secs(120),
|
||||
Some(Duration::from_secs(60)),
|
||||
&mut seen,
|
||||
ScannerCycleObservedGenerations {
|
||||
dirty_usage: None,
|
||||
runtime_config: crate::runtime_config::scanner_runtime_config_generation(),
|
||||
maintenance: crate::scanner_io::scanner_maintenance_generation(),
|
||||
defer_cluster_activity: true,
|
||||
},
|
||||
|| false,
|
||||
|| std::future::ready(Ok(changed.clone())),
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(reason, ScannerCycleWakeReason::ClusterActivity);
|
||||
assert_eq!(seen, Some(changed));
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn superseded_retry_wait_wakes_when_remote_restart_clears_movement_state() {
|
||||
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
||||
let ctx = CancellationToken::new();
|
||||
let blocked = BTreeMap::from([(
|
||||
"node-2".to_string(),
|
||||
ScannerNodeActivity {
|
||||
data_movement_active: true,
|
||||
publication_blocked: true,
|
||||
..scanner_node_activity("remote-a", 7, 3)
|
||||
},
|
||||
)]);
|
||||
let restarted = BTreeMap::from([("node-2".to_string(), scanner_node_activity("remote-b", 7, 3))]);
|
||||
let mut seen = Some(blocked);
|
||||
|
||||
let reason = wait_for_next_scanner_cycle_with_activity(
|
||||
&ctx,
|
||||
Duration::from_secs(120),
|
||||
Some(Duration::from_secs(60)),
|
||||
&mut seen,
|
||||
ScannerCycleObservedGenerations {
|
||||
dirty_usage: None,
|
||||
runtime_config: crate::runtime_config::scanner_runtime_config_generation(),
|
||||
maintenance: crate::scanner_io::scanner_maintenance_generation(),
|
||||
defer_cluster_activity: true,
|
||||
},
|
||||
|| false,
|
||||
|| std::future::ready(Ok(restarted.clone())),
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(reason, ScannerCycleWakeReason::ClusterActivity);
|
||||
assert_eq!(seen, Some(restarted));
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn distributed_clean_idle_wait_blocks_backoff_for_unpropagated_maintenance() {
|
||||
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
||||
|
||||
@@ -13,6 +13,7 @@
|
||||
// limitations under the License.
|
||||
/// Data-usage snapshot persistence: CAS store pipeline, epoch baselines, and observed-snapshot cleanup.
|
||||
use super::*;
|
||||
use std::collections::HashMap;
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
||||
pub(super) enum DataUsagePersistOutcome {
|
||||
@@ -29,12 +30,40 @@ pub(super) enum DataUsagePersistOutcome {
|
||||
Failed,
|
||||
}
|
||||
|
||||
fn remote_lease_expired(deadline: Option<std::time::Instant>) -> bool {
|
||||
deadline.is_some_and(|deadline| std::time::Instant::now() >= deadline)
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub(super) struct DataUsagePersistBaseline {
|
||||
pub(super) data: Option<Bytes>,
|
||||
pub(super) revision: DataUsageCacheRevision,
|
||||
}
|
||||
|
||||
/// Short-lived publication inputs captured for one usage persistence attempt.
|
||||
/// Keeping the movement epoch, lease deadline, and target fence together makes
|
||||
/// it explicit that they are one proof rather than independent options.
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub(super) struct ScannerPublicationFence {
|
||||
pub(super) expected_publication_epoch: Option<u64>,
|
||||
pub(super) remote_lease_deadline: Option<std::time::Instant>,
|
||||
pub(super) scanner_publication_lease_fence: Option<String>,
|
||||
}
|
||||
|
||||
impl ScannerPublicationFence {
|
||||
pub(super) fn new(
|
||||
expected_publication_epoch: Option<u64>,
|
||||
remote_lease_deadline: Option<std::time::Instant>,
|
||||
scanner_publication_lease_fence: Option<String>,
|
||||
) -> Self {
|
||||
Self {
|
||||
expected_publication_epoch,
|
||||
remote_lease_deadline,
|
||||
scanner_publication_lease_fence,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub(super) enum DataUsagePersistTaskResult {
|
||||
Completed(DataUsagePersistOutcome),
|
||||
@@ -129,7 +158,7 @@ where
|
||||
receiver,
|
||||
leader_epoch,
|
||||
initial_baseline,
|
||||
None,
|
||||
ScannerPublicationFence::default(),
|
||||
route_probe,
|
||||
)
|
||||
.await
|
||||
@@ -141,16 +170,49 @@ pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_basel
|
||||
>(
|
||||
ctx: CancellationToken,
|
||||
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
|
||||
mut receiver: mpsc::Receiver<DataUsageInfo>,
|
||||
receiver: mpsc::Receiver<DataUsageInfo>,
|
||||
leader_epoch: Option<u64>,
|
||||
initial_baseline: Option<DataUsagePersistBaseline>,
|
||||
expected_publication_epoch: Option<u64>,
|
||||
publication_fence: ScannerPublicationFence,
|
||||
route_probe: F,
|
||||
) -> DataUsagePersistOutcome
|
||||
where
|
||||
F: Fn() -> Fut + Send + Sync,
|
||||
Fut: Future<Output = bool> + Send,
|
||||
{
|
||||
store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe_for_publication_epoch_and_lease_fence(
|
||||
ctx,
|
||||
storeapi,
|
||||
receiver,
|
||||
leader_epoch,
|
||||
initial_baseline,
|
||||
publication_fence,
|
||||
route_probe,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe_for_publication_epoch_and_lease_fence<
|
||||
F,
|
||||
Fut,
|
||||
>(
|
||||
ctx: CancellationToken,
|
||||
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
|
||||
mut receiver: mpsc::Receiver<DataUsageInfo>,
|
||||
leader_epoch: Option<u64>,
|
||||
initial_baseline: Option<DataUsagePersistBaseline>,
|
||||
publication_fence: ScannerPublicationFence,
|
||||
route_probe: F,
|
||||
) -> DataUsagePersistOutcome
|
||||
where
|
||||
F: Fn() -> Fut + Send + Sync,
|
||||
Fut: Future<Output = bool> + Send,
|
||||
{
|
||||
let ScannerPublicationFence {
|
||||
expected_publication_epoch,
|
||||
remote_lease_deadline,
|
||||
scanner_publication_lease_fence,
|
||||
} = publication_fence;
|
||||
let mut outcome = DataUsagePersistOutcome::NoUpdate;
|
||||
let mut next_baseline = initial_baseline;
|
||||
|
||||
@@ -162,6 +224,10 @@ where
|
||||
if let Some(leader_epoch) = leader_epoch {
|
||||
data_usage_info.scanner_epoch = Some(leader_epoch);
|
||||
}
|
||||
if remote_lease_expired(remote_lease_deadline) {
|
||||
outcome = DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable);
|
||||
break 'updates;
|
||||
}
|
||||
if let Some(expected_epoch) = expected_publication_epoch
|
||||
&& scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch)
|
||||
.await
|
||||
@@ -430,6 +496,9 @@ where
|
||||
);
|
||||
break DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
|
||||
}
|
||||
if remote_lease_expired(remote_lease_deadline) {
|
||||
break DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable);
|
||||
}
|
||||
|
||||
let done_save = Metrics::time(Metric::SaveUsage);
|
||||
let save_result = {
|
||||
@@ -439,12 +508,17 @@ where
|
||||
done_save();
|
||||
break DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
|
||||
};
|
||||
save_config_shared_with_preconditions(
|
||||
if remote_lease_expired(remote_lease_deadline) {
|
||||
done_save();
|
||||
break DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable);
|
||||
}
|
||||
save_config_shared_with_preconditions_and_lease_fence(
|
||||
storeapi.clone(),
|
||||
target_path,
|
||||
data.clone(),
|
||||
sha256hex.clone(),
|
||||
revision.preconditions(),
|
||||
scanner_publication_lease_fence.as_deref(),
|
||||
)
|
||||
.await
|
||||
};
|
||||
@@ -538,10 +612,12 @@ where
|
||||
if observational {
|
||||
invalidate_admin_data_usage_snapshot_cache().await;
|
||||
} else {
|
||||
let cleanup_ok = cleanup_observed_data_usage_snapshot_for_epoch(
|
||||
let cleanup_ok = cleanup_observed_data_usage_snapshot_for_epoch_and_lease(
|
||||
storeapi.clone(),
|
||||
&data_usage_info,
|
||||
expected_publication_epoch,
|
||||
remote_lease_deadline,
|
||||
scanner_publication_lease_fence.as_deref(),
|
||||
)
|
||||
.await;
|
||||
if expected_publication_epoch.is_some() && !cleanup_ok {
|
||||
@@ -559,10 +635,12 @@ where
|
||||
if observational {
|
||||
invalidate_admin_data_usage_snapshot_cache().await;
|
||||
} else {
|
||||
let cleanup_ok = cleanup_observed_data_usage_snapshot_for_epoch(
|
||||
let cleanup_ok = cleanup_observed_data_usage_snapshot_for_epoch_and_lease(
|
||||
storeapi.clone(),
|
||||
&data_usage_info,
|
||||
expected_publication_epoch,
|
||||
remote_lease_deadline,
|
||||
scanner_publication_lease_fence.as_deref(),
|
||||
)
|
||||
.await;
|
||||
if expected_publication_epoch.is_some() && !cleanup_ok {
|
||||
@@ -599,10 +677,12 @@ where
|
||||
if observational {
|
||||
invalidate_admin_data_usage_snapshot_cache().await;
|
||||
} else {
|
||||
let cleanup_ok = cleanup_observed_data_usage_snapshot_for_epoch(
|
||||
let cleanup_ok = cleanup_observed_data_usage_snapshot_for_epoch_and_lease(
|
||||
storeapi.clone(),
|
||||
&data_usage_info,
|
||||
expected_publication_epoch,
|
||||
remote_lease_deadline,
|
||||
scanner_publication_lease_fence.as_deref(),
|
||||
)
|
||||
.await;
|
||||
if expected_publication_epoch.is_some() && !cleanup_ok {
|
||||
@@ -620,8 +700,14 @@ where
|
||||
|
||||
if backup_due {
|
||||
let done_save = Metrics::time(Metric::SaveUsage);
|
||||
let backup_result =
|
||||
sync_data_usage_backup_from_primary_for_epoch(&ctx, storeapi.clone(), expected_publication_epoch).await;
|
||||
let backup_result = sync_data_usage_backup_from_primary_for_epoch_and_lease_and_fence(
|
||||
&ctx,
|
||||
storeapi.clone(),
|
||||
expected_publication_epoch,
|
||||
remote_lease_deadline,
|
||||
scanner_publication_lease_fence.as_deref(),
|
||||
)
|
||||
.await;
|
||||
done_save();
|
||||
if let Err(e) = backup_result {
|
||||
warn!(
|
||||
@@ -647,11 +733,16 @@ where
|
||||
outcome
|
||||
}
|
||||
|
||||
pub(super) async fn cleanup_observed_data_usage_snapshot_for_epoch(
|
||||
async fn cleanup_observed_data_usage_snapshot_for_epoch_and_lease(
|
||||
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
|
||||
authoritative: &DataUsageInfo,
|
||||
expected_publication_epoch: Option<u64>,
|
||||
remote_lease_deadline: Option<std::time::Instant>,
|
||||
scanner_publication_lease_fence: Option<&str>,
|
||||
) -> bool {
|
||||
if remote_lease_expired(remote_lease_deadline) {
|
||||
return false;
|
||||
}
|
||||
let read_epoch = match expected_publication_epoch {
|
||||
Some(expected_epoch) => {
|
||||
if scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch)
|
||||
@@ -667,10 +758,11 @@ pub(super) async fn cleanup_observed_data_usage_snapshot_for_epoch(
|
||||
None => return false,
|
||||
},
|
||||
};
|
||||
if expected_publication_epoch.is_some()
|
||||
&& scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch)
|
||||
.await
|
||||
.is_none()
|
||||
if remote_lease_expired(remote_lease_deadline)
|
||||
|| expected_publication_epoch.is_some()
|
||||
&& scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch)
|
||||
.await
|
||||
.is_none()
|
||||
{
|
||||
return false;
|
||||
}
|
||||
@@ -711,6 +803,9 @@ pub(super) async fn cleanup_observed_data_usage_snapshot_for_epoch(
|
||||
if observed_data_usage_is_newer(&observed, authoritative) {
|
||||
return true;
|
||||
}
|
||||
if remote_lease_expired(remote_lease_deadline) {
|
||||
return false;
|
||||
}
|
||||
|
||||
let result = delete_config_with_publication_admission_for_epoch(
|
||||
storeapi,
|
||||
@@ -720,6 +815,14 @@ pub(super) async fn cleanup_observed_data_usage_snapshot_for_epoch(
|
||||
delete_prefix: true,
|
||||
delete_prefix_object: true,
|
||||
http_preconditions: Some(revision.preconditions()),
|
||||
user_defined: scanner_publication_lease_fence
|
||||
.map(|fence| {
|
||||
HashMap::from([(
|
||||
crate::storage_api::owner::SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY.to_string(),
|
||||
fence.to_string(),
|
||||
)])
|
||||
})
|
||||
.unwrap_or_default(),
|
||||
..Default::default()
|
||||
},
|
||||
read_epoch,
|
||||
|
||||
@@ -239,6 +239,13 @@ fn classify_nsscanner_cycle(
|
||||
dirty_usage_status: DirtyUsageSnapshotStatus,
|
||||
activity_status: ScannerCycleActivityStatus,
|
||||
) -> ScannerCycleStatus {
|
||||
// The post-scan activity proof is required regardless of why the scan was
|
||||
// incomplete. Returning Incomplete first would apply the long ordinary
|
||||
// retry/backoff path to an unverifiable publication and could acknowledge
|
||||
// a cycle without a movement-generation proof.
|
||||
if activity_status == ScannerCycleActivityStatus::Unverified {
|
||||
return ScannerCycleStatus::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable);
|
||||
}
|
||||
if budget_elapsed
|
||||
|| cancelled
|
||||
|| !matches!(bucket_scan_status, ScannerBucketScanStatus::Complete)
|
||||
@@ -252,7 +259,6 @@ fn classify_nsscanner_cycle(
|
||||
|
||||
match (activity_status, dirty_usage_status) {
|
||||
(ScannerCycleActivityStatus::Unchanged, DirtyUsageSnapshotStatus::Current) => ScannerCycleStatus::Complete,
|
||||
(ScannerCycleActivityStatus::Unverified, _) => ScannerCycleStatus::Incomplete,
|
||||
_ => ScannerCycleStatus::Superseded,
|
||||
}
|
||||
}
|
||||
@@ -307,10 +313,16 @@ async fn scanner_cycle_activity_status(
|
||||
store: &ECStore,
|
||||
distributed: bool,
|
||||
before: &crate::scanner::ScannerActivitySnapshot,
|
||||
) -> ScannerCycleActivityStatus {
|
||||
) -> (ScannerCycleActivityStatus, Vec<(String, String, u64)>) {
|
||||
match crate::scanner::probe_scanner_activity(store, distributed).await {
|
||||
Ok(after) if after == *before => ScannerCycleActivityStatus::Unchanged,
|
||||
Ok(_) => ScannerCycleActivityStatus::Changed,
|
||||
Ok(after) => {
|
||||
let status = if after == *before {
|
||||
ScannerCycleActivityStatus::Unchanged
|
||||
} else {
|
||||
ScannerCycleActivityStatus::Changed
|
||||
};
|
||||
(status, crate::scanner::scanner_activity_publication_lease_targets(&after))
|
||||
}
|
||||
Err(err) => {
|
||||
warn!(
|
||||
target: "rustfs::scanner::io",
|
||||
@@ -321,7 +333,7 @@ async fn scanner_cycle_activity_status(
|
||||
error = %err,
|
||||
"Scanner cycle activity verification failed"
|
||||
);
|
||||
ScannerCycleActivityStatus::Unverified
|
||||
(ScannerCycleActivityStatus::Unverified, Vec::new())
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -599,6 +611,7 @@ pub(crate) struct ScannerCycleResult {
|
||||
publication_epoch: Option<u64>,
|
||||
dirty_usage_clear: Option<DirtyUsageBuckets>,
|
||||
remote_dirty_usage_acknowledgements: Vec<crate::scanner::ScannerDirtyUsageAcknowledgement>,
|
||||
remote_publication_lease_targets: Vec<(String, String, u64)>,
|
||||
failed_dirty_usage: bool,
|
||||
pending_maintenance_work: bool,
|
||||
required_cycle_floor: Option<u64>,
|
||||
@@ -611,6 +624,7 @@ impl ScannerCycleResult {
|
||||
publication_epoch: None,
|
||||
dirty_usage_clear,
|
||||
remote_dirty_usage_acknowledgements: Vec::new(),
|
||||
remote_publication_lease_targets: Vec::new(),
|
||||
failed_dirty_usage: false,
|
||||
pending_maintenance_work: false,
|
||||
required_cycle_floor: None,
|
||||
@@ -649,6 +663,15 @@ impl ScannerCycleResult {
|
||||
self
|
||||
}
|
||||
|
||||
pub(crate) fn with_remote_publication_lease_targets(mut self, targets: Vec<(String, String, u64)>) -> Self {
|
||||
self.remote_publication_lease_targets = targets;
|
||||
self
|
||||
}
|
||||
|
||||
pub(crate) fn remote_publication_lease_targets(&self) -> &[(String, String, u64)] {
|
||||
&self.remote_publication_lease_targets
|
||||
}
|
||||
|
||||
pub(crate) fn acknowledge_durable_usage(self) -> Vec<crate::scanner::ScannerDirtyUsageAcknowledgement> {
|
||||
if let Some(snapshot) = self.dirty_usage_clear {
|
||||
clear_dirty_usage_buckets(&snapshot);
|
||||
|
||||
@@ -151,7 +151,8 @@ impl ScannerIOCycle for ECStore {
|
||||
ScannerCycleResult::new(ScannerCycleStatus::Incomplete, None).with_publication_epoch(publication_epoch)
|
||||
);
|
||||
}
|
||||
let activity_status = scanner_cycle_activity_status(self, distributed, &activity_before).await;
|
||||
let (activity_status, remote_publication_lease_targets) =
|
||||
scanner_cycle_activity_status(self, distributed, &activity_before).await;
|
||||
let dirty_usage_status = dirty_usage_snapshot_status(&dirty_usage_snapshot);
|
||||
let status = classify_nsscanner_cycle(
|
||||
true,
|
||||
@@ -187,6 +188,7 @@ impl ScannerIOCycle for ECStore {
|
||||
};
|
||||
return Ok(ScannerCycleResult::new(status, dirty_usage_clear)
|
||||
.with_publication_epoch(publication_epoch)
|
||||
.with_remote_publication_lease_targets(remote_publication_lease_targets)
|
||||
.with_remote_dirty_usage_acknowledgements(remote_dirty_usage_acknowledgements));
|
||||
}
|
||||
|
||||
@@ -400,7 +402,8 @@ impl ScannerIOCycle for ECStore {
|
||||
let budget_elapsed = budget.budget_elapsed();
|
||||
let dirty_usage_status = dirty_usage_snapshot_status(&dirty_usage_snapshot);
|
||||
let dirty_usage_current = dirty_usage_status == DirtyUsageSnapshotStatus::Current;
|
||||
let activity_status = scanner_cycle_activity_status(self, distributed, &activity_before).await;
|
||||
let (activity_status, remote_publication_lease_targets) =
|
||||
scanner_cycle_activity_status(self, distributed, &activity_before).await;
|
||||
let all_bucket_names = all_buckets.iter().map(|bucket| bucket.name.clone()).collect::<Vec<_>>();
|
||||
let completed_usage = completed_data_usage_info(
|
||||
&results,
|
||||
@@ -460,6 +463,7 @@ impl ScannerIOCycle for ECStore {
|
||||
};
|
||||
Ok(ScannerCycleResult::new(cycle_status, dirty_usage_clear)
|
||||
.with_publication_epoch(publication_epoch)
|
||||
.with_remote_publication_lease_targets(remote_publication_lease_targets)
|
||||
.with_remote_dirty_usage_acknowledgements(remote_dirty_usage_acknowledgements)
|
||||
.with_failed_dirty_usage(!failed_buckets.is_empty())
|
||||
.with_pending_maintenance_work(pending_maintenance_work)
|
||||
|
||||
@@ -766,7 +766,7 @@ fn scanner_cycle_status_requires_a_clean_complete_snapshot() {
|
||||
DirtyUsageSnapshotStatus::Current,
|
||||
ScannerCycleActivityStatus::Unverified,
|
||||
),
|
||||
ScannerCycleStatus::Incomplete
|
||||
ScannerCycleStatus::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable)
|
||||
);
|
||||
|
||||
for status in [
|
||||
@@ -815,6 +815,34 @@ fn scanner_cycle_status_requires_a_clean_complete_snapshot() {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unverified_activity_defers_partial_and_floor_cycles() {
|
||||
let expected = ScannerCycleStatus::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable);
|
||||
|
||||
assert_eq!(
|
||||
classify_nsscanner_cycle(
|
||||
true,
|
||||
false,
|
||||
false,
|
||||
ScannerBucketScanStatus::Partial,
|
||||
DirtyUsageSnapshotStatus::Current,
|
||||
ScannerCycleActivityStatus::Unverified,
|
||||
),
|
||||
expected
|
||||
);
|
||||
assert_eq!(
|
||||
classify_nsscanner_cycle(
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
ScannerBucketScanStatus::Complete,
|
||||
DirtyUsageSnapshotStatus::Current,
|
||||
ScannerCycleActivityStatus::Unverified,
|
||||
),
|
||||
expected
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn structurally_complete_superseded_cycles_publish_without_claiming_convergence() {
|
||||
let (updates, mut receiver) = mpsc::channel(2);
|
||||
@@ -834,6 +862,15 @@ async fn structurally_complete_superseded_cycles_publish_without_claiming_conver
|
||||
.await
|
||||
.expect("incomplete snapshot suppression should succeed")
|
||||
);
|
||||
assert!(
|
||||
!publish_usage_snapshot(
|
||||
&updates,
|
||||
ScannerCycleStatus::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable),
|
||||
DataUsageInfo::default(),
|
||||
)
|
||||
.await
|
||||
.expect("unverified activity suppression should succeed")
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
receiver
|
||||
@@ -855,11 +892,7 @@ async fn structurally_complete_superseded_cycles_publish_without_claiming_conver
|
||||
|
||||
#[test]
|
||||
fn scanner_cycle_fails_closed_for_namespace_disappearance() {
|
||||
for activity_status in [
|
||||
ScannerCycleActivityStatus::Changed,
|
||||
ScannerCycleActivityStatus::Unchanged,
|
||||
ScannerCycleActivityStatus::Unverified,
|
||||
] {
|
||||
for activity_status in [ScannerCycleActivityStatus::Changed, ScannerCycleActivityStatus::Unchanged] {
|
||||
assert_eq!(
|
||||
classify_nsscanner_cycle(
|
||||
false,
|
||||
@@ -872,6 +905,17 @@ fn scanner_cycle_fails_closed_for_namespace_disappearance() {
|
||||
ScannerCycleStatus::Incomplete
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
classify_nsscanner_cycle(
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
ScannerBucketScanStatus::NamespaceNotFound,
|
||||
DirtyUsageSnapshotStatus::Changed,
|
||||
ScannerCycleActivityStatus::Unverified,
|
||||
),
|
||||
ScannerCycleStatus::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable)
|
||||
);
|
||||
assert_eq!(
|
||||
classify_nsscanner_cycle(
|
||||
true,
|
||||
|
||||
@@ -85,6 +85,7 @@ pub(crate) use rustfs_ecstore::api::event::{EventArgs as EcstoreEventArgs, send_
|
||||
pub(crate) use rustfs_ecstore::api::layout::{
|
||||
EndpointServerPools as EcstoreEndpointServerPools, Endpoints as EcstoreEndpoints, PoolEndpoints as EcstorePoolEndpoints,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::api::object::SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY;
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::rebalance::{
|
||||
RebalStatus as EcstoreRebalStatus, RebalanceInfo as EcstoreRebalanceInfo, RebalanceMeta as EcstoreRebalanceMeta,
|
||||
@@ -98,9 +99,9 @@ pub(crate) use rustfs_ecstore::api::runtime::{
|
||||
setup_is_erasure_sd as ecstore_is_erasure_sd,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::api::set_disk::SetDisks as EcstoreSetDisks;
|
||||
pub(crate) use rustfs_ecstore::api::storage::ECStore as EcstoreStore;
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::storage::init_local_disks_with_instance_ctx as ecstore_init_local_disks_with_instance_ctx;
|
||||
pub(crate) use rustfs_ecstore::api::storage::{ECStore as EcstoreStore, SCANNER_PUBLICATION_LEASE_TTL_MS};
|
||||
use rustfs_storage_api as storage_contracts;
|
||||
|
||||
pub(crate) mod owner {
|
||||
@@ -115,10 +116,11 @@ pub(crate) mod owner {
|
||||
EcstoreDiskLocation, EcstoreDiskResult, EcstoreErrorType, EcstoreEvaluator, EcstoreEvent, EcstoreEventArgs,
|
||||
EcstoreLcEventSrc, EcstoreLifecycle, EcstoreListPathRawOptions, EcstoreNsScannerOpenRequest, EcstoreObjectOpts,
|
||||
EcstoreReplicationConfigurationExt, EcstoreReplicationScannerBridge, EcstoreResultType, EcstoreScanGuard,
|
||||
EcstoreSetDisks, EcstoreStorageError, EcstoreStore, EcstoreVersioningApi, ScannerReplicationHealObject,
|
||||
ScannerReplicationHealResult, ScannerReplicationQueueAdmission, ecstore_apply_expiry_rule, ecstore_apply_transition_rule,
|
||||
ecstore_expiry_state_handle, ecstore_get_global_tier_config_mgr, ecstore_get_lifecycle_config,
|
||||
ecstore_get_object_lock_config, ecstore_get_replication_config, ecstore_invalidate_admin_data_usage_snapshot_cache,
|
||||
EcstoreSetDisks, EcstoreStorageError, EcstoreStore, EcstoreVersioningApi, SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY,
|
||||
SCANNER_PUBLICATION_LEASE_TTL_MS, ScannerReplicationHealObject, ScannerReplicationHealResult,
|
||||
ScannerReplicationQueueAdmission, ecstore_apply_expiry_rule, ecstore_apply_transition_rule, ecstore_expiry_state_handle,
|
||||
ecstore_get_global_tier_config_mgr, ecstore_get_lifecycle_config, ecstore_get_object_lock_config,
|
||||
ecstore_get_replication_config, ecstore_invalidate_admin_data_usage_snapshot_cache,
|
||||
ecstore_invalidate_data_usage_snapshot_cache, ecstore_is_erasure, ecstore_is_erasure_sd,
|
||||
ecstore_is_reserved_or_invalid_bucket, ecstore_list_path_raw, ecstore_object_opts_from_object_info,
|
||||
ecstore_path2_bucket_object, ecstore_path2_bucket_object_with_base_path, ecstore_read_config,
|
||||
@@ -274,13 +276,13 @@ impl From<EcstoreReplicationHealQueueResult> for ScannerReplicationHealResult {
|
||||
}
|
||||
|
||||
pub(crate) mod scan {
|
||||
pub use super::storage_contracts::SCANNER_ACTIVITY_PROTOCOL_VERSION;
|
||||
pub(crate) use super::storage_contracts::{
|
||||
BucketOperations, BucketOptions, NamespaceLocking, SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION,
|
||||
SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use super::storage_contracts::{DeleteBucketOptions, MakeBucketOptions, ObjectIO};
|
||||
pub use super::storage_contracts::{SCANNER_ACTIVITY_PROTOCOL_VERSION, SCANNER_ACTIVITY_V6_PROTOCOL_VERSION};
|
||||
}
|
||||
|
||||
pub(crate) mod scanner_io {
|
||||
|
||||
@@ -50,7 +50,12 @@ pub const NS_SCANNER_TIER_REGISTRY_GENERATION_QUERY: &str = "ns_scanner_tier_reg
|
||||
pub const NS_SCANNER_PROTOCOL_VERSION: u16 = 3;
|
||||
pub const SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION: u32 = 0;
|
||||
pub const SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION: u32 = 5;
|
||||
pub const SCANNER_ACTIVITY_PROTOCOL_VERSION: u32 = 6;
|
||||
/// Protocol v6 carries the activity fields that predate the storage-owned
|
||||
/// movement generation. It remains readable during the rolling upgrade, but
|
||||
/// a scanner must not use it as a publication proof because terminal movement
|
||||
/// state is not authenticated by that version.
|
||||
pub const SCANNER_ACTIVITY_V6_PROTOCOL_VERSION: u32 = 6;
|
||||
pub const SCANNER_ACTIVITY_PROTOCOL_VERSION: u32 = 7;
|
||||
|
||||
#[derive(Debug, serde::Deserialize, serde::Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
|
||||
Reference in New Issue
Block a user