fix(scanner): fence movement generation publication (#6461)

* feat(scanner): add movement generation fencing

* fix(scanner): prioritize unverified cycle deferral

* feat(ecstore): add scanner publication lease fence

* feat(rpc): add scanner publication lease protocol

* feat(scanner): hold remote leases through usage publish

* test(scanner): cover publication lease fencing

* fix(scanner): fence remote leases across restart and delay

* feat(rpc): fence scanner publication rename writes

* fix(scanner): fence observed cleanup deletes

* fix(proto): qualify lease release test types

* fix(scanner): pin movement notifications

* fix(scanner): clean publication imports

* fix(ecstore): satisfy scanner fence clippy

* refactor(scanner): group wait and publication options

* fix(scanner): satisfy final lint and facade guards

* fix(rpc): resolve facade export conflicts

* fix(ci): remove unused decommission and healing facades

* fix(ci): cfg-gate test-only usage overlay import

* fix(scanner): wake on remote scanner restart
This commit is contained in:
cxymds
2026-08-24 14:17:35 +08:00
committed by GitHub
parent 1c5c28842a
commit eec0e0e056
30 changed files with 2926 additions and 230 deletions
+19 -17
View File
@@ -415,9 +415,9 @@ pub mod notification {
#[cfg(any(test, feature = "test-util"))]
pub use crate::services::notification_sys::rotate_cross_pool_fence_fleet_proof_for_test;
pub use crate::services::notification_sys::{
CrossPoolFenceFleetProofToken, NotificationPeerErr, NotificationSys, acquire_cross_pool_fence_fleet_proof,
cross_pool_fence_fleet_proof_matches, get_global_notification_sys, new_global_notification_sys,
start_remote_version_state_fleet_probe,
CrossPoolFenceFleetProofToken, NotificationPeerErr, NotificationSys, ScannerPublicationLeaseGrant,
acquire_cross_pool_fence_fleet_proof, cross_pool_fence_fleet_proof_matches, get_global_notification_sys,
new_global_notification_sys, start_remote_version_state_fleet_probe,
};
}
@@ -426,9 +426,10 @@ pub mod object {
BLOCK_SIZE_V2, ERASURE_ALGORITHM, EncryptionResolutionError, EncryptionResolutionErrorKind, GetObjectBodyCacheHook,
GetObjectBodyCacheHookLookup, GetObjectBodySource, GetObjectReader, NamespaceLockFence, ObjectEncryptionResolver,
ObjectInfo, ObjectLockConfigSnapshot, ObjectMutationHook, ObjectOptions, PutObjReader, QuotaAdmission,
RangedDecompressReader, ReadEncryptionMaterial, ReadEncryptionMode, ReadEncryptionRequest, StreamConsumer,
get_object_body_cache_plaintext_len, lookup_get_object_body_cache_hook, register_get_object_body_cache_hook,
register_object_mutation_hook, unregister_get_object_body_cache_hook, unregister_object_mutation_hook,
RangedDecompressReader, ReadEncryptionMaterial, ReadEncryptionMode, ReadEncryptionRequest,
SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY, StreamConsumer, get_object_body_cache_plaintext_len,
lookup_get_object_body_cache_hook, register_get_object_body_cache_hook, register_object_mutation_hook,
unregister_get_object_body_cache_hook, unregister_object_mutation_hook,
};
pub use crate::store::{
PrepareSelectObjectSnapshotError, PreparedGetObjectReader, SelectObjectSnapshot, SelectObjectSnapshotReadError,
@@ -461,16 +462,17 @@ pub mod rpc {
pub use crate::cluster::rpc::{
AuthenticatedChannel, KMS_SIGNAL_SUBSYSTEM, LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS,
PeerRestClient, PeerS3Client, S3PeerSys, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC,
ScannerBucketListing, ScannerPeerActivity, TONIC_RPC_PREFIX, TonicInterceptor, build_put_file_auth_trailer,
check_and_record_signed_rpc_nonce, decode_heal_bucket_rpc_options, encode_heal_bucket_rpc_options, gen_signature_headers,
gen_tonic_replay_scope_headers, gen_tonic_signature_headers, gen_tonic_signature_interceptor,
node_service_time_out_client, node_service_time_out_client_no_auth, normalize_tonic_rpc_audience,
set_tonic_canonical_body_digest, sign_ns_scanner_capability, sign_ns_scanner_capability_with_tier_registry_generation,
sign_put_file_capability, sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers,
tonic_rpc_auth_failure_reason, verify_ns_scanner_capability, verify_ns_scanner_capability_with_tier_registry_generation,
verify_put_file_auth_trailer, verify_put_file_capability, verify_rpc_signature, verify_tonic_boot_epoch_response,
verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof,
verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
ScannerBucketListing, ScannerPeerActivity, ScannerPublicationLease, TONIC_RPC_PREFIX, TonicInterceptor,
build_put_file_auth_trailer, check_and_record_signed_rpc_nonce, decode_heal_bucket_rpc_options,
encode_heal_bucket_rpc_options, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
gen_tonic_signature_interceptor, node_service_time_out_client, node_service_time_out_client_no_auth,
normalize_tonic_rpc_audience, set_tonic_canonical_body_digest, sign_ns_scanner_capability,
sign_ns_scanner_capability_with_tier_registry_generation, sign_put_file_capability, sign_tonic_rpc_response_proof,
tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason,
verify_ns_scanner_capability, verify_ns_scanner_capability_with_tier_registry_generation, verify_put_file_auth_trailer,
verify_put_file_capability, verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
verify_tonic_rpc_signature_with_bootstrap,
};
}
@@ -497,7 +499,7 @@ pub mod storage {
pub use crate::core::pools::HealLifecycleExpiryContext;
pub use crate::store::HealWalkVersion;
pub use crate::store::{
ECStore, all_local_disk, all_local_disk_path, find_local_disk_by_ref, init_local_disks,
ECStore, SCANNER_PUBLICATION_LEASE_TTL_MS, all_local_disk, all_local_disk_path, find_local_disk_by_ref, init_local_disks,
init_local_disks_with_instance_ctx, init_lock_clients, prewarm_local_disk_id_map,
prewarm_local_disk_id_map_with_instance_ctx,
};
@@ -2994,6 +2994,7 @@ mod tests {
dst_volume: "bucket".to_string(),
dst_path: "object".to_string(),
file_info_bin: vec![0x81, 0xA1, 0x76, 0x01].into(),
scanner_publication_lease_token: Vec::new().into(),
};
let body = rustfs_protos::canonical_rename_data_request_body(&message).expect("small request should encode");
let mut request = tonic::Request::new(());
+1 -1
View File
@@ -48,7 +48,7 @@ pub use internode_data_transport::build_internode_data_transport_from_env;
pub(crate) use peer_rest_client::TierConfigReloadOutcome;
pub use peer_rest_client::{
KMS_SIGNAL_SUBSYSTEM, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, SERVICE_SIGNAL_REFRESH_CONFIG,
SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerPeerActivity,
SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerPeerActivity, ScannerPublicationLease,
};
pub(crate) use peer_s3_client::heal_bucket_local_on_disks;
pub use peer_s3_client::{
@@ -20,6 +20,7 @@ use crate::cluster::rpc::{set_tonic_canonical_body_digest, set_tonic_mutation_bo
use crate::error::{Error, Result};
use crate::storage_api_contracts::internode::{
SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION, SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SCANNER_ACTIVITY_PROTOCOL_VERSION,
SCANNER_ACTIVITY_V6_PROTOCOL_VERSION,
};
use crate::{
bucket::replication::BucketStats,
@@ -45,8 +46,9 @@ use rustfs_protos::proto_gen::node_service::{
HealControlRequest, LoadBucketMetadataRequest, LoadGroupRequest, LoadPolicyMappingRequest, LoadPolicyRequest,
LoadRebalanceMetaRequest, LoadServiceAccountRequest, LoadTransitionTierConfigRequest, LoadUserRequest,
LocalStorageInfoRequest, Mss, ReloadPoolMetaRequest, ReloadSiteReplicationConfigRequest, ReplacementRecoveryStatusRequest,
ScannerActivityRequest, ScannerActivityResponse, ServerInfoRequest, SignalServiceRequest, SignalServiceResponse,
StartDecommissionRequest, StartProfilingRequest, StopRebalanceRequest, TierMutationAbortRequest, TierMutationCommitRequest,
ScannerActivityRequest, ScannerActivityResponse, ScannerPublicationLeaseReleaseRequest, ScannerPublicationLeaseRequest,
ScannerPublicationLeaseResponse, ServerInfoRequest, SignalServiceRequest, SignalServiceResponse, StartDecommissionRequest,
StartProfilingRequest, StopRebalanceRequest, TierMutationAbortRequest, TierMutationCommitRequest,
TierMutationControlResponse, TierMutationPeerState, TierMutationPrepareRequest, node_service_client::NodeServiceClient,
tier_mutation_control_service_client::TierMutationControlServiceClient,
};
@@ -84,6 +86,11 @@ const HEAL_CONTROL_PAYLOAD_MAX_SIZE: usize = 64 * 1024;
const PEER_REST_RECOVERY_MAX_ATTEMPTS: u32 = 60;
const PEER_REST_RECOVERY_MAX_BACKOFF: Duration = Duration::from_secs(30);
const SCANNER_ACTIVITY_MAX_MESSAGE_SIZE: usize = 1024;
/// Reserve time for the acquire response's network/clock uncertainty. The
/// server owns the real expiry; this local deadline is intentionally earlier
/// so a coordinator never starts a bounded persistence operation at the edge
/// of a remote lease.
const SCANNER_PUBLICATION_LEASE_SAFETY_MARGIN: Duration = Duration::from_secs(5);
const REPLICATION_STATS_MAX_MESSAGE_SIZE: usize = 8 * 1024 * 1024;
const BUCKET_METADATA_RELOAD_TIMEOUT: Duration = Duration::from_secs(5);
@@ -150,6 +157,8 @@ pub struct ScannerPeerActivity {
pub data_movement_active: Option<bool>,
pub dirty_usage_generation: Option<u64>,
pub dirty_usage_pending: Option<bool>,
pub movement_generation: Option<u64>,
pub publication_blocked: Option<bool>,
}
fn decode_scanner_activity_with_verifier(
@@ -166,7 +175,14 @@ fn decode_scanner_activity_with_verifier(
{
return Err(Error::other("peer returned an invalid scanner activity instance ID"));
}
let (topology_digest, data_movement_active, dirty_usage_generation, dirty_usage_pending) = match response.protocol_version {
let (
topology_digest,
data_movement_active,
dirty_usage_generation,
dirty_usage_pending,
movement_generation,
publication_blocked,
) = match response.protocol_version {
// RUSTFS_COMPAT_TODO(ns-scanner-rpc-v3): legacy response fields are unauthenticated. Remove after protocol v0 peers are unsupported.
SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION
if response.topology_digest.is_empty()
@@ -175,7 +191,7 @@ fn decode_scanner_activity_with_verifier(
&& response.dirty_usage_generation == 0
&& !response.dirty_usage_pending =>
{
(None, None, None, None)
(None, None, None, None, None, None)
}
SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION => {
return Err(Error::other("legacy scanner activity peer returned unexpected extended fields"));
@@ -198,9 +214,11 @@ fn decode_scanner_activity_with_verifier(
Some(response.data_movement_active),
None,
None,
None,
None,
)
}
SCANNER_ACTIVITY_PROTOCOL_VERSION => {
SCANNER_ACTIVITY_V6_PROTOCOL_VERSION => {
if response.dirty_usage_pending && response.dirty_usage_generation == 0 {
return Err(Error::other("scanner activity peer returned pending dirty usage without a generation"));
}
@@ -218,11 +236,42 @@ fn decode_scanner_activity_with_verifier(
Some(response.data_movement_active),
Some(response.dirty_usage_generation),
Some(response.dirty_usage_pending),
None,
None,
)
}
version => {
return Err(Error::other(format!("peer returned unsupported scanner activity protocol {version}")));
SCANNER_ACTIVITY_PROTOCOL_VERSION => {
if response.dirty_usage_pending && response.dirty_usage_generation == 0 {
return Err(Error::other("scanner activity peer returned pending dirty usage without a generation"));
}
let movement_generation = response
.movement_generation
.ok_or_else(|| Error::other("scanner activity peer omitted its movement generation"))?;
let publication_blocked = response
.publication_blocked
.ok_or_else(|| Error::other("scanner activity peer omitted its publication blocked state"))?;
if movement_generation == u64::MAX {
return Err(Error::other("scanner activity peer exhausted its movement generation"));
}
let canonical = rustfs_protos::canonical_scanner_activity_v7_response_body(challenge, &response)
.map_err(|_| Error::other("scanner activity peer response is too large to authenticate"))?;
verify_proof(&canonical, &response.response_proof)?;
(
Some(
response
.topology_digest
.as_ref()
.try_into()
.map_err(|_| Error::other("peer returned an invalid scanner topology digest"))?,
),
Some(response.data_movement_active),
Some(response.dirty_usage_generation),
Some(response.dirty_usage_pending),
Some(movement_generation),
Some(publication_blocked),
)
}
version => return Err(Error::other(format!("peer returned unsupported scanner activity protocol {version}"))),
};
Ok(ScannerPeerActivity {
instance_id: response.instance_id,
@@ -233,6 +282,8 @@ fn decode_scanner_activity_with_verifier(
data_movement_active,
dirty_usage_generation,
dirty_usage_pending,
movement_generation,
publication_blocked,
})
}
@@ -243,6 +294,17 @@ fn decode_scanner_activity(response: ScannerActivityResponse, challenge: &[u8; 1
})
}
fn scanner_activity_protocol_unsupported(err: &Error) -> bool {
matches!(
err,
Error::Io(io_err)
if embedded_tonic_status(io_err).is_some_and(|status| {
status.code() == tonic::Code::FailedPrecondition
&& status.message().starts_with("unsupported scanner activity request protocol")
})
)
}
fn validate_heal_control_capability_proof(canonical_ack: &[u8], proof: &[u8]) -> Result<()> {
verify_tonic_rpc_response_proof(canonical_ack, proof)
.map_err(|_| Error::other("peer returned an invalid heal control capability proof"))
@@ -285,6 +347,76 @@ pub struct PeerLiveEventsBatch {
pub truncated: bool,
}
#[derive(Clone, Debug)]
pub struct ScannerPublicationLease {
pub token: Uuid,
pub movement_generation: u64,
/// Stable storage owner identity. This is distinct from the activity
/// session and is bound into both acquire and release proofs.
pub owner_id: String,
/// Process/session nonce observed by the final activity probe.
pub session_id: String,
pub expires_at: std::time::Instant,
}
impl ScannerPublicationLease {
pub fn is_valid(&self) -> bool {
std::time::Instant::now() < self.expires_at
}
}
fn validate_scanner_publication_lease_response_fields(
response: &ScannerPublicationLeaseResponse,
expected_session_id: &str,
expected_generation: u64,
) -> Result<(Uuid, String)> {
if !response.success {
return Err(Error::other(
response
.error
.as_ref()
.map(|error| error.error_info.clone())
.unwrap_or_else(|| "peer rejected scanner publication lease".to_string()),
));
}
if response.movement_generation != expected_generation {
return Err(Error::other("peer returned a different scanner publication lease generation"));
}
if response.session_id != expected_session_id {
return Err(Error::other("peer returned a different scanner publication lease session"));
}
let owner_id = Uuid::parse_str(&response.owner_id)
.ok()
.filter(|owner_id| !owner_id.is_nil())
.map(|owner_id| owner_id.to_string())
.ok_or_else(|| Error::other("peer returned an invalid scanner publication lease owner"))?;
if response.lease_ttl_ms != crate::store::SCANNER_PUBLICATION_LEASE_TTL_MS {
return Err(Error::other("peer returned an unsupported scanner publication lease TTL"));
}
let token = Uuid::from_slice(response.token.as_ref())
.map_err(|_| Error::other("peer returned an invalid scanner publication lease token"))?;
Ok((token, owner_id))
}
fn scanner_publication_lease_deadline(
request_started: std::time::Instant,
response_received: std::time::Instant,
lease_ttl_ms: u64,
) -> Result<std::time::Instant> {
let lease_window = Duration::from_millis(lease_ttl_ms)
.checked_sub(SCANNER_PUBLICATION_LEASE_SAFETY_MARGIN)
.ok_or_else(|| Error::other("scanner publication lease TTL is shorter than its safety margin"))?;
let elapsed = response_received
.checked_duration_since(request_started)
.ok_or_else(|| Error::other("scanner publication lease response clock moved backwards"))?;
if elapsed >= lease_window {
return Err(Error::other("scanner publication lease response arrived after its safety window"));
}
request_started
.checked_add(lease_window)
.ok_or_else(|| Error::other("scanner publication lease deadline overflowed"))
}
#[derive(Clone, Debug)]
pub struct PeerRestClient {
pub host: XHost,
@@ -1640,10 +1772,11 @@ impl PeerRestClient {
.await
}
async fn scanner_activity_request(
async fn scanner_activity_request_with_protocol(
&self,
acknowledge_instance_id: String,
acknowledge_dirty_usage_generation: u64,
protocol_version: u32,
) -> Result<ScannerPeerActivity> {
self.finalize_result(
async {
@@ -1655,7 +1788,7 @@ impl PeerRestClient {
.max_encoding_message_size(SCANNER_ACTIVITY_MAX_MESSAGE_SIZE);
let mut request = Request::new(ScannerActivityRequest {
challenge: challenge.as_bytes().to_vec().into(),
protocol_version: SCANNER_ACTIVITY_PROTOCOL_VERSION,
protocol_version,
acknowledge_instance_id,
acknowledge_dirty_usage_generation,
});
@@ -1671,11 +1804,168 @@ impl PeerRestClient {
}
pub async fn scanner_activity(&self) -> Result<ScannerPeerActivity> {
self.scanner_activity_request(String::new(), 0).await
let result = self
.scanner_activity_request_with_protocol(String::new(), 0, SCANNER_ACTIVITY_PROTOCOL_VERSION)
.await;
if result.as_ref().err().is_some_and(scanner_activity_protocol_unsupported) {
// A v6 peer cannot parse the v7 marker. Its authenticated
// response is still decoded as untrusted terminal state, so the
// scanner will defer publication until every peer is v7.
self.scanner_activity_request_with_protocol(String::new(), 0, SCANNER_ACTIVITY_V6_PROTOCOL_VERSION)
.await
} else {
result
}
}
pub async fn acknowledge_scanner_dirty_usage(&self, instance_id: String, generation: u64) -> Result<ScannerPeerActivity> {
self.scanner_activity_request(instance_id, generation).await
let result = self
.scanner_activity_request_with_protocol(instance_id.clone(), generation, SCANNER_ACTIVITY_PROTOCOL_VERSION)
.await;
if result.as_ref().err().is_some_and(scanner_activity_protocol_unsupported) {
self.scanner_activity_request_with_protocol(instance_id, generation, SCANNER_ACTIVITY_V6_PROTOCOL_VERSION)
.await
} else {
result
}
}
/// Acquire a bounded, storage-owned read admission on the peer that
/// produced the final activity generation. Older peers do not implement
/// the lease form and are rejected rather than downgraded.
pub async fn acquire_scanner_publication_lease(
&self,
expected_session_id: &str,
expected_generation: u64,
) -> Result<ScannerPublicationLease> {
let request_started = std::time::Instant::now();
self.finalize_result(
async {
let challenge = Uuid::new_v4();
let mut client = self
.get_client()
.await?
.max_decoding_message_size(SCANNER_ACTIVITY_MAX_MESSAGE_SIZE)
.max_encoding_message_size(SCANNER_ACTIVITY_MAX_MESSAGE_SIZE);
let mut request = Request::new(ScannerPublicationLeaseRequest {
challenge: challenge.as_bytes().to_vec().into(),
expected_movement_generation: expected_generation,
ttl_ms: crate::store::SCANNER_PUBLICATION_LEASE_TTL_MS,
expected_session_id: expected_session_id.to_string(),
token: Bytes::new(),
});
let canonical = rustfs_protos::canonical_scanner_publication_lease_request_body(request.get_ref())
.map_err(|_| Error::other("scanner publication lease request is too large to authenticate"))?;
set_tonic_canonical_body_digest(&mut request, &canonical)?;
let response = client.acquire_scanner_publication_lease(request).await?.into_inner();
let response_body =
rustfs_protos::canonical_scanner_publication_lease_response_body(challenge.as_bytes(), &response)
.map_err(|_| Error::other("scanner publication lease response is too large to authenticate"))?;
verify_tonic_rpc_response_proof(&response_body, &response.response_proof)
.map_err(|_| Error::other("peer returned an invalid scanner publication lease proof"))?;
let (token, owner_id) =
validate_scanner_publication_lease_response_fields(&response, expected_session_id, expected_generation)?;
Ok(ScannerPublicationLease {
token,
movement_generation: response.movement_generation,
owner_id,
session_id: response.session_id,
expires_at: scanner_publication_lease_deadline(
request_started,
std::time::Instant::now(),
response.lease_ttl_ms,
)?,
})
}
.await,
)
.await
}
/// Revalidate the exact token immediately before the coordinator commits
/// its final publication. The peer keeps the original movement read
/// guard in its token table; a restart drops that table and changes the
/// activity session, so this proof fails closed instead of accepting an
/// ABA generation value.
pub async fn validate_scanner_publication_lease(&self, lease: &ScannerPublicationLease) -> Result<()> {
self.finalize_result(
async {
let challenge = Uuid::new_v4();
let mut client = self
.get_client()
.await?
.max_decoding_message_size(SCANNER_ACTIVITY_MAX_MESSAGE_SIZE)
.max_encoding_message_size(SCANNER_ACTIVITY_MAX_MESSAGE_SIZE);
let mut request = Request::new(ScannerPublicationLeaseRequest {
challenge: challenge.as_bytes().to_vec().into(),
expected_movement_generation: lease.movement_generation,
ttl_ms: crate::store::SCANNER_PUBLICATION_LEASE_TTL_MS,
expected_session_id: lease.session_id.clone(),
token: lease.token.as_bytes().to_vec().into(),
});
let canonical = rustfs_protos::canonical_scanner_publication_lease_request_body(request.get_ref())
.map_err(|_| Error::other("scanner publication lease validation request is too large to authenticate"))?;
set_tonic_canonical_body_digest(&mut request, &canonical)?;
let response = client.acquire_scanner_publication_lease(request).await?.into_inner();
let response_body =
rustfs_protos::canonical_scanner_publication_lease_response_body(challenge.as_bytes(), &response).map_err(
|_| Error::other("scanner publication lease validation response is too large to authenticate"),
)?;
verify_tonic_rpc_response_proof(&response_body, &response.response_proof)
.map_err(|_| Error::other("peer returned an invalid scanner publication lease validation proof"))?;
let (token, owner_id) =
validate_scanner_publication_lease_response_fields(&response, &lease.session_id, lease.movement_generation)?;
if token != lease.token {
return Err(Error::other("peer returned a different scanner publication lease token"));
}
if owner_id != lease.owner_id {
return Err(Error::other("peer returned a different scanner publication lease owner"));
}
Ok(())
}
.await,
)
.await
}
pub async fn release_scanner_publication_lease(&self, lease: &ScannerPublicationLease) -> Result<()> {
self.finalize_result(
async {
let challenge = Uuid::new_v4();
let mut client = self.get_client().await?;
let mut request = Request::new(ScannerPublicationLeaseReleaseRequest {
challenge: challenge.as_bytes().to_vec().into(),
token: lease.token.as_bytes().to_vec().into(),
owner_id: lease.owner_id.clone(),
session_id: lease.session_id.clone(),
});
let canonical = rustfs_protos::canonical_scanner_publication_lease_release_request_body(request.get_ref())
.map_err(|_| Error::other("scanner publication lease release request is too large to authenticate"))?;
set_tonic_canonical_body_digest(&mut request, &canonical)?;
let request_body = request.get_ref().clone();
let response = client.release_scanner_publication_lease(request).await?.into_inner();
let response_body = rustfs_protos::canonical_scanner_publication_lease_release_response_body(
challenge.as_bytes(),
&request_body,
&response,
)
.map_err(|_| Error::other("scanner publication lease release response is too large to authenticate"))?;
verify_tonic_rpc_response_proof(&response_body, &response.response_proof)
.map_err(|_| Error::other("peer returned an invalid scanner publication lease release proof"))?;
if response.success {
Ok(())
} else {
Err(Error::other(
response
.error
.map(|error| error.error_info)
.unwrap_or_else(|| "peer rejected scanner publication lease release".to_string()),
))
}
}
.await,
)
.await
}
pub async fn get_metacache_listing(&self) -> Result<()> {
@@ -1991,6 +2281,52 @@ mod tests {
use temp_env::async_with_vars;
use tracing_subscriber::{Registry, fmt::MakeWriter, layer::SubscriberExt};
#[test]
fn scanner_publication_lease_response_rejects_stale_generation_and_session() {
let token = Uuid::new_v4();
let response = ScannerPublicationLeaseResponse {
success: true,
token: token.as_bytes().to_vec().into(),
movement_generation: 7,
lease_ttl_ms: crate::store::SCANNER_PUBLICATION_LEASE_TTL_MS,
error: None,
response_proof: Bytes::new(),
owner_id: Uuid::new_v4().to_string(),
session_id: "session-a".to_string(),
};
assert!(validate_scanner_publication_lease_response_fields(&response, "session-a", 7).is_ok());
let stale_generation = ScannerPublicationLeaseResponse {
movement_generation: 6,
..response.clone()
};
let error = validate_scanner_publication_lease_response_fields(&stale_generation, "session-a", 7)
.expect_err("a response from an older movement generation must be rejected");
assert!(error.to_string().contains("different scanner publication lease generation"));
let stale_session = ScannerPublicationLeaseResponse {
session_id: "session-b".to_string(),
..response
};
let error = validate_scanner_publication_lease_response_fields(&stale_session, "session-a", 7)
.expect_err("a response from an older scanner session must be rejected");
assert!(error.to_string().contains("different scanner publication lease session"));
}
#[test]
fn scanner_publication_lease_deadline_accounts_for_delayed_rpc_response() {
let started = std::time::Instant::now();
let expected_deadline = started + Duration::from_secs(55);
let deadline = scanner_publication_lease_deadline(started, started + Duration::from_secs(10), 60_000)
.expect("a response inside the safety window should retain the original deadline");
assert_eq!(deadline, expected_deadline);
let error = scanner_publication_lease_deadline(started, started + Duration::from_secs(55), 60_000)
.expect_err("a response arriving at the safety boundary must fail closed");
assert!(error.to_string().contains("after its safety window"));
}
#[test]
fn replication_stats_response_decodes_valid_empty_provider() {
let mut stats = BucketStats::default();
@@ -2229,6 +2565,8 @@ mod tests {
response_proof: Vec::new().into(),
dirty_usage_generation: 0,
dirty_usage_pending: false,
movement_generation: None,
publication_blocked: None,
})
.expect("legacy peers should retain their activity generations during a rolling upgrade");
assert_eq!(
@@ -2242,6 +2580,8 @@ mod tests {
data_movement_active: None,
dirty_usage_generation: None,
dirty_usage_pending: None,
movement_generation: None,
publication_blocked: None,
}
);
@@ -2255,6 +2595,8 @@ mod tests {
response_proof: b"proof".to_vec().into(),
dirty_usage_generation: 0,
dirty_usage_pending: false,
movement_generation: None,
publication_blocked: None,
})
.expect("protocol v4 peers should remain observable during a rolling upgrade");
assert_eq!(
@@ -2268,9 +2610,29 @@ mod tests {
data_movement_active: Some(true),
dirty_usage_generation: None,
dirty_usage_pending: None,
movement_generation: None,
publication_blocked: None,
}
);
let v6 = decode_test_scanner_activity(ScannerActivityResponse {
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
namespace_generation: 7,
maintenance_generation: 3,
protocol_version: SCANNER_ACTIVITY_V6_PROTOCOL_VERSION,
topology_digest: vec![7; 32].into(),
data_movement_active: true,
response_proof: b"proof".to_vec().into(),
dirty_usage_generation: 11,
dirty_usage_pending: true,
movement_generation: None,
publication_blocked: None,
})
.expect("v6 peers should remain readable without a v7 publication proof");
assert_eq!(v6.movement_generation, None);
assert_eq!(v6.publication_blocked, None);
assert_eq!(v6.dirty_usage_generation, Some(11));
let malformed_topology = ScannerActivityResponse {
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
namespace_generation: 7,
@@ -2281,6 +2643,8 @@ mod tests {
response_proof: b"proof".to_vec().into(),
dirty_usage_generation: 11,
dirty_usage_pending: true,
movement_generation: Some(19),
publication_blocked: Some(false),
};
assert!(
decode_test_scanner_activity(malformed_topology)
@@ -2299,6 +2663,8 @@ mod tests {
response_proof: b"proof".to_vec().into(),
dirty_usage_generation: 11,
dirty_usage_pending: true,
movement_generation: Some(19),
publication_blocked: Some(false),
};
assert!(
decode_test_scanner_activity(missing_instance)
@@ -2317,6 +2683,8 @@ mod tests {
response_proof: b"proof".to_vec().into(),
dirty_usage_generation: 11,
dirty_usage_pending: true,
movement_generation: Some(19),
publication_blocked: Some(false),
};
assert!(
decode_test_scanner_activity(malformed_instance)
@@ -2335,6 +2703,8 @@ mod tests {
response_proof: b"proof".to_vec().into(),
dirty_usage_generation: 11,
dirty_usage_pending: true,
movement_generation: Some(19),
publication_blocked: Some(false),
})
.expect("complete activity responses should be accepted");
assert_eq!(
@@ -2348,9 +2718,31 @@ mod tests {
data_movement_active: Some(true),
dirty_usage_generation: Some(11),
dirty_usage_pending: Some(true),
movement_generation: Some(19),
publication_blocked: Some(false),
}
);
let missing_movement_generation = ScannerActivityResponse {
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
namespace_generation: 7,
maintenance_generation: 3,
protocol_version: SCANNER_ACTIVITY_PROTOCOL_VERSION,
topology_digest: vec![7; 32].into(),
data_movement_active: false,
response_proof: b"proof".to_vec().into(),
dirty_usage_generation: 11,
dirty_usage_pending: false,
movement_generation: None,
publication_blocked: Some(false),
};
assert!(
decode_test_scanner_activity(missing_movement_generation)
.expect_err("v7 activity must carry movement generation")
.to_string()
.contains("movement generation")
);
let pending_without_generation = ScannerActivityResponse {
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
namespace_generation: 7,
@@ -2361,6 +2753,8 @@ mod tests {
response_proof: b"proof".to_vec().into(),
dirty_usage_generation: 0,
dirty_usage_pending: true,
movement_generation: Some(19),
publication_blocked: Some(false),
};
assert!(
decode_test_scanner_activity(pending_without_generation)
@@ -2379,6 +2773,8 @@ mod tests {
response_proof: b"proof".to_vec().into(),
dirty_usage_generation: 11,
dirty_usage_pending: true,
movement_generation: None,
publication_blocked: None,
};
assert!(
decode_test_scanner_activity(previous_with_dirty_usage)
@@ -2397,6 +2793,8 @@ mod tests {
response_proof: b"proof".to_vec().into(),
dirty_usage_generation: 0,
dirty_usage_pending: false,
movement_generation: None,
publication_blocked: None,
};
assert!(
decode_test_scanner_activity(legacy_with_topology)
@@ -2415,6 +2813,8 @@ mod tests {
response_proof: b"proof".to_vec().into(),
dirty_usage_generation: 11,
dirty_usage_pending: true,
movement_generation: None,
publication_blocked: None,
};
assert!(
decode_test_scanner_activity(unsupported_protocol)
@@ -2433,6 +2833,8 @@ mod tests {
response_proof: Vec::new().into(),
dirty_usage_generation: 11,
dirty_usage_pending: true,
movement_generation: Some(19),
publication_blocked: Some(false),
};
assert!(
decode_test_scanner_activity(missing_proof)
+89 -42
View File
@@ -1981,6 +1981,20 @@ impl RemoteDisk {
fi: &FileInfo,
dst_volume: &str,
dst_path: &str,
) -> Result<RenameDataResp> {
self.rename_data_borrowed_with_fence(src_volume, src_path, fi, dst_volume, dst_path, None)
.await
}
#[tracing::instrument(level = "trace", skip_all)]
pub(crate) async fn rename_data_borrowed_with_fence(
&self,
src_volume: &str,
src_path: &str,
fi: &FileInfo,
dst_volume: &str,
dst_path: &str,
scanner_publication_lease_token: Option<Uuid>,
) -> Result<RenameDataResp> {
trace!(
event = EVENT_REMOTE_DISK_RPC,
@@ -2013,9 +2027,18 @@ impl RemoteDisk {
dst_volume: dst_volume.to_string(),
dst_path: dst_path.to_string(),
file_info_bin: file_info_bin.into(),
scanner_publication_lease_token: scanner_publication_lease_token
.map(|token| token.as_bytes().to_vec().into())
.unwrap_or_default(),
});
let canonical_body = rustfs_protos::canonical_rename_data_request_body(request.get_ref());
attach_mutation_body_digest(&mut request, canonical_body, "rename_data")?;
if scanner_publication_lease_token.is_some() {
let canonical_body =
canonical_body.map_err(|_| Error::other("rename_data request length cannot be represented"))?;
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &canonical_body).map_err(Error::other)?;
} else {
attach_mutation_body_digest(&mut request, canonical_body, "rename_data")?;
}
let response = client.rename_data(request).await?.into_inner();
@@ -2035,6 +2058,70 @@ impl RemoteDisk {
)
.await
}
/// Delete a path while binding the target-side operation to a scanner
/// publication lease. The ordinary `DiskAPI::delete` path keeps the
/// legacy digest/compatibility behavior by passing no token.
#[tracing::instrument(level = "trace", skip_all)]
pub(crate) async fn delete_with_scanner_publication_lease(
&self,
volume: &str,
path: &str,
opt: DeleteOptions,
scanner_publication_lease_token: Option<Uuid>,
) -> Result<()> {
trace!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
volume,
path,
recursive = opt.recursive,
immediate = opt.immediate,
fenced = scanner_publication_lease_token.is_some(),
op = "delete",
state = "started",
"Remote disk RPC started"
);
self.execute_with_timeout(
|| async {
let options = serde_json::to_string(&opt)?;
let mut client = self
.get_client()
.await
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
let mut request = Request::new(DeleteRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
path: path.to_string(),
options,
scanner_publication_lease_token: scanner_publication_lease_token
.map(|token| token.as_bytes().to_vec().into())
.unwrap_or_default(),
});
let canonical_body = rustfs_protos::canonical_delete_request_body(request.get_ref());
if scanner_publication_lease_token.is_some() {
let canonical_body =
canonical_body.map_err(|_| Error::other("delete request length cannot be represented"))?;
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &canonical_body).map_err(Error::other)?;
} else {
attach_mutation_body_digest(&mut request, canonical_body, "delete")?;
}
let response = client.delete(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
Ok(())
},
get_max_timeout_duration(),
)
.await
}
}
#[async_trait::async_trait]
@@ -3446,47 +3533,7 @@ impl DiskAPI for RemoteDisk {
#[tracing::instrument(level = "trace", skip_all)]
async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()> {
trace!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
volume,
path,
recursive = opt.recursive,
immediate = opt.immediate,
op = "delete",
state = "started",
"Remote disk RPC started"
);
self.execute_with_timeout(
|| async {
let options = serde_json::to_string(&opt)?;
let mut client = self
.get_client()
.await
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
let mut request = Request::new(DeleteRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
path: path.to_string(),
options,
});
let canonical_body = rustfs_protos::canonical_delete_request_body(request.get_ref());
attach_mutation_body_digest(&mut request, canonical_body, "delete")?;
let response = client.delete(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
Ok(())
},
get_max_timeout_duration(),
)
.await
self.delete_with_scanner_publication_lease(volume, path, opt, None).await
}
#[tracing::instrument(level = "trace", skip_all)]
+38 -1
View File
@@ -677,6 +677,23 @@ impl DiskAPI for Disk {
}
impl Disk {
pub(crate) async fn delete_with_scanner_publication_lease(
&self,
volume: &str,
path: &str,
opts: DeleteOptions,
scanner_publication_lease_token: Option<Uuid>,
) -> Result<()> {
match self {
Disk::Local(local_disk) => local_disk.delete(volume, path, opts).await,
Disk::Remote(remote_disk) => {
remote_disk
.delete_with_scanner_publication_lease(volume, path, opts, scanner_publication_lease_token)
.await
}
}
}
pub(crate) async fn rename_data_borrowed(
&self,
src_volume: &str,
@@ -684,6 +701,19 @@ impl Disk {
fi: &FileInfo,
dst_volume: &str,
dst_path: &str,
) -> Result<RenameDataResp> {
self.rename_data_borrowed_with_fence(src_volume, src_path, fi, dst_volume, dst_path, None)
.await
}
pub(crate) async fn rename_data_borrowed_with_fence(
&self,
src_volume: &str,
src_path: &str,
fi: &FileInfo,
dst_volume: &str,
dst_path: &str,
scanner_publication_lease_token: Option<Uuid>,
) -> Result<RenameDataResp> {
match self {
Disk::Local(local_disk) => {
@@ -693,7 +723,14 @@ impl Disk {
}
Disk::Remote(remote_disk) => {
remote_disk
.rename_data_borrowed(src_volume, src_path, fi, dst_volume, dst_path)
.rename_data_borrowed_with_fence(
src_volume,
src_path,
fi,
dst_volume,
dst_path,
scanner_publication_lease_token,
)
.await
}
}
+5
View File
@@ -446,6 +446,11 @@ pub struct ObjectOptions {
pub tier_delete_journal_api: Option<Arc<crate::store::ECStore>>,
}
/// Transient scanner-only carrier for target-side publication lease tokens.
/// SetDisks consumes and removes this key before constructing durable
/// FileInfo metadata; it must never appear in an S3-visible object.
pub const SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY: &str = "x-rustfs-internal-scanner-publication-lease-fence-v1";
impl ObjectOptions {
pub fn set_quota_admission(&mut self, current_usage: u64, quota_limit: u64) -> bool {
self.quota_admission = (current_usage <= quota_limit).then_some(QuotaAdmission {
+154 -1
View File
@@ -56,7 +56,8 @@ use std::sync::{
Arc, OnceLock,
atomic::{AtomicBool, AtomicU8, AtomicU64, Ordering},
};
use tokio::sync::{OnceCell, RwLock};
use tokio::sync::{Mutex, Notify, OnceCell, OwnedRwLockReadGuard, RwLock};
use tokio::time::Instant;
use tokio_util::sync::CancellationToken;
use uuid::Uuid;
@@ -64,6 +65,19 @@ const SCANNER_PUBLICATION_STATE_UNKNOWN: u8 = 0;
const SCANNER_PUBLICATION_STATE_ALLOWED: u8 = 1;
const SCANNER_PUBLICATION_STATE_BLOCKED: u8 = 2;
pub(crate) const SCANNER_PUBLICATION_LEASE_MAX_ENTRIES: usize = 256;
/// A lease is deliberately short-lived. The coordinator treats expiry as a
/// failed publication rather than silently continuing with a peer that may
/// have started movement after the lease was abandoned.
pub(crate) const SCANNER_PUBLICATION_LEASE_TTL: std::time::Duration = std::time::Duration::from_secs(60);
pub(crate) struct ScannerPublicationLeaseEntry {
pub(crate) expires_at: Instant,
pub(crate) movement_generation: u64,
pub(crate) _operation_guard: OwnedRwLockReadGuard<()>,
}
/// Runtime state owned by a single `ECStore` instance.
///
/// This is intentionally minimal in the first migration slice; subsequent
@@ -171,6 +185,11 @@ pub struct InstanceContext {
/// Readers are held across one publication commit; movement transitions
/// take the writer at their durable state commit boundary.
data_movement_operation_gate: Arc<RwLock<()>>,
/// Remote scanner publication leases own a read guard until explicit
/// release or bounded expiry. Keeping the guard in storage-owned state
/// makes a remote movement transition wait on the same fence as a local
/// scanner commit.
scanner_publication_leases: Arc<Mutex<HashMap<Uuid, ScannerPublicationLeaseEntry>>>,
/// Monotonic admission epoch paired with the operation gate. A
/// publication admitted before a movement transition must never be
/// mistaken for one admitted after the transition.
@@ -180,6 +199,13 @@ pub struct InstanceContext {
/// saturating counter prevents an unchanged `u64::MAX` value from being
/// mistaken for a fresh epoch after overflow.
data_movement_operation_epoch_exhausted: AtomicBool,
/// Storage-owned generation for movement state changes. This is separate
/// from the publication admission epoch so scanners can wait for a
/// terminal/clear transition without treating the wake as a publication
/// permit.
data_movement_generation: AtomicU64,
data_movement_generation_exhausted: AtomicBool,
data_movement_generation_notify: Arc<Notify>,
/// Last storage-owned movement snapshot observed under the operation
/// gate. SetDisks cache writers fail closed until ECStore refreshes it.
scanner_publication_state: AtomicU8,
@@ -224,8 +250,12 @@ impl InstanceContext {
bucket_metadata_sys: std::sync::Mutex::new(None),
background_cancel_token: OnceLock::new(),
data_movement_operation_gate: Arc::new(RwLock::new(())),
scanner_publication_leases: Arc::new(Mutex::new(HashMap::new())),
data_movement_operation_epoch: AtomicU64::new(0),
data_movement_operation_epoch_exhausted: AtomicBool::new(false),
data_movement_generation: AtomicU64::new(0),
data_movement_generation_exhausted: AtomicBool::new(false),
data_movement_generation_notify: Arc::new(Notify::new()),
scanner_publication_state: AtomicU8::new(SCANNER_PUBLICATION_STATE_UNKNOWN),
object_encryption_resolver: OnceLock::new(),
tier_delete_journal_recovery_stores: std::sync::Mutex::new(HashSet::new()),
@@ -249,6 +279,75 @@ impl InstanceContext {
Arc::clone(&self.data_movement_operation_gate)
}
pub(crate) async fn install_scanner_publication_lease(
&self,
token: Uuid,
expires_at: Instant,
movement_generation: u64,
operation_guard: OwnedRwLockReadGuard<()>,
) -> bool {
let mut leases = self.scanner_publication_leases.lock().await;
if leases.len() >= SCANNER_PUBLICATION_LEASE_MAX_ENTRIES {
return false;
}
leases.insert(
token,
ScannerPublicationLeaseEntry {
expires_at,
movement_generation,
_operation_guard: operation_guard,
},
);
true
}
pub(crate) async fn remove_scanner_publication_lease(&self, token: Uuid) -> bool {
self.scanner_publication_leases.lock().await.remove(&token).is_some()
}
/// Check a lease token while the caller holds the movement read guard.
///
/// The token table is deliberately process-owned and non-persistent: a
/// restarted instance has no entries from the previous process, so an old
/// coordinator proof cannot become valid again merely because the
/// movement generation counter restarted at zero.
pub(crate) async fn scanner_publication_lease_is_active(&self, token: Uuid) -> bool {
let mut leases = self.scanner_publication_leases.lock().await;
let now = Instant::now();
let Some(expires_at) = leases.get(&token).map(|entry| entry.expires_at) else {
return false;
};
if expires_at <= now {
leases.remove(&token);
return false;
}
true
}
/// Return the generation bound to a live lease. The lease entry owns the
/// movement read guard, so a successful lookup remains valid for the
/// caller's guard-protected operation; expiry is still fail-closed.
pub(crate) async fn scanner_publication_lease_generation(&self, token: Uuid) -> Option<u64> {
let mut leases = self.scanner_publication_leases.lock().await;
let now = Instant::now();
let (expires_at, movement_generation) = leases
.get(&token)
.map(|entry| (entry.expires_at, entry.movement_generation))?;
if expires_at <= now {
leases.remove(&token);
return None;
}
Some(movement_generation)
}
pub(crate) async fn expire_scanner_publication_lease(&self, token: Uuid, expires_at: Instant) {
let mut leases = self.scanner_publication_leases.lock().await;
let should_remove = leases.get(&token).is_some_and(|entry| entry.expires_at <= expires_at);
if should_remove {
leases.remove(&token);
}
}
pub(crate) fn data_movement_operation_epoch(&self) -> u64 {
self.data_movement_operation_epoch.load(Ordering::Acquire)
}
@@ -257,8 +356,21 @@ impl InstanceContext {
self.data_movement_operation_epoch_exhausted.load(Ordering::Acquire)
}
pub(crate) fn data_movement_generation(&self) -> u64 {
self.data_movement_generation.load(Ordering::Acquire)
}
pub(crate) fn data_movement_generation_exhausted(&self) -> bool {
self.data_movement_generation_exhausted.load(Ordering::Acquire)
}
pub(crate) fn data_movement_generation_notify(&self) -> Arc<Notify> {
Arc::clone(&self.data_movement_generation_notify)
}
pub(crate) fn scanner_publication_state_allowed(&self) -> bool {
!self.data_movement_operation_epoch_exhausted()
&& !self.data_movement_generation_exhausted()
&& self.scanner_publication_state.load(Ordering::Acquire) == SCANNER_PUBLICATION_STATE_ALLOWED
}
@@ -276,6 +388,7 @@ impl InstanceContext {
pub(crate) fn advance_data_movement_operation_epoch(&self) -> u64 {
self.scanner_publication_state
.store(SCANNER_PUBLICATION_STATE_UNKNOWN, Ordering::Release);
let previous = self.data_movement_operation_epoch.load(Ordering::Acquire);
let _ = self
.data_movement_operation_epoch
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |epoch| Some(epoch.saturating_add(1)));
@@ -283,9 +396,42 @@ impl InstanceContext {
if result == u64::MAX {
self.data_movement_operation_epoch_exhausted.store(true, Ordering::Release);
}
if result != previous {
let _ = self.advance_data_movement_generation();
}
result
}
/// Advance the movement generation after a durable movement transition.
/// The generation is deliberately bounded: once it reaches `u64::MAX`,
/// publication and generation-based waits fail closed rather than reusing
/// an indistinguishable saturated value.
pub(crate) fn advance_data_movement_generation(&self) -> Option<u64> {
if self.data_movement_generation_exhausted.load(Ordering::Acquire) {
return None;
}
let updated = self
.data_movement_generation
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |generation| generation.checked_add(1));
match updated {
Ok(previous) => {
let Some(generation) = previous.checked_add(1) else {
self.data_movement_generation_exhausted.store(true, Ordering::Release);
return None;
};
if generation == u64::MAX {
self.data_movement_generation_exhausted.store(true, Ordering::Release);
}
self.data_movement_generation_notify.notify_waiters();
Some(generation)
}
Err(_) => {
self.data_movement_generation_exhausted.store(true, Ordering::Release);
None
}
}
}
#[cfg(test)]
pub(crate) fn set_data_movement_operation_epoch_for_test(&self, epoch: u64) {
self.data_movement_operation_epoch.store(epoch, Ordering::Release);
@@ -295,6 +441,13 @@ impl InstanceContext {
.store(SCANNER_PUBLICATION_STATE_UNKNOWN, Ordering::Release);
}
#[cfg(test)]
pub(crate) fn set_data_movement_generation_for_test(&self, generation: u64) {
self.data_movement_generation.store(generation, Ordering::Release);
self.data_movement_generation_exhausted
.store(generation == u64::MAX, Ordering::Release);
}
/// Install the application-owned object-encryption resolver once.
pub fn set_object_encryption_resolver(
&self,
+137 -1
View File
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::cluster::rpc::{PeerRestClient, ScannerPeerActivity, TierConfigReloadOutcome};
use crate::cluster::rpc::{PeerRestClient, ScannerPeerActivity, ScannerPublicationLease, TierConfigReloadOutcome};
use crate::diagnostics::admin_server_info::get_commit_id;
use crate::disk::DiskAPI;
use crate::error::{Error, Result};
@@ -53,6 +53,12 @@ const REMOTE_VERSION_STATE_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
const REMOTE_VERSION_STATE_PROOF_TTL: Duration = Duration::from_secs(30);
const CROSS_POOL_FENCE_SUPPORTED_VERSION: u32 = 2;
#[derive(Clone, Debug)]
pub struct ScannerPublicationLeaseGrant {
pub host: String,
pub lease: ScannerPublicationLease,
}
/// Cached result from the last successful admin call to a peer.
struct PeerAdminCache {
last_storage_info: Option<StorageInfo>,
@@ -1491,6 +1497,102 @@ impl NotificationSys {
aggregate_scanner_dirty_usage_acknowledgement_results(join_all(futures).await, failures)
}
/// Acquire remote publication leases in a deterministic host order. A
/// missing/legacy peer is a hard publication deferral; already acquired
/// leases are released before returning so a partial acquisition cannot
/// pin movement on one peer.
pub async fn acquire_scanner_publication_leases(
&self,
mut targets: Vec<(String, String, u64)>,
) -> Result<Vec<ScannerPublicationLeaseGrant>> {
targets.sort_by(|left, right| left.0.cmp(&right.0));
for pair in targets.windows(2) {
if pair[0].0 == pair[1].0 {
return Err(Error::other(format!("duplicate scanner publication lease target: {}", pair[0].0)));
}
}
let mut grants = Vec::with_capacity(targets.len());
for (host, session_id, generation) in targets {
let Some(client) = self
.peer_clients
.iter()
.flatten()
.find(|client| client.grid_host == host)
.cloned()
else {
let _ = self.release_scanner_publication_leases(grants).await;
return Err(Error::other(format!("scanner publication lease peer {host} is unavailable")));
};
match client.acquire_scanner_publication_lease(&session_id, generation).await {
Ok(lease) => grants.push(ScannerPublicationLeaseGrant { host, lease }),
Err(err) => {
let _ = self.release_scanner_publication_leases(grants).await;
return Err(Error::other(format!("scanner publication lease acquisition failed: {err}")));
}
}
}
Ok(grants)
}
pub async fn release_scanner_publication_leases(&self, mut grants: Vec<ScannerPublicationLeaseGrant>) -> Result<()> {
grants.sort_by(|left, right| right.host.cmp(&left.host));
let mut failures = Vec::new();
for grant in grants {
let Some(client) = self
.peer_clients
.iter()
.flatten()
.find(|client| client.grid_host == grant.host)
else {
failures.push(format!("peer {} is unavailable", grant.host));
continue;
};
if let Err(err) = client.release_scanner_publication_lease(&grant.lease).await {
failures.push(format!("peer {} release failed: {err}", grant.host));
}
}
if failures.is_empty() {
Ok(())
} else {
Err(Error::other(format!(
"scanner publication lease release failures: {}",
failures.join("; ")
)))
}
}
/// Revalidate every remote lease in deterministic host order immediately
/// before a final scanner metadata write. A peer restart removes its
/// process-owned token table and changes its activity session, so an old
/// generation cannot pass this proof even when the numeric generation is
/// reused.
pub async fn validate_scanner_publication_leases(&self, grants: &[ScannerPublicationLeaseGrant]) -> Result<()> {
let mut grants = grants.to_vec();
grants.sort_by(|left, right| left.host.cmp(&right.host));
for pair in grants.windows(2) {
if pair[0].host == pair[1].host {
return Err(Error::other(format!("duplicate scanner publication lease target: {}", pair[0].host)));
}
}
for grant in grants {
let Some(client) = self
.peer_clients
.iter()
.flatten()
.find(|client| client.grid_host == grant.host)
.cloned()
else {
return Err(Error::other(format!("scanner publication lease peer {} is unavailable", grant.host)));
};
client
.validate_scanner_publication_lease(&grant.lease)
.await
.map_err(|err| Error::other(format!("scanner publication lease validation failed for {}: {err}", grant.host)))?;
}
Ok(())
}
pub async fn reload_site_replication_config(&self) -> Vec<NotificationPeerErr> {
let mut futures = Vec::with_capacity(self.peer_clients.len());
for client in self.peer_clients.iter() {
@@ -2633,6 +2735,38 @@ mod tests {
assert!(err.to_string().contains("no remote peers"));
}
#[tokio::test]
async fn scanner_publication_lease_release_reports_all_unavailable_peers() {
let sys = NotificationSys {
peer_clients: Vec::new(),
all_peer_clients: Vec::new(),
peer_topology_hosts: Vec::new(),
peer_admin_caches: Vec::new(),
tier_config_reload_workers: Default::default(),
};
let grants = ["peer-a", "peer-b"]
.into_iter()
.map(|host| ScannerPublicationLeaseGrant {
host: host.to_string(),
lease: ScannerPublicationLease {
token: Uuid::new_v4(),
movement_generation: 3,
owner_id: Uuid::new_v4().to_string(),
session_id: "session-a".to_string(),
expires_at: Instant::now() + Duration::from_secs(30),
},
})
.collect();
let error = sys
.release_scanner_publication_leases(grants)
.await
.expect_err("an unavailable peer must not silently release a remote lease");
let message = error.to_string();
assert!(message.contains("peer-a"));
assert!(message.contains("peer-b"));
}
#[tokio::test]
async fn scanner_activity_probe_rejects_an_incomplete_peer_topology() {
let client = PeerRestClient::new(
@@ -2755,6 +2889,8 @@ mod tests {
data_movement_active: Some(false),
dirty_usage_generation: Some(2),
dirty_usage_pending,
movement_generation: Some(1),
publication_blocked: Some(false),
};
let pending = aggregate_scanner_dirty_usage_acknowledgement_results(
+129 -11
View File
@@ -3499,6 +3499,26 @@ pub(in crate::set_disk) struct RenameDataCommit {
pub(in crate::set_disk) tail_drain: Option<tokio::task::JoinHandle<()>>,
}
/// Options shared by the normal and early-ack rename fanouts. Keeping the
/// quorum and optional scanner lease map together avoids widening either
/// fanout helper's argument list while preserving the fence semantics.
pub(in crate::set_disk) struct RenameDataFenceOptions<'a> {
write_quorum: usize,
scanner_publication_lease_tokens: Option<&'a HashMap<String, Uuid>>,
}
impl<'a> RenameDataFenceOptions<'a> {
pub(in crate::set_disk) fn new(
write_quorum: usize,
scanner_publication_lease_tokens: Option<&'a HashMap<String, Uuid>>,
) -> Self {
Self {
write_quorum,
scanner_publication_lease_tokens,
}
}
}
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
type RenameDataLegacyTuple = (
Vec<Option<DiskStore>>,
@@ -3690,16 +3710,51 @@ impl SetDisks {
.map(RenameDataCommit::into_legacy_tuple)
}
#[tracing::instrument(level = "debug", skip(disks, file_infos))]
async fn rename_data_owned_early_ack(
pub(in crate::set_disk) fn scanner_publication_lease_token_for_disk(
disk: Option<&DiskStore>,
scanner_publication_lease_tokens: Option<&HashMap<String, Uuid>>,
) -> disk::error::Result<Option<Uuid>> {
let Some(disk) = disk else {
return Ok(None);
};
if disk.is_local() {
return Ok(None);
}
let Some(tokens) = scanner_publication_lease_tokens else {
return Ok(None);
};
let host = disk.endpoint().grid_host();
tokens
.get(&host)
.copied()
.ok_or_else(|| DiskError::other(format!("scanner publication lease token missing for remote disk host {host}")))
.map(Some)
}
pub(in crate::set_disk) fn scanner_publication_lease_tokens_for_disks(
disks: &[Option<DiskStore>],
scanner_publication_lease_tokens: Option<&HashMap<String, Uuid>>,
) -> disk::error::Result<Vec<Option<Uuid>>> {
disks
.iter()
.map(|disk| Self::scanner_publication_lease_token_for_disk(disk.as_ref(), scanner_publication_lease_tokens))
.collect()
}
#[tracing::instrument(level = "debug", skip(disks, file_infos, fence_options))]
async fn rename_data_owned_early_ack_with_fence(
disks: &[Option<DiskStore>],
src_bucket: &str,
src_object: &str,
file_infos: Vec<FileInfo>,
dst_bucket: &str,
dst_object: &str,
write_quorum: usize,
fence_options: RenameDataFenceOptions<'_>,
) -> disk::error::Result<RenameDataCommit> {
let RenameDataFenceOptions {
write_quorum,
scanner_publication_lease_tokens,
} = fence_options;
if let Some(file_info) = disks
.iter()
.zip(file_infos.iter())
@@ -3714,6 +3769,7 @@ impl SetDisks {
let disk_count = disks.len();
let fanout_disks = disks.to_vec();
let fanout_fence_tokens = Self::scanner_publication_lease_tokens_for_disks(disks, scanner_publication_lease_tokens)?;
let coordinator_disks = fanout_disks.clone();
let src_bucket = Arc::new(src_bucket.to_string());
let src_object = Arc::new(src_object.to_string());
@@ -3730,7 +3786,12 @@ impl SetDisks {
let successful_rename_completion_rank =
rustfs_io_metrics::put_stage_metrics_enabled().then(|| Arc::new(AtomicUsize::new(0)));
let mut tasks = JoinSet::new();
for (i, (disk, file_info)) in fanout_disks.into_iter().zip(file_infos.iter()).enumerate() {
for (i, ((disk, file_info), scanner_publication_lease_token)) in fanout_disks
.into_iter()
.zip(file_infos.iter())
.zip(fanout_fence_tokens.into_iter())
.enumerate()
{
let src_bucket = fanout_src_bucket.clone();
let src_object = fanout_src_object.clone();
let dst_bucket = fanout_dst_bucket.clone();
@@ -3767,7 +3828,14 @@ impl SetDisks {
let disk_wait_started = rustfs_io_metrics::put_stage_timer();
let result = disk
.rename_data_borrowed(&src_bucket, &src_object, file_info, &dst_bucket, &dst_object)
.rename_data_borrowed_with_fence(
&src_bucket,
&src_object,
file_info,
&dst_bucket,
&dst_object,
scanner_publication_lease_token,
)
.await;
if let Some(disk_wait_started) = disk_wait_started {
let duration_ms = disk_wait_started.elapsed().as_secs_f64() * 1000.0;
@@ -3967,19 +4035,45 @@ impl SetDisks {
dst_bucket: &str,
dst_object: &str,
write_quorum: usize,
) -> disk::error::Result<RenameDataCommit> {
Self::rename_data_owned_with_fence(
disks,
src_bucket,
src_object,
file_infos,
dst_bucket,
dst_object,
RenameDataFenceOptions::new(write_quorum, None),
)
.await
}
#[tracing::instrument(level = "debug", skip(disks, file_infos, fence_options))]
pub(in crate::set_disk) async fn rename_data_owned_with_fence(
disks: &[Option<DiskStore>],
src_bucket: &str,
src_object: &str,
file_infos: Vec<FileInfo>,
dst_bucket: &str,
dst_object: &str,
fence_options: RenameDataFenceOptions<'_>,
) -> disk::error::Result<RenameDataCommit> {
if put_rename_early_ack_enabled() {
return Self::rename_data_owned_early_ack(
return Self::rename_data_owned_early_ack_with_fence(
disks,
src_bucket,
src_object,
file_infos,
dst_bucket,
dst_object,
write_quorum,
fence_options,
)
.await;
}
let RenameDataFenceOptions {
write_quorum,
scanner_publication_lease_tokens,
} = fence_options;
if let Some(file_info) = disks
.iter()
.zip(file_infos.iter())
@@ -4005,6 +4099,7 @@ impl SetDisks {
let disk_count = disks.len();
let fanout_disks = disks.to_vec();
let fanout_file_infos = file_infos;
let fanout_fence_tokens = Self::scanner_publication_lease_tokens_for_disks(disks, scanner_publication_lease_tokens)?;
let fanout_src_bucket = src_bucket.clone();
let fanout_src_object = src_object.clone();
let fanout_dst_bucket = dst_bucket.clone();
@@ -4019,8 +4114,9 @@ impl SetDisks {
let futures = fanout_disks
.into_iter()
.zip(fanout_file_infos.iter())
.zip(fanout_fence_tokens.into_iter())
.enumerate()
.map(|(i, (disk, file_info))| {
.map(|(i, ((disk, file_info), scanner_publication_lease_token))| {
let src_bucket = fanout_src_bucket.clone();
let src_object = fanout_src_object.clone();
let dst_object = fanout_dst_object.clone();
@@ -4060,7 +4156,14 @@ impl SetDisks {
let disk_wait_started = rustfs_io_metrics::put_stage_timer();
let result = disk
.rename_data_borrowed(&src_bucket, &src_object, file_info, &dst_bucket, &dst_object)
.rename_data_borrowed_with_fence(
&src_bucket,
&src_object,
file_info,
&dst_bucket,
&dst_object,
scanner_publication_lease_token,
)
.await;
if let Some(disk_wait_started) = disk_wait_started {
let duration_ms = disk_wait_started.elapsed().as_secs_f64() * 1000.0;
@@ -5426,18 +5529,32 @@ impl SetDisks {
Ok(())
}
#[cfg(test)]
pub(in crate::set_disk) async fn delete_prefix(&self, bucket: &str, prefix: &str) -> disk::error::Result<()> {
self.delete_prefix_with_scanner_publication_lease(bucket, prefix, None).await
}
/// Delete a prefix with an optional per-remote-disk scanner publication
/// lease fence. The fence is transient and never reaches disk metadata;
/// remote deletes without a matching token fail closed before the RPC.
pub(in crate::set_disk) async fn delete_prefix_with_scanner_publication_lease(
&self,
bucket: &str,
prefix: &str,
scanner_publication_lease_tokens: Option<&HashMap<String, Uuid>>,
) -> disk::error::Result<()> {
let disks = self.get_disks_internal().await;
let write_quorum = disks.len() / 2 + 1;
let fanout_fence_tokens = Self::scanner_publication_lease_tokens_for_disks(&disks, scanner_publication_lease_tokens)?;
let mut futures = Vec::with_capacity(disks.len());
for disk_op in disks.iter() {
for (disk_op, scanner_publication_lease_token) in disks.iter().zip(fanout_fence_tokens) {
let bucket = bucket.to_string();
let prefix = prefix.to_string();
futures.push(async move {
if let Some(disk) = disk_op {
disk.delete(
disk.delete_with_scanner_publication_lease(
&bucket,
&prefix,
DeleteOptions {
@@ -5445,6 +5562,7 @@ impl SetDisks {
immediate: true,
..Default::default()
},
scanner_publication_lease_token,
)
.await
} else {
+67 -4
View File
@@ -49,8 +49,8 @@ use crate::data_usage::quota_object_size;
use crate::diagnostics::get::GetObjectFailureReason;
use crate::disk::{DataDirDeleteStatus, OldCurrentSize};
use crate::error::is_err_invalid_upload_id;
use crate::object_api::NamespaceLockFence;
use crate::object_api::{GetObjectBodySource, get_object_body_cache_hook_suppressed};
use crate::object_api::{NamespaceLockFence, SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY};
use crate::services::notification_sys::RemoteVersionStateFleetProofToken;
use crate::services::tier::tier::{TierConfigMgr, TierOperationLease};
use crate::store::ECStore;
@@ -63,6 +63,60 @@ use std::sync::OnceLock;
use tokio_util::sync::CancellationToken;
const OLD_DATA_CLEANUP_RECEIPT_FILE: &str = ".rustfs-old-data-cleanup-receipt.json";
const SCANNER_PUBLICATION_LEASE_FENCE_MAX_BYTES: usize = 64 * 1024;
const SCANNER_PUBLICATION_LEASE_FENCE_MAX_ENTRIES: usize = 256;
fn take_scanner_publication_lease_tokens(user_defined: &mut HashMap<String, String>) -> Result<Option<HashMap<String, Uuid>>> {
let Some(encoded) = user_defined.remove(SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY) else {
return Ok(None);
};
if encoded.len() > SCANNER_PUBLICATION_LEASE_FENCE_MAX_BYTES {
return Err(Error::other("scanner publication lease fence is too large"));
}
let encoded_tokens = serde_json::from_str::<HashMap<String, String>>(&encoded)
.map_err(|err| Error::other(format!("invalid scanner publication lease fence: {err}")))?;
if encoded_tokens.len() > SCANNER_PUBLICATION_LEASE_FENCE_MAX_ENTRIES {
return Err(Error::other("scanner publication lease fence has too many entries"));
}
let mut tokens = HashMap::with_capacity(encoded_tokens.len());
for (host, token) in encoded_tokens {
if host.is_empty() || host.len() > 1024 {
return Err(Error::other("invalid scanner publication lease fence host"));
}
let token = Uuid::parse_str(&token)
.map_err(|err| Error::other(format!("invalid scanner publication lease fence token: {err}")))?;
if token.is_nil() {
return Err(Error::other("invalid scanner publication lease fence token"));
}
tokens.insert(host, token);
}
Ok(Some(tokens))
}
#[cfg(test)]
mod scanner_publication_lease_fence_tests {
use super::*;
#[test]
fn scanner_publication_lease_fence_is_transient_and_validated() {
let token = Uuid::new_v4();
let mut metadata = HashMap::from([(
SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY.to_string(),
serde_json::json!({"http://node-a:9000": token.to_string()}).to_string(),
)]);
let parsed = take_scanner_publication_lease_tokens(&mut metadata)
.expect("valid scanner publication lease fence should parse")
.expect("scanner publication lease fence should be present");
assert_eq!(parsed.get("http://node-a:9000"), Some(&token));
assert!(!metadata.contains_key(SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY));
let mut malformed = HashMap::from([(
SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY.to_string(),
r#"{"http://node-a:9000":"not-a-uuid"}"#.to_string(),
)]);
assert!(take_scanner_publication_lease_tokens(&mut malformed).is_err());
}
}
struct PutObjectCommitCancellation {
token: CancellationToken,
@@ -2177,6 +2231,7 @@ impl SetDisks {
user_defined.insert(key.clone(), value.clone());
}
}
let scanner_publication_lease_tokens = take_scanner_publication_lease_tokens(&mut user_defined)?;
if replication_lww_applicable(opts) {
// Object Lock evaluation stamps category timestamps with this
// receiver's clock. Pin them back to the source-authored times
@@ -2896,6 +2951,7 @@ impl SetDisks {
let commit_bucket_lifecycle_lock_fence = opts.bucket_lifecycle_lock_fence.clone();
let commit_capacity_scope_token = opts.capacity_scope_token;
let commit_replication_state = replication_state_to_filemeta(&opts.put_replication_state());
let commit_scanner_publication_lease_tokens = scanner_publication_lease_tokens;
tmp_cleanup_owned = true;
let commit = move |cancellation: Option<CancellationToken>| async move {
@@ -3015,14 +3071,17 @@ impl SetDisks {
}
Self::assign_rename_data_indexes(&mut parts_metadatas);
let rename_result = SetDisks::rename_data_owned(
let rename_result = SetDisks::rename_data_owned_with_fence(
&commit_disks,
RUSTFS_META_TMP_BUCKET,
commit_tmp_dir.as_str(),
parts_metadatas,
&commit_bucket,
&commit_object,
write_quorum,
crate::set_disk::core::io_primitives::RenameDataFenceOptions::new(
write_quorum,
commit_scanner_publication_lease_tokens.as_ref(),
),
)
.await;
if quota_mutation_fence {
@@ -6418,6 +6477,10 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
#[tracing::instrument(skip(self))]
async fn delete_object(&self, bucket: &str, object: &str, mut opts: ObjectOptions) -> Result<ObjectInfo> {
// Scanner cleanup carries the per-peer lease fence as transient
// request metadata. Consume it before any delete-prefix fanout so it
// cannot be persisted or treated as user metadata.
let scanner_publication_lease_tokens = take_scanner_publication_lease_tokens(&mut opts.user_defined)?;
let preserve_delete_replication_state = should_preserve_delete_replication_state(&opts);
let delete_config_snapshot = if opts.delete_prefix || opts.transition.expire_restored || preserve_delete_replication_state
{
@@ -6544,7 +6607,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
self.validate_bucket_incarnation(bucket, expected_incarnation_id).await?;
}
ensure_delete_commit_locks_held(_lock_guard.as_ref(), bucket, object, &opts)?;
self.delete_prefix(bucket, object)
self.delete_prefix_with_scanner_publication_lease(bucket, object, scanner_publication_lease_tokens.as_ref())
.await
.map_err(|e| to_object_err(e.into(), vec![bucket, object]))?;
@@ -32,7 +32,8 @@ pub(crate) mod internode {
PUT_FILE_AUTH_TRAILER_MAGIC, PUT_FILE_AUTH_V1, PUT_FILE_CAPABILITY_CHALLENGE_QUERY, PUT_FILE_CAPABILITY_QUERY,
PUT_FILE_CAPABILITY_VERSION, PUT_FILE_NONCE_QUERY, PUT_FILE_SERVER_EPOCH_QUERY, PutFileCapabilityResponse,
SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION, SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SCANNER_ACTIVITY_PROTOCOL_VERSION,
WALK_DIR_BODY_SHA256_QUERY, WALK_DIR_STREAM_COMPLETION_QUERY, WALK_DIR_STREAM_COMPLETION_V1,
SCANNER_ACTIVITY_V6_PROTOCOL_VERSION, WALK_DIR_BODY_SHA256_QUERY, WALK_DIR_STREAM_COMPLETION_QUERY,
WALK_DIR_STREAM_COMPLETION_V1,
};
}
+273 -4
View File
@@ -87,6 +87,8 @@ type ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>;
type WalkOptions = StorageWalkOptions<fn(&FileInfo) -> bool>;
pub const SCANNER_PUBLICATION_LEASE_TTL_MS: u64 = 60_000;
/// Check if a directory contains any xl.meta files (indicating actual S3 objects)
/// This is used to determine if a bucket is empty for deletion purposes.
pub(crate) async fn has_xlmeta_files(path: &std::path::Path) -> std::io::Result<bool> {
@@ -344,6 +346,33 @@ impl ECStore {
decommission || rebalance
}
/// Return the storage-owned movement state and generation as one
/// authenticated activity snapshot. The read lock is acquired before
/// the state locks (cancelers, pool metadata, then rebalance metadata),
/// matching the transition writer order and preventing a terminal state
/// from being reported with the preceding generation.
pub async fn scanner_data_movement_activity(&self) -> (bool, bool, u64) {
let operation_gate = self.ctx.data_movement_operation_gate();
let _operation_guard = operation_gate.read_owned().await;
let (active, blocked) = self.scanner_data_movement_snapshot_locked().await;
let blocked =
blocked || self.ctx.data_movement_operation_epoch_exhausted() || self.ctx.data_movement_generation_exhausted();
self.ctx.set_scanner_publication_state(blocked);
(active, blocked, self.ctx.data_movement_generation())
}
pub fn scanner_data_movement_generation(&self) -> u64 {
self.ctx.data_movement_generation()
}
pub fn scanner_data_movement_generation_exhausted(&self) -> bool {
self.ctx.data_movement_generation_exhausted()
}
pub fn scanner_data_movement_changed(&self) -> std::sync::Arc<tokio::sync::Notify> {
self.ctx.data_movement_generation_notify()
}
/// Returns whether scanner metadata may still be hidden by a local
/// data-movement state. Terminal failed/canceled decommission entries
/// remain suspended until an operator clears or retries them, so they are
@@ -355,10 +384,16 @@ impl ECStore {
}
async fn scanner_data_usage_publication_snapshot_blocked(&self) -> bool {
if self.ctx.data_movement_operation_epoch_exhausted() {
if self.ctx.data_movement_operation_epoch_exhausted() || self.ctx.data_movement_generation_exhausted() {
self.ctx.set_scanner_publication_state(true);
return true;
}
let (_, blocked) = self.scanner_data_movement_snapshot_locked().await;
self.ctx.set_scanner_publication_state(blocked);
blocked
}
async fn scanner_data_movement_snapshot_locked(&self) -> (bool, bool) {
let decommission_cancelers = self.decommission_cancelers.read().await;
let decommission_active = decommission_cancelers
.iter()
@@ -385,8 +420,7 @@ impl ECStore {
.is_some_and(is_rebalance_conflicting_with_decommission);
let blocked = decommission_active || decommission_terminal || rebalance_active;
self.ctx.set_scanner_publication_state(blocked);
blocked
(decommission_active || rebalance_active, blocked)
}
/// Admit one short data-usage publication commit under the same
@@ -407,7 +441,7 @@ impl ECStore {
pub async fn scanner_data_usage_publication_admission_guard(&self) -> Option<(tokio::sync::OwnedRwLockReadGuard<()>, u64)> {
let operation_gate = self.ctx.data_movement_operation_gate();
let operation_guard = operation_gate.read_owned().await;
if self.ctx.data_movement_operation_epoch_exhausted() {
if self.ctx.data_movement_operation_epoch_exhausted() || self.ctx.data_movement_generation_exhausted() {
return None;
}
if self.scanner_data_usage_publication_snapshot_blocked().await {
@@ -425,6 +459,98 @@ impl ECStore {
drop(operation_guard);
Some(epoch)
}
/// Acquire a storage-owned read admission for a coordinator's final
/// scanner publication. The guard remains in the context's lease table,
/// so a local movement writer cannot pass the peer while its authoritative
/// PUT is in flight.
pub async fn acquire_scanner_publication_lease(
&self,
expected_generation: u64,
ttl: std::time::Duration,
) -> Result<(Uuid, u64)> {
if ttl != crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL {
return Err(Error::other("scanner publication lease TTL is not supported"));
}
let operation_gate = self.ctx.data_movement_operation_gate();
let operation_guard = operation_gate.read_owned().await;
if self.ctx.data_movement_generation_exhausted()
|| self.ctx.data_movement_operation_epoch_exhausted()
|| self.ctx.data_movement_generation() != expected_generation
{
return Err(Error::other("scanner publication lease generation is stale"));
}
if self.scanner_data_movement_snapshot_locked().await.1 {
return Err(Error::other("scanner publication lease is blocked by data movement"));
}
let token = Uuid::new_v4();
let expires_at = tokio::time::Instant::now() + ttl;
if !self
.ctx
.install_scanner_publication_lease(token, expires_at, expected_generation, operation_guard)
.await
{
return Err(Error::other("scanner publication lease capacity is exhausted"));
}
let context = Arc::clone(&self.ctx);
tokio::spawn(async move {
tokio::time::sleep_until(expires_at).await;
context.expire_scanner_publication_lease(token, expires_at).await;
});
Ok((token, expected_generation))
}
pub async fn release_scanner_publication_lease(&self, token: Uuid) -> bool {
self.ctx.remove_scanner_publication_lease(token).await
}
/// Revalidate a previously acquired remote publication lease immediately
/// before the coordinator's final metadata write. The operation read
/// guard makes the movement snapshot and token lookup one storage-owned
/// admission; a restarted context has no old token and therefore fails
/// closed even if its generation counter has returned to zero.
pub async fn validate_scanner_publication_lease(&self, token: Uuid, expected_generation: u64) -> Result<()> {
let _operation_guard = self.acquire_scanner_publication_lease_guard(token).await?;
if self.ctx.data_movement_generation_exhausted()
|| self.ctx.data_movement_operation_epoch_exhausted()
|| self.ctx.data_movement_generation() != expected_generation
{
return Err(Error::other("scanner publication lease generation is stale"));
}
if self.scanner_data_movement_snapshot_locked().await.1 {
return Err(Error::other("scanner publication lease is blocked by data movement"));
}
if !self.ctx.scanner_publication_lease_is_active(token).await {
return Err(Error::other("scanner publication lease is unknown or expired"));
}
Ok(())
}
/// Acquire the target-side read guard bound to a previously granted lease.
/// The guard is returned to the RPC handler and must remain alive through
/// the complete rename/write operation. A lease token is process-owned;
/// restart, expiry, generation changes, or blocked movement all reject it
/// before the target disk is touched.
pub async fn acquire_scanner_publication_lease_guard(&self, token: Uuid) -> Result<tokio::sync::OwnedRwLockReadGuard<()>> {
let operation_gate = self.ctx.data_movement_operation_gate();
let operation_guard = operation_gate.read_owned().await;
if self.ctx.data_movement_generation_exhausted() || self.ctx.data_movement_operation_epoch_exhausted() {
return Err(Error::other("scanner publication lease generation is exhausted"));
}
if self.scanner_data_movement_snapshot_locked().await.1 {
return Err(Error::other("scanner publication lease is blocked by data movement"));
}
let Some(lease_generation) = self.ctx.scanner_publication_lease_generation(token).await else {
return Err(Error::other("scanner publication lease is unknown or expired"));
};
if lease_generation != self.ctx.data_movement_generation() {
return Err(Error::other("scanner publication lease generation is stale"));
}
Ok(operation_guard)
}
}
// impl Clone for ECStore {
@@ -1084,6 +1210,149 @@ mod tests {
.await
.expect("idle store should admit the next publication");
assert_eq!(next_epoch, 1);
assert_eq!(store.scanner_data_movement_generation(), 1);
}
#[tokio::test(start_paused = true)]
async fn scanner_publication_lease_blocks_movement_writer_until_release_or_expiry() {
let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
let (token, generation) = store
.acquire_scanner_publication_lease(0, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
.await
.expect("an idle store should grant a publication lease");
assert_eq!(generation, 0);
let gate = store.ctx.data_movement_operation_gate();
let (writer_started, writer_started_rx) = tokio::sync::oneshot::channel();
let movement_writer = tokio::spawn(async move {
let _ = writer_started.send(());
gate.write_owned().await
});
writer_started_rx
.await
.expect("movement writer should reach the gate before waiting");
assert!(
!movement_writer.is_finished(),
"a movement writer must wait while the remote lease owns the read guard"
);
assert!(store.release_scanner_publication_lease(token).await);
tokio::time::timeout(Duration::from_secs(1), movement_writer)
.await
.expect("movement writer should proceed after lease release")
.expect("movement writer task should not panic");
let (expiring_token, _) = store
.acquire_scanner_publication_lease(0, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
.await
.expect("the store should grant a second publication lease");
let expiry_gate = store.ctx.data_movement_operation_gate();
let (expiry_started, expiry_started_rx) = tokio::sync::oneshot::channel();
let mut expiry_writer = tokio::spawn(async move {
let _ = expiry_started.send(());
expiry_gate.write_owned().await
});
expiry_started_rx
.await
.expect("expiry writer should reach the gate before waiting");
assert!(!expiry_writer.is_finished());
tokio::time::advance(crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL + Duration::from_millis(1)).await;
tokio::task::yield_now().await;
tokio::time::timeout(Duration::from_secs(1), &mut expiry_writer)
.await
.expect("movement writer should proceed after lease expiry")
.expect("expiry writer task should not panic");
assert!(!store.release_scanner_publication_lease(expiring_token).await);
}
#[tokio::test]
async fn scanner_publication_lease_rejects_stale_generation_before_install() {
let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
let error = store
.acquire_scanner_publication_lease(1, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
.await
.expect_err("a stale movement generation must not acquire a lease");
assert!(error.to_string().contains("generation is stale"));
}
#[tokio::test]
async fn scanner_target_guard_keeps_movement_writer_fenced_after_lease_release() {
let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
let (token, _) = store
.acquire_scanner_publication_lease(0, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
.await
.expect("an idle store should grant a publication lease");
let target_guard = store
.acquire_scanner_publication_lease_guard(token)
.await
.expect("the target-side rename should acquire its short guard");
assert!(store.release_scanner_publication_lease(token).await);
let movement_gate = store.ctx.data_movement_operation_gate();
let movement_writer = tokio::spawn(async move { movement_gate.write_owned().await });
tokio::task::yield_now().await;
assert!(!movement_writer.is_finished(), "the target guard must span the rename operation");
drop(target_guard);
tokio::time::timeout(std::time::Duration::from_secs(1), movement_writer)
.await
.expect("movement writer should proceed after the target rename guard is dropped")
.expect("movement writer task should not panic");
}
#[tokio::test]
async fn scanner_publication_lease_rejects_restart_aba_token() {
let first_store = build_store_with_ctx(Arc::new(InstanceContext::new()));
let (token, generation) = first_store
.acquire_scanner_publication_lease(0, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
.await
.expect("the initial instance should grant a publication lease");
first_store
.validate_scanner_publication_lease(token, generation)
.await
.expect("the current instance should validate its own live token");
first_store
.acquire_scanner_publication_lease_guard(token)
.await
.expect("the current instance should admit the target-side rename");
// A restarted storage instance starts its local generation at zero,
// but its process-owned lease table is empty. The old token must not
// pass validation just because the numeric generation matches again.
let restarted_store = build_store_with_ctx(Arc::new(InstanceContext::new()));
let error = restarted_store
.validate_scanner_publication_lease(token, generation)
.await
.expect_err("a token from a prior instance must fail closed after restart");
assert!(error.to_string().contains("unknown or expired"));
let error = restarted_store
.acquire_scanner_publication_lease_guard(token)
.await
.expect_err("a restarted instance must reject the target-side rename token");
assert!(error.to_string().contains("unknown or expired"));
}
#[tokio::test]
async fn movement_generation_notifies_waiters_and_fails_closed_at_maximum() {
let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
let notify = store.scanner_data_movement_changed();
let notified = notify.notified();
tokio::pin!(notified);
notified.as_mut().enable();
assert_eq!(store.ctx.advance_data_movement_operation_epoch(), 1);
tokio::time::timeout(std::time::Duration::from_secs(1), notified)
.await
.expect("movement transition should wake scanner waiters");
assert_eq!(store.scanner_data_movement_generation(), 1);
store.ctx.set_data_movement_generation_for_test(u64::MAX - 1);
assert_eq!(store.ctx.advance_data_movement_generation(), Some(u64::MAX));
assert!(store.scanner_data_movement_generation_exhausted());
assert_eq!(store.ctx.advance_data_movement_generation(), None);
assert!(
store.scanner_data_usage_publication_admission_guard().await.is_none(),
"generation exhaustion must close publication admission"
);
}
#[tokio::test]
@@ -142,6 +142,9 @@ pub struct DeleteRequest {
pub path: ::prost::alloc::string::String,
#[prost(string, tag = "4")]
pub options: ::prost::alloc::string::String,
/// Optional scanner publication lease token.
#[prost(bytes = "bytes", tag = "5")]
pub scanner_publication_lease_token: ::prost::bytes::Bytes,
}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct DeleteResponse {
@@ -393,6 +396,8 @@ pub struct RenameDataRequest {
pub dst_path: ::prost::alloc::string::String,
#[prost(bytes = "bytes", tag = "7")]
pub file_info_bin: ::prost::bytes::Bytes,
#[prost(bytes = "bytes", tag = "8")]
pub scanner_publication_lease_token: ::prost::bytes::Bytes,
}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct RenameDataResponse {
@@ -1213,6 +1218,62 @@ pub struct ScannerActivityResponse {
pub dirty_usage_generation: u64,
#[prost(bool, tag = "9")]
pub dirty_usage_pending: bool,
#[prost(uint64, optional, tag = "10")]
pub movement_generation: ::core::option::Option<u64>,
#[prost(bool, optional, tag = "11")]
pub publication_blocked: ::core::option::Option<bool>,
}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct ScannerPublicationLeaseRequest {
#[prost(bytes = "bytes", tag = "1")]
pub challenge: ::prost::bytes::Bytes,
#[prost(uint64, tag = "2")]
pub expected_movement_generation: u64,
#[prost(uint64, tag = "3")]
pub ttl_ms: u64,
#[prost(string, tag = "4")]
pub expected_session_id: ::prost::alloc::string::String,
#[prost(bytes = "bytes", tag = "5")]
pub token: ::prost::bytes::Bytes,
}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct ScannerPublicationLeaseResponse {
#[prost(bool, tag = "1")]
pub success: bool,
#[prost(bytes = "bytes", tag = "2")]
pub token: ::prost::bytes::Bytes,
#[prost(uint64, tag = "3")]
pub movement_generation: u64,
#[prost(uint64, tag = "4")]
pub lease_ttl_ms: u64,
#[prost(message, optional, tag = "5")]
pub error: ::core::option::Option<Error>,
#[prost(bytes = "bytes", tag = "6")]
pub response_proof: ::prost::bytes::Bytes,
#[prost(string, tag = "7")]
pub owner_id: ::prost::alloc::string::String,
#[prost(string, tag = "8")]
pub session_id: ::prost::alloc::string::String,
}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct ScannerPublicationLeaseReleaseRequest {
#[prost(bytes = "bytes", tag = "1")]
pub challenge: ::prost::bytes::Bytes,
#[prost(bytes = "bytes", tag = "2")]
pub token: ::prost::bytes::Bytes,
#[prost(string, tag = "3")]
pub owner_id: ::prost::alloc::string::String,
#[prost(string, tag = "4")]
pub session_id: ::prost::alloc::string::String,
}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct ScannerPublicationLeaseReleaseResponse {
#[prost(bool, tag = "1")]
pub success: bool,
#[prost(message, optional, tag = "2")]
pub error: ::core::option::Option<Error>,
#[prost(bytes = "bytes", tag = "3")]
pub response_proof: ::prost::bytes::Bytes,
}
#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
pub struct BackgroundHealStatusRequest {
@@ -2695,6 +2756,36 @@ pub mod node_service_client {
.insert(GrpcMethod::new("node_service.NodeService", "ScannerActivity"));
self.inner.unary(req, path, codec).await
}
pub async fn acquire_scanner_publication_lease(
&mut self,
request: impl tonic::IntoRequest<super::ScannerPublicationLeaseRequest>,
) -> std::result::Result<tonic::Response<super::ScannerPublicationLeaseResponse>, tonic::Status> {
self.inner
.ready()
.await
.map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?;
let codec = tonic_prost::ProstCodec::default();
let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/AcquireScannerPublicationLease");
let mut req = request.into_request();
req.extensions_mut()
.insert(GrpcMethod::new("node_service.NodeService", "AcquireScannerPublicationLease"));
self.inner.unary(req, path, codec).await
}
pub async fn release_scanner_publication_lease(
&mut self,
request: impl tonic::IntoRequest<super::ScannerPublicationLeaseReleaseRequest>,
) -> std::result::Result<tonic::Response<super::ScannerPublicationLeaseReleaseResponse>, tonic::Status> {
self.inner
.ready()
.await
.map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?;
let codec = tonic_prost::ProstCodec::default();
let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/ReleaseScannerPublicationLease");
let mut req = request.into_request();
req.extensions_mut()
.insert(GrpcMethod::new("node_service.NodeService", "ReleaseScannerPublicationLease"));
self.inner.unary(req, path, codec).await
}
pub async fn background_heal_status(
&mut self,
request: impl tonic::IntoRequest<super::BackgroundHealStatusRequest>,
@@ -3208,6 +3299,18 @@ pub mod node_service_server {
&self,
request: tonic::Request<super::ScannerActivityRequest>,
) -> std::result::Result<tonic::Response<super::ScannerActivityResponse>, tonic::Status>;
async fn acquire_scanner_publication_lease(
&self,
_request: tonic::Request<super::ScannerPublicationLeaseRequest>,
) -> std::result::Result<tonic::Response<super::ScannerPublicationLeaseResponse>, tonic::Status> {
Err(tonic::Status::unimplemented("scanner publication leases are unsupported"))
}
async fn release_scanner_publication_lease(
&self,
_request: tonic::Request<super::ScannerPublicationLeaseReleaseRequest>,
) -> std::result::Result<tonic::Response<super::ScannerPublicationLeaseReleaseResponse>, tonic::Status> {
Err(tonic::Status::unimplemented("scanner publication leases are unsupported"))
}
async fn background_heal_status(
&self,
request: tonic::Request<super::BackgroundHealStatusRequest>,
@@ -5488,6 +5591,67 @@ pub mod node_service_server {
};
Box::pin(fut)
}
"/node_service.NodeService/AcquireScannerPublicationLease" => {
#[allow(non_camel_case_types)]
struct AcquireScannerPublicationLeaseSvc<T: NodeService>(pub Arc<T>);
impl<T: NodeService> tonic::server::UnaryService<super::ScannerPublicationLeaseRequest> for AcquireScannerPublicationLeaseSvc<T> {
type Response = super::ScannerPublicationLeaseResponse;
type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
fn call(&mut self, request: tonic::Request<super::ScannerPublicationLeaseRequest>) -> Self::Future {
let inner = Arc::clone(&self.0);
let fut = async move { <T as NodeService>::acquire_scanner_publication_lease(&inner, request).await };
Box::pin(fut)
}
}
let accept_compression_encodings = self.accept_compression_encodings;
let send_compression_encodings = self.send_compression_encodings;
let max_decoding_message_size = self.max_decoding_message_size;
let max_encoding_message_size = self.max_encoding_message_size;
let inner = self.inner.clone();
let fut = async move {
let method = AcquireScannerPublicationLeaseSvc(inner);
let codec = tonic_prost::ProstCodec::default();
let mut grpc = tonic::server::Grpc::new(codec)
.apply_compression_config(accept_compression_encodings, send_compression_encodings)
.apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size);
let res = grpc.unary(method, req).await;
Ok(res)
};
Box::pin(fut)
}
"/node_service.NodeService/ReleaseScannerPublicationLease" => {
#[allow(non_camel_case_types)]
struct ReleaseScannerPublicationLeaseSvc<T: NodeService>(pub Arc<T>);
impl<T: NodeService> tonic::server::UnaryService<super::ScannerPublicationLeaseReleaseRequest>
for ReleaseScannerPublicationLeaseSvc<T>
{
type Response = super::ScannerPublicationLeaseReleaseResponse;
type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
fn call(
&mut self,
request: tonic::Request<super::ScannerPublicationLeaseReleaseRequest>,
) -> Self::Future {
let inner = Arc::clone(&self.0);
let fut = async move { <T as NodeService>::release_scanner_publication_lease(&inner, request).await };
Box::pin(fut)
}
}
let accept_compression_encodings = self.accept_compression_encodings;
let send_compression_encodings = self.send_compression_encodings;
let max_decoding_message_size = self.max_decoding_message_size;
let max_encoding_message_size = self.max_encoding_message_size;
let inner = self.inner.clone();
let fut = async move {
let method = ReleaseScannerPublicationLeaseSvc(inner);
let codec = tonic_prost::ProstCodec::default();
let mut grpc = tonic::server::Grpc::new(codec)
.apply_compression_config(accept_compression_encodings, send_compression_encodings)
.apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size);
let res = grpc.unary(method, req).await;
Ok(res)
};
Box::pin(fut)
}
"/node_service.NodeService/BackgroundHealStatus" => {
#[allow(non_camel_case_types)]
struct BackgroundHealStatusSvc<T: NodeService>(pub Arc<T>);
+261 -2
View File
@@ -486,6 +486,160 @@ pub fn canonical_scanner_activity_response_body(
Ok(body)
}
/// Builds the protocol-v7 response body. The optional movement fields are
/// presence-bound so a missing terminal-generation proof cannot authenticate
/// as the value zero.
pub fn canonical_scanner_activity_v7_response_body(
challenge: &[u8],
response: &proto_gen::node_service::ScannerActivityResponse,
) -> Result<Vec<u8>, std::num::TryFromIntError> {
const DOMAIN: &[u8] = b"rustfs-scanner-activity-response-v3\0";
let instance_id = response.instance_id.as_bytes();
let topology_digest = response.topology_digest.as_ref();
let mut body = Vec::with_capacity(DOMAIN.len() + challenge.len() + instance_id.len() + topology_digest.len() + 4 + 8 * 8 + 4);
body.extend_from_slice(DOMAIN);
body.extend_from_slice(&u64::try_from(challenge.len())?.to_be_bytes());
body.extend_from_slice(challenge);
body.extend_from_slice(&u64::try_from(instance_id.len())?.to_be_bytes());
body.extend_from_slice(instance_id);
body.extend_from_slice(&response.namespace_generation.to_be_bytes());
body.extend_from_slice(&response.maintenance_generation.to_be_bytes());
body.extend_from_slice(&response.protocol_version.to_be_bytes());
body.extend_from_slice(&u64::try_from(topology_digest.len())?.to_be_bytes());
body.extend_from_slice(topology_digest);
body.push(u8::from(response.data_movement_active));
body.extend_from_slice(&response.dirty_usage_generation.to_be_bytes());
body.push(u8::from(response.dirty_usage_pending));
body.push(u8::from(response.movement_generation.is_some()));
if let Some(generation) = response.movement_generation {
body.extend_from_slice(&generation.to_be_bytes());
}
body.push(u8::from(response.publication_blocked.is_some()));
if let Some(blocked) = response.publication_blocked {
body.push(u8::from(blocked));
}
Ok(body)
}
/// Builds the body authenticated by the short-lived remote scanner publication
/// lease request. This is a separate domain from ScannerActivity so v6/v7
/// observation proofs remain byte-for-byte compatible.
pub fn canonical_scanner_publication_lease_request_body(
request: &proto_gen::node_service::ScannerPublicationLeaseRequest,
) -> Result<Vec<u8>, std::num::TryFromIntError> {
const DOMAIN: &[u8] = b"rustfs-scanner-publication-lease-request-v1\0";
let challenge = request.challenge.as_ref();
let session_id = request.expected_session_id.as_bytes();
let mut body = Vec::with_capacity(DOMAIN.len() + challenge.len() + session_id.len() + 40);
body.extend_from_slice(DOMAIN);
body.extend_from_slice(&u64::try_from(challenge.len())?.to_be_bytes());
body.extend_from_slice(challenge);
body.extend_from_slice(&request.expected_movement_generation.to_be_bytes());
body.extend_from_slice(&request.ttl_ms.to_be_bytes());
body.extend_from_slice(&u64::try_from(session_id.len())?.to_be_bytes());
body.extend_from_slice(session_id);
// Empty keeps the original acquire body byte-for-byte compatible. A
// non-empty token is the authenticated validation form used immediately
// before a coordinator commits its final publication.
let token = request.token.as_ref();
if !token.is_empty() {
body.extend_from_slice(&u64::try_from(token.len())?.to_be_bytes());
body.extend_from_slice(token);
}
Ok(body)
}
/// Builds the body authenticated by a remote scanner publication lease
/// release request.
pub fn canonical_scanner_publication_lease_release_request_body(
request: &proto_gen::node_service::ScannerPublicationLeaseReleaseRequest,
) -> Result<Vec<u8>, std::num::TryFromIntError> {
const DOMAIN: &[u8] = b"rustfs-scanner-publication-lease-release-request-v1\0";
let challenge = request.challenge.as_ref();
let token = request.token.as_ref();
let mut body = Vec::with_capacity(DOMAIN.len() + challenge.len() + token.len() + 16);
body.extend_from_slice(DOMAIN);
body.extend_from_slice(&u64::try_from(challenge.len())?.to_be_bytes());
body.extend_from_slice(challenge);
body.extend_from_slice(&u64::try_from(token.len())?.to_be_bytes());
body.extend_from_slice(token);
let owner_id = request.owner_id.as_bytes();
let session_id = request.session_id.as_bytes();
body.extend_from_slice(&u64::try_from(owner_id.len())?.to_be_bytes());
body.extend_from_slice(owner_id);
body.extend_from_slice(&u64::try_from(session_id.len())?.to_be_bytes());
body.extend_from_slice(session_id);
Ok(body)
}
pub fn canonical_scanner_publication_lease_response_body(
challenge: &[u8],
response: &proto_gen::node_service::ScannerPublicationLeaseResponse,
) -> Result<Vec<u8>, std::num::TryFromIntError> {
const DOMAIN: &[u8] = b"rustfs-scanner-publication-lease-response-v1\0";
let token = response.token.as_ref();
let owner_id = response.owner_id.as_bytes();
let session_id = response.session_id.as_bytes();
let error_info = response.error.as_ref().map(|error| error.error_info.as_bytes());
let error_code = response.error.as_ref().map_or(0, |error| error.code);
let mut body = Vec::with_capacity(
DOMAIN.len() + challenge.len() + token.len() + owner_id.len() + session_id.len() + error_info.map_or(0, |v| v.len()) + 72,
);
body.extend_from_slice(DOMAIN);
body.extend_from_slice(&u64::try_from(challenge.len())?.to_be_bytes());
body.extend_from_slice(challenge);
body.push(u8::from(response.success));
body.extend_from_slice(&u64::try_from(token.len())?.to_be_bytes());
body.extend_from_slice(token);
body.extend_from_slice(&response.movement_generation.to_be_bytes());
body.extend_from_slice(&response.lease_ttl_ms.to_be_bytes());
body.extend_from_slice(&u64::try_from(owner_id.len())?.to_be_bytes());
body.extend_from_slice(owner_id);
body.extend_from_slice(&u64::try_from(session_id.len())?.to_be_bytes());
body.extend_from_slice(session_id);
body.push(u8::from(response.error.is_some()));
body.extend_from_slice(&error_code.to_be_bytes());
body.extend_from_slice(&u64::try_from(error_info.map_or(0, |value| value.len()))?.to_be_bytes());
if let Some(error_info) = error_info {
body.extend_from_slice(error_info);
}
Ok(body)
}
pub fn canonical_scanner_publication_lease_release_response_body(
challenge: &[u8],
request: &proto_gen::node_service::ScannerPublicationLeaseReleaseRequest,
response: &proto_gen::node_service::ScannerPublicationLeaseReleaseResponse,
) -> Result<Vec<u8>, std::num::TryFromIntError> {
const DOMAIN: &[u8] = b"rustfs-scanner-publication-lease-release-response-v1\0";
let token = request.token.as_ref();
let owner_id = request.owner_id.as_bytes();
let session_id = request.session_id.as_bytes();
let error_info = response.error.as_ref().map(|error| error.error_info.as_bytes());
let error_code = response.error.as_ref().map_or(0, |error| error.code);
let mut body = Vec::with_capacity(
DOMAIN.len() + challenge.len() + token.len() + owner_id.len() + session_id.len() + error_info.map_or(0, |v| v.len()) + 56,
);
body.extend_from_slice(DOMAIN);
body.extend_from_slice(&u64::try_from(challenge.len())?.to_be_bytes());
body.extend_from_slice(challenge);
body.extend_from_slice(&u64::try_from(token.len())?.to_be_bytes());
body.extend_from_slice(token);
body.extend_from_slice(&u64::try_from(owner_id.len())?.to_be_bytes());
body.extend_from_slice(owner_id);
body.extend_from_slice(&u64::try_from(session_id.len())?.to_be_bytes());
body.extend_from_slice(session_id);
body.push(u8::from(response.success));
body.push(u8::from(response.error.is_some()));
body.extend_from_slice(&error_code.to_be_bytes());
body.extend_from_slice(&u64::try_from(error_info.map_or(0, |value| value.len()))?.to_be_bytes());
if let Some(error_info) = error_info {
body.extend_from_slice(error_info);
}
Ok(body)
}
/// Length-prefixed, domain-separated byte builder for the disk-mutation canonical bodies below.
/// Every variable-length field is u64-length-prefixed and every list u64-count-prefixed, so
/// distinct field values can never collide into the same canonical bytes.
@@ -758,6 +912,11 @@ pub fn canonical_rename_data_request_body(
body.push_str(&request.dst_volume)?;
body.push_str(&request.dst_path)?;
body.push_bytes(&request.file_info_bin)?;
// Keep legacy rename requests byte-for-byte compatible. The optional
// token is included only for the scanner's target-side lease fence.
if !request.scanner_publication_lease_token.is_empty() {
body.push_bytes(&request.scanner_publication_lease_token)?;
}
Ok(body.finish())
}
@@ -840,6 +999,9 @@ pub fn canonical_delete_request_body(
body.push_str(&request.volume)?;
body.push_str(&request.path)?;
body.push_str(&request.options)?;
if !request.scanner_publication_lease_token.is_empty() {
body.push_bytes(&request.scanner_publication_lease_token)?;
}
Ok(body.finish())
}
@@ -1002,6 +1164,7 @@ mod disk_mutation_canonical_tests {
dst_volume: "dst-vol".into(),
dst_path: "dst-path".into(),
file_info_bin: vec![0x81, 0x01].into(),
scanner_publication_lease_token: Vec::new().into(),
};
let mut bodies = vec![canonical_rename_data_request_body(&baseline).unwrap()];
for mutate in [
@@ -1013,6 +1176,7 @@ mod disk_mutation_canonical_tests {
|r: &mut RenameDataRequest| r.dst_path = "dst-path2".into(),
|r: &mut RenameDataRequest| r.file_info_bin = vec![0x81, 0x02].into(),
|r: &mut RenameDataRequest| r.file_info_bin = Vec::new().into(),
|r: &mut RenameDataRequest| r.scanner_publication_lease_token = vec![0x01; 16].into(),
] {
let mut request = baseline.clone();
mutate(&mut request);
@@ -1173,6 +1337,7 @@ mod disk_mutation_canonical_tests {
volume: "v".into(),
path: "p".into(),
options: "{\"o\":1}".into(),
scanner_publication_lease_token: Vec::new().into(),
};
let mut bodies = vec![canonical_delete_request_body(&delete).unwrap()];
for mutate in [
@@ -1180,6 +1345,7 @@ mod disk_mutation_canonical_tests {
|r: &mut DeleteRequest| r.volume = "v2".into(),
|r: &mut DeleteRequest| r.path = "p2".into(),
|r: &mut DeleteRequest| r.options = "{\"recursive\":true}".into(),
|r: &mut DeleteRequest| r.scanner_publication_lease_token = vec![0x01; 16].into(),
] {
let mut request = delete.clone();
mutate(&mut request);
@@ -1565,8 +1731,12 @@ mod non_disk_mutation_canonical_tests {
mod scanner_activity_tests {
use super::{
canonical_scanner_activity_request_body, canonical_scanner_activity_response_body,
canonical_scanner_activity_v4_response_body,
proto_gen::node_service::{ScannerActivityRequest, ScannerActivityResponse},
canonical_scanner_activity_v4_response_body, canonical_scanner_activity_v7_response_body,
canonical_scanner_publication_lease_release_request_body, canonical_scanner_publication_lease_request_body,
canonical_scanner_publication_lease_response_body,
proto_gen::node_service::{
ScannerActivityRequest, ScannerActivityResponse, ScannerPublicationLeaseRequest, ScannerPublicationLeaseResponse,
},
};
#[test]
@@ -1617,6 +1787,8 @@ mod scanner_activity_tests {
response_proof: Vec::new().into(),
dirty_usage_generation: 11,
dirty_usage_pending: true,
movement_generation: Some(19),
publication_blocked: Some(false),
};
let baseline =
canonical_scanner_activity_response_body(&[1; 16], &response).expect("scanner activity response should encode");
@@ -1667,6 +1839,25 @@ mod scanner_activity_tests {
canonical_scanner_activity_response_body(&[2; 16], &response)
.expect("scanner activity response with a different challenge should encode")
);
let v7_baseline =
canonical_scanner_activity_v7_response_body(&[1; 16], &response).expect("scanner activity v7 response should encode");
for variant in [
ScannerActivityResponse {
movement_generation: Some(20),
..response.clone()
},
ScannerActivityResponse {
publication_blocked: Some(true),
..response
},
] {
assert_ne!(
v7_baseline,
canonical_scanner_activity_v7_response_body(&[1; 16], &variant)
.expect("scanner activity v7 response variant should encode")
);
}
}
#[test]
@@ -1681,6 +1872,8 @@ mod scanner_activity_tests {
response_proof: Vec::new().into(),
dirty_usage_generation: 0,
dirty_usage_pending: false,
movement_generation: None,
publication_blocked: None,
};
let baseline =
canonical_scanner_activity_v4_response_body(&[1; 16], &response).expect("scanner activity v4 response should encode");
@@ -1714,6 +1907,72 @@ mod scanner_activity_tests {
.expect("scanner activity v4 response should ignore v5 fields")
);
}
#[test]
fn scanner_publication_lease_canonical_bodies_bind_session_owner_and_generation() {
let request = ScannerPublicationLeaseRequest {
challenge: vec![1; 16].into(),
expected_movement_generation: 7,
ttl_ms: 60_000,
expected_session_id: "session-a".to_string(),
token: Vec::new().into(),
};
let baseline = canonical_scanner_publication_lease_request_body(&request).unwrap();
for variant in [
ScannerPublicationLeaseRequest {
expected_movement_generation: 8,
..request.clone()
},
ScannerPublicationLeaseRequest {
expected_session_id: "session-b".to_string(),
..request.clone()
},
ScannerPublicationLeaseRequest {
ttl_ms: 30_000,
..request.clone()
},
ScannerPublicationLeaseRequest {
token: vec![3; 16].into(),
..request
},
] {
assert_ne!(baseline, canonical_scanner_publication_lease_request_body(&variant).unwrap());
}
let release_a = crate::proto_gen::node_service::ScannerPublicationLeaseReleaseRequest {
challenge: vec![2; 16].into(),
token: vec![3; 16].into(),
owner_id: "owner-a".to_string(),
session_id: "session-a".to_string(),
};
let release_b = crate::proto_gen::node_service::ScannerPublicationLeaseReleaseRequest {
owner_id: "owner-b".to_string(),
..release_a.clone()
};
assert_ne!(
canonical_scanner_publication_lease_release_request_body(&release_a).unwrap(),
canonical_scanner_publication_lease_release_request_body(&release_b).unwrap()
);
let response = ScannerPublicationLeaseResponse {
success: true,
token: vec![4; 16].into(),
movement_generation: 7,
lease_ttl_ms: 60_000,
error: None,
response_proof: Vec::new().into(),
owner_id: "owner-a".to_string(),
session_id: "session-a".to_string(),
};
let response_changed = ScannerPublicationLeaseResponse {
owner_id: "owner-b".to_string(),
..response.clone()
};
assert_ne!(
canonical_scanner_publication_lease_response_body(&[1; 16], &response).unwrap(),
canonical_scanner_publication_lease_response_body(&[1; 16], &response_changed).unwrap()
);
}
}
#[cfg(test)]
+55
View File
@@ -111,6 +111,9 @@ message DeleteRequest {
string volume = 2;
string path = 3;
string options = 4;
// Optional scanner publication lease token. When present, the target binds
// the complete delete operation to its movement read admission.
bytes scanner_publication_lease_token = 5;
}
message DeleteResponse {
@@ -281,6 +284,10 @@ message RenameDataRequest {
string dst_volume = 5;
string dst_path = 6;
bytes file_info_bin = 7;
// Optional target-side scanner publication lease. Empty preserves the
// legacy rename request body; a non-empty token is checked at the target's
// rename linearization point.
bytes scanner_publication_lease_token = 8;
}
message RenameDataResponse {
@@ -844,6 +851,52 @@ message ScannerActivityResponse {
bytes response_proof = 7;
uint64 dirty_usage_generation = 8;
bool dirty_usage_pending = 9;
// v7 fields. They are optional so v6 peers can continue to decode the
// response shape while newer readers fail closed when they are absent.
optional uint64 movement_generation = 10;
optional bool publication_blocked = 11;
}
// A short-lived storage-owned read admission used only around a final
// scanner metadata publication. It is intentionally separate from the
// ScannerActivity observation wire so v6/v7 rolling compatibility remains
// unchanged.
message ScannerPublicationLeaseRequest {
bytes challenge = 1;
uint64 expected_movement_generation = 2;
uint64 ttl_ms = 3;
// The activity instance is a process session nonce. It is intentionally
// separate from the storage-owned deployment identity returned by the
// lease response so a restart cannot reuse an old session token.
string expected_session_id = 4;
// A non-empty token turns the acquire RPC into an in-place validation of an
// existing lease. Keeping this on the existing RPC lets old peers reject
// the proof without changing the v7 activity wire shape.
bytes token = 5;
}
message ScannerPublicationLeaseResponse {
bool success = 1;
bytes token = 2;
uint64 movement_generation = 3;
uint64 lease_ttl_ms = 4;
optional Error error = 5;
bytes response_proof = 6;
string owner_id = 7;
string session_id = 8;
}
message ScannerPublicationLeaseReleaseRequest {
bytes challenge = 1;
bytes token = 2;
string owner_id = 3;
string session_id = 4;
}
message ScannerPublicationLeaseReleaseResponse {
bool success = 1;
optional Error error = 2;
bytes response_proof = 3;
}
message BackgroundHealStatusRequest {
@@ -1096,6 +1149,8 @@ service NodeService {
// rpc CommitBinary() returns () {};
rpc SignalService(SignalServiceRequest) returns (SignalServiceResponse) {}; // auth-policy: body-bound
rpc ScannerActivity(ScannerActivityRequest) returns (ScannerActivityResponse) {}; // auth-policy: body-bound
rpc AcquireScannerPublicationLease(ScannerPublicationLeaseRequest) returns (ScannerPublicationLeaseResponse) {}; // auth-policy: body-bound
rpc ReleaseScannerPublicationLease(ScannerPublicationLeaseReleaseRequest) returns (ScannerPublicationLeaseReleaseResponse) {}; // auth-policy: body-bound
rpc BackgroundHealStatus(BackgroundHealStatusRequest) returns (BackgroundHealStatusResponse) {}; // auth-policy: read-only
rpc ReplacementRecoveryStatus(ReplacementRecoveryStatusRequest) returns (ReplacementRecoveryStatusResponse) {}; // auth-policy: read-only
rpc GetMetacacheListing(GetMetacacheListingRequest) returns (GetMetacacheListingResponse) {}; // auth-policy: unimplemented
+11 -2
View File
@@ -92,7 +92,7 @@ pub use scanner_io::{
pub use sleeper::{DynamicSleeper, SCANNER_IDLE_MODE, SCANNER_SLEEPER};
use std::sync::atomic::{AtomicU64, Ordering};
pub use storage_api::ScannerReplicationConfig as ReplicationConfig;
pub use storage_api::scan::SCANNER_ACTIVITY_PROTOCOL_VERSION;
pub use storage_api::scan::{SCANNER_ACTIVITY_PROTOCOL_VERSION, SCANNER_ACTIVITY_V6_PROTOCOL_VERSION};
static SCANNER_ACTIVE_WORK_UNITS: AtomicU64 = AtomicU64::new(0);
static SCANNER_RUNTIME_INSTANCES: AtomicU64 = AtomicU64::new(0);
@@ -796,17 +796,25 @@ where
Some(admission)
}
pub(crate) async fn save_config_shared_with_preconditions<S>(
pub(crate) async fn save_config_shared_with_preconditions_and_lease_fence<S>(
api: Arc<S>,
file: &str,
data: Bytes,
sha256hex: Option<String>,
preconditions: HTTPPreconditions,
scanner_publication_lease_fence: Option<&str>,
) -> EcstoreResult<ScannerObjectInfo>
where
S: ScannerObjectIO,
{
let mut reader = ScannerPutObjReader::from_prehashed_bytes(data, sha256hex)?;
let mut user_defined = HashMap::new();
if let Some(fence) = scanner_publication_lease_fence {
user_defined.insert(
storage_api::owner::SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY.to_string(),
fence.to_string(),
);
}
api.put_object(
RUSTFS_META_BUCKET,
file,
@@ -814,6 +822,7 @@ where
&ScannerObjectOptions {
max_parity: true,
http_preconditions: Some(preconditions),
user_defined,
..Default::default()
},
)
+159 -13
View File
@@ -63,6 +63,7 @@ use tokio_util::sync::CancellationToken;
use tokio_util::task::AbortOnDropHandle;
use tracing::{debug, error, info, instrument, warn};
use crate::storage_api::owner::SCANNER_PUBLICATION_LEASE_TTL_MS;
use crate::storage_api::scan::{
BucketOperations, BucketOptions, NamespaceLocking as _, SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION,
SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SCANNER_ACTIVITY_PROTOCOL_VERSION,
@@ -71,9 +72,9 @@ use crate::{
ECStore, EcstoreError, RUSTFS_META_BUCKET, SCANNER_PUBLICATION_EPOCH_CHANGED, ScannerLifecycleConfigExt as _,
ScannerReplicationConfigExt as _, delete_config_with_publication_admission_for_epoch, get_lifecycle_config,
get_replication_config, invalidate_admin_data_usage_snapshot_cache, invalidate_data_usage_snapshot_cache, read_config,
replace_bucket_usage_memory_from_info, save_config, save_config_shared_with_preconditions, save_config_with_preconditions,
save_config_with_publication_admission_for_epoch, scanner_is_erasure_sd, scanner_publication_admission_for_epoch,
scanner_publication_epoch, scanner_publication_epoch_changed,
replace_bucket_usage_memory_from_info, save_config, save_config_shared_with_preconditions_and_lease_fence,
save_config_with_preconditions, save_config_with_publication_admission_for_epoch, scanner_is_erasure_sd,
scanner_publication_admission_for_epoch, scanner_publication_epoch, scanner_publication_epoch_changed,
};
const LOG_COMPONENT_SCANNER: &str = "scanner";
@@ -125,6 +126,8 @@ const MAINTENANCE_FEATURE_INSPECTION_RETRY_MAX_INTERVAL: Duration = Duration::fr
const MAX_MAINTENANCE_FEATURE_INSPECTION_ATTEMPTS: usize = 2;
const SCANNER_PERSIST_CAS_RETRIES: usize = 2;
const DATA_USAGE_BACKUP_INTERVAL_CYCLES: u64 = 10;
const SCANNER_PUBLICATION_LEASE_FENCE_MAX_ENTRIES: usize = 256;
const SCANNER_PUBLICATION_LEASE_FENCE_MAX_BYTES: usize = 64 * 1024;
const SCANNER_CYCLE_STATE_MAGIC: &[u8; 8] = b"RSCYC001";
const SCANNER_CYCLE_STATE_HEADER_LEN: usize = 24;
#[cfg(test)]
@@ -137,6 +140,10 @@ static SCANNER_CYCLE_STATE_PERSIST_TEST_HOOK: LazyLock<StdMutex<Option<ScannerCy
static SCANNER_CYCLE_RECOVERY_WAKE: LazyLock<Notify> = LazyLock::new(Notify::new);
fn remote_publication_lease_fence_targets_are_required(target_count: usize, grants_present: bool, fence_present: bool) -> bool {
target_count > 0 && (!grants_present || !fence_present)
}
pub(super) fn notify_scanner_cycle_recovery_wake() {
SCANNER_CYCLE_RECOVERY_WAKE.notify_one();
}
@@ -407,19 +414,24 @@ async fn sync_data_usage_backup_from_primary(
ctx: &CancellationToken,
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
) -> Result<(), EcstoreError> {
sync_data_usage_backup_from_primary_for_epoch(ctx, storeapi, None).await
sync_data_usage_backup_from_primary_for_epoch_and_lease_and_fence(ctx, storeapi, None, None, None).await
}
async fn sync_data_usage_backup_from_primary_for_epoch(
async fn sync_data_usage_backup_from_primary_for_epoch_and_lease_and_fence(
ctx: &CancellationToken,
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
expected_publication_epoch: Option<u64>,
remote_lease_deadline: Option<std::time::Instant>,
scanner_publication_lease_fence: Option<&str>,
) -> Result<(), EcstoreError> {
let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str());
for retry in 0..=SCANNER_PERSIST_CAS_RETRIES {
if ctx.is_cancelled() {
return Ok(());
}
if remote_lease_deadline.is_some_and(|deadline| std::time::Instant::now() >= deadline) {
return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED));
}
let read_epoch = match expected_publication_epoch {
Some(expected_epoch) => {
@@ -446,6 +458,10 @@ async fn sync_data_usage_backup_from_primary_for_epoch(
}
let primary = Bytes::from(primary);
if remote_lease_deadline.is_some_and(|deadline| std::time::Instant::now() >= deadline) {
return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED));
}
let (backup, revision) = read_config_with_revision(storeapi.clone(), &backup_path).await?;
if backup.as_deref() == Some(primary.as_ref()) {
if scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch)
@@ -462,18 +478,22 @@ async fn sync_data_usage_backup_from_primary_for_epoch(
let sha256hex = Some(hex_simd::encode_to_string(Sha256::digest(&primary), hex_simd::AsciiCase::Lower));
let save_result = {
if remote_lease_deadline.is_some_and(|deadline| std::time::Instant::now() >= deadline) {
return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED));
}
let Some(_publication_admission) = scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch).await else {
if retry < SCANNER_PERSIST_CAS_RETRIES {
continue;
}
return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED));
};
save_config_shared_with_preconditions(
save_config_shared_with_preconditions_and_lease_fence(
storeapi.clone(),
&backup_path,
primary.clone(),
sha256hex,
revision.preconditions(),
scanner_publication_lease_fence,
)
.await
};
@@ -1377,8 +1397,89 @@ async fn run_data_scanner_cycle_with_budget(
};
let publication_deferred = publication_defer_reason.is_some();
let publication_epoch = scan_result.as_ref().ok().and_then(ScannerCycleResult::publication_epoch);
let remote_publication_lease_targets = if publication_defer_reason.is_none() {
scan_result
.as_ref()
.ok()
.map(|result| result.remote_publication_lease_targets().to_vec())
.unwrap_or_default()
} else {
Vec::new()
};
let mut remote_publication_leases = None;
let remote_lease_defer_reason = if remote_publication_lease_targets.is_empty() {
None
} else if usage_persist_timeout >= Duration::from_millis(SCANNER_PUBLICATION_LEASE_TTL_MS) {
// The lease is intentionally fixed-duration and has no renewal path.
// Refuse a persistence budget that could outlive it instead of
// allowing the peer to admit movement while a local PUT is in flight.
Some(ScannerCycleDeferReason::ActivityBaselineUnavailable)
} else if let Some(notification_system) = storeapi.notification_system() {
match notification_system
.acquire_scanner_publication_leases(remote_publication_lease_targets.clone())
.await
{
Ok(grants) => {
remote_publication_leases = Some((notification_system, grants));
None
}
Err(_) => Some(ScannerCycleDeferReason::ActivityBaselineUnavailable),
}
} else {
Some(ScannerCycleDeferReason::ActivityBaselineUnavailable)
};
let remote_lease_deadline = remote_publication_leases
.as_ref()
.and_then(|(_, grants)| grants.iter().map(|grant| grant.lease.expires_at).min());
// The transient fence is carried only to the SetDisks rename boundary;
// it is never inserted into FileInfo metadata. Keep the representation
// bounded and require one authenticated token per remote target so a
// partial grant can never silently fall back to an unfenced rename.
let remote_lease_fence = remote_publication_leases.as_ref().and_then(|(_, grants)| {
if grants.len() != remote_publication_lease_targets.len() || grants.len() > SCANNER_PUBLICATION_LEASE_FENCE_MAX_ENTRIES {
return None;
}
let mut fence = BTreeMap::new();
for grant in grants {
if grant.host.is_empty() || grant.host.len() > 1024 {
return None;
}
if fence.insert(grant.host.clone(), grant.lease.token.to_string()).is_some() {
return None;
}
}
if remote_publication_lease_targets
.iter()
.any(|(host, _, _)| !fence.contains_key(host))
{
return None;
}
serde_json::to_string(&fence)
.ok()
.filter(|encoded| encoded.len() <= SCANNER_PUBLICATION_LEASE_FENCE_MAX_BYTES)
});
let remote_lease_fence_defer_reason = (remote_publication_lease_fence_targets_are_required(
remote_publication_lease_targets.len(),
remote_publication_leases.is_some(),
remote_lease_fence.is_some(),
))
.then_some(ScannerCycleDeferReason::ActivityBaselineUnavailable);
let remote_lease_covers_persistence = remote_lease_deadline.is_none_or(|deadline| {
std::time::Instant::now()
.checked_add(usage_persist_timeout)
.is_some_and(|latest_finish| latest_finish < deadline)
});
let publication_defer_reason = publication_defer_reason
.or(remote_lease_defer_reason)
.or(remote_lease_fence_defer_reason);
let publication_defer_reason = (!remote_lease_covers_persistence)
.then_some(ScannerCycleDeferReason::ActivityBaselineUnavailable)
.or(publication_defer_reason);
let budget_elapsed = cycle_budget.budget_elapsed() && !ctx.is_cancelled();
let usage_persist_outcome = match publication_defer_reason {
let remote_lease_probe = remote_publication_leases
.as_ref()
.map(|(notification_system, grants)| (Arc::clone(notification_system), grants.clone()));
let mut usage_persist_outcome = match publication_defer_reason {
Some(reason) => {
drop(receiver);
DataUsagePersistOutcome::Deferred(reason)
@@ -1390,17 +1491,33 @@ async fn run_data_scanner_cycle_with_budget(
let storeapi_clone = storeapi.clone();
let ctx_clone = ctx.clone();
let route_probe_store = storeapi.clone();
let remote_lease_fence = remote_lease_fence.clone();
let mut usage_persist_task = AbortOnDropHandle::new(tokio::spawn(async move {
store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe_for_publication_epoch(
store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe_for_publication_epoch_and_lease_fence(
ctx_clone,
storeapi_clone,
receiver,
Some(leader_epoch),
Some(usage_persist_baseline),
publication_epoch,
ScannerPublicationFence::new(
publication_epoch,
remote_lease_deadline,
remote_lease_fence,
),
move || {
let storeapi = route_probe_store.clone();
async move { storeapi.scanner_data_usage_publication_blocked().await }
let remote_lease_probe = remote_lease_probe.clone();
async move {
if let Some((notification_system, grants)) = remote_lease_probe.as_ref()
&& notification_system.validate_scanner_publication_leases(grants).await.is_err()
{
// A remote restart or movement flip invalidates
// the token proof; usage_store interprets this
// as a publication barrier and performs no PUT.
return true;
}
storeapi.scanner_data_usage_publication_blocked().await
}
},
)
.await
@@ -1448,6 +1565,22 @@ async fn run_data_scanner_cycle_with_budget(
}
}
};
let lease_expired = remote_publication_leases
.as_ref()
.is_some_and(|(_, grants)| grants.iter().any(|grant| !grant.lease.is_valid()));
if let Some((notification_system, grants)) = remote_publication_leases.take() {
let release_result = notification_system.release_scanner_publication_leases(grants).await;
if lease_expired || release_result.is_err() {
// A lease that expired or could not be released is never treated
// as a successful authoritative publication. The peer may have
// admitted movement immediately after the lease ended.
usage_persist_outcome = if usage_persist_outcome == DataUsagePersistOutcome::Failed {
DataUsagePersistOutcome::Failed
} else {
DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable)
};
}
}
let unresolved_heal_work = global_metrics().current_scan_cycle_has_unresolved_heal_work();
let scan_cycle_result = match scan_result {
@@ -2199,7 +2332,16 @@ async fn run_data_scanner_with_maintenance_state(
);
let activity_poll_interval = backoff_enabled.then_some(runtime_config.cycle_interval.max(Duration::from_secs(1)));
let wake_reason = wait_for_next_scanner_cycle_with_activity(
let movement_generation_before_wait = storeapi.scanner_data_movement_generation();
let movement_changed = storeapi.scanner_data_movement_changed();
let movement_store = storeapi.clone();
let movement = ScannerMovementWaitContext {
movement_generation_seen: Some(movement_generation_before_wait),
movement_changed,
current_movement_generation: move || movement_store.scanner_data_movement_generation(),
is_lock_lost: || guard.is_lock_lost(),
};
let wake_reason = wait_for_next_scanner_cycle_with_activity_and_movement(
&ctx,
wait_plan.delay,
activity_poll_interval,
@@ -2211,7 +2353,7 @@ async fn run_data_scanner_with_maintenance_state(
runtime_config_generation_seen,
maintenance_generation_before_wait,
),
|| guard.is_lock_lost(),
movement,
|| probe_scanner_activity(storeapi.as_ref(), distributed),
)
.await;
@@ -2239,6 +2381,10 @@ async fn run_data_scanner_with_maintenance_state(
ScannerCycleWakeReason::ClusterMaintenance => {
clean_idle_backoff.reset();
}
ScannerCycleWakeReason::MovementGeneration => {
scanner_activity_seen = None;
clean_idle_backoff.reset();
}
ScannerCycleWakeReason::Timer
| ScannerCycleWakeReason::DirtyUsage
| ScannerCycleWakeReason::ClusterActivity
@@ -2612,7 +2758,7 @@ use usage_store::*;
pub use activity::scanner_topology_digest;
pub(crate) use activity::{
ScannerActivitySnapshot, ScannerDirtyUsageAcknowledgement, probe_scanner_activity, scanner_activity_allows_usage_publication,
scanner_activity_snapshot_digest, scanner_dirty_usage_acknowledgements,
scanner_activity_publication_lease_targets, scanner_activity_snapshot_digest, scanner_dirty_usage_acknowledgements,
};
pub(crate) use activity::{ScannerCycleOutcome, scanner_cycle_outcome_with_pending_maintenance};
#[cfg(test)]
+225 -57
View File
@@ -13,11 +13,13 @@
// limitations under the License.
/// Cycle wake/backoff policy and scanner activity observation (probing, generations, topology digest).
use super::*;
use crate::storage_api::scan::SCANNER_ACTIVITY_V6_PROTOCOL_VERSION;
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(super) enum ScannerCycleWakeReason {
Timer,
DirtyUsage,
MovementGeneration,
ClusterActivity,
ClusterMaintenance,
ClusterActivityUnavailable,
@@ -250,6 +252,17 @@ impl ScannerCycleObservedGenerations {
}
}
/// Movement state observed while a scanner waits for the next cycle.
///
/// Keeping the movement inputs together makes it harder for callers to pair a
/// generation with the wrong notification or lock predicate.
pub(super) struct ScannerMovementWaitContext<G, F> {
pub(super) movement_generation_seen: Option<u64>,
pub(super) movement_changed: Arc<Notify>,
pub(super) current_movement_generation: G,
pub(super) is_lock_lost: F,
}
pub(super) const LOCAL_SCANNER_ACTIVITY_NODE: &str = "<local>";
#[derive(Clone, Debug, PartialEq, Eq)]
@@ -262,6 +275,8 @@ pub(crate) struct ScannerNodeActivity {
pub(super) data_movement_active: bool,
pub(super) dirty_usage_generation: u64,
pub(super) dirty_usage_pending: bool,
pub(super) movement_generation: u64,
pub(super) publication_blocked: bool,
}
pub(crate) type ScannerActivitySnapshot = BTreeMap<String, ScannerNodeActivity>;
@@ -278,6 +293,14 @@ pub(super) enum ScannerActivityObservation {
NotRequired,
Unchanged,
Changed,
/// A storage-owned movement generation changed. This wake must bypass the
/// ordinary deferred cluster-activity backoff so publication can retry
/// after a transition reaches its terminal state.
MovementChanged,
/// A remote scanner process restarted. Publication leases are bound to the
/// process instance, so this must bypass deferred cluster-activity backoff
/// even when the restarted peer reports otherwise ordinary activity.
RemoteRestarted,
MaintenanceChanged,
Unverified,
}
@@ -373,6 +396,8 @@ pub(super) fn scanner_activity_observed_work(observation: ScannerActivityObserva
matches!(
observation,
ScannerActivityObservation::Changed
| ScannerActivityObservation::MovementChanged
| ScannerActivityObservation::RemoteRestarted
| ScannerActivityObservation::MaintenanceChanged
| ScannerActivityObservation::Unverified
)
@@ -386,6 +411,7 @@ pub(super) fn scanner_activity_backoff_blocked_after_wake(currently_blocked: boo
}
}
#[cfg(test)]
pub(super) async fn wait_for_next_scanner_cycle<F>(
ctx: &CancellationToken,
delay: Duration,
@@ -396,6 +422,36 @@ pub(super) async fn wait_for_next_scanner_cycle<F>(
) -> ScannerCycleWakeReason
where
F: Fn() -> bool,
{
let movement = ScannerMovementWaitContext {
movement_generation_seen: None,
movement_changed: Arc::new(Notify::new()),
current_movement_generation: || 0,
is_lock_lost,
};
wait_for_next_scanner_cycle_with_movement(
ctx,
delay,
ScannerCycleObservedGenerations {
dirty_usage: dirty_usage_generation_seen,
runtime_config: runtime_config_generation,
maintenance: maintenance_generation,
defer_cluster_activity: false,
},
&movement,
)
.await
}
pub(super) async fn wait_for_next_scanner_cycle_with_movement<G, F>(
ctx: &CancellationToken,
delay: Duration,
generations: ScannerCycleObservedGenerations,
movement: &ScannerMovementWaitContext<G, F>,
) -> ScannerCycleWakeReason
where
F: Fn() -> bool,
G: Fn() -> u64,
{
let sleep = tokio::time::sleep(delay);
tokio::pin!(sleep);
@@ -403,55 +459,86 @@ where
tokio::pin!(lock_poll);
loop {
if is_lock_lost() {
if (movement.is_lock_lost)() {
return ScannerCycleWakeReason::LeaderLockLost;
}
if scanner_runtime_config_generation() != runtime_config_generation {
if scanner_runtime_config_generation() != generations.runtime_config {
return ScannerCycleWakeReason::RuntimeConfig;
}
if scanner_maintenance_generation() != maintenance_generation {
if scanner_maintenance_generation() != generations.maintenance {
return ScannerCycleWakeReason::MaintenanceConfig;
}
if dirty_usage_generation_seen.is_some_and(|seen| dirty_usage_buckets_pending() && dirty_usage_generation() != seen) {
if generations
.dirty_usage
.is_some_and(|seen| dirty_usage_buckets_pending() && dirty_usage_generation() != seen)
{
return ScannerCycleWakeReason::DirtyUsage;
}
if movement
.movement_generation_seen
.is_some_and(|seen| (movement.current_movement_generation)() != seen)
{
return ScannerCycleWakeReason::MovementGeneration;
}
let movement_notification = movement.movement_changed.notified();
tokio::pin!(movement_notification);
movement_notification.as_mut().enable();
// A transition may finish between the initial generation read and
// registration with Notify. Re-check after `enable()` so that such a
// transition cannot be lost when it used `notify_waiters()`.
if movement
.movement_generation_seen
.is_some_and(|seen| (movement.current_movement_generation)() != seen)
{
return ScannerCycleWakeReason::MovementGeneration;
}
tokio::select! {
_ = ctx.cancelled() => return ScannerCycleWakeReason::Cancelled,
_ = &mut sleep => return ScannerCycleWakeReason::Timer,
_ = &mut lock_poll => {
if is_lock_lost() {
if (movement.is_lock_lost)() {
return ScannerCycleWakeReason::LeaderLockLost;
}
lock_poll.as_mut().reset(Instant::now() + SCANNER_LEADER_LOCK_POLL_INTERVAL);
}
_ = dirty_usage_bucket_notified() => {
if scanner_runtime_config_generation() != runtime_config_generation {
if scanner_runtime_config_generation() != generations.runtime_config {
return ScannerCycleWakeReason::RuntimeConfig;
}
if scanner_maintenance_generation() != maintenance_generation {
if scanner_maintenance_generation() != generations.maintenance {
return ScannerCycleWakeReason::MaintenanceConfig;
}
if dirty_usage_generation_seen
if generations
.dirty_usage
.is_some_and(|seen| dirty_usage_buckets_pending() && dirty_usage_generation() != seen)
{
return ScannerCycleWakeReason::DirtyUsage;
}
}
_ = scanner_runtime_config_changed() => {
if scanner_runtime_config_generation() != runtime_config_generation {
if scanner_runtime_config_generation() != generations.runtime_config {
return ScannerCycleWakeReason::RuntimeConfig;
}
}
_ = scanner_maintenance_changed() => {
if scanner_maintenance_generation() != maintenance_generation {
if scanner_maintenance_generation() != generations.maintenance {
return ScannerCycleWakeReason::MaintenanceConfig;
}
}
_ = &mut movement_notification => {
if movement
.movement_generation_seen
.is_some_and(|seen| (movement.current_movement_generation)() != seen)
{
return ScannerCycleWakeReason::MovementGeneration;
}
}
}
}
}
#[cfg(test)]
pub(super) async fn wait_for_next_scanner_cycle_with_activity<F, Probe, ProbeFuture>(
ctx: &CancellationToken,
delay: Duration,
@@ -459,10 +546,43 @@ pub(super) async fn wait_for_next_scanner_cycle_with_activity<F, Probe, ProbeFut
activity_seen: &mut Option<ScannerActivitySnapshot>,
generations: ScannerCycleObservedGenerations,
is_lock_lost: F,
probe_activity: Probe,
) -> ScannerCycleWakeReason
where
F: Fn() -> bool,
Probe: FnMut() -> ProbeFuture,
ProbeFuture: Future<Output = Result<ScannerActivitySnapshot, String>>,
{
let movement = ScannerMovementWaitContext {
movement_generation_seen: None,
movement_changed: Arc::new(Notify::new()),
current_movement_generation: || 0,
is_lock_lost,
};
wait_for_next_scanner_cycle_with_activity_and_movement(
ctx,
delay,
activity_poll_interval,
activity_seen,
generations,
movement,
probe_activity,
)
.await
}
pub(super) async fn wait_for_next_scanner_cycle_with_activity_and_movement<F, G, Probe, ProbeFuture>(
ctx: &CancellationToken,
delay: Duration,
activity_poll_interval: Option<Duration>,
activity_seen: &mut Option<ScannerActivitySnapshot>,
generations: ScannerCycleObservedGenerations,
movement: ScannerMovementWaitContext<G, F>,
mut probe_activity: Probe,
) -> ScannerCycleWakeReason
where
F: Fn() -> bool,
G: Fn() -> u64,
Probe: FnMut() -> ProbeFuture,
ProbeFuture: Future<Output = Result<ScannerActivitySnapshot, String>>,
{
@@ -475,15 +595,7 @@ where
let wait_slice = activity_poll_interval
.map(|interval| interval.max(Duration::from_secs(1)).min(remaining))
.unwrap_or(remaining);
let wake_reason = wait_for_next_scanner_cycle(
ctx,
wait_slice,
generations.dirty_usage,
generations.runtime_config,
generations.maintenance,
&is_lock_lost,
)
.await;
let wake_reason = wait_for_next_scanner_cycle_with_movement(ctx, wait_slice, generations, &movement).await;
if wake_reason != ScannerCycleWakeReason::Timer || Instant::now() >= deadline {
return wake_reason;
}
@@ -491,7 +603,7 @@ where
let Some(_) = activity_poll_interval else {
return ScannerCycleWakeReason::Timer;
};
if is_lock_lost() {
if (movement.is_lock_lost)() {
return ScannerCycleWakeReason::LeaderLockLost;
}
@@ -500,7 +612,7 @@ where
let lock_lost = async {
loop {
tokio::time::sleep(SCANNER_LEADER_LOCK_POLL_INTERVAL).await;
if is_lock_lost() {
if (movement.is_lock_lost)() {
break;
}
}
@@ -519,6 +631,9 @@ where
}
match observation {
ScannerActivityObservation::Unchanged | ScannerActivityObservation::NotRequired => {}
ScannerActivityObservation::MovementChanged | ScannerActivityObservation::RemoteRestarted => {
return ScannerCycleWakeReason::ClusterActivity;
}
ScannerActivityObservation::Changed if !generations.defer_cluster_activity => {
return ScannerCycleWakeReason::ClusterActivity;
}
@@ -568,12 +683,23 @@ pub(super) fn compare_scanner_activity(
let Some(previous_activity) = previous.get(host) else {
continue;
};
if host != LOCAL_SCANNER_ACTIVITY_NODE && previous_activity.instance_id != current_activity.instance_id {
return ScannerActivityObservation::RemoteRestarted;
}
if host != LOCAL_SCANNER_ACTIVITY_NODE
&& previous_activity.instance_id == current_activity.instance_id
&& previous_activity.maintenance_generation != current_activity.maintenance_generation
{
return ScannerActivityObservation::MaintenanceChanged;
}
if previous_activity.instance_id == current_activity.instance_id
&& (previous_activity.data_movement_active != current_activity.data_movement_active
|| previous_activity.movement_generation != current_activity.movement_generation
|| previous_activity.publication_blocked != current_activity.publication_blocked)
{
return ScannerActivityObservation::MovementChanged;
}
}
ScannerActivityObservation::Changed
@@ -630,12 +756,28 @@ pub(crate) fn scanner_activity_snapshot_digest(snapshot: &ScannerActivitySnapsho
hasher.update([u8::from(activity.data_movement_active)]);
hasher.update(activity.dirty_usage_generation.to_be_bytes());
hasher.update([u8::from(activity.dirty_usage_pending)]);
hasher.update(activity.movement_generation.to_be_bytes());
hasher.update([u8::from(activity.publication_blocked)]);
}
hasher.finalize().into()
}
pub(crate) fn scanner_activity_allows_usage_publication(snapshot: &ScannerActivitySnapshot) -> bool {
snapshot.values().all(|activity| !activity.data_movement_active)
!snapshot.is_empty()
&& snapshot.values().all(|activity| {
activity.protocol_version == SCANNER_ACTIVITY_PROTOCOL_VERSION
&& activity.movement_generation != u64::MAX
&& !activity.data_movement_active
&& !activity.publication_blocked
})
}
pub(crate) fn scanner_activity_publication_lease_targets(snapshot: &ScannerActivitySnapshot) -> Vec<(String, String, u64)> {
snapshot
.iter()
.filter(|(host, _)| host.as_str() != LOCAL_SCANNER_ACTIVITY_NODE)
.map(|(host, activity)| (host.clone(), activity.instance_id.clone(), activity.movement_generation))
.collect()
}
pub(crate) fn scanner_dirty_usage_acknowledgements(snapshot: &ScannerActivitySnapshot) -> Vec<ScannerDirtyUsageAcknowledgement> {
@@ -695,11 +837,15 @@ pub(super) fn record_scanner_activity_instance(
pub(crate) async fn probe_scanner_activity(storeapi: &ECStore, distributed: bool) -> Result<ScannerActivitySnapshot, String> {
let topology_digest = scanner_topology_digest(storeapi);
let data_movement_active = storeapi.scanner_data_movement_active().await;
let (data_movement_active, publication_blocked, movement_generation) = storeapi.scanner_data_movement_activity().await;
let namespace_generation = storeapi.scanner_namespace_mutation_generation();
let maintenance_generation = scanner_maintenance_generation();
let dirty_usage = scanner_dirty_usage_state();
if namespace_generation == u64::MAX || maintenance_generation == u64::MAX || dirty_usage.generation == u64::MAX {
if namespace_generation == u64::MAX
|| maintenance_generation == u64::MAX
|| dirty_usage.generation == u64::MAX
|| movement_generation == u64::MAX
{
return Err("local scanner activity generation is exhausted".to_string());
}
let local_instance_id = crate::scanner_io::scanner_activity_epoch().to_string();
@@ -715,6 +861,8 @@ pub(crate) async fn probe_scanner_activity(storeapi: &ECStore, distributed: bool
data_movement_active,
dirty_usage_generation: dirty_usage.generation,
dirty_usage_pending: dirty_usage.pending,
movement_generation,
publication_blocked,
},
)]);
if !distributed {
@@ -732,39 +880,57 @@ pub(crate) async fn probe_scanner_activity(storeapi: &ECStore, distributed: bool
if activity.namespace_generation == u64::MAX || activity.maintenance_generation == u64::MAX {
return Err(format!("scanner activity peer {host} exhausted its activity generation"));
}
let (peer_topology_digest, peer_data_movement_active, peer_dirty_usage_generation, peer_dirty_usage_pending) =
match activity.protocol_version {
SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION => {
return Err(format!("scanner activity peer {host} cannot verify data movement publication fencing"));
}
SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION => {
return Err(format!(
"scanner activity peer {host} cannot safely share scanner cache locks with protocol {}",
SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION
));
}
SCANNER_ACTIVITY_PROTOCOL_VERSION => (
activity
.topology_digest
.ok_or_else(|| format!("scanner activity peer {host} omitted its storage topology"))?,
activity
.data_movement_active
.ok_or_else(|| format!("scanner activity peer {host} omitted its data movement state"))?,
activity
.dirty_usage_generation
.ok_or_else(|| format!("scanner activity peer {host} omitted its dirty usage generation"))?,
activity
.dirty_usage_pending
.ok_or_else(|| format!("scanner activity peer {host} omitted its dirty usage state"))?,
),
version => {
return Err(format!(
"scanner activity peer {host} uses protocol {version}, expected {}",
SCANNER_ACTIVITY_PROTOCOL_VERSION
));
}
};
if peer_dirty_usage_generation == u64::MAX {
let (
peer_topology_digest,
peer_data_movement_active,
peer_dirty_usage_generation,
peer_dirty_usage_pending,
peer_movement_generation,
peer_publication_blocked,
) = match activity.protocol_version {
SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION => {
return Err(format!("scanner activity peer {host} cannot verify data movement publication fencing"));
}
SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION => {
return Err(format!(
"scanner activity peer {host} cannot safely share scanner cache locks with protocol {}",
SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION
));
}
SCANNER_ACTIVITY_V6_PROTOCOL_VERSION => {
return Err(format!(
"scanner activity peer {host} cannot verify terminal movement state with protocol {}",
SCANNER_ACTIVITY_V6_PROTOCOL_VERSION
));
}
SCANNER_ACTIVITY_PROTOCOL_VERSION => (
activity
.topology_digest
.ok_or_else(|| format!("scanner activity peer {host} omitted its storage topology"))?,
activity
.data_movement_active
.ok_or_else(|| format!("scanner activity peer {host} omitted its data movement state"))?,
activity
.dirty_usage_generation
.ok_or_else(|| format!("scanner activity peer {host} omitted its dirty usage generation"))?,
activity
.dirty_usage_pending
.ok_or_else(|| format!("scanner activity peer {host} omitted its dirty usage state"))?,
activity
.movement_generation
.ok_or_else(|| format!("scanner activity peer {host} omitted its movement generation"))?,
activity
.publication_blocked
.ok_or_else(|| format!("scanner activity peer {host} omitted its publication blocked state"))?,
),
version => {
return Err(format!(
"scanner activity peer {host} uses protocol {version}, expected {}",
SCANNER_ACTIVITY_PROTOCOL_VERSION
));
}
};
if peer_dirty_usage_generation == u64::MAX || peer_movement_generation == u64::MAX {
return Err(format!("scanner activity peer {host} exhausted its dirty usage generation"));
}
if peer_topology_digest != topology_digest {
@@ -783,6 +949,8 @@ pub(crate) async fn probe_scanner_activity(storeapi: &ECStore, distributed: bool
data_movement_active: peer_data_movement_active,
dirty_usage_generation: peer_dirty_usage_generation,
dirty_usage_pending: peer_dirty_usage_pending,
movement_generation: peer_movement_generation,
publication_blocked: peer_publication_blocked,
},
)
.is_some()
+179 -2
View File
@@ -23,7 +23,7 @@ use crate::{
};
use std::collections::{HashMap, HashSet};
use std::io::Cursor;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
use std::task::Poll;
use temp_env::{with_var, with_var_unset};
use tokio::io::AsyncReadExt;
@@ -2888,6 +2888,47 @@ async fn test_usage_route_barrier_precedes_durable_reconciliation() {
assert_eq!(store.put_counts.lock().await.get(&key), None);
}
#[tokio::test]
async fn coordinator_does_not_put_after_remote_generation_flip() {
let store = Arc::new(MemoryConfigStore::default());
let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str());
let (sender, receiver) = mpsc::channel(1);
sender
.send(complete_usage_with_bucket_count(
Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)),
1,
))
.await
.expect("usage snapshot should enqueue");
drop(sender);
let route_store = store.clone();
let outcome = store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe_for_publication_epoch(
CancellationToken::new(),
store.clone(),
receiver,
None,
Some(DataUsagePersistBaseline {
data: None,
revision: DataUsageCacheRevision::Missing,
}),
ScannerPublicationFence::new(Some(0), None, None),
move || {
let route_store = route_store.clone();
async move {
// Model the remote lease holder flipping its movement generation
// after the activity probe but before the coordinator's PUT.
route_store.publication_admission_blocked.store(true, Ordering::Release);
false
}
},
)
.await;
assert_eq!(outcome, DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement));
assert_eq!(store.put_counts.lock().await.get(&key), None);
}
#[tokio::test]
async fn test_deferred_usage_save_keeps_last_real_save_metric() {
let metrics = global_metrics();
@@ -4974,6 +5015,41 @@ async fn test_wait_for_next_scanner_cycle_stops_after_leader_lock_loss() {
assert_eq!(reason, ScannerCycleWakeReason::LeaderLockLost);
}
#[tokio::test]
async fn movement_generation_wakes_deferred_wait_without_dirty_bucket() {
let ctx = CancellationToken::new();
let movement_generation = Arc::new(AtomicU64::new(7));
let movement_changed = Arc::new(Notify::new());
let next_generation = Arc::clone(&movement_generation);
let next_changed = Arc::clone(&movement_changed);
tokio::spawn(async move {
tokio::task::yield_now().await;
next_generation.store(8, Ordering::Release);
next_changed.notify_waiters();
});
let movement = ScannerMovementWaitContext {
movement_generation_seen: Some(7),
movement_changed,
current_movement_generation: move || movement_generation.load(Ordering::Acquire),
is_lock_lost: || false,
};
let reason = wait_for_next_scanner_cycle_with_movement(
&ctx,
Duration::from_secs(60),
ScannerCycleObservedGenerations {
dirty_usage: None,
runtime_config: crate::runtime_config::scanner_runtime_config_generation(),
maintenance: crate::scanner_io::scanner_maintenance_generation(),
defer_cluster_activity: false,
},
&movement,
)
.await;
assert_eq!(reason, ScannerCycleWakeReason::MovementGeneration);
}
fn scanner_node_activity(epoch: &str, namespace_generation: u64, maintenance_generation: u64) -> ScannerNodeActivity {
ScannerNodeActivity {
instance_id: epoch.to_string(),
@@ -4984,6 +5060,8 @@ fn scanner_node_activity(epoch: &str, namespace_generation: u64, maintenance_gen
data_movement_active: false,
dirty_usage_generation: 5,
dirty_usage_pending: false,
movement_generation: 9,
publication_blocked: false,
}
}
@@ -5024,6 +5102,7 @@ fn scanner_activity_snapshot_fences_data_movement() {
let mut moving = idle.clone();
moving.get_mut("node-2").expect("node should exist").data_movement_active = true;
assert!(!scanner_activity_allows_usage_publication(&BTreeMap::new()));
assert!(scanner_activity_allows_usage_publication(&idle));
assert!(!scanner_activity_allows_usage_publication(&moving));
assert_ne!(scanner_activity_snapshot_digest(&idle), scanner_activity_snapshot_digest(&moving));
@@ -5107,7 +5186,7 @@ fn scanner_activity_observation_requires_a_complete_baseline() {
let restarted = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-b", 8, 0))]);
let (observation, error) = apply_scanner_activity_probe_result(&mut seen, Ok(restarted));
assert_eq!(observation, ScannerActivityObservation::Changed);
assert_eq!(observation, ScannerActivityObservation::RemoteRestarted);
assert!(error.is_none());
let (observation, error) =
@@ -5142,6 +5221,36 @@ fn remote_maintenance_change_is_distinct_from_namespace_activity() {
);
}
#[test]
fn remote_movement_generation_change_is_distinct_from_cluster_activity() {
let previous = BTreeMap::from([("node-2".to_string(), scanner_node_activity("remote", 7, 3))]);
let movement_changed = BTreeMap::from([(
"node-2".to_string(),
ScannerNodeActivity {
movement_generation: 10,
..scanner_node_activity("remote", 7, 3)
},
)]);
assert_eq!(
compare_scanner_activity(&previous, &movement_changed),
ScannerActivityObservation::MovementChanged
);
assert!(scanner_activity_observed_work(ScannerActivityObservation::MovementChanged));
}
#[test]
fn remote_restart_is_distinct_from_deferred_cluster_activity() {
let previous = BTreeMap::from([("node-2".to_string(), scanner_node_activity("remote-a", 7, 3))]);
let restarted = BTreeMap::from([("node-2".to_string(), scanner_node_activity("remote-b", 7, 3))]);
assert_eq!(
compare_scanner_activity(&previous, &restarted),
ScannerActivityObservation::RemoteRestarted
);
assert!(scanner_activity_observed_work(ScannerActivityObservation::RemoteRestarted));
}
#[test]
fn local_maintenance_wakeup_releases_a_remote_maintenance_block() {
let blocked = scanner_activity_backoff_blocked_after_wake(false, ScannerCycleWakeReason::ClusterMaintenance);
@@ -5231,6 +5340,74 @@ async fn superseded_retry_wait_defers_dirty_cluster_activity_until_timer() {
assert_eq!(seen, Some(changed));
}
#[tokio::test(start_paused = true)]
async fn superseded_retry_wait_wakes_for_remote_movement_generation() {
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
let ctx = CancellationToken::new();
let mut seen = Some(BTreeMap::from([("node-2".to_string(), scanner_node_activity("remote", 7, 3))]));
let changed = BTreeMap::from([(
"node-2".to_string(),
ScannerNodeActivity {
movement_generation: 10,
..scanner_node_activity("remote", 7, 3)
},
)]);
let reason = wait_for_next_scanner_cycle_with_activity(
&ctx,
Duration::from_secs(120),
Some(Duration::from_secs(60)),
&mut seen,
ScannerCycleObservedGenerations {
dirty_usage: None,
runtime_config: crate::runtime_config::scanner_runtime_config_generation(),
maintenance: crate::scanner_io::scanner_maintenance_generation(),
defer_cluster_activity: true,
},
|| false,
|| std::future::ready(Ok(changed.clone())),
)
.await;
assert_eq!(reason, ScannerCycleWakeReason::ClusterActivity);
assert_eq!(seen, Some(changed));
}
#[tokio::test(start_paused = true)]
async fn superseded_retry_wait_wakes_when_remote_restart_clears_movement_state() {
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
let ctx = CancellationToken::new();
let blocked = BTreeMap::from([(
"node-2".to_string(),
ScannerNodeActivity {
data_movement_active: true,
publication_blocked: true,
..scanner_node_activity("remote-a", 7, 3)
},
)]);
let restarted = BTreeMap::from([("node-2".to_string(), scanner_node_activity("remote-b", 7, 3))]);
let mut seen = Some(blocked);
let reason = wait_for_next_scanner_cycle_with_activity(
&ctx,
Duration::from_secs(120),
Some(Duration::from_secs(60)),
&mut seen,
ScannerCycleObservedGenerations {
dirty_usage: None,
runtime_config: crate::runtime_config::scanner_runtime_config_generation(),
maintenance: crate::scanner_io::scanner_maintenance_generation(),
defer_cluster_activity: true,
},
|| false,
|| std::future::ready(Ok(restarted.clone())),
)
.await;
assert_eq!(reason, ScannerCycleWakeReason::ClusterActivity);
assert_eq!(seen, Some(restarted));
}
#[tokio::test(start_paused = true)]
async fn distributed_clean_idle_wait_blocks_backoff_for_unpropagated_maintenance() {
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
+117 -14
View File
@@ -13,6 +13,7 @@
// limitations under the License.
/// Data-usage snapshot persistence: CAS store pipeline, epoch baselines, and observed-snapshot cleanup.
use super::*;
use std::collections::HashMap;
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub(super) enum DataUsagePersistOutcome {
@@ -29,12 +30,40 @@ pub(super) enum DataUsagePersistOutcome {
Failed,
}
fn remote_lease_expired(deadline: Option<std::time::Instant>) -> bool {
deadline.is_some_and(|deadline| std::time::Instant::now() >= deadline)
}
#[derive(Clone, Debug)]
pub(super) struct DataUsagePersistBaseline {
pub(super) data: Option<Bytes>,
pub(super) revision: DataUsageCacheRevision,
}
/// Short-lived publication inputs captured for one usage persistence attempt.
/// Keeping the movement epoch, lease deadline, and target fence together makes
/// it explicit that they are one proof rather than independent options.
#[derive(Clone, Debug, Default)]
pub(super) struct ScannerPublicationFence {
pub(super) expected_publication_epoch: Option<u64>,
pub(super) remote_lease_deadline: Option<std::time::Instant>,
pub(super) scanner_publication_lease_fence: Option<String>,
}
impl ScannerPublicationFence {
pub(super) fn new(
expected_publication_epoch: Option<u64>,
remote_lease_deadline: Option<std::time::Instant>,
scanner_publication_lease_fence: Option<String>,
) -> Self {
Self {
expected_publication_epoch,
remote_lease_deadline,
scanner_publication_lease_fence,
}
}
}
#[derive(Debug)]
pub(super) enum DataUsagePersistTaskResult {
Completed(DataUsagePersistOutcome),
@@ -129,7 +158,7 @@ where
receiver,
leader_epoch,
initial_baseline,
None,
ScannerPublicationFence::default(),
route_probe,
)
.await
@@ -141,16 +170,49 @@ pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_basel
>(
ctx: CancellationToken,
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
mut receiver: mpsc::Receiver<DataUsageInfo>,
receiver: mpsc::Receiver<DataUsageInfo>,
leader_epoch: Option<u64>,
initial_baseline: Option<DataUsagePersistBaseline>,
expected_publication_epoch: Option<u64>,
publication_fence: ScannerPublicationFence,
route_probe: F,
) -> DataUsagePersistOutcome
where
F: Fn() -> Fut + Send + Sync,
Fut: Future<Output = bool> + Send,
{
store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe_for_publication_epoch_and_lease_fence(
ctx,
storeapi,
receiver,
leader_epoch,
initial_baseline,
publication_fence,
route_probe,
)
.await
}
pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe_for_publication_epoch_and_lease_fence<
F,
Fut,
>(
ctx: CancellationToken,
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
mut receiver: mpsc::Receiver<DataUsageInfo>,
leader_epoch: Option<u64>,
initial_baseline: Option<DataUsagePersistBaseline>,
publication_fence: ScannerPublicationFence,
route_probe: F,
) -> DataUsagePersistOutcome
where
F: Fn() -> Fut + Send + Sync,
Fut: Future<Output = bool> + Send,
{
let ScannerPublicationFence {
expected_publication_epoch,
remote_lease_deadline,
scanner_publication_lease_fence,
} = publication_fence;
let mut outcome = DataUsagePersistOutcome::NoUpdate;
let mut next_baseline = initial_baseline;
@@ -162,6 +224,10 @@ where
if let Some(leader_epoch) = leader_epoch {
data_usage_info.scanner_epoch = Some(leader_epoch);
}
if remote_lease_expired(remote_lease_deadline) {
outcome = DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable);
break 'updates;
}
if let Some(expected_epoch) = expected_publication_epoch
&& scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch)
.await
@@ -430,6 +496,9 @@ where
);
break DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
}
if remote_lease_expired(remote_lease_deadline) {
break DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable);
}
let done_save = Metrics::time(Metric::SaveUsage);
let save_result = {
@@ -439,12 +508,17 @@ where
done_save();
break DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
};
save_config_shared_with_preconditions(
if remote_lease_expired(remote_lease_deadline) {
done_save();
break DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable);
}
save_config_shared_with_preconditions_and_lease_fence(
storeapi.clone(),
target_path,
data.clone(),
sha256hex.clone(),
revision.preconditions(),
scanner_publication_lease_fence.as_deref(),
)
.await
};
@@ -538,10 +612,12 @@ where
if observational {
invalidate_admin_data_usage_snapshot_cache().await;
} else {
let cleanup_ok = cleanup_observed_data_usage_snapshot_for_epoch(
let cleanup_ok = cleanup_observed_data_usage_snapshot_for_epoch_and_lease(
storeapi.clone(),
&data_usage_info,
expected_publication_epoch,
remote_lease_deadline,
scanner_publication_lease_fence.as_deref(),
)
.await;
if expected_publication_epoch.is_some() && !cleanup_ok {
@@ -559,10 +635,12 @@ where
if observational {
invalidate_admin_data_usage_snapshot_cache().await;
} else {
let cleanup_ok = cleanup_observed_data_usage_snapshot_for_epoch(
let cleanup_ok = cleanup_observed_data_usage_snapshot_for_epoch_and_lease(
storeapi.clone(),
&data_usage_info,
expected_publication_epoch,
remote_lease_deadline,
scanner_publication_lease_fence.as_deref(),
)
.await;
if expected_publication_epoch.is_some() && !cleanup_ok {
@@ -599,10 +677,12 @@ where
if observational {
invalidate_admin_data_usage_snapshot_cache().await;
} else {
let cleanup_ok = cleanup_observed_data_usage_snapshot_for_epoch(
let cleanup_ok = cleanup_observed_data_usage_snapshot_for_epoch_and_lease(
storeapi.clone(),
&data_usage_info,
expected_publication_epoch,
remote_lease_deadline,
scanner_publication_lease_fence.as_deref(),
)
.await;
if expected_publication_epoch.is_some() && !cleanup_ok {
@@ -620,8 +700,14 @@ where
if backup_due {
let done_save = Metrics::time(Metric::SaveUsage);
let backup_result =
sync_data_usage_backup_from_primary_for_epoch(&ctx, storeapi.clone(), expected_publication_epoch).await;
let backup_result = sync_data_usage_backup_from_primary_for_epoch_and_lease_and_fence(
&ctx,
storeapi.clone(),
expected_publication_epoch,
remote_lease_deadline,
scanner_publication_lease_fence.as_deref(),
)
.await;
done_save();
if let Err(e) = backup_result {
warn!(
@@ -647,11 +733,16 @@ where
outcome
}
pub(super) async fn cleanup_observed_data_usage_snapshot_for_epoch(
async fn cleanup_observed_data_usage_snapshot_for_epoch_and_lease(
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
authoritative: &DataUsageInfo,
expected_publication_epoch: Option<u64>,
remote_lease_deadline: Option<std::time::Instant>,
scanner_publication_lease_fence: Option<&str>,
) -> bool {
if remote_lease_expired(remote_lease_deadline) {
return false;
}
let read_epoch = match expected_publication_epoch {
Some(expected_epoch) => {
if scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch)
@@ -667,10 +758,11 @@ pub(super) async fn cleanup_observed_data_usage_snapshot_for_epoch(
None => return false,
},
};
if expected_publication_epoch.is_some()
&& scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch)
.await
.is_none()
if remote_lease_expired(remote_lease_deadline)
|| expected_publication_epoch.is_some()
&& scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch)
.await
.is_none()
{
return false;
}
@@ -711,6 +803,9 @@ pub(super) async fn cleanup_observed_data_usage_snapshot_for_epoch(
if observed_data_usage_is_newer(&observed, authoritative) {
return true;
}
if remote_lease_expired(remote_lease_deadline) {
return false;
}
let result = delete_config_with_publication_admission_for_epoch(
storeapi,
@@ -720,6 +815,14 @@ pub(super) async fn cleanup_observed_data_usage_snapshot_for_epoch(
delete_prefix: true,
delete_prefix_object: true,
http_preconditions: Some(revision.preconditions()),
user_defined: scanner_publication_lease_fence
.map(|fence| {
HashMap::from([(
crate::storage_api::owner::SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY.to_string(),
fence.to_string(),
)])
})
.unwrap_or_default(),
..Default::default()
},
read_epoch,
+28 -5
View File
@@ -239,6 +239,13 @@ fn classify_nsscanner_cycle(
dirty_usage_status: DirtyUsageSnapshotStatus,
activity_status: ScannerCycleActivityStatus,
) -> ScannerCycleStatus {
// The post-scan activity proof is required regardless of why the scan was
// incomplete. Returning Incomplete first would apply the long ordinary
// retry/backoff path to an unverifiable publication and could acknowledge
// a cycle without a movement-generation proof.
if activity_status == ScannerCycleActivityStatus::Unverified {
return ScannerCycleStatus::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable);
}
if budget_elapsed
|| cancelled
|| !matches!(bucket_scan_status, ScannerBucketScanStatus::Complete)
@@ -252,7 +259,6 @@ fn classify_nsscanner_cycle(
match (activity_status, dirty_usage_status) {
(ScannerCycleActivityStatus::Unchanged, DirtyUsageSnapshotStatus::Current) => ScannerCycleStatus::Complete,
(ScannerCycleActivityStatus::Unverified, _) => ScannerCycleStatus::Incomplete,
_ => ScannerCycleStatus::Superseded,
}
}
@@ -307,10 +313,16 @@ async fn scanner_cycle_activity_status(
store: &ECStore,
distributed: bool,
before: &crate::scanner::ScannerActivitySnapshot,
) -> ScannerCycleActivityStatus {
) -> (ScannerCycleActivityStatus, Vec<(String, String, u64)>) {
match crate::scanner::probe_scanner_activity(store, distributed).await {
Ok(after) if after == *before => ScannerCycleActivityStatus::Unchanged,
Ok(_) => ScannerCycleActivityStatus::Changed,
Ok(after) => {
let status = if after == *before {
ScannerCycleActivityStatus::Unchanged
} else {
ScannerCycleActivityStatus::Changed
};
(status, crate::scanner::scanner_activity_publication_lease_targets(&after))
}
Err(err) => {
warn!(
target: "rustfs::scanner::io",
@@ -321,7 +333,7 @@ async fn scanner_cycle_activity_status(
error = %err,
"Scanner cycle activity verification failed"
);
ScannerCycleActivityStatus::Unverified
(ScannerCycleActivityStatus::Unverified, Vec::new())
}
}
}
@@ -599,6 +611,7 @@ pub(crate) struct ScannerCycleResult {
publication_epoch: Option<u64>,
dirty_usage_clear: Option<DirtyUsageBuckets>,
remote_dirty_usage_acknowledgements: Vec<crate::scanner::ScannerDirtyUsageAcknowledgement>,
remote_publication_lease_targets: Vec<(String, String, u64)>,
failed_dirty_usage: bool,
pending_maintenance_work: bool,
required_cycle_floor: Option<u64>,
@@ -611,6 +624,7 @@ impl ScannerCycleResult {
publication_epoch: None,
dirty_usage_clear,
remote_dirty_usage_acknowledgements: Vec::new(),
remote_publication_lease_targets: Vec::new(),
failed_dirty_usage: false,
pending_maintenance_work: false,
required_cycle_floor: None,
@@ -649,6 +663,15 @@ impl ScannerCycleResult {
self
}
pub(crate) fn with_remote_publication_lease_targets(mut self, targets: Vec<(String, String, u64)>) -> Self {
self.remote_publication_lease_targets = targets;
self
}
pub(crate) fn remote_publication_lease_targets(&self) -> &[(String, String, u64)] {
&self.remote_publication_lease_targets
}
pub(crate) fn acknowledge_durable_usage(self) -> Vec<crate::scanner::ScannerDirtyUsageAcknowledgement> {
if let Some(snapshot) = self.dirty_usage_clear {
clear_dirty_usage_buckets(&snapshot);
+6 -2
View File
@@ -151,7 +151,8 @@ impl ScannerIOCycle for ECStore {
ScannerCycleResult::new(ScannerCycleStatus::Incomplete, None).with_publication_epoch(publication_epoch)
);
}
let activity_status = scanner_cycle_activity_status(self, distributed, &activity_before).await;
let (activity_status, remote_publication_lease_targets) =
scanner_cycle_activity_status(self, distributed, &activity_before).await;
let dirty_usage_status = dirty_usage_snapshot_status(&dirty_usage_snapshot);
let status = classify_nsscanner_cycle(
true,
@@ -187,6 +188,7 @@ impl ScannerIOCycle for ECStore {
};
return Ok(ScannerCycleResult::new(status, dirty_usage_clear)
.with_publication_epoch(publication_epoch)
.with_remote_publication_lease_targets(remote_publication_lease_targets)
.with_remote_dirty_usage_acknowledgements(remote_dirty_usage_acknowledgements));
}
@@ -400,7 +402,8 @@ impl ScannerIOCycle for ECStore {
let budget_elapsed = budget.budget_elapsed();
let dirty_usage_status = dirty_usage_snapshot_status(&dirty_usage_snapshot);
let dirty_usage_current = dirty_usage_status == DirtyUsageSnapshotStatus::Current;
let activity_status = scanner_cycle_activity_status(self, distributed, &activity_before).await;
let (activity_status, remote_publication_lease_targets) =
scanner_cycle_activity_status(self, distributed, &activity_before).await;
let all_bucket_names = all_buckets.iter().map(|bucket| bucket.name.clone()).collect::<Vec<_>>();
let completed_usage = completed_data_usage_info(
&results,
@@ -460,6 +463,7 @@ impl ScannerIOCycle for ECStore {
};
Ok(ScannerCycleResult::new(cycle_status, dirty_usage_clear)
.with_publication_epoch(publication_epoch)
.with_remote_publication_lease_targets(remote_publication_lease_targets)
.with_remote_dirty_usage_acknowledgements(remote_dirty_usage_acknowledgements)
.with_failed_dirty_usage(!failed_buckets.is_empty())
.with_pending_maintenance_work(pending_maintenance_work)
+50 -6
View File
@@ -766,7 +766,7 @@ fn scanner_cycle_status_requires_a_clean_complete_snapshot() {
DirtyUsageSnapshotStatus::Current,
ScannerCycleActivityStatus::Unverified,
),
ScannerCycleStatus::Incomplete
ScannerCycleStatus::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable)
);
for status in [
@@ -815,6 +815,34 @@ fn scanner_cycle_status_requires_a_clean_complete_snapshot() {
}
}
#[test]
fn unverified_activity_defers_partial_and_floor_cycles() {
let expected = ScannerCycleStatus::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable);
assert_eq!(
classify_nsscanner_cycle(
true,
false,
false,
ScannerBucketScanStatus::Partial,
DirtyUsageSnapshotStatus::Current,
ScannerCycleActivityStatus::Unverified,
),
expected
);
assert_eq!(
classify_nsscanner_cycle(
false,
false,
false,
ScannerBucketScanStatus::Complete,
DirtyUsageSnapshotStatus::Current,
ScannerCycleActivityStatus::Unverified,
),
expected
);
}
#[tokio::test]
async fn structurally_complete_superseded_cycles_publish_without_claiming_convergence() {
let (updates, mut receiver) = mpsc::channel(2);
@@ -834,6 +862,15 @@ async fn structurally_complete_superseded_cycles_publish_without_claiming_conver
.await
.expect("incomplete snapshot suppression should succeed")
);
assert!(
!publish_usage_snapshot(
&updates,
ScannerCycleStatus::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable),
DataUsageInfo::default(),
)
.await
.expect("unverified activity suppression should succeed")
);
assert_eq!(
receiver
@@ -855,11 +892,7 @@ async fn structurally_complete_superseded_cycles_publish_without_claiming_conver
#[test]
fn scanner_cycle_fails_closed_for_namespace_disappearance() {
for activity_status in [
ScannerCycleActivityStatus::Changed,
ScannerCycleActivityStatus::Unchanged,
ScannerCycleActivityStatus::Unverified,
] {
for activity_status in [ScannerCycleActivityStatus::Changed, ScannerCycleActivityStatus::Unchanged] {
assert_eq!(
classify_nsscanner_cycle(
false,
@@ -872,6 +905,17 @@ fn scanner_cycle_fails_closed_for_namespace_disappearance() {
ScannerCycleStatus::Incomplete
);
}
assert_eq!(
classify_nsscanner_cycle(
false,
false,
false,
ScannerBucketScanStatus::NamespaceNotFound,
DirtyUsageSnapshotStatus::Changed,
ScannerCycleActivityStatus::Unverified,
),
ScannerCycleStatus::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable)
);
assert_eq!(
classify_nsscanner_cycle(
true,
+8 -6
View File
@@ -85,6 +85,7 @@ pub(crate) use rustfs_ecstore::api::event::{EventArgs as EcstoreEventArgs, send_
pub(crate) use rustfs_ecstore::api::layout::{
EndpointServerPools as EcstoreEndpointServerPools, Endpoints as EcstoreEndpoints, PoolEndpoints as EcstorePoolEndpoints,
};
pub(crate) use rustfs_ecstore::api::object::SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY;
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::rebalance::{
RebalStatus as EcstoreRebalStatus, RebalanceInfo as EcstoreRebalanceInfo, RebalanceMeta as EcstoreRebalanceMeta,
@@ -98,9 +99,9 @@ pub(crate) use rustfs_ecstore::api::runtime::{
setup_is_erasure_sd as ecstore_is_erasure_sd,
};
pub(crate) use rustfs_ecstore::api::set_disk::SetDisks as EcstoreSetDisks;
pub(crate) use rustfs_ecstore::api::storage::ECStore as EcstoreStore;
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::storage::init_local_disks_with_instance_ctx as ecstore_init_local_disks_with_instance_ctx;
pub(crate) use rustfs_ecstore::api::storage::{ECStore as EcstoreStore, SCANNER_PUBLICATION_LEASE_TTL_MS};
use rustfs_storage_api as storage_contracts;
pub(crate) mod owner {
@@ -115,10 +116,11 @@ pub(crate) mod owner {
EcstoreDiskLocation, EcstoreDiskResult, EcstoreErrorType, EcstoreEvaluator, EcstoreEvent, EcstoreEventArgs,
EcstoreLcEventSrc, EcstoreLifecycle, EcstoreListPathRawOptions, EcstoreNsScannerOpenRequest, EcstoreObjectOpts,
EcstoreReplicationConfigurationExt, EcstoreReplicationScannerBridge, EcstoreResultType, EcstoreScanGuard,
EcstoreSetDisks, EcstoreStorageError, EcstoreStore, EcstoreVersioningApi, ScannerReplicationHealObject,
ScannerReplicationHealResult, ScannerReplicationQueueAdmission, ecstore_apply_expiry_rule, ecstore_apply_transition_rule,
ecstore_expiry_state_handle, ecstore_get_global_tier_config_mgr, ecstore_get_lifecycle_config,
ecstore_get_object_lock_config, ecstore_get_replication_config, ecstore_invalidate_admin_data_usage_snapshot_cache,
EcstoreSetDisks, EcstoreStorageError, EcstoreStore, EcstoreVersioningApi, SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY,
SCANNER_PUBLICATION_LEASE_TTL_MS, ScannerReplicationHealObject, ScannerReplicationHealResult,
ScannerReplicationQueueAdmission, ecstore_apply_expiry_rule, ecstore_apply_transition_rule, ecstore_expiry_state_handle,
ecstore_get_global_tier_config_mgr, ecstore_get_lifecycle_config, ecstore_get_object_lock_config,
ecstore_get_replication_config, ecstore_invalidate_admin_data_usage_snapshot_cache,
ecstore_invalidate_data_usage_snapshot_cache, ecstore_is_erasure, ecstore_is_erasure_sd,
ecstore_is_reserved_or_invalid_bucket, ecstore_list_path_raw, ecstore_object_opts_from_object_info,
ecstore_path2_bucket_object, ecstore_path2_bucket_object_with_base_path, ecstore_read_config,
@@ -274,13 +276,13 @@ impl From<EcstoreReplicationHealQueueResult> for ScannerReplicationHealResult {
}
pub(crate) mod scan {
pub use super::storage_contracts::SCANNER_ACTIVITY_PROTOCOL_VERSION;
pub(crate) use super::storage_contracts::{
BucketOperations, BucketOptions, NamespaceLocking, SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION,
SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION,
};
#[cfg(test)]
pub(crate) use super::storage_contracts::{DeleteBucketOptions, MakeBucketOptions, ObjectIO};
pub use super::storage_contracts::{SCANNER_ACTIVITY_PROTOCOL_VERSION, SCANNER_ACTIVITY_V6_PROTOCOL_VERSION};
}
pub(crate) mod scanner_io {
+6 -1
View File
@@ -50,7 +50,12 @@ pub const NS_SCANNER_TIER_REGISTRY_GENERATION_QUERY: &str = "ns_scanner_tier_reg
pub const NS_SCANNER_PROTOCOL_VERSION: u16 = 3;
pub const SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION: u32 = 0;
pub const SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION: u32 = 5;
pub const SCANNER_ACTIVITY_PROTOCOL_VERSION: u32 = 6;
/// Protocol v6 carries the activity fields that predate the storage-owned
/// movement generation. It remains readable during the rolling upgrade, but
/// a scanner must not use it as a publication proof because terminal movement
/// state is not authenticated by that version.
pub const SCANNER_ACTIVITY_V6_PROTOCOL_VERSION: u32 = 6;
pub const SCANNER_ACTIVITY_PROTOCOL_VERSION: u32 = 7;
#[derive(Debug, serde::Deserialize, serde::Serialize)]
#[serde(deny_unknown_fields)]
+1 -1
View File
@@ -25,7 +25,7 @@ for later deletion.
- `rustfs-5416-kubernetes-alias-dns` Kubernetes endpoint identity fallback: deployments created before explicit local endpoint identity may use resolvable aliases that do not match the Pod hostname. An implicit auto-mode zero match retains legacy DNS locality with a bounded deadline, while ambiguous matches and invalid explicit anchors still fail closed. Remove the fallback after every supported direct-upgrade chart and deployment manifest provides a canonical RUSTFS_LOCAL_ENDPOINT_HOST for domain-based distributed topologies.
- `rustfs-5416-zero-retry-delay` startup retry-delay validation: releases before bounded topology convergence accept RUSTFS_STARTUP_TOPOLOGY_RETRY_MAX_DELAY values of 0 or 0ms. New servers replace those values with the safe nonzero default so a direct upgrade neither fails startup nor enters a busy loop. Reject zero after the minimum supported direct-upgrade release validates or rewrites this setting before rollout.
- `scanner-usage-v2` persisted scanner usage migration: pre-v2 scanners write `.usage.json`, so upgraded clusters read that primary/backup pair only while `.usage.v2.json` is absent and continue removing deleted buckets from legacy copies that still exist. The additive usage_snapshot_complete field in `.usage.v2.json` must remain optional while mixed-version clusters are supported; a missing field means the snapshot is not authoritative. The legacy read also feeds the degraded quota-admission baseline (issue #5716): while no authoritative usage exists, quota checks admit against the pre-discard sizes of the last loaded snapshot, including a legacy one. Remove the legacy object fallback and cleanup only after every supported direct-upgrade source writes `.usage.v2.json`; the baseline then feeds from incomplete v2 snapshots alone.
- `ns-scanner-rpc-v3` namespace scanner capability and activity handshake: old peers and legacy internode transports lack the authenticated startup-epoch handshake. The oldest peers send an empty activity request and receive a field-empty protocol-0 response. Protocol v4 binds the challenge and response topology but cannot authenticate distributed dirty-usage state. Protocol v5 binds the request version, acknowledgement target and generation, and the response dirty-usage state, but predates set-scoped scanner cache locks. Current protocol v6 additionally fences scanner cache lock-domain changes, so distributed scanner cycles publish usage only after every peer reports protocol v6 state. Servers retain protocol-0 and protocol-v4 codecs for rolling upgrades, while protocol-v5 peers are treated as previous-version peers that cannot safely participate in the new cache lock domain. Scanner selection treats HTTP 404/405/426 and the legacy MethodNotAllowed default as an explicit lack of remote scanner v3 support and assigns those disks to coordinator-driven workers; transient capability failures remain incomplete and do not activate the fallback. Remove the coordinator fallback after the minimum supported RustFS peer version implements namespace scanner protocol v3, remove protocol-0 activity requests and responses after every supported peer implements authenticated scanner activity protocol v4, remove the protocol-v4 activity codec after every supported peer implements protocol v5, and remove protocol-v5 previous-version rejection after every supported peer implements protocol v6; future protocol revisions must keep the same dual-version server/codec window before changing the advertised version.
- `ns-scanner-rpc-v3` namespace scanner capability and activity handshake: old peers and legacy internode transports lack the authenticated startup-epoch handshake. The oldest peers send an empty activity request and receive a field-empty protocol-0 response. Protocol v4 binds the challenge and response topology but cannot authenticate distributed dirty-usage state. Protocol v5 binds the request version, acknowledgement target and generation, and the response dirty-usage state, but predates set-scoped scanner cache locks. Protocol v6 additionally fences scanner cache lock-domain changes. Current protocol v7 binds the storage-owned movement generation and publication-blocked state, so distributed scanner cycles publish usage only after every peer reports a complete v7 activity proof; v6 responses remain readable but are treated as unverified for publication. Servers retain protocol-0, protocol-v4, and protocol-v6 codecs alongside the current v7 codec for rolling upgrades, while protocol-v5 peers are treated as previous-version peers that cannot safely participate in the new cache lock domain. Scanner selection treats HTTP 404/405/426 and the legacy MethodNotAllowed default as an explicit lack of remote scanner v3 support and assigns those disks to coordinator-driven workers; transient capability failures remain incomplete and do not activate the fallback. Remove the coordinator fallback after the minimum supported RustFS peer version implements namespace scanner protocol v3, remove protocol-0 activity requests and responses after every supported peer implements authenticated scanner activity protocol v4, remove the protocol-v4 activity codec after every supported peer implements protocol v5, and remove protocol-v5 previous-version rejection after every supported peer implements protocol v6; future protocol revisions must keep the same dual-version server/codec window before changing the advertised version.
- `#4648` walk-dir stream completion capability: old clients can append fallback output to an already-used metacache writer after a terminal body error, so servers emit terminal walk errors only to clients that sign the `walk_dir_stream_completion=error-v1` query capability and its request-body digest. Remove the legacy clean-EOF path after the minimum supported RustFS peer version always advertises this capability.
- `heal-rpc-auth-v2` internode gRPC authentication: servers temporarily accept legacy prefix signatures so old peers remain available during rolling upgrades. Remove the legacy fallback after the minimum supported RustFS peer version sends v2 authentication on every internode gRPC request.
- `disk-mutation-body-digest` internode mutating disk RPCs: servers temporarily accept mutating disk RPCs (RenameData, DeleteVersion, DeleteVersions, WriteMetadata, UpdateMetadata, WriteAll, Delete, DeletePaths, RenameFile, RenamePart, DeleteVolume, MakeVolume, MakeVolumes) that carry no signature-bound canonical body digest, so peers from releases that predate body-digest signing remain available during rolling upgrades. Accepted digestless mutations increment the internode body-digest fallback counter; that counter must read zero fleet-wide across a release window before RUSTFS_INTERNODE_RPC_BODY_DIGEST_STRICT is enabled. Because body-bound requests now consume replay-cache nonces on the receiver, deploy the raised RUSTFS_INTERNODE_RPC_REPLAY_CACHE_CAPACITY default fleet-wide before enabling strict mode, and watch the internode replay-cache overflow counter for undersized capacity during the rollout. Remove the digestless fallback after the minimum supported RustFS peer version body-binds every mutating disk RPC.
+230 -19
View File
@@ -24,9 +24,9 @@ use crate::storage::storage_api::rpc_consumer::node_service::STORAGE_CLASS_SUB_S
use crate::storage::storage_api::rpc_consumer::node_service::{CollectMetricsOpts, MetricType};
use crate::storage::storage_api::rpc_consumer::node_service::{
DiskStore, ECStore, Error, KMS_SIGNAL_SUBSYSTEM, LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS,
SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION, SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SERVICE_SIGNAL_REFRESH_CONFIG,
SERVICE_SIGNAL_RELOAD_DYNAMIC, StorageDiskRpcExt as _, StorageResult, all_local_disk_path, find_local_disk_by_ref,
reload_transition_tier_config,
SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION, SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SCANNER_ACTIVITY_V6_PROTOCOL_VERSION,
SCANNER_PUBLICATION_LEASE_TTL_MS, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, StorageDiskRpcExt as _,
StorageResult, all_local_disk_path, find_local_disk_by_ref, reload_transition_tier_config,
};
use crate::storage::storage_api::runtime_sources_consumer::{EndpointServerPools, runtime_sources};
use crate::storage::storage_api::{
@@ -237,9 +237,25 @@ fn scanner_activity_response(
response_proof: Bytes::new(),
dirty_usage_generation: dirty_usage.generation,
dirty_usage_pending: dirty_usage.pending,
movement_generation: None,
publication_blocked: None,
}
}
fn scanner_activity_response_v7(
namespace_generation: u64,
topology_digest: [u8; 32],
data_movement_active: bool,
dirty_usage: rustfs_scanner::ScannerDirtyUsageState,
movement_generation: u64,
publication_blocked: bool,
) -> ScannerActivityResponse {
let mut response = scanner_activity_response(namespace_generation, topology_digest, data_movement_active, dirty_usage);
response.movement_generation = Some(movement_generation);
response.publication_blocked = Some(publication_blocked);
response
}
fn previous_scanner_activity_response(
namespace_generation: u64,
topology_digest: [u8; 32],
@@ -255,6 +271,8 @@ fn previous_scanner_activity_response(
response_proof: Bytes::new(),
dirty_usage_generation: 0,
dirty_usage_pending: false,
movement_generation: None,
publication_blocked: None,
}
}
@@ -269,6 +287,29 @@ fn legacy_scanner_activity_response(namespace_generation: u64) -> ScannerActivit
response_proof: Bytes::new(),
dirty_usage_generation: 0,
dirty_usage_pending: false,
movement_generation: None,
publication_blocked: None,
}
}
fn v6_scanner_activity_response(
namespace_generation: u64,
topology_digest: [u8; 32],
data_movement_active: bool,
dirty_usage: rustfs_scanner::ScannerDirtyUsageState,
) -> ScannerActivityResponse {
ScannerActivityResponse {
instance_id: rustfs_scanner::scanner_activity_epoch().to_string(),
namespace_generation,
maintenance_generation: rustfs_scanner::scanner_maintenance_generation(),
protocol_version: SCANNER_ACTIVITY_V6_PROTOCOL_VERSION,
topology_digest: topology_digest.to_vec().into(),
data_movement_active,
response_proof: Bytes::new(),
dirty_usage_generation: dirty_usage.generation,
dirty_usage_pending: dirty_usage.pending,
movement_generation: None,
publication_blocked: None,
}
}
@@ -1826,7 +1867,7 @@ impl Node for NodeService {
return Err(Status::invalid_argument("scanner activity protocol v4 cannot acknowledge dirty usage"));
}
}
rustfs_scanner::SCANNER_ACTIVITY_PROTOCOL_VERSION => {
SCANNER_ACTIVITY_V6_PROTOCOL_VERSION | rustfs_scanner::SCANNER_ACTIVITY_PROTOCOL_VERSION => {
let canonical = rustfs_protos::canonical_scanner_activity_request_body(request.get_ref())
.map_err(|_| Status::invalid_argument("scanner activity request is too large to authenticate"))?;
verify_tonic_canonical_body_digest(&request, &canonical)
@@ -1873,17 +1914,25 @@ impl Node for NodeService {
}
let namespace_generation = store.scanner_namespace_mutation_generation();
let topology_digest = rustfs_scanner::scanner_topology_digest(store.as_ref());
let data_movement_active = store.scanner_data_movement_active().await;
let (data_movement_active, publication_blocked, movement_generation) = store.scanner_data_movement_activity().await;
let mut response = match request_protocol {
SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION | SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION => {
previous_scanner_activity_response(namespace_generation, topology_digest, data_movement_active)
}
rustfs_scanner::SCANNER_ACTIVITY_PROTOCOL_VERSION => scanner_activity_response(
SCANNER_ACTIVITY_V6_PROTOCOL_VERSION => v6_scanner_activity_response(
namespace_generation,
topology_digest,
data_movement_active,
rustfs_scanner::scanner_dirty_usage_state(),
),
rustfs_scanner::SCANNER_ACTIVITY_PROTOCOL_VERSION => scanner_activity_response_v7(
namespace_generation,
topology_digest,
data_movement_active,
rustfs_scanner::scanner_dirty_usage_state(),
movement_generation,
publication_blocked || store.scanner_data_movement_generation_exhausted(),
),
version => {
return Err(Status::failed_precondition(format!(
"unsupported scanner activity request protocol {version}"
@@ -1894,9 +1943,12 @@ impl Node for NodeService {
SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION => {
rustfs_protos::canonical_scanner_activity_v4_response_body(&challenge, &response)
}
rustfs_scanner::SCANNER_ACTIVITY_PROTOCOL_VERSION => {
SCANNER_ACTIVITY_V6_PROTOCOL_VERSION => {
rustfs_protos::canonical_scanner_activity_response_body(&challenge, &response)
}
rustfs_scanner::SCANNER_ACTIVITY_PROTOCOL_VERSION => {
rustfs_protos::canonical_scanner_activity_v7_response_body(&challenge, &response)
}
version => {
return Err(Status::internal(format!(
"scanner activity response selected unsupported protocol {version}"
@@ -1910,6 +1962,133 @@ impl Node for NodeService {
Ok(Response::new(response))
}
async fn acquire_scanner_publication_lease(
&self,
request: Request<ScannerPublicationLeaseRequest>,
) -> Result<Response<ScannerPublicationLeaseResponse>, Status> {
let canonical = rustfs_protos::canonical_scanner_publication_lease_request_body(request.get_ref())
.map_err(|_| Status::invalid_argument("scanner publication lease request is too large to authenticate"))?;
verify_tonic_canonical_body_digest(&request, &canonical)
.map_err(|err| Status::permission_denied(format!("scanner publication lease authentication failed: {err}")))?;
if request.get_ref().challenge.len() != 16 {
return Err(Status::invalid_argument("scanner publication lease challenge must be 16 bytes"));
}
if request.get_ref().ttl_ms != SCANNER_PUBLICATION_LEASE_TTL_MS {
return Err(Status::invalid_argument("scanner publication lease TTL is unsupported"));
}
let session_id = rustfs_scanner::scanner_activity_epoch().to_string();
if request.get_ref().expected_session_id != session_id {
return Err(Status::failed_precondition("scanner publication lease session is stale"));
}
let validation_token = if request.get_ref().token.is_empty() {
None
} else {
Some(
Uuid::from_slice(request.get_ref().token.as_ref())
.map_err(|_| Status::invalid_argument("scanner publication lease token must be a UUID"))?,
)
};
let challenge = request.get_ref().challenge.clone();
let request = request.into_inner();
let store = self
.resolve_object_store()
.ok_or_else(|| Status::unavailable("storage layer is not initialized"))?;
if store.id.is_nil() {
return Err(Status::unavailable("storage owner identity is not initialized"));
}
let owner_id = store.id.to_string();
let result = match validation_token {
Some(token) => store
.validate_scanner_publication_lease(token, request.expected_movement_generation)
.await
.map(|()| (token, request.expected_movement_generation)),
None => {
store
.acquire_scanner_publication_lease(
request.expected_movement_generation,
Duration::from_millis(request.ttl_ms),
)
.await
}
};
let mut response = match result {
Ok((token, generation)) => ScannerPublicationLeaseResponse {
success: true,
token: token.as_bytes().to_vec().into(),
movement_generation: generation,
lease_ttl_ms: request.ttl_ms,
error: None,
response_proof: Bytes::new(),
owner_id: owner_id.clone(),
session_id: session_id.clone(),
},
Err(err) => ScannerPublicationLeaseResponse {
success: false,
token: Bytes::new(),
movement_generation: store.scanner_data_movement_generation(),
lease_ttl_ms: 0,
error: Some(rustfs_protos::proto_gen::node_service::Error {
code: 1,
error_info: err.to_string(),
}),
response_proof: Bytes::new(),
owner_id: owner_id.clone(),
session_id: session_id.clone(),
},
};
let response_body = rustfs_protos::canonical_scanner_publication_lease_response_body(&challenge, &response)
.map_err(|_| Status::internal("scanner publication lease response is too large to authenticate"))?;
response.response_proof = sign_tonic_rpc_response_proof(&response_body)
.map_err(|_| Status::unavailable("scanner publication lease response authentication is unavailable"))?
.into();
Ok(Response::new(response))
}
async fn release_scanner_publication_lease(
&self,
request: Request<ScannerPublicationLeaseReleaseRequest>,
) -> Result<Response<ScannerPublicationLeaseReleaseResponse>, Status> {
let canonical = rustfs_protos::canonical_scanner_publication_lease_release_request_body(request.get_ref())
.map_err(|_| Status::invalid_argument("scanner publication lease release request is too large to authenticate"))?;
verify_tonic_canonical_body_digest(&request, &canonical).map_err(|err| {
Status::permission_denied(format!("scanner publication lease release authentication failed: {err}"))
})?;
if request.get_ref().challenge.len() != 16 {
return Err(Status::invalid_argument("scanner publication lease challenge must be 16 bytes"));
}
let store = self
.resolve_object_store()
.ok_or_else(|| Status::unavailable("storage layer is not initialized"))?;
if store.id.is_nil() {
return Err(Status::unavailable("storage owner identity is not initialized"));
}
let owner_id = store.id.to_string();
let session_id = rustfs_scanner::scanner_activity_epoch().to_string();
if request.get_ref().owner_id != owner_id || request.get_ref().session_id != session_id {
return Err(Status::failed_precondition("scanner publication lease owner or session is stale"));
}
let token = Uuid::from_slice(request.get_ref().token.as_ref())
.map_err(|_| Status::invalid_argument("scanner publication lease token must be a UUID"))?;
let challenge = request.get_ref().challenge.clone();
let request = request.into_inner();
let released = store.release_scanner_publication_lease(token).await;
let mut response = ScannerPublicationLeaseReleaseResponse {
success: released,
error: (!released).then(|| rustfs_protos::proto_gen::node_service::Error {
code: 1,
error_info: "scanner publication lease is unknown or expired".to_string(),
}),
response_proof: Bytes::new(),
};
let response_body =
rustfs_protos::canonical_scanner_publication_lease_release_response_body(&challenge, &request, &response)
.map_err(|_| Status::internal("scanner publication lease response is too large to authenticate"))?;
response.response_proof = sign_tonic_rpc_response_proof(&response_body)
.map_err(|_| Status::unavailable("scanner publication lease response authentication is unavailable"))?
.into();
Ok(Response::new(response))
}
async fn background_heal_status(
&self,
request: Request<BackgroundHealStatusRequest>,
@@ -2204,12 +2383,13 @@ mod tests {
use super::{
CollectMetricsOpts, DiskStore, Error, HEAL_CONTROL_PAYLOAD_MAX_SIZE, KMS_SIGNAL_SUBSYSTEM, MetricType, Node as _,
NodeService, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION,
SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC,
STORAGE_CLASS_SUB_SYS, admit_heal_control_replay, background_rebalance_start_error_message,
execute_heal_control_envelope_with_manager, initialize_heal_topology_fingerprint,
initialize_heal_topology_fingerprint_with_probe, legacy_scanner_activity_response, make_heal_control_server,
make_heal_control_server_with_cache, make_server, make_server_for_context, make_tier_mutation_control_server_for_context,
previous_scanner_activity_response, remove_heal_control_replay, scanner_activity_response, stop_rebalance_response,
SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SCANNER_PUBLICATION_LEASE_TTL_MS, SERVICE_SIGNAL_REFRESH_CONFIG,
SERVICE_SIGNAL_RELOAD_DYNAMIC, STORAGE_CLASS_SUB_SYS, admit_heal_control_replay,
background_rebalance_start_error_message, execute_heal_control_envelope_with_manager,
initialize_heal_topology_fingerprint, initialize_heal_topology_fingerprint_with_probe, legacy_scanner_activity_response,
make_heal_control_server, make_heal_control_server_with_cache, make_server, make_server_for_context,
make_tier_mutation_control_server_for_context, previous_scanner_activity_response, remove_heal_control_replay,
scanner_activity_response_v7, stop_rebalance_response,
};
use crate::storage::rpc::node_service::heal::heal_topology_fingerprint;
use crate::storage::storage_api::rpc_consumer::node_service::{DiskError, HealBucketInfo};
@@ -2243,11 +2423,12 @@ mod tests {
LoadTransitionTierConfigRequest, LoadUserRequest, LocalStorageInfoRequest, MakeBucketRequest, MakeVolumeRequest,
MakeVolumesRequest, Mss, PingRequest, PreparePartTransactionRequest, ReadAllRequest, ReadAtRequest, ReadMultipleRequest,
ReadVersionRequest, ReadXlRequest, ReloadPoolMetaRequest, ReloadSiteReplicationConfigRequest, RenameDataRequest,
RenameFileRequest, RenamePartRequest, ScannerActivityRequest, ServerInfoRequest, SettlePartTransactionRequest,
SignalServiceRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, SnapshotLeaseRequest,
StartDecommissionRequest, StartProfilingRequest, StatVolumeRequest, StopRebalanceRequest, TierMutationPeerState,
TierMutationPrepareRequest, UpdateMetacacheListingRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest,
WriteMetadataRequest, WriteRequest,
RenameFileRequest, RenamePartRequest, ScannerActivityRequest, ScannerPublicationLeaseReleaseRequest,
ScannerPublicationLeaseRequest, ServerInfoRequest, SettlePartTransactionRequest, SignalServiceRequest,
SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, SnapshotLeaseRequest, StartDecommissionRequest,
StartProfilingRequest, StatVolumeRequest, StopRebalanceRequest, TierMutationPeerState, TierMutationPrepareRequest,
UpdateMetacacheListingRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteMetadataRequest,
WriteRequest,
heal_control_service_client::HealControlServiceClient,
heal_control_service_server::{HealControlService as _, HealControlServiceServer},
node_service_client::NodeServiceClient,
@@ -2621,6 +2802,7 @@ mod tests {
volume: "bucket".to_string(),
path: "object".to_string(),
options: options.to_string(),
scanner_publication_lease_token: Vec::new().into(),
}
}
@@ -2720,6 +2902,7 @@ mod tests {
dst_volume: "dst".into(),
dst_path: "dp".into(),
file_info_bin: vec![0x80].into(),
scanner_publication_lease_token: Vec::new().into(),
},
rustfs_protos::canonical_rename_data_request_body
);
@@ -2790,6 +2973,7 @@ mod tests {
volume: "v".into(),
path: "p".into(),
options: "{}".into(),
scanner_publication_lease_token: Vec::new().into(),
},
rustfs_protos::canonical_delete_request_body
);
@@ -3698,6 +3882,7 @@ mod tests {
volume: "test-volume".to_string(),
path: "test-path".to_string(),
options: "{}".to_string(),
scanner_publication_lease_token: Vec::new().into(),
});
let response = service.delete(request).await;
@@ -3717,6 +3902,7 @@ mod tests {
volume: "test-volume".to_string(),
path: "test-path".to_string(),
options: "invalid json".to_string(),
scanner_publication_lease_token: Vec::new().into(),
});
let response = service.delete(request).await;
@@ -3891,6 +4077,7 @@ mod tests {
dst_path: "dst-path".to_string(),
file_info: "{}".to_string(),
file_info_bin: Vec::new().into(),
scanner_publication_lease_token: Vec::new().into(),
});
let response = service.rename_data(request).await;
@@ -3913,6 +4100,7 @@ mod tests {
dst_path: "dst-path".to_string(),
file_info: "invalid json".to_string(),
file_info_bin: Vec::new().into(),
scanner_publication_lease_token: Vec::new().into(),
});
let response = service.rename_data(request).await;
@@ -5333,6 +5521,25 @@ mod tests {
acknowledge_dirty_usage_generation: 0,
}
);
assert_tampered!(
acquire_scanner_publication_lease,
ScannerPublicationLeaseRequest {
challenge: vec![7; 16].into(),
expected_movement_generation: 0,
ttl_ms: SCANNER_PUBLICATION_LEASE_TTL_MS,
expected_session_id: String::new(),
token: Bytes::new(),
}
);
assert_tampered!(
release_scanner_publication_lease,
ScannerPublicationLeaseReleaseRequest {
challenge: vec![7; 16].into(),
token: vec![1; 16].into(),
owner_id: String::new(),
session_id: String::new(),
}
);
assert_tampered!(reload_pool_meta, ReloadPoolMetaRequest::default());
assert_tampered!(stop_rebalance, StopRebalanceRequest::default());
assert_tampered!(load_rebalance_meta, LoadRebalanceMetaRequest::default());
@@ -5489,7 +5696,7 @@ mod tests {
#[test]
fn test_scanner_activity_response_uses_process_epoch_and_generations() {
let response = scanner_activity_response(
let response = scanner_activity_response_v7(
17,
[7; 32],
true,
@@ -5497,6 +5704,8 @@ mod tests {
generation: 11,
pending: true,
},
23,
true,
);
assert_eq!(response.instance_id, rustfs_scanner::scanner_activity_epoch());
@@ -5507,6 +5716,8 @@ mod tests {
assert!(response.data_movement_active);
assert_eq!(response.dirty_usage_generation, 11);
assert!(response.dirty_usage_pending);
assert_eq!(response.movement_generation, Some(23));
assert_eq!(response.publication_blocked, Some(true));
}
#[test]
@@ -36,6 +36,7 @@ use std::io::Cursor;
use std::time::Instant;
use tonic::{Request, Response, Status};
use tracing::debug;
use uuid::Uuid;
/// Initial capacity hint (bytes) for typical small msgpack requests and responses.
const MSGPACK_ENCODE_CAPACITY_HINT: usize = 512;
@@ -1188,6 +1189,16 @@ impl NodeService {
&self,
request: Request<RenameDataRequest>,
) -> Result<Response<RenameDataResponse>, Status> {
if !request.get_ref().scanner_publication_lease_token.is_empty() {
let has_body_digest = request
.metadata()
.get("x-rustfs-content-sha256")
.and_then(|value| value.to_str().ok())
.is_some_and(|value| value != "UNSIGNED-PAYLOAD");
if !has_body_digest {
return Err(Status::permission_denied("scanner publication lease rename requires a body-bound digest"));
}
}
verify_disk_mutation_digest(
&request,
rustfs_protos::canonical_rename_data_request_body(request.get_ref()),
@@ -1206,6 +1217,42 @@ impl NodeService {
}));
}
};
let scanner_publication_lease_token = if request.scanner_publication_lease_token.is_empty() {
None
} else {
let token = Uuid::from_slice(&request.scanner_publication_lease_token)
.map_err(|_| Status::invalid_argument("scanner publication lease token must be a UUID"))?;
if token.is_nil() {
return Err(Status::invalid_argument("scanner publication lease token must not be nil"));
}
Some(token)
};
// The target owns this read guard. It must span the complete
// disk rename, not merely the preflight, so a movement transition
// cannot restart after validation and before rename linearization.
let _scanner_publication_lease_guard = if let Some(token) = scanner_publication_lease_token {
let Some(store) = self.resolve_object_store() else {
return Ok(Response::new(RenameDataResponse {
success: false,
rename_data_resp: String::new(),
rename_data_resp_bin: Vec::new().into(),
error: Some(DiskError::other("scanner publication lease owner is unavailable").into()),
}));
};
match store.acquire_scanner_publication_lease_guard(token).await {
Ok(guard) => Some(guard),
Err(err) => {
return Ok(Response::new(RenameDataResponse {
success: false,
rename_data_resp: String::new(),
rename_data_resp_bin: Vec::new().into(),
error: Some(DiskError::other(err.to_string()).into()),
}));
}
}
} else {
None
};
let request_decoded_from_msgpack = decoded_file_info.from_msgpack;
match disk
.rename_data(
@@ -1559,6 +1606,16 @@ impl NodeService {
}
pub(super) async fn handle_delete(&self, request: Request<DeleteRequest>) -> Result<Response<DeleteResponse>, Status> {
if !request.get_ref().scanner_publication_lease_token.is_empty() {
let has_body_digest = request
.metadata()
.get("x-rustfs-content-sha256")
.and_then(|value| value.to_str().ok())
.is_some_and(|value| value != "UNSIGNED-PAYLOAD");
if !has_body_digest {
return Err(Status::permission_denied("scanner publication lease delete requires a body-bound digest"));
}
}
verify_disk_mutation_digest(&request, rustfs_protos::canonical_delete_request_body(request.get_ref()), "delete")?;
let request = request.into_inner();
if let Some(disk) = self.find_disk(&request.disk).await {
@@ -1571,6 +1628,38 @@ impl NodeService {
}));
}
};
let scanner_publication_lease_token = if request.scanner_publication_lease_token.is_empty() {
None
} else {
let token = Uuid::from_slice(&request.scanner_publication_lease_token)
.map_err(|_| Status::invalid_argument("scanner publication lease token must be a UUID"))?;
if token.is_nil() {
return Err(Status::invalid_argument("scanner publication lease token must not be nil"));
}
Some(token)
};
// The target-side guard spans the complete delete operation. A
// lease expiry or movement transition cannot occur between this
// validation and the disk delete linearization point.
let _scanner_publication_lease_guard = if let Some(token) = scanner_publication_lease_token {
let Some(store) = self.resolve_object_store() else {
return Ok(Response::new(DeleteResponse {
success: false,
error: Some(DiskError::other("scanner publication lease owner is unavailable").into()),
}));
};
match store.acquire_scanner_publication_lease_guard(token).await {
Ok(guard) => Some(guard),
Err(err) => {
return Ok(Response::new(DeleteResponse {
success: false,
error: Some(DiskError::other(err.to_string()).into()),
}));
}
}
} else {
None
};
match disk.delete(&request.volume, &request.path, options).await {
Ok(_) => Ok(Response::new(DeleteResponse {
success: true,
+9 -6
View File
@@ -242,14 +242,16 @@ pub(crate) mod rpc_consumer {
pub(crate) use super::super::ecstore_rpc::decode_heal_bucket_rpc_options;
pub(crate) use super::super::storage_contracts::{
SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION, SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION,
SCANNER_ACTIVITY_V6_PROTOCOL_VERSION,
};
pub(crate) use super::super::{
BatchReadVersionReq, BatchReadVersionResp, CollectMetricsOpts, DeleteOptions, DiskError, DiskInfoOptions, DiskStore,
ECStore, Error, FileInfoVersions, KMS_SIGNAL_SUBSYSTEM, LocalPeerS3Client, MetricType, PEER_RESTDRY_RUN,
PEER_RESTSIGNAL, PEER_RESTSUB_SYS, ReadMultipleReq, ReadMultipleResp, ReadOptions, SERVICE_SIGNAL_REFRESH_CONFIG,
SERVICE_SIGNAL_RELOAD_DYNAMIC, StorageDiskRpcExt, StoragePeerS3ClientExt, UpdateMetadataOpts, all_local_disk_path,
collect_local_metrics, find_local_disk_by_ref, get_local_server_property, reload_bucket_metadata,
reload_transition_tier_config, remove_bucket_metadata, validate_batch_read_version_item_count,
PEER_RESTSIGNAL, PEER_RESTSUB_SYS, ReadMultipleReq, ReadMultipleResp, ReadOptions, SCANNER_PUBLICATION_LEASE_TTL_MS,
SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, StorageDiskRpcExt, StoragePeerS3ClientExt,
UpdateMetadataOpts, all_local_disk_path, collect_local_metrics, find_local_disk_by_ref, get_local_server_property,
reload_bucket_metadata, reload_transition_tier_config, remove_bucket_metadata,
validate_batch_read_version_item_count,
};
pub(crate) type StorageResult<T> = super::super::Result<T>;
@@ -581,8 +583,8 @@ pub(crate) mod ecstore_storage {
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::storage::init_local_disks;
pub(crate) use rustfs_ecstore::api::storage::{
ECStore, all_local_disk, all_local_disk_path, find_local_disk_by_ref, init_local_disks_with_instance_ctx,
init_lock_clients, prewarm_local_disk_id_map_with_instance_ctx,
ECStore, SCANNER_PUBLICATION_LEASE_TTL_MS, all_local_disk, all_local_disk_path, find_local_disk_by_ref,
init_local_disks_with_instance_ctx, init_lock_clients, prewarm_local_disk_id_map_with_instance_ctx,
};
}
@@ -614,6 +616,7 @@ pub(crate) const KMS_SIGNAL_SUBSYSTEM: &str = ecstore_rpc::KMS_SIGNAL_SUBSYSTEM;
pub(crate) const SERVICE_SIGNAL_REFRESH_CONFIG: u64 = ecstore_rpc::SERVICE_SIGNAL_REFRESH_CONFIG;
pub(crate) const SERVICE_SIGNAL_RELOAD_DYNAMIC: u64 = ecstore_rpc::SERVICE_SIGNAL_RELOAD_DYNAMIC;
pub(crate) const RUSTFS_META_BUCKET: &str = ecstore_disk::RUSTFS_META_BUCKET;
pub(crate) const SCANNER_PUBLICATION_LEASE_TTL_MS: u64 = ecstore_storage::SCANNER_PUBLICATION_LEASE_TTL_MS;
pub(crate) const TONIC_RPC_PREFIX: &str = ecstore_rpc::TONIC_RPC_PREFIX;
pub(crate) fn normalize_tonic_rpc_audience(value: &str) -> std::io::Result<String> {