fix(scanner): fence movement generation publication (#6461)

* feat(scanner): add movement generation fencing

* fix(scanner): prioritize unverified cycle deferral

* feat(ecstore): add scanner publication lease fence

* feat(rpc): add scanner publication lease protocol

* feat(scanner): hold remote leases through usage publish

* test(scanner): cover publication lease fencing

* fix(scanner): fence remote leases across restart and delay

* feat(rpc): fence scanner publication rename writes

* fix(scanner): fence observed cleanup deletes

* fix(proto): qualify lease release test types

* fix(scanner): pin movement notifications

* fix(scanner): clean publication imports

* fix(ecstore): satisfy scanner fence clippy

* refactor(scanner): group wait and publication options

* fix(scanner): satisfy final lint and facade guards

* fix(rpc): resolve facade export conflicts

* fix(ci): remove unused decommission and healing facades

* fix(ci): cfg-gate test-only usage overlay import

* fix(scanner): wake on remote scanner restart
This commit is contained in:
cxymds
2026-08-24 14:17:35 +08:00
committed by GitHub
parent 1c5c28842a
commit eec0e0e056
30 changed files with 2926 additions and 230 deletions
+19 -17
View File
@@ -415,9 +415,9 @@ pub mod notification {
#[cfg(any(test, feature = "test-util"))]
pub use crate::services::notification_sys::rotate_cross_pool_fence_fleet_proof_for_test;
pub use crate::services::notification_sys::{
CrossPoolFenceFleetProofToken, NotificationPeerErr, NotificationSys, acquire_cross_pool_fence_fleet_proof,
cross_pool_fence_fleet_proof_matches, get_global_notification_sys, new_global_notification_sys,
start_remote_version_state_fleet_probe,
CrossPoolFenceFleetProofToken, NotificationPeerErr, NotificationSys, ScannerPublicationLeaseGrant,
acquire_cross_pool_fence_fleet_proof, cross_pool_fence_fleet_proof_matches, get_global_notification_sys,
new_global_notification_sys, start_remote_version_state_fleet_probe,
};
}
@@ -426,9 +426,10 @@ pub mod object {
BLOCK_SIZE_V2, ERASURE_ALGORITHM, EncryptionResolutionError, EncryptionResolutionErrorKind, GetObjectBodyCacheHook,
GetObjectBodyCacheHookLookup, GetObjectBodySource, GetObjectReader, NamespaceLockFence, ObjectEncryptionResolver,
ObjectInfo, ObjectLockConfigSnapshot, ObjectMutationHook, ObjectOptions, PutObjReader, QuotaAdmission,
RangedDecompressReader, ReadEncryptionMaterial, ReadEncryptionMode, ReadEncryptionRequest, StreamConsumer,
get_object_body_cache_plaintext_len, lookup_get_object_body_cache_hook, register_get_object_body_cache_hook,
register_object_mutation_hook, unregister_get_object_body_cache_hook, unregister_object_mutation_hook,
RangedDecompressReader, ReadEncryptionMaterial, ReadEncryptionMode, ReadEncryptionRequest,
SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY, StreamConsumer, get_object_body_cache_plaintext_len,
lookup_get_object_body_cache_hook, register_get_object_body_cache_hook, register_object_mutation_hook,
unregister_get_object_body_cache_hook, unregister_object_mutation_hook,
};
pub use crate::store::{
PrepareSelectObjectSnapshotError, PreparedGetObjectReader, SelectObjectSnapshot, SelectObjectSnapshotReadError,
@@ -461,16 +462,17 @@ pub mod rpc {
pub use crate::cluster::rpc::{
AuthenticatedChannel, KMS_SIGNAL_SUBSYSTEM, LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS,
PeerRestClient, PeerS3Client, S3PeerSys, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC,
ScannerBucketListing, ScannerPeerActivity, TONIC_RPC_PREFIX, TonicInterceptor, build_put_file_auth_trailer,
check_and_record_signed_rpc_nonce, decode_heal_bucket_rpc_options, encode_heal_bucket_rpc_options, gen_signature_headers,
gen_tonic_replay_scope_headers, gen_tonic_signature_headers, gen_tonic_signature_interceptor,
node_service_time_out_client, node_service_time_out_client_no_auth, normalize_tonic_rpc_audience,
set_tonic_canonical_body_digest, sign_ns_scanner_capability, sign_ns_scanner_capability_with_tier_registry_generation,
sign_put_file_capability, sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers,
tonic_rpc_auth_failure_reason, verify_ns_scanner_capability, verify_ns_scanner_capability_with_tier_registry_generation,
verify_put_file_auth_trailer, verify_put_file_capability, verify_rpc_signature, verify_tonic_boot_epoch_response,
verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof,
verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
ScannerBucketListing, ScannerPeerActivity, ScannerPublicationLease, TONIC_RPC_PREFIX, TonicInterceptor,
build_put_file_auth_trailer, check_and_record_signed_rpc_nonce, decode_heal_bucket_rpc_options,
encode_heal_bucket_rpc_options, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
gen_tonic_signature_interceptor, node_service_time_out_client, node_service_time_out_client_no_auth,
normalize_tonic_rpc_audience, set_tonic_canonical_body_digest, sign_ns_scanner_capability,
sign_ns_scanner_capability_with_tier_registry_generation, sign_put_file_capability, sign_tonic_rpc_response_proof,
tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason,
verify_ns_scanner_capability, verify_ns_scanner_capability_with_tier_registry_generation, verify_put_file_auth_trailer,
verify_put_file_capability, verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
verify_tonic_rpc_signature_with_bootstrap,
};
}
@@ -497,7 +499,7 @@ pub mod storage {
pub use crate::core::pools::HealLifecycleExpiryContext;
pub use crate::store::HealWalkVersion;
pub use crate::store::{
ECStore, all_local_disk, all_local_disk_path, find_local_disk_by_ref, init_local_disks,
ECStore, SCANNER_PUBLICATION_LEASE_TTL_MS, all_local_disk, all_local_disk_path, find_local_disk_by_ref, init_local_disks,
init_local_disks_with_instance_ctx, init_lock_clients, prewarm_local_disk_id_map,
prewarm_local_disk_id_map_with_instance_ctx,
};
@@ -2994,6 +2994,7 @@ mod tests {
dst_volume: "bucket".to_string(),
dst_path: "object".to_string(),
file_info_bin: vec![0x81, 0xA1, 0x76, 0x01].into(),
scanner_publication_lease_token: Vec::new().into(),
};
let body = rustfs_protos::canonical_rename_data_request_body(&message).expect("small request should encode");
let mut request = tonic::Request::new(());
+1 -1
View File
@@ -48,7 +48,7 @@ pub use internode_data_transport::build_internode_data_transport_from_env;
pub(crate) use peer_rest_client::TierConfigReloadOutcome;
pub use peer_rest_client::{
KMS_SIGNAL_SUBSYSTEM, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, SERVICE_SIGNAL_REFRESH_CONFIG,
SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerPeerActivity,
SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerPeerActivity, ScannerPublicationLease,
};
pub(crate) use peer_s3_client::heal_bucket_local_on_disks;
pub use peer_s3_client::{
@@ -20,6 +20,7 @@ use crate::cluster::rpc::{set_tonic_canonical_body_digest, set_tonic_mutation_bo
use crate::error::{Error, Result};
use crate::storage_api_contracts::internode::{
SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION, SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SCANNER_ACTIVITY_PROTOCOL_VERSION,
SCANNER_ACTIVITY_V6_PROTOCOL_VERSION,
};
use crate::{
bucket::replication::BucketStats,
@@ -45,8 +46,9 @@ use rustfs_protos::proto_gen::node_service::{
HealControlRequest, LoadBucketMetadataRequest, LoadGroupRequest, LoadPolicyMappingRequest, LoadPolicyRequest,
LoadRebalanceMetaRequest, LoadServiceAccountRequest, LoadTransitionTierConfigRequest, LoadUserRequest,
LocalStorageInfoRequest, Mss, ReloadPoolMetaRequest, ReloadSiteReplicationConfigRequest, ReplacementRecoveryStatusRequest,
ScannerActivityRequest, ScannerActivityResponse, ServerInfoRequest, SignalServiceRequest, SignalServiceResponse,
StartDecommissionRequest, StartProfilingRequest, StopRebalanceRequest, TierMutationAbortRequest, TierMutationCommitRequest,
ScannerActivityRequest, ScannerActivityResponse, ScannerPublicationLeaseReleaseRequest, ScannerPublicationLeaseRequest,
ScannerPublicationLeaseResponse, ServerInfoRequest, SignalServiceRequest, SignalServiceResponse, StartDecommissionRequest,
StartProfilingRequest, StopRebalanceRequest, TierMutationAbortRequest, TierMutationCommitRequest,
TierMutationControlResponse, TierMutationPeerState, TierMutationPrepareRequest, node_service_client::NodeServiceClient,
tier_mutation_control_service_client::TierMutationControlServiceClient,
};
@@ -84,6 +86,11 @@ const HEAL_CONTROL_PAYLOAD_MAX_SIZE: usize = 64 * 1024;
const PEER_REST_RECOVERY_MAX_ATTEMPTS: u32 = 60;
const PEER_REST_RECOVERY_MAX_BACKOFF: Duration = Duration::from_secs(30);
const SCANNER_ACTIVITY_MAX_MESSAGE_SIZE: usize = 1024;
/// Reserve time for the acquire response's network/clock uncertainty. The
/// server owns the real expiry; this local deadline is intentionally earlier
/// so a coordinator never starts a bounded persistence operation at the edge
/// of a remote lease.
const SCANNER_PUBLICATION_LEASE_SAFETY_MARGIN: Duration = Duration::from_secs(5);
const REPLICATION_STATS_MAX_MESSAGE_SIZE: usize = 8 * 1024 * 1024;
const BUCKET_METADATA_RELOAD_TIMEOUT: Duration = Duration::from_secs(5);
@@ -150,6 +157,8 @@ pub struct ScannerPeerActivity {
pub data_movement_active: Option<bool>,
pub dirty_usage_generation: Option<u64>,
pub dirty_usage_pending: Option<bool>,
pub movement_generation: Option<u64>,
pub publication_blocked: Option<bool>,
}
fn decode_scanner_activity_with_verifier(
@@ -166,7 +175,14 @@ fn decode_scanner_activity_with_verifier(
{
return Err(Error::other("peer returned an invalid scanner activity instance ID"));
}
let (topology_digest, data_movement_active, dirty_usage_generation, dirty_usage_pending) = match response.protocol_version {
let (
topology_digest,
data_movement_active,
dirty_usage_generation,
dirty_usage_pending,
movement_generation,
publication_blocked,
) = match response.protocol_version {
// RUSTFS_COMPAT_TODO(ns-scanner-rpc-v3): legacy response fields are unauthenticated. Remove after protocol v0 peers are unsupported.
SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION
if response.topology_digest.is_empty()
@@ -175,7 +191,7 @@ fn decode_scanner_activity_with_verifier(
&& response.dirty_usage_generation == 0
&& !response.dirty_usage_pending =>
{
(None, None, None, None)
(None, None, None, None, None, None)
}
SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION => {
return Err(Error::other("legacy scanner activity peer returned unexpected extended fields"));
@@ -198,9 +214,11 @@ fn decode_scanner_activity_with_verifier(
Some(response.data_movement_active),
None,
None,
None,
None,
)
}
SCANNER_ACTIVITY_PROTOCOL_VERSION => {
SCANNER_ACTIVITY_V6_PROTOCOL_VERSION => {
if response.dirty_usage_pending && response.dirty_usage_generation == 0 {
return Err(Error::other("scanner activity peer returned pending dirty usage without a generation"));
}
@@ -218,11 +236,42 @@ fn decode_scanner_activity_with_verifier(
Some(response.data_movement_active),
Some(response.dirty_usage_generation),
Some(response.dirty_usage_pending),
None,
None,
)
}
version => {
return Err(Error::other(format!("peer returned unsupported scanner activity protocol {version}")));
SCANNER_ACTIVITY_PROTOCOL_VERSION => {
if response.dirty_usage_pending && response.dirty_usage_generation == 0 {
return Err(Error::other("scanner activity peer returned pending dirty usage without a generation"));
}
let movement_generation = response
.movement_generation
.ok_or_else(|| Error::other("scanner activity peer omitted its movement generation"))?;
let publication_blocked = response
.publication_blocked
.ok_or_else(|| Error::other("scanner activity peer omitted its publication blocked state"))?;
if movement_generation == u64::MAX {
return Err(Error::other("scanner activity peer exhausted its movement generation"));
}
let canonical = rustfs_protos::canonical_scanner_activity_v7_response_body(challenge, &response)
.map_err(|_| Error::other("scanner activity peer response is too large to authenticate"))?;
verify_proof(&canonical, &response.response_proof)?;
(
Some(
response
.topology_digest
.as_ref()
.try_into()
.map_err(|_| Error::other("peer returned an invalid scanner topology digest"))?,
),
Some(response.data_movement_active),
Some(response.dirty_usage_generation),
Some(response.dirty_usage_pending),
Some(movement_generation),
Some(publication_blocked),
)
}
version => return Err(Error::other(format!("peer returned unsupported scanner activity protocol {version}"))),
};
Ok(ScannerPeerActivity {
instance_id: response.instance_id,
@@ -233,6 +282,8 @@ fn decode_scanner_activity_with_verifier(
data_movement_active,
dirty_usage_generation,
dirty_usage_pending,
movement_generation,
publication_blocked,
})
}
@@ -243,6 +294,17 @@ fn decode_scanner_activity(response: ScannerActivityResponse, challenge: &[u8; 1
})
}
fn scanner_activity_protocol_unsupported(err: &Error) -> bool {
matches!(
err,
Error::Io(io_err)
if embedded_tonic_status(io_err).is_some_and(|status| {
status.code() == tonic::Code::FailedPrecondition
&& status.message().starts_with("unsupported scanner activity request protocol")
})
)
}
fn validate_heal_control_capability_proof(canonical_ack: &[u8], proof: &[u8]) -> Result<()> {
verify_tonic_rpc_response_proof(canonical_ack, proof)
.map_err(|_| Error::other("peer returned an invalid heal control capability proof"))
@@ -285,6 +347,76 @@ pub struct PeerLiveEventsBatch {
pub truncated: bool,
}
#[derive(Clone, Debug)]
pub struct ScannerPublicationLease {
pub token: Uuid,
pub movement_generation: u64,
/// Stable storage owner identity. This is distinct from the activity
/// session and is bound into both acquire and release proofs.
pub owner_id: String,
/// Process/session nonce observed by the final activity probe.
pub session_id: String,
pub expires_at: std::time::Instant,
}
impl ScannerPublicationLease {
pub fn is_valid(&self) -> bool {
std::time::Instant::now() < self.expires_at
}
}
fn validate_scanner_publication_lease_response_fields(
response: &ScannerPublicationLeaseResponse,
expected_session_id: &str,
expected_generation: u64,
) -> Result<(Uuid, String)> {
if !response.success {
return Err(Error::other(
response
.error
.as_ref()
.map(|error| error.error_info.clone())
.unwrap_or_else(|| "peer rejected scanner publication lease".to_string()),
));
}
if response.movement_generation != expected_generation {
return Err(Error::other("peer returned a different scanner publication lease generation"));
}
if response.session_id != expected_session_id {
return Err(Error::other("peer returned a different scanner publication lease session"));
}
let owner_id = Uuid::parse_str(&response.owner_id)
.ok()
.filter(|owner_id| !owner_id.is_nil())
.map(|owner_id| owner_id.to_string())
.ok_or_else(|| Error::other("peer returned an invalid scanner publication lease owner"))?;
if response.lease_ttl_ms != crate::store::SCANNER_PUBLICATION_LEASE_TTL_MS {
return Err(Error::other("peer returned an unsupported scanner publication lease TTL"));
}
let token = Uuid::from_slice(response.token.as_ref())
.map_err(|_| Error::other("peer returned an invalid scanner publication lease token"))?;
Ok((token, owner_id))
}
fn scanner_publication_lease_deadline(
request_started: std::time::Instant,
response_received: std::time::Instant,
lease_ttl_ms: u64,
) -> Result<std::time::Instant> {
let lease_window = Duration::from_millis(lease_ttl_ms)
.checked_sub(SCANNER_PUBLICATION_LEASE_SAFETY_MARGIN)
.ok_or_else(|| Error::other("scanner publication lease TTL is shorter than its safety margin"))?;
let elapsed = response_received
.checked_duration_since(request_started)
.ok_or_else(|| Error::other("scanner publication lease response clock moved backwards"))?;
if elapsed >= lease_window {
return Err(Error::other("scanner publication lease response arrived after its safety window"));
}
request_started
.checked_add(lease_window)
.ok_or_else(|| Error::other("scanner publication lease deadline overflowed"))
}
#[derive(Clone, Debug)]
pub struct PeerRestClient {
pub host: XHost,
@@ -1640,10 +1772,11 @@ impl PeerRestClient {
.await
}
async fn scanner_activity_request(
async fn scanner_activity_request_with_protocol(
&self,
acknowledge_instance_id: String,
acknowledge_dirty_usage_generation: u64,
protocol_version: u32,
) -> Result<ScannerPeerActivity> {
self.finalize_result(
async {
@@ -1655,7 +1788,7 @@ impl PeerRestClient {
.max_encoding_message_size(SCANNER_ACTIVITY_MAX_MESSAGE_SIZE);
let mut request = Request::new(ScannerActivityRequest {
challenge: challenge.as_bytes().to_vec().into(),
protocol_version: SCANNER_ACTIVITY_PROTOCOL_VERSION,
protocol_version,
acknowledge_instance_id,
acknowledge_dirty_usage_generation,
});
@@ -1671,11 +1804,168 @@ impl PeerRestClient {
}
pub async fn scanner_activity(&self) -> Result<ScannerPeerActivity> {
self.scanner_activity_request(String::new(), 0).await
let result = self
.scanner_activity_request_with_protocol(String::new(), 0, SCANNER_ACTIVITY_PROTOCOL_VERSION)
.await;
if result.as_ref().err().is_some_and(scanner_activity_protocol_unsupported) {
// A v6 peer cannot parse the v7 marker. Its authenticated
// response is still decoded as untrusted terminal state, so the
// scanner will defer publication until every peer is v7.
self.scanner_activity_request_with_protocol(String::new(), 0, SCANNER_ACTIVITY_V6_PROTOCOL_VERSION)
.await
} else {
result
}
}
pub async fn acknowledge_scanner_dirty_usage(&self, instance_id: String, generation: u64) -> Result<ScannerPeerActivity> {
self.scanner_activity_request(instance_id, generation).await
let result = self
.scanner_activity_request_with_protocol(instance_id.clone(), generation, SCANNER_ACTIVITY_PROTOCOL_VERSION)
.await;
if result.as_ref().err().is_some_and(scanner_activity_protocol_unsupported) {
self.scanner_activity_request_with_protocol(instance_id, generation, SCANNER_ACTIVITY_V6_PROTOCOL_VERSION)
.await
} else {
result
}
}
/// Acquire a bounded, storage-owned read admission on the peer that
/// produced the final activity generation. Older peers do not implement
/// the lease form and are rejected rather than downgraded.
pub async fn acquire_scanner_publication_lease(
&self,
expected_session_id: &str,
expected_generation: u64,
) -> Result<ScannerPublicationLease> {
let request_started = std::time::Instant::now();
self.finalize_result(
async {
let challenge = Uuid::new_v4();
let mut client = self
.get_client()
.await?
.max_decoding_message_size(SCANNER_ACTIVITY_MAX_MESSAGE_SIZE)
.max_encoding_message_size(SCANNER_ACTIVITY_MAX_MESSAGE_SIZE);
let mut request = Request::new(ScannerPublicationLeaseRequest {
challenge: challenge.as_bytes().to_vec().into(),
expected_movement_generation: expected_generation,
ttl_ms: crate::store::SCANNER_PUBLICATION_LEASE_TTL_MS,
expected_session_id: expected_session_id.to_string(),
token: Bytes::new(),
});
let canonical = rustfs_protos::canonical_scanner_publication_lease_request_body(request.get_ref())
.map_err(|_| Error::other("scanner publication lease request is too large to authenticate"))?;
set_tonic_canonical_body_digest(&mut request, &canonical)?;
let response = client.acquire_scanner_publication_lease(request).await?.into_inner();
let response_body =
rustfs_protos::canonical_scanner_publication_lease_response_body(challenge.as_bytes(), &response)
.map_err(|_| Error::other("scanner publication lease response is too large to authenticate"))?;
verify_tonic_rpc_response_proof(&response_body, &response.response_proof)
.map_err(|_| Error::other("peer returned an invalid scanner publication lease proof"))?;
let (token, owner_id) =
validate_scanner_publication_lease_response_fields(&response, expected_session_id, expected_generation)?;
Ok(ScannerPublicationLease {
token,
movement_generation: response.movement_generation,
owner_id,
session_id: response.session_id,
expires_at: scanner_publication_lease_deadline(
request_started,
std::time::Instant::now(),
response.lease_ttl_ms,
)?,
})
}
.await,
)
.await
}
/// Revalidate the exact token immediately before the coordinator commits
/// its final publication. The peer keeps the original movement read
/// guard in its token table; a restart drops that table and changes the
/// activity session, so this proof fails closed instead of accepting an
/// ABA generation value.
pub async fn validate_scanner_publication_lease(&self, lease: &ScannerPublicationLease) -> Result<()> {
self.finalize_result(
async {
let challenge = Uuid::new_v4();
let mut client = self
.get_client()
.await?
.max_decoding_message_size(SCANNER_ACTIVITY_MAX_MESSAGE_SIZE)
.max_encoding_message_size(SCANNER_ACTIVITY_MAX_MESSAGE_SIZE);
let mut request = Request::new(ScannerPublicationLeaseRequest {
challenge: challenge.as_bytes().to_vec().into(),
expected_movement_generation: lease.movement_generation,
ttl_ms: crate::store::SCANNER_PUBLICATION_LEASE_TTL_MS,
expected_session_id: lease.session_id.clone(),
token: lease.token.as_bytes().to_vec().into(),
});
let canonical = rustfs_protos::canonical_scanner_publication_lease_request_body(request.get_ref())
.map_err(|_| Error::other("scanner publication lease validation request is too large to authenticate"))?;
set_tonic_canonical_body_digest(&mut request, &canonical)?;
let response = client.acquire_scanner_publication_lease(request).await?.into_inner();
let response_body =
rustfs_protos::canonical_scanner_publication_lease_response_body(challenge.as_bytes(), &response).map_err(
|_| Error::other("scanner publication lease validation response is too large to authenticate"),
)?;
verify_tonic_rpc_response_proof(&response_body, &response.response_proof)
.map_err(|_| Error::other("peer returned an invalid scanner publication lease validation proof"))?;
let (token, owner_id) =
validate_scanner_publication_lease_response_fields(&response, &lease.session_id, lease.movement_generation)?;
if token != lease.token {
return Err(Error::other("peer returned a different scanner publication lease token"));
}
if owner_id != lease.owner_id {
return Err(Error::other("peer returned a different scanner publication lease owner"));
}
Ok(())
}
.await,
)
.await
}
pub async fn release_scanner_publication_lease(&self, lease: &ScannerPublicationLease) -> Result<()> {
self.finalize_result(
async {
let challenge = Uuid::new_v4();
let mut client = self.get_client().await?;
let mut request = Request::new(ScannerPublicationLeaseReleaseRequest {
challenge: challenge.as_bytes().to_vec().into(),
token: lease.token.as_bytes().to_vec().into(),
owner_id: lease.owner_id.clone(),
session_id: lease.session_id.clone(),
});
let canonical = rustfs_protos::canonical_scanner_publication_lease_release_request_body(request.get_ref())
.map_err(|_| Error::other("scanner publication lease release request is too large to authenticate"))?;
set_tonic_canonical_body_digest(&mut request, &canonical)?;
let request_body = request.get_ref().clone();
let response = client.release_scanner_publication_lease(request).await?.into_inner();
let response_body = rustfs_protos::canonical_scanner_publication_lease_release_response_body(
challenge.as_bytes(),
&request_body,
&response,
)
.map_err(|_| Error::other("scanner publication lease release response is too large to authenticate"))?;
verify_tonic_rpc_response_proof(&response_body, &response.response_proof)
.map_err(|_| Error::other("peer returned an invalid scanner publication lease release proof"))?;
if response.success {
Ok(())
} else {
Err(Error::other(
response
.error
.map(|error| error.error_info)
.unwrap_or_else(|| "peer rejected scanner publication lease release".to_string()),
))
}
}
.await,
)
.await
}
pub async fn get_metacache_listing(&self) -> Result<()> {
@@ -1991,6 +2281,52 @@ mod tests {
use temp_env::async_with_vars;
use tracing_subscriber::{Registry, fmt::MakeWriter, layer::SubscriberExt};
#[test]
fn scanner_publication_lease_response_rejects_stale_generation_and_session() {
let token = Uuid::new_v4();
let response = ScannerPublicationLeaseResponse {
success: true,
token: token.as_bytes().to_vec().into(),
movement_generation: 7,
lease_ttl_ms: crate::store::SCANNER_PUBLICATION_LEASE_TTL_MS,
error: None,
response_proof: Bytes::new(),
owner_id: Uuid::new_v4().to_string(),
session_id: "session-a".to_string(),
};
assert!(validate_scanner_publication_lease_response_fields(&response, "session-a", 7).is_ok());
let stale_generation = ScannerPublicationLeaseResponse {
movement_generation: 6,
..response.clone()
};
let error = validate_scanner_publication_lease_response_fields(&stale_generation, "session-a", 7)
.expect_err("a response from an older movement generation must be rejected");
assert!(error.to_string().contains("different scanner publication lease generation"));
let stale_session = ScannerPublicationLeaseResponse {
session_id: "session-b".to_string(),
..response
};
let error = validate_scanner_publication_lease_response_fields(&stale_session, "session-a", 7)
.expect_err("a response from an older scanner session must be rejected");
assert!(error.to_string().contains("different scanner publication lease session"));
}
#[test]
fn scanner_publication_lease_deadline_accounts_for_delayed_rpc_response() {
let started = std::time::Instant::now();
let expected_deadline = started + Duration::from_secs(55);
let deadline = scanner_publication_lease_deadline(started, started + Duration::from_secs(10), 60_000)
.expect("a response inside the safety window should retain the original deadline");
assert_eq!(deadline, expected_deadline);
let error = scanner_publication_lease_deadline(started, started + Duration::from_secs(55), 60_000)
.expect_err("a response arriving at the safety boundary must fail closed");
assert!(error.to_string().contains("after its safety window"));
}
#[test]
fn replication_stats_response_decodes_valid_empty_provider() {
let mut stats = BucketStats::default();
@@ -2229,6 +2565,8 @@ mod tests {
response_proof: Vec::new().into(),
dirty_usage_generation: 0,
dirty_usage_pending: false,
movement_generation: None,
publication_blocked: None,
})
.expect("legacy peers should retain their activity generations during a rolling upgrade");
assert_eq!(
@@ -2242,6 +2580,8 @@ mod tests {
data_movement_active: None,
dirty_usage_generation: None,
dirty_usage_pending: None,
movement_generation: None,
publication_blocked: None,
}
);
@@ -2255,6 +2595,8 @@ mod tests {
response_proof: b"proof".to_vec().into(),
dirty_usage_generation: 0,
dirty_usage_pending: false,
movement_generation: None,
publication_blocked: None,
})
.expect("protocol v4 peers should remain observable during a rolling upgrade");
assert_eq!(
@@ -2268,9 +2610,29 @@ mod tests {
data_movement_active: Some(true),
dirty_usage_generation: None,
dirty_usage_pending: None,
movement_generation: None,
publication_blocked: None,
}
);
let v6 = decode_test_scanner_activity(ScannerActivityResponse {
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
namespace_generation: 7,
maintenance_generation: 3,
protocol_version: SCANNER_ACTIVITY_V6_PROTOCOL_VERSION,
topology_digest: vec![7; 32].into(),
data_movement_active: true,
response_proof: b"proof".to_vec().into(),
dirty_usage_generation: 11,
dirty_usage_pending: true,
movement_generation: None,
publication_blocked: None,
})
.expect("v6 peers should remain readable without a v7 publication proof");
assert_eq!(v6.movement_generation, None);
assert_eq!(v6.publication_blocked, None);
assert_eq!(v6.dirty_usage_generation, Some(11));
let malformed_topology = ScannerActivityResponse {
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
namespace_generation: 7,
@@ -2281,6 +2643,8 @@ mod tests {
response_proof: b"proof".to_vec().into(),
dirty_usage_generation: 11,
dirty_usage_pending: true,
movement_generation: Some(19),
publication_blocked: Some(false),
};
assert!(
decode_test_scanner_activity(malformed_topology)
@@ -2299,6 +2663,8 @@ mod tests {
response_proof: b"proof".to_vec().into(),
dirty_usage_generation: 11,
dirty_usage_pending: true,
movement_generation: Some(19),
publication_blocked: Some(false),
};
assert!(
decode_test_scanner_activity(missing_instance)
@@ -2317,6 +2683,8 @@ mod tests {
response_proof: b"proof".to_vec().into(),
dirty_usage_generation: 11,
dirty_usage_pending: true,
movement_generation: Some(19),
publication_blocked: Some(false),
};
assert!(
decode_test_scanner_activity(malformed_instance)
@@ -2335,6 +2703,8 @@ mod tests {
response_proof: b"proof".to_vec().into(),
dirty_usage_generation: 11,
dirty_usage_pending: true,
movement_generation: Some(19),
publication_blocked: Some(false),
})
.expect("complete activity responses should be accepted");
assert_eq!(
@@ -2348,9 +2718,31 @@ mod tests {
data_movement_active: Some(true),
dirty_usage_generation: Some(11),
dirty_usage_pending: Some(true),
movement_generation: Some(19),
publication_blocked: Some(false),
}
);
let missing_movement_generation = ScannerActivityResponse {
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
namespace_generation: 7,
maintenance_generation: 3,
protocol_version: SCANNER_ACTIVITY_PROTOCOL_VERSION,
topology_digest: vec![7; 32].into(),
data_movement_active: false,
response_proof: b"proof".to_vec().into(),
dirty_usage_generation: 11,
dirty_usage_pending: false,
movement_generation: None,
publication_blocked: Some(false),
};
assert!(
decode_test_scanner_activity(missing_movement_generation)
.expect_err("v7 activity must carry movement generation")
.to_string()
.contains("movement generation")
);
let pending_without_generation = ScannerActivityResponse {
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
namespace_generation: 7,
@@ -2361,6 +2753,8 @@ mod tests {
response_proof: b"proof".to_vec().into(),
dirty_usage_generation: 0,
dirty_usage_pending: true,
movement_generation: Some(19),
publication_blocked: Some(false),
};
assert!(
decode_test_scanner_activity(pending_without_generation)
@@ -2379,6 +2773,8 @@ mod tests {
response_proof: b"proof".to_vec().into(),
dirty_usage_generation: 11,
dirty_usage_pending: true,
movement_generation: None,
publication_blocked: None,
};
assert!(
decode_test_scanner_activity(previous_with_dirty_usage)
@@ -2397,6 +2793,8 @@ mod tests {
response_proof: b"proof".to_vec().into(),
dirty_usage_generation: 0,
dirty_usage_pending: false,
movement_generation: None,
publication_blocked: None,
};
assert!(
decode_test_scanner_activity(legacy_with_topology)
@@ -2415,6 +2813,8 @@ mod tests {
response_proof: b"proof".to_vec().into(),
dirty_usage_generation: 11,
dirty_usage_pending: true,
movement_generation: None,
publication_blocked: None,
};
assert!(
decode_test_scanner_activity(unsupported_protocol)
@@ -2433,6 +2833,8 @@ mod tests {
response_proof: Vec::new().into(),
dirty_usage_generation: 11,
dirty_usage_pending: true,
movement_generation: Some(19),
publication_blocked: Some(false),
};
assert!(
decode_test_scanner_activity(missing_proof)
+89 -42
View File
@@ -1981,6 +1981,20 @@ impl RemoteDisk {
fi: &FileInfo,
dst_volume: &str,
dst_path: &str,
) -> Result<RenameDataResp> {
self.rename_data_borrowed_with_fence(src_volume, src_path, fi, dst_volume, dst_path, None)
.await
}
#[tracing::instrument(level = "trace", skip_all)]
pub(crate) async fn rename_data_borrowed_with_fence(
&self,
src_volume: &str,
src_path: &str,
fi: &FileInfo,
dst_volume: &str,
dst_path: &str,
scanner_publication_lease_token: Option<Uuid>,
) -> Result<RenameDataResp> {
trace!(
event = EVENT_REMOTE_DISK_RPC,
@@ -2013,9 +2027,18 @@ impl RemoteDisk {
dst_volume: dst_volume.to_string(),
dst_path: dst_path.to_string(),
file_info_bin: file_info_bin.into(),
scanner_publication_lease_token: scanner_publication_lease_token
.map(|token| token.as_bytes().to_vec().into())
.unwrap_or_default(),
});
let canonical_body = rustfs_protos::canonical_rename_data_request_body(request.get_ref());
attach_mutation_body_digest(&mut request, canonical_body, "rename_data")?;
if scanner_publication_lease_token.is_some() {
let canonical_body =
canonical_body.map_err(|_| Error::other("rename_data request length cannot be represented"))?;
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &canonical_body).map_err(Error::other)?;
} else {
attach_mutation_body_digest(&mut request, canonical_body, "rename_data")?;
}
let response = client.rename_data(request).await?.into_inner();
@@ -2035,6 +2058,70 @@ impl RemoteDisk {
)
.await
}
/// Delete a path while binding the target-side operation to a scanner
/// publication lease. The ordinary `DiskAPI::delete` path keeps the
/// legacy digest/compatibility behavior by passing no token.
#[tracing::instrument(level = "trace", skip_all)]
pub(crate) async fn delete_with_scanner_publication_lease(
&self,
volume: &str,
path: &str,
opt: DeleteOptions,
scanner_publication_lease_token: Option<Uuid>,
) -> Result<()> {
trace!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
volume,
path,
recursive = opt.recursive,
immediate = opt.immediate,
fenced = scanner_publication_lease_token.is_some(),
op = "delete",
state = "started",
"Remote disk RPC started"
);
self.execute_with_timeout(
|| async {
let options = serde_json::to_string(&opt)?;
let mut client = self
.get_client()
.await
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
let mut request = Request::new(DeleteRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
path: path.to_string(),
options,
scanner_publication_lease_token: scanner_publication_lease_token
.map(|token| token.as_bytes().to_vec().into())
.unwrap_or_default(),
});
let canonical_body = rustfs_protos::canonical_delete_request_body(request.get_ref());
if scanner_publication_lease_token.is_some() {
let canonical_body =
canonical_body.map_err(|_| Error::other("delete request length cannot be represented"))?;
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &canonical_body).map_err(Error::other)?;
} else {
attach_mutation_body_digest(&mut request, canonical_body, "delete")?;
}
let response = client.delete(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
Ok(())
},
get_max_timeout_duration(),
)
.await
}
}
#[async_trait::async_trait]
@@ -3446,47 +3533,7 @@ impl DiskAPI for RemoteDisk {
#[tracing::instrument(level = "trace", skip_all)]
async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()> {
trace!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
volume,
path,
recursive = opt.recursive,
immediate = opt.immediate,
op = "delete",
state = "started",
"Remote disk RPC started"
);
self.execute_with_timeout(
|| async {
let options = serde_json::to_string(&opt)?;
let mut client = self
.get_client()
.await
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
let mut request = Request::new(DeleteRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
path: path.to_string(),
options,
});
let canonical_body = rustfs_protos::canonical_delete_request_body(request.get_ref());
attach_mutation_body_digest(&mut request, canonical_body, "delete")?;
let response = client.delete(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
Ok(())
},
get_max_timeout_duration(),
)
.await
self.delete_with_scanner_publication_lease(volume, path, opt, None).await
}
#[tracing::instrument(level = "trace", skip_all)]
+38 -1
View File
@@ -677,6 +677,23 @@ impl DiskAPI for Disk {
}
impl Disk {
pub(crate) async fn delete_with_scanner_publication_lease(
&self,
volume: &str,
path: &str,
opts: DeleteOptions,
scanner_publication_lease_token: Option<Uuid>,
) -> Result<()> {
match self {
Disk::Local(local_disk) => local_disk.delete(volume, path, opts).await,
Disk::Remote(remote_disk) => {
remote_disk
.delete_with_scanner_publication_lease(volume, path, opts, scanner_publication_lease_token)
.await
}
}
}
pub(crate) async fn rename_data_borrowed(
&self,
src_volume: &str,
@@ -684,6 +701,19 @@ impl Disk {
fi: &FileInfo,
dst_volume: &str,
dst_path: &str,
) -> Result<RenameDataResp> {
self.rename_data_borrowed_with_fence(src_volume, src_path, fi, dst_volume, dst_path, None)
.await
}
pub(crate) async fn rename_data_borrowed_with_fence(
&self,
src_volume: &str,
src_path: &str,
fi: &FileInfo,
dst_volume: &str,
dst_path: &str,
scanner_publication_lease_token: Option<Uuid>,
) -> Result<RenameDataResp> {
match self {
Disk::Local(local_disk) => {
@@ -693,7 +723,14 @@ impl Disk {
}
Disk::Remote(remote_disk) => {
remote_disk
.rename_data_borrowed(src_volume, src_path, fi, dst_volume, dst_path)
.rename_data_borrowed_with_fence(
src_volume,
src_path,
fi,
dst_volume,
dst_path,
scanner_publication_lease_token,
)
.await
}
}
+5
View File
@@ -446,6 +446,11 @@ pub struct ObjectOptions {
pub tier_delete_journal_api: Option<Arc<crate::store::ECStore>>,
}
/// Transient scanner-only carrier for target-side publication lease tokens.
/// SetDisks consumes and removes this key before constructing durable
/// FileInfo metadata; it must never appear in an S3-visible object.
pub const SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY: &str = "x-rustfs-internal-scanner-publication-lease-fence-v1";
impl ObjectOptions {
pub fn set_quota_admission(&mut self, current_usage: u64, quota_limit: u64) -> bool {
self.quota_admission = (current_usage <= quota_limit).then_some(QuotaAdmission {
+154 -1
View File
@@ -56,7 +56,8 @@ use std::sync::{
Arc, OnceLock,
atomic::{AtomicBool, AtomicU8, AtomicU64, Ordering},
};
use tokio::sync::{OnceCell, RwLock};
use tokio::sync::{Mutex, Notify, OnceCell, OwnedRwLockReadGuard, RwLock};
use tokio::time::Instant;
use tokio_util::sync::CancellationToken;
use uuid::Uuid;
@@ -64,6 +65,19 @@ const SCANNER_PUBLICATION_STATE_UNKNOWN: u8 = 0;
const SCANNER_PUBLICATION_STATE_ALLOWED: u8 = 1;
const SCANNER_PUBLICATION_STATE_BLOCKED: u8 = 2;
pub(crate) const SCANNER_PUBLICATION_LEASE_MAX_ENTRIES: usize = 256;
/// A lease is deliberately short-lived. The coordinator treats expiry as a
/// failed publication rather than silently continuing with a peer that may
/// have started movement after the lease was abandoned.
pub(crate) const SCANNER_PUBLICATION_LEASE_TTL: std::time::Duration = std::time::Duration::from_secs(60);
pub(crate) struct ScannerPublicationLeaseEntry {
pub(crate) expires_at: Instant,
pub(crate) movement_generation: u64,
pub(crate) _operation_guard: OwnedRwLockReadGuard<()>,
}
/// Runtime state owned by a single `ECStore` instance.
///
/// This is intentionally minimal in the first migration slice; subsequent
@@ -171,6 +185,11 @@ pub struct InstanceContext {
/// Readers are held across one publication commit; movement transitions
/// take the writer at their durable state commit boundary.
data_movement_operation_gate: Arc<RwLock<()>>,
/// Remote scanner publication leases own a read guard until explicit
/// release or bounded expiry. Keeping the guard in storage-owned state
/// makes a remote movement transition wait on the same fence as a local
/// scanner commit.
scanner_publication_leases: Arc<Mutex<HashMap<Uuid, ScannerPublicationLeaseEntry>>>,
/// Monotonic admission epoch paired with the operation gate. A
/// publication admitted before a movement transition must never be
/// mistaken for one admitted after the transition.
@@ -180,6 +199,13 @@ pub struct InstanceContext {
/// saturating counter prevents an unchanged `u64::MAX` value from being
/// mistaken for a fresh epoch after overflow.
data_movement_operation_epoch_exhausted: AtomicBool,
/// Storage-owned generation for movement state changes. This is separate
/// from the publication admission epoch so scanners can wait for a
/// terminal/clear transition without treating the wake as a publication
/// permit.
data_movement_generation: AtomicU64,
data_movement_generation_exhausted: AtomicBool,
data_movement_generation_notify: Arc<Notify>,
/// Last storage-owned movement snapshot observed under the operation
/// gate. SetDisks cache writers fail closed until ECStore refreshes it.
scanner_publication_state: AtomicU8,
@@ -224,8 +250,12 @@ impl InstanceContext {
bucket_metadata_sys: std::sync::Mutex::new(None),
background_cancel_token: OnceLock::new(),
data_movement_operation_gate: Arc::new(RwLock::new(())),
scanner_publication_leases: Arc::new(Mutex::new(HashMap::new())),
data_movement_operation_epoch: AtomicU64::new(0),
data_movement_operation_epoch_exhausted: AtomicBool::new(false),
data_movement_generation: AtomicU64::new(0),
data_movement_generation_exhausted: AtomicBool::new(false),
data_movement_generation_notify: Arc::new(Notify::new()),
scanner_publication_state: AtomicU8::new(SCANNER_PUBLICATION_STATE_UNKNOWN),
object_encryption_resolver: OnceLock::new(),
tier_delete_journal_recovery_stores: std::sync::Mutex::new(HashSet::new()),
@@ -249,6 +279,75 @@ impl InstanceContext {
Arc::clone(&self.data_movement_operation_gate)
}
pub(crate) async fn install_scanner_publication_lease(
&self,
token: Uuid,
expires_at: Instant,
movement_generation: u64,
operation_guard: OwnedRwLockReadGuard<()>,
) -> bool {
let mut leases = self.scanner_publication_leases.lock().await;
if leases.len() >= SCANNER_PUBLICATION_LEASE_MAX_ENTRIES {
return false;
}
leases.insert(
token,
ScannerPublicationLeaseEntry {
expires_at,
movement_generation,
_operation_guard: operation_guard,
},
);
true
}
pub(crate) async fn remove_scanner_publication_lease(&self, token: Uuid) -> bool {
self.scanner_publication_leases.lock().await.remove(&token).is_some()
}
/// Check a lease token while the caller holds the movement read guard.
///
/// The token table is deliberately process-owned and non-persistent: a
/// restarted instance has no entries from the previous process, so an old
/// coordinator proof cannot become valid again merely because the
/// movement generation counter restarted at zero.
pub(crate) async fn scanner_publication_lease_is_active(&self, token: Uuid) -> bool {
let mut leases = self.scanner_publication_leases.lock().await;
let now = Instant::now();
let Some(expires_at) = leases.get(&token).map(|entry| entry.expires_at) else {
return false;
};
if expires_at <= now {
leases.remove(&token);
return false;
}
true
}
/// Return the generation bound to a live lease. The lease entry owns the
/// movement read guard, so a successful lookup remains valid for the
/// caller's guard-protected operation; expiry is still fail-closed.
pub(crate) async fn scanner_publication_lease_generation(&self, token: Uuid) -> Option<u64> {
let mut leases = self.scanner_publication_leases.lock().await;
let now = Instant::now();
let (expires_at, movement_generation) = leases
.get(&token)
.map(|entry| (entry.expires_at, entry.movement_generation))?;
if expires_at <= now {
leases.remove(&token);
return None;
}
Some(movement_generation)
}
pub(crate) async fn expire_scanner_publication_lease(&self, token: Uuid, expires_at: Instant) {
let mut leases = self.scanner_publication_leases.lock().await;
let should_remove = leases.get(&token).is_some_and(|entry| entry.expires_at <= expires_at);
if should_remove {
leases.remove(&token);
}
}
pub(crate) fn data_movement_operation_epoch(&self) -> u64 {
self.data_movement_operation_epoch.load(Ordering::Acquire)
}
@@ -257,8 +356,21 @@ impl InstanceContext {
self.data_movement_operation_epoch_exhausted.load(Ordering::Acquire)
}
pub(crate) fn data_movement_generation(&self) -> u64 {
self.data_movement_generation.load(Ordering::Acquire)
}
pub(crate) fn data_movement_generation_exhausted(&self) -> bool {
self.data_movement_generation_exhausted.load(Ordering::Acquire)
}
pub(crate) fn data_movement_generation_notify(&self) -> Arc<Notify> {
Arc::clone(&self.data_movement_generation_notify)
}
pub(crate) fn scanner_publication_state_allowed(&self) -> bool {
!self.data_movement_operation_epoch_exhausted()
&& !self.data_movement_generation_exhausted()
&& self.scanner_publication_state.load(Ordering::Acquire) == SCANNER_PUBLICATION_STATE_ALLOWED
}
@@ -276,6 +388,7 @@ impl InstanceContext {
pub(crate) fn advance_data_movement_operation_epoch(&self) -> u64 {
self.scanner_publication_state
.store(SCANNER_PUBLICATION_STATE_UNKNOWN, Ordering::Release);
let previous = self.data_movement_operation_epoch.load(Ordering::Acquire);
let _ = self
.data_movement_operation_epoch
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |epoch| Some(epoch.saturating_add(1)));
@@ -283,9 +396,42 @@ impl InstanceContext {
if result == u64::MAX {
self.data_movement_operation_epoch_exhausted.store(true, Ordering::Release);
}
if result != previous {
let _ = self.advance_data_movement_generation();
}
result
}
/// Advance the movement generation after a durable movement transition.
/// The generation is deliberately bounded: once it reaches `u64::MAX`,
/// publication and generation-based waits fail closed rather than reusing
/// an indistinguishable saturated value.
pub(crate) fn advance_data_movement_generation(&self) -> Option<u64> {
if self.data_movement_generation_exhausted.load(Ordering::Acquire) {
return None;
}
let updated = self
.data_movement_generation
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |generation| generation.checked_add(1));
match updated {
Ok(previous) => {
let Some(generation) = previous.checked_add(1) else {
self.data_movement_generation_exhausted.store(true, Ordering::Release);
return None;
};
if generation == u64::MAX {
self.data_movement_generation_exhausted.store(true, Ordering::Release);
}
self.data_movement_generation_notify.notify_waiters();
Some(generation)
}
Err(_) => {
self.data_movement_generation_exhausted.store(true, Ordering::Release);
None
}
}
}
#[cfg(test)]
pub(crate) fn set_data_movement_operation_epoch_for_test(&self, epoch: u64) {
self.data_movement_operation_epoch.store(epoch, Ordering::Release);
@@ -295,6 +441,13 @@ impl InstanceContext {
.store(SCANNER_PUBLICATION_STATE_UNKNOWN, Ordering::Release);
}
#[cfg(test)]
pub(crate) fn set_data_movement_generation_for_test(&self, generation: u64) {
self.data_movement_generation.store(generation, Ordering::Release);
self.data_movement_generation_exhausted
.store(generation == u64::MAX, Ordering::Release);
}
/// Install the application-owned object-encryption resolver once.
pub fn set_object_encryption_resolver(
&self,
+137 -1
View File
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::cluster::rpc::{PeerRestClient, ScannerPeerActivity, TierConfigReloadOutcome};
use crate::cluster::rpc::{PeerRestClient, ScannerPeerActivity, ScannerPublicationLease, TierConfigReloadOutcome};
use crate::diagnostics::admin_server_info::get_commit_id;
use crate::disk::DiskAPI;
use crate::error::{Error, Result};
@@ -53,6 +53,12 @@ const REMOTE_VERSION_STATE_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
const REMOTE_VERSION_STATE_PROOF_TTL: Duration = Duration::from_secs(30);
const CROSS_POOL_FENCE_SUPPORTED_VERSION: u32 = 2;
#[derive(Clone, Debug)]
pub struct ScannerPublicationLeaseGrant {
pub host: String,
pub lease: ScannerPublicationLease,
}
/// Cached result from the last successful admin call to a peer.
struct PeerAdminCache {
last_storage_info: Option<StorageInfo>,
@@ -1491,6 +1497,102 @@ impl NotificationSys {
aggregate_scanner_dirty_usage_acknowledgement_results(join_all(futures).await, failures)
}
/// Acquire remote publication leases in a deterministic host order. A
/// missing/legacy peer is a hard publication deferral; already acquired
/// leases are released before returning so a partial acquisition cannot
/// pin movement on one peer.
pub async fn acquire_scanner_publication_leases(
&self,
mut targets: Vec<(String, String, u64)>,
) -> Result<Vec<ScannerPublicationLeaseGrant>> {
targets.sort_by(|left, right| left.0.cmp(&right.0));
for pair in targets.windows(2) {
if pair[0].0 == pair[1].0 {
return Err(Error::other(format!("duplicate scanner publication lease target: {}", pair[0].0)));
}
}
let mut grants = Vec::with_capacity(targets.len());
for (host, session_id, generation) in targets {
let Some(client) = self
.peer_clients
.iter()
.flatten()
.find(|client| client.grid_host == host)
.cloned()
else {
let _ = self.release_scanner_publication_leases(grants).await;
return Err(Error::other(format!("scanner publication lease peer {host} is unavailable")));
};
match client.acquire_scanner_publication_lease(&session_id, generation).await {
Ok(lease) => grants.push(ScannerPublicationLeaseGrant { host, lease }),
Err(err) => {
let _ = self.release_scanner_publication_leases(grants).await;
return Err(Error::other(format!("scanner publication lease acquisition failed: {err}")));
}
}
}
Ok(grants)
}
pub async fn release_scanner_publication_leases(&self, mut grants: Vec<ScannerPublicationLeaseGrant>) -> Result<()> {
grants.sort_by(|left, right| right.host.cmp(&left.host));
let mut failures = Vec::new();
for grant in grants {
let Some(client) = self
.peer_clients
.iter()
.flatten()
.find(|client| client.grid_host == grant.host)
else {
failures.push(format!("peer {} is unavailable", grant.host));
continue;
};
if let Err(err) = client.release_scanner_publication_lease(&grant.lease).await {
failures.push(format!("peer {} release failed: {err}", grant.host));
}
}
if failures.is_empty() {
Ok(())
} else {
Err(Error::other(format!(
"scanner publication lease release failures: {}",
failures.join("; ")
)))
}
}
/// Revalidate every remote lease in deterministic host order immediately
/// before a final scanner metadata write. A peer restart removes its
/// process-owned token table and changes its activity session, so an old
/// generation cannot pass this proof even when the numeric generation is
/// reused.
pub async fn validate_scanner_publication_leases(&self, grants: &[ScannerPublicationLeaseGrant]) -> Result<()> {
let mut grants = grants.to_vec();
grants.sort_by(|left, right| left.host.cmp(&right.host));
for pair in grants.windows(2) {
if pair[0].host == pair[1].host {
return Err(Error::other(format!("duplicate scanner publication lease target: {}", pair[0].host)));
}
}
for grant in grants {
let Some(client) = self
.peer_clients
.iter()
.flatten()
.find(|client| client.grid_host == grant.host)
.cloned()
else {
return Err(Error::other(format!("scanner publication lease peer {} is unavailable", grant.host)));
};
client
.validate_scanner_publication_lease(&grant.lease)
.await
.map_err(|err| Error::other(format!("scanner publication lease validation failed for {}: {err}", grant.host)))?;
}
Ok(())
}
pub async fn reload_site_replication_config(&self) -> Vec<NotificationPeerErr> {
let mut futures = Vec::with_capacity(self.peer_clients.len());
for client in self.peer_clients.iter() {
@@ -2633,6 +2735,38 @@ mod tests {
assert!(err.to_string().contains("no remote peers"));
}
#[tokio::test]
async fn scanner_publication_lease_release_reports_all_unavailable_peers() {
let sys = NotificationSys {
peer_clients: Vec::new(),
all_peer_clients: Vec::new(),
peer_topology_hosts: Vec::new(),
peer_admin_caches: Vec::new(),
tier_config_reload_workers: Default::default(),
};
let grants = ["peer-a", "peer-b"]
.into_iter()
.map(|host| ScannerPublicationLeaseGrant {
host: host.to_string(),
lease: ScannerPublicationLease {
token: Uuid::new_v4(),
movement_generation: 3,
owner_id: Uuid::new_v4().to_string(),
session_id: "session-a".to_string(),
expires_at: Instant::now() + Duration::from_secs(30),
},
})
.collect();
let error = sys
.release_scanner_publication_leases(grants)
.await
.expect_err("an unavailable peer must not silently release a remote lease");
let message = error.to_string();
assert!(message.contains("peer-a"));
assert!(message.contains("peer-b"));
}
#[tokio::test]
async fn scanner_activity_probe_rejects_an_incomplete_peer_topology() {
let client = PeerRestClient::new(
@@ -2755,6 +2889,8 @@ mod tests {
data_movement_active: Some(false),
dirty_usage_generation: Some(2),
dirty_usage_pending,
movement_generation: Some(1),
publication_blocked: Some(false),
};
let pending = aggregate_scanner_dirty_usage_acknowledgement_results(
+129 -11
View File
@@ -3499,6 +3499,26 @@ pub(in crate::set_disk) struct RenameDataCommit {
pub(in crate::set_disk) tail_drain: Option<tokio::task::JoinHandle<()>>,
}
/// Options shared by the normal and early-ack rename fanouts. Keeping the
/// quorum and optional scanner lease map together avoids widening either
/// fanout helper's argument list while preserving the fence semantics.
pub(in crate::set_disk) struct RenameDataFenceOptions<'a> {
write_quorum: usize,
scanner_publication_lease_tokens: Option<&'a HashMap<String, Uuid>>,
}
impl<'a> RenameDataFenceOptions<'a> {
pub(in crate::set_disk) fn new(
write_quorum: usize,
scanner_publication_lease_tokens: Option<&'a HashMap<String, Uuid>>,
) -> Self {
Self {
write_quorum,
scanner_publication_lease_tokens,
}
}
}
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
type RenameDataLegacyTuple = (
Vec<Option<DiskStore>>,
@@ -3690,16 +3710,51 @@ impl SetDisks {
.map(RenameDataCommit::into_legacy_tuple)
}
#[tracing::instrument(level = "debug", skip(disks, file_infos))]
async fn rename_data_owned_early_ack(
pub(in crate::set_disk) fn scanner_publication_lease_token_for_disk(
disk: Option<&DiskStore>,
scanner_publication_lease_tokens: Option<&HashMap<String, Uuid>>,
) -> disk::error::Result<Option<Uuid>> {
let Some(disk) = disk else {
return Ok(None);
};
if disk.is_local() {
return Ok(None);
}
let Some(tokens) = scanner_publication_lease_tokens else {
return Ok(None);
};
let host = disk.endpoint().grid_host();
tokens
.get(&host)
.copied()
.ok_or_else(|| DiskError::other(format!("scanner publication lease token missing for remote disk host {host}")))
.map(Some)
}
pub(in crate::set_disk) fn scanner_publication_lease_tokens_for_disks(
disks: &[Option<DiskStore>],
scanner_publication_lease_tokens: Option<&HashMap<String, Uuid>>,
) -> disk::error::Result<Vec<Option<Uuid>>> {
disks
.iter()
.map(|disk| Self::scanner_publication_lease_token_for_disk(disk.as_ref(), scanner_publication_lease_tokens))
.collect()
}
#[tracing::instrument(level = "debug", skip(disks, file_infos, fence_options))]
async fn rename_data_owned_early_ack_with_fence(
disks: &[Option<DiskStore>],
src_bucket: &str,
src_object: &str,
file_infos: Vec<FileInfo>,
dst_bucket: &str,
dst_object: &str,
write_quorum: usize,
fence_options: RenameDataFenceOptions<'_>,
) -> disk::error::Result<RenameDataCommit> {
let RenameDataFenceOptions {
write_quorum,
scanner_publication_lease_tokens,
} = fence_options;
if let Some(file_info) = disks
.iter()
.zip(file_infos.iter())
@@ -3714,6 +3769,7 @@ impl SetDisks {
let disk_count = disks.len();
let fanout_disks = disks.to_vec();
let fanout_fence_tokens = Self::scanner_publication_lease_tokens_for_disks(disks, scanner_publication_lease_tokens)?;
let coordinator_disks = fanout_disks.clone();
let src_bucket = Arc::new(src_bucket.to_string());
let src_object = Arc::new(src_object.to_string());
@@ -3730,7 +3786,12 @@ impl SetDisks {
let successful_rename_completion_rank =
rustfs_io_metrics::put_stage_metrics_enabled().then(|| Arc::new(AtomicUsize::new(0)));
let mut tasks = JoinSet::new();
for (i, (disk, file_info)) in fanout_disks.into_iter().zip(file_infos.iter()).enumerate() {
for (i, ((disk, file_info), scanner_publication_lease_token)) in fanout_disks
.into_iter()
.zip(file_infos.iter())
.zip(fanout_fence_tokens.into_iter())
.enumerate()
{
let src_bucket = fanout_src_bucket.clone();
let src_object = fanout_src_object.clone();
let dst_bucket = fanout_dst_bucket.clone();
@@ -3767,7 +3828,14 @@ impl SetDisks {
let disk_wait_started = rustfs_io_metrics::put_stage_timer();
let result = disk
.rename_data_borrowed(&src_bucket, &src_object, file_info, &dst_bucket, &dst_object)
.rename_data_borrowed_with_fence(
&src_bucket,
&src_object,
file_info,
&dst_bucket,
&dst_object,
scanner_publication_lease_token,
)
.await;
if let Some(disk_wait_started) = disk_wait_started {
let duration_ms = disk_wait_started.elapsed().as_secs_f64() * 1000.0;
@@ -3967,19 +4035,45 @@ impl SetDisks {
dst_bucket: &str,
dst_object: &str,
write_quorum: usize,
) -> disk::error::Result<RenameDataCommit> {
Self::rename_data_owned_with_fence(
disks,
src_bucket,
src_object,
file_infos,
dst_bucket,
dst_object,
RenameDataFenceOptions::new(write_quorum, None),
)
.await
}
#[tracing::instrument(level = "debug", skip(disks, file_infos, fence_options))]
pub(in crate::set_disk) async fn rename_data_owned_with_fence(
disks: &[Option<DiskStore>],
src_bucket: &str,
src_object: &str,
file_infos: Vec<FileInfo>,
dst_bucket: &str,
dst_object: &str,
fence_options: RenameDataFenceOptions<'_>,
) -> disk::error::Result<RenameDataCommit> {
if put_rename_early_ack_enabled() {
return Self::rename_data_owned_early_ack(
return Self::rename_data_owned_early_ack_with_fence(
disks,
src_bucket,
src_object,
file_infos,
dst_bucket,
dst_object,
write_quorum,
fence_options,
)
.await;
}
let RenameDataFenceOptions {
write_quorum,
scanner_publication_lease_tokens,
} = fence_options;
if let Some(file_info) = disks
.iter()
.zip(file_infos.iter())
@@ -4005,6 +4099,7 @@ impl SetDisks {
let disk_count = disks.len();
let fanout_disks = disks.to_vec();
let fanout_file_infos = file_infos;
let fanout_fence_tokens = Self::scanner_publication_lease_tokens_for_disks(disks, scanner_publication_lease_tokens)?;
let fanout_src_bucket = src_bucket.clone();
let fanout_src_object = src_object.clone();
let fanout_dst_bucket = dst_bucket.clone();
@@ -4019,8 +4114,9 @@ impl SetDisks {
let futures = fanout_disks
.into_iter()
.zip(fanout_file_infos.iter())
.zip(fanout_fence_tokens.into_iter())
.enumerate()
.map(|(i, (disk, file_info))| {
.map(|(i, ((disk, file_info), scanner_publication_lease_token))| {
let src_bucket = fanout_src_bucket.clone();
let src_object = fanout_src_object.clone();
let dst_object = fanout_dst_object.clone();
@@ -4060,7 +4156,14 @@ impl SetDisks {
let disk_wait_started = rustfs_io_metrics::put_stage_timer();
let result = disk
.rename_data_borrowed(&src_bucket, &src_object, file_info, &dst_bucket, &dst_object)
.rename_data_borrowed_with_fence(
&src_bucket,
&src_object,
file_info,
&dst_bucket,
&dst_object,
scanner_publication_lease_token,
)
.await;
if let Some(disk_wait_started) = disk_wait_started {
let duration_ms = disk_wait_started.elapsed().as_secs_f64() * 1000.0;
@@ -5426,18 +5529,32 @@ impl SetDisks {
Ok(())
}
#[cfg(test)]
pub(in crate::set_disk) async fn delete_prefix(&self, bucket: &str, prefix: &str) -> disk::error::Result<()> {
self.delete_prefix_with_scanner_publication_lease(bucket, prefix, None).await
}
/// Delete a prefix with an optional per-remote-disk scanner publication
/// lease fence. The fence is transient and never reaches disk metadata;
/// remote deletes without a matching token fail closed before the RPC.
pub(in crate::set_disk) async fn delete_prefix_with_scanner_publication_lease(
&self,
bucket: &str,
prefix: &str,
scanner_publication_lease_tokens: Option<&HashMap<String, Uuid>>,
) -> disk::error::Result<()> {
let disks = self.get_disks_internal().await;
let write_quorum = disks.len() / 2 + 1;
let fanout_fence_tokens = Self::scanner_publication_lease_tokens_for_disks(&disks, scanner_publication_lease_tokens)?;
let mut futures = Vec::with_capacity(disks.len());
for disk_op in disks.iter() {
for (disk_op, scanner_publication_lease_token) in disks.iter().zip(fanout_fence_tokens) {
let bucket = bucket.to_string();
let prefix = prefix.to_string();
futures.push(async move {
if let Some(disk) = disk_op {
disk.delete(
disk.delete_with_scanner_publication_lease(
&bucket,
&prefix,
DeleteOptions {
@@ -5445,6 +5562,7 @@ impl SetDisks {
immediate: true,
..Default::default()
},
scanner_publication_lease_token,
)
.await
} else {
+67 -4
View File
@@ -49,8 +49,8 @@ use crate::data_usage::quota_object_size;
use crate::diagnostics::get::GetObjectFailureReason;
use crate::disk::{DataDirDeleteStatus, OldCurrentSize};
use crate::error::is_err_invalid_upload_id;
use crate::object_api::NamespaceLockFence;
use crate::object_api::{GetObjectBodySource, get_object_body_cache_hook_suppressed};
use crate::object_api::{NamespaceLockFence, SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY};
use crate::services::notification_sys::RemoteVersionStateFleetProofToken;
use crate::services::tier::tier::{TierConfigMgr, TierOperationLease};
use crate::store::ECStore;
@@ -63,6 +63,60 @@ use std::sync::OnceLock;
use tokio_util::sync::CancellationToken;
const OLD_DATA_CLEANUP_RECEIPT_FILE: &str = ".rustfs-old-data-cleanup-receipt.json";
const SCANNER_PUBLICATION_LEASE_FENCE_MAX_BYTES: usize = 64 * 1024;
const SCANNER_PUBLICATION_LEASE_FENCE_MAX_ENTRIES: usize = 256;
fn take_scanner_publication_lease_tokens(user_defined: &mut HashMap<String, String>) -> Result<Option<HashMap<String, Uuid>>> {
let Some(encoded) = user_defined.remove(SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY) else {
return Ok(None);
};
if encoded.len() > SCANNER_PUBLICATION_LEASE_FENCE_MAX_BYTES {
return Err(Error::other("scanner publication lease fence is too large"));
}
let encoded_tokens = serde_json::from_str::<HashMap<String, String>>(&encoded)
.map_err(|err| Error::other(format!("invalid scanner publication lease fence: {err}")))?;
if encoded_tokens.len() > SCANNER_PUBLICATION_LEASE_FENCE_MAX_ENTRIES {
return Err(Error::other("scanner publication lease fence has too many entries"));
}
let mut tokens = HashMap::with_capacity(encoded_tokens.len());
for (host, token) in encoded_tokens {
if host.is_empty() || host.len() > 1024 {
return Err(Error::other("invalid scanner publication lease fence host"));
}
let token = Uuid::parse_str(&token)
.map_err(|err| Error::other(format!("invalid scanner publication lease fence token: {err}")))?;
if token.is_nil() {
return Err(Error::other("invalid scanner publication lease fence token"));
}
tokens.insert(host, token);
}
Ok(Some(tokens))
}
#[cfg(test)]
mod scanner_publication_lease_fence_tests {
use super::*;
#[test]
fn scanner_publication_lease_fence_is_transient_and_validated() {
let token = Uuid::new_v4();
let mut metadata = HashMap::from([(
SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY.to_string(),
serde_json::json!({"http://node-a:9000": token.to_string()}).to_string(),
)]);
let parsed = take_scanner_publication_lease_tokens(&mut metadata)
.expect("valid scanner publication lease fence should parse")
.expect("scanner publication lease fence should be present");
assert_eq!(parsed.get("http://node-a:9000"), Some(&token));
assert!(!metadata.contains_key(SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY));
let mut malformed = HashMap::from([(
SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY.to_string(),
r#"{"http://node-a:9000":"not-a-uuid"}"#.to_string(),
)]);
assert!(take_scanner_publication_lease_tokens(&mut malformed).is_err());
}
}
struct PutObjectCommitCancellation {
token: CancellationToken,
@@ -2177,6 +2231,7 @@ impl SetDisks {
user_defined.insert(key.clone(), value.clone());
}
}
let scanner_publication_lease_tokens = take_scanner_publication_lease_tokens(&mut user_defined)?;
if replication_lww_applicable(opts) {
// Object Lock evaluation stamps category timestamps with this
// receiver's clock. Pin them back to the source-authored times
@@ -2896,6 +2951,7 @@ impl SetDisks {
let commit_bucket_lifecycle_lock_fence = opts.bucket_lifecycle_lock_fence.clone();
let commit_capacity_scope_token = opts.capacity_scope_token;
let commit_replication_state = replication_state_to_filemeta(&opts.put_replication_state());
let commit_scanner_publication_lease_tokens = scanner_publication_lease_tokens;
tmp_cleanup_owned = true;
let commit = move |cancellation: Option<CancellationToken>| async move {
@@ -3015,14 +3071,17 @@ impl SetDisks {
}
Self::assign_rename_data_indexes(&mut parts_metadatas);
let rename_result = SetDisks::rename_data_owned(
let rename_result = SetDisks::rename_data_owned_with_fence(
&commit_disks,
RUSTFS_META_TMP_BUCKET,
commit_tmp_dir.as_str(),
parts_metadatas,
&commit_bucket,
&commit_object,
write_quorum,
crate::set_disk::core::io_primitives::RenameDataFenceOptions::new(
write_quorum,
commit_scanner_publication_lease_tokens.as_ref(),
),
)
.await;
if quota_mutation_fence {
@@ -6418,6 +6477,10 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
#[tracing::instrument(skip(self))]
async fn delete_object(&self, bucket: &str, object: &str, mut opts: ObjectOptions) -> Result<ObjectInfo> {
// Scanner cleanup carries the per-peer lease fence as transient
// request metadata. Consume it before any delete-prefix fanout so it
// cannot be persisted or treated as user metadata.
let scanner_publication_lease_tokens = take_scanner_publication_lease_tokens(&mut opts.user_defined)?;
let preserve_delete_replication_state = should_preserve_delete_replication_state(&opts);
let delete_config_snapshot = if opts.delete_prefix || opts.transition.expire_restored || preserve_delete_replication_state
{
@@ -6544,7 +6607,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
self.validate_bucket_incarnation(bucket, expected_incarnation_id).await?;
}
ensure_delete_commit_locks_held(_lock_guard.as_ref(), bucket, object, &opts)?;
self.delete_prefix(bucket, object)
self.delete_prefix_with_scanner_publication_lease(bucket, object, scanner_publication_lease_tokens.as_ref())
.await
.map_err(|e| to_object_err(e.into(), vec![bucket, object]))?;
@@ -32,7 +32,8 @@ pub(crate) mod internode {
PUT_FILE_AUTH_TRAILER_MAGIC, PUT_FILE_AUTH_V1, PUT_FILE_CAPABILITY_CHALLENGE_QUERY, PUT_FILE_CAPABILITY_QUERY,
PUT_FILE_CAPABILITY_VERSION, PUT_FILE_NONCE_QUERY, PUT_FILE_SERVER_EPOCH_QUERY, PutFileCapabilityResponse,
SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION, SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SCANNER_ACTIVITY_PROTOCOL_VERSION,
WALK_DIR_BODY_SHA256_QUERY, WALK_DIR_STREAM_COMPLETION_QUERY, WALK_DIR_STREAM_COMPLETION_V1,
SCANNER_ACTIVITY_V6_PROTOCOL_VERSION, WALK_DIR_BODY_SHA256_QUERY, WALK_DIR_STREAM_COMPLETION_QUERY,
WALK_DIR_STREAM_COMPLETION_V1,
};
}
+273 -4
View File
@@ -87,6 +87,8 @@ type ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>;
type WalkOptions = StorageWalkOptions<fn(&FileInfo) -> bool>;
pub const SCANNER_PUBLICATION_LEASE_TTL_MS: u64 = 60_000;
/// Check if a directory contains any xl.meta files (indicating actual S3 objects)
/// This is used to determine if a bucket is empty for deletion purposes.
pub(crate) async fn has_xlmeta_files(path: &std::path::Path) -> std::io::Result<bool> {
@@ -344,6 +346,33 @@ impl ECStore {
decommission || rebalance
}
/// Return the storage-owned movement state and generation as one
/// authenticated activity snapshot. The read lock is acquired before
/// the state locks (cancelers, pool metadata, then rebalance metadata),
/// matching the transition writer order and preventing a terminal state
/// from being reported with the preceding generation.
pub async fn scanner_data_movement_activity(&self) -> (bool, bool, u64) {
let operation_gate = self.ctx.data_movement_operation_gate();
let _operation_guard = operation_gate.read_owned().await;
let (active, blocked) = self.scanner_data_movement_snapshot_locked().await;
let blocked =
blocked || self.ctx.data_movement_operation_epoch_exhausted() || self.ctx.data_movement_generation_exhausted();
self.ctx.set_scanner_publication_state(blocked);
(active, blocked, self.ctx.data_movement_generation())
}
pub fn scanner_data_movement_generation(&self) -> u64 {
self.ctx.data_movement_generation()
}
pub fn scanner_data_movement_generation_exhausted(&self) -> bool {
self.ctx.data_movement_generation_exhausted()
}
pub fn scanner_data_movement_changed(&self) -> std::sync::Arc<tokio::sync::Notify> {
self.ctx.data_movement_generation_notify()
}
/// Returns whether scanner metadata may still be hidden by a local
/// data-movement state. Terminal failed/canceled decommission entries
/// remain suspended until an operator clears or retries them, so they are
@@ -355,10 +384,16 @@ impl ECStore {
}
async fn scanner_data_usage_publication_snapshot_blocked(&self) -> bool {
if self.ctx.data_movement_operation_epoch_exhausted() {
if self.ctx.data_movement_operation_epoch_exhausted() || self.ctx.data_movement_generation_exhausted() {
self.ctx.set_scanner_publication_state(true);
return true;
}
let (_, blocked) = self.scanner_data_movement_snapshot_locked().await;
self.ctx.set_scanner_publication_state(blocked);
blocked
}
async fn scanner_data_movement_snapshot_locked(&self) -> (bool, bool) {
let decommission_cancelers = self.decommission_cancelers.read().await;
let decommission_active = decommission_cancelers
.iter()
@@ -385,8 +420,7 @@ impl ECStore {
.is_some_and(is_rebalance_conflicting_with_decommission);
let blocked = decommission_active || decommission_terminal || rebalance_active;
self.ctx.set_scanner_publication_state(blocked);
blocked
(decommission_active || rebalance_active, blocked)
}
/// Admit one short data-usage publication commit under the same
@@ -407,7 +441,7 @@ impl ECStore {
pub async fn scanner_data_usage_publication_admission_guard(&self) -> Option<(tokio::sync::OwnedRwLockReadGuard<()>, u64)> {
let operation_gate = self.ctx.data_movement_operation_gate();
let operation_guard = operation_gate.read_owned().await;
if self.ctx.data_movement_operation_epoch_exhausted() {
if self.ctx.data_movement_operation_epoch_exhausted() || self.ctx.data_movement_generation_exhausted() {
return None;
}
if self.scanner_data_usage_publication_snapshot_blocked().await {
@@ -425,6 +459,98 @@ impl ECStore {
drop(operation_guard);
Some(epoch)
}
/// Acquire a storage-owned read admission for a coordinator's final
/// scanner publication. The guard remains in the context's lease table,
/// so a local movement writer cannot pass the peer while its authoritative
/// PUT is in flight.
pub async fn acquire_scanner_publication_lease(
&self,
expected_generation: u64,
ttl: std::time::Duration,
) -> Result<(Uuid, u64)> {
if ttl != crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL {
return Err(Error::other("scanner publication lease TTL is not supported"));
}
let operation_gate = self.ctx.data_movement_operation_gate();
let operation_guard = operation_gate.read_owned().await;
if self.ctx.data_movement_generation_exhausted()
|| self.ctx.data_movement_operation_epoch_exhausted()
|| self.ctx.data_movement_generation() != expected_generation
{
return Err(Error::other("scanner publication lease generation is stale"));
}
if self.scanner_data_movement_snapshot_locked().await.1 {
return Err(Error::other("scanner publication lease is blocked by data movement"));
}
let token = Uuid::new_v4();
let expires_at = tokio::time::Instant::now() + ttl;
if !self
.ctx
.install_scanner_publication_lease(token, expires_at, expected_generation, operation_guard)
.await
{
return Err(Error::other("scanner publication lease capacity is exhausted"));
}
let context = Arc::clone(&self.ctx);
tokio::spawn(async move {
tokio::time::sleep_until(expires_at).await;
context.expire_scanner_publication_lease(token, expires_at).await;
});
Ok((token, expected_generation))
}
pub async fn release_scanner_publication_lease(&self, token: Uuid) -> bool {
self.ctx.remove_scanner_publication_lease(token).await
}
/// Revalidate a previously acquired remote publication lease immediately
/// before the coordinator's final metadata write. The operation read
/// guard makes the movement snapshot and token lookup one storage-owned
/// admission; a restarted context has no old token and therefore fails
/// closed even if its generation counter has returned to zero.
pub async fn validate_scanner_publication_lease(&self, token: Uuid, expected_generation: u64) -> Result<()> {
let _operation_guard = self.acquire_scanner_publication_lease_guard(token).await?;
if self.ctx.data_movement_generation_exhausted()
|| self.ctx.data_movement_operation_epoch_exhausted()
|| self.ctx.data_movement_generation() != expected_generation
{
return Err(Error::other("scanner publication lease generation is stale"));
}
if self.scanner_data_movement_snapshot_locked().await.1 {
return Err(Error::other("scanner publication lease is blocked by data movement"));
}
if !self.ctx.scanner_publication_lease_is_active(token).await {
return Err(Error::other("scanner publication lease is unknown or expired"));
}
Ok(())
}
/// Acquire the target-side read guard bound to a previously granted lease.
/// The guard is returned to the RPC handler and must remain alive through
/// the complete rename/write operation. A lease token is process-owned;
/// restart, expiry, generation changes, or blocked movement all reject it
/// before the target disk is touched.
pub async fn acquire_scanner_publication_lease_guard(&self, token: Uuid) -> Result<tokio::sync::OwnedRwLockReadGuard<()>> {
let operation_gate = self.ctx.data_movement_operation_gate();
let operation_guard = operation_gate.read_owned().await;
if self.ctx.data_movement_generation_exhausted() || self.ctx.data_movement_operation_epoch_exhausted() {
return Err(Error::other("scanner publication lease generation is exhausted"));
}
if self.scanner_data_movement_snapshot_locked().await.1 {
return Err(Error::other("scanner publication lease is blocked by data movement"));
}
let Some(lease_generation) = self.ctx.scanner_publication_lease_generation(token).await else {
return Err(Error::other("scanner publication lease is unknown or expired"));
};
if lease_generation != self.ctx.data_movement_generation() {
return Err(Error::other("scanner publication lease generation is stale"));
}
Ok(operation_guard)
}
}
// impl Clone for ECStore {
@@ -1084,6 +1210,149 @@ mod tests {
.await
.expect("idle store should admit the next publication");
assert_eq!(next_epoch, 1);
assert_eq!(store.scanner_data_movement_generation(), 1);
}
#[tokio::test(start_paused = true)]
async fn scanner_publication_lease_blocks_movement_writer_until_release_or_expiry() {
let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
let (token, generation) = store
.acquire_scanner_publication_lease(0, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
.await
.expect("an idle store should grant a publication lease");
assert_eq!(generation, 0);
let gate = store.ctx.data_movement_operation_gate();
let (writer_started, writer_started_rx) = tokio::sync::oneshot::channel();
let movement_writer = tokio::spawn(async move {
let _ = writer_started.send(());
gate.write_owned().await
});
writer_started_rx
.await
.expect("movement writer should reach the gate before waiting");
assert!(
!movement_writer.is_finished(),
"a movement writer must wait while the remote lease owns the read guard"
);
assert!(store.release_scanner_publication_lease(token).await);
tokio::time::timeout(Duration::from_secs(1), movement_writer)
.await
.expect("movement writer should proceed after lease release")
.expect("movement writer task should not panic");
let (expiring_token, _) = store
.acquire_scanner_publication_lease(0, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
.await
.expect("the store should grant a second publication lease");
let expiry_gate = store.ctx.data_movement_operation_gate();
let (expiry_started, expiry_started_rx) = tokio::sync::oneshot::channel();
let mut expiry_writer = tokio::spawn(async move {
let _ = expiry_started.send(());
expiry_gate.write_owned().await
});
expiry_started_rx
.await
.expect("expiry writer should reach the gate before waiting");
assert!(!expiry_writer.is_finished());
tokio::time::advance(crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL + Duration::from_millis(1)).await;
tokio::task::yield_now().await;
tokio::time::timeout(Duration::from_secs(1), &mut expiry_writer)
.await
.expect("movement writer should proceed after lease expiry")
.expect("expiry writer task should not panic");
assert!(!store.release_scanner_publication_lease(expiring_token).await);
}
#[tokio::test]
async fn scanner_publication_lease_rejects_stale_generation_before_install() {
let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
let error = store
.acquire_scanner_publication_lease(1, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
.await
.expect_err("a stale movement generation must not acquire a lease");
assert!(error.to_string().contains("generation is stale"));
}
#[tokio::test]
async fn scanner_target_guard_keeps_movement_writer_fenced_after_lease_release() {
let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
let (token, _) = store
.acquire_scanner_publication_lease(0, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
.await
.expect("an idle store should grant a publication lease");
let target_guard = store
.acquire_scanner_publication_lease_guard(token)
.await
.expect("the target-side rename should acquire its short guard");
assert!(store.release_scanner_publication_lease(token).await);
let movement_gate = store.ctx.data_movement_operation_gate();
let movement_writer = tokio::spawn(async move { movement_gate.write_owned().await });
tokio::task::yield_now().await;
assert!(!movement_writer.is_finished(), "the target guard must span the rename operation");
drop(target_guard);
tokio::time::timeout(std::time::Duration::from_secs(1), movement_writer)
.await
.expect("movement writer should proceed after the target rename guard is dropped")
.expect("movement writer task should not panic");
}
#[tokio::test]
async fn scanner_publication_lease_rejects_restart_aba_token() {
let first_store = build_store_with_ctx(Arc::new(InstanceContext::new()));
let (token, generation) = first_store
.acquire_scanner_publication_lease(0, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
.await
.expect("the initial instance should grant a publication lease");
first_store
.validate_scanner_publication_lease(token, generation)
.await
.expect("the current instance should validate its own live token");
first_store
.acquire_scanner_publication_lease_guard(token)
.await
.expect("the current instance should admit the target-side rename");
// A restarted storage instance starts its local generation at zero,
// but its process-owned lease table is empty. The old token must not
// pass validation just because the numeric generation matches again.
let restarted_store = build_store_with_ctx(Arc::new(InstanceContext::new()));
let error = restarted_store
.validate_scanner_publication_lease(token, generation)
.await
.expect_err("a token from a prior instance must fail closed after restart");
assert!(error.to_string().contains("unknown or expired"));
let error = restarted_store
.acquire_scanner_publication_lease_guard(token)
.await
.expect_err("a restarted instance must reject the target-side rename token");
assert!(error.to_string().contains("unknown or expired"));
}
#[tokio::test]
async fn movement_generation_notifies_waiters_and_fails_closed_at_maximum() {
let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
let notify = store.scanner_data_movement_changed();
let notified = notify.notified();
tokio::pin!(notified);
notified.as_mut().enable();
assert_eq!(store.ctx.advance_data_movement_operation_epoch(), 1);
tokio::time::timeout(std::time::Duration::from_secs(1), notified)
.await
.expect("movement transition should wake scanner waiters");
assert_eq!(store.scanner_data_movement_generation(), 1);
store.ctx.set_data_movement_generation_for_test(u64::MAX - 1);
assert_eq!(store.ctx.advance_data_movement_generation(), Some(u64::MAX));
assert!(store.scanner_data_movement_generation_exhausted());
assert_eq!(store.ctx.advance_data_movement_generation(), None);
assert!(
store.scanner_data_usage_publication_admission_guard().await.is_none(),
"generation exhaustion must close publication admission"
);
}
#[tokio::test]