mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-25 05:26:50 +00:00
fix(scanner): fence movement generation publication (#6461)
* feat(scanner): add movement generation fencing * fix(scanner): prioritize unverified cycle deferral * feat(ecstore): add scanner publication lease fence * feat(rpc): add scanner publication lease protocol * feat(scanner): hold remote leases through usage publish * test(scanner): cover publication lease fencing * fix(scanner): fence remote leases across restart and delay * feat(rpc): fence scanner publication rename writes * fix(scanner): fence observed cleanup deletes * fix(proto): qualify lease release test types * fix(scanner): pin movement notifications * fix(scanner): clean publication imports * fix(ecstore): satisfy scanner fence clippy * refactor(scanner): group wait and publication options * fix(scanner): satisfy final lint and facade guards * fix(rpc): resolve facade export conflicts * fix(ci): remove unused decommission and healing facades * fix(ci): cfg-gate test-only usage overlay import * fix(scanner): wake on remote scanner restart
This commit is contained in:
@@ -415,9 +415,9 @@ pub mod notification {
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub use crate::services::notification_sys::rotate_cross_pool_fence_fleet_proof_for_test;
|
||||
pub use crate::services::notification_sys::{
|
||||
CrossPoolFenceFleetProofToken, NotificationPeerErr, NotificationSys, acquire_cross_pool_fence_fleet_proof,
|
||||
cross_pool_fence_fleet_proof_matches, get_global_notification_sys, new_global_notification_sys,
|
||||
start_remote_version_state_fleet_probe,
|
||||
CrossPoolFenceFleetProofToken, NotificationPeerErr, NotificationSys, ScannerPublicationLeaseGrant,
|
||||
acquire_cross_pool_fence_fleet_proof, cross_pool_fence_fleet_proof_matches, get_global_notification_sys,
|
||||
new_global_notification_sys, start_remote_version_state_fleet_probe,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -426,9 +426,10 @@ pub mod object {
|
||||
BLOCK_SIZE_V2, ERASURE_ALGORITHM, EncryptionResolutionError, EncryptionResolutionErrorKind, GetObjectBodyCacheHook,
|
||||
GetObjectBodyCacheHookLookup, GetObjectBodySource, GetObjectReader, NamespaceLockFence, ObjectEncryptionResolver,
|
||||
ObjectInfo, ObjectLockConfigSnapshot, ObjectMutationHook, ObjectOptions, PutObjReader, QuotaAdmission,
|
||||
RangedDecompressReader, ReadEncryptionMaterial, ReadEncryptionMode, ReadEncryptionRequest, StreamConsumer,
|
||||
get_object_body_cache_plaintext_len, lookup_get_object_body_cache_hook, register_get_object_body_cache_hook,
|
||||
register_object_mutation_hook, unregister_get_object_body_cache_hook, unregister_object_mutation_hook,
|
||||
RangedDecompressReader, ReadEncryptionMaterial, ReadEncryptionMode, ReadEncryptionRequest,
|
||||
SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY, StreamConsumer, get_object_body_cache_plaintext_len,
|
||||
lookup_get_object_body_cache_hook, register_get_object_body_cache_hook, register_object_mutation_hook,
|
||||
unregister_get_object_body_cache_hook, unregister_object_mutation_hook,
|
||||
};
|
||||
pub use crate::store::{
|
||||
PrepareSelectObjectSnapshotError, PreparedGetObjectReader, SelectObjectSnapshot, SelectObjectSnapshotReadError,
|
||||
@@ -461,16 +462,17 @@ pub mod rpc {
|
||||
pub use crate::cluster::rpc::{
|
||||
AuthenticatedChannel, KMS_SIGNAL_SUBSYSTEM, LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS,
|
||||
PeerRestClient, PeerS3Client, S3PeerSys, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC,
|
||||
ScannerBucketListing, ScannerPeerActivity, TONIC_RPC_PREFIX, TonicInterceptor, build_put_file_auth_trailer,
|
||||
check_and_record_signed_rpc_nonce, decode_heal_bucket_rpc_options, encode_heal_bucket_rpc_options, gen_signature_headers,
|
||||
gen_tonic_replay_scope_headers, gen_tonic_signature_headers, gen_tonic_signature_interceptor,
|
||||
node_service_time_out_client, node_service_time_out_client_no_auth, normalize_tonic_rpc_audience,
|
||||
set_tonic_canonical_body_digest, sign_ns_scanner_capability, sign_ns_scanner_capability_with_tier_registry_generation,
|
||||
sign_put_file_capability, sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers,
|
||||
tonic_rpc_auth_failure_reason, verify_ns_scanner_capability, verify_ns_scanner_capability_with_tier_registry_generation,
|
||||
verify_put_file_auth_trailer, verify_put_file_capability, verify_rpc_signature, verify_tonic_boot_epoch_response,
|
||||
verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof,
|
||||
verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
|
||||
ScannerBucketListing, ScannerPeerActivity, ScannerPublicationLease, TONIC_RPC_PREFIX, TonicInterceptor,
|
||||
build_put_file_auth_trailer, check_and_record_signed_rpc_nonce, decode_heal_bucket_rpc_options,
|
||||
encode_heal_bucket_rpc_options, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
|
||||
gen_tonic_signature_interceptor, node_service_time_out_client, node_service_time_out_client_no_auth,
|
||||
normalize_tonic_rpc_audience, set_tonic_canonical_body_digest, sign_ns_scanner_capability,
|
||||
sign_ns_scanner_capability_with_tier_registry_generation, sign_put_file_capability, sign_tonic_rpc_response_proof,
|
||||
tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason,
|
||||
verify_ns_scanner_capability, verify_ns_scanner_capability_with_tier_registry_generation, verify_put_file_auth_trailer,
|
||||
verify_put_file_capability, verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
|
||||
verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
|
||||
verify_tonic_rpc_signature_with_bootstrap,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -497,7 +499,7 @@ pub mod storage {
|
||||
pub use crate::core::pools::HealLifecycleExpiryContext;
|
||||
pub use crate::store::HealWalkVersion;
|
||||
pub use crate::store::{
|
||||
ECStore, all_local_disk, all_local_disk_path, find_local_disk_by_ref, init_local_disks,
|
||||
ECStore, SCANNER_PUBLICATION_LEASE_TTL_MS, all_local_disk, all_local_disk_path, find_local_disk_by_ref, init_local_disks,
|
||||
init_local_disks_with_instance_ctx, init_lock_clients, prewarm_local_disk_id_map,
|
||||
prewarm_local_disk_id_map_with_instance_ctx,
|
||||
};
|
||||
|
||||
@@ -2994,6 +2994,7 @@ mod tests {
|
||||
dst_volume: "bucket".to_string(),
|
||||
dst_path: "object".to_string(),
|
||||
file_info_bin: vec![0x81, 0xA1, 0x76, 0x01].into(),
|
||||
scanner_publication_lease_token: Vec::new().into(),
|
||||
};
|
||||
let body = rustfs_protos::canonical_rename_data_request_body(&message).expect("small request should encode");
|
||||
let mut request = tonic::Request::new(());
|
||||
|
||||
@@ -48,7 +48,7 @@ pub use internode_data_transport::build_internode_data_transport_from_env;
|
||||
pub(crate) use peer_rest_client::TierConfigReloadOutcome;
|
||||
pub use peer_rest_client::{
|
||||
KMS_SIGNAL_SUBSYSTEM, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, SERVICE_SIGNAL_REFRESH_CONFIG,
|
||||
SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerPeerActivity,
|
||||
SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerPeerActivity, ScannerPublicationLease,
|
||||
};
|
||||
pub(crate) use peer_s3_client::heal_bucket_local_on_disks;
|
||||
pub use peer_s3_client::{
|
||||
|
||||
@@ -20,6 +20,7 @@ use crate::cluster::rpc::{set_tonic_canonical_body_digest, set_tonic_mutation_bo
|
||||
use crate::error::{Error, Result};
|
||||
use crate::storage_api_contracts::internode::{
|
||||
SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION, SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SCANNER_ACTIVITY_PROTOCOL_VERSION,
|
||||
SCANNER_ACTIVITY_V6_PROTOCOL_VERSION,
|
||||
};
|
||||
use crate::{
|
||||
bucket::replication::BucketStats,
|
||||
@@ -45,8 +46,9 @@ use rustfs_protos::proto_gen::node_service::{
|
||||
HealControlRequest, LoadBucketMetadataRequest, LoadGroupRequest, LoadPolicyMappingRequest, LoadPolicyRequest,
|
||||
LoadRebalanceMetaRequest, LoadServiceAccountRequest, LoadTransitionTierConfigRequest, LoadUserRequest,
|
||||
LocalStorageInfoRequest, Mss, ReloadPoolMetaRequest, ReloadSiteReplicationConfigRequest, ReplacementRecoveryStatusRequest,
|
||||
ScannerActivityRequest, ScannerActivityResponse, ServerInfoRequest, SignalServiceRequest, SignalServiceResponse,
|
||||
StartDecommissionRequest, StartProfilingRequest, StopRebalanceRequest, TierMutationAbortRequest, TierMutationCommitRequest,
|
||||
ScannerActivityRequest, ScannerActivityResponse, ScannerPublicationLeaseReleaseRequest, ScannerPublicationLeaseRequest,
|
||||
ScannerPublicationLeaseResponse, ServerInfoRequest, SignalServiceRequest, SignalServiceResponse, StartDecommissionRequest,
|
||||
StartProfilingRequest, StopRebalanceRequest, TierMutationAbortRequest, TierMutationCommitRequest,
|
||||
TierMutationControlResponse, TierMutationPeerState, TierMutationPrepareRequest, node_service_client::NodeServiceClient,
|
||||
tier_mutation_control_service_client::TierMutationControlServiceClient,
|
||||
};
|
||||
@@ -84,6 +86,11 @@ const HEAL_CONTROL_PAYLOAD_MAX_SIZE: usize = 64 * 1024;
|
||||
const PEER_REST_RECOVERY_MAX_ATTEMPTS: u32 = 60;
|
||||
const PEER_REST_RECOVERY_MAX_BACKOFF: Duration = Duration::from_secs(30);
|
||||
const SCANNER_ACTIVITY_MAX_MESSAGE_SIZE: usize = 1024;
|
||||
/// Reserve time for the acquire response's network/clock uncertainty. The
|
||||
/// server owns the real expiry; this local deadline is intentionally earlier
|
||||
/// so a coordinator never starts a bounded persistence operation at the edge
|
||||
/// of a remote lease.
|
||||
const SCANNER_PUBLICATION_LEASE_SAFETY_MARGIN: Duration = Duration::from_secs(5);
|
||||
const REPLICATION_STATS_MAX_MESSAGE_SIZE: usize = 8 * 1024 * 1024;
|
||||
const BUCKET_METADATA_RELOAD_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
|
||||
@@ -150,6 +157,8 @@ pub struct ScannerPeerActivity {
|
||||
pub data_movement_active: Option<bool>,
|
||||
pub dirty_usage_generation: Option<u64>,
|
||||
pub dirty_usage_pending: Option<bool>,
|
||||
pub movement_generation: Option<u64>,
|
||||
pub publication_blocked: Option<bool>,
|
||||
}
|
||||
|
||||
fn decode_scanner_activity_with_verifier(
|
||||
@@ -166,7 +175,14 @@ fn decode_scanner_activity_with_verifier(
|
||||
{
|
||||
return Err(Error::other("peer returned an invalid scanner activity instance ID"));
|
||||
}
|
||||
let (topology_digest, data_movement_active, dirty_usage_generation, dirty_usage_pending) = match response.protocol_version {
|
||||
let (
|
||||
topology_digest,
|
||||
data_movement_active,
|
||||
dirty_usage_generation,
|
||||
dirty_usage_pending,
|
||||
movement_generation,
|
||||
publication_blocked,
|
||||
) = match response.protocol_version {
|
||||
// RUSTFS_COMPAT_TODO(ns-scanner-rpc-v3): legacy response fields are unauthenticated. Remove after protocol v0 peers are unsupported.
|
||||
SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION
|
||||
if response.topology_digest.is_empty()
|
||||
@@ -175,7 +191,7 @@ fn decode_scanner_activity_with_verifier(
|
||||
&& response.dirty_usage_generation == 0
|
||||
&& !response.dirty_usage_pending =>
|
||||
{
|
||||
(None, None, None, None)
|
||||
(None, None, None, None, None, None)
|
||||
}
|
||||
SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION => {
|
||||
return Err(Error::other("legacy scanner activity peer returned unexpected extended fields"));
|
||||
@@ -198,9 +214,11 @@ fn decode_scanner_activity_with_verifier(
|
||||
Some(response.data_movement_active),
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
}
|
||||
SCANNER_ACTIVITY_PROTOCOL_VERSION => {
|
||||
SCANNER_ACTIVITY_V6_PROTOCOL_VERSION => {
|
||||
if response.dirty_usage_pending && response.dirty_usage_generation == 0 {
|
||||
return Err(Error::other("scanner activity peer returned pending dirty usage without a generation"));
|
||||
}
|
||||
@@ -218,11 +236,42 @@ fn decode_scanner_activity_with_verifier(
|
||||
Some(response.data_movement_active),
|
||||
Some(response.dirty_usage_generation),
|
||||
Some(response.dirty_usage_pending),
|
||||
None,
|
||||
None,
|
||||
)
|
||||
}
|
||||
version => {
|
||||
return Err(Error::other(format!("peer returned unsupported scanner activity protocol {version}")));
|
||||
SCANNER_ACTIVITY_PROTOCOL_VERSION => {
|
||||
if response.dirty_usage_pending && response.dirty_usage_generation == 0 {
|
||||
return Err(Error::other("scanner activity peer returned pending dirty usage without a generation"));
|
||||
}
|
||||
let movement_generation = response
|
||||
.movement_generation
|
||||
.ok_or_else(|| Error::other("scanner activity peer omitted its movement generation"))?;
|
||||
let publication_blocked = response
|
||||
.publication_blocked
|
||||
.ok_or_else(|| Error::other("scanner activity peer omitted its publication blocked state"))?;
|
||||
if movement_generation == u64::MAX {
|
||||
return Err(Error::other("scanner activity peer exhausted its movement generation"));
|
||||
}
|
||||
let canonical = rustfs_protos::canonical_scanner_activity_v7_response_body(challenge, &response)
|
||||
.map_err(|_| Error::other("scanner activity peer response is too large to authenticate"))?;
|
||||
verify_proof(&canonical, &response.response_proof)?;
|
||||
(
|
||||
Some(
|
||||
response
|
||||
.topology_digest
|
||||
.as_ref()
|
||||
.try_into()
|
||||
.map_err(|_| Error::other("peer returned an invalid scanner topology digest"))?,
|
||||
),
|
||||
Some(response.data_movement_active),
|
||||
Some(response.dirty_usage_generation),
|
||||
Some(response.dirty_usage_pending),
|
||||
Some(movement_generation),
|
||||
Some(publication_blocked),
|
||||
)
|
||||
}
|
||||
version => return Err(Error::other(format!("peer returned unsupported scanner activity protocol {version}"))),
|
||||
};
|
||||
Ok(ScannerPeerActivity {
|
||||
instance_id: response.instance_id,
|
||||
@@ -233,6 +282,8 @@ fn decode_scanner_activity_with_verifier(
|
||||
data_movement_active,
|
||||
dirty_usage_generation,
|
||||
dirty_usage_pending,
|
||||
movement_generation,
|
||||
publication_blocked,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -243,6 +294,17 @@ fn decode_scanner_activity(response: ScannerActivityResponse, challenge: &[u8; 1
|
||||
})
|
||||
}
|
||||
|
||||
fn scanner_activity_protocol_unsupported(err: &Error) -> bool {
|
||||
matches!(
|
||||
err,
|
||||
Error::Io(io_err)
|
||||
if embedded_tonic_status(io_err).is_some_and(|status| {
|
||||
status.code() == tonic::Code::FailedPrecondition
|
||||
&& status.message().starts_with("unsupported scanner activity request protocol")
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
fn validate_heal_control_capability_proof(canonical_ack: &[u8], proof: &[u8]) -> Result<()> {
|
||||
verify_tonic_rpc_response_proof(canonical_ack, proof)
|
||||
.map_err(|_| Error::other("peer returned an invalid heal control capability proof"))
|
||||
@@ -285,6 +347,76 @@ pub struct PeerLiveEventsBatch {
|
||||
pub truncated: bool,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct ScannerPublicationLease {
|
||||
pub token: Uuid,
|
||||
pub movement_generation: u64,
|
||||
/// Stable storage owner identity. This is distinct from the activity
|
||||
/// session and is bound into both acquire and release proofs.
|
||||
pub owner_id: String,
|
||||
/// Process/session nonce observed by the final activity probe.
|
||||
pub session_id: String,
|
||||
pub expires_at: std::time::Instant,
|
||||
}
|
||||
|
||||
impl ScannerPublicationLease {
|
||||
pub fn is_valid(&self) -> bool {
|
||||
std::time::Instant::now() < self.expires_at
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_scanner_publication_lease_response_fields(
|
||||
response: &ScannerPublicationLeaseResponse,
|
||||
expected_session_id: &str,
|
||||
expected_generation: u64,
|
||||
) -> Result<(Uuid, String)> {
|
||||
if !response.success {
|
||||
return Err(Error::other(
|
||||
response
|
||||
.error
|
||||
.as_ref()
|
||||
.map(|error| error.error_info.clone())
|
||||
.unwrap_or_else(|| "peer rejected scanner publication lease".to_string()),
|
||||
));
|
||||
}
|
||||
if response.movement_generation != expected_generation {
|
||||
return Err(Error::other("peer returned a different scanner publication lease generation"));
|
||||
}
|
||||
if response.session_id != expected_session_id {
|
||||
return Err(Error::other("peer returned a different scanner publication lease session"));
|
||||
}
|
||||
let owner_id = Uuid::parse_str(&response.owner_id)
|
||||
.ok()
|
||||
.filter(|owner_id| !owner_id.is_nil())
|
||||
.map(|owner_id| owner_id.to_string())
|
||||
.ok_or_else(|| Error::other("peer returned an invalid scanner publication lease owner"))?;
|
||||
if response.lease_ttl_ms != crate::store::SCANNER_PUBLICATION_LEASE_TTL_MS {
|
||||
return Err(Error::other("peer returned an unsupported scanner publication lease TTL"));
|
||||
}
|
||||
let token = Uuid::from_slice(response.token.as_ref())
|
||||
.map_err(|_| Error::other("peer returned an invalid scanner publication lease token"))?;
|
||||
Ok((token, owner_id))
|
||||
}
|
||||
|
||||
fn scanner_publication_lease_deadline(
|
||||
request_started: std::time::Instant,
|
||||
response_received: std::time::Instant,
|
||||
lease_ttl_ms: u64,
|
||||
) -> Result<std::time::Instant> {
|
||||
let lease_window = Duration::from_millis(lease_ttl_ms)
|
||||
.checked_sub(SCANNER_PUBLICATION_LEASE_SAFETY_MARGIN)
|
||||
.ok_or_else(|| Error::other("scanner publication lease TTL is shorter than its safety margin"))?;
|
||||
let elapsed = response_received
|
||||
.checked_duration_since(request_started)
|
||||
.ok_or_else(|| Error::other("scanner publication lease response clock moved backwards"))?;
|
||||
if elapsed >= lease_window {
|
||||
return Err(Error::other("scanner publication lease response arrived after its safety window"));
|
||||
}
|
||||
request_started
|
||||
.checked_add(lease_window)
|
||||
.ok_or_else(|| Error::other("scanner publication lease deadline overflowed"))
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct PeerRestClient {
|
||||
pub host: XHost,
|
||||
@@ -1640,10 +1772,11 @@ impl PeerRestClient {
|
||||
.await
|
||||
}
|
||||
|
||||
async fn scanner_activity_request(
|
||||
async fn scanner_activity_request_with_protocol(
|
||||
&self,
|
||||
acknowledge_instance_id: String,
|
||||
acknowledge_dirty_usage_generation: u64,
|
||||
protocol_version: u32,
|
||||
) -> Result<ScannerPeerActivity> {
|
||||
self.finalize_result(
|
||||
async {
|
||||
@@ -1655,7 +1788,7 @@ impl PeerRestClient {
|
||||
.max_encoding_message_size(SCANNER_ACTIVITY_MAX_MESSAGE_SIZE);
|
||||
let mut request = Request::new(ScannerActivityRequest {
|
||||
challenge: challenge.as_bytes().to_vec().into(),
|
||||
protocol_version: SCANNER_ACTIVITY_PROTOCOL_VERSION,
|
||||
protocol_version,
|
||||
acknowledge_instance_id,
|
||||
acknowledge_dirty_usage_generation,
|
||||
});
|
||||
@@ -1671,11 +1804,168 @@ impl PeerRestClient {
|
||||
}
|
||||
|
||||
pub async fn scanner_activity(&self) -> Result<ScannerPeerActivity> {
|
||||
self.scanner_activity_request(String::new(), 0).await
|
||||
let result = self
|
||||
.scanner_activity_request_with_protocol(String::new(), 0, SCANNER_ACTIVITY_PROTOCOL_VERSION)
|
||||
.await;
|
||||
if result.as_ref().err().is_some_and(scanner_activity_protocol_unsupported) {
|
||||
// A v6 peer cannot parse the v7 marker. Its authenticated
|
||||
// response is still decoded as untrusted terminal state, so the
|
||||
// scanner will defer publication until every peer is v7.
|
||||
self.scanner_activity_request_with_protocol(String::new(), 0, SCANNER_ACTIVITY_V6_PROTOCOL_VERSION)
|
||||
.await
|
||||
} else {
|
||||
result
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn acknowledge_scanner_dirty_usage(&self, instance_id: String, generation: u64) -> Result<ScannerPeerActivity> {
|
||||
self.scanner_activity_request(instance_id, generation).await
|
||||
let result = self
|
||||
.scanner_activity_request_with_protocol(instance_id.clone(), generation, SCANNER_ACTIVITY_PROTOCOL_VERSION)
|
||||
.await;
|
||||
if result.as_ref().err().is_some_and(scanner_activity_protocol_unsupported) {
|
||||
self.scanner_activity_request_with_protocol(instance_id, generation, SCANNER_ACTIVITY_V6_PROTOCOL_VERSION)
|
||||
.await
|
||||
} else {
|
||||
result
|
||||
}
|
||||
}
|
||||
|
||||
/// Acquire a bounded, storage-owned read admission on the peer that
|
||||
/// produced the final activity generation. Older peers do not implement
|
||||
/// the lease form and are rejected rather than downgraded.
|
||||
pub async fn acquire_scanner_publication_lease(
|
||||
&self,
|
||||
expected_session_id: &str,
|
||||
expected_generation: u64,
|
||||
) -> Result<ScannerPublicationLease> {
|
||||
let request_started = std::time::Instant::now();
|
||||
self.finalize_result(
|
||||
async {
|
||||
let challenge = Uuid::new_v4();
|
||||
let mut client = self
|
||||
.get_client()
|
||||
.await?
|
||||
.max_decoding_message_size(SCANNER_ACTIVITY_MAX_MESSAGE_SIZE)
|
||||
.max_encoding_message_size(SCANNER_ACTIVITY_MAX_MESSAGE_SIZE);
|
||||
let mut request = Request::new(ScannerPublicationLeaseRequest {
|
||||
challenge: challenge.as_bytes().to_vec().into(),
|
||||
expected_movement_generation: expected_generation,
|
||||
ttl_ms: crate::store::SCANNER_PUBLICATION_LEASE_TTL_MS,
|
||||
expected_session_id: expected_session_id.to_string(),
|
||||
token: Bytes::new(),
|
||||
});
|
||||
let canonical = rustfs_protos::canonical_scanner_publication_lease_request_body(request.get_ref())
|
||||
.map_err(|_| Error::other("scanner publication lease request is too large to authenticate"))?;
|
||||
set_tonic_canonical_body_digest(&mut request, &canonical)?;
|
||||
let response = client.acquire_scanner_publication_lease(request).await?.into_inner();
|
||||
let response_body =
|
||||
rustfs_protos::canonical_scanner_publication_lease_response_body(challenge.as_bytes(), &response)
|
||||
.map_err(|_| Error::other("scanner publication lease response is too large to authenticate"))?;
|
||||
verify_tonic_rpc_response_proof(&response_body, &response.response_proof)
|
||||
.map_err(|_| Error::other("peer returned an invalid scanner publication lease proof"))?;
|
||||
let (token, owner_id) =
|
||||
validate_scanner_publication_lease_response_fields(&response, expected_session_id, expected_generation)?;
|
||||
Ok(ScannerPublicationLease {
|
||||
token,
|
||||
movement_generation: response.movement_generation,
|
||||
owner_id,
|
||||
session_id: response.session_id,
|
||||
expires_at: scanner_publication_lease_deadline(
|
||||
request_started,
|
||||
std::time::Instant::now(),
|
||||
response.lease_ttl_ms,
|
||||
)?,
|
||||
})
|
||||
}
|
||||
.await,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Revalidate the exact token immediately before the coordinator commits
|
||||
/// its final publication. The peer keeps the original movement read
|
||||
/// guard in its token table; a restart drops that table and changes the
|
||||
/// activity session, so this proof fails closed instead of accepting an
|
||||
/// ABA generation value.
|
||||
pub async fn validate_scanner_publication_lease(&self, lease: &ScannerPublicationLease) -> Result<()> {
|
||||
self.finalize_result(
|
||||
async {
|
||||
let challenge = Uuid::new_v4();
|
||||
let mut client = self
|
||||
.get_client()
|
||||
.await?
|
||||
.max_decoding_message_size(SCANNER_ACTIVITY_MAX_MESSAGE_SIZE)
|
||||
.max_encoding_message_size(SCANNER_ACTIVITY_MAX_MESSAGE_SIZE);
|
||||
let mut request = Request::new(ScannerPublicationLeaseRequest {
|
||||
challenge: challenge.as_bytes().to_vec().into(),
|
||||
expected_movement_generation: lease.movement_generation,
|
||||
ttl_ms: crate::store::SCANNER_PUBLICATION_LEASE_TTL_MS,
|
||||
expected_session_id: lease.session_id.clone(),
|
||||
token: lease.token.as_bytes().to_vec().into(),
|
||||
});
|
||||
let canonical = rustfs_protos::canonical_scanner_publication_lease_request_body(request.get_ref())
|
||||
.map_err(|_| Error::other("scanner publication lease validation request is too large to authenticate"))?;
|
||||
set_tonic_canonical_body_digest(&mut request, &canonical)?;
|
||||
let response = client.acquire_scanner_publication_lease(request).await?.into_inner();
|
||||
let response_body =
|
||||
rustfs_protos::canonical_scanner_publication_lease_response_body(challenge.as_bytes(), &response).map_err(
|
||||
|_| Error::other("scanner publication lease validation response is too large to authenticate"),
|
||||
)?;
|
||||
verify_tonic_rpc_response_proof(&response_body, &response.response_proof)
|
||||
.map_err(|_| Error::other("peer returned an invalid scanner publication lease validation proof"))?;
|
||||
let (token, owner_id) =
|
||||
validate_scanner_publication_lease_response_fields(&response, &lease.session_id, lease.movement_generation)?;
|
||||
if token != lease.token {
|
||||
return Err(Error::other("peer returned a different scanner publication lease token"));
|
||||
}
|
||||
if owner_id != lease.owner_id {
|
||||
return Err(Error::other("peer returned a different scanner publication lease owner"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
.await,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn release_scanner_publication_lease(&self, lease: &ScannerPublicationLease) -> Result<()> {
|
||||
self.finalize_result(
|
||||
async {
|
||||
let challenge = Uuid::new_v4();
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(ScannerPublicationLeaseReleaseRequest {
|
||||
challenge: challenge.as_bytes().to_vec().into(),
|
||||
token: lease.token.as_bytes().to_vec().into(),
|
||||
owner_id: lease.owner_id.clone(),
|
||||
session_id: lease.session_id.clone(),
|
||||
});
|
||||
let canonical = rustfs_protos::canonical_scanner_publication_lease_release_request_body(request.get_ref())
|
||||
.map_err(|_| Error::other("scanner publication lease release request is too large to authenticate"))?;
|
||||
set_tonic_canonical_body_digest(&mut request, &canonical)?;
|
||||
let request_body = request.get_ref().clone();
|
||||
let response = client.release_scanner_publication_lease(request).await?.into_inner();
|
||||
let response_body = rustfs_protos::canonical_scanner_publication_lease_release_response_body(
|
||||
challenge.as_bytes(),
|
||||
&request_body,
|
||||
&response,
|
||||
)
|
||||
.map_err(|_| Error::other("scanner publication lease release response is too large to authenticate"))?;
|
||||
verify_tonic_rpc_response_proof(&response_body, &response.response_proof)
|
||||
.map_err(|_| Error::other("peer returned an invalid scanner publication lease release proof"))?;
|
||||
if response.success {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(Error::other(
|
||||
response
|
||||
.error
|
||||
.map(|error| error.error_info)
|
||||
.unwrap_or_else(|| "peer rejected scanner publication lease release".to_string()),
|
||||
))
|
||||
}
|
||||
}
|
||||
.await,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn get_metacache_listing(&self) -> Result<()> {
|
||||
@@ -1991,6 +2281,52 @@ mod tests {
|
||||
use temp_env::async_with_vars;
|
||||
use tracing_subscriber::{Registry, fmt::MakeWriter, layer::SubscriberExt};
|
||||
|
||||
#[test]
|
||||
fn scanner_publication_lease_response_rejects_stale_generation_and_session() {
|
||||
let token = Uuid::new_v4();
|
||||
let response = ScannerPublicationLeaseResponse {
|
||||
success: true,
|
||||
token: token.as_bytes().to_vec().into(),
|
||||
movement_generation: 7,
|
||||
lease_ttl_ms: crate::store::SCANNER_PUBLICATION_LEASE_TTL_MS,
|
||||
error: None,
|
||||
response_proof: Bytes::new(),
|
||||
owner_id: Uuid::new_v4().to_string(),
|
||||
session_id: "session-a".to_string(),
|
||||
};
|
||||
|
||||
assert!(validate_scanner_publication_lease_response_fields(&response, "session-a", 7).is_ok());
|
||||
|
||||
let stale_generation = ScannerPublicationLeaseResponse {
|
||||
movement_generation: 6,
|
||||
..response.clone()
|
||||
};
|
||||
let error = validate_scanner_publication_lease_response_fields(&stale_generation, "session-a", 7)
|
||||
.expect_err("a response from an older movement generation must be rejected");
|
||||
assert!(error.to_string().contains("different scanner publication lease generation"));
|
||||
|
||||
let stale_session = ScannerPublicationLeaseResponse {
|
||||
session_id: "session-b".to_string(),
|
||||
..response
|
||||
};
|
||||
let error = validate_scanner_publication_lease_response_fields(&stale_session, "session-a", 7)
|
||||
.expect_err("a response from an older scanner session must be rejected");
|
||||
assert!(error.to_string().contains("different scanner publication lease session"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_publication_lease_deadline_accounts_for_delayed_rpc_response() {
|
||||
let started = std::time::Instant::now();
|
||||
let expected_deadline = started + Duration::from_secs(55);
|
||||
let deadline = scanner_publication_lease_deadline(started, started + Duration::from_secs(10), 60_000)
|
||||
.expect("a response inside the safety window should retain the original deadline");
|
||||
assert_eq!(deadline, expected_deadline);
|
||||
|
||||
let error = scanner_publication_lease_deadline(started, started + Duration::from_secs(55), 60_000)
|
||||
.expect_err("a response arriving at the safety boundary must fail closed");
|
||||
assert!(error.to_string().contains("after its safety window"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replication_stats_response_decodes_valid_empty_provider() {
|
||||
let mut stats = BucketStats::default();
|
||||
@@ -2229,6 +2565,8 @@ mod tests {
|
||||
response_proof: Vec::new().into(),
|
||||
dirty_usage_generation: 0,
|
||||
dirty_usage_pending: false,
|
||||
movement_generation: None,
|
||||
publication_blocked: None,
|
||||
})
|
||||
.expect("legacy peers should retain their activity generations during a rolling upgrade");
|
||||
assert_eq!(
|
||||
@@ -2242,6 +2580,8 @@ mod tests {
|
||||
data_movement_active: None,
|
||||
dirty_usage_generation: None,
|
||||
dirty_usage_pending: None,
|
||||
movement_generation: None,
|
||||
publication_blocked: None,
|
||||
}
|
||||
);
|
||||
|
||||
@@ -2255,6 +2595,8 @@ mod tests {
|
||||
response_proof: b"proof".to_vec().into(),
|
||||
dirty_usage_generation: 0,
|
||||
dirty_usage_pending: false,
|
||||
movement_generation: None,
|
||||
publication_blocked: None,
|
||||
})
|
||||
.expect("protocol v4 peers should remain observable during a rolling upgrade");
|
||||
assert_eq!(
|
||||
@@ -2268,9 +2610,29 @@ mod tests {
|
||||
data_movement_active: Some(true),
|
||||
dirty_usage_generation: None,
|
||||
dirty_usage_pending: None,
|
||||
movement_generation: None,
|
||||
publication_blocked: None,
|
||||
}
|
||||
);
|
||||
|
||||
let v6 = decode_test_scanner_activity(ScannerActivityResponse {
|
||||
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
|
||||
namespace_generation: 7,
|
||||
maintenance_generation: 3,
|
||||
protocol_version: SCANNER_ACTIVITY_V6_PROTOCOL_VERSION,
|
||||
topology_digest: vec![7; 32].into(),
|
||||
data_movement_active: true,
|
||||
response_proof: b"proof".to_vec().into(),
|
||||
dirty_usage_generation: 11,
|
||||
dirty_usage_pending: true,
|
||||
movement_generation: None,
|
||||
publication_blocked: None,
|
||||
})
|
||||
.expect("v6 peers should remain readable without a v7 publication proof");
|
||||
assert_eq!(v6.movement_generation, None);
|
||||
assert_eq!(v6.publication_blocked, None);
|
||||
assert_eq!(v6.dirty_usage_generation, Some(11));
|
||||
|
||||
let malformed_topology = ScannerActivityResponse {
|
||||
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
|
||||
namespace_generation: 7,
|
||||
@@ -2281,6 +2643,8 @@ mod tests {
|
||||
response_proof: b"proof".to_vec().into(),
|
||||
dirty_usage_generation: 11,
|
||||
dirty_usage_pending: true,
|
||||
movement_generation: Some(19),
|
||||
publication_blocked: Some(false),
|
||||
};
|
||||
assert!(
|
||||
decode_test_scanner_activity(malformed_topology)
|
||||
@@ -2299,6 +2663,8 @@ mod tests {
|
||||
response_proof: b"proof".to_vec().into(),
|
||||
dirty_usage_generation: 11,
|
||||
dirty_usage_pending: true,
|
||||
movement_generation: Some(19),
|
||||
publication_blocked: Some(false),
|
||||
};
|
||||
assert!(
|
||||
decode_test_scanner_activity(missing_instance)
|
||||
@@ -2317,6 +2683,8 @@ mod tests {
|
||||
response_proof: b"proof".to_vec().into(),
|
||||
dirty_usage_generation: 11,
|
||||
dirty_usage_pending: true,
|
||||
movement_generation: Some(19),
|
||||
publication_blocked: Some(false),
|
||||
};
|
||||
assert!(
|
||||
decode_test_scanner_activity(malformed_instance)
|
||||
@@ -2335,6 +2703,8 @@ mod tests {
|
||||
response_proof: b"proof".to_vec().into(),
|
||||
dirty_usage_generation: 11,
|
||||
dirty_usage_pending: true,
|
||||
movement_generation: Some(19),
|
||||
publication_blocked: Some(false),
|
||||
})
|
||||
.expect("complete activity responses should be accepted");
|
||||
assert_eq!(
|
||||
@@ -2348,9 +2718,31 @@ mod tests {
|
||||
data_movement_active: Some(true),
|
||||
dirty_usage_generation: Some(11),
|
||||
dirty_usage_pending: Some(true),
|
||||
movement_generation: Some(19),
|
||||
publication_blocked: Some(false),
|
||||
}
|
||||
);
|
||||
|
||||
let missing_movement_generation = ScannerActivityResponse {
|
||||
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
|
||||
namespace_generation: 7,
|
||||
maintenance_generation: 3,
|
||||
protocol_version: SCANNER_ACTIVITY_PROTOCOL_VERSION,
|
||||
topology_digest: vec![7; 32].into(),
|
||||
data_movement_active: false,
|
||||
response_proof: b"proof".to_vec().into(),
|
||||
dirty_usage_generation: 11,
|
||||
dirty_usage_pending: false,
|
||||
movement_generation: None,
|
||||
publication_blocked: Some(false),
|
||||
};
|
||||
assert!(
|
||||
decode_test_scanner_activity(missing_movement_generation)
|
||||
.expect_err("v7 activity must carry movement generation")
|
||||
.to_string()
|
||||
.contains("movement generation")
|
||||
);
|
||||
|
||||
let pending_without_generation = ScannerActivityResponse {
|
||||
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
|
||||
namespace_generation: 7,
|
||||
@@ -2361,6 +2753,8 @@ mod tests {
|
||||
response_proof: b"proof".to_vec().into(),
|
||||
dirty_usage_generation: 0,
|
||||
dirty_usage_pending: true,
|
||||
movement_generation: Some(19),
|
||||
publication_blocked: Some(false),
|
||||
};
|
||||
assert!(
|
||||
decode_test_scanner_activity(pending_without_generation)
|
||||
@@ -2379,6 +2773,8 @@ mod tests {
|
||||
response_proof: b"proof".to_vec().into(),
|
||||
dirty_usage_generation: 11,
|
||||
dirty_usage_pending: true,
|
||||
movement_generation: None,
|
||||
publication_blocked: None,
|
||||
};
|
||||
assert!(
|
||||
decode_test_scanner_activity(previous_with_dirty_usage)
|
||||
@@ -2397,6 +2793,8 @@ mod tests {
|
||||
response_proof: b"proof".to_vec().into(),
|
||||
dirty_usage_generation: 0,
|
||||
dirty_usage_pending: false,
|
||||
movement_generation: None,
|
||||
publication_blocked: None,
|
||||
};
|
||||
assert!(
|
||||
decode_test_scanner_activity(legacy_with_topology)
|
||||
@@ -2415,6 +2813,8 @@ mod tests {
|
||||
response_proof: b"proof".to_vec().into(),
|
||||
dirty_usage_generation: 11,
|
||||
dirty_usage_pending: true,
|
||||
movement_generation: None,
|
||||
publication_blocked: None,
|
||||
};
|
||||
assert!(
|
||||
decode_test_scanner_activity(unsupported_protocol)
|
||||
@@ -2433,6 +2833,8 @@ mod tests {
|
||||
response_proof: Vec::new().into(),
|
||||
dirty_usage_generation: 11,
|
||||
dirty_usage_pending: true,
|
||||
movement_generation: Some(19),
|
||||
publication_blocked: Some(false),
|
||||
};
|
||||
assert!(
|
||||
decode_test_scanner_activity(missing_proof)
|
||||
|
||||
@@ -1981,6 +1981,20 @@ impl RemoteDisk {
|
||||
fi: &FileInfo,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
) -> Result<RenameDataResp> {
|
||||
self.rename_data_borrowed_with_fence(src_volume, src_path, fi, dst_volume, dst_path, None)
|
||||
.await
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
pub(crate) async fn rename_data_borrowed_with_fence(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
src_path: &str,
|
||||
fi: &FileInfo,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
scanner_publication_lease_token: Option<Uuid>,
|
||||
) -> Result<RenameDataResp> {
|
||||
trace!(
|
||||
event = EVENT_REMOTE_DISK_RPC,
|
||||
@@ -2013,9 +2027,18 @@ impl RemoteDisk {
|
||||
dst_volume: dst_volume.to_string(),
|
||||
dst_path: dst_path.to_string(),
|
||||
file_info_bin: file_info_bin.into(),
|
||||
scanner_publication_lease_token: scanner_publication_lease_token
|
||||
.map(|token| token.as_bytes().to_vec().into())
|
||||
.unwrap_or_default(),
|
||||
});
|
||||
let canonical_body = rustfs_protos::canonical_rename_data_request_body(request.get_ref());
|
||||
attach_mutation_body_digest(&mut request, canonical_body, "rename_data")?;
|
||||
if scanner_publication_lease_token.is_some() {
|
||||
let canonical_body =
|
||||
canonical_body.map_err(|_| Error::other("rename_data request length cannot be represented"))?;
|
||||
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &canonical_body).map_err(Error::other)?;
|
||||
} else {
|
||||
attach_mutation_body_digest(&mut request, canonical_body, "rename_data")?;
|
||||
}
|
||||
|
||||
let response = client.rename_data(request).await?.into_inner();
|
||||
|
||||
@@ -2035,6 +2058,70 @@ impl RemoteDisk {
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Delete a path while binding the target-side operation to a scanner
|
||||
/// publication lease. The ordinary `DiskAPI::delete` path keeps the
|
||||
/// legacy digest/compatibility behavior by passing no token.
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
pub(crate) async fn delete_with_scanner_publication_lease(
|
||||
&self,
|
||||
volume: &str,
|
||||
path: &str,
|
||||
opt: DeleteOptions,
|
||||
scanner_publication_lease_token: Option<Uuid>,
|
||||
) -> Result<()> {
|
||||
trace!(
|
||||
event = EVENT_REMOTE_DISK_RPC,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
|
||||
endpoint = %self.endpoint,
|
||||
volume,
|
||||
path,
|
||||
recursive = opt.recursive,
|
||||
immediate = opt.immediate,
|
||||
fenced = scanner_publication_lease_token.is_some(),
|
||||
op = "delete",
|
||||
state = "started",
|
||||
"Remote disk RPC started"
|
||||
);
|
||||
|
||||
self.execute_with_timeout(
|
||||
|| async {
|
||||
let options = serde_json::to_string(&opt)?;
|
||||
let mut client = self
|
||||
.get_client()
|
||||
.await
|
||||
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
|
||||
let mut request = Request::new(DeleteRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
volume: volume.to_string(),
|
||||
path: path.to_string(),
|
||||
options,
|
||||
scanner_publication_lease_token: scanner_publication_lease_token
|
||||
.map(|token| token.as_bytes().to_vec().into())
|
||||
.unwrap_or_default(),
|
||||
});
|
||||
let canonical_body = rustfs_protos::canonical_delete_request_body(request.get_ref());
|
||||
if scanner_publication_lease_token.is_some() {
|
||||
let canonical_body =
|
||||
canonical_body.map_err(|_| Error::other("delete request length cannot be represented"))?;
|
||||
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &canonical_body).map_err(Error::other)?;
|
||||
} else {
|
||||
attach_mutation_body_digest(&mut request, canonical_body, "delete")?;
|
||||
}
|
||||
|
||||
let response = client.delete(request).await?.into_inner();
|
||||
|
||||
if !response.success {
|
||||
return Err(response.error.unwrap_or_default().into());
|
||||
}
|
||||
|
||||
Ok(())
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
@@ -3446,47 +3533,7 @@ impl DiskAPI for RemoteDisk {
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()> {
|
||||
trace!(
|
||||
event = EVENT_REMOTE_DISK_RPC,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
|
||||
endpoint = %self.endpoint,
|
||||
volume,
|
||||
path,
|
||||
recursive = opt.recursive,
|
||||
immediate = opt.immediate,
|
||||
op = "delete",
|
||||
state = "started",
|
||||
"Remote disk RPC started"
|
||||
);
|
||||
|
||||
self.execute_with_timeout(
|
||||
|| async {
|
||||
let options = serde_json::to_string(&opt)?;
|
||||
let mut client = self
|
||||
.get_client()
|
||||
.await
|
||||
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
|
||||
let mut request = Request::new(DeleteRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
volume: volume.to_string(),
|
||||
path: path.to_string(),
|
||||
options,
|
||||
});
|
||||
let canonical_body = rustfs_protos::canonical_delete_request_body(request.get_ref());
|
||||
attach_mutation_body_digest(&mut request, canonical_body, "delete")?;
|
||||
|
||||
let response = client.delete(request).await?.into_inner();
|
||||
|
||||
if !response.success {
|
||||
return Err(response.error.unwrap_or_default().into());
|
||||
}
|
||||
|
||||
Ok(())
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
self.delete_with_scanner_publication_lease(volume, path, opt, None).await
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
|
||||
@@ -677,6 +677,23 @@ impl DiskAPI for Disk {
|
||||
}
|
||||
|
||||
impl Disk {
|
||||
pub(crate) async fn delete_with_scanner_publication_lease(
|
||||
&self,
|
||||
volume: &str,
|
||||
path: &str,
|
||||
opts: DeleteOptions,
|
||||
scanner_publication_lease_token: Option<Uuid>,
|
||||
) -> Result<()> {
|
||||
match self {
|
||||
Disk::Local(local_disk) => local_disk.delete(volume, path, opts).await,
|
||||
Disk::Remote(remote_disk) => {
|
||||
remote_disk
|
||||
.delete_with_scanner_publication_lease(volume, path, opts, scanner_publication_lease_token)
|
||||
.await
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn rename_data_borrowed(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
@@ -684,6 +701,19 @@ impl Disk {
|
||||
fi: &FileInfo,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
) -> Result<RenameDataResp> {
|
||||
self.rename_data_borrowed_with_fence(src_volume, src_path, fi, dst_volume, dst_path, None)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn rename_data_borrowed_with_fence(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
src_path: &str,
|
||||
fi: &FileInfo,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
scanner_publication_lease_token: Option<Uuid>,
|
||||
) -> Result<RenameDataResp> {
|
||||
match self {
|
||||
Disk::Local(local_disk) => {
|
||||
@@ -693,7 +723,14 @@ impl Disk {
|
||||
}
|
||||
Disk::Remote(remote_disk) => {
|
||||
remote_disk
|
||||
.rename_data_borrowed(src_volume, src_path, fi, dst_volume, dst_path)
|
||||
.rename_data_borrowed_with_fence(
|
||||
src_volume,
|
||||
src_path,
|
||||
fi,
|
||||
dst_volume,
|
||||
dst_path,
|
||||
scanner_publication_lease_token,
|
||||
)
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
@@ -446,6 +446,11 @@ pub struct ObjectOptions {
|
||||
pub tier_delete_journal_api: Option<Arc<crate::store::ECStore>>,
|
||||
}
|
||||
|
||||
/// Transient scanner-only carrier for target-side publication lease tokens.
|
||||
/// SetDisks consumes and removes this key before constructing durable
|
||||
/// FileInfo metadata; it must never appear in an S3-visible object.
|
||||
pub const SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY: &str = "x-rustfs-internal-scanner-publication-lease-fence-v1";
|
||||
|
||||
impl ObjectOptions {
|
||||
pub fn set_quota_admission(&mut self, current_usage: u64, quota_limit: u64) -> bool {
|
||||
self.quota_admission = (current_usage <= quota_limit).then_some(QuotaAdmission {
|
||||
|
||||
@@ -56,7 +56,8 @@ use std::sync::{
|
||||
Arc, OnceLock,
|
||||
atomic::{AtomicBool, AtomicU8, AtomicU64, Ordering},
|
||||
};
|
||||
use tokio::sync::{OnceCell, RwLock};
|
||||
use tokio::sync::{Mutex, Notify, OnceCell, OwnedRwLockReadGuard, RwLock};
|
||||
use tokio::time::Instant;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use uuid::Uuid;
|
||||
|
||||
@@ -64,6 +65,19 @@ const SCANNER_PUBLICATION_STATE_UNKNOWN: u8 = 0;
|
||||
const SCANNER_PUBLICATION_STATE_ALLOWED: u8 = 1;
|
||||
const SCANNER_PUBLICATION_STATE_BLOCKED: u8 = 2;
|
||||
|
||||
pub(crate) const SCANNER_PUBLICATION_LEASE_MAX_ENTRIES: usize = 256;
|
||||
|
||||
/// A lease is deliberately short-lived. The coordinator treats expiry as a
|
||||
/// failed publication rather than silently continuing with a peer that may
|
||||
/// have started movement after the lease was abandoned.
|
||||
pub(crate) const SCANNER_PUBLICATION_LEASE_TTL: std::time::Duration = std::time::Duration::from_secs(60);
|
||||
|
||||
pub(crate) struct ScannerPublicationLeaseEntry {
|
||||
pub(crate) expires_at: Instant,
|
||||
pub(crate) movement_generation: u64,
|
||||
pub(crate) _operation_guard: OwnedRwLockReadGuard<()>,
|
||||
}
|
||||
|
||||
/// Runtime state owned by a single `ECStore` instance.
|
||||
///
|
||||
/// This is intentionally minimal in the first migration slice; subsequent
|
||||
@@ -171,6 +185,11 @@ pub struct InstanceContext {
|
||||
/// Readers are held across one publication commit; movement transitions
|
||||
/// take the writer at their durable state commit boundary.
|
||||
data_movement_operation_gate: Arc<RwLock<()>>,
|
||||
/// Remote scanner publication leases own a read guard until explicit
|
||||
/// release or bounded expiry. Keeping the guard in storage-owned state
|
||||
/// makes a remote movement transition wait on the same fence as a local
|
||||
/// scanner commit.
|
||||
scanner_publication_leases: Arc<Mutex<HashMap<Uuid, ScannerPublicationLeaseEntry>>>,
|
||||
/// Monotonic admission epoch paired with the operation gate. A
|
||||
/// publication admitted before a movement transition must never be
|
||||
/// mistaken for one admitted after the transition.
|
||||
@@ -180,6 +199,13 @@ pub struct InstanceContext {
|
||||
/// saturating counter prevents an unchanged `u64::MAX` value from being
|
||||
/// mistaken for a fresh epoch after overflow.
|
||||
data_movement_operation_epoch_exhausted: AtomicBool,
|
||||
/// Storage-owned generation for movement state changes. This is separate
|
||||
/// from the publication admission epoch so scanners can wait for a
|
||||
/// terminal/clear transition without treating the wake as a publication
|
||||
/// permit.
|
||||
data_movement_generation: AtomicU64,
|
||||
data_movement_generation_exhausted: AtomicBool,
|
||||
data_movement_generation_notify: Arc<Notify>,
|
||||
/// Last storage-owned movement snapshot observed under the operation
|
||||
/// gate. SetDisks cache writers fail closed until ECStore refreshes it.
|
||||
scanner_publication_state: AtomicU8,
|
||||
@@ -224,8 +250,12 @@ impl InstanceContext {
|
||||
bucket_metadata_sys: std::sync::Mutex::new(None),
|
||||
background_cancel_token: OnceLock::new(),
|
||||
data_movement_operation_gate: Arc::new(RwLock::new(())),
|
||||
scanner_publication_leases: Arc::new(Mutex::new(HashMap::new())),
|
||||
data_movement_operation_epoch: AtomicU64::new(0),
|
||||
data_movement_operation_epoch_exhausted: AtomicBool::new(false),
|
||||
data_movement_generation: AtomicU64::new(0),
|
||||
data_movement_generation_exhausted: AtomicBool::new(false),
|
||||
data_movement_generation_notify: Arc::new(Notify::new()),
|
||||
scanner_publication_state: AtomicU8::new(SCANNER_PUBLICATION_STATE_UNKNOWN),
|
||||
object_encryption_resolver: OnceLock::new(),
|
||||
tier_delete_journal_recovery_stores: std::sync::Mutex::new(HashSet::new()),
|
||||
@@ -249,6 +279,75 @@ impl InstanceContext {
|
||||
Arc::clone(&self.data_movement_operation_gate)
|
||||
}
|
||||
|
||||
pub(crate) async fn install_scanner_publication_lease(
|
||||
&self,
|
||||
token: Uuid,
|
||||
expires_at: Instant,
|
||||
movement_generation: u64,
|
||||
operation_guard: OwnedRwLockReadGuard<()>,
|
||||
) -> bool {
|
||||
let mut leases = self.scanner_publication_leases.lock().await;
|
||||
if leases.len() >= SCANNER_PUBLICATION_LEASE_MAX_ENTRIES {
|
||||
return false;
|
||||
}
|
||||
leases.insert(
|
||||
token,
|
||||
ScannerPublicationLeaseEntry {
|
||||
expires_at,
|
||||
movement_generation,
|
||||
_operation_guard: operation_guard,
|
||||
},
|
||||
);
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) async fn remove_scanner_publication_lease(&self, token: Uuid) -> bool {
|
||||
self.scanner_publication_leases.lock().await.remove(&token).is_some()
|
||||
}
|
||||
|
||||
/// Check a lease token while the caller holds the movement read guard.
|
||||
///
|
||||
/// The token table is deliberately process-owned and non-persistent: a
|
||||
/// restarted instance has no entries from the previous process, so an old
|
||||
/// coordinator proof cannot become valid again merely because the
|
||||
/// movement generation counter restarted at zero.
|
||||
pub(crate) async fn scanner_publication_lease_is_active(&self, token: Uuid) -> bool {
|
||||
let mut leases = self.scanner_publication_leases.lock().await;
|
||||
let now = Instant::now();
|
||||
let Some(expires_at) = leases.get(&token).map(|entry| entry.expires_at) else {
|
||||
return false;
|
||||
};
|
||||
if expires_at <= now {
|
||||
leases.remove(&token);
|
||||
return false;
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
/// Return the generation bound to a live lease. The lease entry owns the
|
||||
/// movement read guard, so a successful lookup remains valid for the
|
||||
/// caller's guard-protected operation; expiry is still fail-closed.
|
||||
pub(crate) async fn scanner_publication_lease_generation(&self, token: Uuid) -> Option<u64> {
|
||||
let mut leases = self.scanner_publication_leases.lock().await;
|
||||
let now = Instant::now();
|
||||
let (expires_at, movement_generation) = leases
|
||||
.get(&token)
|
||||
.map(|entry| (entry.expires_at, entry.movement_generation))?;
|
||||
if expires_at <= now {
|
||||
leases.remove(&token);
|
||||
return None;
|
||||
}
|
||||
Some(movement_generation)
|
||||
}
|
||||
|
||||
pub(crate) async fn expire_scanner_publication_lease(&self, token: Uuid, expires_at: Instant) {
|
||||
let mut leases = self.scanner_publication_leases.lock().await;
|
||||
let should_remove = leases.get(&token).is_some_and(|entry| entry.expires_at <= expires_at);
|
||||
if should_remove {
|
||||
leases.remove(&token);
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn data_movement_operation_epoch(&self) -> u64 {
|
||||
self.data_movement_operation_epoch.load(Ordering::Acquire)
|
||||
}
|
||||
@@ -257,8 +356,21 @@ impl InstanceContext {
|
||||
self.data_movement_operation_epoch_exhausted.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
pub(crate) fn data_movement_generation(&self) -> u64 {
|
||||
self.data_movement_generation.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
pub(crate) fn data_movement_generation_exhausted(&self) -> bool {
|
||||
self.data_movement_generation_exhausted.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
pub(crate) fn data_movement_generation_notify(&self) -> Arc<Notify> {
|
||||
Arc::clone(&self.data_movement_generation_notify)
|
||||
}
|
||||
|
||||
pub(crate) fn scanner_publication_state_allowed(&self) -> bool {
|
||||
!self.data_movement_operation_epoch_exhausted()
|
||||
&& !self.data_movement_generation_exhausted()
|
||||
&& self.scanner_publication_state.load(Ordering::Acquire) == SCANNER_PUBLICATION_STATE_ALLOWED
|
||||
}
|
||||
|
||||
@@ -276,6 +388,7 @@ impl InstanceContext {
|
||||
pub(crate) fn advance_data_movement_operation_epoch(&self) -> u64 {
|
||||
self.scanner_publication_state
|
||||
.store(SCANNER_PUBLICATION_STATE_UNKNOWN, Ordering::Release);
|
||||
let previous = self.data_movement_operation_epoch.load(Ordering::Acquire);
|
||||
let _ = self
|
||||
.data_movement_operation_epoch
|
||||
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |epoch| Some(epoch.saturating_add(1)));
|
||||
@@ -283,9 +396,42 @@ impl InstanceContext {
|
||||
if result == u64::MAX {
|
||||
self.data_movement_operation_epoch_exhausted.store(true, Ordering::Release);
|
||||
}
|
||||
if result != previous {
|
||||
let _ = self.advance_data_movement_generation();
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
/// Advance the movement generation after a durable movement transition.
|
||||
/// The generation is deliberately bounded: once it reaches `u64::MAX`,
|
||||
/// publication and generation-based waits fail closed rather than reusing
|
||||
/// an indistinguishable saturated value.
|
||||
pub(crate) fn advance_data_movement_generation(&self) -> Option<u64> {
|
||||
if self.data_movement_generation_exhausted.load(Ordering::Acquire) {
|
||||
return None;
|
||||
}
|
||||
let updated = self
|
||||
.data_movement_generation
|
||||
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |generation| generation.checked_add(1));
|
||||
match updated {
|
||||
Ok(previous) => {
|
||||
let Some(generation) = previous.checked_add(1) else {
|
||||
self.data_movement_generation_exhausted.store(true, Ordering::Release);
|
||||
return None;
|
||||
};
|
||||
if generation == u64::MAX {
|
||||
self.data_movement_generation_exhausted.store(true, Ordering::Release);
|
||||
}
|
||||
self.data_movement_generation_notify.notify_waiters();
|
||||
Some(generation)
|
||||
}
|
||||
Err(_) => {
|
||||
self.data_movement_generation_exhausted.store(true, Ordering::Release);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn set_data_movement_operation_epoch_for_test(&self, epoch: u64) {
|
||||
self.data_movement_operation_epoch.store(epoch, Ordering::Release);
|
||||
@@ -295,6 +441,13 @@ impl InstanceContext {
|
||||
.store(SCANNER_PUBLICATION_STATE_UNKNOWN, Ordering::Release);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn set_data_movement_generation_for_test(&self, generation: u64) {
|
||||
self.data_movement_generation.store(generation, Ordering::Release);
|
||||
self.data_movement_generation_exhausted
|
||||
.store(generation == u64::MAX, Ordering::Release);
|
||||
}
|
||||
|
||||
/// Install the application-owned object-encryption resolver once.
|
||||
pub fn set_object_encryption_resolver(
|
||||
&self,
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::cluster::rpc::{PeerRestClient, ScannerPeerActivity, TierConfigReloadOutcome};
|
||||
use crate::cluster::rpc::{PeerRestClient, ScannerPeerActivity, ScannerPublicationLease, TierConfigReloadOutcome};
|
||||
use crate::diagnostics::admin_server_info::get_commit_id;
|
||||
use crate::disk::DiskAPI;
|
||||
use crate::error::{Error, Result};
|
||||
@@ -53,6 +53,12 @@ const REMOTE_VERSION_STATE_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
const REMOTE_VERSION_STATE_PROOF_TTL: Duration = Duration::from_secs(30);
|
||||
const CROSS_POOL_FENCE_SUPPORTED_VERSION: u32 = 2;
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct ScannerPublicationLeaseGrant {
|
||||
pub host: String,
|
||||
pub lease: ScannerPublicationLease,
|
||||
}
|
||||
|
||||
/// Cached result from the last successful admin call to a peer.
|
||||
struct PeerAdminCache {
|
||||
last_storage_info: Option<StorageInfo>,
|
||||
@@ -1491,6 +1497,102 @@ impl NotificationSys {
|
||||
aggregate_scanner_dirty_usage_acknowledgement_results(join_all(futures).await, failures)
|
||||
}
|
||||
|
||||
/// Acquire remote publication leases in a deterministic host order. A
|
||||
/// missing/legacy peer is a hard publication deferral; already acquired
|
||||
/// leases are released before returning so a partial acquisition cannot
|
||||
/// pin movement on one peer.
|
||||
pub async fn acquire_scanner_publication_leases(
|
||||
&self,
|
||||
mut targets: Vec<(String, String, u64)>,
|
||||
) -> Result<Vec<ScannerPublicationLeaseGrant>> {
|
||||
targets.sort_by(|left, right| left.0.cmp(&right.0));
|
||||
for pair in targets.windows(2) {
|
||||
if pair[0].0 == pair[1].0 {
|
||||
return Err(Error::other(format!("duplicate scanner publication lease target: {}", pair[0].0)));
|
||||
}
|
||||
}
|
||||
|
||||
let mut grants = Vec::with_capacity(targets.len());
|
||||
for (host, session_id, generation) in targets {
|
||||
let Some(client) = self
|
||||
.peer_clients
|
||||
.iter()
|
||||
.flatten()
|
||||
.find(|client| client.grid_host == host)
|
||||
.cloned()
|
||||
else {
|
||||
let _ = self.release_scanner_publication_leases(grants).await;
|
||||
return Err(Error::other(format!("scanner publication lease peer {host} is unavailable")));
|
||||
};
|
||||
match client.acquire_scanner_publication_lease(&session_id, generation).await {
|
||||
Ok(lease) => grants.push(ScannerPublicationLeaseGrant { host, lease }),
|
||||
Err(err) => {
|
||||
let _ = self.release_scanner_publication_leases(grants).await;
|
||||
return Err(Error::other(format!("scanner publication lease acquisition failed: {err}")));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(grants)
|
||||
}
|
||||
|
||||
pub async fn release_scanner_publication_leases(&self, mut grants: Vec<ScannerPublicationLeaseGrant>) -> Result<()> {
|
||||
grants.sort_by(|left, right| right.host.cmp(&left.host));
|
||||
let mut failures = Vec::new();
|
||||
for grant in grants {
|
||||
let Some(client) = self
|
||||
.peer_clients
|
||||
.iter()
|
||||
.flatten()
|
||||
.find(|client| client.grid_host == grant.host)
|
||||
else {
|
||||
failures.push(format!("peer {} is unavailable", grant.host));
|
||||
continue;
|
||||
};
|
||||
if let Err(err) = client.release_scanner_publication_lease(&grant.lease).await {
|
||||
failures.push(format!("peer {} release failed: {err}", grant.host));
|
||||
}
|
||||
}
|
||||
if failures.is_empty() {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(Error::other(format!(
|
||||
"scanner publication lease release failures: {}",
|
||||
failures.join("; ")
|
||||
)))
|
||||
}
|
||||
}
|
||||
|
||||
/// Revalidate every remote lease in deterministic host order immediately
|
||||
/// before a final scanner metadata write. A peer restart removes its
|
||||
/// process-owned token table and changes its activity session, so an old
|
||||
/// generation cannot pass this proof even when the numeric generation is
|
||||
/// reused.
|
||||
pub async fn validate_scanner_publication_leases(&self, grants: &[ScannerPublicationLeaseGrant]) -> Result<()> {
|
||||
let mut grants = grants.to_vec();
|
||||
grants.sort_by(|left, right| left.host.cmp(&right.host));
|
||||
for pair in grants.windows(2) {
|
||||
if pair[0].host == pair[1].host {
|
||||
return Err(Error::other(format!("duplicate scanner publication lease target: {}", pair[0].host)));
|
||||
}
|
||||
}
|
||||
for grant in grants {
|
||||
let Some(client) = self
|
||||
.peer_clients
|
||||
.iter()
|
||||
.flatten()
|
||||
.find(|client| client.grid_host == grant.host)
|
||||
.cloned()
|
||||
else {
|
||||
return Err(Error::other(format!("scanner publication lease peer {} is unavailable", grant.host)));
|
||||
};
|
||||
client
|
||||
.validate_scanner_publication_lease(&grant.lease)
|
||||
.await
|
||||
.map_err(|err| Error::other(format!("scanner publication lease validation failed for {}: {err}", grant.host)))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn reload_site_replication_config(&self) -> Vec<NotificationPeerErr> {
|
||||
let mut futures = Vec::with_capacity(self.peer_clients.len());
|
||||
for client in self.peer_clients.iter() {
|
||||
@@ -2633,6 +2735,38 @@ mod tests {
|
||||
assert!(err.to_string().contains("no remote peers"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_publication_lease_release_reports_all_unavailable_peers() {
|
||||
let sys = NotificationSys {
|
||||
peer_clients: Vec::new(),
|
||||
all_peer_clients: Vec::new(),
|
||||
peer_topology_hosts: Vec::new(),
|
||||
peer_admin_caches: Vec::new(),
|
||||
tier_config_reload_workers: Default::default(),
|
||||
};
|
||||
let grants = ["peer-a", "peer-b"]
|
||||
.into_iter()
|
||||
.map(|host| ScannerPublicationLeaseGrant {
|
||||
host: host.to_string(),
|
||||
lease: ScannerPublicationLease {
|
||||
token: Uuid::new_v4(),
|
||||
movement_generation: 3,
|
||||
owner_id: Uuid::new_v4().to_string(),
|
||||
session_id: "session-a".to_string(),
|
||||
expires_at: Instant::now() + Duration::from_secs(30),
|
||||
},
|
||||
})
|
||||
.collect();
|
||||
|
||||
let error = sys
|
||||
.release_scanner_publication_leases(grants)
|
||||
.await
|
||||
.expect_err("an unavailable peer must not silently release a remote lease");
|
||||
let message = error.to_string();
|
||||
assert!(message.contains("peer-a"));
|
||||
assert!(message.contains("peer-b"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_activity_probe_rejects_an_incomplete_peer_topology() {
|
||||
let client = PeerRestClient::new(
|
||||
@@ -2755,6 +2889,8 @@ mod tests {
|
||||
data_movement_active: Some(false),
|
||||
dirty_usage_generation: Some(2),
|
||||
dirty_usage_pending,
|
||||
movement_generation: Some(1),
|
||||
publication_blocked: Some(false),
|
||||
};
|
||||
|
||||
let pending = aggregate_scanner_dirty_usage_acknowledgement_results(
|
||||
|
||||
@@ -3499,6 +3499,26 @@ pub(in crate::set_disk) struct RenameDataCommit {
|
||||
pub(in crate::set_disk) tail_drain: Option<tokio::task::JoinHandle<()>>,
|
||||
}
|
||||
|
||||
/// Options shared by the normal and early-ack rename fanouts. Keeping the
|
||||
/// quorum and optional scanner lease map together avoids widening either
|
||||
/// fanout helper's argument list while preserving the fence semantics.
|
||||
pub(in crate::set_disk) struct RenameDataFenceOptions<'a> {
|
||||
write_quorum: usize,
|
||||
scanner_publication_lease_tokens: Option<&'a HashMap<String, Uuid>>,
|
||||
}
|
||||
|
||||
impl<'a> RenameDataFenceOptions<'a> {
|
||||
pub(in crate::set_disk) fn new(
|
||||
write_quorum: usize,
|
||||
scanner_publication_lease_tokens: Option<&'a HashMap<String, Uuid>>,
|
||||
) -> Self {
|
||||
Self {
|
||||
write_quorum,
|
||||
scanner_publication_lease_tokens,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
type RenameDataLegacyTuple = (
|
||||
Vec<Option<DiskStore>>,
|
||||
@@ -3690,16 +3710,51 @@ impl SetDisks {
|
||||
.map(RenameDataCommit::into_legacy_tuple)
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip(disks, file_infos))]
|
||||
async fn rename_data_owned_early_ack(
|
||||
pub(in crate::set_disk) fn scanner_publication_lease_token_for_disk(
|
||||
disk: Option<&DiskStore>,
|
||||
scanner_publication_lease_tokens: Option<&HashMap<String, Uuid>>,
|
||||
) -> disk::error::Result<Option<Uuid>> {
|
||||
let Some(disk) = disk else {
|
||||
return Ok(None);
|
||||
};
|
||||
if disk.is_local() {
|
||||
return Ok(None);
|
||||
}
|
||||
let Some(tokens) = scanner_publication_lease_tokens else {
|
||||
return Ok(None);
|
||||
};
|
||||
let host = disk.endpoint().grid_host();
|
||||
tokens
|
||||
.get(&host)
|
||||
.copied()
|
||||
.ok_or_else(|| DiskError::other(format!("scanner publication lease token missing for remote disk host {host}")))
|
||||
.map(Some)
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn scanner_publication_lease_tokens_for_disks(
|
||||
disks: &[Option<DiskStore>],
|
||||
scanner_publication_lease_tokens: Option<&HashMap<String, Uuid>>,
|
||||
) -> disk::error::Result<Vec<Option<Uuid>>> {
|
||||
disks
|
||||
.iter()
|
||||
.map(|disk| Self::scanner_publication_lease_token_for_disk(disk.as_ref(), scanner_publication_lease_tokens))
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip(disks, file_infos, fence_options))]
|
||||
async fn rename_data_owned_early_ack_with_fence(
|
||||
disks: &[Option<DiskStore>],
|
||||
src_bucket: &str,
|
||||
src_object: &str,
|
||||
file_infos: Vec<FileInfo>,
|
||||
dst_bucket: &str,
|
||||
dst_object: &str,
|
||||
write_quorum: usize,
|
||||
fence_options: RenameDataFenceOptions<'_>,
|
||||
) -> disk::error::Result<RenameDataCommit> {
|
||||
let RenameDataFenceOptions {
|
||||
write_quorum,
|
||||
scanner_publication_lease_tokens,
|
||||
} = fence_options;
|
||||
if let Some(file_info) = disks
|
||||
.iter()
|
||||
.zip(file_infos.iter())
|
||||
@@ -3714,6 +3769,7 @@ impl SetDisks {
|
||||
|
||||
let disk_count = disks.len();
|
||||
let fanout_disks = disks.to_vec();
|
||||
let fanout_fence_tokens = Self::scanner_publication_lease_tokens_for_disks(disks, scanner_publication_lease_tokens)?;
|
||||
let coordinator_disks = fanout_disks.clone();
|
||||
let src_bucket = Arc::new(src_bucket.to_string());
|
||||
let src_object = Arc::new(src_object.to_string());
|
||||
@@ -3730,7 +3786,12 @@ impl SetDisks {
|
||||
let successful_rename_completion_rank =
|
||||
rustfs_io_metrics::put_stage_metrics_enabled().then(|| Arc::new(AtomicUsize::new(0)));
|
||||
let mut tasks = JoinSet::new();
|
||||
for (i, (disk, file_info)) in fanout_disks.into_iter().zip(file_infos.iter()).enumerate() {
|
||||
for (i, ((disk, file_info), scanner_publication_lease_token)) in fanout_disks
|
||||
.into_iter()
|
||||
.zip(file_infos.iter())
|
||||
.zip(fanout_fence_tokens.into_iter())
|
||||
.enumerate()
|
||||
{
|
||||
let src_bucket = fanout_src_bucket.clone();
|
||||
let src_object = fanout_src_object.clone();
|
||||
let dst_bucket = fanout_dst_bucket.clone();
|
||||
@@ -3767,7 +3828,14 @@ impl SetDisks {
|
||||
|
||||
let disk_wait_started = rustfs_io_metrics::put_stage_timer();
|
||||
let result = disk
|
||||
.rename_data_borrowed(&src_bucket, &src_object, file_info, &dst_bucket, &dst_object)
|
||||
.rename_data_borrowed_with_fence(
|
||||
&src_bucket,
|
||||
&src_object,
|
||||
file_info,
|
||||
&dst_bucket,
|
||||
&dst_object,
|
||||
scanner_publication_lease_token,
|
||||
)
|
||||
.await;
|
||||
if let Some(disk_wait_started) = disk_wait_started {
|
||||
let duration_ms = disk_wait_started.elapsed().as_secs_f64() * 1000.0;
|
||||
@@ -3967,19 +4035,45 @@ impl SetDisks {
|
||||
dst_bucket: &str,
|
||||
dst_object: &str,
|
||||
write_quorum: usize,
|
||||
) -> disk::error::Result<RenameDataCommit> {
|
||||
Self::rename_data_owned_with_fence(
|
||||
disks,
|
||||
src_bucket,
|
||||
src_object,
|
||||
file_infos,
|
||||
dst_bucket,
|
||||
dst_object,
|
||||
RenameDataFenceOptions::new(write_quorum, None),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip(disks, file_infos, fence_options))]
|
||||
pub(in crate::set_disk) async fn rename_data_owned_with_fence(
|
||||
disks: &[Option<DiskStore>],
|
||||
src_bucket: &str,
|
||||
src_object: &str,
|
||||
file_infos: Vec<FileInfo>,
|
||||
dst_bucket: &str,
|
||||
dst_object: &str,
|
||||
fence_options: RenameDataFenceOptions<'_>,
|
||||
) -> disk::error::Result<RenameDataCommit> {
|
||||
if put_rename_early_ack_enabled() {
|
||||
return Self::rename_data_owned_early_ack(
|
||||
return Self::rename_data_owned_early_ack_with_fence(
|
||||
disks,
|
||||
src_bucket,
|
||||
src_object,
|
||||
file_infos,
|
||||
dst_bucket,
|
||||
dst_object,
|
||||
write_quorum,
|
||||
fence_options,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
let RenameDataFenceOptions {
|
||||
write_quorum,
|
||||
scanner_publication_lease_tokens,
|
||||
} = fence_options;
|
||||
if let Some(file_info) = disks
|
||||
.iter()
|
||||
.zip(file_infos.iter())
|
||||
@@ -4005,6 +4099,7 @@ impl SetDisks {
|
||||
let disk_count = disks.len();
|
||||
let fanout_disks = disks.to_vec();
|
||||
let fanout_file_infos = file_infos;
|
||||
let fanout_fence_tokens = Self::scanner_publication_lease_tokens_for_disks(disks, scanner_publication_lease_tokens)?;
|
||||
let fanout_src_bucket = src_bucket.clone();
|
||||
let fanout_src_object = src_object.clone();
|
||||
let fanout_dst_bucket = dst_bucket.clone();
|
||||
@@ -4019,8 +4114,9 @@ impl SetDisks {
|
||||
let futures = fanout_disks
|
||||
.into_iter()
|
||||
.zip(fanout_file_infos.iter())
|
||||
.zip(fanout_fence_tokens.into_iter())
|
||||
.enumerate()
|
||||
.map(|(i, (disk, file_info))| {
|
||||
.map(|(i, ((disk, file_info), scanner_publication_lease_token))| {
|
||||
let src_bucket = fanout_src_bucket.clone();
|
||||
let src_object = fanout_src_object.clone();
|
||||
let dst_object = fanout_dst_object.clone();
|
||||
@@ -4060,7 +4156,14 @@ impl SetDisks {
|
||||
|
||||
let disk_wait_started = rustfs_io_metrics::put_stage_timer();
|
||||
let result = disk
|
||||
.rename_data_borrowed(&src_bucket, &src_object, file_info, &dst_bucket, &dst_object)
|
||||
.rename_data_borrowed_with_fence(
|
||||
&src_bucket,
|
||||
&src_object,
|
||||
file_info,
|
||||
&dst_bucket,
|
||||
&dst_object,
|
||||
scanner_publication_lease_token,
|
||||
)
|
||||
.await;
|
||||
if let Some(disk_wait_started) = disk_wait_started {
|
||||
let duration_ms = disk_wait_started.elapsed().as_secs_f64() * 1000.0;
|
||||
@@ -5426,18 +5529,32 @@ impl SetDisks {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(in crate::set_disk) async fn delete_prefix(&self, bucket: &str, prefix: &str) -> disk::error::Result<()> {
|
||||
self.delete_prefix_with_scanner_publication_lease(bucket, prefix, None).await
|
||||
}
|
||||
|
||||
/// Delete a prefix with an optional per-remote-disk scanner publication
|
||||
/// lease fence. The fence is transient and never reaches disk metadata;
|
||||
/// remote deletes without a matching token fail closed before the RPC.
|
||||
pub(in crate::set_disk) async fn delete_prefix_with_scanner_publication_lease(
|
||||
&self,
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
scanner_publication_lease_tokens: Option<&HashMap<String, Uuid>>,
|
||||
) -> disk::error::Result<()> {
|
||||
let disks = self.get_disks_internal().await;
|
||||
let write_quorum = disks.len() / 2 + 1;
|
||||
let fanout_fence_tokens = Self::scanner_publication_lease_tokens_for_disks(&disks, scanner_publication_lease_tokens)?;
|
||||
|
||||
let mut futures = Vec::with_capacity(disks.len());
|
||||
|
||||
for disk_op in disks.iter() {
|
||||
for (disk_op, scanner_publication_lease_token) in disks.iter().zip(fanout_fence_tokens) {
|
||||
let bucket = bucket.to_string();
|
||||
let prefix = prefix.to_string();
|
||||
futures.push(async move {
|
||||
if let Some(disk) = disk_op {
|
||||
disk.delete(
|
||||
disk.delete_with_scanner_publication_lease(
|
||||
&bucket,
|
||||
&prefix,
|
||||
DeleteOptions {
|
||||
@@ -5445,6 +5562,7 @@ impl SetDisks {
|
||||
immediate: true,
|
||||
..Default::default()
|
||||
},
|
||||
scanner_publication_lease_token,
|
||||
)
|
||||
.await
|
||||
} else {
|
||||
|
||||
@@ -49,8 +49,8 @@ use crate::data_usage::quota_object_size;
|
||||
use crate::diagnostics::get::GetObjectFailureReason;
|
||||
use crate::disk::{DataDirDeleteStatus, OldCurrentSize};
|
||||
use crate::error::is_err_invalid_upload_id;
|
||||
use crate::object_api::NamespaceLockFence;
|
||||
use crate::object_api::{GetObjectBodySource, get_object_body_cache_hook_suppressed};
|
||||
use crate::object_api::{NamespaceLockFence, SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY};
|
||||
use crate::services::notification_sys::RemoteVersionStateFleetProofToken;
|
||||
use crate::services::tier::tier::{TierConfigMgr, TierOperationLease};
|
||||
use crate::store::ECStore;
|
||||
@@ -63,6 +63,60 @@ use std::sync::OnceLock;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
const OLD_DATA_CLEANUP_RECEIPT_FILE: &str = ".rustfs-old-data-cleanup-receipt.json";
|
||||
const SCANNER_PUBLICATION_LEASE_FENCE_MAX_BYTES: usize = 64 * 1024;
|
||||
const SCANNER_PUBLICATION_LEASE_FENCE_MAX_ENTRIES: usize = 256;
|
||||
|
||||
fn take_scanner_publication_lease_tokens(user_defined: &mut HashMap<String, String>) -> Result<Option<HashMap<String, Uuid>>> {
|
||||
let Some(encoded) = user_defined.remove(SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY) else {
|
||||
return Ok(None);
|
||||
};
|
||||
if encoded.len() > SCANNER_PUBLICATION_LEASE_FENCE_MAX_BYTES {
|
||||
return Err(Error::other("scanner publication lease fence is too large"));
|
||||
}
|
||||
let encoded_tokens = serde_json::from_str::<HashMap<String, String>>(&encoded)
|
||||
.map_err(|err| Error::other(format!("invalid scanner publication lease fence: {err}")))?;
|
||||
if encoded_tokens.len() > SCANNER_PUBLICATION_LEASE_FENCE_MAX_ENTRIES {
|
||||
return Err(Error::other("scanner publication lease fence has too many entries"));
|
||||
}
|
||||
let mut tokens = HashMap::with_capacity(encoded_tokens.len());
|
||||
for (host, token) in encoded_tokens {
|
||||
if host.is_empty() || host.len() > 1024 {
|
||||
return Err(Error::other("invalid scanner publication lease fence host"));
|
||||
}
|
||||
let token = Uuid::parse_str(&token)
|
||||
.map_err(|err| Error::other(format!("invalid scanner publication lease fence token: {err}")))?;
|
||||
if token.is_nil() {
|
||||
return Err(Error::other("invalid scanner publication lease fence token"));
|
||||
}
|
||||
tokens.insert(host, token);
|
||||
}
|
||||
Ok(Some(tokens))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod scanner_publication_lease_fence_tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn scanner_publication_lease_fence_is_transient_and_validated() {
|
||||
let token = Uuid::new_v4();
|
||||
let mut metadata = HashMap::from([(
|
||||
SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY.to_string(),
|
||||
serde_json::json!({"http://node-a:9000": token.to_string()}).to_string(),
|
||||
)]);
|
||||
let parsed = take_scanner_publication_lease_tokens(&mut metadata)
|
||||
.expect("valid scanner publication lease fence should parse")
|
||||
.expect("scanner publication lease fence should be present");
|
||||
assert_eq!(parsed.get("http://node-a:9000"), Some(&token));
|
||||
assert!(!metadata.contains_key(SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY));
|
||||
|
||||
let mut malformed = HashMap::from([(
|
||||
SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY.to_string(),
|
||||
r#"{"http://node-a:9000":"not-a-uuid"}"#.to_string(),
|
||||
)]);
|
||||
assert!(take_scanner_publication_lease_tokens(&mut malformed).is_err());
|
||||
}
|
||||
}
|
||||
|
||||
struct PutObjectCommitCancellation {
|
||||
token: CancellationToken,
|
||||
@@ -2177,6 +2231,7 @@ impl SetDisks {
|
||||
user_defined.insert(key.clone(), value.clone());
|
||||
}
|
||||
}
|
||||
let scanner_publication_lease_tokens = take_scanner_publication_lease_tokens(&mut user_defined)?;
|
||||
if replication_lww_applicable(opts) {
|
||||
// Object Lock evaluation stamps category timestamps with this
|
||||
// receiver's clock. Pin them back to the source-authored times
|
||||
@@ -2896,6 +2951,7 @@ impl SetDisks {
|
||||
let commit_bucket_lifecycle_lock_fence = opts.bucket_lifecycle_lock_fence.clone();
|
||||
let commit_capacity_scope_token = opts.capacity_scope_token;
|
||||
let commit_replication_state = replication_state_to_filemeta(&opts.put_replication_state());
|
||||
let commit_scanner_publication_lease_tokens = scanner_publication_lease_tokens;
|
||||
tmp_cleanup_owned = true;
|
||||
|
||||
let commit = move |cancellation: Option<CancellationToken>| async move {
|
||||
@@ -3015,14 +3071,17 @@ impl SetDisks {
|
||||
}
|
||||
|
||||
Self::assign_rename_data_indexes(&mut parts_metadatas);
|
||||
let rename_result = SetDisks::rename_data_owned(
|
||||
let rename_result = SetDisks::rename_data_owned_with_fence(
|
||||
&commit_disks,
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
commit_tmp_dir.as_str(),
|
||||
parts_metadatas,
|
||||
&commit_bucket,
|
||||
&commit_object,
|
||||
write_quorum,
|
||||
crate::set_disk::core::io_primitives::RenameDataFenceOptions::new(
|
||||
write_quorum,
|
||||
commit_scanner_publication_lease_tokens.as_ref(),
|
||||
),
|
||||
)
|
||||
.await;
|
||||
if quota_mutation_fence {
|
||||
@@ -6418,6 +6477,10 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
|
||||
#[tracing::instrument(skip(self))]
|
||||
async fn delete_object(&self, bucket: &str, object: &str, mut opts: ObjectOptions) -> Result<ObjectInfo> {
|
||||
// Scanner cleanup carries the per-peer lease fence as transient
|
||||
// request metadata. Consume it before any delete-prefix fanout so it
|
||||
// cannot be persisted or treated as user metadata.
|
||||
let scanner_publication_lease_tokens = take_scanner_publication_lease_tokens(&mut opts.user_defined)?;
|
||||
let preserve_delete_replication_state = should_preserve_delete_replication_state(&opts);
|
||||
let delete_config_snapshot = if opts.delete_prefix || opts.transition.expire_restored || preserve_delete_replication_state
|
||||
{
|
||||
@@ -6544,7 +6607,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
self.validate_bucket_incarnation(bucket, expected_incarnation_id).await?;
|
||||
}
|
||||
ensure_delete_commit_locks_held(_lock_guard.as_ref(), bucket, object, &opts)?;
|
||||
self.delete_prefix(bucket, object)
|
||||
self.delete_prefix_with_scanner_publication_lease(bucket, object, scanner_publication_lease_tokens.as_ref())
|
||||
.await
|
||||
.map_err(|e| to_object_err(e.into(), vec![bucket, object]))?;
|
||||
|
||||
|
||||
@@ -32,7 +32,8 @@ pub(crate) mod internode {
|
||||
PUT_FILE_AUTH_TRAILER_MAGIC, PUT_FILE_AUTH_V1, PUT_FILE_CAPABILITY_CHALLENGE_QUERY, PUT_FILE_CAPABILITY_QUERY,
|
||||
PUT_FILE_CAPABILITY_VERSION, PUT_FILE_NONCE_QUERY, PUT_FILE_SERVER_EPOCH_QUERY, PutFileCapabilityResponse,
|
||||
SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION, SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SCANNER_ACTIVITY_PROTOCOL_VERSION,
|
||||
WALK_DIR_BODY_SHA256_QUERY, WALK_DIR_STREAM_COMPLETION_QUERY, WALK_DIR_STREAM_COMPLETION_V1,
|
||||
SCANNER_ACTIVITY_V6_PROTOCOL_VERSION, WALK_DIR_BODY_SHA256_QUERY, WALK_DIR_STREAM_COMPLETION_QUERY,
|
||||
WALK_DIR_STREAM_COMPLETION_V1,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -87,6 +87,8 @@ type ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
|
||||
type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>;
|
||||
type WalkOptions = StorageWalkOptions<fn(&FileInfo) -> bool>;
|
||||
|
||||
pub const SCANNER_PUBLICATION_LEASE_TTL_MS: u64 = 60_000;
|
||||
|
||||
/// Check if a directory contains any xl.meta files (indicating actual S3 objects)
|
||||
/// This is used to determine if a bucket is empty for deletion purposes.
|
||||
pub(crate) async fn has_xlmeta_files(path: &std::path::Path) -> std::io::Result<bool> {
|
||||
@@ -344,6 +346,33 @@ impl ECStore {
|
||||
decommission || rebalance
|
||||
}
|
||||
|
||||
/// Return the storage-owned movement state and generation as one
|
||||
/// authenticated activity snapshot. The read lock is acquired before
|
||||
/// the state locks (cancelers, pool metadata, then rebalance metadata),
|
||||
/// matching the transition writer order and preventing a terminal state
|
||||
/// from being reported with the preceding generation.
|
||||
pub async fn scanner_data_movement_activity(&self) -> (bool, bool, u64) {
|
||||
let operation_gate = self.ctx.data_movement_operation_gate();
|
||||
let _operation_guard = operation_gate.read_owned().await;
|
||||
let (active, blocked) = self.scanner_data_movement_snapshot_locked().await;
|
||||
let blocked =
|
||||
blocked || self.ctx.data_movement_operation_epoch_exhausted() || self.ctx.data_movement_generation_exhausted();
|
||||
self.ctx.set_scanner_publication_state(blocked);
|
||||
(active, blocked, self.ctx.data_movement_generation())
|
||||
}
|
||||
|
||||
pub fn scanner_data_movement_generation(&self) -> u64 {
|
||||
self.ctx.data_movement_generation()
|
||||
}
|
||||
|
||||
pub fn scanner_data_movement_generation_exhausted(&self) -> bool {
|
||||
self.ctx.data_movement_generation_exhausted()
|
||||
}
|
||||
|
||||
pub fn scanner_data_movement_changed(&self) -> std::sync::Arc<tokio::sync::Notify> {
|
||||
self.ctx.data_movement_generation_notify()
|
||||
}
|
||||
|
||||
/// Returns whether scanner metadata may still be hidden by a local
|
||||
/// data-movement state. Terminal failed/canceled decommission entries
|
||||
/// remain suspended until an operator clears or retries them, so they are
|
||||
@@ -355,10 +384,16 @@ impl ECStore {
|
||||
}
|
||||
|
||||
async fn scanner_data_usage_publication_snapshot_blocked(&self) -> bool {
|
||||
if self.ctx.data_movement_operation_epoch_exhausted() {
|
||||
if self.ctx.data_movement_operation_epoch_exhausted() || self.ctx.data_movement_generation_exhausted() {
|
||||
self.ctx.set_scanner_publication_state(true);
|
||||
return true;
|
||||
}
|
||||
let (_, blocked) = self.scanner_data_movement_snapshot_locked().await;
|
||||
self.ctx.set_scanner_publication_state(blocked);
|
||||
blocked
|
||||
}
|
||||
|
||||
async fn scanner_data_movement_snapshot_locked(&self) -> (bool, bool) {
|
||||
let decommission_cancelers = self.decommission_cancelers.read().await;
|
||||
let decommission_active = decommission_cancelers
|
||||
.iter()
|
||||
@@ -385,8 +420,7 @@ impl ECStore {
|
||||
.is_some_and(is_rebalance_conflicting_with_decommission);
|
||||
|
||||
let blocked = decommission_active || decommission_terminal || rebalance_active;
|
||||
self.ctx.set_scanner_publication_state(blocked);
|
||||
blocked
|
||||
(decommission_active || rebalance_active, blocked)
|
||||
}
|
||||
|
||||
/// Admit one short data-usage publication commit under the same
|
||||
@@ -407,7 +441,7 @@ impl ECStore {
|
||||
pub async fn scanner_data_usage_publication_admission_guard(&self) -> Option<(tokio::sync::OwnedRwLockReadGuard<()>, u64)> {
|
||||
let operation_gate = self.ctx.data_movement_operation_gate();
|
||||
let operation_guard = operation_gate.read_owned().await;
|
||||
if self.ctx.data_movement_operation_epoch_exhausted() {
|
||||
if self.ctx.data_movement_operation_epoch_exhausted() || self.ctx.data_movement_generation_exhausted() {
|
||||
return None;
|
||||
}
|
||||
if self.scanner_data_usage_publication_snapshot_blocked().await {
|
||||
@@ -425,6 +459,98 @@ impl ECStore {
|
||||
drop(operation_guard);
|
||||
Some(epoch)
|
||||
}
|
||||
|
||||
/// Acquire a storage-owned read admission for a coordinator's final
|
||||
/// scanner publication. The guard remains in the context's lease table,
|
||||
/// so a local movement writer cannot pass the peer while its authoritative
|
||||
/// PUT is in flight.
|
||||
pub async fn acquire_scanner_publication_lease(
|
||||
&self,
|
||||
expected_generation: u64,
|
||||
ttl: std::time::Duration,
|
||||
) -> Result<(Uuid, u64)> {
|
||||
if ttl != crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL {
|
||||
return Err(Error::other("scanner publication lease TTL is not supported"));
|
||||
}
|
||||
|
||||
let operation_gate = self.ctx.data_movement_operation_gate();
|
||||
let operation_guard = operation_gate.read_owned().await;
|
||||
if self.ctx.data_movement_generation_exhausted()
|
||||
|| self.ctx.data_movement_operation_epoch_exhausted()
|
||||
|| self.ctx.data_movement_generation() != expected_generation
|
||||
{
|
||||
return Err(Error::other("scanner publication lease generation is stale"));
|
||||
}
|
||||
if self.scanner_data_movement_snapshot_locked().await.1 {
|
||||
return Err(Error::other("scanner publication lease is blocked by data movement"));
|
||||
}
|
||||
|
||||
let token = Uuid::new_v4();
|
||||
let expires_at = tokio::time::Instant::now() + ttl;
|
||||
if !self
|
||||
.ctx
|
||||
.install_scanner_publication_lease(token, expires_at, expected_generation, operation_guard)
|
||||
.await
|
||||
{
|
||||
return Err(Error::other("scanner publication lease capacity is exhausted"));
|
||||
}
|
||||
|
||||
let context = Arc::clone(&self.ctx);
|
||||
tokio::spawn(async move {
|
||||
tokio::time::sleep_until(expires_at).await;
|
||||
context.expire_scanner_publication_lease(token, expires_at).await;
|
||||
});
|
||||
Ok((token, expected_generation))
|
||||
}
|
||||
|
||||
pub async fn release_scanner_publication_lease(&self, token: Uuid) -> bool {
|
||||
self.ctx.remove_scanner_publication_lease(token).await
|
||||
}
|
||||
|
||||
/// Revalidate a previously acquired remote publication lease immediately
|
||||
/// before the coordinator's final metadata write. The operation read
|
||||
/// guard makes the movement snapshot and token lookup one storage-owned
|
||||
/// admission; a restarted context has no old token and therefore fails
|
||||
/// closed even if its generation counter has returned to zero.
|
||||
pub async fn validate_scanner_publication_lease(&self, token: Uuid, expected_generation: u64) -> Result<()> {
|
||||
let _operation_guard = self.acquire_scanner_publication_lease_guard(token).await?;
|
||||
if self.ctx.data_movement_generation_exhausted()
|
||||
|| self.ctx.data_movement_operation_epoch_exhausted()
|
||||
|| self.ctx.data_movement_generation() != expected_generation
|
||||
{
|
||||
return Err(Error::other("scanner publication lease generation is stale"));
|
||||
}
|
||||
if self.scanner_data_movement_snapshot_locked().await.1 {
|
||||
return Err(Error::other("scanner publication lease is blocked by data movement"));
|
||||
}
|
||||
if !self.ctx.scanner_publication_lease_is_active(token).await {
|
||||
return Err(Error::other("scanner publication lease is unknown or expired"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Acquire the target-side read guard bound to a previously granted lease.
|
||||
/// The guard is returned to the RPC handler and must remain alive through
|
||||
/// the complete rename/write operation. A lease token is process-owned;
|
||||
/// restart, expiry, generation changes, or blocked movement all reject it
|
||||
/// before the target disk is touched.
|
||||
pub async fn acquire_scanner_publication_lease_guard(&self, token: Uuid) -> Result<tokio::sync::OwnedRwLockReadGuard<()>> {
|
||||
let operation_gate = self.ctx.data_movement_operation_gate();
|
||||
let operation_guard = operation_gate.read_owned().await;
|
||||
if self.ctx.data_movement_generation_exhausted() || self.ctx.data_movement_operation_epoch_exhausted() {
|
||||
return Err(Error::other("scanner publication lease generation is exhausted"));
|
||||
}
|
||||
if self.scanner_data_movement_snapshot_locked().await.1 {
|
||||
return Err(Error::other("scanner publication lease is blocked by data movement"));
|
||||
}
|
||||
let Some(lease_generation) = self.ctx.scanner_publication_lease_generation(token).await else {
|
||||
return Err(Error::other("scanner publication lease is unknown or expired"));
|
||||
};
|
||||
if lease_generation != self.ctx.data_movement_generation() {
|
||||
return Err(Error::other("scanner publication lease generation is stale"));
|
||||
}
|
||||
Ok(operation_guard)
|
||||
}
|
||||
}
|
||||
|
||||
// impl Clone for ECStore {
|
||||
@@ -1084,6 +1210,149 @@ mod tests {
|
||||
.await
|
||||
.expect("idle store should admit the next publication");
|
||||
assert_eq!(next_epoch, 1);
|
||||
assert_eq!(store.scanner_data_movement_generation(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn scanner_publication_lease_blocks_movement_writer_until_release_or_expiry() {
|
||||
let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
|
||||
let (token, generation) = store
|
||||
.acquire_scanner_publication_lease(0, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
|
||||
.await
|
||||
.expect("an idle store should grant a publication lease");
|
||||
assert_eq!(generation, 0);
|
||||
|
||||
let gate = store.ctx.data_movement_operation_gate();
|
||||
let (writer_started, writer_started_rx) = tokio::sync::oneshot::channel();
|
||||
let movement_writer = tokio::spawn(async move {
|
||||
let _ = writer_started.send(());
|
||||
gate.write_owned().await
|
||||
});
|
||||
writer_started_rx
|
||||
.await
|
||||
.expect("movement writer should reach the gate before waiting");
|
||||
assert!(
|
||||
!movement_writer.is_finished(),
|
||||
"a movement writer must wait while the remote lease owns the read guard"
|
||||
);
|
||||
|
||||
assert!(store.release_scanner_publication_lease(token).await);
|
||||
tokio::time::timeout(Duration::from_secs(1), movement_writer)
|
||||
.await
|
||||
.expect("movement writer should proceed after lease release")
|
||||
.expect("movement writer task should not panic");
|
||||
|
||||
let (expiring_token, _) = store
|
||||
.acquire_scanner_publication_lease(0, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
|
||||
.await
|
||||
.expect("the store should grant a second publication lease");
|
||||
let expiry_gate = store.ctx.data_movement_operation_gate();
|
||||
let (expiry_started, expiry_started_rx) = tokio::sync::oneshot::channel();
|
||||
let mut expiry_writer = tokio::spawn(async move {
|
||||
let _ = expiry_started.send(());
|
||||
expiry_gate.write_owned().await
|
||||
});
|
||||
expiry_started_rx
|
||||
.await
|
||||
.expect("expiry writer should reach the gate before waiting");
|
||||
assert!(!expiry_writer.is_finished());
|
||||
tokio::time::advance(crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL + Duration::from_millis(1)).await;
|
||||
tokio::task::yield_now().await;
|
||||
tokio::time::timeout(Duration::from_secs(1), &mut expiry_writer)
|
||||
.await
|
||||
.expect("movement writer should proceed after lease expiry")
|
||||
.expect("expiry writer task should not panic");
|
||||
assert!(!store.release_scanner_publication_lease(expiring_token).await);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_publication_lease_rejects_stale_generation_before_install() {
|
||||
let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
|
||||
let error = store
|
||||
.acquire_scanner_publication_lease(1, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
|
||||
.await
|
||||
.expect_err("a stale movement generation must not acquire a lease");
|
||||
assert!(error.to_string().contains("generation is stale"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_target_guard_keeps_movement_writer_fenced_after_lease_release() {
|
||||
let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
|
||||
let (token, _) = store
|
||||
.acquire_scanner_publication_lease(0, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
|
||||
.await
|
||||
.expect("an idle store should grant a publication lease");
|
||||
let target_guard = store
|
||||
.acquire_scanner_publication_lease_guard(token)
|
||||
.await
|
||||
.expect("the target-side rename should acquire its short guard");
|
||||
assert!(store.release_scanner_publication_lease(token).await);
|
||||
|
||||
let movement_gate = store.ctx.data_movement_operation_gate();
|
||||
let movement_writer = tokio::spawn(async move { movement_gate.write_owned().await });
|
||||
tokio::task::yield_now().await;
|
||||
assert!(!movement_writer.is_finished(), "the target guard must span the rename operation");
|
||||
drop(target_guard);
|
||||
tokio::time::timeout(std::time::Duration::from_secs(1), movement_writer)
|
||||
.await
|
||||
.expect("movement writer should proceed after the target rename guard is dropped")
|
||||
.expect("movement writer task should not panic");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_publication_lease_rejects_restart_aba_token() {
|
||||
let first_store = build_store_with_ctx(Arc::new(InstanceContext::new()));
|
||||
let (token, generation) = first_store
|
||||
.acquire_scanner_publication_lease(0, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
|
||||
.await
|
||||
.expect("the initial instance should grant a publication lease");
|
||||
first_store
|
||||
.validate_scanner_publication_lease(token, generation)
|
||||
.await
|
||||
.expect("the current instance should validate its own live token");
|
||||
first_store
|
||||
.acquire_scanner_publication_lease_guard(token)
|
||||
.await
|
||||
.expect("the current instance should admit the target-side rename");
|
||||
|
||||
// A restarted storage instance starts its local generation at zero,
|
||||
// but its process-owned lease table is empty. The old token must not
|
||||
// pass validation just because the numeric generation matches again.
|
||||
let restarted_store = build_store_with_ctx(Arc::new(InstanceContext::new()));
|
||||
let error = restarted_store
|
||||
.validate_scanner_publication_lease(token, generation)
|
||||
.await
|
||||
.expect_err("a token from a prior instance must fail closed after restart");
|
||||
assert!(error.to_string().contains("unknown or expired"));
|
||||
let error = restarted_store
|
||||
.acquire_scanner_publication_lease_guard(token)
|
||||
.await
|
||||
.expect_err("a restarted instance must reject the target-side rename token");
|
||||
assert!(error.to_string().contains("unknown or expired"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn movement_generation_notifies_waiters_and_fails_closed_at_maximum() {
|
||||
let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
|
||||
let notify = store.scanner_data_movement_changed();
|
||||
let notified = notify.notified();
|
||||
tokio::pin!(notified);
|
||||
notified.as_mut().enable();
|
||||
|
||||
assert_eq!(store.ctx.advance_data_movement_operation_epoch(), 1);
|
||||
tokio::time::timeout(std::time::Duration::from_secs(1), notified)
|
||||
.await
|
||||
.expect("movement transition should wake scanner waiters");
|
||||
assert_eq!(store.scanner_data_movement_generation(), 1);
|
||||
|
||||
store.ctx.set_data_movement_generation_for_test(u64::MAX - 1);
|
||||
assert_eq!(store.ctx.advance_data_movement_generation(), Some(u64::MAX));
|
||||
assert!(store.scanner_data_movement_generation_exhausted());
|
||||
assert_eq!(store.ctx.advance_data_movement_generation(), None);
|
||||
assert!(
|
||||
store.scanner_data_usage_publication_admission_guard().await.is_none(),
|
||||
"generation exhaustion must close publication admission"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
Reference in New Issue
Block a user