mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-29 16:37:07 +00:00
fix(scanner): fence movement generation publication (#6461)
* feat(scanner): add movement generation fencing * fix(scanner): prioritize unverified cycle deferral * feat(ecstore): add scanner publication lease fence * feat(rpc): add scanner publication lease protocol * feat(scanner): hold remote leases through usage publish * test(scanner): cover publication lease fencing * fix(scanner): fence remote leases across restart and delay * feat(rpc): fence scanner publication rename writes * fix(scanner): fence observed cleanup deletes * fix(proto): qualify lease release test types * fix(scanner): pin movement notifications * fix(scanner): clean publication imports * fix(ecstore): satisfy scanner fence clippy * refactor(scanner): group wait and publication options * fix(scanner): satisfy final lint and facade guards * fix(rpc): resolve facade export conflicts * fix(ci): remove unused decommission and healing facades * fix(ci): cfg-gate test-only usage overlay import * fix(scanner): wake on remote scanner restart
This commit is contained in:
@@ -24,9 +24,9 @@ use crate::storage::storage_api::rpc_consumer::node_service::STORAGE_CLASS_SUB_S
|
||||
use crate::storage::storage_api::rpc_consumer::node_service::{CollectMetricsOpts, MetricType};
|
||||
use crate::storage::storage_api::rpc_consumer::node_service::{
|
||||
DiskStore, ECStore, Error, KMS_SIGNAL_SUBSYSTEM, LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS,
|
||||
SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION, SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SERVICE_SIGNAL_REFRESH_CONFIG,
|
||||
SERVICE_SIGNAL_RELOAD_DYNAMIC, StorageDiskRpcExt as _, StorageResult, all_local_disk_path, find_local_disk_by_ref,
|
||||
reload_transition_tier_config,
|
||||
SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION, SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SCANNER_ACTIVITY_V6_PROTOCOL_VERSION,
|
||||
SCANNER_PUBLICATION_LEASE_TTL_MS, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, StorageDiskRpcExt as _,
|
||||
StorageResult, all_local_disk_path, find_local_disk_by_ref, reload_transition_tier_config,
|
||||
};
|
||||
use crate::storage::storage_api::runtime_sources_consumer::{EndpointServerPools, runtime_sources};
|
||||
use crate::storage::storage_api::{
|
||||
@@ -237,9 +237,25 @@ fn scanner_activity_response(
|
||||
response_proof: Bytes::new(),
|
||||
dirty_usage_generation: dirty_usage.generation,
|
||||
dirty_usage_pending: dirty_usage.pending,
|
||||
movement_generation: None,
|
||||
publication_blocked: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn scanner_activity_response_v7(
|
||||
namespace_generation: u64,
|
||||
topology_digest: [u8; 32],
|
||||
data_movement_active: bool,
|
||||
dirty_usage: rustfs_scanner::ScannerDirtyUsageState,
|
||||
movement_generation: u64,
|
||||
publication_blocked: bool,
|
||||
) -> ScannerActivityResponse {
|
||||
let mut response = scanner_activity_response(namespace_generation, topology_digest, data_movement_active, dirty_usage);
|
||||
response.movement_generation = Some(movement_generation);
|
||||
response.publication_blocked = Some(publication_blocked);
|
||||
response
|
||||
}
|
||||
|
||||
fn previous_scanner_activity_response(
|
||||
namespace_generation: u64,
|
||||
topology_digest: [u8; 32],
|
||||
@@ -255,6 +271,8 @@ fn previous_scanner_activity_response(
|
||||
response_proof: Bytes::new(),
|
||||
dirty_usage_generation: 0,
|
||||
dirty_usage_pending: false,
|
||||
movement_generation: None,
|
||||
publication_blocked: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -269,6 +287,29 @@ fn legacy_scanner_activity_response(namespace_generation: u64) -> ScannerActivit
|
||||
response_proof: Bytes::new(),
|
||||
dirty_usage_generation: 0,
|
||||
dirty_usage_pending: false,
|
||||
movement_generation: None,
|
||||
publication_blocked: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn v6_scanner_activity_response(
|
||||
namespace_generation: u64,
|
||||
topology_digest: [u8; 32],
|
||||
data_movement_active: bool,
|
||||
dirty_usage: rustfs_scanner::ScannerDirtyUsageState,
|
||||
) -> ScannerActivityResponse {
|
||||
ScannerActivityResponse {
|
||||
instance_id: rustfs_scanner::scanner_activity_epoch().to_string(),
|
||||
namespace_generation,
|
||||
maintenance_generation: rustfs_scanner::scanner_maintenance_generation(),
|
||||
protocol_version: SCANNER_ACTIVITY_V6_PROTOCOL_VERSION,
|
||||
topology_digest: topology_digest.to_vec().into(),
|
||||
data_movement_active,
|
||||
response_proof: Bytes::new(),
|
||||
dirty_usage_generation: dirty_usage.generation,
|
||||
dirty_usage_pending: dirty_usage.pending,
|
||||
movement_generation: None,
|
||||
publication_blocked: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1826,7 +1867,7 @@ impl Node for NodeService {
|
||||
return Err(Status::invalid_argument("scanner activity protocol v4 cannot acknowledge dirty usage"));
|
||||
}
|
||||
}
|
||||
rustfs_scanner::SCANNER_ACTIVITY_PROTOCOL_VERSION => {
|
||||
SCANNER_ACTIVITY_V6_PROTOCOL_VERSION | rustfs_scanner::SCANNER_ACTIVITY_PROTOCOL_VERSION => {
|
||||
let canonical = rustfs_protos::canonical_scanner_activity_request_body(request.get_ref())
|
||||
.map_err(|_| Status::invalid_argument("scanner activity request is too large to authenticate"))?;
|
||||
verify_tonic_canonical_body_digest(&request, &canonical)
|
||||
@@ -1873,17 +1914,25 @@ impl Node for NodeService {
|
||||
}
|
||||
let namespace_generation = store.scanner_namespace_mutation_generation();
|
||||
let topology_digest = rustfs_scanner::scanner_topology_digest(store.as_ref());
|
||||
let data_movement_active = store.scanner_data_movement_active().await;
|
||||
let (data_movement_active, publication_blocked, movement_generation) = store.scanner_data_movement_activity().await;
|
||||
let mut response = match request_protocol {
|
||||
SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION | SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION => {
|
||||
previous_scanner_activity_response(namespace_generation, topology_digest, data_movement_active)
|
||||
}
|
||||
rustfs_scanner::SCANNER_ACTIVITY_PROTOCOL_VERSION => scanner_activity_response(
|
||||
SCANNER_ACTIVITY_V6_PROTOCOL_VERSION => v6_scanner_activity_response(
|
||||
namespace_generation,
|
||||
topology_digest,
|
||||
data_movement_active,
|
||||
rustfs_scanner::scanner_dirty_usage_state(),
|
||||
),
|
||||
rustfs_scanner::SCANNER_ACTIVITY_PROTOCOL_VERSION => scanner_activity_response_v7(
|
||||
namespace_generation,
|
||||
topology_digest,
|
||||
data_movement_active,
|
||||
rustfs_scanner::scanner_dirty_usage_state(),
|
||||
movement_generation,
|
||||
publication_blocked || store.scanner_data_movement_generation_exhausted(),
|
||||
),
|
||||
version => {
|
||||
return Err(Status::failed_precondition(format!(
|
||||
"unsupported scanner activity request protocol {version}"
|
||||
@@ -1894,9 +1943,12 @@ impl Node for NodeService {
|
||||
SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION => {
|
||||
rustfs_protos::canonical_scanner_activity_v4_response_body(&challenge, &response)
|
||||
}
|
||||
rustfs_scanner::SCANNER_ACTIVITY_PROTOCOL_VERSION => {
|
||||
SCANNER_ACTIVITY_V6_PROTOCOL_VERSION => {
|
||||
rustfs_protos::canonical_scanner_activity_response_body(&challenge, &response)
|
||||
}
|
||||
rustfs_scanner::SCANNER_ACTIVITY_PROTOCOL_VERSION => {
|
||||
rustfs_protos::canonical_scanner_activity_v7_response_body(&challenge, &response)
|
||||
}
|
||||
version => {
|
||||
return Err(Status::internal(format!(
|
||||
"scanner activity response selected unsupported protocol {version}"
|
||||
@@ -1910,6 +1962,133 @@ impl Node for NodeService {
|
||||
Ok(Response::new(response))
|
||||
}
|
||||
|
||||
async fn acquire_scanner_publication_lease(
|
||||
&self,
|
||||
request: Request<ScannerPublicationLeaseRequest>,
|
||||
) -> Result<Response<ScannerPublicationLeaseResponse>, Status> {
|
||||
let canonical = rustfs_protos::canonical_scanner_publication_lease_request_body(request.get_ref())
|
||||
.map_err(|_| Status::invalid_argument("scanner publication lease request is too large to authenticate"))?;
|
||||
verify_tonic_canonical_body_digest(&request, &canonical)
|
||||
.map_err(|err| Status::permission_denied(format!("scanner publication lease authentication failed: {err}")))?;
|
||||
if request.get_ref().challenge.len() != 16 {
|
||||
return Err(Status::invalid_argument("scanner publication lease challenge must be 16 bytes"));
|
||||
}
|
||||
if request.get_ref().ttl_ms != SCANNER_PUBLICATION_LEASE_TTL_MS {
|
||||
return Err(Status::invalid_argument("scanner publication lease TTL is unsupported"));
|
||||
}
|
||||
let session_id = rustfs_scanner::scanner_activity_epoch().to_string();
|
||||
if request.get_ref().expected_session_id != session_id {
|
||||
return Err(Status::failed_precondition("scanner publication lease session is stale"));
|
||||
}
|
||||
let validation_token = if request.get_ref().token.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(
|
||||
Uuid::from_slice(request.get_ref().token.as_ref())
|
||||
.map_err(|_| Status::invalid_argument("scanner publication lease token must be a UUID"))?,
|
||||
)
|
||||
};
|
||||
let challenge = request.get_ref().challenge.clone();
|
||||
let request = request.into_inner();
|
||||
let store = self
|
||||
.resolve_object_store()
|
||||
.ok_or_else(|| Status::unavailable("storage layer is not initialized"))?;
|
||||
if store.id.is_nil() {
|
||||
return Err(Status::unavailable("storage owner identity is not initialized"));
|
||||
}
|
||||
let owner_id = store.id.to_string();
|
||||
let result = match validation_token {
|
||||
Some(token) => store
|
||||
.validate_scanner_publication_lease(token, request.expected_movement_generation)
|
||||
.await
|
||||
.map(|()| (token, request.expected_movement_generation)),
|
||||
None => {
|
||||
store
|
||||
.acquire_scanner_publication_lease(
|
||||
request.expected_movement_generation,
|
||||
Duration::from_millis(request.ttl_ms),
|
||||
)
|
||||
.await
|
||||
}
|
||||
};
|
||||
let mut response = match result {
|
||||
Ok((token, generation)) => ScannerPublicationLeaseResponse {
|
||||
success: true,
|
||||
token: token.as_bytes().to_vec().into(),
|
||||
movement_generation: generation,
|
||||
lease_ttl_ms: request.ttl_ms,
|
||||
error: None,
|
||||
response_proof: Bytes::new(),
|
||||
owner_id: owner_id.clone(),
|
||||
session_id: session_id.clone(),
|
||||
},
|
||||
Err(err) => ScannerPublicationLeaseResponse {
|
||||
success: false,
|
||||
token: Bytes::new(),
|
||||
movement_generation: store.scanner_data_movement_generation(),
|
||||
lease_ttl_ms: 0,
|
||||
error: Some(rustfs_protos::proto_gen::node_service::Error {
|
||||
code: 1,
|
||||
error_info: err.to_string(),
|
||||
}),
|
||||
response_proof: Bytes::new(),
|
||||
owner_id: owner_id.clone(),
|
||||
session_id: session_id.clone(),
|
||||
},
|
||||
};
|
||||
let response_body = rustfs_protos::canonical_scanner_publication_lease_response_body(&challenge, &response)
|
||||
.map_err(|_| Status::internal("scanner publication lease response is too large to authenticate"))?;
|
||||
response.response_proof = sign_tonic_rpc_response_proof(&response_body)
|
||||
.map_err(|_| Status::unavailable("scanner publication lease response authentication is unavailable"))?
|
||||
.into();
|
||||
Ok(Response::new(response))
|
||||
}
|
||||
|
||||
async fn release_scanner_publication_lease(
|
||||
&self,
|
||||
request: Request<ScannerPublicationLeaseReleaseRequest>,
|
||||
) -> Result<Response<ScannerPublicationLeaseReleaseResponse>, Status> {
|
||||
let canonical = rustfs_protos::canonical_scanner_publication_lease_release_request_body(request.get_ref())
|
||||
.map_err(|_| Status::invalid_argument("scanner publication lease release request is too large to authenticate"))?;
|
||||
verify_tonic_canonical_body_digest(&request, &canonical).map_err(|err| {
|
||||
Status::permission_denied(format!("scanner publication lease release authentication failed: {err}"))
|
||||
})?;
|
||||
if request.get_ref().challenge.len() != 16 {
|
||||
return Err(Status::invalid_argument("scanner publication lease challenge must be 16 bytes"));
|
||||
}
|
||||
let store = self
|
||||
.resolve_object_store()
|
||||
.ok_or_else(|| Status::unavailable("storage layer is not initialized"))?;
|
||||
if store.id.is_nil() {
|
||||
return Err(Status::unavailable("storage owner identity is not initialized"));
|
||||
}
|
||||
let owner_id = store.id.to_string();
|
||||
let session_id = rustfs_scanner::scanner_activity_epoch().to_string();
|
||||
if request.get_ref().owner_id != owner_id || request.get_ref().session_id != session_id {
|
||||
return Err(Status::failed_precondition("scanner publication lease owner or session is stale"));
|
||||
}
|
||||
let token = Uuid::from_slice(request.get_ref().token.as_ref())
|
||||
.map_err(|_| Status::invalid_argument("scanner publication lease token must be a UUID"))?;
|
||||
let challenge = request.get_ref().challenge.clone();
|
||||
let request = request.into_inner();
|
||||
let released = store.release_scanner_publication_lease(token).await;
|
||||
let mut response = ScannerPublicationLeaseReleaseResponse {
|
||||
success: released,
|
||||
error: (!released).then(|| rustfs_protos::proto_gen::node_service::Error {
|
||||
code: 1,
|
||||
error_info: "scanner publication lease is unknown or expired".to_string(),
|
||||
}),
|
||||
response_proof: Bytes::new(),
|
||||
};
|
||||
let response_body =
|
||||
rustfs_protos::canonical_scanner_publication_lease_release_response_body(&challenge, &request, &response)
|
||||
.map_err(|_| Status::internal("scanner publication lease response is too large to authenticate"))?;
|
||||
response.response_proof = sign_tonic_rpc_response_proof(&response_body)
|
||||
.map_err(|_| Status::unavailable("scanner publication lease response authentication is unavailable"))?
|
||||
.into();
|
||||
Ok(Response::new(response))
|
||||
}
|
||||
|
||||
async fn background_heal_status(
|
||||
&self,
|
||||
request: Request<BackgroundHealStatusRequest>,
|
||||
@@ -2204,12 +2383,13 @@ mod tests {
|
||||
use super::{
|
||||
CollectMetricsOpts, DiskStore, Error, HEAL_CONTROL_PAYLOAD_MAX_SIZE, KMS_SIGNAL_SUBSYSTEM, MetricType, Node as _,
|
||||
NodeService, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION,
|
||||
SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC,
|
||||
STORAGE_CLASS_SUB_SYS, admit_heal_control_replay, background_rebalance_start_error_message,
|
||||
execute_heal_control_envelope_with_manager, initialize_heal_topology_fingerprint,
|
||||
initialize_heal_topology_fingerprint_with_probe, legacy_scanner_activity_response, make_heal_control_server,
|
||||
make_heal_control_server_with_cache, make_server, make_server_for_context, make_tier_mutation_control_server_for_context,
|
||||
previous_scanner_activity_response, remove_heal_control_replay, scanner_activity_response, stop_rebalance_response,
|
||||
SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SCANNER_PUBLICATION_LEASE_TTL_MS, SERVICE_SIGNAL_REFRESH_CONFIG,
|
||||
SERVICE_SIGNAL_RELOAD_DYNAMIC, STORAGE_CLASS_SUB_SYS, admit_heal_control_replay,
|
||||
background_rebalance_start_error_message, execute_heal_control_envelope_with_manager,
|
||||
initialize_heal_topology_fingerprint, initialize_heal_topology_fingerprint_with_probe, legacy_scanner_activity_response,
|
||||
make_heal_control_server, make_heal_control_server_with_cache, make_server, make_server_for_context,
|
||||
make_tier_mutation_control_server_for_context, previous_scanner_activity_response, remove_heal_control_replay,
|
||||
scanner_activity_response_v7, stop_rebalance_response,
|
||||
};
|
||||
use crate::storage::rpc::node_service::heal::heal_topology_fingerprint;
|
||||
use crate::storage::storage_api::rpc_consumer::node_service::{DiskError, HealBucketInfo};
|
||||
@@ -2243,11 +2423,12 @@ mod tests {
|
||||
LoadTransitionTierConfigRequest, LoadUserRequest, LocalStorageInfoRequest, MakeBucketRequest, MakeVolumeRequest,
|
||||
MakeVolumesRequest, Mss, PingRequest, PreparePartTransactionRequest, ReadAllRequest, ReadAtRequest, ReadMultipleRequest,
|
||||
ReadVersionRequest, ReadXlRequest, ReloadPoolMetaRequest, ReloadSiteReplicationConfigRequest, RenameDataRequest,
|
||||
RenameFileRequest, RenamePartRequest, ScannerActivityRequest, ServerInfoRequest, SettlePartTransactionRequest,
|
||||
SignalServiceRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, SnapshotLeaseRequest,
|
||||
StartDecommissionRequest, StartProfilingRequest, StatVolumeRequest, StopRebalanceRequest, TierMutationPeerState,
|
||||
TierMutationPrepareRequest, UpdateMetacacheListingRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest,
|
||||
WriteMetadataRequest, WriteRequest,
|
||||
RenameFileRequest, RenamePartRequest, ScannerActivityRequest, ScannerPublicationLeaseReleaseRequest,
|
||||
ScannerPublicationLeaseRequest, ServerInfoRequest, SettlePartTransactionRequest, SignalServiceRequest,
|
||||
SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, SnapshotLeaseRequest, StartDecommissionRequest,
|
||||
StartProfilingRequest, StatVolumeRequest, StopRebalanceRequest, TierMutationPeerState, TierMutationPrepareRequest,
|
||||
UpdateMetacacheListingRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteMetadataRequest,
|
||||
WriteRequest,
|
||||
heal_control_service_client::HealControlServiceClient,
|
||||
heal_control_service_server::{HealControlService as _, HealControlServiceServer},
|
||||
node_service_client::NodeServiceClient,
|
||||
@@ -2621,6 +2802,7 @@ mod tests {
|
||||
volume: "bucket".to_string(),
|
||||
path: "object".to_string(),
|
||||
options: options.to_string(),
|
||||
scanner_publication_lease_token: Vec::new().into(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2720,6 +2902,7 @@ mod tests {
|
||||
dst_volume: "dst".into(),
|
||||
dst_path: "dp".into(),
|
||||
file_info_bin: vec![0x80].into(),
|
||||
scanner_publication_lease_token: Vec::new().into(),
|
||||
},
|
||||
rustfs_protos::canonical_rename_data_request_body
|
||||
);
|
||||
@@ -2790,6 +2973,7 @@ mod tests {
|
||||
volume: "v".into(),
|
||||
path: "p".into(),
|
||||
options: "{}".into(),
|
||||
scanner_publication_lease_token: Vec::new().into(),
|
||||
},
|
||||
rustfs_protos::canonical_delete_request_body
|
||||
);
|
||||
@@ -3698,6 +3882,7 @@ mod tests {
|
||||
volume: "test-volume".to_string(),
|
||||
path: "test-path".to_string(),
|
||||
options: "{}".to_string(),
|
||||
scanner_publication_lease_token: Vec::new().into(),
|
||||
});
|
||||
|
||||
let response = service.delete(request).await;
|
||||
@@ -3717,6 +3902,7 @@ mod tests {
|
||||
volume: "test-volume".to_string(),
|
||||
path: "test-path".to_string(),
|
||||
options: "invalid json".to_string(),
|
||||
scanner_publication_lease_token: Vec::new().into(),
|
||||
});
|
||||
|
||||
let response = service.delete(request).await;
|
||||
@@ -3891,6 +4077,7 @@ mod tests {
|
||||
dst_path: "dst-path".to_string(),
|
||||
file_info: "{}".to_string(),
|
||||
file_info_bin: Vec::new().into(),
|
||||
scanner_publication_lease_token: Vec::new().into(),
|
||||
});
|
||||
|
||||
let response = service.rename_data(request).await;
|
||||
@@ -3913,6 +4100,7 @@ mod tests {
|
||||
dst_path: "dst-path".to_string(),
|
||||
file_info: "invalid json".to_string(),
|
||||
file_info_bin: Vec::new().into(),
|
||||
scanner_publication_lease_token: Vec::new().into(),
|
||||
});
|
||||
|
||||
let response = service.rename_data(request).await;
|
||||
@@ -5333,6 +5521,25 @@ mod tests {
|
||||
acknowledge_dirty_usage_generation: 0,
|
||||
}
|
||||
);
|
||||
assert_tampered!(
|
||||
acquire_scanner_publication_lease,
|
||||
ScannerPublicationLeaseRequest {
|
||||
challenge: vec![7; 16].into(),
|
||||
expected_movement_generation: 0,
|
||||
ttl_ms: SCANNER_PUBLICATION_LEASE_TTL_MS,
|
||||
expected_session_id: String::new(),
|
||||
token: Bytes::new(),
|
||||
}
|
||||
);
|
||||
assert_tampered!(
|
||||
release_scanner_publication_lease,
|
||||
ScannerPublicationLeaseReleaseRequest {
|
||||
challenge: vec![7; 16].into(),
|
||||
token: vec![1; 16].into(),
|
||||
owner_id: String::new(),
|
||||
session_id: String::new(),
|
||||
}
|
||||
);
|
||||
assert_tampered!(reload_pool_meta, ReloadPoolMetaRequest::default());
|
||||
assert_tampered!(stop_rebalance, StopRebalanceRequest::default());
|
||||
assert_tampered!(load_rebalance_meta, LoadRebalanceMetaRequest::default());
|
||||
@@ -5489,7 +5696,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_scanner_activity_response_uses_process_epoch_and_generations() {
|
||||
let response = scanner_activity_response(
|
||||
let response = scanner_activity_response_v7(
|
||||
17,
|
||||
[7; 32],
|
||||
true,
|
||||
@@ -5497,6 +5704,8 @@ mod tests {
|
||||
generation: 11,
|
||||
pending: true,
|
||||
},
|
||||
23,
|
||||
true,
|
||||
);
|
||||
|
||||
assert_eq!(response.instance_id, rustfs_scanner::scanner_activity_epoch());
|
||||
@@ -5507,6 +5716,8 @@ mod tests {
|
||||
assert!(response.data_movement_active);
|
||||
assert_eq!(response.dirty_usage_generation, 11);
|
||||
assert!(response.dirty_usage_pending);
|
||||
assert_eq!(response.movement_generation, Some(23));
|
||||
assert_eq!(response.publication_blocked, Some(true));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -36,6 +36,7 @@ use std::io::Cursor;
|
||||
use std::time::Instant;
|
||||
use tonic::{Request, Response, Status};
|
||||
use tracing::debug;
|
||||
use uuid::Uuid;
|
||||
|
||||
/// Initial capacity hint (bytes) for typical small msgpack requests and responses.
|
||||
const MSGPACK_ENCODE_CAPACITY_HINT: usize = 512;
|
||||
@@ -1188,6 +1189,16 @@ impl NodeService {
|
||||
&self,
|
||||
request: Request<RenameDataRequest>,
|
||||
) -> Result<Response<RenameDataResponse>, Status> {
|
||||
if !request.get_ref().scanner_publication_lease_token.is_empty() {
|
||||
let has_body_digest = request
|
||||
.metadata()
|
||||
.get("x-rustfs-content-sha256")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.is_some_and(|value| value != "UNSIGNED-PAYLOAD");
|
||||
if !has_body_digest {
|
||||
return Err(Status::permission_denied("scanner publication lease rename requires a body-bound digest"));
|
||||
}
|
||||
}
|
||||
verify_disk_mutation_digest(
|
||||
&request,
|
||||
rustfs_protos::canonical_rename_data_request_body(request.get_ref()),
|
||||
@@ -1206,6 +1217,42 @@ impl NodeService {
|
||||
}));
|
||||
}
|
||||
};
|
||||
let scanner_publication_lease_token = if request.scanner_publication_lease_token.is_empty() {
|
||||
None
|
||||
} else {
|
||||
let token = Uuid::from_slice(&request.scanner_publication_lease_token)
|
||||
.map_err(|_| Status::invalid_argument("scanner publication lease token must be a UUID"))?;
|
||||
if token.is_nil() {
|
||||
return Err(Status::invalid_argument("scanner publication lease token must not be nil"));
|
||||
}
|
||||
Some(token)
|
||||
};
|
||||
// The target owns this read guard. It must span the complete
|
||||
// disk rename, not merely the preflight, so a movement transition
|
||||
// cannot restart after validation and before rename linearization.
|
||||
let _scanner_publication_lease_guard = if let Some(token) = scanner_publication_lease_token {
|
||||
let Some(store) = self.resolve_object_store() else {
|
||||
return Ok(Response::new(RenameDataResponse {
|
||||
success: false,
|
||||
rename_data_resp: String::new(),
|
||||
rename_data_resp_bin: Vec::new().into(),
|
||||
error: Some(DiskError::other("scanner publication lease owner is unavailable").into()),
|
||||
}));
|
||||
};
|
||||
match store.acquire_scanner_publication_lease_guard(token).await {
|
||||
Ok(guard) => Some(guard),
|
||||
Err(err) => {
|
||||
return Ok(Response::new(RenameDataResponse {
|
||||
success: false,
|
||||
rename_data_resp: String::new(),
|
||||
rename_data_resp_bin: Vec::new().into(),
|
||||
error: Some(DiskError::other(err.to_string()).into()),
|
||||
}));
|
||||
}
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let request_decoded_from_msgpack = decoded_file_info.from_msgpack;
|
||||
match disk
|
||||
.rename_data(
|
||||
@@ -1559,6 +1606,16 @@ impl NodeService {
|
||||
}
|
||||
|
||||
pub(super) async fn handle_delete(&self, request: Request<DeleteRequest>) -> Result<Response<DeleteResponse>, Status> {
|
||||
if !request.get_ref().scanner_publication_lease_token.is_empty() {
|
||||
let has_body_digest = request
|
||||
.metadata()
|
||||
.get("x-rustfs-content-sha256")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.is_some_and(|value| value != "UNSIGNED-PAYLOAD");
|
||||
if !has_body_digest {
|
||||
return Err(Status::permission_denied("scanner publication lease delete requires a body-bound digest"));
|
||||
}
|
||||
}
|
||||
verify_disk_mutation_digest(&request, rustfs_protos::canonical_delete_request_body(request.get_ref()), "delete")?;
|
||||
let request = request.into_inner();
|
||||
if let Some(disk) = self.find_disk(&request.disk).await {
|
||||
@@ -1571,6 +1628,38 @@ impl NodeService {
|
||||
}));
|
||||
}
|
||||
};
|
||||
let scanner_publication_lease_token = if request.scanner_publication_lease_token.is_empty() {
|
||||
None
|
||||
} else {
|
||||
let token = Uuid::from_slice(&request.scanner_publication_lease_token)
|
||||
.map_err(|_| Status::invalid_argument("scanner publication lease token must be a UUID"))?;
|
||||
if token.is_nil() {
|
||||
return Err(Status::invalid_argument("scanner publication lease token must not be nil"));
|
||||
}
|
||||
Some(token)
|
||||
};
|
||||
// The target-side guard spans the complete delete operation. A
|
||||
// lease expiry or movement transition cannot occur between this
|
||||
// validation and the disk delete linearization point.
|
||||
let _scanner_publication_lease_guard = if let Some(token) = scanner_publication_lease_token {
|
||||
let Some(store) = self.resolve_object_store() else {
|
||||
return Ok(Response::new(DeleteResponse {
|
||||
success: false,
|
||||
error: Some(DiskError::other("scanner publication lease owner is unavailable").into()),
|
||||
}));
|
||||
};
|
||||
match store.acquire_scanner_publication_lease_guard(token).await {
|
||||
Ok(guard) => Some(guard),
|
||||
Err(err) => {
|
||||
return Ok(Response::new(DeleteResponse {
|
||||
success: false,
|
||||
error: Some(DiskError::other(err.to_string()).into()),
|
||||
}));
|
||||
}
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
match disk.delete(&request.volume, &request.path, options).await {
|
||||
Ok(_) => Ok(Response::new(DeleteResponse {
|
||||
success: true,
|
||||
|
||||
@@ -242,14 +242,16 @@ pub(crate) mod rpc_consumer {
|
||||
pub(crate) use super::super::ecstore_rpc::decode_heal_bucket_rpc_options;
|
||||
pub(crate) use super::super::storage_contracts::{
|
||||
SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION, SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION,
|
||||
SCANNER_ACTIVITY_V6_PROTOCOL_VERSION,
|
||||
};
|
||||
pub(crate) use super::super::{
|
||||
BatchReadVersionReq, BatchReadVersionResp, CollectMetricsOpts, DeleteOptions, DiskError, DiskInfoOptions, DiskStore,
|
||||
ECStore, Error, FileInfoVersions, KMS_SIGNAL_SUBSYSTEM, LocalPeerS3Client, MetricType, PEER_RESTDRY_RUN,
|
||||
PEER_RESTSIGNAL, PEER_RESTSUB_SYS, ReadMultipleReq, ReadMultipleResp, ReadOptions, SERVICE_SIGNAL_REFRESH_CONFIG,
|
||||
SERVICE_SIGNAL_RELOAD_DYNAMIC, StorageDiskRpcExt, StoragePeerS3ClientExt, UpdateMetadataOpts, all_local_disk_path,
|
||||
collect_local_metrics, find_local_disk_by_ref, get_local_server_property, reload_bucket_metadata,
|
||||
reload_transition_tier_config, remove_bucket_metadata, validate_batch_read_version_item_count,
|
||||
PEER_RESTSIGNAL, PEER_RESTSUB_SYS, ReadMultipleReq, ReadMultipleResp, ReadOptions, SCANNER_PUBLICATION_LEASE_TTL_MS,
|
||||
SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, StorageDiskRpcExt, StoragePeerS3ClientExt,
|
||||
UpdateMetadataOpts, all_local_disk_path, collect_local_metrics, find_local_disk_by_ref, get_local_server_property,
|
||||
reload_bucket_metadata, reload_transition_tier_config, remove_bucket_metadata,
|
||||
validate_batch_read_version_item_count,
|
||||
};
|
||||
pub(crate) type StorageResult<T> = super::super::Result<T>;
|
||||
|
||||
@@ -581,8 +583,8 @@ pub(crate) mod ecstore_storage {
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::storage::init_local_disks;
|
||||
pub(crate) use rustfs_ecstore::api::storage::{
|
||||
ECStore, all_local_disk, all_local_disk_path, find_local_disk_by_ref, init_local_disks_with_instance_ctx,
|
||||
init_lock_clients, prewarm_local_disk_id_map_with_instance_ctx,
|
||||
ECStore, SCANNER_PUBLICATION_LEASE_TTL_MS, all_local_disk, all_local_disk_path, find_local_disk_by_ref,
|
||||
init_local_disks_with_instance_ctx, init_lock_clients, prewarm_local_disk_id_map_with_instance_ctx,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -614,6 +616,7 @@ pub(crate) const KMS_SIGNAL_SUBSYSTEM: &str = ecstore_rpc::KMS_SIGNAL_SUBSYSTEM;
|
||||
pub(crate) const SERVICE_SIGNAL_REFRESH_CONFIG: u64 = ecstore_rpc::SERVICE_SIGNAL_REFRESH_CONFIG;
|
||||
pub(crate) const SERVICE_SIGNAL_RELOAD_DYNAMIC: u64 = ecstore_rpc::SERVICE_SIGNAL_RELOAD_DYNAMIC;
|
||||
pub(crate) const RUSTFS_META_BUCKET: &str = ecstore_disk::RUSTFS_META_BUCKET;
|
||||
pub(crate) const SCANNER_PUBLICATION_LEASE_TTL_MS: u64 = ecstore_storage::SCANNER_PUBLICATION_LEASE_TTL_MS;
|
||||
pub(crate) const TONIC_RPC_PREFIX: &str = ecstore_rpc::TONIC_RPC_PREFIX;
|
||||
|
||||
pub(crate) fn normalize_tonic_rpc_audience(value: &str) -> std::io::Result<String> {
|
||||
|
||||
Reference in New Issue
Block a user