mirror of
https://github.com/rustfs/rustfs.git
synced 2026-09-08 04:58:12 +00:00
chore(ecstore): sync heal lock fix with main
This commit is contained in:
@@ -6743,6 +6743,99 @@ async fn test_site_replication_replicates_object_with_bucket_versioning_real_dua
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_site_replication_replays_bucket_created_during_peer_outage_real_dual_node() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
// Keep compilation outside the scenario timeout. Recovery itself waits
|
||||
// for the production 30-second lightweight retry tick.
|
||||
let _rustfs_binary = rustfs_binary_path();
|
||||
|
||||
match timeout(Duration::from_secs(150), async {
|
||||
let mut site_env = replication_fast_env();
|
||||
site_env.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
|
||||
|
||||
let mut site_a_env = RustFSTestEnvironment::new().await?;
|
||||
site_a_env.start_rustfs_server_with_env(vec![], &site_env).await?;
|
||||
|
||||
let mut site_b_env = RustFSTestEnvironment::new().await?;
|
||||
site_b_env.start_rustfs_server_without_cleanup_with_env(&site_env).await?;
|
||||
|
||||
let site_a_client = site_a_env.create_s3_client();
|
||||
let site_b_client = site_b_env.create_s3_client();
|
||||
let bucket = "site-repl-peer-outage";
|
||||
let key = "after-recovery.txt";
|
||||
let payload = b"site replication recovered the missed bucket".to_vec();
|
||||
|
||||
let add_status = site_replication_add(
|
||||
&site_a_env,
|
||||
&[
|
||||
PeerSite {
|
||||
name: "outage-site-a".to_string(),
|
||||
endpoint: site_a_env.url.clone(),
|
||||
access_key: site_a_env.access_key.clone(),
|
||||
secret_key: site_a_env.secret_key.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
PeerSite {
|
||||
name: "outage-site-b".to_string(),
|
||||
endpoint: site_b_env.url.clone(),
|
||||
access_key: site_b_env.access_key.clone(),
|
||||
secret_key: site_b_env.secret_key.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
],
|
||||
)
|
||||
.await?;
|
||||
assert!(add_status.success, "unexpected site add result: {add_status:?}");
|
||||
wait_for_site_replication_enabled(&site_a_env, 2).await?;
|
||||
wait_for_site_replication_enabled(&site_b_env, 2).await?;
|
||||
|
||||
site_b_env.stop_server();
|
||||
site_a_client.create_bucket().bucket(bucket).send().await?;
|
||||
site_a_client.head_bucket().bucket(bucket).send().await?;
|
||||
|
||||
let queued = site_replication_info(&site_a_env)
|
||||
.await?
|
||||
.retry_stats
|
||||
.ok_or("peer outage did not persist a site replication retry event")?;
|
||||
assert!(queued.pending + queued.failed > 0, "peer outage retry queue was unexpectedly empty");
|
||||
|
||||
site_b_env.restart_server_preserving_data(vec![], &site_env).await?;
|
||||
let recovery_deadline = tokio::time::Instant::now() + Duration::from_secs(75);
|
||||
loop {
|
||||
let bucket_recovered = site_b_client.head_bucket().bucket(bucket).send().await.is_ok();
|
||||
let queue_empty = site_replication_info(&site_a_env).await?.retry_stats.is_none();
|
||||
if bucket_recovered && queue_empty {
|
||||
break;
|
||||
}
|
||||
if tokio::time::Instant::now() >= recovery_deadline {
|
||||
return Err(format!(
|
||||
"site replication retry did not settle after peer recovery; bucket_recovered={bucket_recovered}, queue_empty={queue_empty}"
|
||||
)
|
||||
.into());
|
||||
}
|
||||
sleep(Duration::from_millis(250)).await;
|
||||
}
|
||||
|
||||
site_a_client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.body(ByteStream::from(payload.clone()))
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(wait_for_object_on_target(&site_b_client, bucket, key).await?, payload);
|
||||
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(result) => result,
|
||||
Err(_) => Err("site replication peer-outage recovery timed out after 150 seconds".into()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Re-applying a site's own replication config must not disable the peer's reverse direction.
|
||||
///
|
||||
/// `PutBucketReplication` broadcasts the config to every peer — the console's replication
|
||||
|
||||
@@ -196,15 +196,16 @@ pub mod bucket {
|
||||
pub use crate::bucket::metadata_sys::ConfigWriteLockProbe;
|
||||
pub use crate::bucket::metadata_sys::{
|
||||
BucketMetadataMutationGuard, BucketMetadataSys, ObjectLockConfigState, acquire_bucket_metadata_transaction_lock,
|
||||
acquire_bucket_metadata_transaction_lock_for_incarnation, capture_bucket_metadata_incarnation, delete,
|
||||
delete_if_incarnation, delete_under_transaction_lock, get, get_accelerate_config, get_bucket_policy,
|
||||
get_bucket_policy_raw, get_bucket_targets_config, get_config_from_disk, get_cors_config, get_durability_config,
|
||||
get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config, get_notification_config,
|
||||
get_object_lock_config, get_object_lock_config_state, get_on_demand_migration_config, get_public_access_block_config,
|
||||
get_quota_config, get_replication_config, get_request_payment_config, get_sse_config, get_tagging_config,
|
||||
get_versioning_config, get_website_config, init_bucket_metadata_sys, list_bucket_targets, reload_bucket_metadata,
|
||||
remove_bucket_metadata, set_bucket_metadata, update, update_bucket_targets_under_transaction_lock,
|
||||
update_config_with, update_if_incarnation, update_quota_if_incarnation, update_under_transaction_lock,
|
||||
acquire_bucket_metadata_transaction_lock_for_incarnation, acquire_scanner_bucket_incarnation_fence,
|
||||
capture_bucket_metadata_incarnation, delete, delete_if_incarnation, delete_under_transaction_lock, get,
|
||||
get_accelerate_config, get_bucket_policy, get_bucket_policy_raw, get_bucket_targets_config, get_config_from_disk,
|
||||
get_cors_config, get_durability_config, get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config,
|
||||
get_notification_config, get_object_lock_config, get_object_lock_config_state, get_on_demand_migration_config,
|
||||
get_public_access_block_config, get_quota_config, get_replication_config, get_request_payment_config, get_sse_config,
|
||||
get_tagging_config, get_versioning_config, get_website_config, init_bucket_metadata_sys, list_bucket_targets,
|
||||
reload_bucket_metadata, remove_bucket_metadata, set_bucket_metadata, update,
|
||||
update_bucket_targets_under_transaction_lock, update_config_with, update_if_incarnation, update_quota_if_incarnation,
|
||||
update_under_transaction_lock,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -655,6 +655,12 @@ pub struct BucketMetadataMutationGuard {
|
||||
}
|
||||
|
||||
impl BucketMetadataMutationGuard {
|
||||
/// Returns the storage-verified identity while both incarnation fences remain valid.
|
||||
pub fn checked_bucket_incarnation(&self) -> Result<(&str, Uuid)> {
|
||||
self.ensure_valid(&self.bucket)?;
|
||||
Ok((&self.bucket, self.incarnation_id))
|
||||
}
|
||||
|
||||
fn ensure_valid(&self, bucket: &str) -> Result<()> {
|
||||
if self.bucket != bucket {
|
||||
return Err(Error::other("bucket metadata mutation guard does not match bucket"));
|
||||
@@ -674,6 +680,29 @@ async fn acquire_config_write_guard_for_incarnation(
|
||||
sys: Arc<RwLock<BucketMetadataSys>>,
|
||||
bucket: &str,
|
||||
expected_incarnation_id: Option<Uuid>,
|
||||
) -> Result<BucketMetadataMutationGuard> {
|
||||
acquire_config_write_guard_with_migration(sys, bucket, expected_incarnation_id, true).await
|
||||
}
|
||||
|
||||
/// Scanner probes must not create an incarnation to make a capability available.
|
||||
pub async fn acquire_scanner_bucket_incarnation_fence(
|
||||
bucket: &str,
|
||||
expected_incarnation_id: Uuid,
|
||||
expected_owner_id: Uuid,
|
||||
) -> Result<BucketMetadataMutationGuard> {
|
||||
super::utils::check_valid_bucket_name(bucket)?;
|
||||
let sys = get_bucket_metadata_sys()?;
|
||||
if expected_owner_id.is_nil() || sys.read().await.api.id != expected_owner_id || expected_incarnation_id.is_nil() {
|
||||
return Err(Error::other("scanner bucket incarnation owner does not match"));
|
||||
}
|
||||
acquire_config_write_guard_with_migration(sys, bucket, Some(expected_incarnation_id), false).await
|
||||
}
|
||||
|
||||
async fn acquire_config_write_guard_with_migration(
|
||||
sys: Arc<RwLock<BucketMetadataSys>>,
|
||||
bucket: &str,
|
||||
expected_incarnation_id: Option<Uuid>,
|
||||
migrate: bool,
|
||||
) -> Result<BucketMetadataMutationGuard> {
|
||||
let metadata_sys = sys.read().await.clone();
|
||||
let lifecycle_guard = metadata_sys.api.acquire_bucket_lifecycle_read_lock(bucket).await?;
|
||||
@@ -681,13 +710,15 @@ async fn acquire_config_write_guard_for_incarnation(
|
||||
// Legacy buckets are migrated while the lifecycle fence prevents a
|
||||
// same-name replacement. The second read under the write transaction is
|
||||
// the CAS source of truth for the actual rewrite.
|
||||
await_bucket_namespace_operation(
|
||||
Some(&lifecycle_guard),
|
||||
bucket,
|
||||
"bucket config incarnation migration",
|
||||
metadata_sys.get_bucket_incarnation_id(bucket),
|
||||
)
|
||||
.await?;
|
||||
if migrate {
|
||||
await_bucket_namespace_operation(
|
||||
Some(&lifecycle_guard),
|
||||
bucket,
|
||||
"bucket config incarnation migration",
|
||||
metadata_sys.get_bucket_incarnation_id(bucket),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
let transaction_guard = await_bucket_namespace_operation(
|
||||
Some(&lifecycle_guard),
|
||||
bucket,
|
||||
@@ -3176,6 +3207,82 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scoped_dirty_usage_incarnation_probe_does_not_migrate_legacy_metadata() {
|
||||
let (dirs, store) = isolated_store_over_temp_disks().await;
|
||||
let sys = Arc::new(RwLock::new(BucketMetadataSys::new(store.clone())));
|
||||
let bucket = "scoped-ack-legacy";
|
||||
for dir in &dirs {
|
||||
std::fs::create_dir_all(dir.path().join(bucket)).expect("create legacy bucket");
|
||||
}
|
||||
let mut metadata = BucketMetadata::new(bucket);
|
||||
metadata.bucket_incarnation_id = Uuid::nil();
|
||||
sys.read()
|
||||
.await
|
||||
.persist_and_set(metadata)
|
||||
.await
|
||||
.expect("persist legacy metadata");
|
||||
assert!(
|
||||
acquire_config_write_guard_with_migration(sys.clone(), bucket, Some(Uuid::new_v4()), false)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert!(load_bucket_incarnation(store, bucket).await.expect("read sidecar").is_none());
|
||||
assert!(
|
||||
sys.read()
|
||||
.await
|
||||
.get_config_from_disk(bucket)
|
||||
.await
|
||||
.expect("read metadata")
|
||||
.bucket_incarnation_id
|
||||
.is_nil()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn scoped_dirty_usage_incarnation_rejects_deleted_and_recreated_bucket() {
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let sys = bucket_metadata_sys_of(&store.ctx).expect("metadata owner");
|
||||
let bucket = "scoped-ack-recreated";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
let old = store.bucket_incarnation_id_from_disk(bucket).await.expect("old incarnation");
|
||||
let guard = acquire_config_write_guard_with_migration(sys.clone(), bucket, Some(old), false)
|
||||
.await
|
||||
.expect("trusted incarnation fence");
|
||||
assert_eq!(guard.checked_bucket_incarnation().expect("valid fences"), (bucket, old));
|
||||
drop(guard);
|
||||
store
|
||||
.delete_bucket(bucket, &DeleteBucketOptions::default())
|
||||
.await
|
||||
.expect("delete bucket");
|
||||
assert!(
|
||||
acquire_config_write_guard_with_migration(sys.clone(), bucket, Some(old), false)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("recreate bucket");
|
||||
let new = store.bucket_incarnation_id_from_disk(bucket).await.expect("new incarnation");
|
||||
assert_ne!(old, new);
|
||||
assert!(
|
||||
acquire_config_write_guard_with_migration(sys.clone(), bucket, Some(old), false)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
acquire_config_write_guard_with_migration(sys, bucket, Some(new), false)
|
||||
.await
|
||||
.is_ok()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn old_node_metadata_rewrite_cannot_replace_bucket_incarnation_sidecar() {
|
||||
let (dirs, ecstore) = isolated_store_over_temp_disks().await;
|
||||
|
||||
@@ -30,6 +30,7 @@ use rustfs_protos::{
|
||||
ChannelClass, create_new_channel, get_channel_for_class,
|
||||
proto_gen::node_service::{
|
||||
heal_control_service_client::HealControlServiceClient, node_service_client::NodeServiceClient,
|
||||
scanner_control_service_client::ScannerControlServiceClient,
|
||||
tier_mutation_control_service_client::TierMutationControlServiceClient,
|
||||
},
|
||||
};
|
||||
@@ -60,6 +61,24 @@ pub async fn node_service_time_out_client(
|
||||
node_service_time_out_client_for_class(addr, interceptor, ChannelClass::Control).await
|
||||
}
|
||||
|
||||
pub(crate) async fn scanner_control_time_out_client(
|
||||
addr: &str,
|
||||
interceptor: TonicInterceptor,
|
||||
) -> crate::error::Result<ScannerControlServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>> {
|
||||
let interceptor = interceptor.with_rpc_audience(addr)?;
|
||||
let channel = match runtime_sources::cached_node_channel(addr).await {
|
||||
Some(channel) => channel,
|
||||
None => create_new_channel(addr)
|
||||
.await
|
||||
.map_err(|err| crate::error::Error::other(err.to_string()))?,
|
||||
};
|
||||
let channel = ReplayScopeChannel::new(channel, interceptor.replay_scope_audience());
|
||||
let limit = rustfs_protos::scoped_dirty_usage::SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES as usize;
|
||||
Ok(ScannerControlServiceClient::with_interceptor(channel, interceptor)
|
||||
.max_decoding_message_size(limit)
|
||||
.max_encoding_message_size(limit))
|
||||
}
|
||||
|
||||
pub async fn heal_control_time_out_client(
|
||||
addr: &str,
|
||||
interceptor: TonicInterceptor,
|
||||
|
||||
@@ -2050,6 +2050,53 @@ impl PeerRestClient {
|
||||
.await
|
||||
}
|
||||
|
||||
/// Probe only: scoped ACK production requires a durable per-bucket proof.
|
||||
pub async fn scanner_scoped_dirty_usage_capability(
|
||||
&self,
|
||||
owner_id: String,
|
||||
instance_id: String,
|
||||
entries: Vec<rustfs_protos::proto_gen::node_service::ScannerScopedDirtyUsageEntry>,
|
||||
) -> Result<bool> {
|
||||
use rustfs_protos::scoped_dirty_usage::*;
|
||||
let payload = rustfs_protos::proto_gen::node_service::ScannerScopedDirtyUsageAckRequest {
|
||||
challenge: Uuid::new_v4().as_bytes().to_vec().into(),
|
||||
protocol_version: SCOPED_DIRTY_USAGE_PROTOCOL_VERSION,
|
||||
owner_id,
|
||||
instance_id,
|
||||
scope: SCOPED_DIRTY_USAGE_BUCKET_SCOPE,
|
||||
probe_only: true,
|
||||
entries,
|
||||
};
|
||||
let canonical = canonical_scoped_dirty_usage_request(&payload).map_err(|err| Error::other(err.to_string()))?;
|
||||
self.finalize_result(
|
||||
async {
|
||||
let mut client = super::client::scanner_control_time_out_client(
|
||||
&self.grid_host,
|
||||
TonicInterceptor::Signature(gen_tonic_signature_interceptor()),
|
||||
)
|
||||
.await?;
|
||||
let mut request = Request::new(payload.clone());
|
||||
set_tonic_canonical_body_digest(&mut request, &canonical)?;
|
||||
let response = client.scanner_scoped_dirty_usage_ack(request).await?.into_inner();
|
||||
let body = canonical_scoped_dirty_usage_response(&canonical, &response)
|
||||
.map_err(|_| Error::other("scoped dirty usage capability response is too large"))?;
|
||||
verify_tonic_rpc_response_proof(&body, response.response_proof.as_ref())?;
|
||||
if response.protocol_version != SCOPED_DIRTY_USAGE_PROTOCOL_VERSION
|
||||
|| response.owner_id != payload.owner_id
|
||||
|| response.instance_id != payload.instance_id
|
||||
|| response.max_entries != SCOPED_DIRTY_USAGE_MAX_ENTRIES
|
||||
|| response.max_request_bytes != SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES
|
||||
|| response.cleared != 0
|
||||
{
|
||||
return Err(Error::other("scoped dirty usage capability response does not match request"));
|
||||
}
|
||||
Ok(response.supported)
|
||||
}
|
||||
.await,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn acknowledge_scanner_dirty_usage(&self, instance_id: String, generation: u64) -> Result<ScannerPeerActivity> {
|
||||
let result = self
|
||||
.scanner_activity_request_with_protocol(instance_id.clone(), generation, SCANNER_ACTIVITY_PROTOCOL_VERSION)
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -59,27 +59,34 @@ use super::super::{
|
||||
send_heal_request_with_admission, should_prevent_write, to_object_err, try_read_inline_data_shards_direct, warn,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(in crate::set_disk) use super::metadata_quorum::MetadataEarlyStopDecision;
|
||||
pub(in crate::set_disk) use super::metadata_quorum::{
|
||||
MetadataQuorumAccumulator, is_metadata_fanout_ignored_error, metadata_early_stop_candidate_matches,
|
||||
};
|
||||
#[cfg(test)]
|
||||
use crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE;
|
||||
#[cfg(test)]
|
||||
use crate::diagnostics::get::GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_IDENTITY_MISMATCH;
|
||||
#[cfg(test)]
|
||||
use crate::diagnostics::get::GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_PAYLOAD;
|
||||
use crate::diagnostics::get::{
|
||||
GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY,
|
||||
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_DELETED, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY,
|
||||
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_SHARD, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_NOT_INLINE,
|
||||
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_REMOTE,
|
||||
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_TRANSFORMED,
|
||||
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_DELETED,
|
||||
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_SHARD,
|
||||
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_NOT_INLINE, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE,
|
||||
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_REMOTE, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE,
|
||||
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_TRANSFORMED, GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM,
|
||||
GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST, GET_METADATA_RESPONSE_CORRUPT, GET_METADATA_RESPONSE_DISK_NOT_FOUND,
|
||||
GET_METADATA_RESPONSE_ERROR, GET_METADATA_RESPONSE_IGNORED, GET_METADATA_RESPONSE_NOT_FOUND, GET_METADATA_RESPONSE_TIMEOUT,
|
||||
GET_METADATA_RESPONSE_VALID, GET_METADATA_RESPONSE_VERSION_NOT_FOUND, GET_OBJECT_PATH_DIRECT_MEMORY,
|
||||
GET_OBJECT_PATH_INTERNAL_META, GET_OBJECT_PATH_LEGACY_DUPLEX, GET_STAGE_READER_SETUP_DROP_PENDING,
|
||||
GET_STAGE_READER_SETUP_SCHEDULE, GET_STAGE_READER_SETUP_WAIT_QUORUM, GET_STAGE_READER_TASK_BITROT_READER_INIT,
|
||||
GET_STAGE_READER_TASK_FILE_OPEN, GET_STAGE_READER_TASK_READER_CONSTRUCTION, get_stage_timer_if_enabled,
|
||||
record_get_stage_duration_if_enabled,
|
||||
};
|
||||
#[cfg(test)]
|
||||
use crate::diagnostics::get::{
|
||||
GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER, GET_METADATA_EARLY_STOP_REASON_ERROR,
|
||||
GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM, GET_METADATA_EARLY_STOP_REASON_NOT_FOUND,
|
||||
GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST, GET_METADATA_EARLY_STOP_REASON_VALID_QUORUM,
|
||||
GET_METADATA_EARLY_STOP_REASON_VERSION_MATCH_QUORUM, GET_METADATA_EARLY_STOP_REASON_VERSION_NOT_FOUND,
|
||||
GET_METADATA_RESPONSE_CORRUPT, GET_METADATA_RESPONSE_DISK_NOT_FOUND, GET_METADATA_RESPONSE_ERROR,
|
||||
GET_METADATA_RESPONSE_IGNORED, GET_METADATA_RESPONSE_NOT_FOUND, GET_METADATA_RESPONSE_TIMEOUT, GET_METADATA_RESPONSE_VALID,
|
||||
GET_METADATA_RESPONSE_VERSION_NOT_FOUND, GET_OBJECT_PATH_DIRECT_MEMORY, GET_OBJECT_PATH_INTERNAL_META,
|
||||
GET_OBJECT_PATH_LEGACY_DUPLEX, GET_STAGE_READER_SETUP_DROP_PENDING, GET_STAGE_READER_SETUP_SCHEDULE,
|
||||
GET_STAGE_READER_SETUP_WAIT_QUORUM, GET_STAGE_READER_TASK_BITROT_READER_INIT, GET_STAGE_READER_TASK_FILE_OPEN,
|
||||
GET_STAGE_READER_TASK_READER_CONSTRUCTION, get_stage_timer_if_enabled, record_get_stage_duration_if_enabled,
|
||||
GET_METADATA_EARLY_STOP_REASON_VALID_QUORUM,
|
||||
};
|
||||
#[cfg(test)]
|
||||
use crate::disk::CHECK_PART_FILE_NOT_FOUND;
|
||||
@@ -690,324 +697,6 @@ impl MetadataFanoutDiagnostics {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub(in crate::set_disk) struct MetadataEarlyStopDecision {
|
||||
pub(in crate::set_disk) reason: &'static str,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub(in crate::set_disk) struct MetadataQuorumAccumulator {
|
||||
pub(in crate::set_disk) total_disks: usize,
|
||||
pub(in crate::set_disk) default_parity_count: usize,
|
||||
pub(in crate::set_disk) allow_early_stop: bool,
|
||||
pub(in crate::set_disk) valid_responses: usize,
|
||||
pub(in crate::set_disk) not_found_responses: usize,
|
||||
pub(in crate::set_disk) version_not_found_responses: usize,
|
||||
pub(in crate::set_disk) ignored_errors: usize,
|
||||
pub(in crate::set_disk) hard_errors: usize,
|
||||
pub(in crate::set_disk) candidate: Option<FileInfo>,
|
||||
pub(in crate::set_disk) candidate_votes: usize,
|
||||
// Bitset of shard indexes whose metadata matches the candidate. Erasure
|
||||
// layouts are capped at 16 shards, so this stays allocation-free on the
|
||||
// GET metadata hot path.
|
||||
candidate_shard_mask: u16,
|
||||
pub(in crate::set_disk) conflicting_metadata: bool,
|
||||
pub(in crate::set_disk) delete_marker_seen: bool,
|
||||
pub(in crate::set_disk) delete_marker_candidates: Vec<(FileInfo, usize)>,
|
||||
pub(in crate::set_disk) delete_marker_votes: usize,
|
||||
pub(in crate::set_disk) requested_version_id: String,
|
||||
pub(in crate::set_disk) matching_version_votes: usize,
|
||||
}
|
||||
|
||||
impl MetadataQuorumAccumulator {
|
||||
pub(in crate::set_disk) fn new(total_disks: usize, default_parity_count: usize, allow_early_stop: bool) -> Self {
|
||||
Self {
|
||||
total_disks,
|
||||
default_parity_count,
|
||||
allow_early_stop,
|
||||
valid_responses: 0,
|
||||
not_found_responses: 0,
|
||||
version_not_found_responses: 0,
|
||||
ignored_errors: 0,
|
||||
hard_errors: 0,
|
||||
candidate: None,
|
||||
candidate_votes: 0,
|
||||
candidate_shard_mask: 0,
|
||||
conflicting_metadata: false,
|
||||
delete_marker_seen: false,
|
||||
delete_marker_candidates: Vec::new(),
|
||||
delete_marker_votes: 0,
|
||||
requested_version_id: String::new(),
|
||||
matching_version_votes: 0,
|
||||
}
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn with_requested_version_id(mut self, version_id: &str) -> Self {
|
||||
self.requested_version_id = version_id.to_string();
|
||||
self
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn observe_file_info(&mut self, file_info: &FileInfo) {
|
||||
self.observe_file_info_with_index(None, file_info);
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn observe_file_info_at(&mut self, disk_index: usize, file_info: &FileInfo) {
|
||||
self.observe_file_info_with_index(Some(disk_index), file_info);
|
||||
}
|
||||
|
||||
fn observe_file_info_with_index(&mut self, disk_index: Option<usize>, file_info: &FileInfo) {
|
||||
if !file_info_is_valid_for_metadata(file_info) {
|
||||
self.hard_errors = self.hard_errors.saturating_add(1);
|
||||
return;
|
||||
}
|
||||
|
||||
self.valid_responses = self.valid_responses.saturating_add(1);
|
||||
|
||||
// Track version match for versioned requests
|
||||
if !self.requested_version_id.is_empty()
|
||||
&& let Some(ref vid) = file_info.version_id
|
||||
&& vid.to_string() == self.requested_version_id
|
||||
{
|
||||
self.matching_version_votes = self.matching_version_votes.saturating_add(1);
|
||||
}
|
||||
|
||||
if file_info.is_canonical_delete_marker() {
|
||||
self.delete_marker_seen = true;
|
||||
if let Some((_, votes)) = self
|
||||
.delete_marker_candidates
|
||||
.iter_mut()
|
||||
.find(|(candidate, _)| metadata_early_stop_candidate_matches(candidate, file_info))
|
||||
{
|
||||
*votes = votes.saturating_add(1);
|
||||
} else {
|
||||
self.delete_marker_candidates.push((file_info.clone(), 1));
|
||||
}
|
||||
self.delete_marker_votes = self
|
||||
.delete_marker_candidates
|
||||
.iter()
|
||||
.map(|(_, votes)| *votes)
|
||||
.max()
|
||||
.unwrap_or_default();
|
||||
self.conflicting_metadata |= self.delete_marker_candidates.len() > 1;
|
||||
return;
|
||||
}
|
||||
|
||||
match &self.candidate {
|
||||
Some(candidate) if metadata_early_stop_candidate_matches(candidate, file_info) => {
|
||||
self.candidate_votes = self.candidate_votes.saturating_add(1);
|
||||
if let Some(disk_index) = disk_index
|
||||
&& let Some(bit) = Self::candidate_shard_bit(candidate, file_info, disk_index)
|
||||
{
|
||||
self.candidate_shard_mask |= bit;
|
||||
}
|
||||
}
|
||||
Some(_) => {
|
||||
self.conflicting_metadata = true;
|
||||
}
|
||||
None => {
|
||||
self.candidate = Some(file_info.clone());
|
||||
self.candidate_votes = 1;
|
||||
if let Some(disk_index) = disk_index
|
||||
&& let Some(bit) = Self::candidate_shard_bit(file_info, file_info, disk_index)
|
||||
{
|
||||
self.candidate_shard_mask |= bit;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn candidate_shard_bit(candidate: &FileInfo, file_info: &FileInfo, disk_index: usize) -> Option<u16> {
|
||||
let &erasure_index = candidate.erasure.distribution.get(disk_index)?;
|
||||
if erasure_index == 0 || erasure_index > u16::BITS as usize || file_info.erasure.index != erasure_index {
|
||||
return None;
|
||||
}
|
||||
Some(1u16 << (erasure_index - 1))
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn candidate_has_read_reserve(&self) -> bool {
|
||||
self.candidate_read_reserve_target()
|
||||
.is_some_and(|required| self.candidate_shard_mask.count_ones() as usize >= required)
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn candidate_read_reserve_target(&self) -> Option<usize> {
|
||||
let candidate = self.candidate.as_ref()?;
|
||||
Some(
|
||||
candidate
|
||||
.erasure
|
||||
.data_blocks
|
||||
.saturating_add(usize::from(candidate.erasure.parity_blocks > 0)),
|
||||
)
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn observe_error(&mut self, err: &DiskError) {
|
||||
match err {
|
||||
DiskError::FileNotFound | DiskError::VolumeNotFound => {
|
||||
self.not_found_responses = self.not_found_responses.saturating_add(1);
|
||||
}
|
||||
DiskError::FileVersionNotFound => {
|
||||
self.version_not_found_responses = self.version_not_found_responses.saturating_add(1);
|
||||
}
|
||||
_ if is_metadata_fanout_ignored_error(err) => {
|
||||
self.ignored_errors = self.ignored_errors.saturating_add(1);
|
||||
}
|
||||
_ => {
|
||||
self.hard_errors = self.hard_errors.saturating_add(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn early_stop_decision(&self) -> Option<MetadataEarlyStopDecision> {
|
||||
if !self.allow_early_stop {
|
||||
return None;
|
||||
}
|
||||
if self.delete_marker_votes >= self.default_write_quorum() {
|
||||
return Some(MetadataEarlyStopDecision {
|
||||
reason: GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER,
|
||||
});
|
||||
}
|
||||
if self.conflicting_metadata
|
||||
|| self.delete_marker_seen
|
||||
|| self.not_found_responses > 0
|
||||
|| self.version_not_found_responses > 0
|
||||
|| self.hard_errors > 0
|
||||
{
|
||||
return None;
|
||||
}
|
||||
if self
|
||||
.candidate
|
||||
.as_ref()
|
||||
.and_then(|candidate| self.candidate_latest_quorum(candidate))
|
||||
.is_some_and(|latest_quorum| self.candidate_votes >= latest_quorum)
|
||||
{
|
||||
return Some(MetadataEarlyStopDecision {
|
||||
reason: GET_METADATA_EARLY_STOP_REASON_VALID_QUORUM,
|
||||
});
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Check if a versioned request can early-stop because the requested
|
||||
/// version_id has reached quorum across disks.
|
||||
pub(in crate::set_disk) fn version_early_stop_decision(&self) -> Option<MetadataEarlyStopDecision> {
|
||||
if !self.allow_early_stop {
|
||||
return None;
|
||||
}
|
||||
if self.requested_version_id.is_empty() {
|
||||
return None;
|
||||
}
|
||||
if self.conflicting_metadata
|
||||
|| self.delete_marker_seen
|
||||
|| self.not_found_responses > 0
|
||||
|| self.version_not_found_responses > 0
|
||||
|| self.hard_errors > 0
|
||||
{
|
||||
return None;
|
||||
}
|
||||
if self.matching_version_votes >= self.read_quorum_for_version() {
|
||||
return Some(MetadataEarlyStopDecision {
|
||||
reason: GET_METADATA_EARLY_STOP_REASON_VERSION_MATCH_QUORUM,
|
||||
});
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn can_still_reach_early_stop_with_pending(&self, pending: usize) -> bool {
|
||||
if !self.allow_early_stop {
|
||||
return false;
|
||||
}
|
||||
if self.delete_marker_votes.saturating_add(pending) >= self.default_write_quorum() {
|
||||
return true;
|
||||
}
|
||||
if self.conflicting_metadata
|
||||
|| self.delete_marker_seen
|
||||
|| self.not_found_responses > 0
|
||||
|| self.version_not_found_responses > 0
|
||||
|| self.hard_errors > 0
|
||||
{
|
||||
return false;
|
||||
}
|
||||
if !self.requested_version_id.is_empty()
|
||||
&& self.matching_version_votes.saturating_add(pending) >= self.read_quorum_for_version()
|
||||
{
|
||||
return true;
|
||||
}
|
||||
match &self.candidate {
|
||||
Some(candidate) => self
|
||||
.candidate_latest_quorum(candidate)
|
||||
.is_some_and(|latest_quorum| self.candidate_votes.saturating_add(pending) >= latest_quorum),
|
||||
None => pending >= self.default_write_quorum(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Compute the read quorum threshold for version-aware early-stop.
|
||||
/// Uses `total_disks / 2` (like `missing_response_quorum`) when
|
||||
/// `default_parity_count` is set, otherwise requires all disks.
|
||||
pub(in crate::set_disk) fn read_quorum_for_version(&self) -> usize {
|
||||
self.missing_response_quorum()
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn final_miss_reason(&self) -> &'static str {
|
||||
if !self.allow_early_stop {
|
||||
return GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST;
|
||||
}
|
||||
if self.conflicting_metadata {
|
||||
return GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA;
|
||||
}
|
||||
if self.delete_marker_seen {
|
||||
return GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER;
|
||||
}
|
||||
let missing_response_quorum = self.missing_response_quorum();
|
||||
if self.version_not_found_responses >= missing_response_quorum {
|
||||
return GET_METADATA_EARLY_STOP_REASON_VERSION_NOT_FOUND;
|
||||
}
|
||||
if self.not_found_responses >= missing_response_quorum {
|
||||
return GET_METADATA_EARLY_STOP_REASON_NOT_FOUND;
|
||||
}
|
||||
if self.hard_errors > 0 {
|
||||
return GET_METADATA_EARLY_STOP_REASON_ERROR;
|
||||
}
|
||||
if self.ignored_errors > 0 {
|
||||
return GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM;
|
||||
}
|
||||
GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn candidate_latest_quorum(&self, candidate: &FileInfo) -> Option<usize> {
|
||||
if self.default_parity_count == 0 {
|
||||
return Some(self.total_disks);
|
||||
}
|
||||
if candidate.is_canonical_delete_marker() || candidate.size == 0 || candidate.erasure.parity_blocks >= self.total_disks {
|
||||
return None;
|
||||
}
|
||||
let data_blocks = candidate.erasure.data_blocks;
|
||||
Some(if data_blocks == candidate.erasure.parity_blocks {
|
||||
data_blocks.saturating_add(1)
|
||||
} else {
|
||||
data_blocks
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn default_write_quorum(&self) -> usize {
|
||||
if self.default_parity_count == 0 || self.default_parity_count >= self.total_disks {
|
||||
return self.total_disks;
|
||||
}
|
||||
let data_blocks = self.total_disks.saturating_sub(self.default_parity_count);
|
||||
if data_blocks == self.default_parity_count {
|
||||
data_blocks.saturating_add(1)
|
||||
} else {
|
||||
data_blocks
|
||||
}
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn missing_response_quorum(&self) -> usize {
|
||||
if self.default_parity_count == 0 || self.default_parity_count >= self.total_disks {
|
||||
self.total_disks
|
||||
} else {
|
||||
self.total_disks / 2
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub(in crate::set_disk) enum MetadataCacheLookup {
|
||||
Hit(Arc<GetObjectMetadataCacheEntry>),
|
||||
@@ -1015,39 +704,6 @@ pub(in crate::set_disk) enum MetadataCacheLookup {
|
||||
RejectedInsufficientQuorum,
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn metadata_early_stop_candidate_matches(left: &FileInfo, right: &FileInfo) -> bool {
|
||||
left.volume == right.volume
|
||||
&& left.name == right.name
|
||||
&& left.version_id == right.version_id
|
||||
&& left.is_latest == right.is_latest
|
||||
&& left.deleted == right.deleted
|
||||
&& left.mark_deleted == right.mark_deleted
|
||||
&& left.transition_status == right.transition_status
|
||||
&& left.transitioned_objname == right.transitioned_objname
|
||||
&& left.transition_tier == right.transition_tier
|
||||
&& left.transition_version_id == right.transition_version_id
|
||||
&& left.transition_version == right.transition_version
|
||||
&& left.transition_version_state == right.transition_version_state
|
||||
&& left.expire_restored == right.expire_restored
|
||||
&& left.size == right.size
|
||||
&& left.mod_time == right.mod_time
|
||||
&& left.mode == right.mode
|
||||
&& left.written_by_version == right.written_by_version
|
||||
&& left.metadata == right.metadata
|
||||
&& left.replication_state_internal == right.replication_state_internal
|
||||
&& left.parts == right.parts
|
||||
&& left.checksum == right.checksum
|
||||
&& left.versioned == right.versioned
|
||||
&& left.num_versions == right.num_versions
|
||||
&& left.successor_mod_time == right.successor_mod_time
|
||||
&& left.data_dir == right.data_dir
|
||||
&& left.erasure.algorithm == right.erasure.algorithm
|
||||
&& left.erasure.data_blocks == right.erasure.data_blocks
|
||||
&& left.erasure.parity_blocks == right.erasure.parity_blocks
|
||||
&& left.erasure.block_size == right.erasure.block_size
|
||||
&& left.erasure.distribution == right.erasure.distribution
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) async fn data_read_early_stop_inline_body_miss_reason(
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
@@ -1247,10 +903,6 @@ pub(in crate::set_disk) fn classify_metadata_response_error(err: &DiskError) ->
|
||||
}
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn is_metadata_fanout_ignored_error(err: &DiskError) -> bool {
|
||||
OBJECT_OP_IGNORED_ERRS.iter().any(|ignored| ignored == err)
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn is_confirmed_missing_part_error(err: Option<&str>) -> bool {
|
||||
let Some(err) = err else {
|
||||
return false;
|
||||
|
||||
@@ -0,0 +1,385 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Pure metadata quorum and early-stop decisions for `SetDisks` reads.
|
||||
//!
|
||||
//! Disk scheduling, coalescing, cancellation, and late shard materialization
|
||||
//! remain with their existing owners; this module only classifies observations.
|
||||
|
||||
use crate::diagnostics::get::{
|
||||
GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA, GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER,
|
||||
GET_METADATA_EARLY_STOP_REASON_ERROR, GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM,
|
||||
GET_METADATA_EARLY_STOP_REASON_NOT_FOUND, GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST,
|
||||
GET_METADATA_EARLY_STOP_REASON_VALID_QUORUM, GET_METADATA_EARLY_STOP_REASON_VERSION_MATCH_QUORUM,
|
||||
GET_METADATA_EARLY_STOP_REASON_VERSION_NOT_FOUND,
|
||||
};
|
||||
use crate::disk::error::DiskError;
|
||||
use crate::disk::error_reduce::OBJECT_OP_IGNORED_ERRS;
|
||||
use crate::set_disk::file_info_is_valid_for_metadata;
|
||||
use rustfs_filemeta::FileInfo;
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub(in crate::set_disk) struct MetadataEarlyStopDecision {
|
||||
pub(in crate::set_disk) reason: &'static str,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub(in crate::set_disk) struct MetadataQuorumAccumulator {
|
||||
pub(in crate::set_disk) total_disks: usize,
|
||||
pub(in crate::set_disk) default_parity_count: usize,
|
||||
pub(in crate::set_disk) allow_early_stop: bool,
|
||||
pub(in crate::set_disk) valid_responses: usize,
|
||||
pub(in crate::set_disk) not_found_responses: usize,
|
||||
pub(in crate::set_disk) version_not_found_responses: usize,
|
||||
pub(in crate::set_disk) ignored_errors: usize,
|
||||
pub(in crate::set_disk) hard_errors: usize,
|
||||
pub(in crate::set_disk) candidate: Option<FileInfo>,
|
||||
pub(in crate::set_disk) candidate_votes: usize,
|
||||
// Bitset of shard indexes whose metadata matches the candidate. Erasure
|
||||
// layouts are capped at 16 shards, so this stays allocation-free on the
|
||||
// GET metadata hot path.
|
||||
candidate_shard_mask: u16,
|
||||
pub(in crate::set_disk) conflicting_metadata: bool,
|
||||
pub(in crate::set_disk) delete_marker_seen: bool,
|
||||
pub(in crate::set_disk) delete_marker_candidates: Vec<(FileInfo, usize)>,
|
||||
pub(in crate::set_disk) delete_marker_votes: usize,
|
||||
pub(in crate::set_disk) requested_version_id: String,
|
||||
pub(in crate::set_disk) matching_version_votes: usize,
|
||||
}
|
||||
|
||||
impl MetadataQuorumAccumulator {
|
||||
pub(in crate::set_disk) fn new(total_disks: usize, default_parity_count: usize, allow_early_stop: bool) -> Self {
|
||||
Self {
|
||||
total_disks,
|
||||
default_parity_count,
|
||||
allow_early_stop,
|
||||
valid_responses: 0,
|
||||
not_found_responses: 0,
|
||||
version_not_found_responses: 0,
|
||||
ignored_errors: 0,
|
||||
hard_errors: 0,
|
||||
candidate: None,
|
||||
candidate_votes: 0,
|
||||
candidate_shard_mask: 0,
|
||||
conflicting_metadata: false,
|
||||
delete_marker_seen: false,
|
||||
delete_marker_candidates: Vec::new(),
|
||||
delete_marker_votes: 0,
|
||||
requested_version_id: String::new(),
|
||||
matching_version_votes: 0,
|
||||
}
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn with_requested_version_id(mut self, version_id: &str) -> Self {
|
||||
self.requested_version_id = version_id.to_string();
|
||||
self
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn observe_file_info(&mut self, file_info: &FileInfo) {
|
||||
self.observe_file_info_with_index(None, file_info);
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn observe_file_info_at(&mut self, disk_index: usize, file_info: &FileInfo) {
|
||||
self.observe_file_info_with_index(Some(disk_index), file_info);
|
||||
}
|
||||
|
||||
fn observe_file_info_with_index(&mut self, disk_index: Option<usize>, file_info: &FileInfo) {
|
||||
if !file_info_is_valid_for_metadata(file_info) {
|
||||
self.hard_errors = self.hard_errors.saturating_add(1);
|
||||
return;
|
||||
}
|
||||
|
||||
self.valid_responses = self.valid_responses.saturating_add(1);
|
||||
|
||||
// Track version match for versioned requests
|
||||
if !self.requested_version_id.is_empty()
|
||||
&& let Some(ref vid) = file_info.version_id
|
||||
&& vid.to_string() == self.requested_version_id
|
||||
{
|
||||
self.matching_version_votes = self.matching_version_votes.saturating_add(1);
|
||||
}
|
||||
|
||||
if file_info.is_canonical_delete_marker() {
|
||||
self.delete_marker_seen = true;
|
||||
if let Some((_, votes)) = self
|
||||
.delete_marker_candidates
|
||||
.iter_mut()
|
||||
.find(|(candidate, _)| metadata_early_stop_candidate_matches(candidate, file_info))
|
||||
{
|
||||
*votes = votes.saturating_add(1);
|
||||
} else {
|
||||
self.delete_marker_candidates.push((file_info.clone(), 1));
|
||||
}
|
||||
self.delete_marker_votes = self
|
||||
.delete_marker_candidates
|
||||
.iter()
|
||||
.map(|(_, votes)| *votes)
|
||||
.max()
|
||||
.unwrap_or_default();
|
||||
self.conflicting_metadata |= self.delete_marker_candidates.len() > 1;
|
||||
return;
|
||||
}
|
||||
|
||||
match &self.candidate {
|
||||
Some(candidate) if metadata_early_stop_candidate_matches(candidate, file_info) => {
|
||||
self.candidate_votes = self.candidate_votes.saturating_add(1);
|
||||
if let Some(disk_index) = disk_index
|
||||
&& let Some(bit) = Self::candidate_shard_bit(candidate, file_info, disk_index)
|
||||
{
|
||||
self.candidate_shard_mask |= bit;
|
||||
}
|
||||
}
|
||||
Some(_) => {
|
||||
self.conflicting_metadata = true;
|
||||
}
|
||||
None => {
|
||||
self.candidate = Some(file_info.clone());
|
||||
self.candidate_votes = 1;
|
||||
if let Some(disk_index) = disk_index
|
||||
&& let Some(bit) = Self::candidate_shard_bit(file_info, file_info, disk_index)
|
||||
{
|
||||
self.candidate_shard_mask |= bit;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn candidate_shard_bit(candidate: &FileInfo, file_info: &FileInfo, disk_index: usize) -> Option<u16> {
|
||||
let &erasure_index = candidate.erasure.distribution.get(disk_index)?;
|
||||
if erasure_index == 0 || erasure_index > u16::BITS as usize || file_info.erasure.index != erasure_index {
|
||||
return None;
|
||||
}
|
||||
Some(1u16 << (erasure_index - 1))
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn candidate_has_read_reserve(&self) -> bool {
|
||||
self.candidate_read_reserve_target()
|
||||
.is_some_and(|required| self.candidate_shard_mask.count_ones() as usize >= required)
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn candidate_read_reserve_target(&self) -> Option<usize> {
|
||||
let candidate = self.candidate.as_ref()?;
|
||||
Some(
|
||||
candidate
|
||||
.erasure
|
||||
.data_blocks
|
||||
.saturating_add(usize::from(candidate.erasure.parity_blocks > 0)),
|
||||
)
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn observe_error(&mut self, err: &DiskError) {
|
||||
match err {
|
||||
DiskError::FileNotFound | DiskError::VolumeNotFound => {
|
||||
self.not_found_responses = self.not_found_responses.saturating_add(1);
|
||||
}
|
||||
DiskError::FileVersionNotFound => {
|
||||
self.version_not_found_responses = self.version_not_found_responses.saturating_add(1);
|
||||
}
|
||||
_ if is_metadata_fanout_ignored_error(err) => {
|
||||
self.ignored_errors = self.ignored_errors.saturating_add(1);
|
||||
}
|
||||
_ => {
|
||||
self.hard_errors = self.hard_errors.saturating_add(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn early_stop_decision(&self) -> Option<MetadataEarlyStopDecision> {
|
||||
if !self.allow_early_stop {
|
||||
return None;
|
||||
}
|
||||
if self.delete_marker_votes >= self.default_write_quorum() {
|
||||
return Some(MetadataEarlyStopDecision {
|
||||
reason: GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER,
|
||||
});
|
||||
}
|
||||
if self.conflicting_metadata
|
||||
|| self.delete_marker_seen
|
||||
|| self.not_found_responses > 0
|
||||
|| self.version_not_found_responses > 0
|
||||
|| self.hard_errors > 0
|
||||
{
|
||||
return None;
|
||||
}
|
||||
if self
|
||||
.candidate
|
||||
.as_ref()
|
||||
.and_then(|candidate| self.candidate_latest_quorum(candidate))
|
||||
.is_some_and(|latest_quorum| self.candidate_votes >= latest_quorum)
|
||||
{
|
||||
return Some(MetadataEarlyStopDecision {
|
||||
reason: GET_METADATA_EARLY_STOP_REASON_VALID_QUORUM,
|
||||
});
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Check if a versioned request can early-stop because the requested
|
||||
/// version_id has reached quorum across disks.
|
||||
pub(in crate::set_disk) fn version_early_stop_decision(&self) -> Option<MetadataEarlyStopDecision> {
|
||||
if !self.allow_early_stop {
|
||||
return None;
|
||||
}
|
||||
if self.requested_version_id.is_empty() {
|
||||
return None;
|
||||
}
|
||||
if self.conflicting_metadata
|
||||
|| self.delete_marker_seen
|
||||
|| self.not_found_responses > 0
|
||||
|| self.version_not_found_responses > 0
|
||||
|| self.hard_errors > 0
|
||||
{
|
||||
return None;
|
||||
}
|
||||
if self.matching_version_votes >= self.read_quorum_for_version() {
|
||||
return Some(MetadataEarlyStopDecision {
|
||||
reason: GET_METADATA_EARLY_STOP_REASON_VERSION_MATCH_QUORUM,
|
||||
});
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn can_still_reach_early_stop_with_pending(&self, pending: usize) -> bool {
|
||||
if !self.allow_early_stop {
|
||||
return false;
|
||||
}
|
||||
if self.delete_marker_votes.saturating_add(pending) >= self.default_write_quorum() {
|
||||
return true;
|
||||
}
|
||||
if self.conflicting_metadata
|
||||
|| self.delete_marker_seen
|
||||
|| self.not_found_responses > 0
|
||||
|| self.version_not_found_responses > 0
|
||||
|| self.hard_errors > 0
|
||||
{
|
||||
return false;
|
||||
}
|
||||
if !self.requested_version_id.is_empty()
|
||||
&& self.matching_version_votes.saturating_add(pending) >= self.read_quorum_for_version()
|
||||
{
|
||||
return true;
|
||||
}
|
||||
match &self.candidate {
|
||||
Some(candidate) => self
|
||||
.candidate_latest_quorum(candidate)
|
||||
.is_some_and(|latest_quorum| self.candidate_votes.saturating_add(pending) >= latest_quorum),
|
||||
None => pending >= self.default_write_quorum(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Compute the read quorum threshold for version-aware early-stop.
|
||||
/// Uses `total_disks / 2` (like `missing_response_quorum`) when
|
||||
/// `default_parity_count` is set, otherwise requires all disks.
|
||||
pub(in crate::set_disk) fn read_quorum_for_version(&self) -> usize {
|
||||
self.missing_response_quorum()
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn final_miss_reason(&self) -> &'static str {
|
||||
if !self.allow_early_stop {
|
||||
return GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST;
|
||||
}
|
||||
if self.conflicting_metadata {
|
||||
return GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA;
|
||||
}
|
||||
if self.delete_marker_seen {
|
||||
return GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER;
|
||||
}
|
||||
let missing_response_quorum = self.missing_response_quorum();
|
||||
if self.version_not_found_responses >= missing_response_quorum {
|
||||
return GET_METADATA_EARLY_STOP_REASON_VERSION_NOT_FOUND;
|
||||
}
|
||||
if self.not_found_responses >= missing_response_quorum {
|
||||
return GET_METADATA_EARLY_STOP_REASON_NOT_FOUND;
|
||||
}
|
||||
if self.hard_errors > 0 {
|
||||
return GET_METADATA_EARLY_STOP_REASON_ERROR;
|
||||
}
|
||||
if self.ignored_errors > 0 {
|
||||
return GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM;
|
||||
}
|
||||
GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn candidate_latest_quorum(&self, candidate: &FileInfo) -> Option<usize> {
|
||||
if self.default_parity_count == 0 {
|
||||
return Some(self.total_disks);
|
||||
}
|
||||
if candidate.is_canonical_delete_marker() || candidate.size == 0 || candidate.erasure.parity_blocks >= self.total_disks {
|
||||
return None;
|
||||
}
|
||||
let data_blocks = candidate.erasure.data_blocks;
|
||||
Some(if data_blocks == candidate.erasure.parity_blocks {
|
||||
data_blocks.saturating_add(1)
|
||||
} else {
|
||||
data_blocks
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn default_write_quorum(&self) -> usize {
|
||||
if self.default_parity_count == 0 || self.default_parity_count >= self.total_disks {
|
||||
return self.total_disks;
|
||||
}
|
||||
let data_blocks = self.total_disks.saturating_sub(self.default_parity_count);
|
||||
if data_blocks == self.default_parity_count {
|
||||
data_blocks.saturating_add(1)
|
||||
} else {
|
||||
data_blocks
|
||||
}
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn missing_response_quorum(&self) -> usize {
|
||||
if self.default_parity_count == 0 || self.default_parity_count >= self.total_disks {
|
||||
self.total_disks
|
||||
} else {
|
||||
self.total_disks / 2
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn metadata_early_stop_candidate_matches(left: &FileInfo, right: &FileInfo) -> bool {
|
||||
left.volume == right.volume
|
||||
&& left.name == right.name
|
||||
&& left.version_id == right.version_id
|
||||
&& left.is_latest == right.is_latest
|
||||
&& left.deleted == right.deleted
|
||||
&& left.mark_deleted == right.mark_deleted
|
||||
&& left.transition_status == right.transition_status
|
||||
&& left.transitioned_objname == right.transitioned_objname
|
||||
&& left.transition_tier == right.transition_tier
|
||||
&& left.transition_version_id == right.transition_version_id
|
||||
&& left.transition_version == right.transition_version
|
||||
&& left.transition_version_state == right.transition_version_state
|
||||
&& left.expire_restored == right.expire_restored
|
||||
&& left.size == right.size
|
||||
&& left.mod_time == right.mod_time
|
||||
&& left.mode == right.mode
|
||||
&& left.written_by_version == right.written_by_version
|
||||
&& left.metadata == right.metadata
|
||||
&& left.replication_state_internal == right.replication_state_internal
|
||||
&& left.parts == right.parts
|
||||
&& left.checksum == right.checksum
|
||||
&& left.versioned == right.versioned
|
||||
&& left.num_versions == right.num_versions
|
||||
&& left.successor_mod_time == right.successor_mod_time
|
||||
&& left.data_dir == right.data_dir
|
||||
&& left.erasure.algorithm == right.erasure.algorithm
|
||||
&& left.erasure.data_blocks == right.erasure.data_blocks
|
||||
&& left.erasure.parity_blocks == right.erasure.parity_blocks
|
||||
&& left.erasure.block_size == right.erasure.block_size
|
||||
&& left.erasure.distribution == right.erasure.distribution
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn is_metadata_fanout_ignored_error(err: &DiskError) -> bool {
|
||||
OBJECT_OP_IGNORED_ERRS.iter().any(|ignored| ignored == err)
|
||||
}
|
||||
@@ -18,3 +18,4 @@
|
||||
//! duplicating read/write/erasure logic.
|
||||
|
||||
pub(crate) mod io_primitives;
|
||||
mod metadata_quorum;
|
||||
|
||||
@@ -45,6 +45,11 @@ use tracing::{debug, error, info, warn};
|
||||
use super::{DiskError, Endpoint, HealDiskExt as _, local_disk_map_read};
|
||||
|
||||
const KEEP_HEAL_TASK_STATUS_DURATION: Duration = Duration::from_secs(10 * 60);
|
||||
// Each cache includes alias tokens in its count and byte budget. Eviction
|
||||
// removes every token sharing a snapshot; neither cache retains repair state.
|
||||
const MAX_COMPLETED_HEAL_TOKENS: usize = 1024;
|
||||
const MAX_COMPLETED_HEAL_BYTES: usize = 64 * 1024 * 1024;
|
||||
const MAX_COMPLETED_HEAL_RESULT_BYTES: usize = 1024 * 1024;
|
||||
const DISPLACED_HEAL_REASON: &str = "reason=displaced; retry_hint=submit_again";
|
||||
const LOG_COMPONENT_HEAL: &str = "heal";
|
||||
const LOG_SUBSYSTEM_DISK_SCANNER: &str = "disk_scanner";
|
||||
@@ -180,6 +185,8 @@ fn record_displaced_terminal(
|
||||
request: &HealRequest,
|
||||
) -> Arc<CompletedHealStatus> {
|
||||
let terminal = Arc::new(CompletedHealStatus {
|
||||
progress: None,
|
||||
retained_bytes: std::sync::OnceLock::new(),
|
||||
heal_type: request.heal_type.clone(),
|
||||
status: HealTaskStatus::Failed {
|
||||
error: format!("heal task displaced by a higher-priority request ({DISPLACED_HEAL_REASON})"),
|
||||
@@ -193,6 +200,7 @@ fn record_displaced_terminal(
|
||||
let mut terminals = lock_displaced_terminals(registry);
|
||||
prune_completed_heal_statuses(&mut terminals);
|
||||
terminals.insert(request.id.clone(), Arc::clone(&terminal));
|
||||
prune_completed_heal_statuses(&mut terminals);
|
||||
terminal
|
||||
}
|
||||
|
||||
@@ -209,9 +217,15 @@ async fn remove_displaced_task_aliases(
|
||||
.collect::<Vec<_>>();
|
||||
let mut displaced_terminals = lock_displaced_terminals(terminals);
|
||||
prune_completed_heal_statuses(&mut displaced_terminals);
|
||||
for alias_id in alias_ids {
|
||||
displaced_terminals.insert(alias_id, Arc::clone(terminal));
|
||||
if displaced_terminals
|
||||
.get(task_id)
|
||||
.is_some_and(|current| Arc::ptr_eq(current, terminal))
|
||||
{
|
||||
for alias_id in alias_ids {
|
||||
displaced_terminals.insert(alias_id, Arc::clone(terminal));
|
||||
}
|
||||
}
|
||||
prune_completed_heal_statuses(&mut displaced_terminals);
|
||||
aliases.retain(|alias_id, alias| alias_id != task_id && alias.task_id != task_id);
|
||||
}
|
||||
|
||||
@@ -222,6 +236,36 @@ async fn remove_task_aliases_for_task(registry: &Arc<Mutex<HashMap<String, HealT
|
||||
.retain(|alias_id, alias| alias_id != task_id && alias.task_id != task_id);
|
||||
}
|
||||
|
||||
// Callers hold active ownership until publication. Lock order is active ->
|
||||
// retrying (when needed) -> aliases -> completed; queries release aliases
|
||||
// before looking up active state. Publishing aliases before removing their
|
||||
// mapping keeps both an already-resolved token and a new lookup valid.
|
||||
async fn publish_completed_heal(
|
||||
completed_heals: &Mutex<HashMap<String, Arc<CompletedHealStatus>>>,
|
||||
task_aliases: &Mutex<HashMap<String, HealTaskAlias>>,
|
||||
task_id: &str,
|
||||
completed: CompletedHealStatus,
|
||||
terminal: bool,
|
||||
) {
|
||||
let completed = Arc::new(completed);
|
||||
completed.retained_bytes();
|
||||
let mut aliases = task_aliases.lock().await;
|
||||
let mut retained = completed_heals.lock().await;
|
||||
if let Some(previous) = retained.get(task_id).cloned() {
|
||||
for entry in retained.values_mut().filter(|entry| Arc::ptr_eq(entry, &previous)) {
|
||||
*entry = Arc::clone(&completed);
|
||||
}
|
||||
}
|
||||
retained.insert(task_id.to_owned(), Arc::clone(&completed));
|
||||
if terminal {
|
||||
for (alias_id, _) in aliases.iter().filter(|(_, alias)| alias.task_id == task_id) {
|
||||
retained.insert(alias_id.clone(), Arc::clone(&completed));
|
||||
}
|
||||
aliases.retain(|alias_id, alias| alias_id != task_id && alias.task_id != task_id);
|
||||
}
|
||||
prune_completed_heal_statuses(&mut retained);
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct HealTaskReport {
|
||||
pub status: HealTaskStatus,
|
||||
@@ -268,7 +312,7 @@ fn completed_task_report(completed: &CompletedHealStatus, since: Option<u64>) ->
|
||||
let result_items = match since {
|
||||
None => completed.seqed_items.iter().map(|(_, item)| item.clone()).collect(),
|
||||
Some(cursor) => {
|
||||
if cursor + 1 < completed.min_seq {
|
||||
if cursor.saturating_add(1) < completed.min_seq {
|
||||
lagged = true;
|
||||
}
|
||||
completed
|
||||
@@ -283,7 +327,7 @@ fn completed_task_report(completed: &CompletedHealStatus, since: Option<u64>) ->
|
||||
status: completed.status.clone(),
|
||||
result_items,
|
||||
result_items_truncated: completed.result_items_truncated || lagged,
|
||||
progress: None,
|
||||
progress: completed.progress.clone(),
|
||||
next_seq: completed.next_seq,
|
||||
min_seq: completed.min_seq,
|
||||
}
|
||||
@@ -1847,14 +1891,14 @@ impl HealManager {
|
||||
|
||||
pub async fn get_task_progress(&self, task_id: &str) -> Result<HealProgress> {
|
||||
let canonical_task_id = self.canonical_task_id(task_id).await;
|
||||
let active_heals = self.active_heals.lock().await;
|
||||
if let Some(task) = active_heals.get(&canonical_task_id) {
|
||||
Ok(task.get_progress().await)
|
||||
} else {
|
||||
Err(Error::TaskNotFound {
|
||||
task_id: task_id.to_string(),
|
||||
})
|
||||
}
|
||||
let progress = match self.lookup_task_state(&canonical_task_id, None).await {
|
||||
TaskStateLookup::Active(task) => Some(task.get_progress().await),
|
||||
TaskStateLookup::Completed(completed) => completed.progress.clone(),
|
||||
_ => None,
|
||||
};
|
||||
progress.ok_or_else(|| Error::TaskNotFound {
|
||||
task_id: task_id.to_string(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Cancel task
|
||||
@@ -1864,6 +1908,8 @@ impl HealManager {
|
||||
let mut active_heals = self.active_heals.lock().await;
|
||||
if let Some(task) = active_heals.get(&canonical_task_id) {
|
||||
task.cancel().await?;
|
||||
let completed = CompletedHealStatus::snapshot(task, HealTaskStatus::Cancelled).await;
|
||||
publish_completed_heal(&self.completed_heals, &self.task_aliases, &canonical_task_id, completed, true).await;
|
||||
active_heals.remove(&canonical_task_id);
|
||||
publish_active_heal_count(&active_heals);
|
||||
info!(
|
||||
@@ -1940,6 +1986,8 @@ impl HealManager {
|
||||
for task_id in &task_ids {
|
||||
if let Some(task) = active_heals.get(task_id) {
|
||||
task.cancel().await?;
|
||||
let completed = CompletedHealStatus::snapshot(task, HealTaskStatus::Cancelled).await;
|
||||
publish_completed_heal(&self.completed_heals, &self.task_aliases, task_id, completed, true).await;
|
||||
}
|
||||
active_heals.remove(task_id);
|
||||
cancelled += 1;
|
||||
|
||||
@@ -82,6 +82,8 @@ pub(super) enum QueuePushOutcome {
|
||||
pub(super) struct CompletedHealStatus {
|
||||
pub(super) heal_type: HealType,
|
||||
pub(super) status: HealTaskStatus,
|
||||
pub(super) progress: Option<HealProgress>,
|
||||
pub(super) retained_bytes: std::sync::OnceLock<usize>,
|
||||
pub(super) result_items_truncated: bool,
|
||||
pub(super) completed_at: SystemTime,
|
||||
/// Sequence-stamped retained window, archived with the completion so
|
||||
@@ -92,6 +94,133 @@ pub(super) struct CompletedHealStatus {
|
||||
pub(super) min_seq: u64,
|
||||
}
|
||||
|
||||
impl CompletedHealStatus {
|
||||
// Account for owned capacities, including nested drive arrays. Aliases
|
||||
// conservatively charge the shared allocation again, keeping both token
|
||||
// count and retained payload bounded without a second ownership index.
|
||||
pub(super) fn retained_bytes(&self) -> usize {
|
||||
*self.retained_bytes.get_or_init(|| self.measure_retained_bytes())
|
||||
}
|
||||
|
||||
fn measure_retained_bytes(&self) -> usize {
|
||||
let mut bytes = size_of::<Self>();
|
||||
let mut add = |amount: usize| bytes = bytes.saturating_add(amount);
|
||||
match &self.heal_type {
|
||||
HealType::Cluster => {}
|
||||
HealType::Bucket { bucket } => add(bucket.capacity()),
|
||||
HealType::Object {
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
}
|
||||
| HealType::ECDecode {
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
} => {
|
||||
add(bucket.capacity());
|
||||
add(object.capacity());
|
||||
add(version_id.as_ref().map_or(0, String::capacity));
|
||||
}
|
||||
HealType::Prefix { bucket, prefix } => {
|
||||
add(bucket.capacity());
|
||||
add(prefix.capacity());
|
||||
}
|
||||
HealType::Metadata { bucket, object } => {
|
||||
add(bucket.capacity());
|
||||
add(object.capacity());
|
||||
}
|
||||
HealType::ErasureSet { buckets, set_disk_id } => {
|
||||
add(buckets.capacity().saturating_mul(size_of::<String>()));
|
||||
for bucket in buckets {
|
||||
add(bucket.capacity());
|
||||
}
|
||||
add(set_disk_id.capacity());
|
||||
}
|
||||
}
|
||||
if let HealTaskStatus::Failed { error } | HealTaskStatus::Retrying { error, .. } = &self.status {
|
||||
add(error.capacity());
|
||||
}
|
||||
add(self
|
||||
.progress
|
||||
.as_ref()
|
||||
.and_then(|progress| progress.current_object.as_ref())
|
||||
.map_or(0, String::capacity));
|
||||
add(self.seqed_items.capacity().saturating_mul(size_of::<(u64, HealResultItem)>()));
|
||||
for (_, item) in &self.seqed_items {
|
||||
add(Self::result_item_heap_bytes(item));
|
||||
}
|
||||
bytes
|
||||
}
|
||||
|
||||
fn result_item_heap_bytes(item: &HealResultItem) -> usize {
|
||||
let mut bytes = 0usize;
|
||||
let mut add = |amount: usize| bytes = bytes.saturating_add(amount);
|
||||
for value in [
|
||||
&item.heal_item_type,
|
||||
&item.bucket,
|
||||
&item.object,
|
||||
&item.version_id,
|
||||
&item.detail,
|
||||
] {
|
||||
add(value.capacity());
|
||||
}
|
||||
for infos in [&item.before, &item.after] {
|
||||
add(infos
|
||||
.drives
|
||||
.capacity()
|
||||
.saturating_mul(size_of::<rustfs_madmin::heal_commands::HealDriveInfo>()));
|
||||
for drive in &infos.drives {
|
||||
add(drive.uuid.capacity());
|
||||
add(drive.endpoint.capacity());
|
||||
add(drive.state.capacity());
|
||||
}
|
||||
}
|
||||
bytes
|
||||
}
|
||||
|
||||
pub(super) fn bound_result_window(&mut self) {
|
||||
let mut bytes = 0usize;
|
||||
let retained = self
|
||||
.seqed_items
|
||||
.iter()
|
||||
.rev()
|
||||
.take_while(|(_, item)| {
|
||||
bytes = bytes
|
||||
.saturating_add(size_of::<(u64, HealResultItem)>())
|
||||
.saturating_add(Self::result_item_heap_bytes(item));
|
||||
bytes <= MAX_COMPLETED_HEAL_RESULT_BYTES
|
||||
})
|
||||
.count();
|
||||
let truncated = retained < self.seqed_items.len();
|
||||
if truncated {
|
||||
self.seqed_items.drain(..self.seqed_items.len() - retained);
|
||||
self.seqed_items.shrink_to_fit();
|
||||
self.min_seq = self.seqed_items.first().map_or(self.next_seq, |(seq, _)| *seq);
|
||||
self.result_items_truncated = true;
|
||||
self.retained_bytes.take();
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn snapshot(task: &HealTask, status: HealTaskStatus) -> Self {
|
||||
let seqed_items = task.get_seqed_result_items().await;
|
||||
let (next_seq, min_seq) = task.result_seq_cursors();
|
||||
let mut snapshot = Self {
|
||||
heal_type: task.heal_type.clone(),
|
||||
status,
|
||||
progress: Some(task.get_progress().await),
|
||||
retained_bytes: std::sync::OnceLock::new(),
|
||||
result_items_truncated: task.result_items_truncated(),
|
||||
completed_at: SystemTime::now(),
|
||||
seqed_items,
|
||||
next_seq,
|
||||
min_seq,
|
||||
};
|
||||
snapshot.bound_result_window();
|
||||
snapshot
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub(super) struct HealTaskAlias {
|
||||
pub(super) task_id: String,
|
||||
|
||||
@@ -264,7 +264,7 @@ impl HealManager {
|
||||
error: error.clone(),
|
||||
retry_attempt: request.retry_attempts,
|
||||
});
|
||||
let retry_request_for_queue = retry_request;
|
||||
let mut retry_request_for_queue = retry_request;
|
||||
let retry_cancel_token = retry_request_for_queue.as_ref().map(|_| CancellationToken::new());
|
||||
if retry_request_for_queue.is_none() {
|
||||
replacement_recovery_anchors_clone
|
||||
@@ -272,7 +272,35 @@ impl HealManager {
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
.remove(&task_id);
|
||||
}
|
||||
let mut completed_status = match retry_request_for_status {
|
||||
Some(status) => status,
|
||||
None => task.get_status().await,
|
||||
};
|
||||
let mut completed_status_entry = CompletedHealStatus::snapshot(&task, completed_status.clone()).await;
|
||||
let completed_progress = task.get_progress().await;
|
||||
#[cfg(test)]
|
||||
tests::pause_completed_retention_before_publish(&task_id, &completed_status).await;
|
||||
let mut active_heals_guard = active_heals_clone.lock().await;
|
||||
let owns_completion = active_heals_guard.contains_key(&task_id);
|
||||
let cancelled_completion = if owns_completion {
|
||||
false
|
||||
} else {
|
||||
// Cancellation can win while a finished worker waits
|
||||
// for active ownership. It must not resurrect a retry
|
||||
// or replace an acknowledged cancellation with success.
|
||||
retry_request_for_queue = None;
|
||||
completed_heals_clone
|
||||
.lock()
|
||||
.await
|
||||
.get(&task_id)
|
||||
.is_some_and(|completed| completed.status == HealTaskStatus::Cancelled)
|
||||
};
|
||||
if cancelled_completion {
|
||||
completed_status = HealTaskStatus::Cancelled;
|
||||
completed_status_entry.status = HealTaskStatus::Cancelled;
|
||||
}
|
||||
let terminal_completion = !matches!(completed_status, HealTaskStatus::Retrying { .. });
|
||||
let successful_completion = matches!(completed_status, HealTaskStatus::Completed);
|
||||
// Keep retry ownership continuous: status snapshots acquire
|
||||
// these locks in the same active -> retrying order.
|
||||
let mut retrying_heals_guard = if let (Some((request, _, error)), Some(cancel_token)) =
|
||||
@@ -295,6 +323,16 @@ impl HealManager {
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if owns_completion || cancelled_completion {
|
||||
publish_completed_heal(
|
||||
&completed_heals_clone,
|
||||
&task_aliases_clone,
|
||||
&task_id,
|
||||
completed_status_entry,
|
||||
terminal_completion,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
let completed_task = active_heals_guard.remove(&task_id);
|
||||
if let Some(completed_task) = completed_task.as_ref() {
|
||||
publish_active_heal_count(&active_heals_guard);
|
||||
@@ -304,33 +342,10 @@ impl HealManager {
|
||||
drop(retrying_heals_guard.take());
|
||||
drop(active_heals_guard);
|
||||
|
||||
if let Some(completed_task) = completed_task {
|
||||
let completed_status = if let Some(status) = retry_request_for_status {
|
||||
status
|
||||
} else {
|
||||
completed_task.get_status().await
|
||||
};
|
||||
let terminal_completion = !matches!(completed_status, HealTaskStatus::Retrying { .. });
|
||||
let successful_completion = matches!(completed_status, HealTaskStatus::Completed);
|
||||
let completed_progress = completed_task.get_progress().await;
|
||||
// Single snapshot of the retained window: the task is
|
||||
// finished and already off the active map, so there is
|
||||
// no concurrent writer to race with.
|
||||
let seqed_items = completed_task.get_seqed_result_items().await;
|
||||
let (next_seq, min_seq) = completed_task.result_seq_cursors();
|
||||
let completed_status_entry = CompletedHealStatus {
|
||||
heal_type: completed_task.heal_type.clone(),
|
||||
status: completed_status.clone(),
|
||||
result_items_truncated: completed_task.result_items_truncated(),
|
||||
completed_at: SystemTime::now(),
|
||||
seqed_items,
|
||||
next_seq,
|
||||
min_seq,
|
||||
};
|
||||
let mut completed_heals_guard = completed_heals_clone.lock().await;
|
||||
prune_completed_heal_statuses(&mut completed_heals_guard);
|
||||
completed_heals_guard.insert(task_id.clone(), Arc::new(completed_status_entry));
|
||||
drop(completed_heals_guard);
|
||||
#[cfg(test)]
|
||||
tests::pause_completed_retention_handoff(&task_id).await;
|
||||
|
||||
if completed_task.is_some() {
|
||||
// update statistics
|
||||
let mut stats = statistics_clone.write().await;
|
||||
match completed_status {
|
||||
@@ -352,10 +367,6 @@ impl HealManager {
|
||||
} else {
|
||||
release_mrf_repair_notice_targets(notice_targets);
|
||||
}
|
||||
task_aliases_clone
|
||||
.lock()
|
||||
.await
|
||||
.retain(|alias_id, alias| alias_id != &task_id && alias.task_id != task_id);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -718,17 +729,42 @@ pub(super) fn heal_request_set_key_for_task(task: &HealTask) -> Option<String> {
|
||||
}
|
||||
|
||||
pub(super) fn prune_completed_heal_statuses(completed_heals: &mut HashMap<String, Arc<CompletedHealStatus>>) {
|
||||
let Ok(now) = SystemTime::now().duration_since(SystemTime::UNIX_EPOCH) else {
|
||||
return;
|
||||
};
|
||||
prune_completed_heal_statuses_at(completed_heals, SystemTime::now());
|
||||
}
|
||||
|
||||
pub(super) fn prune_completed_heal_statuses_at(completed_heals: &mut HashMap<String, Arc<CompletedHealStatus>>, now: SystemTime) {
|
||||
completed_heals.retain(|_, completed| {
|
||||
completed
|
||||
.completed_at
|
||||
.duration_since(SystemTime::UNIX_EPOCH)
|
||||
.map(|completed_at| now.saturating_sub(completed_at) <= KEEP_HEAL_TASK_STATUS_DURATION)
|
||||
now.duration_since(completed.completed_at)
|
||||
.map(|age| age <= KEEP_HEAL_TASK_STATUS_DURATION)
|
||||
.unwrap_or(false)
|
||||
});
|
||||
let entry_bytes = |key: &String, value: &Arc<CompletedHealStatus>| {
|
||||
key.capacity()
|
||||
.saturating_add(size_of::<(String, Arc<CompletedHealStatus>)>())
|
||||
.saturating_add(value.retained_bytes())
|
||||
};
|
||||
let mut bytes = completed_heals
|
||||
.iter()
|
||||
.fold(0usize, |total, (key, value)| total.saturating_add(entry_bytes(key, value)));
|
||||
while completed_heals.len() > MAX_COMPLETED_HEAL_TOKENS || bytes > MAX_COMPLETED_HEAL_BYTES {
|
||||
let Some(oldest) = completed_heals
|
||||
.iter()
|
||||
.min_by(|(left_id, left), (right_id, right)| {
|
||||
left.completed_at.cmp(&right.completed_at).then_with(|| left_id.cmp(right_id))
|
||||
})
|
||||
.map(|(_, value)| Arc::clone(value))
|
||||
else {
|
||||
break;
|
||||
};
|
||||
completed_heals.retain(|key, value| {
|
||||
if Arc::ptr_eq(value, &oldest) {
|
||||
bytes = bytes.saturating_sub(entry_bytes(key, value));
|
||||
false
|
||||
} else {
|
||||
true
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn can_schedule_request(
|
||||
|
||||
@@ -101,6 +101,326 @@ async fn process_manager_queue_once(manager: &HealManager) {
|
||||
|
||||
struct MockStorage;
|
||||
|
||||
fn completed_retention_fixture(completed_at: SystemTime) -> CompletedHealStatus {
|
||||
CompletedHealStatus {
|
||||
heal_type: HealType::Cluster,
|
||||
status: HealTaskStatus::Completed,
|
||||
progress: Some(HealProgress {
|
||||
objects_scanned: 9,
|
||||
objects_healed: 8,
|
||||
objects_failed: 1,
|
||||
..Default::default()
|
||||
}),
|
||||
retained_bytes: std::sync::OnceLock::new(),
|
||||
result_items_truncated: false,
|
||||
completed_at,
|
||||
seqed_items: vec![(3, HealResultItem::default()), (4, HealResultItem::default())],
|
||||
next_seq: 5,
|
||||
min_seq: 3,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn completed_retention_cursor_boundaries_preserve_progress() {
|
||||
let completed = completed_retention_fixture(SystemTime::now());
|
||||
for (cursor, count, lagged) in [
|
||||
(0, 2, true),
|
||||
(1, 2, true),
|
||||
(2, 2, false),
|
||||
(3, 1, false),
|
||||
(4, 0, false),
|
||||
(5, 0, false),
|
||||
(u64::MAX, 0, false),
|
||||
] {
|
||||
let report = completed_task_report(&completed, Some(cursor));
|
||||
assert_eq!(report.result_items.len(), count, "cursor={cursor}");
|
||||
assert_eq!(report.result_items_truncated, lagged, "cursor={cursor}");
|
||||
assert_eq!(report.progress, completed.progress);
|
||||
assert_eq!((report.next_seq, report.min_seq), (5, 3));
|
||||
}
|
||||
assert_eq!(completed_task_report(&completed, None).result_items.len(), 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn completed_retention_displaced_alias_does_not_resurrect_evicted_snapshot() {
|
||||
let manager = HealManager::new(Arc::new(MockStorage), None);
|
||||
let request = HealRequest::bucket("bucket".to_string());
|
||||
manager.insert_task_alias("alias", &request.id).await;
|
||||
let terminal = record_displaced_terminal(&manager.displaced_terminals, &request);
|
||||
lock_displaced_terminals(&manager.displaced_terminals).remove(&request.id);
|
||||
remove_displaced_task_aliases(&manager.task_aliases, &manager.displaced_terminals, &request.id, &terminal).await;
|
||||
for token in [&request.id, &"alias".to_string()] {
|
||||
assert!(matches!(manager.get_task_report(token).await, Err(Error::TaskNotFound { .. })));
|
||||
}
|
||||
assert!(manager.task_aliases.lock().await.is_empty());
|
||||
assert!(lock_displaced_terminals(&manager.displaced_terminals).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn completed_retention_count_ttl_and_alias_eviction_are_bounded() {
|
||||
let now = SystemTime::now();
|
||||
let mut entries = HashMap::new();
|
||||
let oldest = Arc::new(completed_retention_fixture(now - KEEP_HEAL_TASK_STATUS_DURATION));
|
||||
entries.insert("oldest".to_string(), Arc::clone(&oldest));
|
||||
entries.insert("oldest-alias".to_string(), Arc::clone(&oldest));
|
||||
for index in 2..MAX_COMPLETED_HEAL_TOKENS {
|
||||
entries.insert(format!("task-{index}"), Arc::new(completed_retention_fixture(now)));
|
||||
}
|
||||
prune_completed_heal_statuses_at(&mut entries, now);
|
||||
assert_eq!(entries.len(), MAX_COMPLETED_HEAL_TOKENS);
|
||||
entries.insert("cap-plus-one".to_string(), Arc::new(completed_retention_fixture(now)));
|
||||
prune_completed_heal_statuses_at(&mut entries, now);
|
||||
assert_eq!(entries.len(), MAX_COMPLETED_HEAL_TOKENS - 1);
|
||||
assert!(!entries.contains_key("oldest"));
|
||||
assert!(!entries.contains_key("oldest-alias"));
|
||||
entries.clear();
|
||||
entries.insert("ttl-boundary".to_string(), oldest);
|
||||
entries.insert(
|
||||
"expired".to_string(),
|
||||
Arc::new(completed_retention_fixture(
|
||||
now - KEEP_HEAL_TASK_STATUS_DURATION - Duration::from_nanos(1),
|
||||
)),
|
||||
);
|
||||
entries.insert("future".to_string(), Arc::new(completed_retention_fixture(now + Duration::from_nanos(1))));
|
||||
prune_completed_heal_statuses_at(&mut entries, now);
|
||||
assert_eq!(entries.len(), 1);
|
||||
assert!(entries.contains_key("ttl-boundary"));
|
||||
prune_completed_heal_statuses_at(&mut entries, now + Duration::from_nanos(1));
|
||||
assert!(entries.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn completed_retention_total_byte_cap_and_cap_plus_one() {
|
||||
let now = SystemTime::now();
|
||||
let key = "large".to_string();
|
||||
let mut entry = completed_retention_fixture(now);
|
||||
let base_bytes = entry.retained_bytes() + key.capacity() + size_of::<(String, Arc<CompletedHealStatus>)>();
|
||||
entry.retained_bytes.take();
|
||||
entry.status = HealTaskStatus::Failed {
|
||||
error: "x".repeat(MAX_COMPLETED_HEAL_BYTES - base_bytes),
|
||||
};
|
||||
assert_eq!(
|
||||
entry.retained_bytes() + key.capacity() + size_of::<(String, Arc<CompletedHealStatus>)>(),
|
||||
MAX_COMPLETED_HEAL_BYTES
|
||||
);
|
||||
let mut entries = HashMap::from([(key, Arc::new(entry))]);
|
||||
prune_completed_heal_statuses_at(&mut entries, now);
|
||||
assert_eq!(entries.len(), 1, "exact byte cap remains retained");
|
||||
let mut over = Arc::try_unwrap(entries.remove("large").expect("entry retained")).expect("entry not shared");
|
||||
over.retained_bytes.take();
|
||||
if let HealTaskStatus::Failed { error } = &mut over.status {
|
||||
*error = "x".repeat(error.len() + 1);
|
||||
}
|
||||
entries.insert("large".to_string(), Arc::new(over));
|
||||
prune_completed_heal_statuses_at(&mut entries, now);
|
||||
assert!(entries.is_empty(), "oversized metadata cannot escape total byte bound");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn completed_retention_large_window_keeps_cursors_and_progress() {
|
||||
let task = HealTask::from_request(HealRequest::bucket("bucket".to_string()), Arc::new(MockStorage));
|
||||
let mut snapshot = completed_retention_fixture(SystemTime::now());
|
||||
snapshot.seqed_items[0].1.detail = "x".repeat(MAX_COMPLETED_HEAL_RESULT_BYTES);
|
||||
snapshot.bound_result_window();
|
||||
assert_eq!(snapshot.seqed_items.len(), 1);
|
||||
assert_eq!((snapshot.min_seq, snapshot.next_seq), (4, 5));
|
||||
assert!(snapshot.result_items_truncated);
|
||||
assert!(snapshot.retained_bytes() < MAX_COMPLETED_HEAL_RESULT_BYTES);
|
||||
let report = completed_task_report(&snapshot, Some(0));
|
||||
assert_eq!(report.progress.expect("progress retained").objects_scanned, 9);
|
||||
assert!(report.result_items_truncated);
|
||||
let active_max = task.get_result_items_since(Some(u64::MAX)).await;
|
||||
assert!(active_max.items.is_empty());
|
||||
assert!(!active_max.lagged);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn completed_retention_result_byte_cap_and_cap_plus_one() {
|
||||
for extra in [0, 1] {
|
||||
let mut snapshot = completed_retention_fixture(SystemTime::now());
|
||||
snapshot.seqed_items = vec![(
|
||||
4,
|
||||
HealResultItem {
|
||||
detail: "x".repeat(MAX_COMPLETED_HEAL_RESULT_BYTES - size_of::<(u64, HealResultItem)>() + extra),
|
||||
..Default::default()
|
||||
},
|
||||
)];
|
||||
snapshot.min_seq = 4;
|
||||
snapshot.bound_result_window();
|
||||
assert_eq!(snapshot.seqed_items.len(), 1 - extra);
|
||||
assert_eq!(snapshot.result_items_truncated, extra == 1);
|
||||
assert_eq!(snapshot.min_seq, if extra == 0 { 4 } else { 5 });
|
||||
assert_eq!(snapshot.next_seq, 5);
|
||||
assert_eq!(snapshot.progress.as_ref().expect("progress retained").objects_scanned, 9);
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct CompletedRetentionHook {
|
||||
started: Notify,
|
||||
execute: Notify,
|
||||
handoff: Notify,
|
||||
finish: Notify,
|
||||
pause_before_publish: bool,
|
||||
before_publish: Notify,
|
||||
publish: Notify,
|
||||
prepared_status: Mutex<Option<HealTaskStatus>>,
|
||||
}
|
||||
|
||||
static COMPLETED_RETENTION_HOOKS: LazyLock<Mutex<HashMap<String, Arc<CompletedRetentionHook>>>> =
|
||||
LazyLock::new(|| Mutex::new(HashMap::new()));
|
||||
|
||||
pub(super) async fn pause_completed_retention_handoff(task_id: &str) {
|
||||
let hook = COMPLETED_RETENTION_HOOKS.lock().await.get(task_id).cloned();
|
||||
if let Some(hook) = hook {
|
||||
hook.handoff.notify_one();
|
||||
hook.finish.notified().await;
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn pause_completed_retention_before_publish(task_id: &str, status: &HealTaskStatus) {
|
||||
let hook = COMPLETED_RETENTION_HOOKS.lock().await.get(task_id).cloned();
|
||||
if let Some(hook) = hook.filter(|hook| hook.pause_before_publish) {
|
||||
*hook.prepared_status.lock().await = Some(status.clone());
|
||||
hook.before_publish.notify_one();
|
||||
hook.publish.notified().await;
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn completed_retention_cancel_wins_over_a_prepared_retry_snapshot() {
|
||||
let bucket = "completed-retention-retry-cancel";
|
||||
let manager = HealManager::new(Arc::new(MockStorage), None);
|
||||
let request = HealRequest::object(bucket.to_string(), "object".to_string(), None);
|
||||
let task_id = request.id.clone();
|
||||
let duplicate = HealRequest::object(bucket.to_string(), "object".to_string(), None);
|
||||
let alias = duplicate.id.clone();
|
||||
let hook = Arc::new(CompletedRetentionHook {
|
||||
pause_before_publish: true,
|
||||
..Default::default()
|
||||
});
|
||||
{
|
||||
let mut hooks = COMPLETED_RETENTION_HOOKS.lock().await;
|
||||
hooks.insert(bucket.to_string(), Arc::clone(&hook));
|
||||
hooks.insert(task_id.clone(), Arc::clone(&hook));
|
||||
}
|
||||
manager.submit_heal_request(request).await.expect("admit original");
|
||||
manager.submit_heal_request(duplicate).await.expect("admit alias");
|
||||
process_manager_queue_once(&manager).await;
|
||||
tokio::time::timeout(Duration::from_secs(5), hook.started.notified())
|
||||
.await
|
||||
.expect("scheduler starts");
|
||||
let task = manager.active_heals.lock().await.get(&task_id).cloned().expect("active task");
|
||||
task.progress.write().await.update_object_progress(1, 1, 0, 0, 4096);
|
||||
hook.execute.notify_one();
|
||||
tokio::time::timeout(Duration::from_secs(5), hook.before_publish.notified())
|
||||
.await
|
||||
.expect("retry snapshot prepared");
|
||||
manager.cancel_task(&alias).await.expect("cancel wins active ownership");
|
||||
assert!(matches!(*hook.prepared_status.lock().await, Some(HealTaskStatus::Retrying { .. })));
|
||||
hook.publish.notify_one();
|
||||
tokio::time::timeout(Duration::from_secs(5), hook.handoff.notified())
|
||||
.await
|
||||
.expect("scheduler finishes handoff");
|
||||
for token in [&task_id, &alias] {
|
||||
let report = manager.get_task_report(token).await.expect("cancelled token retained");
|
||||
assert_eq!(report.status, HealTaskStatus::Cancelled);
|
||||
assert_eq!(report.progress.expect("frozen progress").objects_scanned, 1);
|
||||
}
|
||||
assert!(!manager.retrying_heals.lock().await.contains_key(&task_id));
|
||||
assert!(!manager.heal_queue.lock().await.contains_request_id(&task_id));
|
||||
hook.finish.notify_one();
|
||||
COMPLETED_RETENTION_HOOKS
|
||||
.lock()
|
||||
.await
|
||||
.retain(|key, _| key != bucket && key != &task_id);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn completed_retention_scheduler_preserves_progress_aliases_and_atomic_handoff() {
|
||||
for outcome in ["success", "failed", "cancelled"] {
|
||||
let bucket = format!("completed-retention-{outcome}");
|
||||
let hook = Arc::new(CompletedRetentionHook::default());
|
||||
let manager = Arc::new(HealManager::new(Arc::new(MockStorage), None));
|
||||
let request = HealRequest::object(bucket.clone(), "object".to_string(), None);
|
||||
let task_id = request.id.clone();
|
||||
let duplicate = HealRequest::object(bucket.clone(), "object".to_string(), None);
|
||||
let alias = duplicate.id.clone();
|
||||
{
|
||||
let mut hooks = COMPLETED_RETENTION_HOOKS.lock().await;
|
||||
hooks.insert(bucket.clone(), Arc::clone(&hook));
|
||||
hooks.insert(task_id.clone(), Arc::clone(&hook));
|
||||
}
|
||||
manager.submit_heal_request(request).await.expect("admit original");
|
||||
manager.submit_heal_request(duplicate).await.expect("admit alias");
|
||||
process_manager_queue_once(&manager).await;
|
||||
tokio::time::timeout(Duration::from_secs(5), hook.started.notified())
|
||||
.await
|
||||
.expect("scheduler reaches storage");
|
||||
let task = manager
|
||||
.active_heals
|
||||
.lock()
|
||||
.await
|
||||
.get(&task_id)
|
||||
.cloned()
|
||||
.expect("task is active");
|
||||
task.progress.write().await.update_object_progress(1, 1, 0, 0, 4096);
|
||||
let before = manager.get_task_report(&alias).await.expect("alias resolves active progress");
|
||||
assert_eq!(before.progress.as_ref().expect("active progress").objects_scanned, 1);
|
||||
let poll_manager = Arc::clone(&manager);
|
||||
let poll_alias = alias.clone();
|
||||
let stop = CancellationToken::new();
|
||||
let poll_stop = stop.clone();
|
||||
let polling = tokio::spawn(async move {
|
||||
while !poll_stop.is_cancelled() {
|
||||
let report = poll_manager
|
||||
.get_task_report(&poll_alias)
|
||||
.await
|
||||
.expect("handoff must never return NotFound");
|
||||
assert!(report.progress.expect("progress never disappears").objects_scanned >= 1);
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
});
|
||||
if outcome == "cancelled" {
|
||||
manager.cancel_task(&alias).await.expect("cancel active task by alias");
|
||||
} else {
|
||||
hook.execute.notify_one();
|
||||
}
|
||||
tokio::time::timeout(Duration::from_secs(5), hook.handoff.notified())
|
||||
.await
|
||||
.expect("scheduler archives terminal");
|
||||
assert!(!manager.active_heals.lock().await.contains_key(&task_id));
|
||||
let expected = task.get_progress().await;
|
||||
for token in [&task_id, &alias] {
|
||||
assert_eq!(manager.get_task_progress(token).await.expect("terminal progress query"), expected);
|
||||
let report = manager
|
||||
.get_task_report_for_path_since(&format!("{bucket}/object"), token, Some(u64::MAX))
|
||||
.await
|
||||
.expect("terminal token remains queryable at handoff");
|
||||
assert_eq!(report.progress.as_ref(), Some(&expected));
|
||||
assert!(report.result_items.is_empty());
|
||||
match outcome {
|
||||
"success" => assert_eq!(report.status, HealTaskStatus::Completed),
|
||||
"failed" => assert!(matches!(report.status, HealTaskStatus::Failed { .. })),
|
||||
_ => assert_eq!(report.status, HealTaskStatus::Cancelled),
|
||||
}
|
||||
}
|
||||
let retained = manager.completed_heals.lock().await;
|
||||
assert!(Arc::ptr_eq(&retained[&task_id], &retained[&alias]));
|
||||
drop(retained);
|
||||
stop.cancel();
|
||||
polling.await.expect("concurrent polling succeeds");
|
||||
// Archived progress must not alias a mutable live progress object.
|
||||
task.progress.write().await.objects_scanned = 999;
|
||||
assert_eq!(manager.get_task_report(&alias).await.expect("frozen report").progress, Some(expected));
|
||||
hook.finish.notify_one();
|
||||
COMPLETED_RETENTION_HOOKS
|
||||
.lock()
|
||||
.await
|
||||
.retain(|key, _| key != &bucket && key != &task_id);
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl HealStorageAPI for MockStorage {
|
||||
async fn get_object_meta(&self, _bucket: &str, _object: &str) -> Result<Option<HealObjectInfo>> {
|
||||
@@ -123,6 +443,12 @@ impl HealStorageAPI for MockStorage {
|
||||
}
|
||||
|
||||
async fn object_exists(&self, bucket: &str, _object: &str) -> Result<bool> {
|
||||
let hook = COMPLETED_RETENTION_HOOKS.lock().await.get(bucket).cloned();
|
||||
if let Some(hook) = hook {
|
||||
hook.started.notify_one();
|
||||
hook.execute.notified().await;
|
||||
return Ok(true);
|
||||
}
|
||||
Ok(bucket == "retry-transition")
|
||||
}
|
||||
|
||||
@@ -133,13 +459,18 @@ impl HealStorageAPI for MockStorage {
|
||||
_version_id: Option<&str>,
|
||||
_opts: &HealOpts,
|
||||
) -> Result<(HealResultItem, Option<Error>)> {
|
||||
if bucket == "completed-retention-failed" {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: "retention fixture failure".to_string(),
|
||||
});
|
||||
}
|
||||
if let Some(hook) = manager_recovery_test_hook() {
|
||||
*hook
|
||||
.heal_object_calls
|
||||
.lock()
|
||||
.expect("manager recovery object call lock should not poison") += 1;
|
||||
}
|
||||
if bucket == "retry-transition" {
|
||||
if matches!(bucket, "retry-transition" | "completed-retention-retry-cancel") {
|
||||
return Ok((
|
||||
HealResultItem::default(),
|
||||
Some(Error::Storage(EcstoreError::InsufficientReadQuorum(
|
||||
@@ -1145,7 +1476,13 @@ async fn test_active_duplicate_token_can_query_and_cancel_original_task() {
|
||||
.expect("duplicate token should cancel merged active task");
|
||||
|
||||
assert!(manager.active_heals.lock().await.get(&active_task_id).is_none());
|
||||
assert!(matches!(manager.get_task_status(&active_task_id).await, Err(Error::TaskNotFound { .. })));
|
||||
assert_eq!(
|
||||
manager
|
||||
.get_task_status(&active_task_id)
|
||||
.await
|
||||
.expect("cancelled task remains queryable"),
|
||||
HealTaskStatus::Cancelled
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -1638,6 +1975,8 @@ async fn insert_retrying_request(manager: &HealManager, request: HealRequest) ->
|
||||
manager.completed_heals.lock().await.insert(
|
||||
task_id,
|
||||
Arc::new(CompletedHealStatus {
|
||||
progress: None,
|
||||
retained_bytes: std::sync::OnceLock::new(),
|
||||
heal_type: request.heal_type,
|
||||
status: HealTaskStatus::Retrying {
|
||||
error: "Lock acquisition timeout".to_string(),
|
||||
@@ -2053,7 +2392,7 @@ async fn admin_force_start_cancels_overlapping_active_task_first() {
|
||||
"the overlapping admin task must be cancelled (removed from the active table) before the new one starts"
|
||||
);
|
||||
assert!(
|
||||
matches!(manager.get_task_status(&old_id).await, Err(Error::TaskNotFound { .. })),
|
||||
matches!(manager.get_task_status(&old_id).await, Ok(HealTaskStatus::Cancelled)),
|
||||
"a cancelled task must no longer resolve as an active heal"
|
||||
);
|
||||
}
|
||||
@@ -2360,6 +2699,8 @@ async fn test_retrying_completion_outranks_the_queue_for_the_same_id() {
|
||||
manager.completed_heals.lock().await.insert(
|
||||
task_id.clone(),
|
||||
Arc::new(CompletedHealStatus {
|
||||
progress: None,
|
||||
retained_bytes: std::sync::OnceLock::new(),
|
||||
heal_type: request.heal_type.clone(),
|
||||
status: HealTaskStatus::Retrying {
|
||||
error: "transient disk failure".to_string(),
|
||||
@@ -2395,6 +2736,8 @@ async fn test_get_task_status_reads_recent_completed_status() {
|
||||
manager.completed_heals.lock().await.insert(
|
||||
"completed-token".to_string(),
|
||||
Arc::new(CompletedHealStatus {
|
||||
progress: None,
|
||||
retained_bytes: std::sync::OnceLock::new(),
|
||||
heal_type: HealType::Bucket {
|
||||
bucket: "bucket".to_string(),
|
||||
},
|
||||
@@ -2424,6 +2767,8 @@ async fn test_get_task_report_for_path_reads_completed_items() {
|
||||
manager.completed_heals.lock().await.insert(
|
||||
"completed-token".to_string(),
|
||||
Arc::new(CompletedHealStatus {
|
||||
progress: None,
|
||||
retained_bytes: std::sync::OnceLock::new(),
|
||||
heal_type: HealType::Object {
|
||||
bucket: "bucket".to_string(),
|
||||
object: "object".to_string(),
|
||||
|
||||
@@ -999,7 +999,7 @@ impl HealTask {
|
||||
let items = match since {
|
||||
None => result_items.iter().map(|(_, item)| item.clone()).collect::<Vec<_>>(),
|
||||
Some(cursor) => {
|
||||
if cursor + 1 < min_seq {
|
||||
if cursor.saturating_add(1) < min_seq {
|
||||
lagged = true;
|
||||
}
|
||||
result_items
|
||||
|
||||
@@ -182,6 +182,9 @@ pub struct BackgroundHealStatus {
|
||||
pub heal_active_tasks: u64,
|
||||
#[serde(default)]
|
||||
pub cluster_status_complete: bool,
|
||||
/// Missing on older servers; absent coverage or counts mean unknown.
|
||||
#[serde(default)]
|
||||
pub coverage: Option<BackgroundHealCoverage>,
|
||||
#[serde(default)]
|
||||
pub progress: Option<serde_json::Value>,
|
||||
/// Remaining wire fields (flattened `BackgroundHealInfo` plus the
|
||||
@@ -190,6 +193,22 @@ pub struct BackgroundHealStatus {
|
||||
pub extra: serde_json::Map<String, serde_json::Value>,
|
||||
}
|
||||
|
||||
/// Node coverage of a background heal status snapshot. Counters describe only
|
||||
/// nodes with usable snapshots; unknown peers may still be running heal work.
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct BackgroundHealCoverage {
|
||||
#[serde(default)]
|
||||
pub expected: Option<usize>,
|
||||
#[serde(default)]
|
||||
pub responded: Option<usize>,
|
||||
#[serde(default)]
|
||||
pub unknown: Option<usize>,
|
||||
/// Stable reason codes; unknown future codes are preserved verbatim.
|
||||
#[serde(default)]
|
||||
pub reasons: Vec<String>,
|
||||
}
|
||||
|
||||
/// `GET /v3/scanner/status` response, typed at the fields operators branch
|
||||
/// on; everything else passes through verbatim.
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
@@ -630,9 +649,39 @@ mod tests {
|
||||
assert_eq!(status.state, "active");
|
||||
assert_eq!(status.heal_queue_length, 3);
|
||||
assert!(status.cluster_status_complete);
|
||||
assert!(status.coverage.is_none(), "legacy payloads have unknown coverage");
|
||||
assert!(status.extra.contains_key("healOperations"), "unknown nested payloads must pass through");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn background_heal_status_missing_coverage_fields_remain_unknown() {
|
||||
for raw in [json!({"state": "degraded"}), json!({"state": "degraded", "coverage": {}})] {
|
||||
let status: BackgroundHealStatus = serde_json::from_value(raw).expect("partial legacy payload decodes");
|
||||
assert!(!status.cluster_status_complete);
|
||||
if let Some(coverage) = status.coverage {
|
||||
assert_eq!(coverage.expected, None);
|
||||
assert_eq!(coverage.responded, None);
|
||||
assert_eq!(coverage.unknown, None);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn background_heal_status_preserves_future_fields_and_reasons() {
|
||||
let raw = json!({
|
||||
"state": "degraded", "clusterStatusComplete": false,
|
||||
"coverage": {"expected": 3, "responded": 1, "unknown": 2, "reasons": ["future_reason"], "futureCoverage": true},
|
||||
"futureStatus": {"value": 7}
|
||||
});
|
||||
let status: BackgroundHealStatus = serde_json::from_value(raw).expect("future additive fields decode");
|
||||
assert_eq!(status.extra["futureStatus"]["value"], 7);
|
||||
let coverage = status.coverage.expect("coverage supplied");
|
||||
assert_eq!(coverage.expected, Some(3));
|
||||
assert_eq!(coverage.responded, Some(1));
|
||||
assert_eq!(coverage.unknown, Some(2));
|
||||
assert_eq!(coverage.reasons, ["future_reason"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_status_defaults_freshness_to_unknown() {
|
||||
let raw = json!({"enabled": true, "freshness": {"state": "stale"}, "metrics": {}});
|
||||
@@ -721,6 +770,7 @@ mod tests {
|
||||
|
||||
let status = client.background_heal_status().await.expect("status decodes");
|
||||
assert_eq!(status.state, "idle");
|
||||
assert!(status.coverage.is_none(), "older HTTP responses retain unknown coverage");
|
||||
let request = server.recorded();
|
||||
// The server registers this route POST-only; a GET here answers 405.
|
||||
assert_eq!(request.method, "POST");
|
||||
@@ -728,6 +778,28 @@ mod tests {
|
||||
assert_eq!(request.query, "");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn background_heal_status_decodes_partial_coverage_over_http() {
|
||||
let body = r#"{"state":"degraded","healQueueLength":0,"healActiveTasks":0,"clusterStatusComplete":false,"coverage":{"expected":3,"responded":1,"unknown":2,"reasons":["notification_system_unavailable"]},"futureStatus":true}"#;
|
||||
let server = TestServer::spawn(body, 200).await;
|
||||
let client = AdminClient::new(&format!("http://{}", server.addr), "ak", "sk").expect("client builds");
|
||||
let status = client
|
||||
.background_heal_status()
|
||||
.await
|
||||
.expect("partial status is a successful response");
|
||||
assert_eq!(status.state, "degraded");
|
||||
assert!(!status.cluster_status_complete);
|
||||
assert_eq!(status.extra["futureStatus"], true);
|
||||
let coverage = status.coverage.expect("partial coverage supplied");
|
||||
assert_eq!(coverage.expected, Some(3));
|
||||
assert_eq!(coverage.responded, Some(1));
|
||||
assert_eq!(coverage.unknown, Some(2));
|
||||
assert_eq!(coverage.reasons, ["notification_system_unavailable"]);
|
||||
let request = server.recorded();
|
||||
assert_eq!(request.method, "POST");
|
||||
assert_eq!(request.query, "", "reading status must not send heal control parameters");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn http_error_status_maps_to_a_typed_error_with_body() {
|
||||
let server = TestServer::spawn(r#"{"code":"AccessDenied","message":"denied"}"#, 403).await;
|
||||
|
||||
@@ -1283,6 +1283,54 @@ pub struct ScannerDirtyUsageSnapshotResponse {
|
||||
#[prost(bytes = "bytes", tag = "7")]
|
||||
pub response_proof: ::prost::bytes::Bytes,
|
||||
}
|
||||
/// Receiver-only protocol. Producers must retain whole-cycle ACK until they
|
||||
/// have a durable per-bucket publication proof.
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct ScannerScopedDirtyUsageEntry {
|
||||
#[prost(string, tag = "1")]
|
||||
pub bucket: ::prost::alloc::string::String,
|
||||
#[prost(bytes = "bytes", tag = "2")]
|
||||
pub bucket_incarnation: ::prost::bytes::Bytes,
|
||||
#[prost(uint64, tag = "3")]
|
||||
pub generation: u64,
|
||||
}
|
||||
#[derive(Clone, PartialEq, ::prost::Message)]
|
||||
pub struct ScannerScopedDirtyUsageAckRequest {
|
||||
#[prost(bytes = "bytes", tag = "1")]
|
||||
pub challenge: ::prost::bytes::Bytes,
|
||||
#[prost(uint32, tag = "2")]
|
||||
pub protocol_version: u32,
|
||||
#[prost(string, tag = "3")]
|
||||
pub owner_id: ::prost::alloc::string::String,
|
||||
#[prost(string, tag = "4")]
|
||||
pub instance_id: ::prost::alloc::string::String,
|
||||
/// Only scope 1 (a complete bucket) is supported; zero is invalid.
|
||||
#[prost(uint32, tag = "5")]
|
||||
pub scope: u32,
|
||||
#[prost(bool, tag = "6")]
|
||||
pub probe_only: bool,
|
||||
#[prost(message, repeated, tag = "7")]
|
||||
pub entries: ::prost::alloc::vec::Vec<ScannerScopedDirtyUsageEntry>,
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct ScannerScopedDirtyUsageAckResponse {
|
||||
#[prost(uint32, tag = "1")]
|
||||
pub protocol_version: u32,
|
||||
#[prost(string, tag = "2")]
|
||||
pub owner_id: ::prost::alloc::string::String,
|
||||
#[prost(string, tag = "3")]
|
||||
pub instance_id: ::prost::alloc::string::String,
|
||||
#[prost(bool, tag = "4")]
|
||||
pub supported: bool,
|
||||
#[prost(uint32, tag = "5")]
|
||||
pub max_entries: u32,
|
||||
#[prost(uint32, tag = "6")]
|
||||
pub max_request_bytes: u32,
|
||||
#[prost(uint64, tag = "7")]
|
||||
pub cleared: u64,
|
||||
#[prost(bytes = "bytes", tag = "8")]
|
||||
pub response_proof: ::prost::bytes::Bytes,
|
||||
}
|
||||
/// A short-lived storage-owned read admission used only around a final
|
||||
/// scanner metadata publication. It is intentionally separate from the
|
||||
/// ScannerActivity observation wire so v6/v7 rolling compatibility remains
|
||||
@@ -6282,6 +6330,244 @@ pub mod node_service_server {
|
||||
}
|
||||
}
|
||||
/// Generated client implementations.
|
||||
pub mod scanner_control_service_client {
|
||||
#![allow(unused_variables, dead_code, missing_docs, clippy::wildcard_imports, clippy::let_unit_value)]
|
||||
use tonic::codegen::http::Uri;
|
||||
use tonic::codegen::*;
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ScannerControlServiceClient<T> {
|
||||
inner: tonic::client::Grpc<T>,
|
||||
}
|
||||
impl ScannerControlServiceClient<tonic::transport::Channel> {
|
||||
/// Attempt to create a new client by connecting to a given endpoint.
|
||||
pub async fn connect<D>(dst: D) -> Result<Self, tonic::transport::Error>
|
||||
where
|
||||
D: TryInto<tonic::transport::Endpoint>,
|
||||
D::Error: Into<StdError>,
|
||||
{
|
||||
let conn = tonic::transport::Endpoint::new(dst)?.connect().await?;
|
||||
Ok(Self::new(conn))
|
||||
}
|
||||
}
|
||||
impl<T> ScannerControlServiceClient<T>
|
||||
where
|
||||
T: tonic::client::GrpcService<tonic::body::Body>,
|
||||
T::Error: Into<StdError>,
|
||||
T::ResponseBody: Body<Data = Bytes> + std::marker::Send + 'static,
|
||||
<T::ResponseBody as Body>::Error: Into<StdError> + std::marker::Send,
|
||||
{
|
||||
pub fn new(inner: T) -> Self {
|
||||
let inner = tonic::client::Grpc::new(inner);
|
||||
Self { inner }
|
||||
}
|
||||
pub fn with_origin(inner: T, origin: Uri) -> Self {
|
||||
let inner = tonic::client::Grpc::with_origin(inner, origin);
|
||||
Self { inner }
|
||||
}
|
||||
pub fn with_interceptor<F>(inner: T, interceptor: F) -> ScannerControlServiceClient<InterceptedService<T, F>>
|
||||
where
|
||||
F: tonic::service::Interceptor,
|
||||
T::ResponseBody: Default,
|
||||
T: tonic::codegen::Service<
|
||||
http::Request<tonic::body::Body>,
|
||||
Response = http::Response<<T as tonic::client::GrpcService<tonic::body::Body>>::ResponseBody>,
|
||||
>,
|
||||
<T as tonic::codegen::Service<http::Request<tonic::body::Body>>>::Error:
|
||||
Into<StdError> + std::marker::Send + std::marker::Sync,
|
||||
{
|
||||
ScannerControlServiceClient::new(InterceptedService::new(inner, interceptor))
|
||||
}
|
||||
/// Compress requests with the given encoding.
|
||||
///
|
||||
/// This requires the server to support it otherwise it might respond with an
|
||||
/// error.
|
||||
#[must_use]
|
||||
pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self {
|
||||
self.inner = self.inner.send_compressed(encoding);
|
||||
self
|
||||
}
|
||||
/// Enable decompressing responses.
|
||||
#[must_use]
|
||||
pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self {
|
||||
self.inner = self.inner.accept_compressed(encoding);
|
||||
self
|
||||
}
|
||||
/// Limits the maximum size of a decoded message.
|
||||
///
|
||||
/// Default: `4MB`
|
||||
#[must_use]
|
||||
pub fn max_decoding_message_size(mut self, limit: usize) -> Self {
|
||||
self.inner = self.inner.max_decoding_message_size(limit);
|
||||
self
|
||||
}
|
||||
/// Limits the maximum size of an encoded message.
|
||||
///
|
||||
/// Default: `usize::MAX`
|
||||
#[must_use]
|
||||
pub fn max_encoding_message_size(mut self, limit: usize) -> Self {
|
||||
self.inner = self.inner.max_encoding_message_size(limit);
|
||||
self
|
||||
}
|
||||
pub async fn scanner_scoped_dirty_usage_ack(
|
||||
&mut self,
|
||||
request: impl tonic::IntoRequest<super::ScannerScopedDirtyUsageAckRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::ScannerScopedDirtyUsageAckResponse>, tonic::Status> {
|
||||
self.inner
|
||||
.ready()
|
||||
.await
|
||||
.map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?;
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let path = http::uri::PathAndQuery::from_static("/node_service.ScannerControlService/ScannerScopedDirtyUsageAck");
|
||||
let mut req = request.into_request();
|
||||
req.extensions_mut()
|
||||
.insert(GrpcMethod::new("node_service.ScannerControlService", "ScannerScopedDirtyUsageAck"));
|
||||
self.inner.unary(req, path, codec).await
|
||||
}
|
||||
}
|
||||
}
|
||||
/// Generated server implementations.
|
||||
pub mod scanner_control_service_server {
|
||||
#![allow(unused_variables, dead_code, missing_docs, clippy::wildcard_imports, clippy::let_unit_value)]
|
||||
use tonic::codegen::*;
|
||||
/// Generated trait containing gRPC methods that should be implemented for use with ScannerControlServiceServer.
|
||||
#[async_trait]
|
||||
pub trait ScannerControlService: std::marker::Send + std::marker::Sync + 'static {
|
||||
async fn scanner_scoped_dirty_usage_ack(
|
||||
&self,
|
||||
request: tonic::Request<super::ScannerScopedDirtyUsageAckRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::ScannerScopedDirtyUsageAckResponse>, tonic::Status>;
|
||||
}
|
||||
#[derive(Debug)]
|
||||
pub struct ScannerControlServiceServer<T> {
|
||||
inner: Arc<T>,
|
||||
accept_compression_encodings: EnabledCompressionEncodings,
|
||||
send_compression_encodings: EnabledCompressionEncodings,
|
||||
max_decoding_message_size: Option<usize>,
|
||||
max_encoding_message_size: Option<usize>,
|
||||
}
|
||||
impl<T> ScannerControlServiceServer<T> {
|
||||
pub fn new(inner: T) -> Self {
|
||||
Self::from_arc(Arc::new(inner))
|
||||
}
|
||||
pub fn from_arc(inner: Arc<T>) -> Self {
|
||||
Self {
|
||||
inner,
|
||||
accept_compression_encodings: Default::default(),
|
||||
send_compression_encodings: Default::default(),
|
||||
max_decoding_message_size: None,
|
||||
max_encoding_message_size: None,
|
||||
}
|
||||
}
|
||||
pub fn with_interceptor<F>(inner: T, interceptor: F) -> InterceptedService<Self, F>
|
||||
where
|
||||
F: tonic::service::Interceptor,
|
||||
{
|
||||
InterceptedService::new(Self::new(inner), interceptor)
|
||||
}
|
||||
/// Enable decompressing requests with the given encoding.
|
||||
#[must_use]
|
||||
pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self {
|
||||
self.accept_compression_encodings.enable(encoding);
|
||||
self
|
||||
}
|
||||
/// Compress responses with the given encoding, if the client supports it.
|
||||
#[must_use]
|
||||
pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self {
|
||||
self.send_compression_encodings.enable(encoding);
|
||||
self
|
||||
}
|
||||
/// Limits the maximum size of a decoded message.
|
||||
///
|
||||
/// Default: `4MB`
|
||||
#[must_use]
|
||||
pub fn max_decoding_message_size(mut self, limit: usize) -> Self {
|
||||
self.max_decoding_message_size = Some(limit);
|
||||
self
|
||||
}
|
||||
/// Limits the maximum size of an encoded message.
|
||||
///
|
||||
/// Default: `usize::MAX`
|
||||
#[must_use]
|
||||
pub fn max_encoding_message_size(mut self, limit: usize) -> Self {
|
||||
self.max_encoding_message_size = Some(limit);
|
||||
self
|
||||
}
|
||||
}
|
||||
impl<T, B> tonic::codegen::Service<http::Request<B>> for ScannerControlServiceServer<T>
|
||||
where
|
||||
T: ScannerControlService,
|
||||
B: Body + std::marker::Send + 'static,
|
||||
B::Error: Into<StdError> + std::marker::Send + 'static,
|
||||
{
|
||||
type Response = http::Response<tonic::body::Body>;
|
||||
type Error = std::convert::Infallible;
|
||||
type Future = BoxFuture<Self::Response, Self::Error>;
|
||||
fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<std::result::Result<(), Self::Error>> {
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
fn call(&mut self, req: http::Request<B>) -> Self::Future {
|
||||
match req.uri().path() {
|
||||
"/node_service.ScannerControlService/ScannerScopedDirtyUsageAck" => {
|
||||
#[allow(non_camel_case_types)]
|
||||
struct ScannerScopedDirtyUsageAckSvc<T: ScannerControlService>(pub Arc<T>);
|
||||
impl<T: ScannerControlService> tonic::server::UnaryService<super::ScannerScopedDirtyUsageAckRequest>
|
||||
for ScannerScopedDirtyUsageAckSvc<T>
|
||||
{
|
||||
type Response = super::ScannerScopedDirtyUsageAckResponse;
|
||||
type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
|
||||
fn call(&mut self, request: tonic::Request<super::ScannerScopedDirtyUsageAckRequest>) -> Self::Future {
|
||||
let inner = Arc::clone(&self.0);
|
||||
let fut = async move {
|
||||
<T as ScannerControlService>::scanner_scoped_dirty_usage_ack(&inner, request).await
|
||||
};
|
||||
Box::pin(fut)
|
||||
}
|
||||
}
|
||||
let accept_compression_encodings = self.accept_compression_encodings;
|
||||
let send_compression_encodings = self.send_compression_encodings;
|
||||
let max_decoding_message_size = self.max_decoding_message_size;
|
||||
let max_encoding_message_size = self.max_encoding_message_size;
|
||||
let inner = self.inner.clone();
|
||||
let fut = async move {
|
||||
let method = ScannerScopedDirtyUsageAckSvc(inner);
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let mut grpc = tonic::server::Grpc::new(codec)
|
||||
.apply_compression_config(accept_compression_encodings, send_compression_encodings)
|
||||
.apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size);
|
||||
let res = grpc.unary(method, req).await;
|
||||
Ok(res)
|
||||
};
|
||||
Box::pin(fut)
|
||||
}
|
||||
_ => Box::pin(async move {
|
||||
let mut response = http::Response::new(tonic::body::Body::default());
|
||||
let headers = response.headers_mut();
|
||||
headers.insert(tonic::Status::GRPC_STATUS, (tonic::Code::Unimplemented as i32).into());
|
||||
headers.insert(http::header::CONTENT_TYPE, tonic::metadata::GRPC_CONTENT_TYPE);
|
||||
Ok(response)
|
||||
}),
|
||||
}
|
||||
}
|
||||
}
|
||||
impl<T> Clone for ScannerControlServiceServer<T> {
|
||||
fn clone(&self) -> Self {
|
||||
let inner = self.inner.clone();
|
||||
Self {
|
||||
inner,
|
||||
accept_compression_encodings: self.accept_compression_encodings,
|
||||
send_compression_encodings: self.send_compression_encodings,
|
||||
max_decoding_message_size: self.max_decoding_message_size,
|
||||
max_encoding_message_size: self.max_encoding_message_size,
|
||||
}
|
||||
}
|
||||
}
|
||||
/// Generated gRPC service name
|
||||
pub const SERVICE_NAME: &str = "node_service.ScannerControlService";
|
||||
impl<T> tonic::server::NamedService for ScannerControlServiceServer<T> {
|
||||
const NAME: &'static str = SERVICE_NAME;
|
||||
}
|
||||
}
|
||||
/// Generated client implementations.
|
||||
pub mod heal_control_service_client {
|
||||
#![allow(unused_variables, dead_code, missing_docs, clippy::wildcard_imports, clippy::let_unit_value)]
|
||||
use tonic::codegen::http::Uri;
|
||||
|
||||
@@ -541,6 +541,8 @@ pub fn canonical_scanner_activity_v7_response_body(
|
||||
Ok(body)
|
||||
}
|
||||
|
||||
pub mod scoped_dirty_usage;
|
||||
|
||||
pub fn canonical_scanner_dirty_usage_snapshot_request_body(
|
||||
request: &proto_gen::node_service::ScannerDirtyUsageSnapshotRequest,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
|
||||
@@ -903,6 +903,36 @@ message ScannerDirtyUsageSnapshotResponse {
|
||||
bytes response_proof = 7;
|
||||
}
|
||||
|
||||
// Receiver-only protocol. Producers must retain whole-cycle ACK until they
|
||||
// have a durable per-bucket publication proof.
|
||||
message ScannerScopedDirtyUsageEntry {
|
||||
string bucket = 1;
|
||||
bytes bucket_incarnation = 2;
|
||||
uint64 generation = 3;
|
||||
}
|
||||
|
||||
message ScannerScopedDirtyUsageAckRequest {
|
||||
bytes challenge = 1;
|
||||
uint32 protocol_version = 2;
|
||||
string owner_id = 3;
|
||||
string instance_id = 4;
|
||||
// Only scope 1 (a complete bucket) is supported; zero is invalid.
|
||||
uint32 scope = 5;
|
||||
bool probe_only = 6;
|
||||
repeated ScannerScopedDirtyUsageEntry entries = 7;
|
||||
}
|
||||
|
||||
message ScannerScopedDirtyUsageAckResponse {
|
||||
uint32 protocol_version = 1;
|
||||
string owner_id = 2;
|
||||
string instance_id = 3;
|
||||
bool supported = 4;
|
||||
uint32 max_entries = 5;
|
||||
uint32 max_request_bytes = 6;
|
||||
uint64 cleared = 7;
|
||||
bytes response_proof = 8;
|
||||
}
|
||||
|
||||
// A short-lived storage-owned read admission used only around a final
|
||||
// scanner metadata publication. It is intentionally separate from the
|
||||
// ScannerActivity observation wire so v6/v7 rolling compatibility remains
|
||||
@@ -1245,6 +1275,10 @@ service NodeService {
|
||||
rpc GetLiveEvents(GetLiveEventsRequest) returns (GetLiveEventsResponse) {}; // auth-policy: read-only
|
||||
}
|
||||
|
||||
service ScannerControlService {
|
||||
rpc ScannerScopedDirtyUsageAck(ScannerScopedDirtyUsageAckRequest) returns (ScannerScopedDirtyUsageAckResponse) {}; // auth-policy: body-bound
|
||||
}
|
||||
|
||||
service HealControlService {
|
||||
rpc HealControl(HealControlRequest) returns (HealControlResponse) {};
|
||||
}
|
||||
|
||||
@@ -0,0 +1,213 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
//! Bounded, authenticated receiver contract for per-bucket dirty acknowledgements.
|
||||
|
||||
use crate::CanonicalBodyBuilder;
|
||||
use crate::proto_gen::node_service::{ScannerScopedDirtyUsageAckRequest, ScannerScopedDirtyUsageAckResponse};
|
||||
use prost::Message;
|
||||
|
||||
pub const SCOPED_DIRTY_USAGE_PROTOCOL_VERSION: u32 = 1;
|
||||
pub const SCOPED_DIRTY_USAGE_BUCKET_SCOPE: u32 = 1;
|
||||
pub const SCOPED_DIRTY_USAGE_MAX_ENTRIES: u32 = 32;
|
||||
pub const SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES: u32 = 8192;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum ScopedDirtyUsageRequestError {
|
||||
UnsupportedProtocol,
|
||||
UnsupportedScope,
|
||||
InvalidIdentity,
|
||||
InvalidGeneration,
|
||||
InvalidEntries,
|
||||
TooLarge,
|
||||
}
|
||||
|
||||
impl std::fmt::Display for ScopedDirtyUsageRequestError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.write_str(match self {
|
||||
Self::UnsupportedProtocol => "unsupported scoped dirty usage protocol",
|
||||
Self::UnsupportedScope => "unsupported scoped dirty usage scope",
|
||||
Self::InvalidIdentity => "invalid scoped dirty usage identity",
|
||||
Self::InvalidGeneration => "invalid scoped dirty usage generation",
|
||||
Self::InvalidEntries => "scoped dirty usage entries must be nonempty and strictly ordered",
|
||||
Self::TooLarge => "scoped dirty usage request exceeds its budget",
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for ScopedDirtyUsageRequestError {}
|
||||
|
||||
pub fn validate_scoped_dirty_usage_request(
|
||||
request: &ScannerScopedDirtyUsageAckRequest,
|
||||
) -> Result<(), ScopedDirtyUsageRequestError> {
|
||||
use ScopedDirtyUsageRequestError as E;
|
||||
if request.entries.len() > SCOPED_DIRTY_USAGE_MAX_ENTRIES as usize
|
||||
|| request.encoded_len() > SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES as usize
|
||||
{
|
||||
return Err(E::TooLarge);
|
||||
}
|
||||
if request.protocol_version != SCOPED_DIRTY_USAGE_PROTOCOL_VERSION {
|
||||
return Err(E::UnsupportedProtocol);
|
||||
}
|
||||
if request.scope != SCOPED_DIRTY_USAGE_BUCKET_SCOPE {
|
||||
return Err(E::UnsupportedScope);
|
||||
}
|
||||
if request.challenge.len() != 16 || request.owner_id.len() != 36 || request.instance_id.len() != 32 {
|
||||
return Err(E::InvalidIdentity);
|
||||
}
|
||||
if request.entries.is_empty() || request.entries.windows(2).any(|pair| pair[0].bucket >= pair[1].bucket) {
|
||||
return Err(E::InvalidEntries);
|
||||
}
|
||||
for entry in &request.entries {
|
||||
if entry.bucket.is_empty()
|
||||
|| entry.bucket.len() > 63
|
||||
|| entry.bucket_incarnation.len() != 16
|
||||
|| entry.bucket_incarnation.iter().all(|byte| *byte == 0)
|
||||
{
|
||||
return Err(E::InvalidIdentity);
|
||||
}
|
||||
if entry.generation == 0 || entry.generation == u64::MAX {
|
||||
return Err(E::InvalidGeneration);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn canonical_scoped_dirty_usage_request(
|
||||
request: &ScannerScopedDirtyUsageAckRequest,
|
||||
) -> Result<Vec<u8>, ScopedDirtyUsageRequestError> {
|
||||
validate_scoped_dirty_usage_request(request)?;
|
||||
let mut body = CanonicalBodyBuilder::new(b"rustfs-scoped-dirty-usage-ack-request-v1\0");
|
||||
let encode = |_: std::num::TryFromIntError| ScopedDirtyUsageRequestError::TooLarge;
|
||||
body.push_bytes(request.challenge.as_ref()).map_err(encode)?;
|
||||
body.push_u32(request.protocol_version);
|
||||
body.push_str(&request.owner_id).map_err(encode)?;
|
||||
body.push_str(&request.instance_id).map_err(encode)?;
|
||||
body.push_u32(request.scope);
|
||||
body.push_bool(request.probe_only);
|
||||
body.push_count(request.entries.len()).map_err(encode)?;
|
||||
for entry in &request.entries {
|
||||
body.push_str(&entry.bucket).map_err(encode)?;
|
||||
body.push_bytes(entry.bucket_incarnation.as_ref()).map_err(encode)?;
|
||||
body.push_u64(entry.generation);
|
||||
}
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
pub fn canonical_scoped_dirty_usage_response(
|
||||
request_body: &[u8],
|
||||
response: &ScannerScopedDirtyUsageAckResponse,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
let mut body = CanonicalBodyBuilder::new(b"rustfs-scoped-dirty-usage-ack-response-v1\0");
|
||||
body.push_bytes(request_body)?;
|
||||
body.push_u32(response.protocol_version);
|
||||
body.push_str(&response.owner_id)?;
|
||||
body.push_str(&response.instance_id)?;
|
||||
body.push_bool(response.supported);
|
||||
body.push_u32(response.max_entries);
|
||||
body.push_u32(response.max_request_bytes);
|
||||
body.push_u64(response.cleared);
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::proto_gen::node_service::ScannerScopedDirtyUsageEntry;
|
||||
|
||||
fn request() -> ScannerScopedDirtyUsageAckRequest {
|
||||
ScannerScopedDirtyUsageAckRequest {
|
||||
challenge: vec![1; 16].into(),
|
||||
protocol_version: 1,
|
||||
owner_id: "11111111-1111-1111-1111-111111111111".into(),
|
||||
instance_id: "a".repeat(32),
|
||||
scope: 1,
|
||||
probe_only: false,
|
||||
entries: vec![ScannerScopedDirtyUsageEntry {
|
||||
bucket: "photos".into(),
|
||||
bucket_incarnation: vec![2; 16].into(),
|
||||
generation: 8,
|
||||
}],
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scoped_dirty_usage_binds_every_request_field() {
|
||||
let base = request();
|
||||
let baseline = canonical_scoped_dirty_usage_request(&base).expect("valid request");
|
||||
for field in 0..9 {
|
||||
let mut changed = base.clone();
|
||||
match field {
|
||||
0 => changed.challenge = vec![3; 16].into(),
|
||||
1 => changed.protocol_version += 1,
|
||||
2 => changed.owner_id = "22222222-2222-2222-2222-222222222222".into(),
|
||||
3 => changed.instance_id = "b".repeat(32),
|
||||
4 => changed.scope += 1,
|
||||
5 => changed.probe_only = true,
|
||||
6 => changed.entries[0].bucket = "videos".into(),
|
||||
7 => changed.entries[0].bucket_incarnation = vec![3; 16].into(),
|
||||
_ => changed.entries[0].generation += 1,
|
||||
}
|
||||
assert!(canonical_scoped_dirty_usage_request(&changed).map_or(true, |body| body != baseline));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scoped_dirty_usage_binds_capability_and_ack_to_exact_request() {
|
||||
let request = canonical_scoped_dirty_usage_request(&request()).expect("valid request");
|
||||
let response = ScannerScopedDirtyUsageAckResponse {
|
||||
protocol_version: 1,
|
||||
owner_id: "owner".into(),
|
||||
instance_id: "process".into(),
|
||||
supported: true,
|
||||
max_entries: 32,
|
||||
max_request_bytes: 8192,
|
||||
cleared: 1,
|
||||
response_proof: vec![1; 32].into(),
|
||||
};
|
||||
let baseline = canonical_scoped_dirty_usage_response(&request, &response).expect("valid response");
|
||||
for field in 0..7 {
|
||||
let mut changed = response.clone();
|
||||
match field {
|
||||
0 => changed.protocol_version += 1,
|
||||
1 => changed.owner_id.push('x'),
|
||||
2 => changed.instance_id.push('x'),
|
||||
3 => changed.supported = false,
|
||||
4 => changed.max_entries += 1,
|
||||
5 => changed.max_request_bytes += 1,
|
||||
_ => changed.cleared += 1,
|
||||
}
|
||||
assert_ne!(
|
||||
canonical_scoped_dirty_usage_response(&request, &changed).expect("response variant"),
|
||||
baseline
|
||||
);
|
||||
}
|
||||
assert_ne!(
|
||||
canonical_scoped_dirty_usage_response(b"another request", &response).expect("request variant"),
|
||||
baseline
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scoped_dirty_usage_rejects_overflow_unknown_and_duplicate_entries() {
|
||||
let base = request();
|
||||
let mut invalid = base.clone();
|
||||
invalid.entries = vec![base.entries[0].clone(); SCOPED_DIRTY_USAGE_MAX_ENTRIES as usize + 1];
|
||||
assert_eq!(validate_scoped_dirty_usage_request(&invalid), Err(ScopedDirtyUsageRequestError::TooLarge));
|
||||
invalid = base.clone();
|
||||
invalid.entries[0].bucket = "x".repeat(SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES as usize);
|
||||
assert_eq!(validate_scoped_dirty_usage_request(&invalid), Err(ScopedDirtyUsageRequestError::TooLarge));
|
||||
invalid = base.clone();
|
||||
invalid.entries.push(base.entries[0].clone());
|
||||
assert_eq!(
|
||||
validate_scoped_dirty_usage_request(&invalid),
|
||||
Err(ScopedDirtyUsageRequestError::InvalidEntries)
|
||||
);
|
||||
invalid = base;
|
||||
invalid.entries[0].bucket_incarnation = vec![0; 16].into();
|
||||
assert_eq!(
|
||||
validate_scoped_dirty_usage_request(&invalid),
|
||||
Err(ScopedDirtyUsageRequestError::InvalidIdentity)
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -90,8 +90,9 @@ pub use scanner::{
|
||||
};
|
||||
pub use scanner_io::{
|
||||
ScannerDirtyUsageAckError, ScannerDirtyUsageBucket, ScannerDirtyUsageSnapshot, ScannerDirtyUsageState,
|
||||
acknowledge_dirty_usage_generation, clear_dirty_usage_bucket, record_dirty_usage_bucket, record_scanner_maintenance_change,
|
||||
scanner_activity_epoch, scanner_dirty_usage_snapshot, scanner_dirty_usage_state, scanner_maintenance_generation,
|
||||
acknowledge_dirty_usage_generation, acknowledge_scoped_dirty_usage, clear_dirty_usage_bucket, record_dirty_usage_bucket,
|
||||
record_scanner_maintenance_change, scanner_activity_epoch, scanner_dirty_usage_snapshot, scanner_dirty_usage_state,
|
||||
scanner_maintenance_generation,
|
||||
};
|
||||
pub use sleeper::{DynamicSleeper, SCANNER_IDLE_MODE, SCANNER_SLEEPER};
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
|
||||
@@ -1703,14 +1703,18 @@ where
|
||||
let (sender, receiver) = mpsc::channel::<DataUsageInfo>(1);
|
||||
|
||||
let done_cycle = Metrics::time(Metric::ScanCycle);
|
||||
let scan_result = crate::scanner_io::nsscanner_with_storage_status(
|
||||
let scan_result = crate::scanner_io::nsscanner_with_storage_status_scoped(
|
||||
storeapi.as_ref(),
|
||||
cycle_budget.token(),
|
||||
cycle_budget.clone(),
|
||||
sender,
|
||||
cycle_info.current,
|
||||
leader_epoch,
|
||||
scan_mode,
|
||||
crate::scanner_io::ScannerCycleRequest {
|
||||
ctx: cycle_budget.token(),
|
||||
budget: cycle_budget.clone(),
|
||||
updates: sender,
|
||||
want_cycle: cycle_info.current,
|
||||
leader_epoch,
|
||||
scan_mode,
|
||||
scan_scope: crate::scanner_io::ScannerBucketScanScope::default(),
|
||||
persisted_usage_baseline: usage_persist_baseline.data.clone(),
|
||||
},
|
||||
)
|
||||
.await;
|
||||
let publication_defer_reason = match &scan_result {
|
||||
@@ -3424,10 +3428,13 @@ use cycle_state::*;
|
||||
use leadership::*;
|
||||
use usage_store::*;
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) use activity::scanner_activity_snapshot_digest;
|
||||
pub use activity::scanner_topology_digest;
|
||||
pub(crate) use activity::{
|
||||
ScannerActivitySnapshot, ScannerDirtyUsageAcknowledgement, probe_scanner_activity, scanner_activity_allows_usage_publication,
|
||||
scanner_activity_publication_lease_targets, scanner_activity_snapshot_digest, scanner_dirty_usage_acknowledgements,
|
||||
scanner_activity_dirty_usage_state_for_host, scanner_activity_publication_lease_targets, scanner_activity_structural_digest,
|
||||
scanner_dirty_usage_acknowledgements,
|
||||
};
|
||||
pub(crate) use activity::{ScannerCycleOutcome, scanner_cycle_outcome_with_pending_maintenance};
|
||||
pub use backlog::{
|
||||
|
||||
@@ -902,6 +902,7 @@ where
|
||||
observation
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn scanner_activity_snapshot_digest(snapshot: &ScannerActivitySnapshot) -> [u8; 32] {
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(u64::try_from(snapshot.len()).unwrap_or(u64::MAX).to_be_bytes());
|
||||
@@ -925,6 +926,30 @@ pub(crate) fn scanner_activity_snapshot_digest(snapshot: &ScannerActivitySnapsho
|
||||
hasher.finalize().into()
|
||||
}
|
||||
|
||||
/// Hash the activity inputs that make an existing scanner cache unsafe to
|
||||
/// reuse. Regular namespace writes and dirty-usage generations are omitted:
|
||||
/// their affected buckets are tracked separately and may be refreshed from a
|
||||
/// complete authoritative cache baseline.
|
||||
pub(crate) fn scanner_activity_structural_digest(snapshot: &ScannerActivitySnapshot) -> [u8; 32] {
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(u64::try_from(snapshot.len()).unwrap_or(u64::MAX).to_be_bytes());
|
||||
for (host, activity) in snapshot {
|
||||
let host = host.as_bytes();
|
||||
let instance_id = activity.instance_id.as_bytes();
|
||||
hasher.update(u64::try_from(host.len()).unwrap_or(u64::MAX).to_be_bytes());
|
||||
hasher.update(host);
|
||||
hasher.update(u64::try_from(instance_id.len()).unwrap_or(u64::MAX).to_be_bytes());
|
||||
hasher.update(instance_id);
|
||||
hasher.update(activity.maintenance_generation.to_be_bytes());
|
||||
hasher.update(activity.protocol_version.to_be_bytes());
|
||||
hasher.update(activity.topology_digest);
|
||||
hasher.update([u8::from(activity.data_movement_active)]);
|
||||
hasher.update(activity.movement_generation.to_be_bytes());
|
||||
hasher.update([u8::from(activity.publication_blocked)]);
|
||||
}
|
||||
hasher.finalize().into()
|
||||
}
|
||||
|
||||
pub(crate) fn scanner_activity_allows_usage_publication(snapshot: &ScannerActivitySnapshot) -> bool {
|
||||
!snapshot.is_empty()
|
||||
&& snapshot.values().all(|activity| {
|
||||
@@ -955,6 +980,22 @@ pub(crate) fn scanner_dirty_usage_acknowledgements(snapshot: &ScannerActivitySna
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub(crate) fn scanner_activity_dirty_usage_state_for_host<'a>(
|
||||
snapshot: &'a ScannerActivitySnapshot,
|
||||
host: &str,
|
||||
) -> Option<(&'a str, u64, bool)> {
|
||||
snapshot
|
||||
.get(host)
|
||||
.filter(|_| host != LOCAL_SCANNER_ACTIVITY_NODE)
|
||||
.map(|activity| {
|
||||
(
|
||||
activity.instance_id.as_str(),
|
||||
activity.dirty_usage_generation,
|
||||
activity.dirty_usage_pending,
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
pub fn scanner_topology_digest(storeapi: &ECStore) -> [u8; 32] {
|
||||
let endpoint_pools = storeapi.endpoints();
|
||||
let mut hasher = Sha256::new();
|
||||
|
||||
@@ -379,6 +379,12 @@ pub(super) fn decode_recovery_marker_for_reset(
|
||||
if !matches!(marker_revision, DataUsageCacheRevision::Etag(_)) {
|
||||
return Err(ScannerError::Other("cycle recovery marker has no object revision".to_string()));
|
||||
}
|
||||
if let Ok(value) = serde_json::from_slice::<serde_json::Value>(data)
|
||||
&& let Some(state) = value.get("state")
|
||||
&& !matches!(state.as_str(), Some("blocked" | "cleanup-pending"))
|
||||
{
|
||||
return Err(ScannerError::Other("cycle recovery marker state is unsupported".to_string()));
|
||||
}
|
||||
let compat = serde_json::from_slice::<ScannerCycleRecoveryMarkerCompat>(data).ok();
|
||||
let _schema_version = compat.as_ref().and_then(|marker| marker.schema_version);
|
||||
let primary_revision = compat
|
||||
@@ -406,7 +412,10 @@ pub(super) fn decode_recovery_marker_for_reset(
|
||||
};
|
||||
let state = match compat.as_ref().and_then(|marker| marker.state.as_deref()) {
|
||||
Some("cleanup-pending") => "cleanup-pending",
|
||||
_ => "blocked",
|
||||
Some("blocked") | None => "blocked",
|
||||
Some(_) => {
|
||||
return Err(ScannerError::Other("cycle recovery marker state is unsupported".to_string()));
|
||||
}
|
||||
};
|
||||
let now = unix_now_secs();
|
||||
Ok(ScannerCycleRecoveryMarker {
|
||||
@@ -721,17 +730,19 @@ async fn mark_cycle_recovery_cleanup_pending(
|
||||
mut marker: ScannerCycleRecoveryMarker,
|
||||
marker_revision: &DataUsageCacheRevision,
|
||||
expected_epoch: u64,
|
||||
owns_reset: &(impl Fn() -> bool + Sync),
|
||||
) -> Result<(ScannerCycleRecoveryMarker, DataUsageCacheRevision), ScannerError> {
|
||||
marker.state = "cleanup-pending".to_string();
|
||||
marker.last_attempt_at_unix_secs = unix_now_secs();
|
||||
let bytes = serde_json::to_vec(&marker)
|
||||
.map_err(|err| ScannerError::Other(format!("failed to encode cycle recovery marker: {err}")))?;
|
||||
let info = save_config_with_publication_admission_for_epoch(
|
||||
let info = save_reset_config(
|
||||
storeapi.clone(),
|
||||
DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(),
|
||||
bytes,
|
||||
marker_revision.preconditions(),
|
||||
expected_epoch,
|
||||
owns_reset,
|
||||
)
|
||||
.await
|
||||
.map_err(|err| ScannerError::Other(format!("failed to mark cycle recovery cleanup pending: {err}")))?;
|
||||
@@ -933,6 +944,7 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
|
||||
.get_write_lock_quiet(Duration::from_secs(5))
|
||||
.await
|
||||
.map_err(|err| ScannerError::Other(format!("scanner leader lock is busy: {err}")))?;
|
||||
let owns_reset = || !guard.is_lock_lost() && !ctx.is_cancelled();
|
||||
|
||||
if guard.is_lock_lost() {
|
||||
return Err(ScannerError::Other("scanner leader lock was lost before recovery reset".to_string()));
|
||||
@@ -952,7 +964,27 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
|
||||
}
|
||||
Err(err) => return Err(ScannerError::Other(format!("failed to read cycle recovery marker: {err}"))),
|
||||
};
|
||||
let marker_data = marker_data.ok_or_else(|| ScannerError::Other("scanner cycle recovery marker is absent".to_string()))?;
|
||||
let Some(marker_data) = marker_data else {
|
||||
// A delete may commit before its reply is lost. Confirm both durable
|
||||
// fences before treating a retry without its marker as completed.
|
||||
let (cycle, epoch, revision) = read_cycle_state_for_usage_reset(storeapi.clone()).await?;
|
||||
let floor = persisted_usage_floor(storeapi.clone()).await?;
|
||||
if !matches!(revision, DataUsageCacheRevision::Etag(_))
|
||||
|| epoch < floor.leader_epoch
|
||||
|| cycle.next < floor.next_cycle
|
||||
|| !owns_reset()
|
||||
|| scanner_publication_admission_for_epoch(storeapi.clone(), reset_epoch)
|
||||
.await
|
||||
.is_none()
|
||||
{
|
||||
return Err(ScannerError::Other(
|
||||
"scanner cycle recovery marker is absent without a completed reset fence".to_string(),
|
||||
));
|
||||
}
|
||||
set_scanner_cycle_recovery_status(recovery_status("healthy", None, false));
|
||||
super::notify_scanner_cycle_recovery_wake();
|
||||
return Ok(());
|
||||
};
|
||||
let (marker, force_full_rescan) = match serde_json::from_slice::<ScannerCycleRecoveryMarker>(&marker_data) {
|
||||
Ok(marker) if validate_recovery_marker(&marker).is_ok() => (marker, false),
|
||||
_ => (decode_recovery_marker_for_reset(&marker_data, &marker_revision)?, true),
|
||||
@@ -1026,8 +1058,10 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
|
||||
}
|
||||
};
|
||||
if let Some((primary_cycle, primary_epoch)) = primary_state {
|
||||
verify_cycle_reset_intent(storeapi.clone(), &marker_revision, &owns_reset).await?;
|
||||
let (cleanup_marker, cleanup_marker_revision) =
|
||||
mark_cycle_recovery_cleanup_pending(storeapi.clone(), marker.clone(), &marker_revision, reset_epoch).await?;
|
||||
mark_cycle_recovery_cleanup_pending(storeapi.clone(), marker.clone(), &marker_revision, reset_epoch, &owns_reset)
|
||||
.await?;
|
||||
set_scanner_cycle_recovery_status(recovery_status_from_marker(&cleanup_marker, "cleanup-pending"));
|
||||
let usage_floor = persisted_usage_floor(storeapi.clone()).await?;
|
||||
let fence_epoch = primary_epoch
|
||||
@@ -1047,12 +1081,14 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
|
||||
"preserved scanner cycle state exceeds the bounded object size".to_string(),
|
||||
));
|
||||
}
|
||||
let preserved_info = save_config_with_publication_admission_for_epoch(
|
||||
verify_cycle_reset_intent(storeapi.clone(), &cleanup_marker_revision, &owns_reset).await?;
|
||||
let preserved_info = save_reset_config(
|
||||
storeapi.clone(),
|
||||
DATA_USAGE_BLOOM_NAME_PATH.as_str(),
|
||||
preserved_data,
|
||||
primary_revision.preconditions(),
|
||||
reset_epoch,
|
||||
&owns_reset,
|
||||
)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
@@ -1072,9 +1108,17 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
|
||||
"scanner leader lock was lost after fencing newer cycle state".to_string(),
|
||||
));
|
||||
}
|
||||
fence_scanner_usage_epoch_with_expected_epoch(&ctx, storeapi.clone(), fence_epoch, Some(reset_epoch), false)
|
||||
.await
|
||||
.map_err(|err| ScannerError::Other(format!("failed to fence preserved scanner usage epoch: {err}")))?;
|
||||
verify_cycle_reset_intent(storeapi.clone(), &cleanup_marker_revision, &owns_reset).await?;
|
||||
fence_scanner_usage_epoch_with_expected_epoch(
|
||||
&ctx,
|
||||
storeapi.clone(),
|
||||
fence_epoch,
|
||||
Some(reset_epoch),
|
||||
false,
|
||||
&owns_reset,
|
||||
)
|
||||
.await
|
||||
.map_err(|err| ScannerError::Other(format!("failed to fence preserved scanner usage epoch: {err}")))?;
|
||||
if guard.is_lock_lost() {
|
||||
return Err(ScannerError::Other(
|
||||
"scanner leader lock was lost after fencing newer cycle state".to_string(),
|
||||
@@ -1088,7 +1132,8 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
|
||||
"scanner cycle state changed before recovery marker cleanup".to_string(),
|
||||
));
|
||||
}
|
||||
delete_config_with_publication_admission_for_epoch(
|
||||
verify_cycle_reset_intent(storeapi.clone(), &cleanup_marker_revision, &owns_reset).await?;
|
||||
delete_reset_config(
|
||||
storeapi.clone(),
|
||||
RUSTFS_META_BUCKET,
|
||||
DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(),
|
||||
@@ -1100,6 +1145,7 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
|
||||
..Default::default()
|
||||
},
|
||||
reset_epoch,
|
||||
&owns_reset,
|
||||
)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
@@ -1149,17 +1195,20 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
|
||||
// Persist the cleanup-pending phase before rewriting the primary. If the
|
||||
// process dies after the rewrite, startup still sees a durable fence and
|
||||
// cannot mistake the partially completed reset for a healthy state.
|
||||
verify_cycle_reset_intent(storeapi.clone(), &marker_revision, &owns_reset).await?;
|
||||
let (marker, marker_revision) = if marker.state == "cleanup-pending" {
|
||||
(marker, marker_revision)
|
||||
} else {
|
||||
mark_cycle_recovery_cleanup_pending(storeapi.clone(), marker, &marker_revision, reset_epoch).await?
|
||||
mark_cycle_recovery_cleanup_pending(storeapi.clone(), marker, &marker_revision, reset_epoch, &owns_reset).await?
|
||||
};
|
||||
let rebuilt_info = save_config_with_publication_admission_for_epoch(
|
||||
verify_cycle_reset_intent(storeapi.clone(), &marker_revision, &owns_reset).await?;
|
||||
let rebuilt_info = save_reset_config(
|
||||
storeapi.clone(),
|
||||
DATA_USAGE_BLOOM_NAME_PATH.as_str(),
|
||||
data,
|
||||
primary_revision.preconditions(),
|
||||
reset_epoch,
|
||||
&owns_reset,
|
||||
)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
@@ -1178,8 +1227,10 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
|
||||
"scanner leader lock was lost after rebuilding cycle state".to_string(),
|
||||
));
|
||||
}
|
||||
verify_cycle_reset_intent(storeapi.clone(), &marker_revision, &owns_reset).await?;
|
||||
if let Err(err) =
|
||||
fence_scanner_usage_epoch_with_expected_epoch(&ctx, storeapi.clone(), leader_epoch, Some(reset_epoch), false).await
|
||||
fence_scanner_usage_epoch_with_expected_epoch(&ctx, storeapi.clone(), leader_epoch, Some(reset_epoch), false, &owns_reset)
|
||||
.await
|
||||
{
|
||||
set_scanner_cycle_recovery_status(ScannerCycleRecoveryStatus {
|
||||
path: DATA_USAGE_BLOOM_NAME_PATH.clone(),
|
||||
@@ -1249,7 +1300,8 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
|
||||
));
|
||||
}
|
||||
|
||||
if let Err(err) = delete_config_with_publication_admission_for_epoch(
|
||||
verify_cycle_reset_intent(storeapi.clone(), &marker_revision, &owns_reset).await?;
|
||||
if let Err(err) = delete_reset_config(
|
||||
storeapi.clone(),
|
||||
RUSTFS_META_BUCKET,
|
||||
DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(),
|
||||
@@ -1261,6 +1313,7 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
|
||||
..Default::default()
|
||||
},
|
||||
reset_epoch,
|
||||
&owns_reset,
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -1310,6 +1363,57 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn verify_cycle_reset_intent(
|
||||
storeapi: Arc<impl ScannerObjectIO>,
|
||||
expected_revision: &DataUsageCacheRevision,
|
||||
owns_reset: &(impl Fn() -> bool + Sync),
|
||||
) -> Result<(), ScannerError> {
|
||||
let revision = read_config_revision(storeapi, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str())
|
||||
.await
|
||||
.map_err(|err| ScannerError::Other(format!("failed to verify scanner cycle reset intent: {err}")))?;
|
||||
if &revision != expected_revision {
|
||||
return Err(ScannerError::Other("scanner cycle reset intent changed".to_string()));
|
||||
}
|
||||
if !owns_reset() {
|
||||
return Err(ScannerError::Other("scanner cycle reset ownership was lost".to_string()));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn save_reset_config(
|
||||
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
|
||||
path: &str,
|
||||
data: Vec<u8>,
|
||||
preconditions: crate::HTTPPreconditions,
|
||||
expected_epoch: u64,
|
||||
owns_reset: &(impl Fn() -> bool + Sync),
|
||||
) -> Result<crate::ScannerObjectInfo, EcstoreError> {
|
||||
let Some(_admission) = scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch).await else {
|
||||
return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED));
|
||||
};
|
||||
if !owns_reset() {
|
||||
return Err(EcstoreError::other("scanner reset ownership was lost before write"));
|
||||
}
|
||||
save_config_with_preconditions(storeapi, path, data, preconditions).await
|
||||
}
|
||||
|
||||
async fn delete_reset_config(
|
||||
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
|
||||
bucket: &str,
|
||||
path: &str,
|
||||
options: ScannerObjectOptions,
|
||||
expected_epoch: u64,
|
||||
owns_reset: &(impl Fn() -> bool + Sync),
|
||||
) -> Result<crate::ScannerObjectInfo, EcstoreError> {
|
||||
let Some(_admission) = scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch).await else {
|
||||
return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED));
|
||||
};
|
||||
if !owns_reset() {
|
||||
return Err(EcstoreError::other("scanner reset ownership was lost before delete"));
|
||||
}
|
||||
storeapi.delete_config_object(bucket, path, options).await
|
||||
}
|
||||
|
||||
fn scanner_usage_state_reset_paths() -> Vec<String> {
|
||||
vec![
|
||||
DATA_USAGE_OBJ_NAME_PATH.as_str().to_string(),
|
||||
@@ -1333,8 +1437,14 @@ pub(super) async fn read_usage_state_reset_slots(
|
||||
Ok(slots)
|
||||
}
|
||||
|
||||
fn usage_state_reset_floor(slots: &[ScannerUsageStateResetSlot]) -> Result<PersistedUsageFloor, ScannerError> {
|
||||
let mut floor = PersistedUsageFloor::default();
|
||||
enum ScannerUsageResetFloor {
|
||||
Missing,
|
||||
Trusted(PersistedUsageFloor),
|
||||
Corrupt,
|
||||
}
|
||||
|
||||
fn usage_state_reset_floor(slots: &[ScannerUsageStateResetSlot]) -> Result<ScannerUsageResetFloor, ScannerError> {
|
||||
let mut floor = None;
|
||||
for slot in slots {
|
||||
let Some(data) = slot.data.as_deref() else {
|
||||
continue;
|
||||
@@ -1342,9 +1452,21 @@ fn usage_state_reset_floor(slots: &[ScannerUsageStateResetSlot]) -> Result<Persi
|
||||
let Ok(usage) = serde_json::from_slice::<DataUsageInfo>(data) else {
|
||||
continue;
|
||||
};
|
||||
update_persisted_usage_floor(&mut floor, &usage, &slot.path)?;
|
||||
if !data_usage_info_has_persisted_baseline_identity(&usage)
|
||||
&& !(slot.path == DATA_USAGE_OBJ_NAME_PATH.as_str() && data_usage_info_is_bootstrap_pending(&usage))
|
||||
&& legacy_incomplete_usage_fence(data, &usage)
|
||||
.and_then(|fence| fence.claimable_epoch())
|
||||
.is_none()
|
||||
{
|
||||
continue;
|
||||
}
|
||||
update_persisted_usage_floor(floor.get_or_insert_with(PersistedUsageFloor::default), &usage, &slot.path)?;
|
||||
}
|
||||
Ok(floor)
|
||||
Ok(match floor {
|
||||
Some(floor) => ScannerUsageResetFloor::Trusted(floor),
|
||||
None if slots.iter().any(|slot| slot.data.is_some()) => ScannerUsageResetFloor::Corrupt,
|
||||
None => ScannerUsageResetFloor::Missing,
|
||||
})
|
||||
}
|
||||
|
||||
async fn read_cycle_state_for_usage_reset(
|
||||
@@ -1401,11 +1523,12 @@ async fn delete_usage_state_reset_slot(
|
||||
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
|
||||
slot: &ScannerUsageStateResetSlot,
|
||||
expected_epoch: u64,
|
||||
owns_reset: &(impl Fn() -> bool + Sync),
|
||||
) -> Result<bool, ScannerError> {
|
||||
if matches!(slot.revision, DataUsageCacheRevision::Missing) {
|
||||
return Ok(false);
|
||||
}
|
||||
let delete_result = delete_config_with_publication_admission_for_epoch(
|
||||
let delete_result = delete_reset_config(
|
||||
storeapi.clone(),
|
||||
RUSTFS_META_BUCKET,
|
||||
&slot.path,
|
||||
@@ -1415,6 +1538,7 @@ async fn delete_usage_state_reset_slot(
|
||||
..Default::default()
|
||||
},
|
||||
expected_epoch,
|
||||
owns_reset,
|
||||
)
|
||||
.await;
|
||||
match delete_result {
|
||||
@@ -1486,21 +1610,24 @@ pub(super) async fn publish_scanner_usage_bootstrap_primary(
|
||||
expected_publication_epoch: u64,
|
||||
leader_epoch: Option<u64>,
|
||||
context: ScannerUsageBootstrapPublishContext,
|
||||
owns_publication: impl Fn() -> bool + Sync,
|
||||
) -> Result<(), ScannerError> {
|
||||
async fn inner(
|
||||
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
|
||||
expected_revision: &DataUsageCacheRevision,
|
||||
expected_publication_epoch: u64,
|
||||
leader_epoch: Option<u64>,
|
||||
owns_publication: &(impl Fn() -> bool + Sync),
|
||||
) -> Result<(), ScannerUsageBootstrapPublishError> {
|
||||
let marker = scanner_usage_bootstrap_marker(std::time::SystemTime::now(), leader_epoch);
|
||||
let data = serde_json::to_vec(&marker).map_err(ScannerUsageBootstrapPublishError::Encode)?;
|
||||
let save_result = save_config_with_publication_admission_for_epoch(
|
||||
let save_result = save_reset_config(
|
||||
storeapi.clone(),
|
||||
DATA_USAGE_OBJ_NAME_PATH.as_str(),
|
||||
data.clone(),
|
||||
expected_revision.preconditions(),
|
||||
expected_publication_epoch,
|
||||
owns_publication,
|
||||
)
|
||||
.await;
|
||||
if save_result
|
||||
@@ -1524,7 +1651,7 @@ pub(super) async fn publish_scanner_usage_bootstrap_primary(
|
||||
})
|
||||
}
|
||||
|
||||
inner(storeapi, expected_revision, expected_publication_epoch, leader_epoch)
|
||||
inner(storeapi, expected_revision, expected_publication_epoch, leader_epoch, &owns_publication)
|
||||
.await
|
||||
.map_err(|err| err.into_scanner_error(context))
|
||||
}
|
||||
@@ -1534,32 +1661,108 @@ pub(super) async fn reset_scanner_usage_state_slots_for_full_rebuild(
|
||||
slots: &[ScannerUsageStateResetSlot],
|
||||
expected_epoch: u64,
|
||||
leader_epoch: u64,
|
||||
owns_reset: impl Fn() -> bool + Sync,
|
||||
) -> Result<Vec<String>, ScannerError> {
|
||||
let mut reset_paths = Vec::new();
|
||||
let primary = slots
|
||||
.iter()
|
||||
.find(|slot| slot.path == DATA_USAGE_OBJ_NAME_PATH.as_str())
|
||||
.ok_or_else(|| ScannerError::Other("scanner usage reset primary slot was not inspected".to_string()))?;
|
||||
publish_scanner_usage_bootstrap_primary(
|
||||
storeapi.clone(),
|
||||
&primary.revision,
|
||||
expected_epoch,
|
||||
Some(leader_epoch),
|
||||
ScannerUsageBootstrapPublishContext::Reset,
|
||||
)
|
||||
.await?;
|
||||
if !owns_reset() {
|
||||
return Err(ScannerError::Other("scanner usage reset ownership was lost".to_string()));
|
||||
}
|
||||
let resume_epoch = usage_state_reset_resume_epoch(slots)?;
|
||||
match resume_epoch {
|
||||
Some(epoch) if epoch == leader_epoch => {}
|
||||
Some(_) => return Err(ScannerError::Other("scanner usage reset bootstrap epoch changed".to_string())),
|
||||
None => {
|
||||
publish_scanner_usage_bootstrap_primary(
|
||||
storeapi.clone(),
|
||||
&primary.revision,
|
||||
expected_epoch,
|
||||
Some(leader_epoch),
|
||||
ScannerUsageBootstrapPublishContext::Reset,
|
||||
&owns_reset,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
let (data, intent_revision) = read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
|
||||
.await
|
||||
.map_err(|err| ScannerError::Other(format!("failed to inspect scanner usage reset intent: {err}")))?;
|
||||
data.as_deref()
|
||||
.and_then(|data| serde_json::from_slice::<DataUsageInfo>(data).ok())
|
||||
.filter(|usage| data_usage_info_is_bootstrap_pending(usage) && usage.scanner_epoch == Some(leader_epoch))
|
||||
.ok_or_else(|| ScannerError::Other("scanner usage reset intent changed before cleanup".to_string()))?;
|
||||
if !matches!(intent_revision, DataUsageCacheRevision::Etag(_))
|
||||
|| (resume_epoch.is_some() && intent_revision != primary.revision)
|
||||
{
|
||||
return Err(ScannerError::Other("scanner usage reset intent revision changed".to_string()));
|
||||
}
|
||||
reset_paths.push(DATA_USAGE_OBJ_NAME_PATH.as_str().to_string());
|
||||
|
||||
for slot in slots.iter().filter(|slot| slot.path != DATA_USAGE_OBJ_NAME_PATH.as_str()) {
|
||||
if delete_usage_state_reset_slot(storeapi.clone(), slot, expected_epoch).await? {
|
||||
if let Some(usage) = slot
|
||||
.data
|
||||
.as_deref()
|
||||
.and_then(|data| serde_json::from_slice::<DataUsageInfo>(data).ok())
|
||||
&& usage_epoch(&usage) >= leader_epoch
|
||||
{
|
||||
return Err(ScannerError::Other(format!(
|
||||
"scanner usage reset slot is not older than its intent: {}",
|
||||
slot.path
|
||||
)));
|
||||
}
|
||||
let revision = read_config_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
|
||||
.await
|
||||
.map_err(|err| ScannerError::Other(format!("failed to verify scanner usage reset intent: {err}")))?;
|
||||
if revision != intent_revision {
|
||||
return Err(ScannerError::Other("scanner usage reset intent changed during cleanup".to_string()));
|
||||
}
|
||||
if !owns_reset() {
|
||||
return Err(ScannerError::Other("scanner usage reset ownership was lost".to_string()));
|
||||
}
|
||||
if delete_usage_state_reset_slot(storeapi.clone(), slot, expected_epoch, &owns_reset).await? {
|
||||
reset_paths.push(slot.path.clone());
|
||||
}
|
||||
}
|
||||
let revision = read_config_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
|
||||
.await
|
||||
.map_err(|err| ScannerError::Other(format!("failed to confirm scanner usage reset intent: {err}")))?;
|
||||
if revision != intent_revision || !owns_reset() {
|
||||
return Err(ScannerError::Other(
|
||||
"scanner usage reset intent or ownership changed before completion".to_string(),
|
||||
));
|
||||
}
|
||||
invalidate_admin_data_usage_snapshot_cache().await;
|
||||
invalidate_data_usage_snapshot_cache().await;
|
||||
Ok(reset_paths)
|
||||
}
|
||||
|
||||
fn usage_state_reset_resume_epoch(slots: &[ScannerUsageStateResetSlot]) -> Result<Option<u64>, ScannerError> {
|
||||
let primary = slots.iter().find(|slot| slot.path == DATA_USAGE_OBJ_NAME_PATH.as_str());
|
||||
let usage = primary
|
||||
.and_then(|slot| slot.data.as_deref())
|
||||
.and_then(|data| serde_json::from_slice::<DataUsageInfo>(data).ok());
|
||||
match usage {
|
||||
Some(usage) if usage.usage_snapshot_bootstrap_pending => {
|
||||
if !data_usage_info_is_bootstrap_pending(&usage) {
|
||||
return Err(ScannerError::Other("scanner usage reset bootstrap is invalid".to_string()));
|
||||
}
|
||||
if usage.scanner_epoch.is_none() {
|
||||
// Initial bootstrap has no reset owner yet.
|
||||
return Ok(None);
|
||||
}
|
||||
usage
|
||||
.scanner_epoch
|
||||
.filter(|epoch| *epoch > 0 && *epoch < u64::MAX)
|
||||
.map(Some)
|
||||
.ok_or_else(|| ScannerError::Other("scanner usage reset bootstrap has no valid epoch".to_string()))
|
||||
}
|
||||
_ => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn reset_scanner_usage_state_for_full_rebuild(
|
||||
ctx: CancellationToken,
|
||||
storeapi: Arc<ECStore>,
|
||||
@@ -1584,12 +1787,31 @@ pub async fn reset_scanner_usage_state_for_full_rebuild(
|
||||
};
|
||||
let (cycle, cycle_epoch, cycle_revision) = read_cycle_state_for_usage_reset(storeapi.clone()).await?;
|
||||
let slots = read_usage_state_reset_slots(storeapi.clone()).await?;
|
||||
let usage_floor = usage_state_reset_floor(&slots)?;
|
||||
let leader_epoch = cycle_epoch
|
||||
.max(usage_floor.leader_epoch)
|
||||
.checked_add(1)
|
||||
.filter(|epoch| *epoch < u64::MAX)
|
||||
.ok_or_else(|| ScannerError::Other("scanner leader epoch is exhausted".to_string()))?;
|
||||
let usage_floor = match usage_state_reset_floor(&slots)? {
|
||||
ScannerUsageResetFloor::Trusted(floor) => floor,
|
||||
ScannerUsageResetFloor::Corrupt if matches!(cycle_revision, DataUsageCacheRevision::Missing) => {
|
||||
return Err(ScannerError::Other("scanner usage reset has no trusted cycle or usage floor".to_string()));
|
||||
}
|
||||
ScannerUsageResetFloor::Missing | ScannerUsageResetFloor::Corrupt => PersistedUsageFloor {
|
||||
next_cycle: cycle.next,
|
||||
leader_epoch: cycle_epoch,
|
||||
},
|
||||
};
|
||||
let resume_epoch = usage_state_reset_resume_epoch(&slots)?;
|
||||
let leader_epoch = if let Some(epoch) = resume_epoch {
|
||||
if epoch != cycle_epoch || usage_floor.leader_epoch > epoch || usage_floor.next_cycle > cycle.next {
|
||||
return Err(ScannerError::Other(
|
||||
"scanner usage reset bootstrap conflicts with the persisted cycle fence".to_string(),
|
||||
));
|
||||
}
|
||||
epoch
|
||||
} else {
|
||||
cycle_epoch
|
||||
.max(usage_floor.leader_epoch)
|
||||
.checked_add(1)
|
||||
.filter(|epoch| *epoch < u64::MAX)
|
||||
.ok_or_else(|| ScannerError::Other("scanner leader epoch is exhausted".to_string()))?
|
||||
};
|
||||
let rebuilt_cycle = CurrentCycle {
|
||||
next: cycle.next.max(usage_floor.next_cycle),
|
||||
..Default::default()
|
||||
@@ -1602,21 +1824,24 @@ pub async fn reset_scanner_usage_state_for_full_rebuild(
|
||||
"scanner leader lock was lost before fencing usage reset cycle state".to_string(),
|
||||
));
|
||||
}
|
||||
save_config_with_publication_admission_for_epoch(
|
||||
storeapi.clone(),
|
||||
DATA_USAGE_BLOOM_NAME_PATH.as_str(),
|
||||
cycle_data,
|
||||
cycle_revision.preconditions(),
|
||||
reset_epoch,
|
||||
)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
if scanner_publication_epoch_changed(&err) {
|
||||
ScannerError::Other("scanner usage reset deferred by a movement epoch change".to_string())
|
||||
} else {
|
||||
ScannerError::Other(format!("failed to fence scanner cycle state for usage reset: {err}"))
|
||||
}
|
||||
})?;
|
||||
if resume_epoch.is_none() {
|
||||
save_reset_config(
|
||||
storeapi.clone(),
|
||||
DATA_USAGE_BLOOM_NAME_PATH.as_str(),
|
||||
cycle_data,
|
||||
cycle_revision.preconditions(),
|
||||
reset_epoch,
|
||||
&|| !guard.is_lock_lost() && !ctx.is_cancelled(),
|
||||
)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
if scanner_publication_epoch_changed(&err) {
|
||||
ScannerError::Other("scanner usage reset deferred by a movement epoch change".to_string())
|
||||
} else {
|
||||
ScannerError::Other(format!("failed to fence scanner cycle state for usage reset: {err}"))
|
||||
}
|
||||
})?;
|
||||
}
|
||||
|
||||
if guard.is_lock_lost() {
|
||||
return Err(ScannerError::Other(
|
||||
@@ -1624,7 +1849,10 @@ pub async fn reset_scanner_usage_state_for_full_rebuild(
|
||||
));
|
||||
}
|
||||
let reset_paths =
|
||||
reset_scanner_usage_state_slots_for_full_rebuild(storeapi.clone(), &slots, reset_epoch, leader_epoch).await?;
|
||||
reset_scanner_usage_state_slots_for_full_rebuild(storeapi.clone(), &slots, reset_epoch, leader_epoch, || {
|
||||
!guard.is_lock_lost() && !ctx.is_cancelled()
|
||||
})
|
||||
.await?;
|
||||
if guard.is_lock_lost() {
|
||||
return Err(ScannerError::Other(
|
||||
"scanner leader lock was lost after publishing usage reset marker".to_string(),
|
||||
@@ -2135,6 +2363,7 @@ async fn recover_legacy_incomplete_usage_floor(
|
||||
expected_publication_epoch,
|
||||
Some(primary.epoch),
|
||||
ScannerUsageBootstrapPublishContext::Recovery,
|
||||
|| true,
|
||||
)
|
||||
.await?;
|
||||
warn!(
|
||||
|
||||
@@ -191,6 +191,7 @@ pub(super) async fn initialize_usage_baseline_bootstrap(
|
||||
expected_epoch,
|
||||
None,
|
||||
ScannerUsageBootstrapPublishContext::Initial,
|
||||
|| true,
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -201,9 +202,10 @@ pub(super) async fn fence_scanner_usage_epoch_with_expected_epoch(
|
||||
claimed_epoch: u64,
|
||||
expected_publication_epoch: Option<u64>,
|
||||
allow_bootstrap_pending: bool,
|
||||
owns_fence: impl Fn() -> bool,
|
||||
) -> Result<(), ScannerError> {
|
||||
for retry in 0..=SCANNER_PERSIST_CAS_RETRIES {
|
||||
if ctx.is_cancelled() {
|
||||
if ctx.is_cancelled() || !owns_fence() {
|
||||
return Err(ScannerError::Other("scanner leadership was cancelled before usage fencing".to_string()));
|
||||
}
|
||||
|
||||
@@ -264,6 +266,9 @@ pub(super) async fn fence_scanner_usage_epoch_with_expected_epoch(
|
||||
"scanner usage epoch fence changed while preparing its conditional write".to_string(),
|
||||
));
|
||||
};
|
||||
if ctx.is_cancelled() || !owns_fence() {
|
||||
return Err(ScannerError::Other("scanner leadership was lost before usage fencing".to_string()));
|
||||
}
|
||||
save_config_with_preconditions(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), data, revision.preconditions())
|
||||
.await
|
||||
};
|
||||
@@ -319,6 +324,7 @@ pub(super) async fn complete_scanner_leadership_claim(
|
||||
claimed_epoch,
|
||||
expected_publication_epoch,
|
||||
allow_bootstrap_pending,
|
||||
|| true,
|
||||
)
|
||||
.await
|
||||
{
|
||||
|
||||
@@ -634,6 +634,8 @@ struct MemoryConfigStore {
|
||||
cancel_after_successful_puts: Mutex<HashMap<String, (usize, CancellationToken)>>,
|
||||
replace_after_successful_puts: Mutex<HashMap<String, (usize, Vec<u8>)>>,
|
||||
error_after_commit_deletes: Mutex<HashSet<String>>,
|
||||
cancel_after_deletes: Mutex<HashMap<String, CancellationToken>>,
|
||||
pause_next_publication_admission: Mutex<Option<(Arc<tokio::sync::Notify>, Arc<tokio::sync::Notify>)>>,
|
||||
put_counts: Mutex<HashMap<String, usize>>,
|
||||
publication_admission_blocked: AtomicBool,
|
||||
block_publication_after_admissions: AtomicUsize,
|
||||
@@ -4081,6 +4083,9 @@ impl crate::ScannerConfigObjectDelete for MemoryConfigStore {
|
||||
revisions.remove(&key);
|
||||
drop(revisions);
|
||||
drop(objects);
|
||||
if let Some(token) = self.cancel_after_deletes.lock().await.remove(&key) {
|
||||
token.cancel();
|
||||
}
|
||||
if self.error_after_commit_deletes.lock().await.remove(&key) {
|
||||
return Err(EcstoreError::other("injected delete error after commit"));
|
||||
}
|
||||
@@ -4088,6 +4093,11 @@ impl crate::ScannerConfigObjectDelete for MemoryConfigStore {
|
||||
}
|
||||
|
||||
async fn scanner_data_usage_publication_admission(&self) -> Option<crate::ScannerDataUsagePublicationAdmission> {
|
||||
let pause = self.pause_next_publication_admission.lock().await.take();
|
||||
if let Some((entered, resume)) = pause {
|
||||
entered.notify_one();
|
||||
resume.notified().await;
|
||||
}
|
||||
if self.publication_admission_blocked.load(Ordering::Acquire) {
|
||||
return None;
|
||||
}
|
||||
@@ -4589,7 +4599,7 @@ async fn scanner_legacy_usage_backup_survives_fencing_and_restart_after_real_met
|
||||
.expect("publication must also read the intact backup");
|
||||
assert_eq!(baseline.data.as_deref(), Some(data.as_slice()));
|
||||
assert_eq!(baseline.revision, DataUsageCacheRevision::Missing);
|
||||
fence_scanner_usage_epoch_with_expected_epoch(&CancellationToken::new(), store.clone(), 7, None, false)
|
||||
fence_scanner_usage_epoch_with_expected_epoch(&CancellationToken::new(), store.clone(), 7, None, false, || true)
|
||||
.await
|
||||
.expect("legacy backup must be fenced into v2");
|
||||
let fenced = read_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
|
||||
@@ -4818,6 +4828,28 @@ async fn scanner_usage_state_reset_publishes_fenced_bootstrap_marker() {
|
||||
);
|
||||
}
|
||||
|
||||
let cycle_before_retry = read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
|
||||
.await
|
||||
.expect("cycle should remain before retry");
|
||||
let marker_before_retry = read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
|
||||
.await
|
||||
.expect("bootstrap should remain before retry");
|
||||
let retry = reset_scanner_usage_state_for_full_rebuild(CancellationToken::new(), store.clone())
|
||||
.await
|
||||
.expect("completed cleanup should be reentrant");
|
||||
assert_eq!(retry.leader_epoch, result.leader_epoch);
|
||||
assert_eq!(
|
||||
read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
|
||||
.await
|
||||
.expect("cycle should remain"),
|
||||
cycle_before_retry
|
||||
);
|
||||
assert_eq!(
|
||||
read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
|
||||
.await
|
||||
.expect("bootstrap should remain"),
|
||||
marker_before_retry
|
||||
);
|
||||
let (floor, state) = persisted_usage_floor_for_startup(store, false)
|
||||
.await
|
||||
.expect("reset marker should be resumable");
|
||||
@@ -4973,7 +5005,7 @@ async fn scanner_usage_state_reset_slots_reject_primary_aba() {
|
||||
|
||||
store.objects.lock().await.insert(key.clone(), b"newer-json".to_vec());
|
||||
store.revisions.lock().await.insert(key, 2);
|
||||
let err = reset_scanner_usage_state_slots_for_full_rebuild(store, &slots, 0, 3)
|
||||
let err = reset_scanner_usage_state_slots_for_full_rebuild(store, &slots, 0, 3, || true)
|
||||
.await
|
||||
.expect_err("stale primary revision must not be overwritten");
|
||||
assert!(
|
||||
@@ -4983,6 +5015,348 @@ async fn scanner_usage_state_reset_slots_reject_primary_aba() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_usage_state_reset_resumes_every_cleanup_boundary_without_rewriting_intent() {
|
||||
for completed in 0..=4 {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
let primary_path = DATA_USAGE_OBJ_NAME_PATH.as_str();
|
||||
let cleanup_paths = [
|
||||
format!("{primary_path}.bkp"),
|
||||
LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str().to_string(),
|
||||
format!("{}.bkp", LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str()),
|
||||
DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str().to_string(),
|
||||
];
|
||||
for path in std::iter::once(primary_path).chain(cleanup_paths.iter().map(String::as_str)) {
|
||||
let mut usage = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0);
|
||||
usage.scanner_epoch = Some(1);
|
||||
save_config(store.clone(), path, serde_json::to_vec(&usage).expect("fixture should encode"))
|
||||
.await
|
||||
.expect("fixture should persist");
|
||||
}
|
||||
// These objects belong to other owners, even when reset cleanup resumes.
|
||||
for path in ["buckets/quota-reservations/ledger", "buckets/example/incarnation"] {
|
||||
save_config(store.clone(), path, b"retain".to_vec())
|
||||
.await
|
||||
.expect("unrelated state should persist");
|
||||
}
|
||||
let slots = read_usage_state_reset_slots(store.clone()).await.expect("slots should load");
|
||||
let cancelled = CancellationToken::new();
|
||||
if completed == 0 {
|
||||
store
|
||||
.cancel_after_successful_puts
|
||||
.lock()
|
||||
.await
|
||||
.insert(memory_config_key(RUSTFS_META_BUCKET, primary_path), (2, cancelled.clone()));
|
||||
} else {
|
||||
store
|
||||
.cancel_after_deletes
|
||||
.lock()
|
||||
.await
|
||||
.insert(memory_config_key(RUSTFS_META_BUCKET, &cleanup_paths[completed - 1]), cancelled.clone());
|
||||
}
|
||||
let err = reset_scanner_usage_state_slots_for_full_rebuild(store.clone(), &slots, 0, 3, || !cancelled.is_cancelled())
|
||||
.await
|
||||
.expect_err("interruption should stop cleanup");
|
||||
assert!(err.to_string().contains("ownership"), "boundary {completed}: {err}");
|
||||
for (index, path) in cleanup_paths.iter().enumerate() {
|
||||
assert_eq!(
|
||||
store
|
||||
.objects
|
||||
.lock()
|
||||
.await
|
||||
.contains_key(&memory_config_key(RUSTFS_META_BUCKET, path)),
|
||||
index >= completed,
|
||||
"boundary {completed}, slot {index}"
|
||||
);
|
||||
}
|
||||
let intent = read_config_with_revision(store.clone(), primary_path)
|
||||
.await
|
||||
.expect("intent should persist");
|
||||
let slots = read_usage_state_reset_slots(store.clone())
|
||||
.await
|
||||
.expect("restart should reload slots");
|
||||
reset_scanner_usage_state_slots_for_full_rebuild(store.clone(), &slots, 0, 3, || true)
|
||||
.await
|
||||
.expect("restart should complete the same intent");
|
||||
assert_eq!(
|
||||
read_config_with_revision(store.clone(), primary_path)
|
||||
.await
|
||||
.expect("intent should remain"),
|
||||
intent
|
||||
);
|
||||
assert_eq!(store.put_counts.lock().await[&memory_config_key(RUSTFS_META_BUCKET, primary_path)], 2);
|
||||
for path in cleanup_paths {
|
||||
assert!(
|
||||
!store
|
||||
.objects
|
||||
.lock()
|
||||
.await
|
||||
.contains_key(&memory_config_key(RUSTFS_META_BUCKET, &path))
|
||||
);
|
||||
}
|
||||
for path in ["buckets/quota-reservations/ledger", "buckets/example/incarnation"] {
|
||||
assert_eq!(read_config(store.clone(), path).await.expect("unrelated state should remain"), b"retain");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_usage_state_reset_stops_usage_fence_after_owner_loss() {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
let mut usage = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0);
|
||||
usage.scanner_epoch = Some(1);
|
||||
let bytes = serde_json::to_vec(&usage).expect("baseline should encode");
|
||||
save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), bytes.clone())
|
||||
.await
|
||||
.expect("baseline should persist");
|
||||
let checks = AtomicUsize::new(0);
|
||||
let err = fence_scanner_usage_epoch_with_expected_epoch(&CancellationToken::new(), store.clone(), 3, Some(0), false, || {
|
||||
checks.fetch_add(1, Ordering::SeqCst) == 0
|
||||
})
|
||||
.await
|
||||
.expect_err("ownership lost during reads must prevent the write");
|
||||
assert!(err.to_string().contains("leadership was lost"), "{err}");
|
||||
assert_eq!(
|
||||
read_config(store, DATA_USAGE_OBJ_NAME_PATH.as_str())
|
||||
.await
|
||||
.expect("baseline should remain"),
|
||||
bytes
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_usage_state_reset_cancels_during_publication_admission() {
|
||||
for resuming in [false, true] {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
let usage = if resuming {
|
||||
scanner_usage_bootstrap_marker(std::time::SystemTime::UNIX_EPOCH, Some(3))
|
||||
} else {
|
||||
complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0)
|
||||
};
|
||||
save_config(
|
||||
store.clone(),
|
||||
DATA_USAGE_OBJ_NAME_PATH.as_str(),
|
||||
serde_json::to_vec(&usage).expect("primary should encode"),
|
||||
)
|
||||
.await
|
||||
.expect("primary should persist");
|
||||
save_config(store.clone(), LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str(), b"corrupt".to_vec())
|
||||
.await
|
||||
.expect("cleanup target should persist");
|
||||
let slots = read_usage_state_reset_slots(store.clone()).await.expect("slots should load");
|
||||
let before = store.objects.lock().await.clone();
|
||||
let revisions_before = store.revisions.lock().await.clone();
|
||||
let entered = Arc::new(tokio::sync::Notify::new());
|
||||
let resume = Arc::new(tokio::sync::Notify::new());
|
||||
*store.pause_next_publication_admission.lock().await = Some((entered.clone(), resume.clone()));
|
||||
let cancelled = CancellationToken::new();
|
||||
let (result, ()) = tokio::join!(
|
||||
reset_scanner_usage_state_slots_for_full_rebuild(store.clone(), &slots, 0, 3, || !cancelled.is_cancelled()),
|
||||
async {
|
||||
entered.notified().await;
|
||||
cancelled.cancel();
|
||||
resume.notify_one();
|
||||
}
|
||||
);
|
||||
let err = result.expect_err("losing ownership during admission must prevent mutation");
|
||||
assert!(err.to_string().contains("ownership was lost"), "resuming={resuming}: {err}");
|
||||
assert_eq!(*store.objects.lock().await, before);
|
||||
assert_eq!(*store.revisions.lock().await, revisions_before);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn scanner_usage_state_reset_rejects_corruption_without_a_trusted_floor() {
|
||||
let (_temp_dir, store) = setup_scanner_cycle_store_with_usage_baseline(false).await;
|
||||
save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), b"{corrupt".to_vec())
|
||||
.await
|
||||
.expect("corrupt primary should persist");
|
||||
let before = read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
|
||||
.await
|
||||
.expect("evidence should load");
|
||||
let err = reset_scanner_usage_state_for_full_rebuild(CancellationToken::new(), store.clone())
|
||||
.await
|
||||
.expect_err("corruption must not become a zero floor");
|
||||
assert!(err.to_string().contains("no trusted cycle or usage floor"), "{err}");
|
||||
assert_eq!(
|
||||
read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
|
||||
.await
|
||||
.expect("evidence should remain"),
|
||||
before
|
||||
);
|
||||
assert!(matches!(
|
||||
read_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str()).await,
|
||||
Err(EcstoreError::ConfigNotFound)
|
||||
));
|
||||
let mut backup = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0);
|
||||
backup.scanner_epoch = Some(7);
|
||||
backup.scanner_cycle = Some(40);
|
||||
save_config(
|
||||
store.clone(),
|
||||
&format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str()),
|
||||
serde_json::to_vec(&backup).expect("backup should encode"),
|
||||
)
|
||||
.await
|
||||
.expect("valid backup should persist");
|
||||
let result = reset_scanner_usage_state_for_full_rebuild(CancellationToken::new(), store)
|
||||
.await
|
||||
.expect("valid backup should supply the recovery floor");
|
||||
assert_eq!(result.leader_epoch, 8);
|
||||
assert_eq!(result.next_cycle, 41);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_usage_state_reset_rejects_replaced_intent_and_newer_cleanup_slot() {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
let marker = scanner_usage_bootstrap_marker(std::time::SystemTime::UNIX_EPOCH, Some(3));
|
||||
let bytes = serde_json::to_vec(&marker).expect("marker should encode");
|
||||
save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), bytes.clone())
|
||||
.await
|
||||
.expect("intent should persist");
|
||||
let slots = read_usage_state_reset_slots(store.clone()).await.expect("slots should load");
|
||||
save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), bytes)
|
||||
.await
|
||||
.expect("another intent should persist");
|
||||
let err = reset_scanner_usage_state_slots_for_full_rebuild(store.clone(), &slots, 0, 3, || true)
|
||||
.await
|
||||
.expect_err("same epoch cannot replace an intent revision");
|
||||
assert!(err.to_string().contains("intent revision changed"), "{err}");
|
||||
|
||||
let mut newer = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0);
|
||||
newer.scanner_epoch = Some(3);
|
||||
let path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str());
|
||||
let bytes = serde_json::to_vec(&newer).expect("newer snapshot should encode");
|
||||
save_config(store.clone(), &path, bytes.clone())
|
||||
.await
|
||||
.expect("newer snapshot should persist");
|
||||
let slots = read_usage_state_reset_slots(store.clone())
|
||||
.await
|
||||
.expect("slots should reload");
|
||||
let err = reset_scanner_usage_state_slots_for_full_rebuild(store.clone(), &slots, 0, 3, || true)
|
||||
.await
|
||||
.expect_err("cleanup cannot delete same-epoch progress");
|
||||
assert!(err.to_string().contains("not older than its intent"), "{err}");
|
||||
assert_eq!(read_config(store, &path).await.expect("newer snapshot should remain"), bytes);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn scanner_usage_state_reset_rejects_decodable_untrusted_floor() {
|
||||
let (_temp_dir, store) = setup_scanner_cycle_store_with_usage_baseline(false).await;
|
||||
let invalid_identity = DataUsageInfo {
|
||||
usage_snapshot_complete: true,
|
||||
buckets_count: 1,
|
||||
last_update: Some(std::time::SystemTime::UNIX_EPOCH),
|
||||
..Default::default()
|
||||
};
|
||||
for usage in [DataUsageInfo::default(), invalid_identity] {
|
||||
save_config(
|
||||
store.clone(),
|
||||
DATA_USAGE_OBJ_NAME_PATH.as_str(),
|
||||
serde_json::to_vec(&usage).expect("fixture should encode"),
|
||||
)
|
||||
.await
|
||||
.expect("untrusted primary should persist");
|
||||
let before = read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
|
||||
.await
|
||||
.expect("primary should load");
|
||||
let err = reset_scanner_usage_state_for_full_rebuild(CancellationToken::new(), store.clone())
|
||||
.await
|
||||
.expect_err("valid JSON alone cannot prove a usage floor");
|
||||
assert!(err.to_string().contains("no trusted cycle or usage floor"), "{err}");
|
||||
assert_eq!(
|
||||
read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
|
||||
.await
|
||||
.expect("evidence should remain"),
|
||||
before
|
||||
);
|
||||
assert!(matches!(
|
||||
read_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str()).await,
|
||||
Err(EcstoreError::ConfigNotFound)
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn full_rescan_reset_rejects_unknown_marker_phase_even_with_invalid_compat_fields() {
|
||||
for state in [serde_json::json!("rewrite-v2"), serde_json::json!(7), serde_json::Value::Null] {
|
||||
let marker = serde_json::json!({"state": state, "retry_count": "future-type", "schema_version": 99});
|
||||
let err = super::cycle_state::decode_recovery_marker_for_reset(
|
||||
&serde_json::to_vec(&marker).expect("future marker should encode"),
|
||||
&DataUsageCacheRevision::Etag("intent-1".to_string()),
|
||||
)
|
||||
.expect_err("unknown persistent phases must remain fenced");
|
||||
assert!(err.to_string().contains("state is unsupported"), "{err}");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn full_rescan_reset_preserves_unknown_phase_and_retries_completed_cleanup() {
|
||||
let (_temp_dir, store) = setup_scanner_cycle_store().await;
|
||||
save_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str(), b"corrupt".to_vec())
|
||||
.await
|
||||
.expect("corrupt primary should persist");
|
||||
save_config(
|
||||
store.clone(),
|
||||
DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(),
|
||||
br#"{"state":"future-rewrite"}"#.to_vec(),
|
||||
)
|
||||
.await
|
||||
.expect("future marker should persist");
|
||||
let primary_before = read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
|
||||
.await
|
||||
.expect("primary should load");
|
||||
let marker_before = read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str())
|
||||
.await
|
||||
.expect("marker should load");
|
||||
let err = reset_scanner_cycle_recovery(CancellationToken::new(), store.clone())
|
||||
.await
|
||||
.expect_err("unknown phase must block explicit reset");
|
||||
assert!(err.to_string().contains("state is unsupported"), "{err}");
|
||||
assert_eq!(
|
||||
read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
|
||||
.await
|
||||
.expect("primary should remain"),
|
||||
primary_before
|
||||
);
|
||||
assert_eq!(
|
||||
read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str())
|
||||
.await
|
||||
.expect("marker should remain"),
|
||||
marker_before
|
||||
);
|
||||
|
||||
save_config(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), b"{malformed".to_vec())
|
||||
.await
|
||||
.expect("recoverable marker should persist");
|
||||
reset_scanner_cycle_recovery(CancellationToken::new(), store.clone())
|
||||
.await
|
||||
.expect("reset should complete");
|
||||
let primary = read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
|
||||
.await
|
||||
.expect("rebuilt primary should load");
|
||||
let usage = read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
|
||||
.await
|
||||
.expect("fenced usage should load");
|
||||
reset_scanner_cycle_recovery(CancellationToken::new(), store.clone())
|
||||
.await
|
||||
.expect("retry after marker deletion should complete");
|
||||
assert_eq!(
|
||||
read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
|
||||
.await
|
||||
.expect("rebuilt primary should remain"),
|
||||
primary
|
||||
);
|
||||
assert_eq!(
|
||||
read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
|
||||
.await
|
||||
.expect("fenced usage should remain"),
|
||||
usage
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_usage_state_reset_slots_defer_when_publication_epoch_moves() {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
@@ -4993,7 +5367,7 @@ async fn scanner_usage_state_reset_slots_defer_when_publication_epoch_moves() {
|
||||
.expect("usage reset slots should be inspected");
|
||||
store.publication_admission_blocked.store(true, Ordering::Release);
|
||||
|
||||
let err = reset_scanner_usage_state_slots_for_full_rebuild(store, &slots, 0, 3)
|
||||
let err = reset_scanner_usage_state_slots_for_full_rebuild(store, &slots, 0, 3, || true)
|
||||
.await
|
||||
.expect_err("movement admission loss must defer reset");
|
||||
assert!(
|
||||
@@ -8169,6 +8543,44 @@ fn scanner_activity_snapshot_digest_fences_dirty_usage_state() {
|
||||
assert_ne!(scanner_activity_snapshot_digest(&clean), scanner_activity_snapshot_digest(&pending));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_activity_structural_digest_ignores_regular_bucket_writes() {
|
||||
let baseline = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))]);
|
||||
let mut written = baseline.clone();
|
||||
let activity = written.get_mut("node-2").expect("node should exist");
|
||||
activity.namespace_generation = 8;
|
||||
activity.dirty_usage_generation = 6;
|
||||
activity.dirty_usage_pending = true;
|
||||
|
||||
assert_ne!(scanner_activity_snapshot_digest(&baseline), scanner_activity_snapshot_digest(&written));
|
||||
assert_eq!(
|
||||
scanner_activity_structural_digest(&baseline),
|
||||
scanner_activity_structural_digest(&written),
|
||||
"bucket writes are refreshed through the dirty-bucket scope rather than invalidating every cache"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_activity_structural_digest_fences_restart_and_maintenance() {
|
||||
let baseline = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))]);
|
||||
let mut restarted = baseline.clone();
|
||||
restarted.get_mut("node-2").expect("node should exist").instance_id = "epoch-b".to_string();
|
||||
let mut maintained = baseline.clone();
|
||||
maintained
|
||||
.get_mut("node-2")
|
||||
.expect("node should exist")
|
||||
.maintenance_generation = 4;
|
||||
|
||||
assert_ne!(
|
||||
scanner_activity_structural_digest(&baseline),
|
||||
scanner_activity_structural_digest(&restarted)
|
||||
);
|
||||
assert_ne!(
|
||||
scanner_activity_structural_digest(&baseline),
|
||||
scanner_activity_structural_digest(&maintained)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_dirty_usage_acknowledgements_exclude_local_and_clean_nodes() {
|
||||
let snapshot = BTreeMap::from([
|
||||
|
||||
@@ -21,6 +21,7 @@ use crate::{
|
||||
DataUsageCacheSource, DataUsageEntry, DataUsageEntryInfo, DataUsageInfo, DataUsageScanPlanDigest, DataUsageSnapshotSetState,
|
||||
ScannerError, SizeSummary, TierStats,
|
||||
};
|
||||
use bytes::Bytes;
|
||||
use futures::future::join_all;
|
||||
use metrics::counter;
|
||||
use rand::seq::SliceRandom as _;
|
||||
@@ -54,6 +55,7 @@ use tokio_util::task::AbortOnDropHandle;
|
||||
use tracing::{debug, error, warn};
|
||||
|
||||
use crate::ScannerObjectInfo as ObjectInfo;
|
||||
use crate::storage_api::EcstoreScannerPeerDirtyUsageSnapshot;
|
||||
use crate::storage_api::ScannerStorage;
|
||||
use crate::storage_api::scan::NamespaceLocking as _;
|
||||
use crate::storage_api::scanner_io::{BucketInfo, BucketOptions};
|
||||
@@ -111,6 +113,121 @@ pub(crate) struct ScannerBucketScanScope {
|
||||
baseline_scan_plan_digest: Option<DataUsageScanPlanDigest>,
|
||||
}
|
||||
|
||||
impl ScannerBucketScanScope {
|
||||
fn is_default(&self) -> bool {
|
||||
self.selected_buckets.is_none() && self.baseline_scan_plan_digest.is_none()
|
||||
}
|
||||
|
||||
fn from_dirty_buckets(selected_buckets: HashSet<String>, baseline_scan_plan_digest: DataUsageScanPlanDigest) -> Self {
|
||||
Self {
|
||||
selected_buckets: Some(Arc::new(selected_buckets)),
|
||||
baseline_scan_plan_digest: Some(baseline_scan_plan_digest),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
pub(super) struct ScannerCacheBaselineProof<'a> {
|
||||
pub(super) data: Option<&'a Bytes>,
|
||||
pub(super) expected_sources: &'a HashSet<DataUsageCacheSource>,
|
||||
pub(super) leader_epoch: u64,
|
||||
pub(super) want_cycle: u64,
|
||||
pub(super) scan_plan_digest: DataUsageScanPlanDigest,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
struct ScannerPeerDirtyUsageExpectation {
|
||||
instance_id: String,
|
||||
generation: u64,
|
||||
pending: bool,
|
||||
}
|
||||
|
||||
fn verified_remote_dirty_usage_buckets(
|
||||
expected_peers: &HashMap<String, ScannerPeerDirtyUsageExpectation>,
|
||||
peer_snapshots: Vec<(String, EcstoreScannerPeerDirtyUsageSnapshot)>,
|
||||
) -> Option<HashSet<String>> {
|
||||
if expected_peers.is_empty() || peer_snapshots.len() != expected_peers.len() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut received_peers = HashSet::with_capacity(peer_snapshots.len());
|
||||
let mut dirty_buckets = HashSet::new();
|
||||
for (host, snapshot) in peer_snapshots {
|
||||
let expected = expected_peers.get(&host)?;
|
||||
if !received_peers.insert(host)
|
||||
|| snapshot.instance_id != expected.instance_id
|
||||
|| snapshot.generation != expected.generation
|
||||
|| snapshot.generation == u64::MAX
|
||||
|| snapshot.protocol_version != crate::SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION
|
||||
|| !snapshot.complete
|
||||
|| snapshot.pending_bucket_count != u64::try_from(snapshot.buckets.len()).unwrap_or(u64::MAX)
|
||||
|| (expected.pending && snapshot.pending_bucket_count == 0)
|
||||
{
|
||||
return None;
|
||||
}
|
||||
dirty_buckets.extend(snapshot.buckets.into_keys());
|
||||
}
|
||||
|
||||
(received_peers.len() == expected_peers.len()).then_some(dirty_buckets)
|
||||
}
|
||||
|
||||
fn complete_scanner_cache_baseline_plan_digest(proof: ScannerCacheBaselineProof<'_>) -> Option<DataUsageScanPlanDigest> {
|
||||
let data = proof.data?;
|
||||
let baseline = serde_json::from_slice::<DataUsageInfo>(data).ok()?;
|
||||
if !baseline.is_complete_bucket_usage_snapshot()
|
||||
|| baseline.usage_snapshot_partial
|
||||
|| baseline.usage_snapshot_converged != Some(true)
|
||||
|| baseline.scanner_epoch != Some(proof.leader_epoch)
|
||||
|| baseline.usage_snapshot_set_states.len() != proof.expected_sources.len()
|
||||
{
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut states = HashSet::with_capacity(baseline.usage_snapshot_set_states.len());
|
||||
for state in &baseline.usage_snapshot_set_states {
|
||||
let source = DataUsageCacheSource::new(usize::try_from(state.pool_index).ok()?, usize::try_from(state.set_index).ok()?);
|
||||
if !proof.expected_sources.contains(&source)
|
||||
|| !states.insert(source)
|
||||
|| !state.complete
|
||||
|| state.tombstone
|
||||
|| state.scanner_epoch != Some(proof.leader_epoch)
|
||||
|| state.scanner_cycle.is_none_or(|cycle| cycle > proof.want_cycle)
|
||||
|| state.scan_plan_digest != Some(proof.scan_plan_digest.0)
|
||||
{
|
||||
return None;
|
||||
}
|
||||
}
|
||||
|
||||
(states == *proof.expected_sources).then_some(proof.scan_plan_digest)
|
||||
}
|
||||
|
||||
fn scoped_scan_scope_from_dirty_buckets(
|
||||
requested_scope: ScannerBucketScanScope,
|
||||
dirty_buckets: HashSet<String>,
|
||||
dirty_snapshot_complete: bool,
|
||||
all_buckets: &[BucketInfo],
|
||||
baseline_proof: ScannerCacheBaselineProof<'_>,
|
||||
) -> ScannerBucketScanScope {
|
||||
if !requested_scope.is_default() || !dirty_snapshot_complete {
|
||||
return requested_scope;
|
||||
}
|
||||
|
||||
let current_buckets = all_buckets.iter().map(|bucket| bucket.name.as_str()).collect::<HashSet<_>>();
|
||||
let selected_buckets = dirty_buckets
|
||||
.into_iter()
|
||||
.filter(|bucket| current_buckets.contains(bucket.as_str()))
|
||||
.collect::<HashSet<_>>();
|
||||
if selected_buckets.is_empty() {
|
||||
return requested_scope;
|
||||
}
|
||||
|
||||
let Some(baseline_scan_plan_digest) = complete_scanner_cache_baseline_plan_digest(baseline_proof) else {
|
||||
return requested_scope;
|
||||
};
|
||||
|
||||
ScannerBucketScanScope::from_dirty_buckets(selected_buckets, baseline_scan_plan_digest)
|
||||
}
|
||||
|
||||
pub(crate) fn is_scanner_metadata_corrupt_error(err: &StorageError) -> bool {
|
||||
matches!(err, StorageError::Io(io) if io.to_string().starts_with(SCANNER_METADATA_CORRUPT_ERROR))
|
||||
}
|
||||
@@ -749,7 +866,7 @@ mod io_cache;
|
||||
mod io_cycle;
|
||||
#[cfg(test)]
|
||||
use io_cache::{ScannerSetCacheGeneration, prepare_scoped_set_scan};
|
||||
pub(crate) use io_cycle::nsscanner_with_storage_status;
|
||||
pub(crate) use io_cycle::{ScannerCycleRequest, nsscanner_with_storage_status_scoped};
|
||||
mod io_disk;
|
||||
#[cfg(test)]
|
||||
mod publish_gate_tests;
|
||||
@@ -766,8 +883,9 @@ pub(crate) use cache::{
|
||||
};
|
||||
pub use dirty_usage::{
|
||||
ScannerDirtyUsageAckError, ScannerDirtyUsageBucket, ScannerDirtyUsageSnapshot, ScannerDirtyUsageState,
|
||||
acknowledge_dirty_usage_generation, clear_dirty_usage_bucket, record_dirty_usage_bucket, record_scanner_maintenance_change,
|
||||
scanner_activity_epoch, scanner_dirty_usage_snapshot, scanner_dirty_usage_state, scanner_maintenance_generation,
|
||||
acknowledge_dirty_usage_generation, acknowledge_scoped_dirty_usage, clear_dirty_usage_bucket, record_dirty_usage_bucket,
|
||||
record_scanner_maintenance_change, scanner_activity_epoch, scanner_dirty_usage_snapshot, scanner_dirty_usage_state,
|
||||
scanner_maintenance_generation,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use dirty_usage::{clear_dirty_usage_buckets_for_tests, dirty_usage_buckets_for_tests};
|
||||
|
||||
@@ -282,9 +282,26 @@ pub(super) fn completed_data_usage_info(
|
||||
.iter()
|
||||
.map(|(bucket, usage)| (bucket.clone(), usage.size))
|
||||
.collect();
|
||||
let mut usage_snapshot_set_states = results
|
||||
.iter()
|
||||
.map(|result| {
|
||||
let source = result.info.source?;
|
||||
Some(DataUsageSnapshotSetState {
|
||||
pool_index: u64::try_from(source.pool_index).ok()?,
|
||||
set_index: u64::try_from(source.set_index).ok()?,
|
||||
scanner_cycle: Some(result.info.next_cycle),
|
||||
scanner_epoch: Some(result.info.leader_epoch),
|
||||
scan_plan_digest: Some(result.info.scan_plan_digest?.0),
|
||||
complete: true,
|
||||
tombstone: false,
|
||||
})
|
||||
})
|
||||
.collect::<Option<Vec<_>>>()?;
|
||||
usage_snapshot_set_states.sort_by_key(|state| (state.pool_index, state.set_index));
|
||||
let data_usage_info = DataUsageInfo {
|
||||
last_update: Some(merged_last_update),
|
||||
scanner_cycle: Some(results.first()?.info.next_cycle),
|
||||
scanner_epoch: Some(results.first()?.info.leader_epoch),
|
||||
objects_total_count: u64::try_from(total.objects).ok()?,
|
||||
versions_total_count: u64::try_from(total.versions).ok()?,
|
||||
delete_markers_total_count: u64::try_from(total.delete_markers).ok()?,
|
||||
@@ -295,6 +312,7 @@ pub(super) fn completed_data_usage_info(
|
||||
bucket_sizes,
|
||||
buckets_usage,
|
||||
usage_snapshot_complete: true,
|
||||
usage_snapshot_set_states,
|
||||
..Default::default()
|
||||
};
|
||||
Some((data_usage_info, merged_last_update))
|
||||
|
||||
@@ -52,6 +52,112 @@ pub enum ScannerDirtyUsageAckError {
|
||||
ProcessChanged,
|
||||
#[error("scanner dirty usage generation cannot be acknowledged")]
|
||||
InvalidGeneration,
|
||||
#[error("scanner dirty usage bucket incarnation fence is unavailable")]
|
||||
IncarnationUnavailable,
|
||||
}
|
||||
|
||||
/// A scoped ACK requires storage-owned lifecycle and incarnation fences.
|
||||
/// Callers must only send ACKs backed by durable per-bucket publication.
|
||||
pub fn acknowledge_scoped_dirty_usage(
|
||||
instance_id: &str,
|
||||
entries: &[(&crate::storage_api::EcstoreBucketMetadataMutationGuard, u64)],
|
||||
probe_only: bool,
|
||||
) -> std::result::Result<u64, ScannerDirtyUsageAckError> {
|
||||
// Lock order: sorted bucket lifecycle/metadata fences (caller), then dirty map.
|
||||
// No await or storage operation occurs while the dirty map is locked.
|
||||
let (cleared, pending) = {
|
||||
let mut dirty = dirty_usage_buckets();
|
||||
let checked = entries
|
||||
.iter()
|
||||
.map(|(guard, generation)| {
|
||||
guard
|
||||
.checked_bucket_incarnation()
|
||||
.map(|(bucket, _)| (bucket, *generation))
|
||||
.map_err(|_| ScannerDirtyUsageAckError::IncarnationUnavailable)
|
||||
})
|
||||
.collect::<std::result::Result<Vec<_>, _>>()?;
|
||||
let cleared = apply_scoped_dirty_usage_ack(
|
||||
instance_id,
|
||||
scanner_activity_epoch(),
|
||||
DIRTY_USAGE_BUCKET_GENERATION.load(Ordering::Acquire),
|
||||
&mut dirty,
|
||||
&checked,
|
||||
probe_only,
|
||||
)?;
|
||||
if cleared > 0 {
|
||||
advance_generation(&DIRTY_USAGE_BUCKET_GENERATION);
|
||||
}
|
||||
(cleared, dirty.len())
|
||||
};
|
||||
if !probe_only {
|
||||
global_metrics().record_scanner_dirty_usage_cycle_clear(usize_to_u64_saturated(cleared), usize_to_u64_saturated(pending));
|
||||
}
|
||||
Ok(usize_to_u64_saturated(cleared))
|
||||
}
|
||||
|
||||
fn apply_scoped_dirty_usage_ack(
|
||||
instance_id: &str,
|
||||
current_instance: &str,
|
||||
current_generation: u64,
|
||||
dirty: &mut DirtyUsageBuckets,
|
||||
entries: &[(&str, u64)],
|
||||
probe_only: bool,
|
||||
) -> std::result::Result<usize, ScannerDirtyUsageAckError> {
|
||||
if instance_id != current_instance {
|
||||
return Err(ScannerDirtyUsageAckError::ProcessChanged);
|
||||
}
|
||||
if current_generation == u64::MAX
|
||||
|| entries
|
||||
.iter()
|
||||
.any(|(_, generation)| *generation == 0 || *generation == u64::MAX || *generation > current_generation)
|
||||
{
|
||||
return Err(ScannerDirtyUsageAckError::InvalidGeneration);
|
||||
}
|
||||
let mut cleared = 0;
|
||||
if !probe_only {
|
||||
for (bucket, generation) in entries {
|
||||
if dirty.get(*bucket) == Some(generation) {
|
||||
dirty.remove(*bucket);
|
||||
cleared += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(cleared)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod scoped_dirty_usage_tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn scoped_dirty_usage_preserves_uncovered_newer_and_replayed_generations() {
|
||||
let mut dirty = HashMap::from([("hot".to_string(), 7), ("cold".to_string(), 8)]);
|
||||
assert_eq!(apply_scoped_dirty_usage_ack("p", "p", 8, &mut dirty, &[("cold", 8)], true), Ok(0));
|
||||
assert_eq!(dirty.len(), 2);
|
||||
assert_eq!(apply_scoped_dirty_usage_ack("p", "p", 8, &mut dirty, &[("cold", 8)], false), Ok(1));
|
||||
assert_eq!(dirty.get("hot"), Some(&7));
|
||||
assert_eq!(apply_scoped_dirty_usage_ack("p", "p", 8, &mut dirty, &[("cold", 8)], false), Ok(0));
|
||||
dirty.insert("cold".to_string(), 9);
|
||||
assert_eq!(apply_scoped_dirty_usage_ack("p", "p", 9, &mut dirty, &[("cold", 8)], false), Ok(0));
|
||||
assert_eq!(dirty.get("cold"), Some(&9));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scoped_dirty_usage_rejects_restart_and_invalid_batch_before_clearing() {
|
||||
let original = HashMap::from([("hot".to_string(), 7), ("cold".to_string(), 8)]);
|
||||
let mut dirty = original.clone();
|
||||
assert_eq!(
|
||||
apply_scoped_dirty_usage_ack("old", "new", 8, &mut dirty, &[("cold", 8)], false),
|
||||
Err(ScannerDirtyUsageAckError::ProcessChanged)
|
||||
);
|
||||
for generation in [0, 9, u64::MAX] {
|
||||
assert_eq!(
|
||||
apply_scoped_dirty_usage_ack("p", "p", 8, &mut dirty, &[("cold", 8), ("hot", generation)], false),
|
||||
Err(ScannerDirtyUsageAckError::InvalidGeneration)
|
||||
);
|
||||
assert_eq!(dirty, original);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn dirty_usage_buckets() -> MutexGuard<'static, DirtyUsageBuckets> {
|
||||
|
||||
@@ -71,6 +71,7 @@ where
|
||||
leader_epoch,
|
||||
scan_mode,
|
||||
scan_scope: ScannerBucketScanScope::default(),
|
||||
persisted_usage_baseline: None,
|
||||
};
|
||||
nsscanner_with_storage_status_scoped(store, request).await
|
||||
}
|
||||
@@ -83,6 +84,79 @@ pub(crate) struct ScannerCycleRequest {
|
||||
pub(crate) leader_epoch: u64,
|
||||
pub(crate) scan_mode: HealScanMode,
|
||||
pub(crate) scan_scope: ScannerBucketScanScope,
|
||||
pub(crate) persisted_usage_baseline: Option<Bytes>,
|
||||
}
|
||||
|
||||
struct ScannerBucketScopeResolution<'a> {
|
||||
requested_scope: ScannerBucketScanScope,
|
||||
baseline_proof: ScannerCacheBaselineProof<'a>,
|
||||
activity_before: &'a crate::scanner::ScannerActivitySnapshot,
|
||||
dirty_usage_snapshot: &'a DirtyUsageSnapshot,
|
||||
all_buckets: &'a [BucketInfo],
|
||||
}
|
||||
|
||||
async fn resolve_scanner_bucket_scan_scope<S>(
|
||||
store: &S,
|
||||
distributed: bool,
|
||||
resolution: ScannerBucketScopeResolution<'_>,
|
||||
) -> ScannerBucketScanScope
|
||||
where
|
||||
S: ScannerStorage,
|
||||
{
|
||||
if !resolution.requested_scope.is_default()
|
||||
|| !resolution.dirty_usage_snapshot.covers_all_pending
|
||||
|| resolution.dirty_usage_snapshot.generation == u64::MAX
|
||||
|| resolution.dirty_usage_snapshot.buckets.len() > crate::SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES
|
||||
{
|
||||
return resolution.requested_scope;
|
||||
}
|
||||
|
||||
let mut dirty_buckets = resolution
|
||||
.dirty_usage_snapshot
|
||||
.buckets
|
||||
.keys()
|
||||
.cloned()
|
||||
.collect::<HashSet<_>>();
|
||||
if distributed {
|
||||
let Some(notification_system) = store.scanner_notification_system() else {
|
||||
return resolution.requested_scope;
|
||||
};
|
||||
let Ok(peer_snapshots) = notification_system.scanner_dirty_usage_snapshots().await else {
|
||||
return resolution.requested_scope;
|
||||
};
|
||||
let mut expected_peers = HashMap::new();
|
||||
for (host, lease_instance_id, _) in crate::scanner::scanner_activity_publication_lease_targets(resolution.activity_before)
|
||||
{
|
||||
let Some((activity_instance_id, generation, pending)) =
|
||||
crate::scanner::scanner_activity_dirty_usage_state_for_host(resolution.activity_before, &host)
|
||||
else {
|
||||
return resolution.requested_scope;
|
||||
};
|
||||
if activity_instance_id != lease_instance_id || expected_peers.contains_key(&host) {
|
||||
return resolution.requested_scope;
|
||||
}
|
||||
expected_peers.insert(
|
||||
host,
|
||||
ScannerPeerDirtyUsageExpectation {
|
||||
instance_id: activity_instance_id.to_string(),
|
||||
generation,
|
||||
pending,
|
||||
},
|
||||
);
|
||||
}
|
||||
let Some(remote_dirty_buckets) = verified_remote_dirty_usage_buckets(&expected_peers, peer_snapshots) else {
|
||||
return resolution.requested_scope;
|
||||
};
|
||||
dirty_buckets.extend(remote_dirty_buckets);
|
||||
}
|
||||
|
||||
scoped_scan_scope_from_dirty_buckets(
|
||||
resolution.requested_scope,
|
||||
dirty_buckets,
|
||||
true,
|
||||
resolution.all_buckets,
|
||||
resolution.baseline_proof,
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) async fn nsscanner_with_storage_status_scoped<S>(store: &S, request: ScannerCycleRequest) -> Result<ScannerCycleResult>
|
||||
@@ -97,6 +171,7 @@ where
|
||||
leader_epoch,
|
||||
scan_mode,
|
||||
scan_scope,
|
||||
persisted_usage_baseline,
|
||||
} = request;
|
||||
let child_token = ctx.child_token();
|
||||
let _tier_cycle_guard = begin_tier_registry_cycle(want_cycle, leader_epoch);
|
||||
@@ -186,8 +261,26 @@ where
|
||||
}
|
||||
bucket_plan_complete &= buckets_by_source.keys().copied().collect::<HashSet<_>>() == *expected_sources;
|
||||
let scan_plan_digest =
|
||||
scanner_bucket_plan_digest(&all_buckets, crate::scanner::scanner_activity_snapshot_digest(&activity_before));
|
||||
scanner_bucket_plan_digest(&all_buckets, crate::scanner::scanner_activity_structural_digest(&activity_before));
|
||||
let dirty_usage_snapshot = Arc::new(snapshot_dirty_usage_buckets(&all_buckets, dirty_generation_before_bucket_list));
|
||||
let scan_scope = resolve_scanner_bucket_scan_scope(
|
||||
store,
|
||||
distributed,
|
||||
ScannerBucketScopeResolution {
|
||||
requested_scope: scan_scope,
|
||||
baseline_proof: ScannerCacheBaselineProof {
|
||||
data: persisted_usage_baseline.as_ref(),
|
||||
expected_sources: &expected_sources,
|
||||
leader_epoch,
|
||||
want_cycle,
|
||||
scan_plan_digest,
|
||||
},
|
||||
activity_before: &activity_before,
|
||||
dirty_usage_snapshot: &dirty_usage_snapshot,
|
||||
all_buckets: &all_buckets,
|
||||
},
|
||||
)
|
||||
.await;
|
||||
let cache_cycle_floor = Arc::new(AtomicU64::new(want_cycle));
|
||||
let tier_registry = runtime_tier_registry_for_cycle(want_cycle, leader_epoch).await;
|
||||
let tier_registry_generation = tier_registry.generation;
|
||||
|
||||
@@ -655,9 +655,31 @@ fn completed_data_usage_info_requires_every_set_before_publish() {
|
||||
.expect("all completed sets should produce a publishable data usage snapshot");
|
||||
assert_eq!(last_update, SystemTime::UNIX_EPOCH + Duration::from_secs(20));
|
||||
assert_eq!(data_usage_info.scanner_cycle, Some(0));
|
||||
assert_eq!(data_usage_info.scanner_epoch, Some(0));
|
||||
assert_eq!(data_usage_info.objects_total_count, 3);
|
||||
assert_eq!(data_usage_info.buckets_usage.len(), 3);
|
||||
assert!(data_usage_info.usage_snapshot_complete);
|
||||
assert_eq!(
|
||||
data_usage_info
|
||||
.usage_snapshot_set_states
|
||||
.iter()
|
||||
.map(|state| {
|
||||
(
|
||||
state.pool_index,
|
||||
state.set_index,
|
||||
state.scanner_cycle,
|
||||
state.scanner_epoch,
|
||||
state.scan_plan_digest,
|
||||
state.complete,
|
||||
state.tombstone,
|
||||
)
|
||||
})
|
||||
.collect::<Vec<_>>(),
|
||||
vec![
|
||||
(0, 0, Some(0), Some(0), Some(TEST_PLAN_DIGEST.0), true, false),
|
||||
(1, 0, Some(0), Some(0), Some(TEST_PLAN_DIGEST.0), true, false),
|
||||
]
|
||||
);
|
||||
assert_eq!(
|
||||
data_usage_info
|
||||
.buckets_usage
|
||||
|
||||
@@ -17,6 +17,7 @@ use super::io_disk::tier_stats_template;
|
||||
use super::*;
|
||||
use crate::scanner_budget::ScannerCycleBudgetConfig;
|
||||
use crate::scanner_folder::ScannerItem;
|
||||
use crate::storage_api::EcstoreScannerPeerDirtyUsageSnapshot;
|
||||
use crate::storage_api::owner::{
|
||||
EcstorePoolDecommissionInfo, EcstoreRebalStatus, EcstoreRebalanceInfo, EcstoreRebalanceMeta, EcstoreRebalanceStats,
|
||||
};
|
||||
@@ -796,6 +797,195 @@ fn complete_set_usage_cache(buckets: &[(&str, usize)], scan_plan_digest: DataUsa
|
||||
cache
|
||||
}
|
||||
|
||||
fn complete_usage_baseline(
|
||||
source: DataUsageCacheSource,
|
||||
scan_plan_digest: DataUsageScanPlanDigest,
|
||||
scanner_cycle: u64,
|
||||
scanner_epoch: u64,
|
||||
) -> bytes::Bytes {
|
||||
let baseline = DataUsageInfo {
|
||||
last_update: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(10)),
|
||||
scanner_cycle: Some(scanner_cycle),
|
||||
scanner_epoch: Some(scanner_epoch),
|
||||
buckets_count: 1,
|
||||
buckets_usage: HashMap::from([("photos".to_string(), Default::default())]),
|
||||
usage_snapshot_complete: true,
|
||||
usage_snapshot_converged: Some(true),
|
||||
usage_snapshot_set_states: vec![DataUsageSnapshotSetState {
|
||||
pool_index: u64::try_from(source.pool_index).expect("test pool index should fit"),
|
||||
set_index: u64::try_from(source.set_index).expect("test set index should fit"),
|
||||
scanner_cycle: Some(scanner_cycle),
|
||||
scanner_epoch: Some(scanner_epoch),
|
||||
scan_plan_digest: Some(scan_plan_digest.0),
|
||||
complete: true,
|
||||
tombstone: false,
|
||||
}],
|
||||
..Default::default()
|
||||
};
|
||||
bytes::Bytes::from(serde_json::to_vec(&baseline).expect("test baseline should encode"))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scoped_scan_requires_a_converged_complete_baseline_with_exact_set_provenance() {
|
||||
let source = DataUsageCacheSource::new(1, 2);
|
||||
let expected_sources = HashSet::from([source]);
|
||||
let scan_plan_digest = DataUsageScanPlanDigest([9; 32]);
|
||||
let baseline = complete_usage_baseline(source, scan_plan_digest, 7, 11);
|
||||
|
||||
assert_eq!(
|
||||
complete_scanner_cache_baseline_plan_digest(ScannerCacheBaselineProof {
|
||||
data: Some(&baseline),
|
||||
expected_sources: &expected_sources,
|
||||
leader_epoch: 11,
|
||||
want_cycle: 8,
|
||||
scan_plan_digest,
|
||||
}),
|
||||
Some(scan_plan_digest)
|
||||
);
|
||||
|
||||
let mut incomplete = serde_json::from_slice::<DataUsageInfo>(&baseline).expect("test baseline should decode");
|
||||
incomplete.usage_snapshot_converged = Some(false);
|
||||
let incomplete = bytes::Bytes::from(serde_json::to_vec(&incomplete).expect("test baseline should encode"));
|
||||
assert_eq!(
|
||||
complete_scanner_cache_baseline_plan_digest(ScannerCacheBaselineProof {
|
||||
data: Some(&incomplete),
|
||||
expected_sources: &expected_sources,
|
||||
leader_epoch: 11,
|
||||
want_cycle: 8,
|
||||
scan_plan_digest,
|
||||
}),
|
||||
None
|
||||
);
|
||||
|
||||
let mut wrong_provenance = serde_json::from_slice::<DataUsageInfo>(&baseline).expect("test baseline should decode");
|
||||
wrong_provenance.usage_snapshot_set_states[0].scan_plan_digest = Some([8; 32]);
|
||||
let wrong_provenance = bytes::Bytes::from(serde_json::to_vec(&wrong_provenance).expect("test baseline should encode"));
|
||||
assert_eq!(
|
||||
complete_scanner_cache_baseline_plan_digest(ScannerCacheBaselineProof {
|
||||
data: Some(&wrong_provenance),
|
||||
expected_sources: &expected_sources,
|
||||
leader_epoch: 11,
|
||||
want_cycle: 8,
|
||||
scan_plan_digest,
|
||||
}),
|
||||
None
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scoped_scan_selects_only_current_dirty_buckets_after_baseline_validation() {
|
||||
let source = DataUsageCacheSource::new(1, 2);
|
||||
let expected_sources = HashSet::from([source]);
|
||||
let baseline_scan_plan_digest = DataUsageScanPlanDigest([4; 32]);
|
||||
let current_scan_plan_digest = DataUsageScanPlanDigest([5; 32]);
|
||||
let baseline = complete_usage_baseline(source, current_scan_plan_digest, 7, 11);
|
||||
let scope = scoped_scan_scope_from_dirty_buckets(
|
||||
ScannerBucketScanScope::default(),
|
||||
HashSet::from(["photos".to_string(), "deleted".to_string()]),
|
||||
true,
|
||||
&[bucket_info("photos")],
|
||||
ScannerCacheBaselineProof {
|
||||
data: Some(&baseline),
|
||||
expected_sources: &expected_sources,
|
||||
leader_epoch: 11,
|
||||
want_cycle: 8,
|
||||
scan_plan_digest: current_scan_plan_digest,
|
||||
},
|
||||
);
|
||||
|
||||
assert_eq!(scope.baseline_scan_plan_digest, Some(current_scan_plan_digest));
|
||||
assert_eq!(
|
||||
scope
|
||||
.selected_buckets
|
||||
.as_deref()
|
||||
.expect("validated scope should select a bucket"),
|
||||
&HashSet::from(["photos".to_string()])
|
||||
);
|
||||
assert_ne!(scope.baseline_scan_plan_digest, Some(baseline_scan_plan_digest));
|
||||
}
|
||||
|
||||
fn peer_dirty_usage_snapshot(
|
||||
instance_id: &str,
|
||||
generation: u64,
|
||||
complete: bool,
|
||||
buckets: &[(&str, u64)],
|
||||
) -> EcstoreScannerPeerDirtyUsageSnapshot {
|
||||
EcstoreScannerPeerDirtyUsageSnapshot {
|
||||
instance_id: instance_id.to_string(),
|
||||
generation,
|
||||
pending_bucket_count: u64::try_from(buckets.len()).expect("test bucket count should fit"),
|
||||
protocol_version: crate::SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION,
|
||||
complete,
|
||||
buckets: buckets
|
||||
.iter()
|
||||
.map(|(bucket, generation)| ((*bucket).to_string(), *generation))
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn verified_remote_dirty_usage_buckets_merges_only_complete_current_snapshots() {
|
||||
let expected_peers = HashMap::from([
|
||||
(
|
||||
"node-a:9000".to_string(),
|
||||
ScannerPeerDirtyUsageExpectation {
|
||||
instance_id: "instance-a".to_string(),
|
||||
generation: 7,
|
||||
pending: true,
|
||||
},
|
||||
),
|
||||
(
|
||||
"node-b:9000".to_string(),
|
||||
ScannerPeerDirtyUsageExpectation {
|
||||
instance_id: "instance-b".to_string(),
|
||||
generation: 3,
|
||||
pending: false,
|
||||
},
|
||||
),
|
||||
]);
|
||||
|
||||
assert_eq!(
|
||||
verified_remote_dirty_usage_buckets(
|
||||
&expected_peers,
|
||||
vec![
|
||||
(
|
||||
"node-a:9000".to_string(),
|
||||
peer_dirty_usage_snapshot("instance-a", 7, true, &[("photos", 7)]),
|
||||
),
|
||||
(
|
||||
"node-b:9000".to_string(),
|
||||
peer_dirty_usage_snapshot("instance-b", 3, true, &[("archive", 3)]),
|
||||
),
|
||||
],
|
||||
),
|
||||
Some(HashSet::from(["photos".to_string(), "archive".to_string()]))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn verified_remote_dirty_usage_buckets_rejects_incomplete_or_stale_peer_state() {
|
||||
let expected_peers = HashMap::from([(
|
||||
"node-a:9000".to_string(),
|
||||
ScannerPeerDirtyUsageExpectation {
|
||||
instance_id: "instance-a".to_string(),
|
||||
generation: 7,
|
||||
pending: true,
|
||||
},
|
||||
)]);
|
||||
|
||||
for snapshot in [
|
||||
peer_dirty_usage_snapshot("instance-a", 7, false, &[("photos", 7)]),
|
||||
peer_dirty_usage_snapshot("instance-a", 6, true, &[("photos", 6)]),
|
||||
peer_dirty_usage_snapshot("instance-b", 7, true, &[("photos", 7)]),
|
||||
peer_dirty_usage_snapshot("instance-a", 7, true, &[]),
|
||||
] {
|
||||
assert!(
|
||||
verified_remote_dirty_usage_buckets(&expected_peers, vec![("node-a:9000".to_string(), snapshot)]).is_none(),
|
||||
"incomplete, stale, mismatched, or empty pending peer state must fall back to a full scan"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scoped_set_scan_preserves_unselected_usage_and_drops_deleted_buckets() {
|
||||
let baseline_digest = DataUsageScanPlanDigest([1; 32]);
|
||||
|
||||
@@ -38,8 +38,8 @@ pub(crate) use rustfs_ecstore::api::bucket::lifecycle::lifecycle::object_opts_fr
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::init_bucket_metadata_sys as ecstore_init_bucket_metadata_sys;
|
||||
pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::{
|
||||
get_lifecycle_config as ecstore_get_lifecycle_config, get_object_lock_config as ecstore_get_object_lock_config,
|
||||
get_replication_config as ecstore_get_replication_config,
|
||||
BucketMetadataMutationGuard as EcstoreBucketMetadataMutationGuard, get_lifecycle_config as ecstore_get_lifecycle_config,
|
||||
get_object_lock_config as ecstore_get_object_lock_config, get_replication_config as ecstore_get_replication_config,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::api::bucket::replication::{
|
||||
ReplicateObjectInfo, ReplicationConfig as EcstoreReplicationConfig,
|
||||
@@ -103,7 +103,9 @@ pub(crate) use rustfs_ecstore::api::rebalance::{
|
||||
RebalStatus as EcstoreRebalStatus, RebalanceInfo as EcstoreRebalanceInfo, RebalanceMeta as EcstoreRebalanceMeta,
|
||||
RebalanceStats as EcstoreRebalanceStats,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::api::rpc::ScannerBucketListing as EcstoreScannerBucketListing;
|
||||
pub(crate) use rustfs_ecstore::api::rpc::{
|
||||
ScannerBucketListing as EcstoreScannerBucketListing, ScannerPeerDirtyUsageSnapshot as EcstoreScannerPeerDirtyUsageSnapshot,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::runtime::InstanceContext as EcstoreInstanceContext;
|
||||
pub(crate) use rustfs_ecstore::api::runtime::{
|
||||
|
||||
Reference in New Issue
Block a user