mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-18 14:05:27 +00:00
Compare commits
6 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| bd8855d0e4 | |||
| f57449197b | |||
| c4a50138ea | |||
| a5a5d0aac0 | |||
| 5e04936a88 | |||
| d098f82bf5 |
Generated
+1
@@ -10843,6 +10843,7 @@ dependencies = [
|
||||
"tracing",
|
||||
"tracing-subscriber",
|
||||
"url",
|
||||
"uuid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
@@ -864,6 +864,70 @@ async fn test_webhook_event_delivery_and_filtering() -> TestResult {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_webhook_subscriptions_follow_bucket_incarnation() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
let (endpoint, mut rx, collector) = spawn_event_collector().await?;
|
||||
let allowed_origin = endpoint_origin(&endpoint)?;
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server_with_env(vec![], &[(ENV_OUTBOUND_ALLOW_ORIGINS, allowed_origin.as_str())])
|
||||
.await?;
|
||||
enable_notify_module(&env).await?;
|
||||
let target = "incarnation";
|
||||
configure_webhook_target(&env, target, &endpoint).await?;
|
||||
wait_for_target_registered(&env, target).await?;
|
||||
|
||||
let client = env.create_s3_client();
|
||||
let bucket = "notify-incarnation";
|
||||
let control_bucket = "notify-incarnation-control";
|
||||
for name in [bucket, control_bucket] {
|
||||
client.create_bucket().bucket(name).send().await?;
|
||||
put_notification_config(&client, name, target, "events/", ".txt").await?;
|
||||
}
|
||||
let put = |bucket, key| {
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.body(ByteStream::from_static(b"event fixture"))
|
||||
.send()
|
||||
};
|
||||
let wait = Duration::from_secs(20);
|
||||
let original_key = "events/original.txt";
|
||||
put(bucket, original_key).await?;
|
||||
wait_for_event(&mut rx, original_key, "s3:ObjectCreated:", wait).await?;
|
||||
|
||||
assert!(client.delete_bucket().bucket(bucket).send().await.is_err());
|
||||
let after_failed_delete = "events/retained.txt";
|
||||
put(bucket, after_failed_delete).await?;
|
||||
wait_for_event(&mut rx, after_failed_delete, "s3:ObjectCreated:", wait).await?;
|
||||
|
||||
for key in [original_key, after_failed_delete] {
|
||||
client.delete_object().bucket(bucket).key(key).send().await?;
|
||||
}
|
||||
client.delete_bucket().bucket(bucket).send().await?;
|
||||
client.create_bucket().bucket(bucket).send().await?;
|
||||
let config = client.get_bucket_notification_configuration().bucket(bucket).send().await?;
|
||||
assert!(config.queue_configurations().is_empty());
|
||||
|
||||
let new_key = "events/fresh.txt";
|
||||
let sentinel_key = "events/sentinel.txt";
|
||||
put(bucket, new_key).await?;
|
||||
put(control_bucket, sentinel_key).await?;
|
||||
let events = collect_until(&mut rx, sentinel_key, wait, Duration::from_secs(2)).await;
|
||||
assert!(events.iter().any(|event| event_key(event).as_deref() == Some(sentinel_key)));
|
||||
assert!(events.iter().all(|event| event_key(event).as_deref() != Some(new_key)));
|
||||
|
||||
put_notification_config(&client, bucket, target, "events/", ".txt").await?;
|
||||
let subscribed_key = "events/subscribed.txt";
|
||||
put(bucket, subscribed_key).await?;
|
||||
wait_for_event(&mut rx, subscribed_key, "s3:ObjectCreated:", wait).await?;
|
||||
env.stop_server();
|
||||
collector.abort();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// An event queued while the target endpoint rejects delivery survives on the
|
||||
/// durable store and is redelivered once the endpoint comes back.
|
||||
#[tokio::test]
|
||||
|
||||
@@ -51,15 +51,20 @@ impl LcAuditEvent {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn emit_transition_failed_event(object: ObjectInfo) {
|
||||
emit_lifecycle_event(EventName::ObjectTransitionFailed, object, LIFECYCLE_TRANSITION_USER_AGENT);
|
||||
pub(crate) fn emit_transition_failed_event(object: ObjectInfo, incarnation_id: Option<uuid::Uuid>) {
|
||||
emit_lifecycle_event(EventName::ObjectTransitionFailed, object, LIFECYCLE_TRANSITION_USER_AGENT, incarnation_id);
|
||||
}
|
||||
|
||||
pub(crate) fn emit_transition_complete_event(object: ObjectInfo) {
|
||||
emit_lifecycle_event(EventName::ObjectTransitionComplete, object, LIFECYCLE_TRANSITION_USER_AGENT);
|
||||
pub(crate) fn emit_transition_complete_event(object: ObjectInfo, incarnation_id: Option<uuid::Uuid>) {
|
||||
emit_lifecycle_event(
|
||||
EventName::ObjectTransitionComplete,
|
||||
object,
|
||||
LIFECYCLE_TRANSITION_USER_AGENT,
|
||||
incarnation_id,
|
||||
);
|
||||
}
|
||||
|
||||
pub(crate) fn emit_transitioned_expiration_event(source: &ObjectInfo, deleted: &ObjectInfo) {
|
||||
pub(crate) fn emit_transitioned_expiration_event(source: &ObjectInfo, deleted: &ObjectInfo, incarnation_id: uuid::Uuid) {
|
||||
let event_name = transitioned_expiration_event_name(source.delete_marker, deleted.delete_marker);
|
||||
let object = ObjectInfo {
|
||||
bucket: source.bucket.clone(),
|
||||
@@ -69,12 +74,17 @@ pub(crate) fn emit_transitioned_expiration_event(source: &ObjectInfo, deleted: &
|
||||
delete_marker: source.delete_marker,
|
||||
..Default::default()
|
||||
};
|
||||
emit_lifecycle_event(event_name, object, LIFECYCLE_EXPIRY_USER_AGENT);
|
||||
emit_lifecycle_event(event_name, object, LIFECYCLE_EXPIRY_USER_AGENT, Some(incarnation_id));
|
||||
}
|
||||
|
||||
pub(crate) fn emit_non_transitioned_expiration_event(action: IlmAction, source: &ObjectInfo, deleted: ObjectInfo) {
|
||||
pub(crate) fn emit_non_transitioned_expiration_event(
|
||||
action: IlmAction,
|
||||
source: &ObjectInfo,
|
||||
deleted: ObjectInfo,
|
||||
incarnation_id: uuid::Uuid,
|
||||
) {
|
||||
let event_name = non_transitioned_expiration_event_name(action, source.delete_marker, deleted.delete_marker);
|
||||
emit_lifecycle_event(event_name, deleted, LIFECYCLE_EXPIRY_USER_AGENT);
|
||||
emit_lifecycle_event(event_name, deleted, LIFECYCLE_EXPIRY_USER_AGENT, Some(incarnation_id));
|
||||
}
|
||||
|
||||
/// Emit the lifecycle expiration event for one version removed by the batch
|
||||
@@ -83,9 +93,15 @@ pub(crate) fn emit_non_transitioned_expiration_event(action: IlmAction, source:
|
||||
/// That path never sent events, so a successful noncurrent-version expiry was
|
||||
/// invisible to notification subscribers even though the equivalent
|
||||
/// current-version path emits one (backlog#2202).
|
||||
pub(crate) fn emit_noncurrent_expiration_event(bucket: &str, target: &ObjectToDelete, deleted: &DeletedObject, failed: bool) {
|
||||
pub(crate) fn emit_noncurrent_expiration_event(
|
||||
bucket: &str,
|
||||
target: &ObjectToDelete,
|
||||
deleted: &DeletedObject,
|
||||
failed: bool,
|
||||
incarnation_id: uuid::Uuid,
|
||||
) {
|
||||
if let Some((event_name, object)) = noncurrent_expiration_event(bucket, target, deleted, failed) {
|
||||
emit_lifecycle_event(event_name, object, LIFECYCLE_EXPIRY_USER_AGENT);
|
||||
emit_lifecycle_event(event_name, object, LIFECYCLE_EXPIRY_USER_AGENT, Some(incarnation_id));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -131,9 +147,10 @@ fn noncurrent_expiration_event(
|
||||
Some((event_name, object))
|
||||
}
|
||||
|
||||
fn emit_lifecycle_event(event_name: EventName, object: ObjectInfo, user_agent: &str) {
|
||||
fn emit_lifecycle_event(event_name: EventName, object: ObjectInfo, user_agent: &str, incarnation_id: Option<uuid::Uuid>) {
|
||||
send_event(EventArgs {
|
||||
event_name: event_name.to_string(),
|
||||
bucket_incarnation_id: incarnation_id,
|
||||
bucket_name: object.bucket.clone(),
|
||||
object,
|
||||
user_agent: user_agent.to_string(),
|
||||
|
||||
@@ -2166,9 +2166,10 @@ impl TransitionState {
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
if let Err(err) =
|
||||
let (incarnation_id, transition_result) =
|
||||
transition_object(api.clone(), &task.obj_info, LcAuditEvent::new(task.event.clone(), task.src.clone()))
|
||||
.await
|
||||
.await;
|
||||
if let Err(err) = transition_result
|
||||
{
|
||||
if let (Some(job_id), Some(result_key)) = (task.manual_job_id, task.manual_result_key.as_deref()) {
|
||||
record_manual_transition_worker_result_for_task_with_reason(
|
||||
@@ -2195,7 +2196,7 @@ impl TransitionState {
|
||||
"Lifecycle tier operation failed"
|
||||
);
|
||||
}
|
||||
emit_transition_failed_event(obj_info_for_event);
|
||||
emit_transition_failed_event(obj_info_for_event, incarnation_id);
|
||||
} else {
|
||||
if let (Some(job_id), Some(result_key)) = (task.manual_job_id, task.manual_result_key.as_deref()) {
|
||||
record_manual_transition_worker_result_for_task(
|
||||
@@ -2217,7 +2218,7 @@ impl TransitionState {
|
||||
}
|
||||
transition_state.add_lastday_stats(&task.event.storage_class, ts);
|
||||
|
||||
emit_transition_complete_event(obj_info_for_event);
|
||||
emit_transition_complete_event(obj_info_for_event, incarnation_id);
|
||||
}
|
||||
// Release the dedup claim so a later lifecycle pass can
|
||||
// re-transition this object if needed (rustfs/backlog#1268).
|
||||
@@ -5121,7 +5122,7 @@ async fn expire_transitioned_object_with_lock_lost_signal(
|
||||
}
|
||||
};
|
||||
|
||||
schedule_lifecycle_replication_delete_if_needed(oi, &dobj).await;
|
||||
schedule_lifecycle_replication_delete_if_needed(oi, &dobj, bucket_incarnation_id).await;
|
||||
|
||||
// The transitioned version is gone; evict any cached body for this object
|
||||
// so it does not linger until TTL (ODC-26).
|
||||
@@ -5134,7 +5135,7 @@ async fn expire_transitioned_object_with_lock_lost_signal(
|
||||
|
||||
//audit_log_lifecycle(oi, ILMExpiry, tags);
|
||||
|
||||
emit_transitioned_expiration_event(oi, &dobj);
|
||||
emit_transitioned_expiration_event(oi, &dobj, bucket_incarnation_id);
|
||||
|
||||
Ok(dobj)
|
||||
}
|
||||
@@ -5152,12 +5153,12 @@ pub fn gen_transition_objname(bucket: &str) -> Result<String, Error> {
|
||||
Ok(obj)
|
||||
}
|
||||
|
||||
pub async fn transition_object(api: Arc<ECStore>, oi: &ObjectInfo, lae: LcAuditEvent) -> Result<(), Error> {
|
||||
async fn transition_object(api: Arc<ECStore>, oi: &ObjectInfo, lae: LcAuditEvent) -> (Option<Uuid>, Result<(), Error>) {
|
||||
let time_ilm = Metrics::time_ilm(lae.event.action);
|
||||
|
||||
let (publication_guard, bucket_incarnation_id) = lifecycle_transition_publication_guard(&api, oi)
|
||||
.await
|
||||
.ok_or_else(|| Error::other("lifecycle transition is not allowed for this bucket"))?;
|
||||
let Some((publication_guard, bucket_incarnation_id)) = lifecycle_transition_publication_guard(&api, oi).await else {
|
||||
return (None, Err(Error::other("lifecycle transition is not allowed for this bucket")));
|
||||
};
|
||||
|
||||
let etag = if let Some(etag) = &oi.etag { etag } else { "" };
|
||||
let etag = etag.to_string();
|
||||
@@ -5180,7 +5181,7 @@ pub async fn transition_object(api: Arc<ECStore>, oi: &ObjectInfo, lae: LcAuditE
|
||||
opts.add_namespace_lock_guard(&publication_guard);
|
||||
let result = api.transition_object(&oi.bucket, &oi.name, &opts).await;
|
||||
time_ilm(1)();
|
||||
result
|
||||
(Some(bucket_incarnation_id), result)
|
||||
}
|
||||
|
||||
#[allow(
|
||||
@@ -6034,7 +6035,7 @@ async fn apply_expiry_on_non_transitioned_objects_with_lock_lost_signal(
|
||||
{
|
||||
accounting.commit(size).await;
|
||||
}
|
||||
schedule_lifecycle_replication_delete_if_needed(oi, &dobj).await;
|
||||
schedule_lifecycle_replication_delete_if_needed(oi, &dobj, bucket_incarnation_id).await;
|
||||
|
||||
// The object (or all its versions, for delete_all) was expired; evict any
|
||||
// cached body so dead bytes do not sit resident until TTL (ODC-26). The
|
||||
@@ -6050,7 +6051,7 @@ async fn apply_expiry_on_non_transitioned_objects_with_lock_lost_signal(
|
||||
//let tags = LcAuditEvent::new(lc_event.clone(), src.clone()).tags();
|
||||
//tags["version-id"] = dobj.version_id;
|
||||
|
||||
emit_non_transitioned_expiration_event(lc_event.action, oi, dobj);
|
||||
emit_non_transitioned_expiration_event(lc_event.action, oi, dobj, bucket_incarnation_id);
|
||||
|
||||
if lc_event.action != IlmAction::NoneAction {
|
||||
let mut num_versions = 1_u64;
|
||||
@@ -6202,12 +6203,12 @@ fn lifecycle_deleted_object(oi: &ObjectInfo, dobj: &ObjectInfo) -> DeletedObject
|
||||
}
|
||||
}
|
||||
|
||||
async fn schedule_lifecycle_replication_delete_if_needed(oi: &ObjectInfo, dobj: &ObjectInfo) {
|
||||
async fn schedule_lifecycle_replication_delete_if_needed(oi: &ObjectInfo, dobj: &ObjectInfo, bucket_incarnation_id: Uuid) {
|
||||
let delete_object = lifecycle_deleted_object(oi, dobj);
|
||||
if delete_object.replication_state.is_none() {
|
||||
return;
|
||||
}
|
||||
replication_sink::schedule_delete(oi.bucket.clone(), delete_object).await;
|
||||
replication_sink::schedule_delete(oi.bucket.clone(), delete_object, Some(bucket_incarnation_id)).await;
|
||||
}
|
||||
|
||||
async fn lifecycle_delete_config_snapshot(api: &ECStore, oi: &ObjectInfo) -> Result<DeleteReplicationConfigSnapshot, Error> {
|
||||
|
||||
@@ -128,7 +128,7 @@ pub async fn delete_object_versions(
|
||||
// order and admission; the event is derived from the committed
|
||||
// result, and a send failure never rolls back a delete that
|
||||
// already happened (backlog#2202).
|
||||
emit_noncurrent_expiration_event(bucket, target, deleted_obj, false);
|
||||
emit_noncurrent_expiration_event(bucket, target, deleted_obj, false, bucket_incarnation_id);
|
||||
}
|
||||
if !deleted_obj
|
||||
.replication_state
|
||||
@@ -137,7 +137,7 @@ pub async fn delete_object_versions(
|
||||
{
|
||||
continue;
|
||||
}
|
||||
replication_sink::schedule_delete(bucket.to_string(), deleted_obj.clone()).await;
|
||||
replication_sink::schedule_delete(bucket.to_string(), deleted_obj.clone(), Some(bucket_incarnation_id)).await;
|
||||
}
|
||||
|
||||
for (i, err) in errors.iter().enumerate() {
|
||||
|
||||
@@ -66,8 +66,12 @@ pub(crate) fn lifecycle_action_waits_for_replication(action: IlmAction) -> bool
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) async fn schedule_delete(bucket: String, delete_object: DeletedObject) {
|
||||
ReplicationLifecycleBridge::schedule_delete(bucket, delete_object).await;
|
||||
pub(crate) async fn schedule_delete(
|
||||
bucket: String,
|
||||
delete_object: DeletedObject,
|
||||
notification_bucket_incarnation: Option<uuid::Uuid>,
|
||||
) {
|
||||
ReplicationLifecycleBridge::schedule_delete(bucket, delete_object, notification_bucket_incarnation).await;
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -460,7 +460,7 @@ fn clear_bucket_overrides(bucket: &str) {
|
||||
}
|
||||
|
||||
/// Publish application-owned bytes on every cache install path.
|
||||
fn sync_on_demand_migration(bucket: &str, bm: &BucketMetadata) {
|
||||
fn sync_application_bucket_configs(bucket: &str, bm: &BucketMetadata) {
|
||||
if let Some(hook) = BUCKET_CONFIG_PUBLISH_HOOK.get() {
|
||||
hook(
|
||||
bucket,
|
||||
@@ -468,12 +468,22 @@ fn sync_on_demand_migration(bucket: &str, bm: &BucketMetadata) {
|
||||
bm.on_demand_migration_config()
|
||||
.map(|(bytes, stamp)| (bytes, stamp, bm.bucket_incarnation_id)),
|
||||
);
|
||||
hook(
|
||||
bucket,
|
||||
super::metadata::BUCKET_NOTIFICATION_CONFIG,
|
||||
(!bm.notification_config_xml.is_empty()).then_some((
|
||||
bm.notification_config_xml.as_slice(),
|
||||
bm.notification_config_updated_at,
|
||||
bm.bucket_incarnation_id,
|
||||
)),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn clear_on_demand_migration(bucket: &str) {
|
||||
fn clear_application_bucket_configs(bucket: &str) {
|
||||
if let Some(hook) = BUCKET_CONFIG_PUBLISH_HOOK.get() {
|
||||
hook(bucket, super::metadata::BUCKET_ON_DEMAND_MIGRATION_CONFIG, None);
|
||||
hook(bucket, super::metadata::BUCKET_NOTIFICATION_CONFIG, None);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2253,6 +2263,7 @@ impl BucketMetadataSys {
|
||||
.await?;
|
||||
self.publish_if_unchanged(bucket, expected.as_ref(), metadata, persisted, &guard)
|
||||
.await
|
||||
.map(|_| ())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -2486,7 +2497,7 @@ impl BucketMetadataSys {
|
||||
if removed {
|
||||
BucketTargetSys::get().delete(bucket).await;
|
||||
clear_bucket_overrides(bucket);
|
||||
clear_on_demand_migration(bucket);
|
||||
clear_application_bucket_configs(bucket);
|
||||
}
|
||||
}
|
||||
return Ok(());
|
||||
@@ -2524,7 +2535,7 @@ impl BucketMetadataSys {
|
||||
self.missing_buckets.invalidate(bucket).await;
|
||||
sync_bucket_target_sys(bucket, &bm).await;
|
||||
sync_bucket_overrides(bucket, &bm);
|
||||
sync_on_demand_migration(bucket, &bm);
|
||||
sync_application_bucket_configs(bucket, &bm);
|
||||
}
|
||||
MetadataLoadMode::Initial => {
|
||||
let _publish_guard = self
|
||||
@@ -2550,7 +2561,7 @@ impl BucketMetadataSys {
|
||||
metadata: BucketMetadata,
|
||||
persisted: bool,
|
||||
namespace_guard: &rustfs_lock::NamespaceLockGuard,
|
||||
) -> Result<()> {
|
||||
) -> Result<bool> {
|
||||
self.revoke_pending_bucket_creation(bucket).await;
|
||||
if !persisted {
|
||||
let _publish_guard = self
|
||||
@@ -2563,7 +2574,7 @@ impl BucketMetadataSys {
|
||||
_ => false,
|
||||
};
|
||||
if !unchanged {
|
||||
return Ok(());
|
||||
return Ok(false);
|
||||
}
|
||||
let removed = map.remove(bucket).is_some();
|
||||
drop(map);
|
||||
@@ -2572,9 +2583,9 @@ impl BucketMetadataSys {
|
||||
if removed {
|
||||
BucketTargetSys::get().delete(bucket).await;
|
||||
clear_bucket_overrides(bucket);
|
||||
clear_on_demand_migration(bucket);
|
||||
clear_application_bucket_configs(bucket);
|
||||
}
|
||||
return Ok(());
|
||||
return Ok(true);
|
||||
}
|
||||
let _publish_guard = self
|
||||
.lock_metadata_publish(bucket, namespace_guard, "refreshed bucket metadata publish")
|
||||
@@ -2587,7 +2598,7 @@ impl BucketMetadataSys {
|
||||
_ => false,
|
||||
};
|
||||
if !unchanged {
|
||||
return Ok(());
|
||||
return Ok(false);
|
||||
}
|
||||
map.insert(bucket.to_string(), Arc::clone(&metadata));
|
||||
drop(map);
|
||||
@@ -2595,8 +2606,8 @@ impl BucketMetadataSys {
|
||||
self.missing_buckets.invalidate(bucket).await;
|
||||
sync_bucket_target_sys(bucket, &metadata).await;
|
||||
sync_bucket_overrides(bucket, &metadata);
|
||||
sync_on_demand_migration(bucket, &metadata);
|
||||
Ok(())
|
||||
sync_application_bucket_configs(bucket, &metadata);
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
pub async fn get(&self, bucket: &str) -> Result<Arc<BucketMetadata>> {
|
||||
@@ -2625,7 +2636,7 @@ impl BucketMetadataSys {
|
||||
self.missing_buckets.invalidate(&bucket).await;
|
||||
sync_bucket_target_sys(&bucket, &bm).await;
|
||||
sync_bucket_overrides(&bucket, &bm);
|
||||
sync_on_demand_migration(&bucket, &bm);
|
||||
sync_application_bucket_configs(&bucket, &bm);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2649,7 +2660,7 @@ impl BucketMetadataSys {
|
||||
if removed {
|
||||
BucketTargetSys::get().delete(bucket).await;
|
||||
clear_bucket_overrides(bucket);
|
||||
clear_on_demand_migration(bucket);
|
||||
clear_application_bucket_configs(bucket);
|
||||
}
|
||||
removed || removed_fabricated
|
||||
}
|
||||
@@ -2876,25 +2887,35 @@ impl BucketMetadataSys {
|
||||
namespace_guard: &rustfs_lock::NamespaceLockGuard,
|
||||
) -> Result<()> {
|
||||
self.revoke_pending_bucket_creation(bucket).await;
|
||||
let expected = self.metadata_map.read().await.get(bucket).cloned();
|
||||
if !self
|
||||
.bucket_exists(bucket, namespace_guard, "peer bucket metadata existence check")
|
||||
.await?
|
||||
{
|
||||
return Err(Error::other(PEER_METADATA_NOT_PERSISTED));
|
||||
}
|
||||
let (metadata, persisted) = await_bucket_namespace_operation(
|
||||
Some(namespace_guard),
|
||||
bucket,
|
||||
"peer bucket metadata load",
|
||||
load_bucket_metadata_parse_with_presence(self.object_store(), bucket, true),
|
||||
)
|
||||
.await?;
|
||||
if !persisted {
|
||||
return Err(Error::other(PEER_METADATA_NOT_PERSISTED));
|
||||
// A concurrent publication can reject the snapshot we read. Re-read
|
||||
// from storage so a newer committed config does not lose its reload to
|
||||
// an older one that happened to publish first.
|
||||
for _ in 0..3 {
|
||||
let expected = self.metadata_map.read().await.get(bucket).cloned();
|
||||
let (metadata, persisted) = await_bucket_namespace_operation(
|
||||
Some(namespace_guard),
|
||||
bucket,
|
||||
"peer bucket metadata load",
|
||||
load_bucket_metadata_parse_with_presence(self.object_store(), bucket, true),
|
||||
)
|
||||
.await?;
|
||||
if !persisted {
|
||||
return Err(Error::other(PEER_METADATA_NOT_PERSISTED));
|
||||
}
|
||||
if self
|
||||
.publish_if_unchanged(bucket, expected.as_ref(), metadata, true, namespace_guard)
|
||||
.await?
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
self.publish_if_unchanged(bucket, expected.as_ref(), metadata, true, namespace_guard)
|
||||
.await
|
||||
Err(Error::other("bucket metadata changed repeatedly during peer reload"))
|
||||
}
|
||||
|
||||
pub async fn get_config(&self, bucket: &str) -> Result<(Arc<BucketMetadata>, bool)> {
|
||||
@@ -2992,7 +3013,7 @@ impl BucketMetadataSys {
|
||||
self.missing_buckets.invalidate(bucket).await;
|
||||
sync_bucket_target_sys(bucket, &bm).await;
|
||||
sync_bucket_overrides(bucket, &bm);
|
||||
sync_on_demand_migration(bucket, &bm);
|
||||
sync_application_bucket_configs(bucket, &bm);
|
||||
return Ok((bm, true));
|
||||
} else {
|
||||
let exists = self
|
||||
@@ -3443,7 +3464,7 @@ impl BucketMetadataSys {
|
||||
self.missing_buckets.invalidate(bucket).await;
|
||||
sync_bucket_target_sys(bucket, &metadata).await;
|
||||
sync_bucket_overrides(bucket, &metadata);
|
||||
sync_on_demand_migration(bucket, &metadata);
|
||||
sync_application_bucket_configs(bucket, &metadata);
|
||||
Ok(BucketMetadataAuthority::Authoritative(metadata))
|
||||
}
|
||||
|
||||
@@ -6792,31 +6813,33 @@ mod tests {
|
||||
|
||||
const ODM_JSON: &[u8] = br#"{"source":{"provider":"minio","endpoint":"https://legacy.example.com:9000","region":"auto","bucket":"legacy-bucket","credentials":{"access_key":"AK","secret_key":"SK"}}}"#;
|
||||
|
||||
type RecordedOdmConfig = Option<(Vec<u8>, OffsetDateTime, Uuid)>;
|
||||
type RecordedOdmHookCall = (String, RecordedOdmConfig);
|
||||
const NOTIFY_XML: &[u8] = br#"<NotificationConfiguration><QueueConfiguration><Id>test</Id><Queue>arn:rustfs:sqs:us-east-1:primary:webhook</Queue><Event>s3:ObjectCreated:*</Event></QueueConfiguration></NotificationConfiguration>"#;
|
||||
const PUBLISH_TEST_BUCKET: &str = "application-config-hook-paths";
|
||||
|
||||
/// Every `(bucket, config)` the recording hook has seen. Tests filter by
|
||||
/// their own bucket name; the hook is process-wide and set once.
|
||||
static ODM_HOOK_CALLS: std::sync::Mutex<Vec<RecordedOdmHookCall>> = std::sync::Mutex::new(Vec::new());
|
||||
type RecordedConfig = Option<(Vec<u8>, OffsetDateTime, Uuid)>;
|
||||
type RecordedConfigCall = (String, RecordedConfig);
|
||||
|
||||
fn install_recording_odm_hook() {
|
||||
static CONFIG_HOOK_CALLS: std::sync::Mutex<Vec<RecordedConfigCall>> = std::sync::Mutex::new(Vec::new());
|
||||
|
||||
fn install_recording_config_hook() {
|
||||
BUCKET_CONFIG_PUBLISH_HOOK.get_or_init(|| {
|
||||
Box::new(|bucket, config_file, config| {
|
||||
assert_eq!(config_file, super::super::metadata::BUCKET_ON_DEMAND_MIGRATION_CONFIG);
|
||||
ODM_HOOK_CALLS.lock().unwrap().push((
|
||||
bucket.to_string(),
|
||||
config.map(|(bytes, stamp, incarnation)| (bytes.to_vec(), stamp, incarnation)),
|
||||
));
|
||||
if bucket == PUBLISH_TEST_BUCKET {
|
||||
CONFIG_HOOK_CALLS.lock().unwrap().push((
|
||||
config_file.to_string(),
|
||||
config.map(|(bytes, stamp, incarnation)| (bytes.to_vec(), stamp, incarnation)),
|
||||
));
|
||||
}
|
||||
})
|
||||
});
|
||||
}
|
||||
|
||||
fn odm_hook_calls(bucket: &str) -> Vec<RecordedOdmConfig> {
|
||||
ODM_HOOK_CALLS
|
||||
fn config_hook_calls(config_file: &str) -> Vec<RecordedConfig> {
|
||||
CONFIG_HOOK_CALLS
|
||||
.lock()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.filter(|(name, _)| name == bucket)
|
||||
.filter(|(file, _)| file == config_file)
|
||||
.map(|(_, config)| config.clone())
|
||||
.collect()
|
||||
}
|
||||
@@ -6826,51 +6849,56 @@ mod tests {
|
||||
/// reload, refresh loop, lazy load) and withdraws on removal, mirroring
|
||||
/// `sync_bucket_durability`.
|
||||
#[tokio::test]
|
||||
async fn on_demand_migration_hook_fires_on_every_cache_install_path() {
|
||||
install_recording_odm_hook();
|
||||
async fn application_config_hook_fires_on_every_cache_install_path() {
|
||||
use crate::bucket::metadata::{BUCKET_NOTIFICATION_CONFIG, BUCKET_ON_DEMAND_MIGRATION_CONFIG};
|
||||
|
||||
install_recording_config_hook();
|
||||
|
||||
let (dirs, ecstore) = isolated_store_over_temp_disks().await;
|
||||
let bucket = "odm-hook-paths";
|
||||
let bucket = PUBLISH_TEST_BUCKET;
|
||||
for dir in &dirs {
|
||||
std::fs::create_dir_all(dir.path().join(bucket)).expect("physical bucket should exist");
|
||||
}
|
||||
|
||||
let incarnation = Uuid::new_v4();
|
||||
let expect_publish = |before: usize, label: &str| {
|
||||
let calls = odm_hook_calls(bucket);
|
||||
assert_eq!(calls.len(), before + 1, "{label} must publish exactly once");
|
||||
assert_eq!(
|
||||
calls.last().unwrap().as_ref().map(|(bytes, _, _)| bytes.as_slice()),
|
||||
Some(ODM_JSON),
|
||||
"{label} must publish the stored bytes"
|
||||
);
|
||||
assert_eq!(calls.last().unwrap().as_ref().map(|(_, _, id)| *id), Some(incarnation));
|
||||
for (file, bytes) in [
|
||||
(BUCKET_ON_DEMAND_MIGRATION_CONFIG, ODM_JSON),
|
||||
(BUCKET_NOTIFICATION_CONFIG, NOTIFY_XML),
|
||||
] {
|
||||
let calls = config_hook_calls(file);
|
||||
assert_eq!(calls.len(), before + 1, "{label} must publish {file} exactly once");
|
||||
let (stored, _, id) = calls.last().unwrap().as_ref().expect("stored configuration");
|
||||
assert_eq!(stored, bytes, "{label} must publish the stored config");
|
||||
assert_eq!(*id, incarnation);
|
||||
}
|
||||
};
|
||||
|
||||
// set (via persist_new_and_set, which installs through `set`).
|
||||
let mut bm = BucketMetadata::new(bucket);
|
||||
bm.bucket_incarnation_id = incarnation;
|
||||
bm.update_config(crate::bucket::metadata::BUCKET_ON_DEMAND_MIGRATION_CONFIG, ODM_JSON.to_vec())
|
||||
bm.update_config(BUCKET_ON_DEMAND_MIGRATION_CONFIG, ODM_JSON.to_vec())
|
||||
.unwrap();
|
||||
bm.update_config(BUCKET_NOTIFICATION_CONFIG, NOTIFY_XML.to_vec()).unwrap();
|
||||
let writer = BucketMetadataSys::new(ecstore.clone());
|
||||
let before = odm_hook_calls(bucket).len();
|
||||
let before = config_hook_calls(BUCKET_ON_DEMAND_MIGRATION_CONFIG).len();
|
||||
writer.persist_new_and_set(bm).await.expect("metadata should persist");
|
||||
expect_publish(before, "set");
|
||||
|
||||
// init (initial load on a cold system).
|
||||
let mut cold = BucketMetadataSys::new(ecstore.clone());
|
||||
let before = odm_hook_calls(bucket).len();
|
||||
let before = config_hook_calls(BUCKET_ON_DEMAND_MIGRATION_CONFIG).len();
|
||||
cold.init(vec![bucket.to_string()]).await;
|
||||
assert!(cold.get(bucket).await.is_ok(), "initial load must cache the bucket");
|
||||
expect_publish(before, "init");
|
||||
|
||||
// peer reload.
|
||||
let before = odm_hook_calls(bucket).len();
|
||||
let before = config_hook_calls(BUCKET_ON_DEMAND_MIGRATION_CONFIG).len();
|
||||
cold.reload_from_store(bucket).await.expect("peer reload should publish");
|
||||
expect_publish(before, "peer reload");
|
||||
|
||||
// refresh loop.
|
||||
let before = odm_hook_calls(bucket).len();
|
||||
let before = config_hook_calls(BUCKET_ON_DEMAND_MIGRATION_CONFIG).len();
|
||||
let mut failed = HashSet::new();
|
||||
cold.concurrent_load(&[bucket.to_string()], &mut failed, MetadataLoadMode::Refresh)
|
||||
.await;
|
||||
@@ -6879,24 +6907,28 @@ mod tests {
|
||||
|
||||
// lazy load on another cold system.
|
||||
let lazy = BucketMetadataSys::new(ecstore);
|
||||
let before = odm_hook_calls(bucket).len();
|
||||
let before = config_hook_calls(BUCKET_ON_DEMAND_MIGRATION_CONFIG).len();
|
||||
let (_, loaded) = lazy.get_config(bucket).await.expect("lazy load should publish");
|
||||
assert!(loaded, "the lazy path must have gone to disk");
|
||||
expect_publish(before, "lazy load");
|
||||
|
||||
// Removal withdraws the config.
|
||||
let before = odm_hook_calls(bucket).len();
|
||||
let before = config_hook_calls(BUCKET_ON_DEMAND_MIGRATION_CONFIG).len();
|
||||
assert!(lazy.remove(bucket).await);
|
||||
let calls = odm_hook_calls(bucket);
|
||||
let calls = config_hook_calls(BUCKET_ON_DEMAND_MIGRATION_CONFIG);
|
||||
assert_eq!(calls.len(), before + 1, "remove must withdraw exactly once");
|
||||
assert_eq!(calls.last().unwrap(), &None);
|
||||
|
||||
let notify_calls = config_hook_calls(BUCKET_NOTIFICATION_CONFIG);
|
||||
assert_eq!(notify_calls.len(), before + 1);
|
||||
assert_eq!(notify_calls.last(), Some(&None));
|
||||
|
||||
// Opaque bytes reach the application even if they are not valid JSON.
|
||||
let mut corrupt = BucketMetadata::new(bucket);
|
||||
corrupt.on_demand_migration_config_json = b"not-json".to_vec();
|
||||
let before = odm_hook_calls(bucket).len();
|
||||
let before = config_hook_calls(BUCKET_ON_DEMAND_MIGRATION_CONFIG).len();
|
||||
lazy.set(bucket.to_string(), Arc::new(corrupt)).await;
|
||||
let calls = odm_hook_calls(bucket);
|
||||
let calls = config_hook_calls(BUCKET_ON_DEMAND_MIGRATION_CONFIG);
|
||||
assert_eq!(calls.len(), before + 1);
|
||||
assert_eq!(
|
||||
calls.last().unwrap().as_ref().map(|(bytes, _, _)| bytes.as_slice()),
|
||||
|
||||
@@ -13,6 +13,8 @@
|
||||
// limitations under the License.
|
||||
|
||||
pub(crate) use crate::services::event_notification::EventArgs;
|
||||
#[cfg(test)]
|
||||
pub(crate) use crate::services::event_notification::test_recorder;
|
||||
|
||||
use super::runtime_boundary;
|
||||
|
||||
|
||||
@@ -89,7 +89,11 @@ impl ReplicationLifecycleBridge {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn schedule_delete(bucket: String, delete_object: DeletedObject) {
|
||||
pub(crate) async fn schedule_delete(
|
||||
bucket: String,
|
||||
delete_object: DeletedObject,
|
||||
notification_bucket_incarnation: Option<uuid::Uuid>,
|
||||
) {
|
||||
#[cfg(test)]
|
||||
{
|
||||
SCHEDULED_DELETE_OBJECTS
|
||||
@@ -98,6 +102,7 @@ impl ReplicationLifecycleBridge {
|
||||
.push(delete_object.clone());
|
||||
}
|
||||
super::replication_pool::schedule_replication_delete(DeletedObjectReplicationInfo {
|
||||
notification_bucket_incarnation,
|
||||
delete_object: deleted_object_for_replication(delete_object),
|
||||
bucket,
|
||||
event_type: REPLICATE_INCOMING_DELETE.to_string(),
|
||||
|
||||
@@ -22,6 +22,10 @@ use rustfs_utils::path::path_join_buf;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
use super::replication_error_boundary::{Error, Result};
|
||||
#[cfg(test)]
|
||||
pub(crate) use crate::bucket::metadata_sys::{
|
||||
bucket_metadata_sys_of, init_bucket_metadata_sys, set_bucket_metadata_in, test_support,
|
||||
};
|
||||
|
||||
pub(crate) type ReplicationInstanceContext = InstanceContext;
|
||||
|
||||
|
||||
@@ -110,13 +110,16 @@ impl ReplicationObjectBridge {
|
||||
check_replicate_delete_with_snapshot(object, source, opts, source_error, snapshot)
|
||||
}
|
||||
|
||||
/// Admits replication with the bucket identity verified by the originating mutation.
|
||||
/// An absent identity suppresses external notifications; workers never infer one by name.
|
||||
pub async fn schedule_object<S: ReplicationStorage>(
|
||||
object: ObjectInfo,
|
||||
storage: Arc<S>,
|
||||
decision: ReplicateDecision,
|
||||
op_type: ReplicationType,
|
||||
notification_bucket_incarnation: Option<uuid::Uuid>,
|
||||
) {
|
||||
schedule_replication(object, storage, decision, op_type).await;
|
||||
schedule_replication(object, storage, decision, op_type, notification_bucket_incarnation).await;
|
||||
}
|
||||
|
||||
pub async fn schedule_delete(delete_object: DeletedObjectReplicationInfo) {
|
||||
@@ -148,8 +151,16 @@ impl ReplicationObjectBridge {
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn schedule_storage_delete(delete_object: DeletedObject, bucket: String, event_type: String) {
|
||||
/// Schedules a committed delete with its originating bucket identity for notifications.
|
||||
/// An absent identity leaves external notifications unbound.
|
||||
pub async fn schedule_storage_delete(
|
||||
delete_object: DeletedObject,
|
||||
bucket: String,
|
||||
event_type: String,
|
||||
notification_bucket_incarnation: Option<uuid::Uuid>,
|
||||
) {
|
||||
Self::schedule_delete(DeletedObjectReplicationInfo {
|
||||
notification_bucket_incarnation,
|
||||
delete_object: deleted_object_for_replication(delete_object),
|
||||
bucket,
|
||||
event_type,
|
||||
@@ -158,10 +169,18 @@ impl ReplicationObjectBridge {
|
||||
.await;
|
||||
}
|
||||
|
||||
pub async fn schedule_storage_deletes(delete_objects: Vec<DeletedObject>, bucket: String, event_type: String) {
|
||||
/// Schedules committed deletes from one bucket instance, retaining that notification identity.
|
||||
/// An absent identity leaves external notifications unbound.
|
||||
pub async fn schedule_storage_deletes(
|
||||
delete_objects: Vec<DeletedObject>,
|
||||
bucket: String,
|
||||
event_type: String,
|
||||
notification_bucket_incarnation: Option<uuid::Uuid>,
|
||||
) {
|
||||
let delete_objects = delete_objects
|
||||
.into_iter()
|
||||
.map(|delete_object| DeletedObjectReplicationInfo {
|
||||
notification_bucket_incarnation,
|
||||
delete_object: deleted_object_for_replication(delete_object),
|
||||
bucket: bucket.clone(),
|
||||
event_type: event_type.clone(),
|
||||
|
||||
@@ -1016,7 +1016,9 @@ async fn replay_mrf_metadata_entry<S: ReplicationStorage>(
|
||||
/// admitted targets, carrying over the entry's retry count.
|
||||
fn admitted_mrf_replicate_object(oi: ObjectInfo, entry: &MrfReplicateEntry, op_type: ReplicationType) -> ReplicateObjectInfo {
|
||||
let dsc = replicate_decision_for_admitted_targets(&entry.target_arns);
|
||||
let mut roi = replicate_object_info_from_object_info(oi, dsc, op_type);
|
||||
// The journal does not bind this task to a bucket instance. Keep external
|
||||
// notifications unbound instead of attaching an old task to a reused name.
|
||||
let mut roi = replicate_object_info_from_object_info(oi, dsc, op_type, None);
|
||||
roi.retry_count = entry.retry_count.max(0) as u32;
|
||||
roi
|
||||
}
|
||||
@@ -3171,12 +3173,13 @@ pub(crate) async fn schedule_replication<S: ReplicationStorage>(
|
||||
o: Arc<S>,
|
||||
dsc: ReplicateDecision,
|
||||
op_type: ReplicationType,
|
||||
notification_bucket_incarnation: Option<uuid::Uuid>,
|
||||
) {
|
||||
let (synchronous, asynchronous) = dsc.partition_by_sync();
|
||||
let mut async_oi = oi;
|
||||
|
||||
if synchronous.replicate_any() {
|
||||
let ri = replicate_object_info_from_object_info(async_oi.clone(), synchronous, op_type);
|
||||
let ri = replicate_object_info_from_object_info(async_oi.clone(), synchronous, op_type, notification_bucket_incarnation);
|
||||
let state = replicate_object(ri, o.clone()).await;
|
||||
async_oi.replication_status_internal = state.replication_status_internal;
|
||||
async_oi.version_purge_status_internal = state.version_purge_status_internal;
|
||||
@@ -3185,7 +3188,7 @@ pub(crate) async fn schedule_replication<S: ReplicationStorage>(
|
||||
if asynchronous.replicate_any()
|
||||
&& let Some(pool) = runtime_sources::replication_pool()
|
||||
{
|
||||
let ri = replicate_object_info_from_object_info(async_oi, asynchronous, op_type);
|
||||
let ri = replicate_object_info_from_object_info(async_oi, asynchronous, op_type, notification_bucket_incarnation);
|
||||
let _ = pool.queue_replica_task(ri).await;
|
||||
}
|
||||
}
|
||||
@@ -3194,6 +3197,7 @@ fn replicate_object_info_from_object_info(
|
||||
oi: ObjectInfo,
|
||||
dsc: ReplicateDecision,
|
||||
op_type: ReplicationType,
|
||||
notification_bucket_incarnation: Option<uuid::Uuid>,
|
||||
) -> ReplicateObjectInfo {
|
||||
let tgt_statuses = replication_statuses_map(&oi.replication_status_internal.clone().unwrap_or_default());
|
||||
let purge_statuses = version_purge_statuses_map(&oi.version_purge_status_internal.clone().unwrap_or_default());
|
||||
@@ -3209,6 +3213,7 @@ fn replicate_object_info_from_object_info(
|
||||
let checksum = if ssec { oi.checksum.clone() } else { None };
|
||||
|
||||
ReplicateObjectInfo {
|
||||
notification_bucket_incarnation,
|
||||
name: oi.name,
|
||||
size: oi.size,
|
||||
actual_size: asz,
|
||||
@@ -3350,7 +3355,7 @@ pub async fn queue_replication_metadata(bucket: &str, oi: ObjectInfo, retry_coun
|
||||
return ReplicationQueueAdmission::Skipped;
|
||||
}
|
||||
|
||||
let mut roi = replicate_object_info_from_object_info(oi, dsc, ReplicationType::Metadata);
|
||||
let mut roi = replicate_object_info_from_object_info(oi, dsc, ReplicationType::Metadata, None);
|
||||
roi.retry_count = retry_count;
|
||||
if let Some(pool) = runtime_sources::replication_pool() {
|
||||
pool.queue_replica_task(roi).await
|
||||
@@ -5205,6 +5210,117 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replication_admission_keeps_original_incarnation_and_mrf_replay_does_not_guess() {
|
||||
let incarnation = Uuid::new_v4();
|
||||
let target = "arn:rustfs:replication:target-a";
|
||||
for op_type in [ReplicationType::Object, ReplicationType::Metadata] {
|
||||
let object = ObjectInfo {
|
||||
bucket: "source".to_string(),
|
||||
name: "object".to_string(),
|
||||
version_id: Some(Uuid::new_v4()),
|
||||
..Default::default()
|
||||
};
|
||||
let admitted = replicate_object_info_from_object_info(
|
||||
object.clone(),
|
||||
test_replicate_decision(&[target]),
|
||||
op_type,
|
||||
Some(incarnation),
|
||||
);
|
||||
assert_eq!(admitted.notification_bucket_incarnation, Some(incarnation));
|
||||
let encoded = encode_mrf_file(&[admitted.to_mrf_entry()]).expect("encode queued replication");
|
||||
let entries = decode_mrf_file(&encoded).expect("decode queued replication");
|
||||
let replayed = admitted_mrf_replicate_object(object, &entries[0], op_type);
|
||||
assert_eq!(
|
||||
replayed.notification_bucket_incarnation, None,
|
||||
"MRF replay must not infer a bucket instance from a reused name"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial_test::serial]
|
||||
async fn queued_delete_notifications_keep_origin_across_bucket_recreation() {
|
||||
use super::super::replication_event_sink::test_recorder;
|
||||
use super::super::replication_metadata_boundary::{
|
||||
bucket_metadata_sys_of, init_bucket_metadata_sys, set_bucket_metadata_in,
|
||||
test_support::isolated_store_over_temp_disks,
|
||||
};
|
||||
use super::super::replication_storage_boundary::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions};
|
||||
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
runtime_sources::set_test_object_layer(store.clone()).await;
|
||||
test_recorder::install();
|
||||
let bucket = "queued-delete-notification-origin";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create original bucket");
|
||||
let original = store.bucket_incarnation_id(bucket).await.expect("original incarnation");
|
||||
let queued = DeletedObjectReplicationInfo {
|
||||
notification_bucket_incarnation: Some(original),
|
||||
bucket: bucket.to_string(),
|
||||
target_arn: "arn:rustfs:replication:target-a".to_string(),
|
||||
delete_object: ReplicationDeletedObject {
|
||||
object_name: "original-object".to_string(),
|
||||
version_id: Some(Uuid::new_v4()),
|
||||
replication_state: Some(ReplicationState {
|
||||
replicate_decision_str: "malformed-decision".to_string(),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
let encoded = encode_mrf_file(&[queued.to_mrf_entry()]).expect("persist old delete task");
|
||||
let entries = decode_mrf_file(&encoded).expect("recover old delete task");
|
||||
let mut replayed = reconstructed_heal_delete_info(
|
||||
&entries[0],
|
||||
&ObjectInfo::default(),
|
||||
&replicate_decision_for_admitted_targets(&entries[0].target_arns),
|
||||
);
|
||||
replayed
|
||||
.delete_object
|
||||
.replication_state
|
||||
.as_mut()
|
||||
.expect("reconstructed replication state")
|
||||
.replicate_decision_str = "malformed-decision".to_string();
|
||||
|
||||
store
|
||||
.delete_bucket(bucket, &DeleteBucketOptions::default())
|
||||
.await
|
||||
.expect("delete original bucket");
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create replacement bucket");
|
||||
let replacement = store.bucket_incarnation_id(bucket).await.expect("replacement incarnation");
|
||||
assert_ne!(original, replacement);
|
||||
let sys = bucket_metadata_sys_of(&store.ctx).expect("metadata owner");
|
||||
let mut metadata = (*sys.read().await.get(bucket).await.expect("replacement metadata")).clone();
|
||||
metadata.notification_config_xml = br#"<NotificationConfiguration><QueueConfiguration><Id>replacement</Id><Queue>arn:rustfs:sqs:us-east-1:replacement:webhook</Queue><Event>s3:Replication:*</Event></QueueConfiguration></NotificationConfiguration>"#.to_vec();
|
||||
set_bucket_metadata_in(&store.ctx, metadata)
|
||||
.await
|
||||
.expect("install replacement notification configuration");
|
||||
|
||||
assert!(!replicate_delete_with_outcome(queued, store.clone()).await);
|
||||
assert!(!replicate_delete_with_outcome(replayed, store).await);
|
||||
let events = test_recorder::recorded_for_bucket(bucket);
|
||||
assert_eq!(events.len(), 2, "both worker failures must emit an observable local event");
|
||||
assert!(
|
||||
events
|
||||
.iter()
|
||||
.all(|event| event.object == "original-object" && event.event_name == "s3:Replication:OperationNotTracked")
|
||||
);
|
||||
assert_eq!(events[0].bucket_incarnation_id, Some(original));
|
||||
assert_eq!(
|
||||
events[1].bucket_incarnation_id, None,
|
||||
"a replay without provenance must not acquire the replacement bucket's identity"
|
||||
);
|
||||
assert!(events.iter().all(|event| event.bucket_incarnation_id != Some(replacement)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replicate_object_info_from_object_info_preserves_ssec_checksum() {
|
||||
let checksum = bytes::Bytes::from_static(b"ssec-checksum");
|
||||
@@ -5219,7 +5335,7 @@ mod tests {
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let ri = replicate_object_info_from_object_info(oi, ReplicateDecision::default(), ReplicationType::Object);
|
||||
let ri = replicate_object_info_from_object_info(oi, ReplicateDecision::default(), ReplicationType::Object, None);
|
||||
|
||||
assert!(ri.ssec);
|
||||
assert_eq!(ri.checksum, Some(checksum));
|
||||
@@ -5235,8 +5351,12 @@ mod tests {
|
||||
user_tags: Arc::new("owner=a3".to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
let live =
|
||||
replicate_object_info_from_object_info(object.clone(), test_replicate_decision(&[target]), ReplicationType::Metadata);
|
||||
let live = replicate_object_info_from_object_info(
|
||||
object.clone(),
|
||||
test_replicate_decision(&[target]),
|
||||
ReplicationType::Metadata,
|
||||
None,
|
||||
);
|
||||
let persisted = live.to_mrf_entry();
|
||||
let encoded = encode_mrf_file(std::slice::from_ref(&persisted)).expect("metadata MRF entry should encode");
|
||||
let decoded = decode_mrf_file(&encoded).expect("metadata MRF entry should decode");
|
||||
|
||||
@@ -2275,6 +2275,7 @@ pub async fn get_heal_replicate_object_info(oi: &ObjectInfo, rcfg: &ReplicationC
|
||||
let replication_generation = oi.replication_generation_snapshot();
|
||||
|
||||
Ok(ReplicateObjectInfo {
|
||||
notification_bucket_incarnation: None,
|
||||
name: oi.name.clone(),
|
||||
size: oi.size,
|
||||
actual_size,
|
||||
@@ -2449,6 +2450,7 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
|
||||
"Failed to parse replicate decision"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
bucket_incarnation_id: dobj.notification_bucket_incarnation,
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.clone(),
|
||||
object: ObjectInfo {
|
||||
@@ -2530,6 +2532,7 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
|
||||
"Failed to verify source delete-marker state before replication"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
bucket_incarnation_id: dobj.notification_bucket_incarnation,
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.clone(),
|
||||
object: ObjectInfo {
|
||||
@@ -2564,6 +2567,7 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
|
||||
"Skipping replication delete"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
bucket_incarnation_id: dobj.notification_bucket_incarnation,
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.clone(),
|
||||
object: ObjectInfo {
|
||||
@@ -2594,6 +2598,7 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
|
||||
"Skipping replication delete"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
bucket_incarnation_id: dobj.notification_bucket_incarnation,
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.clone(),
|
||||
object: ObjectInfo {
|
||||
@@ -2693,6 +2698,7 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
|
||||
"Replication resync task failed"
|
||||
);
|
||||
send_event(EventArgs {
|
||||
bucket_incarnation_id: dobj.notification_bucket_incarnation,
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.clone(),
|
||||
object: ObjectInfo {
|
||||
@@ -2801,6 +2807,7 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
|
||||
{
|
||||
Ok(object) => {
|
||||
send_event(EventArgs {
|
||||
bucket_incarnation_id: dobj.notification_bucket_incarnation,
|
||||
event_name,
|
||||
bucket_name: bucket.clone(),
|
||||
object,
|
||||
@@ -2826,6 +2833,7 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
|
||||
);
|
||||
}
|
||||
send_event(EventArgs {
|
||||
bucket_incarnation_id: dobj.notification_bucket_incarnation,
|
||||
event_name,
|
||||
bucket_name: bucket.clone(),
|
||||
object: ObjectInfo {
|
||||
@@ -3157,6 +3165,7 @@ async fn purge_stale_delete_marker_targets(bucket: &str, dobj: &DeletedObjectRep
|
||||
}
|
||||
|
||||
async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &DeletedObjectReplicationInfo, storage: Arc<S>) -> bool {
|
||||
let incarnation_id = dobj.notification_bucket_incarnation;
|
||||
let bucket = &dobj.bucket;
|
||||
let object_name = &dobj.delete_object.object_name;
|
||||
let admitted_target_arns = dobj.admitted_target_arns();
|
||||
@@ -3177,6 +3186,7 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
|
||||
"Skipping replication force-delete because replication config is missing"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
bucket_incarnation_id: incarnation_id,
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.clone(),
|
||||
object: ObjectInfo {
|
||||
@@ -3200,6 +3210,7 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
|
||||
"Skipping replication force-delete because replication config lookup failed"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
bucket_incarnation_id: incarnation_id,
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.clone(),
|
||||
object: ObjectInfo {
|
||||
@@ -3234,6 +3245,7 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
|
||||
"Skipping replication force-delete"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
bucket_incarnation_id: incarnation_id,
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.clone(),
|
||||
object: ObjectInfo {
|
||||
@@ -3262,6 +3274,7 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
|
||||
"Skipping replication force-delete"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
bucket_incarnation_id: incarnation_id,
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.clone(),
|
||||
object: ObjectInfo {
|
||||
@@ -3309,6 +3322,7 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
|
||||
"Skipping replication force-delete because target client is unavailable"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
bucket_incarnation_id: incarnation_id,
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.clone(),
|
||||
object: ObjectInfo {
|
||||
@@ -3339,6 +3353,7 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
|
||||
"Skipping replication force-delete"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
bucket_incarnation_id: incarnation_id,
|
||||
event_name: EventName::ObjectReplicationFailed.to_string(),
|
||||
bucket_name: bucket.clone(),
|
||||
object: ObjectInfo {
|
||||
@@ -3375,6 +3390,7 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
|
||||
"Replication target operation failed"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
bucket_incarnation_id: incarnation_id,
|
||||
event_name: EventName::ObjectReplicationFailed.to_string(),
|
||||
bucket_name: bucket.clone(),
|
||||
object: ObjectInfo {
|
||||
@@ -4239,6 +4255,7 @@ pub(crate) async fn replicate_object_with_outcome<S: ReplicationStorage>(
|
||||
"Skipping replication object"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
bucket_incarnation_id: roi.notification_bucket_incarnation,
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.clone(),
|
||||
object: roi.to_object_info(),
|
||||
@@ -4262,6 +4279,7 @@ pub(crate) async fn replicate_object_with_outcome<S: ReplicationStorage>(
|
||||
"Skipping replication object"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
bucket_incarnation_id: roi.notification_bucket_incarnation,
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.clone(),
|
||||
object: roi.to_object_info(),
|
||||
@@ -4328,6 +4346,7 @@ pub(crate) async fn replicate_object_with_outcome<S: ReplicationStorage>(
|
||||
"Replication resync task failed"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
bucket_incarnation_id: roi.notification_bucket_incarnation,
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.clone(),
|
||||
object: roi.to_object_info(),
|
||||
@@ -4472,6 +4491,7 @@ pub(crate) async fn replicate_object_with_outcome<S: ReplicationStorage>(
|
||||
|
||||
if publication.emit_event {
|
||||
send_local_event(EventArgs {
|
||||
bucket_incarnation_id: roi.notification_bucket_incarnation,
|
||||
event_name,
|
||||
bucket_name: bucket.clone(),
|
||||
object: object_info,
|
||||
@@ -4614,6 +4634,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
|
||||
"Skipping replication object target"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
bucket_incarnation_id: self.notification_bucket_incarnation,
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.clone(),
|
||||
object: self.to_object_info(),
|
||||
@@ -4629,6 +4650,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
|
||||
let Some(ssec_audit_required) = resolve_ssec_passthrough_gate(self.ssec, &tgt_client, &bucket, &object, &mut rinfo).await
|
||||
else {
|
||||
send_local_event(EventArgs {
|
||||
bucket_incarnation_id: self.notification_bucket_incarnation,
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.clone(),
|
||||
object: self.to_object_info(),
|
||||
@@ -4672,6 +4694,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
|
||||
);
|
||||
|
||||
send_local_event(EventArgs {
|
||||
bucket_incarnation_id: self.notification_bucket_incarnation,
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.clone(),
|
||||
object: self.to_object_info(),
|
||||
@@ -4703,6 +4726,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
|
||||
"Skipping replication object target"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
bucket_incarnation_id: self.notification_bucket_incarnation,
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.clone(),
|
||||
object: object_info,
|
||||
@@ -4728,6 +4752,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
|
||||
"Skipping replication object target"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
bucket_incarnation_id: self.notification_bucket_incarnation,
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.clone(),
|
||||
object: object_info,
|
||||
@@ -4862,6 +4887,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
|
||||
"Replication target operation failed"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
bucket_incarnation_id: self.notification_bucket_incarnation,
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.clone(),
|
||||
object: object_info,
|
||||
@@ -4890,6 +4916,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
|
||||
"Replication target operation failed"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
bucket_incarnation_id: self.notification_bucket_incarnation,
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.clone(),
|
||||
object: object_info,
|
||||
@@ -4987,6 +5014,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
|
||||
let Some(ssec_audit_required) = resolve_ssec_passthrough_gate(self.ssec, &tgt_client, &bucket, &object, &mut rinfo).await
|
||||
else {
|
||||
send_local_event(EventArgs {
|
||||
bucket_incarnation_id: self.notification_bucket_incarnation,
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.clone(),
|
||||
object: self.to_object_info(),
|
||||
@@ -5020,7 +5048,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
|
||||
let size = match object_info.get_actual_size() {
|
||||
Ok(size) => size,
|
||||
Err(e) => {
|
||||
note_replicate_all_size_unavailable(&bucket, &tgt_client, object_info, &e);
|
||||
note_replicate_all_size_unavailable(&bucket, &tgt_client, object_info, &e, self.notification_bucket_incarnation);
|
||||
return rinfo;
|
||||
}
|
||||
};
|
||||
@@ -5029,7 +5057,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
|
||||
let transfer_size = if self.ssec { object_info.size } else { size };
|
||||
|
||||
if tgt_client.bucket.is_empty() {
|
||||
note_replicate_all_target_bucket_empty(&bucket, &tgt_client, object_info);
|
||||
note_replicate_all_target_bucket_empty(&bucket, &tgt_client, object_info, self.notification_bucket_incarnation);
|
||||
return rinfo;
|
||||
}
|
||||
|
||||
@@ -5075,7 +5103,15 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
|
||||
let (put_opts, is_multipart) = match replication_put_object_options_for_target(&tgt_client, &object_info) {
|
||||
Ok((put_opts, is_mp)) => (put_opts, is_mp),
|
||||
Err(e) => {
|
||||
fail_replicate_all_put_options(&mut rinfo, &tgt_client, &bucket, object_info, &e, start_time);
|
||||
fail_replicate_all_put_options(
|
||||
&mut rinfo,
|
||||
&tgt_client,
|
||||
&bucket,
|
||||
object_info,
|
||||
&e,
|
||||
start_time,
|
||||
self.notification_bucket_incarnation,
|
||||
);
|
||||
return rinfo;
|
||||
}
|
||||
};
|
||||
@@ -5179,6 +5215,7 @@ fn note_replicate_all_target_offline(roi: &ReplicateObjectInfo, bucket: &str, tg
|
||||
"Skipped replication because target is offline"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
bucket_incarnation_id: roi.notification_bucket_incarnation,
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.to_string(),
|
||||
object: roi.to_object_info(),
|
||||
@@ -5217,6 +5254,7 @@ fn note_replicate_all_reader_unavailable(roi: &ReplicateObjectInfo, bucket: &str
|
||||
"Skipped replication because object reader is unavailable"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
bucket_incarnation_id: roi.notification_bucket_incarnation,
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.to_string(),
|
||||
object: roi.to_object_info(),
|
||||
@@ -5228,7 +5266,13 @@ fn note_replicate_all_reader_unavailable(roi: &ReplicateObjectInfo, bucket: &str
|
||||
|
||||
/// Log and notify that replication was skipped because the actual object size
|
||||
/// is unavailable.
|
||||
fn note_replicate_all_size_unavailable(bucket: &str, tgt_client: &TargetClient, object_info: ObjectInfo, e: &std::io::Error) {
|
||||
fn note_replicate_all_size_unavailable(
|
||||
bucket: &str,
|
||||
tgt_client: &TargetClient,
|
||||
object_info: ObjectInfo,
|
||||
e: &std::io::Error,
|
||||
incarnation_id: Option<uuid::Uuid>,
|
||||
) {
|
||||
debug!(
|
||||
event = EVENT_RESYNC_RUNTIME_SKIPPED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
@@ -5241,6 +5285,7 @@ fn note_replicate_all_size_unavailable(bucket: &str, tgt_client: &TargetClient,
|
||||
"Skipped replication because actual object size is unavailable"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
bucket_incarnation_id: incarnation_id,
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.to_string(),
|
||||
object: object_info,
|
||||
@@ -5251,7 +5296,12 @@ fn note_replicate_all_size_unavailable(bucket: &str, tgt_client: &TargetClient,
|
||||
|
||||
/// Log and notify that replication was skipped because the target bucket is
|
||||
/// empty.
|
||||
fn note_replicate_all_target_bucket_empty(bucket: &str, tgt_client: &TargetClient, object_info: ObjectInfo) {
|
||||
fn note_replicate_all_target_bucket_empty(
|
||||
bucket: &str,
|
||||
tgt_client: &TargetClient,
|
||||
object_info: ObjectInfo,
|
||||
incarnation_id: Option<uuid::Uuid>,
|
||||
) {
|
||||
debug!(
|
||||
event = EVENT_RESYNC_RUNTIME_SKIPPED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
@@ -5263,6 +5313,7 @@ fn note_replicate_all_target_bucket_empty(bucket: &str, tgt_client: &TargetClien
|
||||
"Skipped replication because target bucket is empty"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
bucket_incarnation_id: incarnation_id,
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.to_string(),
|
||||
object: object_info,
|
||||
@@ -5376,6 +5427,7 @@ async fn resolve_replicate_all_action(
|
||||
"Skipping replication because newer target version exists"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
bucket_incarnation_id: roi.notification_bucket_incarnation,
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.to_string(),
|
||||
object: object_info.clone(),
|
||||
@@ -5499,6 +5551,7 @@ async fn resolve_replicate_all_action(
|
||||
"Failed replication head-object fallback"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
bucket_incarnation_id: roi.notification_bucket_incarnation,
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.to_string(),
|
||||
object: object_info,
|
||||
@@ -5531,6 +5584,7 @@ async fn resolve_replicate_all_action(
|
||||
);
|
||||
|
||||
send_local_event(EventArgs {
|
||||
bucket_incarnation_id: roi.notification_bucket_incarnation,
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.to_string(),
|
||||
object: object_info,
|
||||
@@ -5559,6 +5613,7 @@ fn fail_replicate_all_put_options(
|
||||
object_info: ObjectInfo,
|
||||
e: &Error,
|
||||
start_time: OffsetDateTime,
|
||||
incarnation_id: Option<uuid::Uuid>,
|
||||
) {
|
||||
rinfo.replication_status = ReplicationStatusType::Failed;
|
||||
rinfo.error = Some(e.to_string());
|
||||
@@ -5574,6 +5629,7 @@ fn fail_replicate_all_put_options(
|
||||
"Replication target operation failed"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
bucket_incarnation_id: incarnation_id,
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.to_string(),
|
||||
object: object_info,
|
||||
|
||||
@@ -24,6 +24,8 @@ pub(crate) use crate::object_api::{
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use crate::object_api::{NamespaceLockFence, NamespaceLockSignalTestFence};
|
||||
#[cfg(test)]
|
||||
pub(crate) use crate::storage_api_contracts::bucket::{BucketOperations, DeleteBucketOptions, MakeBucketOptions};
|
||||
pub(crate) use crate::storage_api_contracts::list::{
|
||||
ListOperations, StorageListObjectVersionsInfo, StorageListObjectsV2Info, StorageObjectInfoOrErr, StorageWalkOptions,
|
||||
};
|
||||
|
||||
@@ -18,6 +18,8 @@ use super::replication_bandwidth_boundary::ReplicationBucketMonitor;
|
||||
use super::replication_pool::DynReplicationPool;
|
||||
use super::replication_state::ReplicationStats;
|
||||
use super::replication_storage_boundary::ReplicationObjectStore;
|
||||
#[cfg(test)]
|
||||
pub(crate) use crate::runtime::global::set_object_layer as set_test_object_layer;
|
||||
use crate::runtime::sources;
|
||||
pub use crate::runtime::sources::{
|
||||
ScannerDirtyUsageMutationObserver, ScannerDirtyUsageMutationSource, set_scanner_dirty_usage_mutation_observer,
|
||||
|
||||
@@ -82,6 +82,8 @@ impl EventNotifier {
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct EventArgs {
|
||||
/// Bucket instance verified by storage when producing this event.
|
||||
pub bucket_incarnation_id: Option<uuid::Uuid>,
|
||||
pub event_name: String,
|
||||
pub bucket_name: String,
|
||||
pub object: ObjectInfo,
|
||||
@@ -135,6 +137,7 @@ pub(crate) mod test_recorder {
|
||||
#[derive(Clone, Debug)]
|
||||
pub(crate) struct RecordedEvent {
|
||||
pub(crate) event_name: String,
|
||||
pub(crate) bucket_incarnation_id: Option<Uuid>,
|
||||
pub(crate) bucket: String,
|
||||
pub(crate) object: String,
|
||||
pub(crate) version_id: Option<Uuid>,
|
||||
@@ -154,6 +157,7 @@ pub(crate) mod test_recorder {
|
||||
register_event_dispatch_hook(|args| {
|
||||
recorded().lock().unwrap_or_else(|err| err.into_inner()).push(RecordedEvent {
|
||||
event_name: args.event_name,
|
||||
bucket_incarnation_id: args.bucket_incarnation_id,
|
||||
bucket: args.bucket_name,
|
||||
object: args.object.name,
|
||||
version_id: args.object.version_id,
|
||||
|
||||
@@ -9294,7 +9294,12 @@ impl SetDisks {
|
||||
)
|
||||
.await?;
|
||||
if let Some((_, deleted_object)) = replication_delete {
|
||||
ReplicationLifecycleBridge::schedule_delete(bucket.to_string(), deleted_object).await;
|
||||
ReplicationLifecycleBridge::schedule_delete(
|
||||
bucket.to_string(),
|
||||
deleted_object,
|
||||
opts.expected_bucket_incarnation_id,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
deleted += 1;
|
||||
}
|
||||
@@ -11563,7 +11568,12 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
)
|
||||
.await?;
|
||||
if let Some((_, deleted_object)) = replication_delete {
|
||||
ReplicationLifecycleBridge::schedule_delete(bucket.to_string(), deleted_object).await;
|
||||
ReplicationLifecycleBridge::schedule_delete(
|
||||
bucket.to_string(),
|
||||
deleted_object,
|
||||
opts.expected_bucket_incarnation_id,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
@@ -12732,6 +12742,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
let obj_info = ObjectInfo::from_file_info(&fi, bucket, object, opts.versioned || opts.version_suspended);
|
||||
send_event(EventArgs {
|
||||
event_name: event_name.to_string(),
|
||||
bucket_incarnation_id: opts.expected_bucket_incarnation_id,
|
||||
bucket_name: bucket.to_string(),
|
||||
object: obj_info,
|
||||
user_agent: "Internal: [ILM-Transition]".to_string(),
|
||||
@@ -12878,6 +12889,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
let restored_info = self_.finalize_restore_metadata(bucket, object, &restored_info, &opts).await?;
|
||||
send_event(EventArgs {
|
||||
event_name: EventName::ObjectRestoreCompleted.as_str().to_string(),
|
||||
bucket_incarnation_id: opts.expected_bucket_incarnation_id,
|
||||
bucket_name: bucket.to_string(),
|
||||
object: restored_info,
|
||||
user_agent: "Internal: [Restore-Completed]".to_string(),
|
||||
@@ -13014,6 +13026,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
let restored_info = self_.finalize_restore_metadata(bucket, object, &restored_info, opts).await?;
|
||||
send_event(EventArgs {
|
||||
event_name: EventName::ObjectRestoreCompleted.as_str().to_string(),
|
||||
bucket_incarnation_id: opts.expected_bucket_incarnation_id,
|
||||
bucket_name: bucket.to_string(),
|
||||
object: restored_info,
|
||||
user_agent: "Internal: [Restore-Completed]".to_string(),
|
||||
|
||||
@@ -74,6 +74,7 @@ jiff = { workspace = true, features = ["serde"] }
|
||||
percent-encoding = { workspace = true }
|
||||
rayon = { workspace = true }
|
||||
rustc-hash = { workspace = true }
|
||||
uuid = { workspace = true }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
starshard = { workspace = true, features = ["rayon", "async", "serde"] }
|
||||
thiserror = { workspace = true }
|
||||
|
||||
@@ -13,9 +13,8 @@
|
||||
// limitations under the License.
|
||||
|
||||
use crate::{
|
||||
BucketNotificationConfig, NotificationError, config_manager::notify_configuration_hint,
|
||||
notification_system_subscriber::NotificationSystemSubscriberView, notifier::EventNotifier, rule_engine::NotifyRuleEngine,
|
||||
rules::ParseConfigError,
|
||||
BucketNotificationConfig, NotificationError, config_manager::notify_configuration_hint, notifier::EventNotifier,
|
||||
rule_engine::NotifyRuleEngine, rules::ParseConfigError,
|
||||
};
|
||||
use rustfs_s3_types::EventName;
|
||||
use std::sync::Arc;
|
||||
@@ -30,26 +29,14 @@ const EVENT_NOTIFY_BUCKET_CONFIG_LOADED: &str = "notify_bucket_config_loaded";
|
||||
pub struct NotifyBucketConfigManager {
|
||||
notifier: Arc<EventNotifier>,
|
||||
rule_engine: NotifyRuleEngine,
|
||||
subscriber_view: Arc<NotificationSystemSubscriberView>,
|
||||
}
|
||||
|
||||
impl NotifyBucketConfigManager {
|
||||
pub fn new(
|
||||
notifier: Arc<EventNotifier>,
|
||||
rule_engine: NotifyRuleEngine,
|
||||
subscriber_view: Arc<NotificationSystemSubscriberView>,
|
||||
) -> Self {
|
||||
Self {
|
||||
notifier,
|
||||
rule_engine,
|
||||
subscriber_view,
|
||||
}
|
||||
pub fn new(notifier: Arc<EventNotifier>, rule_engine: NotifyRuleEngine) -> Self {
|
||||
Self { notifier, rule_engine }
|
||||
}
|
||||
|
||||
pub async fn has_subscriber(&self, bucket: &str, event: &EventName) -> bool {
|
||||
if !self.subscriber_view.has_subscriber(bucket, event) {
|
||||
return false;
|
||||
}
|
||||
self.rule_engine.has_subscriber(bucket, event).await
|
||||
}
|
||||
|
||||
@@ -105,7 +92,6 @@ impl NotifyBucketConfigManager {
|
||||
|
||||
/// Publishes an already validated `cfg` to the runtime rule state.
|
||||
pub async fn apply_bucket_notification_config(&self, bucket: &str, cfg: &BucketNotificationConfig) {
|
||||
self.subscriber_view.apply_bucket_config(bucket, cfg);
|
||||
self.rule_engine.set_bucket_rules(bucket, cfg.get_rules_map().clone()).await;
|
||||
info!(
|
||||
event = EVENT_NOTIFY_BUCKET_CONFIG_LOADED,
|
||||
@@ -128,8 +114,16 @@ impl NotifyBucketConfigManager {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn publish_bucket_notification_config(&self, bucket: &str, cfg: Option<(uuid::Uuid, &BucketNotificationConfig)>) {
|
||||
if let Some((incarnation_id, cfg)) = cfg {
|
||||
self.rule_engine
|
||||
.publish_bucket_rules(bucket, Some(incarnation_id), cfg.get_rules_map().clone());
|
||||
} else {
|
||||
self.rule_engine.withdraw_bucket_rules(bucket);
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn remove_bucket_notification_config(&self, bucket: &str) {
|
||||
self.subscriber_view.clear_bucket(bucket);
|
||||
self.rule_engine.clear_bucket_rules(bucket).await;
|
||||
}
|
||||
}
|
||||
@@ -138,8 +132,8 @@ impl NotifyBucketConfigManager {
|
||||
mod tests {
|
||||
use super::NotifyBucketConfigManager;
|
||||
use crate::{
|
||||
BucketNotificationConfig, NotificationError, integration::NotificationMetrics,
|
||||
notification_system_subscriber::NotificationSystemSubscriberView, notifier::EventNotifier, rule_engine::NotifyRuleEngine,
|
||||
BucketNotificationConfig, NotificationError, integration::NotificationMetrics, notifier::EventNotifier,
|
||||
rule_engine::NotifyRuleEngine,
|
||||
};
|
||||
use rustfs_s3_types::EventName;
|
||||
use rustfs_targets::arn::TargetID;
|
||||
@@ -149,8 +143,7 @@ mod tests {
|
||||
let metrics = Arc::new(NotificationMetrics::new());
|
||||
let rule_engine = NotifyRuleEngine::new();
|
||||
let notifier = Arc::new(EventNotifier::new(metrics, rule_engine.clone()));
|
||||
let subscriber_view = Arc::new(NotificationSystemSubscriberView::new());
|
||||
NotifyBucketConfigManager::new(notifier, rule_engine, subscriber_view)
|
||||
NotifyBucketConfigManager::new(notifier, rule_engine)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -166,11 +159,11 @@ mod tests {
|
||||
let mut cfg = BucketNotificationConfig::new("us-east-1");
|
||||
cfg.add_rule(&[EventName::ObjectCreatedPut], "*".to_string(), target_id);
|
||||
|
||||
manager.subscriber_view.apply_bucket_config("bucket", &cfg);
|
||||
assert!(manager.subscriber_view.has_subscriber("bucket", &EventName::ObjectCreatedPut));
|
||||
manager.apply_bucket_notification_config("bucket", &cfg).await;
|
||||
assert!(manager.has_subscriber("bucket", &EventName::ObjectCreatedPut).await);
|
||||
|
||||
manager.remove_bucket_notification_config("bucket").await;
|
||||
assert!(!manager.subscriber_view.has_subscriber("bucket", &EventName::ObjectCreatedPut));
|
||||
assert!(!manager.has_subscriber("bucket", &EventName::ObjectCreatedPut).await);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -186,7 +179,6 @@ mod tests {
|
||||
.expect_err("validation must fail when the runtime has no notify targets");
|
||||
assert!(matches!(err, NotificationError::Configuration(_)));
|
||||
assert!(!manager.has_subscriber("bucket", &EventName::ObjectCreatedPut).await);
|
||||
assert!(!manager.subscriber_view.has_subscriber("bucket", &EventName::ObjectCreatedPut));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::rules::TargetIdSet;
|
||||
use hashbrown::HashMap;
|
||||
use jiff::{Timestamp, tz::TimeZone};
|
||||
use rustfs_s3_ops::is_object_removed_event;
|
||||
@@ -170,6 +171,10 @@ pub struct Event {
|
||||
pub glacier_event_data: Option<GlacierEventData>,
|
||||
/// Information about the source of the event
|
||||
pub source: Source,
|
||||
#[serde(skip)]
|
||||
pub(crate) bucket_incarnation_id: Option<uuid::Uuid>,
|
||||
#[serde(skip)]
|
||||
pub(crate) bucket_removed_targets: Option<TargetIdSet>,
|
||||
}
|
||||
|
||||
impl Event {
|
||||
@@ -195,6 +200,8 @@ impl Event {
|
||||
user_metadata.insert("x-request-time".to_string(), format_timestamp_rfc3339_millis(&Timestamp::now()));
|
||||
|
||||
Event {
|
||||
bucket_incarnation_id: None,
|
||||
bucket_removed_targets: None,
|
||||
event_version: event_schema_version(event_name).to_string(),
|
||||
event_source: "rustfs:s3".to_string(),
|
||||
aws_region: "us-east-1".to_string(),
|
||||
@@ -320,6 +327,8 @@ impl Event {
|
||||
};
|
||||
|
||||
Self {
|
||||
bucket_incarnation_id: args.bucket_incarnation_id,
|
||||
bucket_removed_targets: args.bucket_removed_targets,
|
||||
event_version: event_schema_version(args.event_name).to_string(),
|
||||
event_source: "rustfs:s3".to_string(),
|
||||
aws_region: args.req_params.get("region").cloned().unwrap_or_default(),
|
||||
@@ -387,6 +396,10 @@ fn initialize_response_elements(elements: &mut HashMap<String, String>, keys: &[
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct EventArgs {
|
||||
/// Bucket identity established by the producing operation.
|
||||
pub bucket_incarnation_id: Option<uuid::Uuid>,
|
||||
/// Delivery targets captured before a successful bucket removal withdraws its rules.
|
||||
pub bucket_removed_targets: Option<TargetIdSet>,
|
||||
pub event_name: EventName,
|
||||
pub bucket_name: String,
|
||||
pub object: NotifyObjectInfo,
|
||||
@@ -436,6 +449,8 @@ impl EventArgs {
|
||||
/// ```
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct EventArgsBuilder {
|
||||
bucket_incarnation_id: Option<uuid::Uuid>,
|
||||
bucket_removed_targets: Option<TargetIdSet>,
|
||||
event_name: EventName,
|
||||
bucket_name: String,
|
||||
object: NotifyObjectInfo,
|
||||
@@ -458,6 +473,18 @@ impl EventArgsBuilder {
|
||||
}
|
||||
}
|
||||
|
||||
/// Carries the operation's confirmed bucket identity to event dispatch.
|
||||
pub fn bucket_incarnation_id(mut self, incarnation_id: Option<uuid::Uuid>) -> Self {
|
||||
self.bucket_incarnation_id = incarnation_id;
|
||||
self
|
||||
}
|
||||
|
||||
/// Uses the subscription targets captured before deletion.
|
||||
pub fn bucket_removed_targets(mut self, targets: TargetIdSet) -> Self {
|
||||
self.bucket_removed_targets = Some(targets);
|
||||
self
|
||||
}
|
||||
|
||||
/// Sets the event name.
|
||||
pub fn event_name(mut self, event_name: EventName) -> Self {
|
||||
self.event_name = event_name;
|
||||
@@ -529,6 +556,8 @@ impl EventArgsBuilder {
|
||||
/// This method consumes the builder and returns the constructed `EventArgs`.
|
||||
pub fn build(self) -> EventArgs {
|
||||
EventArgs {
|
||||
bucket_incarnation_id: self.bucket_incarnation_id,
|
||||
bucket_removed_targets: self.bucket_removed_targets,
|
||||
event_name: self.event_name,
|
||||
bucket_name: self.bucket_name,
|
||||
object: self.object,
|
||||
@@ -823,6 +852,8 @@ mod event_args_tests {
|
||||
req_params.insert((*k).to_string(), (*v).to_string());
|
||||
}
|
||||
EventArgs {
|
||||
bucket_incarnation_id: None,
|
||||
bucket_removed_targets: None,
|
||||
event_name: EventName::ObjectRemovedDelete,
|
||||
bucket_name: "b".to_string(),
|
||||
object: ObjectInfo::default(),
|
||||
|
||||
@@ -239,7 +239,8 @@ pub mod notifier_global {
|
||||
.await
|
||||
}
|
||||
|
||||
fn bucket_notification_config_for_rules(
|
||||
/// Compiles event rules into the bucket configuration used by validation and publication.
|
||||
pub fn bucket_notification_config_for_rules(
|
||||
region: &str,
|
||||
event_rules: &[(Vec<EventName>, String, String, Vec<TargetID>)],
|
||||
) -> BucketNotificationConfig {
|
||||
|
||||
@@ -12,7 +12,6 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::notification_system_subscriber::NotificationSystemSubscriberView;
|
||||
use crate::notifier::{EventNotifier, TargetList};
|
||||
use crate::services::NotifyServices;
|
||||
use crate::{
|
||||
@@ -182,7 +181,6 @@ impl NotificationSystem {
|
||||
rustfs_utils::get_env_usize(ENV_NOTIFY_TARGET_STREAM_CONCURRENCY, DEFAULT_NOTIFY_TARGET_STREAM_CONCURRENCY);
|
||||
let (live_event_sender, _) = broadcast::channel(1024);
|
||||
let metrics = Arc::new(NotificationMetrics::new());
|
||||
let subscriber_view = Arc::new(NotificationSystemSubscriberView::new());
|
||||
let rule_engine = NotifyRuleEngine::new();
|
||||
let notifier = Arc::new(EventNotifier::new(metrics.clone(), rule_engine.clone()));
|
||||
let target_list = notifier.target_list();
|
||||
@@ -200,7 +198,6 @@ impl NotificationSystem {
|
||||
stream_cancellers,
|
||||
concurrency_limiter,
|
||||
metrics,
|
||||
subscriber_view,
|
||||
live_event_sender,
|
||||
live_event_history,
|
||||
);
|
||||
@@ -413,6 +410,13 @@ impl NotificationSystem {
|
||||
.await
|
||||
}
|
||||
|
||||
/// Applies the current committed bucket rules while the caller holds the metadata publication fence.
|
||||
pub fn publish_bucket_notification_config(&self, bucket: &str, cfg: Option<(uuid::Uuid, &BucketNotificationConfig)>) {
|
||||
self.services
|
||||
.bucket_config_manager
|
||||
.publish_bucket_notification_config(bucket, cfg);
|
||||
}
|
||||
|
||||
/// Sends an event
|
||||
pub async fn send_event(&self, event: Arc<Event>) {
|
||||
self.services.pipeline.send_event(event).await;
|
||||
|
||||
@@ -26,7 +26,6 @@ pub mod factory;
|
||||
mod global;
|
||||
pub mod integration;
|
||||
mod lifecycle;
|
||||
mod notification_system_subscriber;
|
||||
pub mod notifier;
|
||||
mod pipeline;
|
||||
pub mod registry;
|
||||
|
||||
@@ -1,74 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::BucketNotificationConfig;
|
||||
use crate::rules::{BucketRulesSnapshot, DynRulesContainer, SubscriberIndex};
|
||||
use rustfs_s3_types::EventName;
|
||||
|
||||
/// NotificationSystemSubscriberView - Provides an interface to manage and query
|
||||
/// the subscription status of buckets in the notification system.
|
||||
#[derive(Debug)]
|
||||
pub struct NotificationSystemSubscriberView {
|
||||
index: SubscriberIndex,
|
||||
}
|
||||
|
||||
impl NotificationSystemSubscriberView {
|
||||
/// Creates a new NotificationSystemSubscriberView with an empty SubscriberIndex.
|
||||
///
|
||||
/// Returns a new instance of NotificationSystemSubscriberView.
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
index: SubscriberIndex::default(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Checks if a bucket has any subscribers for a specific event.
|
||||
/// This is a quick check using the event mask in the snapshot.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `bucket` - The name of the bucket to check.
|
||||
/// * `event` - The event name to check for subscriptions.
|
||||
///
|
||||
/// Returns `true` if there are subscribers for the event, `false` otherwise.
|
||||
#[inline]
|
||||
pub fn has_subscriber(&self, bucket: &str, event: &EventName) -> bool {
|
||||
self.index.has_subscriber(bucket, event)
|
||||
}
|
||||
|
||||
/// Builds and atomically replaces a bucket's subscription snapshot from the configuration.
|
||||
///
|
||||
/// Core principle: masks and rules are calculated and stored together in the same update.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `bucket` - The name of the bucket to update.
|
||||
/// * `cfg` - The bucket notification configuration to compile into a snapshot.
|
||||
pub fn apply_bucket_config(&self, bucket: &str, cfg: &BucketNotificationConfig) {
|
||||
// *It is recommended to merge compile into one function to ensure the same origin.
|
||||
let snapshot: BucketRulesSnapshot<DynRulesContainer> = cfg.compile_snapshot();
|
||||
|
||||
// *debug to prevent inconsistencies from being introduced when modifying the compile logic in the future.
|
||||
snapshot.debug_assert_mask_consistent();
|
||||
|
||||
self.index.store_snapshot(bucket, snapshot);
|
||||
}
|
||||
|
||||
/// Clears a bucket's subscription snapshot.
|
||||
///
|
||||
/// #Arguments
|
||||
/// * `bucket` - The name of the bucket to clear.
|
||||
#[inline]
|
||||
pub fn clear_bucket(&self, bucket: &str) {
|
||||
self.index.clear_bucket(bucket);
|
||||
}
|
||||
}
|
||||
@@ -13,8 +13,9 @@
|
||||
// limitations under the License.
|
||||
|
||||
use crate::error::NotificationError;
|
||||
use crate::{event::Event, integration::NotificationMetrics, rule_engine::NotifyRuleEngine};
|
||||
use crate::{event::Event, integration::NotificationMetrics, rule_engine::NotifyRuleEngine, rules::TargetIdSet};
|
||||
use rustfs_config::notify::{DEFAULT_NOTIFY_SEND_CONCURRENCY, ENV_NOTIFY_SEND_CONCURRENCY};
|
||||
use rustfs_s3_types::EventName;
|
||||
use rustfs_targets::Target;
|
||||
use rustfs_targets::arn::TargetID;
|
||||
use rustfs_targets::target::EntityTarget;
|
||||
@@ -235,6 +236,12 @@ impl EventNotifier {
|
||||
);
|
||||
}
|
||||
|
||||
/// Captures the deleted bucket's final delivery targets before its rules are withdrawn.
|
||||
pub fn bucket_removed_targets(&self, bucket: &str, incarnation_id: uuid::Uuid) -> TargetIdSet {
|
||||
self.rule_engine
|
||||
.match_bucket_targets(bucket, Some(incarnation_id), EventName::BucketRemoved, "")
|
||||
}
|
||||
|
||||
/// Sends an event to the appropriate targets based on the bucket rules
|
||||
///
|
||||
/// # Arguments
|
||||
@@ -245,7 +252,25 @@ impl EventNotifier {
|
||||
let object_key = &event.s3.object.key;
|
||||
let event_name = event.event_name;
|
||||
|
||||
let target_ids = self.rule_engine.match_targets(bucket_name, event_name, object_key).await;
|
||||
let target_ids = match &event.bucket_removed_targets {
|
||||
Some(targets) if event_name == EventName::BucketRemoved => targets.clone(),
|
||||
_ => {
|
||||
let match_targets = || {
|
||||
self.rule_engine
|
||||
.match_bucket_targets(bucket_name, event.bucket_incarnation_id, event_name, object_key)
|
||||
};
|
||||
let targets = match_targets();
|
||||
if targets.is_empty()
|
||||
&& let Some(id) = event.bucket_incarnation_id
|
||||
&& self.rule_engine.request_incarnation_refresh(bucket_name, id)
|
||||
&& crate::storage_api::refresh_notify_bucket_metadata(bucket_name).await
|
||||
{
|
||||
match_targets()
|
||||
} else {
|
||||
targets
|
||||
}
|
||||
}
|
||||
};
|
||||
if target_ids.is_empty() {
|
||||
debug!(
|
||||
event = EVENT_NOTIFY_DISPATCH_SKIPPED,
|
||||
@@ -1110,6 +1135,46 @@ mod tests {
|
||||
assert_eq!(disabled_target.save_call_count(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bucket_removal_targets_survive_withdrawal_without_routing_recreated_objects() {
|
||||
let engine = NotifyRuleEngine::new();
|
||||
let notifier = EventNotifier::new(Arc::new(NotificationMetrics::new()), engine.clone());
|
||||
let target = MockTarget::new("removed-bucket", "webhook");
|
||||
let old_id = uuid::Uuid::new_v4();
|
||||
let new_id = uuid::Uuid::new_v4();
|
||||
let mut rules = RulesMap::new();
|
||||
rules.add_rule_config(
|
||||
&[EventName::BucketRemoved, EventName::ObjectCreatedPut],
|
||||
"*".to_owned(),
|
||||
target.target_id(),
|
||||
);
|
||||
engine.publish_bucket_rules("bucket", Some(old_id), rules);
|
||||
notifier
|
||||
.target_list()
|
||||
.write()
|
||||
.await
|
||||
.add(Arc::new(target.clone()))
|
||||
.expect("register target");
|
||||
|
||||
let targets = notifier.bucket_removed_targets("bucket", old_id);
|
||||
assert!(targets.contains(&target.target_id()));
|
||||
engine.withdraw_bucket_rules("bucket");
|
||||
|
||||
let mut removed = Event::new_test_event("bucket", "", EventName::BucketRemoved);
|
||||
removed.bucket_incarnation_id = Some(old_id);
|
||||
removed.bucket_removed_targets = Some(targets);
|
||||
let encoded = serde_json::to_value(&removed).expect("serialize webhook event");
|
||||
assert!(encoded.get("bucketIncarnationId").is_none());
|
||||
assert!(encoded.get("bucketRemovedTargets").is_none());
|
||||
notifier.send(Arc::new(removed)).await;
|
||||
assert_eq!(target.save_call_count(), 1, "the deleted bucket earns its final removal event");
|
||||
|
||||
let mut created = Event::new_test_event("bucket", "fresh.txt", EventName::ObjectCreatedPut);
|
||||
created.bucket_incarnation_id = Some(new_id);
|
||||
notifier.send(Arc::new(created)).await;
|
||||
assert_eq!(target.save_call_count(), 1, "the new bucket has no subscription");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn send_event_respects_prefix_suffix_filters() {
|
||||
let rule_engine = NotifyRuleEngine::new();
|
||||
|
||||
@@ -16,9 +16,13 @@ use crate::rules::{RulesMap, TargetIdSet};
|
||||
use percent_encoding::percent_decode_str;
|
||||
use rustfs_s3_types::EventName;
|
||||
use rustfs_targets::arn::TargetID;
|
||||
use starshard::{AsyncShardedHashMap, DEFAULT_SHARDS, SnapshotMode};
|
||||
use std::sync::Arc;
|
||||
use starshard::{DEFAULT_SHARDS, ShardedHashMap, SnapshotMode};
|
||||
use std::sync::{
|
||||
Arc,
|
||||
atomic::{AtomicBool, Ordering},
|
||||
};
|
||||
use tracing::info;
|
||||
use uuid::Uuid;
|
||||
|
||||
const LOG_COMPONENT_NOTIFY: &str = "notify";
|
||||
const LOG_SUBSYSTEM_RULE_ENGINE: &str = "rule_engine";
|
||||
@@ -33,34 +37,56 @@ fn decoded_object_key_for_matching(object_key: &str) -> Option<String> {
|
||||
(decoded != object_key).then(|| decoded.into_owned())
|
||||
}
|
||||
|
||||
struct BucketRules {
|
||||
incarnation_id: Option<Uuid>,
|
||||
rules: RulesMap,
|
||||
refresh_requested: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct NotifyRuleEngine {
|
||||
bucket_rules_map: Arc<AsyncShardedHashMap<String, RulesMap, rustc_hash::FxBuildHasher>>,
|
||||
bucket_rules_map: Arc<ShardedHashMap<String, Arc<BucketRules>, rustc_hash::FxBuildHasher>>,
|
||||
}
|
||||
|
||||
impl NotifyRuleEngine {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
bucket_rules_map: Arc::new(AsyncShardedHashMap::with_snapshot_mode(DEFAULT_SHARDS, SnapshotMode::Cached)),
|
||||
bucket_rules_map: Arc::new(ShardedHashMap::with_snapshot_mode(DEFAULT_SHARDS, SnapshotMode::Cached)),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn is_target_bound_to_any_bucket(&self, target_id: &TargetID) -> bool {
|
||||
let items = self.bucket_rules_map.iter().await;
|
||||
for (_bucket, rules_map) in items {
|
||||
if rules_map.contains_target_id(target_id) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
false
|
||||
self.bucket_rules_map
|
||||
.iter()
|
||||
.any(|(_, snapshot)| snapshot.rules.contains_target_id(target_id))
|
||||
}
|
||||
|
||||
pub async fn set_bucket_rules(&self, bucket: &str, rules_map: RulesMap) {
|
||||
self.publish_bucket_rules(bucket, None, rules_map);
|
||||
}
|
||||
|
||||
/// Atomically publishes the rules and the identity used by event dispatch.
|
||||
pub(crate) fn publish_bucket_rules(&self, bucket: &str, incarnation_id: Option<Uuid>, rules_map: RulesMap) {
|
||||
let event_count = rules_map.iter_events().count();
|
||||
let bucket_key = bucket.to_owned();
|
||||
if rules_map.is_empty() {
|
||||
self.bucket_rules_map.remove(&bucket.to_string()).await;
|
||||
self.bucket_rules_map.remove(&bucket_key);
|
||||
} else {
|
||||
self.bucket_rules_map.insert(bucket.to_string(), rules_map).await;
|
||||
// A reload that confirms the current incarnation must not re-arm
|
||||
// the refresh triggered by delayed events from an older bucket.
|
||||
let refresh_requested = self
|
||||
.bucket_rules_map
|
||||
.get(&bucket_key)
|
||||
.filter(|current| current.incarnation_id == incarnation_id)
|
||||
.map_or_else(|| Arc::new(AtomicBool::new(false)), |current| current.refresh_requested.clone());
|
||||
self.bucket_rules_map.insert(
|
||||
bucket_key,
|
||||
Arc::new(BucketRules {
|
||||
incarnation_id,
|
||||
rules: rules_map,
|
||||
refresh_requested,
|
||||
}),
|
||||
);
|
||||
}
|
||||
info!(
|
||||
event = EVENT_NOTIFY_RULES_UPDATED,
|
||||
@@ -74,11 +100,17 @@ impl NotifyRuleEngine {
|
||||
}
|
||||
|
||||
pub async fn get_bucket_rules(&self, bucket: &str) -> Option<RulesMap> {
|
||||
self.bucket_rules_map.get(&bucket.to_string()).await
|
||||
self.bucket_rules_map
|
||||
.get(&bucket.to_string())
|
||||
.map(|snapshot| snapshot.rules.clone())
|
||||
}
|
||||
|
||||
pub async fn clear_bucket_rules(&self, bucket: &str) {
|
||||
if self.bucket_rules_map.remove(&bucket.to_string()).await.is_some() {
|
||||
self.withdraw_bucket_rules(bucket);
|
||||
}
|
||||
|
||||
pub(crate) fn withdraw_bucket_rules(&self, bucket: &str) {
|
||||
if self.bucket_rules_map.remove(&bucket.to_string()).is_some() {
|
||||
info!(
|
||||
event = EVENT_NOTIFY_RULES_UPDATED,
|
||||
component = LOG_COMPONENT_NOTIFY,
|
||||
@@ -91,21 +123,43 @@ impl NotifyRuleEngine {
|
||||
}
|
||||
|
||||
pub async fn has_subscriber(&self, bucket: &str, event: &EventName) -> bool {
|
||||
self.get_bucket_rules(bucket)
|
||||
.await
|
||||
.is_some_and(|rules_map| rules_map.has_subscriber(event))
|
||||
self.bucket_rules_map
|
||||
.get(&bucket.to_string())
|
||||
.is_some_and(|snapshot| snapshot.rules.has_subscriber(event))
|
||||
}
|
||||
|
||||
pub async fn match_targets(&self, bucket: &str, event_name: EventName, object_key: &str) -> TargetIdSet {
|
||||
self.get_bucket_rules(bucket)
|
||||
.await
|
||||
.map_or_else(TargetIdSet::new, |rules_map| {
|
||||
let mut target_ids = rules_map.match_rules(event_name, object_key);
|
||||
if let Some(decoded_key) = decoded_object_key_for_matching(object_key) {
|
||||
target_ids.extend(rules_map.match_rules(event_name, &decoded_key));
|
||||
}
|
||||
target_ids
|
||||
})
|
||||
self.match_bucket_targets(bucket, None, event_name, object_key)
|
||||
}
|
||||
|
||||
/// Allow one targeted reload per installed bucket incarnation. The ordinary
|
||||
/// metadata refresh loop can still recover a failed or interrupted reload.
|
||||
pub(crate) fn request_incarnation_refresh(&self, bucket: &str, incarnation_id: Uuid) -> bool {
|
||||
self.bucket_rules_map.get(&bucket.to_string()).is_some_and(|snapshot| {
|
||||
snapshot.incarnation_id.is_some_and(|current| current != incarnation_id)
|
||||
&& !snapshot.refresh_requested.swap(true, Ordering::Relaxed)
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn match_bucket_targets(
|
||||
&self,
|
||||
bucket: &str,
|
||||
incarnation_id: Option<Uuid>,
|
||||
event_name: EventName,
|
||||
object_key: &str,
|
||||
) -> TargetIdSet {
|
||||
let Some(snapshot) = self
|
||||
.bucket_rules_map
|
||||
.get(&bucket.to_string())
|
||||
.filter(|snapshot| snapshot.incarnation_id == incarnation_id)
|
||||
else {
|
||||
return TargetIdSet::new();
|
||||
};
|
||||
let mut targets = snapshot.rules.match_rules(event_name, object_key);
|
||||
if let Some(decoded_key) = decoded_object_key_for_matching(object_key) {
|
||||
targets.extend(snapshot.rules.match_rules(event_name, &decoded_key));
|
||||
}
|
||||
targets
|
||||
}
|
||||
}
|
||||
|
||||
@@ -122,6 +176,35 @@ mod tests {
|
||||
use rustfs_s3_types::EventName;
|
||||
use rustfs_targets::arn::TargetID;
|
||||
|
||||
#[test]
|
||||
fn bucket_rules_route_events_only_with_the_matching_incarnation() {
|
||||
let engine = NotifyRuleEngine::new();
|
||||
let old_id = uuid::Uuid::new_v4();
|
||||
let new_id = uuid::Uuid::new_v4();
|
||||
let old_target = TargetID::new("old".to_string(), "webhook".to_string());
|
||||
let new_target = TargetID::new("new".to_string(), "webhook".to_string());
|
||||
let rules = |target: TargetID| {
|
||||
let mut rules = RulesMap::new();
|
||||
rules.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_owned(), target);
|
||||
rules
|
||||
};
|
||||
let matched = |id| engine.match_bucket_targets("bucket", id, EventName::ObjectCreatedPut, "object");
|
||||
|
||||
engine.publish_bucket_rules("bucket", Some(old_id), rules(old_target.clone()));
|
||||
assert!(matched(Some(old_id)).contains(&old_target));
|
||||
assert!(matched(Some(new_id)).is_empty(), "a peer may still have the old rules");
|
||||
assert!(engine.request_incarnation_refresh("bucket", new_id));
|
||||
assert!(!engine.request_incarnation_refresh("bucket", new_id));
|
||||
assert!(matched(None).is_empty());
|
||||
|
||||
engine.withdraw_bucket_rules("bucket");
|
||||
assert!(matched(Some(old_id)).is_empty());
|
||||
assert!(matched(Some(new_id)).is_empty());
|
||||
engine.publish_bucket_rules("bucket", Some(new_id), rules(new_target.clone()));
|
||||
assert!(matched(Some(old_id)).is_empty());
|
||||
assert!(matched(Some(new_id)).contains(&new_target));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rule_engine_tracks_bucket_rule_lifecycle() {
|
||||
let engine = NotifyRuleEngine::new();
|
||||
|
||||
@@ -17,7 +17,6 @@ use crate::{
|
||||
bucket_config_manager::NotifyBucketConfigManager,
|
||||
config_manager::NotifyConfigManager,
|
||||
integration::NotificationMetrics,
|
||||
notification_system_subscriber::NotificationSystemSubscriberView,
|
||||
notifier::{EventNotifier, SharedNotifyTargetList},
|
||||
pipeline::{LiveEventHistory, NotifyPipeline},
|
||||
registry::TargetRegistry,
|
||||
@@ -52,7 +51,6 @@ impl NotifyServices {
|
||||
stream_cancellers: Arc<RwLock<ReplayWorkerManager>>,
|
||||
concurrency_limiter: Arc<Semaphore>,
|
||||
metrics: Arc<NotificationMetrics>,
|
||||
subscriber_view: Arc<NotificationSystemSubscriberView>,
|
||||
live_event_sender: broadcast::Sender<Arc<Event>>,
|
||||
live_event_history: Arc<RwLock<LiveEventHistory>>,
|
||||
) -> Self {
|
||||
@@ -65,7 +63,7 @@ impl NotifyServices {
|
||||
metrics.clone(),
|
||||
);
|
||||
let config_manager = NotifyConfigManager::new(config, registry, rule_engine.clone(), runtime_facade.clone());
|
||||
let bucket_config_manager = NotifyBucketConfigManager::new(notifier.clone(), rule_engine, subscriber_view);
|
||||
let bucket_config_manager = NotifyBucketConfigManager::new(notifier.clone(), rule_engine);
|
||||
let pipeline = NotifyPipeline::new(notifier, live_event_sender, live_event_history);
|
||||
let status_view = NotifyStatusView::new(metrics);
|
||||
|
||||
@@ -84,8 +82,8 @@ impl NotifyServices {
|
||||
mod tests {
|
||||
use super::NotifyServices;
|
||||
use crate::{
|
||||
integration::NotificationMetrics, notification_system_subscriber::NotificationSystemSubscriberView,
|
||||
notifier::EventNotifier, pipeline::LiveEventHistory, registry::TargetRegistry, rule_engine::NotifyRuleEngine,
|
||||
integration::NotificationMetrics, notifier::EventNotifier, pipeline::LiveEventHistory, registry::TargetRegistry,
|
||||
rule_engine::NotifyRuleEngine,
|
||||
};
|
||||
use rustfs_config::server_config::Config;
|
||||
use rustfs_targets::ReplayWorkerManager;
|
||||
@@ -102,7 +100,6 @@ mod tests {
|
||||
let config = Arc::new(RwLock::new(Config::default()));
|
||||
let stream_cancellers = Arc::new(RwLock::new(ReplayWorkerManager::new()));
|
||||
let concurrency_limiter = Arc::new(Semaphore::new(4));
|
||||
let subscriber_view = Arc::new(NotificationSystemSubscriberView::new());
|
||||
let (live_event_sender, _) = broadcast::channel(16);
|
||||
let live_event_history = Arc::new(RwLock::new(LiveEventHistory::default()));
|
||||
|
||||
@@ -115,7 +112,6 @@ mod tests {
|
||||
stream_cancellers,
|
||||
concurrency_limiter,
|
||||
metrics,
|
||||
subscriber_view,
|
||||
live_event_sender,
|
||||
live_event_history,
|
||||
);
|
||||
|
||||
@@ -27,6 +27,15 @@ pub(crate) fn resolve_notify_object_store_handle() -> Option<Arc<NotifyStore>> {
|
||||
resolve_notify_object_store_handle_from_backend()
|
||||
}
|
||||
|
||||
pub(crate) async fn refresh_notify_bucket_metadata(bucket: &str) -> bool {
|
||||
let Some(store) = resolve_notify_object_store_handle() else {
|
||||
return false;
|
||||
};
|
||||
rustfs_ecstore::api::bucket::metadata_sys::reload_bucket_metadata(store, bucket)
|
||||
.await
|
||||
.is_ok()
|
||||
}
|
||||
|
||||
pub(crate) type NotifyServerConfigSnapshot = rustfs_ecstore::api::config::com::ServerConfigSnapshot;
|
||||
|
||||
pub(crate) async fn read_notify_server_config_snapshot(store: Arc<NotifyStore>) -> Result<NotifyServerConfigSnapshot, String> {
|
||||
|
||||
@@ -24,6 +24,9 @@ use crate::{
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct DeletedObjectReplicationInfo {
|
||||
/// The originating mutation's bucket instance, retained across in-process retries.
|
||||
/// Replay without this identity leaves it absent and cannot deliver external notifications.
|
||||
pub notification_bucket_incarnation: Option<Uuid>,
|
||||
pub delete_object: DeletedObject,
|
||||
pub bucket: String,
|
||||
pub event_type: String,
|
||||
@@ -182,6 +185,7 @@ pub fn resync_existing_delete_replication_info(roi: &ReplicateObjectInfo, target
|
||||
};
|
||||
|
||||
DeletedObjectReplicationInfo {
|
||||
notification_bucket_incarnation: roi.notification_bucket_incarnation,
|
||||
delete_object: DeletedObject {
|
||||
object_name: roi.name.clone(),
|
||||
delete_marker_version_id: dm_version_id,
|
||||
@@ -519,7 +523,9 @@ mod tests {
|
||||
/// pin that this shape flows into purge — not marker-creation — semantics.
|
||||
#[test]
|
||||
fn resync_rebuilt_purge_entry_keeps_versioned_delete_semantics() {
|
||||
let incarnation = Uuid::new_v4();
|
||||
let roi = ReplicateObjectInfo {
|
||||
notification_bucket_incarnation: Some(incarnation),
|
||||
bucket: "bucket".to_string(),
|
||||
name: "object".to_string(),
|
||||
version_id: Some(Uuid::new_v4()),
|
||||
@@ -530,6 +536,7 @@ mod tests {
|
||||
|
||||
let info = resync_existing_delete_replication_info(&roi, "arn:target-a");
|
||||
|
||||
assert_eq!(info.notification_bucket_incarnation, Some(incarnation));
|
||||
assert!(is_version_delete_replication(&info.delete_object));
|
||||
assert!(
|
||||
!delete_replication_creates_marker(&info.delete_object),
|
||||
|
||||
@@ -891,6 +891,10 @@ pub struct ReplicationGenerationSnapshot {
|
||||
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
||||
pub struct ReplicateObjectInfo {
|
||||
/// The originating mutation's bucket instance, retained across in-process retries.
|
||||
/// Replay without this identity leaves it absent and cannot deliver external notifications.
|
||||
#[serde(skip)]
|
||||
pub notification_bucket_incarnation: Option<Uuid>,
|
||||
pub name: String,
|
||||
pub size: i64,
|
||||
pub actual_size: i64,
|
||||
|
||||
@@ -262,8 +262,16 @@ impl DefaultBucketUsecase {
|
||||
// same-name make while it waits for repair coordination.
|
||||
let operation_bucket = bucket.clone();
|
||||
let operation_store = store.clone();
|
||||
with_site_replication_bucket_mutation_lock(store, &bucket, move || async move {
|
||||
let removed_targets = with_site_replication_bucket_mutation_lock(store, &bucket, move || async move {
|
||||
let intent = prepare_site_replication_delete_bucket(&operation_bucket, force).await?;
|
||||
let removed_targets = match rustfs_notify::notification_system() {
|
||||
Some(system) => operation_store
|
||||
.bucket_incarnation_id(&operation_bucket)
|
||||
.await
|
||||
.ok()
|
||||
.map(|id| system.notifier.bucket_removed_targets(&operation_bucket, id)),
|
||||
None => None,
|
||||
};
|
||||
let delete_result = operation_store
|
||||
.delete_bucket(
|
||||
&operation_bucket,
|
||||
@@ -281,7 +289,7 @@ impl DefaultBucketUsecase {
|
||||
{
|
||||
warn!(bucket = %operation_bucket, error = ?err, "site replication delete bucket hook failed");
|
||||
}
|
||||
Ok::<(), S3Error>(())
|
||||
Ok::<_, S3Error>(removed_targets)
|
||||
}
|
||||
Err(err) => {
|
||||
if let Some(intent) = intent {
|
||||
@@ -306,6 +314,10 @@ impl DefaultBucketUsecase {
|
||||
// Notify peers to drop their cached metadata for the now-deleted bucket.
|
||||
notify_bucket_metadata_delete(bucket, envelope.request_context().cloned());
|
||||
|
||||
let helper = match removed_targets {
|
||||
Some(targets) => helper.with_bucket_removed_targets(targets),
|
||||
None => helper,
|
||||
};
|
||||
let result = Ok(gateway_dto::DeleteBucketOutput::default());
|
||||
let _ = helper.complete_owned(&result, StatusCode::OK, &http::HeaderMap::new());
|
||||
result
|
||||
|
||||
@@ -183,7 +183,7 @@ impl DefaultBucketUsecase {
|
||||
})
|
||||
}
|
||||
|
||||
/// Persists notification configuration, then updates the active rules for the bucket.
|
||||
/// Persists notification configuration and publishes the committed metadata snapshot.
|
||||
pub(crate) async fn execute_put_bucket_notification_configuration(
|
||||
&self,
|
||||
envelope: RequestEnvelope,
|
||||
@@ -217,7 +217,7 @@ impl DefaultBucketUsecase {
|
||||
let notify = current_notify_interface_for_context(self.context.as_deref());
|
||||
let event_rules = build_notification_event_rules(destinations)?;
|
||||
|
||||
// Validate before the write. Clear and add still run after the durable update.
|
||||
// The metadata publisher applies rules only after the durable update.
|
||||
notify
|
||||
.validate_event_specific_rules(bucket, ®ion, &event_rules)
|
||||
.await
|
||||
@@ -230,16 +230,6 @@ impl DefaultBucketUsecase {
|
||||
|
||||
notify_bucket_metadata_reload(bucket.to_owned(), "put bucket notification", request_context, false).await;
|
||||
|
||||
notify
|
||||
.clear_bucket_notification_rules(bucket)
|
||||
.await
|
||||
.map_err(|e| s3_error!(InternalError, "Failed to clear rules: {e}"))?;
|
||||
warn!("notify event rules: {:?}", &event_rules);
|
||||
notify
|
||||
.add_event_specific_rules(bucket, ®ion, &event_rules)
|
||||
.await
|
||||
.map_err(|e| s3_error!(InternalError, "Failed to add rules: {e}"))?;
|
||||
|
||||
Ok(gateway_dto::PutBucketNotificationConfigurationOutput::default())
|
||||
},
|
||||
)
|
||||
|
||||
@@ -51,22 +51,18 @@ enum Callback {
|
||||
Validate,
|
||||
ReloadStarted,
|
||||
ReloadCompleted,
|
||||
Clear,
|
||||
Add,
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
enum Observation {
|
||||
Rules(Callback, String, Vec<RuntimeRule>, Vec<u8>),
|
||||
Reload(Callback, &'static str, bool),
|
||||
Clear(Vec<u8>),
|
||||
}
|
||||
|
||||
struct RecordingNotify {
|
||||
bucket: String,
|
||||
fail_at: Option<Callback>,
|
||||
calls: Arc<Mutex<Vec<Observation>>>,
|
||||
active_rules: Mutex<Vec<RuntimeRule>>,
|
||||
}
|
||||
|
||||
async fn read_notification_xml(bucket: &str) -> Vec<u8> {
|
||||
@@ -113,22 +109,6 @@ impl NotifyInterface for RecordingNotify {
|
||||
self.record_rules(Callback::Validate, bucket, region, rules).await;
|
||||
self.maybe_fail(Callback::Validate)
|
||||
}
|
||||
|
||||
async fn add_event_specific_rules(&self, bucket: &str, region: &str, rules: &[RuntimeRule]) -> Result<(), NotificationError> {
|
||||
self.record_rules(Callback::Add, bucket, region, rules).await;
|
||||
self.maybe_fail(Callback::Add)?;
|
||||
*self.active_rules.lock().unwrap() = rules.to_vec();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn clear_bucket_notification_rules(&self, bucket: &str) -> Result<(), NotificationError> {
|
||||
assert_eq!(bucket, self.bucket);
|
||||
let xml = read_notification_xml(bucket).await;
|
||||
self.calls.lock().unwrap().push(Observation::Clear(xml));
|
||||
self.maybe_fail(Callback::Clear)?;
|
||||
self.active_rules.lock().unwrap().clear();
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn configuration() -> NotificationConfiguration {
|
||||
@@ -219,7 +199,7 @@ fn check_notification_write_order() {
|
||||
run_large_stack_test("notification-write-order", || async {
|
||||
let ambient = shared_gating_ambient().await;
|
||||
let store = ambient.object_store();
|
||||
for fail_at in [None, Some(Callback::Validate), Some(Callback::Clear), Some(Callback::Add)] {
|
||||
for fail_at in [None, Some(Callback::Validate)] {
|
||||
let bucket = format!("notify-order-{}", uuid::Uuid::new_v4().simple());
|
||||
store.make_bucket(&bucket, &MakeBucketOptions::default()).await.unwrap();
|
||||
store
|
||||
@@ -241,9 +221,8 @@ fn check_notification_write_order() {
|
||||
bucket: bucket.clone(),
|
||||
fail_at,
|
||||
calls: calls.clone(),
|
||||
active_rules: Mutex::new(expected_rules()),
|
||||
});
|
||||
let context = Arc::new(app_context_from_current_environment(&ambient).with_test_notify_interface(recorder.clone()));
|
||||
let context = Arc::new(app_context_from_current_environment(&ambient).with_test_notify_interface(recorder));
|
||||
let region = context
|
||||
.region()
|
||||
.get()
|
||||
@@ -261,18 +240,14 @@ fn check_notification_write_order() {
|
||||
let mut timeline = vec![Observation::Rules(
|
||||
Callback::Validate,
|
||||
region.clone(),
|
||||
expected.clone(),
|
||||
expected,
|
||||
OLD_XML.to_vec(),
|
||||
)];
|
||||
if fail_at != Some(Callback::Validate) {
|
||||
timeline.extend([
|
||||
Observation::Reload(Callback::ReloadStarted, "put bucket notification", false),
|
||||
Observation::Reload(Callback::ReloadCompleted, "put bucket notification", false),
|
||||
Observation::Clear(NEW_XML.to_vec()),
|
||||
]);
|
||||
if fail_at != Some(Callback::Clear) {
|
||||
timeline.push(Observation::Rules(Callback::Add, region.clone(), expected.clone(), NEW_XML.to_vec()));
|
||||
}
|
||||
}
|
||||
assert_eq!(*calls.lock().unwrap(), timeline);
|
||||
let stored = if fail_at == Some(Callback::Validate) {
|
||||
@@ -282,7 +257,6 @@ fn check_notification_write_order() {
|
||||
};
|
||||
assert_eq!(read_notification_xml(&bucket).await, stored);
|
||||
let error_message = match fail_at {
|
||||
Some(Callback::Clear) => Some("Failed to clear rules: Rule configuration error: fixture failure"),
|
||||
Some(_) => Some("Failed to add rules: Rule configuration error: fixture failure"),
|
||||
None => None,
|
||||
};
|
||||
@@ -304,7 +278,6 @@ fn check_notification_write_order() {
|
||||
assert!(audit.events.lock().unwrap().is_empty());
|
||||
|
||||
if fail_at.is_none() {
|
||||
assert_eq!(*recorder.active_rules.lock().unwrap(), expected);
|
||||
let mut invalid = configuration();
|
||||
invalid.lambda_function_configurations.as_mut().unwrap()[0].lambda_function_arn = "invalid-arn".into();
|
||||
let mut invalid = notification_request(&bucket, invalid, &slot);
|
||||
@@ -350,12 +323,9 @@ fn check_notification_write_order() {
|
||||
Observation::Rules(Callback::Validate, region.clone(), Vec::new(), NEW_XML.to_vec()),
|
||||
Observation::Reload(Callback::ReloadStarted, "put bucket notification", false),
|
||||
Observation::Reload(Callback::ReloadCompleted, "put bucket notification", false),
|
||||
Observation::Clear(EMPTY_XML.to_vec()),
|
||||
Observation::Rules(Callback::Add, region, Vec::new(), EMPTY_XML.to_vec()),
|
||||
]
|
||||
);
|
||||
assert_eq!(read_notification_xml(&bucket).await, EMPTY_XML);
|
||||
assert!(recorder.active_rules.lock().unwrap().is_empty());
|
||||
}
|
||||
}
|
||||
});
|
||||
@@ -412,7 +382,6 @@ fn legacy_put_notification_does_not_write_after_the_bucket_is_recreated() {
|
||||
bucket: bucket.clone(),
|
||||
fail_at: None,
|
||||
calls: calls.clone(),
|
||||
active_rules: Mutex::new(Vec::new()),
|
||||
});
|
||||
let context = Arc::new(app_context_from_current_environment(&ambient).with_test_notify_interface(recorder));
|
||||
let slot = ServerContextSlot::new();
|
||||
|
||||
@@ -849,6 +849,7 @@ impl RequestEnvelope {
|
||||
self.client_info(),
|
||||
self.remote_addr(),
|
||||
self.request_context(),
|
||||
self.bucket_generation(),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -191,19 +191,6 @@ impl NotifyInterface for NotifyHandle {
|
||||
) -> Result<(), NotificationError> {
|
||||
runtime_sources::validate_event_specific_rules(bucket_name, region, event_rules).await
|
||||
}
|
||||
|
||||
async fn add_event_specific_rules(
|
||||
&self,
|
||||
bucket_name: &str,
|
||||
region: &str,
|
||||
event_rules: &[(Vec<EventName>, String, String, Vec<TargetID>)],
|
||||
) -> Result<(), NotificationError> {
|
||||
runtime_sources::add_event_specific_rules(bucket_name, region, event_rules).await
|
||||
}
|
||||
|
||||
async fn clear_bucket_notification_rules(&self, bucket_name: &str) -> Result<(), NotificationError> {
|
||||
runtime_sources::clear_bucket_notification_rules(bucket_name).await
|
||||
}
|
||||
}
|
||||
|
||||
/// Default notification system handle adapter.
|
||||
|
||||
@@ -90,15 +90,6 @@ pub trait NotifyInterface: Send + Sync {
|
||||
region: &str,
|
||||
event_rules: &[(Vec<EventName>, String, String, Vec<TargetID>)],
|
||||
) -> Result<(), NotificationError>;
|
||||
|
||||
async fn add_event_specific_rules(
|
||||
&self,
|
||||
bucket_name: &str,
|
||||
region: &str,
|
||||
event_rules: &[(Vec<EventName>, String, String, Vec<TargetID>)],
|
||||
) -> Result<(), NotificationError>;
|
||||
|
||||
async fn clear_bucket_notification_rules(&self, bucket_name: &str) -> Result<(), NotificationError>;
|
||||
}
|
||||
|
||||
/// Notification system handle interface for admin peer orchestration.
|
||||
|
||||
@@ -79,18 +79,6 @@ pub async fn validate_event_specific_rules(
|
||||
notifier_global::validate_event_specific_rules(bucket_name, region, event_rules).await
|
||||
}
|
||||
|
||||
pub async fn add_event_specific_rules(
|
||||
bucket_name: &str,
|
||||
region: &str,
|
||||
event_rules: &[(Vec<EventName>, String, String, Vec<TargetID>)],
|
||||
) -> Result<(), NotificationError> {
|
||||
notifier_global::add_event_specific_rules(bucket_name, region, event_rules).await
|
||||
}
|
||||
|
||||
pub async fn clear_bucket_notification_rules(bucket_name: &str) -> Result<(), NotificationError> {
|
||||
notifier_global::clear_bucket_notification_rules(bucket_name).await
|
||||
}
|
||||
|
||||
pub fn notification_system() -> Option<Arc<NotificationSys>> {
|
||||
get_global_notification_sys()
|
||||
}
|
||||
|
||||
@@ -563,7 +563,7 @@ impl DefaultMultipartUsecase {
|
||||
&& decision.replicate_any()
|
||||
{
|
||||
warn!("need multipart replication");
|
||||
schedule_object_replication(obj_info.clone(), store, decision).await;
|
||||
schedule_object_replication(obj_info.clone(), store, decision, opts.expected_bucket_incarnation_id).await;
|
||||
}
|
||||
|
||||
rustfs_scanner::record_dirty_usage_object_from_producer(
|
||||
|
||||
@@ -821,7 +821,8 @@ impl DefaultObjectUsecase {
|
||||
// the persisted pending marker and the schedule always agree, mirroring
|
||||
// the PUT path.
|
||||
if dsc.replicate_any() {
|
||||
schedule_object_replication(oi.clone(), Arc::clone(&store), dsc).await;
|
||||
schedule_object_replication(oi.clone(), Arc::clone(&store), dsc, dst_opts.expected_bucket_incarnation_id)
|
||||
.await;
|
||||
}
|
||||
|
||||
maybe_enqueue_transition_immediate(&oi, LcEventSrc::S3CopyObject).await;
|
||||
|
||||
@@ -945,6 +945,7 @@ impl DefaultObjectUsecase {
|
||||
..Default::default()
|
||||
};
|
||||
apply_bucket_generation_guard(&envelope, &bucket, &mut storage_delete_opts)?;
|
||||
let incarnation_id = storage_delete_opts.expected_bucket_incarnation_id;
|
||||
let storage_started = std::time::Instant::now();
|
||||
let (dobjs, errs, accounting) = store
|
||||
.delete_objects_with_tier_delete_journal_and_accounting(&bucket, object_to_delete.clone(), storage_delete_opts)
|
||||
@@ -1096,8 +1097,13 @@ impl DefaultObjectUsecase {
|
||||
let bucket_for_replication = bucket.clone();
|
||||
let replication_task = tokio::spawn(async move {
|
||||
let _activity_guard = DeleteTailActivityGuard::new(DeleteTailStage::Replication);
|
||||
schedule_replication_deletes(replication_deletes, bucket_for_replication, REPLICATE_INCOMING_DELETE.to_string())
|
||||
.await;
|
||||
schedule_replication_deletes(
|
||||
replication_deletes,
|
||||
bucket_for_replication,
|
||||
REPLICATE_INCOMING_DELETE.to_string(),
|
||||
incarnation_id,
|
||||
)
|
||||
.await;
|
||||
});
|
||||
// The spawned task owns every locally committed delete. Dropping the
|
||||
// join handle on request cancellation therefore cannot lose the tail.
|
||||
@@ -1158,6 +1164,7 @@ impl DefaultObjectUsecase {
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.bucket_incarnation_id(incarnation_id)
|
||||
.version_id(version_id.unwrap_or_default())
|
||||
.req_params(req_params.clone())
|
||||
.resp_elements(resp_elements.clone())
|
||||
@@ -1499,6 +1506,7 @@ impl DefaultObjectUsecase {
|
||||
},
|
||||
bucket.clone(),
|
||||
REPLICATE_INCOMING_DELETE.to_string(),
|
||||
opts.expected_bucket_incarnation_id,
|
||||
)
|
||||
.await;
|
||||
} else if replicate_force_delete {
|
||||
@@ -1521,7 +1529,13 @@ impl DefaultObjectUsecase {
|
||||
) {
|
||||
set_deleted_object_replication_state(&mut delete_object, &replication_state);
|
||||
}
|
||||
schedule_replication_delete(delete_object, bucket.clone(), REPLICATE_INCOMING_DELETE.to_string()).await;
|
||||
schedule_replication_delete(
|
||||
delete_object,
|
||||
bucket.clone(),
|
||||
REPLICATE_INCOMING_DELETE.to_string(),
|
||||
opts.expected_bucket_incarnation_id,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
// Prefix/force-delete returns empty ObjectInfo; still emit bucket notification so webhooks match S3 DELETE.
|
||||
helper = helper
|
||||
@@ -1602,7 +1616,13 @@ impl DefaultObjectUsecase {
|
||||
set_deleted_object_replication_state(&mut deleted_object, &replication_state_source.replication_state());
|
||||
enrich_delete_replication_state_if_needed(&delete_config_snapshot, &mut deleted_object, replication_state_source);
|
||||
}
|
||||
schedule_replication_delete(deleted_object, bucket.clone(), REPLICATE_INCOMING_DELETE.to_string()).await;
|
||||
schedule_replication_delete(
|
||||
deleted_object,
|
||||
bucket.clone(),
|
||||
REPLICATE_INCOMING_DELETE.to_string(),
|
||||
opts.expected_bucket_incarnation_id,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
let delete_marker = obj_info.delete_marker;
|
||||
|
||||
@@ -740,13 +740,16 @@ async fn complete_extract_member_post_commit(
|
||||
let _ = invalidate_object_data_cache_after_put_success(&context.cache_adapter, &context.bucket, &key).await;
|
||||
|
||||
if replication.replicate_any() {
|
||||
schedule_object_replication(obj_info.clone(), context.store.clone(), replication).await;
|
||||
schedule_object_replication(obj_info.clone(), context.store.clone(), replication, opts.expected_bucket_incarnation_id)
|
||||
.await;
|
||||
}
|
||||
|
||||
let actual_version_id = extract_notification_version_id(obj_info.version_id, opts.versioned, opts.version_suspended);
|
||||
let mut event_object = convert_ecstore_object_info(obj_info);
|
||||
event_object.version_id = (!actual_version_id.is_empty()).then_some(actual_version_id.clone());
|
||||
let event = rustfs_notify::EventArgs {
|
||||
bucket_incarnation_id: opts.expected_bucket_incarnation_id,
|
||||
bucket_removed_targets: None,
|
||||
event_name: put_event_name_for_post_object(false),
|
||||
bucket_name: context.bucket.clone(),
|
||||
object: event_object,
|
||||
@@ -2858,19 +2861,6 @@ mod tests {
|
||||
) -> Result<(), rustfs_notify::NotificationError> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn add_event_specific_rules(
|
||||
&self,
|
||||
_bucket_name: &str,
|
||||
_region: &str,
|
||||
_event_rules: &[(Vec<rustfs_targets::EventName>, String, String, Vec<rustfs_targets::arn::TargetID>)],
|
||||
) -> Result<(), rustfs_notify::NotificationError> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn clear_bucket_notification_rules(&self, _bucket_name: &str) -> Result<(), rustfs_notify::NotificationError> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
async fn recording_extract_commit_context(
|
||||
|
||||
@@ -187,6 +187,11 @@ impl InternalPutObjectEvent {
|
||||
.req_param("principalId", principal_id)
|
||||
}
|
||||
|
||||
pub(super) fn with_bucket_incarnation(mut self, incarnation_id: Option<Uuid>) -> Self {
|
||||
self.builder = self.builder.bucket_incarnation_id(incarnation_id);
|
||||
self
|
||||
}
|
||||
|
||||
pub(super) fn object(self, obj_info: ObjectInfo) -> Self {
|
||||
let Self {
|
||||
builder,
|
||||
@@ -709,6 +714,7 @@ impl DefaultObjectUsecase {
|
||||
&key,
|
||||
ctx.principal_id,
|
||||
)
|
||||
.map(|event| event.with_bucket_incarnation(opts.expected_bucket_incarnation_id))
|
||||
});
|
||||
|
||||
// The spawned task owns the commit so a cancelled caller cannot leave
|
||||
@@ -742,7 +748,13 @@ impl DefaultObjectUsecase {
|
||||
enqueue_transition_immediate(&obj_info, LcEventSrc::S3CompleteMultipartUpload).await;
|
||||
|
||||
if completion_replication_decision.replicate_any() {
|
||||
schedule_object_replication(obj_info.clone(), store, completion_replication_decision).await;
|
||||
schedule_object_replication(
|
||||
obj_info.clone(),
|
||||
store,
|
||||
completion_replication_decision,
|
||||
opts.expected_bucket_incarnation_id,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
rustfs_scanner::record_dirty_usage_object_from_producer(
|
||||
|
||||
@@ -2023,6 +2023,7 @@ impl DefaultObjectUsecase {
|
||||
&key,
|
||||
principal_id,
|
||||
)
|
||||
.map(|event| event.with_bucket_incarnation(opts.expected_bucket_incarnation_id))
|
||||
});
|
||||
(PutObjectCompletion::Internal(event.flatten().map(Box::new)), Some(request_context))
|
||||
}
|
||||
@@ -2316,7 +2317,7 @@ impl DefaultObjectUsecase {
|
||||
|
||||
if dsc.replicate_any() {
|
||||
let replication_schedule_stage_start = put_stage_metrics_enabled.then(Instant::now);
|
||||
schedule_object_replication(obj_info.clone(), store, dsc).await;
|
||||
schedule_object_replication(obj_info.clone(), store, dsc, opts.expected_bucket_incarnation_id).await;
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
"app_post_store_replication_schedule",
|
||||
replication_schedule_stage_start,
|
||||
|
||||
@@ -132,7 +132,7 @@ impl ObjectMetaUsecase {
|
||||
|
||||
// The current target transport carries the updated metadata in a full-object PUT.
|
||||
if dsc.replicate_any() {
|
||||
schedule_metadata_replication(info.clone(), store.clone(), dsc).await;
|
||||
schedule_metadata_replication(info.clone(), store.clone(), dsc, popts.expected_bucket_incarnation_id).await;
|
||||
}
|
||||
|
||||
let output = PutObjectLegalHoldOutput {
|
||||
|
||||
@@ -151,7 +151,7 @@ impl ObjectMetaUsecase {
|
||||
|
||||
// The current target transport carries the updated metadata in a full-object PUT.
|
||||
if dsc.replicate_any() {
|
||||
schedule_metadata_replication(object_info.clone(), store.clone(), dsc).await;
|
||||
schedule_metadata_replication(object_info.clone(), store.clone(), dsc, opts.expected_bucket_incarnation_id).await;
|
||||
}
|
||||
|
||||
let output = PutObjectRetentionOutput {
|
||||
|
||||
@@ -29,6 +29,7 @@ use crate::app::storage_api::object_meta_usecase::{
|
||||
};
|
||||
use crate::app::storage_api::object_usecase::ECStore;
|
||||
use crate::app::storage_api::object_usecase::StorageObjectOptions as ObjectOptions;
|
||||
use crate::app::storage_api::object_usecase::access::apply_bucket_generation_guard;
|
||||
use crate::app::storage_api::object_usecase::bucket::replication::{must_replicate_metadata, schedule_metadata_replication};
|
||||
use crate::app::storage_api::object_usecase::bucket::versioning_sys::BucketVersioningSys;
|
||||
use crate::app::storage_api::object_usecase::error::{is_err_object_not_found, is_err_version_not_found};
|
||||
@@ -208,6 +209,7 @@ impl ObjectMetaUsecase {
|
||||
let mut opts = get_opts(bucket, object, version_id.clone(), None, envelope.headers())
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
apply_bucket_generation_guard(&envelope, bucket, &mut opts)?;
|
||||
let existing = match store.get_object_info(bucket, object, &opts).await {
|
||||
Ok(info) => info,
|
||||
Err(error) => {
|
||||
@@ -264,7 +266,7 @@ impl ObjectMetaUsecase {
|
||||
ApiError::from(error)
|
||||
})?;
|
||||
if decision.replicate_any() {
|
||||
schedule_metadata_replication(info.clone(), Arc::clone(&store), decision).await;
|
||||
schedule_metadata_replication(info.clone(), Arc::clone(&store), decision, opts.expected_bucket_incarnation_id).await;
|
||||
}
|
||||
counter!("rustfs_put_object_tagging_success").increment(1);
|
||||
|
||||
@@ -328,6 +330,7 @@ impl ObjectMetaUsecase {
|
||||
let mut opts = get_opts(bucket, object, version_id.clone(), None, envelope.headers())
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
apply_bucket_generation_guard(&envelope, bucket, &mut opts)?;
|
||||
let existing_object_info = match store.get_object_info(bucket, object, &opts).await {
|
||||
Ok(info) => info,
|
||||
Err(e) => {
|
||||
@@ -388,7 +391,8 @@ impl ObjectMetaUsecase {
|
||||
.or_else(|| object_info.version_id.map(|version_id| version_id.to_string()))
|
||||
.unwrap_or_default();
|
||||
if dsc.replicate_any() {
|
||||
schedule_metadata_replication(object_info.clone(), Arc::clone(&store), dsc).await;
|
||||
schedule_metadata_replication(object_info.clone(), Arc::clone(&store), dsc, opts.expected_bucket_incarnation_id)
|
||||
.await;
|
||||
}
|
||||
counter!("rustfs_delete_object_tagging_success").increment(1);
|
||||
helper = helper.object(object_info).version_id(event_version_id);
|
||||
|
||||
@@ -1385,42 +1385,67 @@ pub(crate) mod bucket {
|
||||
oi: crate::storage::storage_api::StorageObjectInfo,
|
||||
store: Arc<crate::storage::storage_api::ECStore>,
|
||||
dsc: ReplicateDecision,
|
||||
notification_bucket_incarnation: Option<Uuid>,
|
||||
) {
|
||||
ReplicationObjectBridge::schedule_object(oi, store, dsc, replication_contracts::ReplicationType::Object).await;
|
||||
ReplicationObjectBridge::schedule_object(
|
||||
oi,
|
||||
store,
|
||||
dsc,
|
||||
replication_contracts::ReplicationType::Object,
|
||||
notification_bucket_incarnation,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
pub(crate) async fn schedule_metadata_replication(
|
||||
oi: crate::storage::storage_api::StorageObjectInfo,
|
||||
store: Arc<crate::storage::storage_api::ECStore>,
|
||||
dsc: ReplicateDecision,
|
||||
notification_bucket_incarnation: Option<Uuid>,
|
||||
) {
|
||||
ReplicationObjectBridge::schedule_object(oi, store, dsc, replication_contracts::ReplicationType::Metadata).await;
|
||||
ReplicationObjectBridge::schedule_object(
|
||||
oi,
|
||||
store,
|
||||
dsc,
|
||||
replication_contracts::ReplicationType::Metadata,
|
||||
notification_bucket_incarnation,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
pub(crate) async fn schedule_replication_delete(
|
||||
delete_object: crate::storage::storage_api::StorageDeletedObject,
|
||||
bucket: String,
|
||||
event_type: String,
|
||||
notification_bucket_incarnation: Option<Uuid>,
|
||||
) {
|
||||
#[cfg(test)]
|
||||
SCHEDULED_REPLICATION_DELETES
|
||||
.lock()
|
||||
.expect("scheduled replication delete lock should not be poisoned")
|
||||
.push(delete_object.clone());
|
||||
ReplicationObjectBridge::schedule_storage_delete(delete_object, bucket, event_type).await;
|
||||
ReplicationObjectBridge::schedule_storage_delete(delete_object, bucket, event_type, notification_bucket_incarnation)
|
||||
.await;
|
||||
}
|
||||
|
||||
pub(crate) async fn schedule_replication_deletes(
|
||||
delete_objects: Vec<crate::storage::storage_api::StorageDeletedObject>,
|
||||
bucket: String,
|
||||
event_type: String,
|
||||
notification_bucket_incarnation: Option<Uuid>,
|
||||
) {
|
||||
#[cfg(test)]
|
||||
SCHEDULED_REPLICATION_DELETES
|
||||
.lock()
|
||||
.expect("scheduled replication delete lock should not be poisoned")
|
||||
.extend(delete_objects.iter().cloned());
|
||||
ReplicationObjectBridge::schedule_storage_deletes(delete_objects, bucket, event_type).await;
|
||||
ReplicationObjectBridge::schedule_storage_deletes(
|
||||
delete_objects,
|
||||
bucket,
|
||||
event_type,
|
||||
notification_bucket_incarnation,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
pub(crate) fn set_deleted_object_replication_state(
|
||||
|
||||
+43
-162
@@ -16,12 +16,9 @@
|
||||
use crate::protocols::ProtocolStorageClient;
|
||||
use crate::runtime_sources::current_region;
|
||||
use crate::server::ShutdownHandle;
|
||||
use crate::server::runtime_sources::current_notify_interface;
|
||||
use crate::storage_api::error::{StorageError, is_unreadable_config_error};
|
||||
use crate::storage_api::startup::bucket_metadata::contract::bucket::{BucketOperations, BucketOptions};
|
||||
use crate::storage_api::startup::init::notification_config_to_s3s;
|
||||
use crate::storage_api::startup::init::{
|
||||
get_bucket_notification_config, process_lambda_configurations, process_queue_configurations, process_topic_configurations,
|
||||
deserialize, process_lambda_configurations, process_queue_configurations, process_topic_configurations,
|
||||
};
|
||||
use crate::storage_api::startup::services::ECStore;
|
||||
#[cfg(any(feature = "ftps", feature = "webdav", feature = "sftp"))]
|
||||
@@ -32,8 +29,7 @@ use rustfs_config::{
|
||||
DEFAULT_BUFFER_MAX_SIZE, DEFAULT_BUFFER_MIN_SIZE, DEFAULT_BUFFER_PROFILE, DEFAULT_BUFFER_UNKNOWN_SIZE, DEFAULT_UPDATE_CHECK,
|
||||
ENV_RUSTFS_BUFFER_DEFAULT_SIZE, ENV_RUSTFS_BUFFER_MAX_SIZE, ENV_RUSTFS_BUFFER_MIN_SIZE, ENV_UPDATE_CHECK, RUSTFS_REGION,
|
||||
};
|
||||
use rustfs_notify::NotificationError;
|
||||
use rustfs_s3_types::{EventName, s3_error};
|
||||
use rustfs_s3_types::EventName;
|
||||
use rustfs_targets::arn::{ARN, TargetID, TargetIDError};
|
||||
use rustfs_utils::get_env_usize;
|
||||
use std::env;
|
||||
@@ -174,148 +170,44 @@ fn notification_config_to_event_rules(
|
||||
Ok(event_rules)
|
||||
}
|
||||
|
||||
/// One bucket's persisted notification configuration as startup sees it.
|
||||
#[derive(Debug)]
|
||||
enum BucketNotificationLookup {
|
||||
Configured(rustfs_gateway_types::persistence::PersistedNotificationConfiguration),
|
||||
Missing,
|
||||
/// Stored bytes cannot be parsed. Deterministic, so a retry cannot help,
|
||||
/// and reading it as missing would clear the bucket's rules.
|
||||
Unreadable,
|
||||
}
|
||||
|
||||
fn classify_bucket_notification_lookup(
|
||||
bucket: &str,
|
||||
lookup: Result<Option<rustfs_gateway_types::persistence::PersistedNotificationConfiguration>, StorageError>,
|
||||
) -> Result<BucketNotificationLookup, NotificationError> {
|
||||
match lookup {
|
||||
Ok(Some(cfg)) => Ok(BucketNotificationLookup::Configured(cfg)),
|
||||
Ok(None) => Ok(BucketNotificationLookup::Missing),
|
||||
Err(err) if is_unreadable_config_error(&err) => {
|
||||
/// Publishes the complete, current configuration under the bucket metadata publication lock.
|
||||
pub(crate) fn publish_bucket_notification_configuration(bucket: &str, stored: Option<(&[u8], uuid::Uuid)>) {
|
||||
let Some((bytes, incarnation_id)) = stored else {
|
||||
if let Some(system) = rustfs_notify::notification_system() {
|
||||
system.publish_bucket_notification_config(bucket, None);
|
||||
}
|
||||
return;
|
||||
};
|
||||
let parsed = (|| {
|
||||
let config = deserialize::<rustfs_gateway_types::persistence::PersistedNotificationConfiguration>(bytes)
|
||||
.map_err(|_| "invalid_xml")?;
|
||||
let config = notification_config_to_s3s(&config).map_err(|_| "invalid_configuration")?;
|
||||
notification_config_to_event_rules(&config).map_err(|_| "invalid_target_arn")
|
||||
})();
|
||||
match parsed {
|
||||
Ok(rules) => {
|
||||
let config = rustfs_notify::notifier_global::bucket_notification_config_for_rules(¬ification_region(), &rules);
|
||||
rustfs_notify::ensure_live_events().publish_bucket_notification_config(bucket, Some((incarnation_id, &config)));
|
||||
}
|
||||
Err(reason) => {
|
||||
error!(
|
||||
target: "rustfs::init",
|
||||
event = "notification_config_unreadable",
|
||||
component = LOG_COMPONENT_INIT,
|
||||
subsystem = LOG_SUBSYSTEM_NOTIFICATION,
|
||||
bucket = %bucket,
|
||||
error = %err,
|
||||
reason,
|
||||
"Bucket notification configuration is unreadable; leaving its rules unchanged"
|
||||
);
|
||||
Ok(BucketNotificationLookup::Unreadable)
|
||||
}
|
||||
Err(err) => Err(NotificationError::StorageNotAvailable(format!(
|
||||
"load bucket notification config for {bucket}: {err}"
|
||||
))),
|
||||
}
|
||||
}
|
||||
|
||||
/// Apply one bucket's persisted notification rules. An unreadable config is
|
||||
/// isolated to its bucket: it neither aborts setup for the other buckets nor
|
||||
/// clears the bucket's rules as if it had none.
|
||||
async fn apply_bucket_notification_configuration(bucket: &str, region: &str) -> Result<bool, NotificationError> {
|
||||
match classify_bucket_notification_lookup(bucket, get_bucket_notification_config(bucket).await)? {
|
||||
BucketNotificationLookup::Unreadable => Ok(false),
|
||||
BucketNotificationLookup::Configured(cfg) => {
|
||||
let cfg = notification_config_to_s3s(&cfg).map_err(|err| NotificationError::BucketNotification(err.to_string()))?;
|
||||
info!(
|
||||
target: "rustfs::init",
|
||||
event = "notification_config_loaded",
|
||||
component = LOG_COMPONENT_INIT,
|
||||
subsystem = LOG_SUBSYSTEM_NOTIFICATION,
|
||||
bucket = %bucket,
|
||||
queue_configuration_count = cfg.queue_configurations.as_ref().map_or(0, Vec::len),
|
||||
topic_configuration_count = cfg.topic_configurations.as_ref().map_or(0, Vec::len),
|
||||
lambda_configuration_count = cfg.lambda_function_configurations.as_ref().map_or(0, Vec::len),
|
||||
"Loaded bucket notification configuration"
|
||||
);
|
||||
|
||||
let event_rules =
|
||||
notification_config_to_event_rules(&cfg).map_err(|err| NotificationError::BucketNotification(err.to_string()))?;
|
||||
current_notify_interface()
|
||||
.add_event_specific_rules(bucket, region, &event_rules)
|
||||
.await?;
|
||||
Ok(true)
|
||||
}
|
||||
BucketNotificationLookup::Missing => {
|
||||
info!(
|
||||
target: "rustfs::init",
|
||||
event = "notification_config_missing",
|
||||
component = LOG_COMPONENT_INIT,
|
||||
subsystem = LOG_SUBSYSTEM_NOTIFICATION,
|
||||
bucket = %bucket,
|
||||
"Bucket notification configuration not found"
|
||||
);
|
||||
current_notify_interface().clear_bucket_notification_rules(bucket).await?;
|
||||
Ok(false)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Add existing bucket notification configurations to the global notifier system.
|
||||
/// This function retrieves notification configurations for each bucket
|
||||
/// and registers the corresponding event rules with the notifier system.
|
||||
/// It processes queue, topic, and lambda configurations and maps them to event rules.
|
||||
/// # Arguments
|
||||
/// * `buckets` - A vector of bucket names to process
|
||||
#[instrument(skip_all)]
|
||||
pub async fn add_bucket_notification_configuration(buckets: Vec<String>) {
|
||||
let region = notification_region();
|
||||
for bucket in buckets.iter() {
|
||||
if let Err(err) = apply_bucket_notification_configuration(bucket, region.as_str()).await {
|
||||
let err = s3_error!(InternalError, "Failed to add rules: {err}");
|
||||
error!(
|
||||
target: "rustfs::init",
|
||||
event = "notification_rules_registration_failed",
|
||||
component = LOG_COMPONENT_INIT,
|
||||
subsystem = LOG_SUBSYSTEM_NOTIFICATION,
|
||||
bucket = %bucket,
|
||||
region,
|
||||
error = ?err,
|
||||
"Failed to register bucket notification rules"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn reconcile_persisted_bucket_notification_configurations(
|
||||
store: Arc<rustfs_notify::NotifyStore>,
|
||||
) -> Result<usize, NotificationError> {
|
||||
let bucket_infos = store
|
||||
.list_bucket(&BucketOptions {
|
||||
no_metadata: true,
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.map_err(|err| NotificationError::StorageNotAvailable(format!("list buckets for notification reconciliation: {err}")))?;
|
||||
let region = notification_region();
|
||||
let mut configured_bucket_count = 0;
|
||||
|
||||
for bucket in bucket_infos {
|
||||
if apply_bucket_notification_configuration(&bucket.name, region.as_str()).await? {
|
||||
configured_bucket_count += 1;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(configured_bucket_count)
|
||||
}
|
||||
|
||||
fn notification_region() -> String {
|
||||
let global_region = current_region();
|
||||
global_region
|
||||
current_region()
|
||||
.as_ref()
|
||||
.filter(|r| !r.as_str().is_empty())
|
||||
.map(|r| r.to_string())
|
||||
.unwrap_or_else(|| {
|
||||
warn!(
|
||||
target: "rustfs::init",
|
||||
event = "notification_region_fallback",
|
||||
component = LOG_COMPONENT_INIT,
|
||||
subsystem = LOG_SUBSYSTEM_NOTIFICATION,
|
||||
fallback_region = RUSTFS_REGION,
|
||||
"Notification configuration falling back to default region"
|
||||
);
|
||||
RUSTFS_REGION.to_string()
|
||||
})
|
||||
.filter(|region| !region.as_str().is_empty())
|
||||
.map_or_else(|| RUSTFS_REGION.to_owned(), ToString::to_string)
|
||||
}
|
||||
|
||||
/// Build KMS configuration for local backend
|
||||
@@ -1423,42 +1315,16 @@ pub async fn init_sftp_system(
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
BucketNotificationLookup, build_aws_kms_config, build_vault_kms_config, build_vault_transit_kms_config,
|
||||
classify_bucket_notification_lookup, notification_config_to_event_rules, resolve_buffer_profile_config,
|
||||
build_aws_kms_config, build_vault_kms_config, build_vault_transit_kms_config, notification_config_to_event_rules,
|
||||
resolve_buffer_profile_config,
|
||||
};
|
||||
use crate::config::{BufferConfig, WorkloadProfile};
|
||||
use crate::storage_api::error::{StorageError, UnreadableBucketConfig};
|
||||
use rustfs_config::KI_B;
|
||||
use rustfs_s3_types::EventName;
|
||||
use s3s::dto::{
|
||||
FilterRule, FilterRuleName, NotificationConfiguration, NotificationConfigurationFilter, QueueConfiguration, S3KeyFilter,
|
||||
};
|
||||
|
||||
/// rustfs/backlog#1734: an unreadable notification config is isolated to
|
||||
/// its bucket. It must not abort setup for every other bucket (a storage
|
||||
/// fault still does, so reconciliation retries) and must not read as
|
||||
/// missing, which would clear the bucket's rules.
|
||||
#[test]
|
||||
fn unreadable_notification_config_is_isolated_to_its_bucket() {
|
||||
let unreadable = StorageError::other(UnreadableBucketConfig {
|
||||
bucket: "b".to_string(),
|
||||
config_file: "notification.xml".to_string(),
|
||||
raw_len: 5,
|
||||
});
|
||||
assert!(matches!(
|
||||
classify_bucket_notification_lookup("b", Err(unreadable)),
|
||||
Ok(BucketNotificationLookup::Unreadable)
|
||||
));
|
||||
assert!(matches!(
|
||||
classify_bucket_notification_lookup("b", Ok(None)),
|
||||
Ok(BucketNotificationLookup::Missing)
|
||||
));
|
||||
assert!(matches!(
|
||||
classify_bucket_notification_lookup("b", Err(StorageError::ErasureReadQuorum)),
|
||||
Err(rustfs_notify::NotificationError::StorageNotAvailable(_))
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_buffer_profile_config_returns_fallback_when_primary_is_invalid() {
|
||||
let invalid_primary = WorkloadProfile::Custom(BufferConfig {
|
||||
@@ -1488,6 +1354,21 @@ mod tests {
|
||||
assert!(resolved.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn corrupt_notification_config_keeps_same_incarnation_rules_until_withdrawn() {
|
||||
let bucket = "unreadable-notify-config";
|
||||
let id = uuid::Uuid::new_v4();
|
||||
let xml = br#"<NotificationConfiguration><QueueConfiguration><Id>test</Id><Queue>arn:rustfs:sqs:us-east-1:primary:webhook</Queue><Event>s3:ObjectCreated:*</Event></QueueConfiguration></NotificationConfiguration>"#;
|
||||
super::publish_bucket_notification_configuration(bucket, Some((xml, id)));
|
||||
let system = rustfs_notify::notification_system().expect("notification system created by metadata install");
|
||||
assert!(system.has_subscriber(bucket, &EventName::ObjectCreatedPut).await);
|
||||
super::publish_bucket_notification_configuration(bucket, Some((b"malformed xml", id)));
|
||||
assert!(system.has_subscriber(bucket, &EventName::ObjectCreatedPut).await);
|
||||
|
||||
super::publish_bucket_notification_configuration(bucket, None);
|
||||
assert!(!system.has_subscriber(bucket, &EventName::ObjectCreatedPut).await);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn notification_config_to_event_rules_preserves_target_and_filters() {
|
||||
let cfg = NotificationConfiguration {
|
||||
|
||||
@@ -17,10 +17,9 @@
|
||||
#[cfg(test)]
|
||||
pub(super) use crate::storage_api::on_demand_migration::test_support;
|
||||
pub(super) use crate::storage_api::on_demand_migration::{
|
||||
BUCKET_CONFIG_PUBLISH_HOOK, BUCKET_META_PREFIX, BUCKET_ON_DEMAND_MIGRATION_CONFIG, ECStore, HTTPPreconditions, HTTPRangeSpec,
|
||||
NamespaceLocking, ObjectOperations, ObjectOptions, RUSTFS_META_BUCKET, StorageError, WriteCompletion,
|
||||
get_lock_acquire_timeout, get_on_demand_migration_config, get_on_demand_migration_config_in, read_config_with_metadata,
|
||||
remote_s3_client, save_config_with_opts,
|
||||
BUCKET_META_PREFIX, ECStore, HTTPPreconditions, HTTPRangeSpec, NamespaceLocking, ObjectOperations, ObjectOptions,
|
||||
RUSTFS_META_BUCKET, StorageError, WriteCompletion, get_lock_acquire_timeout, get_on_demand_migration_config,
|
||||
get_on_demand_migration_config_in, read_config_with_metadata, remote_s3_client, save_config_with_opts,
|
||||
};
|
||||
|
||||
pub(super) async fn local_node_name() -> String {
|
||||
|
||||
@@ -19,7 +19,7 @@
|
||||
//! [`SourceClient`], a circuit breaker, a negative cache, a per-key
|
||||
//! singleflight table, a pull concurrency limit and counters. Its lifecycle
|
||||
//! follows the bucket metadata cache through the publish hook registered in
|
||||
//! [`BUCKET_CONFIG_PUBLISH_HOOK`]; the hook fires on every cache install
|
||||
//! `BUCKET_CONFIG_PUBLISH_HOOK`. It fires on every cache install
|
||||
//! path (initial load, admin update, peer reload, refresh loop, lazy load).
|
||||
//!
|
||||
//! Change detection compares the config by value (`PartialEq`) rather than
|
||||
@@ -52,7 +52,6 @@ use super::stats::{GaugeGuard, OdmStats, OdmStatsSnapshot, PullFailureReason};
|
||||
use super::storage_api::remote_s3_client::{
|
||||
PathStyle as ClientPathStyle, RemoteCredentials, RemoteS3ClientError, RemoteS3RetryPolicy,
|
||||
};
|
||||
use super::storage_api::{BUCKET_CONFIG_PUBLISH_HOOK, BUCKET_ON_DEMAND_MIGRATION_CONFIG};
|
||||
use parking_lot::{Mutex, RwLock};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
@@ -746,20 +745,8 @@ impl OnDemandMigrationSys {
|
||||
self.write_back.read().clone()
|
||||
}
|
||||
|
||||
/// Registers `publish` as the bucket-metadata publish hook. Returns
|
||||
/// `false` when a hook was already registered.
|
||||
pub fn register_config_hook(&'static self) -> bool {
|
||||
BUCKET_CONFIG_PUBLISH_HOOK
|
||||
.set(Box::new(move |bucket, config_file, stored| {
|
||||
if config_file == BUCKET_ON_DEMAND_MIGRATION_CONFIG {
|
||||
self.publish_stored(bucket, stored.map(|(bytes, _, incarnation)| (bytes, incarnation)));
|
||||
}
|
||||
}))
|
||||
.is_ok()
|
||||
}
|
||||
|
||||
/// Corrupt persisted bytes withdraw state synchronously, just like deletion.
|
||||
fn publish_stored(&'static self, bucket: &str, stored: Option<(&[u8], uuid::Uuid)>) {
|
||||
pub(crate) fn publish_stored(&'static self, bucket: &str, stored: Option<(&[u8], uuid::Uuid)>) {
|
||||
let incarnation_id = stored.map(|(_, id)| id).unwrap_or_default();
|
||||
match stored.map(|(bytes, _)| OnDemandMigrationConfig::from_json(bytes)).transpose() {
|
||||
Ok(config) => self.publish_for_incarnation(bucket, incarnation_id, config.as_ref()),
|
||||
|
||||
@@ -16,7 +16,6 @@ use super::{
|
||||
module_switch::{resolve_notify_module_state, validate_notify_module_env, with_refreshed_notify_module_state_from},
|
||||
refresh_persisted_module_switches_from, runtime_sources,
|
||||
};
|
||||
use crate::init::reconcile_persisted_bucket_notification_configurations;
|
||||
use crate::storage_api::server::event::{
|
||||
EventArgs as EcstoreEventArgs, read_existing_server_config_no_lock, register_event_dispatch_hook,
|
||||
with_server_config_read_lock,
|
||||
@@ -36,7 +35,6 @@ use tokio_util::sync::CancellationToken;
|
||||
use tracing::{info, instrument, warn};
|
||||
|
||||
static NOTIFY_RUNTIME_RECONCILED: AtomicBool = AtomicBool::new(false);
|
||||
static NOTIFY_BUCKET_RULES_RECONCILED: AtomicBool = AtomicBool::new(false);
|
||||
static ECSTORE_EVENT_DISPATCH_HOOK: OnceLock<()> = OnceLock::new();
|
||||
|
||||
// Serialize local notification lifecycle publication with the periodic and
|
||||
@@ -59,7 +57,6 @@ pub(crate) fn mark_event_notifier_reconciled() {
|
||||
|
||||
pub(crate) fn mark_event_notifier_unreconciled() {
|
||||
NOTIFY_RUNTIME_RECONCILED.store(false, Ordering::Release);
|
||||
NOTIFY_BUCKET_RULES_RECONCILED.store(false, Ordering::Release);
|
||||
}
|
||||
|
||||
pub(crate) async fn with_notify_runtime_reconcile_lock<Fut, T>(operation: Fut) -> T
|
||||
@@ -70,18 +67,6 @@ where
|
||||
operation.await
|
||||
}
|
||||
|
||||
fn are_bucket_notification_rules_reconciled() -> bool {
|
||||
NOTIFY_BUCKET_RULES_RECONCILED.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
fn mark_bucket_notification_rules_reconciled() {
|
||||
NOTIFY_BUCKET_RULES_RECONCILED.store(true, Ordering::Release);
|
||||
}
|
||||
|
||||
fn should_reconcile_bucket_notification_rules(runtime_changed: bool, notify_enabled: bool) -> bool {
|
||||
notify_enabled && (runtime_changed || !are_bucket_notification_rules_reconciled())
|
||||
}
|
||||
|
||||
pub fn refresh_notify_module_enabled() -> bool {
|
||||
let enabled = resolve_notify_module_state().enabled;
|
||||
crate::module_switches::set_notify_module_enabled(enabled);
|
||||
@@ -111,6 +96,8 @@ fn convert_ecstore_event_args(args: EcstoreEventArgs) -> Option<NotifyEventArgs>
|
||||
};
|
||||
|
||||
Some(NotifyEventArgs {
|
||||
bucket_incarnation_id: args.bucket_incarnation_id,
|
||||
bucket_removed_targets: None,
|
||||
event_name,
|
||||
bucket_name: args.bucket_name,
|
||||
object: convert_ecstore_object_info(args.object),
|
||||
@@ -216,24 +203,11 @@ async fn reconcile_event_notifier_from_store_unlocked(
|
||||
.await
|
||||
.map_err(|err| NotificationError::Initialization(format!("failed to refresh notify module switch: {err}")))??;
|
||||
|
||||
let runtime_changed = transition.is_some();
|
||||
if let Some(transition) = transition {
|
||||
transition.wait().await?;
|
||||
}
|
||||
|
||||
ensure_event_notifier_converged(&system)?;
|
||||
if should_reconcile_bucket_notification_rules(runtime_changed, is_notify_module_enabled()) {
|
||||
let configured_bucket_count = reconcile_persisted_bucket_notification_configurations(store).await?;
|
||||
mark_bucket_notification_rules_reconciled();
|
||||
info!(
|
||||
event = EVENT_NOTIFY_RUNTIME_RECONCILE,
|
||||
component = "notify",
|
||||
subsystem = "bucket_rules",
|
||||
configured_bucket_count,
|
||||
"Persisted bucket notification rules reconciled"
|
||||
);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
.await;
|
||||
@@ -427,9 +401,8 @@ mod tests {
|
||||
set_persisted_module_switches,
|
||||
};
|
||||
use super::{
|
||||
convert_ecstore_object_info, init_event_notifier_with_store, mark_bucket_notification_rules_reconciled,
|
||||
parse_host_and_port, run_persisted_event_notifier_reconciler, should_reconcile_bucket_notification_rules,
|
||||
with_notify_runtime_reconcile_lock,
|
||||
convert_ecstore_object_info, init_event_notifier_with_store, parse_host_and_port,
|
||||
run_persisted_event_notifier_reconciler, with_notify_runtime_reconcile_lock,
|
||||
};
|
||||
use crate::server::is_event_notifier_reconciled;
|
||||
use crate::storage_api::server::event::StorageObjectInfo;
|
||||
@@ -594,30 +567,6 @@ mod tests {
|
||||
assert_eq!(converted.transitioned_tier.as_deref(), Some("DEEP_ARCHIVE"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bucket_rule_reconcile_runs_once_and_after_runtime_change() {
|
||||
super::mark_event_notifier_unreconciled();
|
||||
|
||||
assert!(
|
||||
should_reconcile_bucket_notification_rules(false, true),
|
||||
"enabled notify must restore bucket rules until the first successful replay"
|
||||
);
|
||||
|
||||
mark_bucket_notification_rules_reconciled();
|
||||
assert!(
|
||||
!should_reconcile_bucket_notification_rules(false, true),
|
||||
"steady-state reconcile must not rescan buckets every tick"
|
||||
);
|
||||
assert!(
|
||||
should_reconcile_bucket_notification_rules(true, true),
|
||||
"target runtime changes must replay persisted bucket rules"
|
||||
);
|
||||
assert!(
|
||||
!should_reconcile_bucket_notification_rules(true, false),
|
||||
"disabled notify must not load external target rules"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn persisted_reconciler_converges_after_one_injected_tick() {
|
||||
let persisted_generation = Arc::new(AtomicUsize::new(1));
|
||||
|
||||
@@ -16,8 +16,9 @@ use crate::app::object::OnDemandMigrationWriteBack;
|
||||
use crate::module_switches::{on_demand_migration_enabled_from_env, set_on_demand_migration_module_enabled};
|
||||
use crate::on_demand_migration::OnDemandMigrationSys;
|
||||
use crate::storage_api::startup::bucket_metadata::{
|
||||
ECStore, Error as StorageError, Result as StorageResult, get_global_replication_pool, init_bucket_metadata_sys,
|
||||
reconcile_bucket_resync_target_intents, try_migrate_bucket_metadata, try_migrate_iam_config,
|
||||
BUCKET_CONFIG_PUBLISH_HOOK, BUCKET_NOTIFICATION_CONFIG, BUCKET_ON_DEMAND_MIGRATION_CONFIG, ECStore, Error as StorageError,
|
||||
Result as StorageResult, get_global_replication_pool, init_bucket_metadata_sys, reconcile_bucket_resync_target_intents,
|
||||
try_migrate_bucket_metadata, try_migrate_iam_config,
|
||||
};
|
||||
use std::{
|
||||
future::Future,
|
||||
@@ -55,7 +56,7 @@ const STARTUP_BACKGROUND_STATUS_SUCCEEDED: f64 = 1.0;
|
||||
const STARTUP_BACKGROUND_STATUS_RUNNING: f64 = 2.0;
|
||||
const STARTUP_BACKGROUND_STATUS_CANCELED: f64 = 3.0;
|
||||
|
||||
pub(crate) async fn init_embedded_bucket_metadata_runtime(store: Arc<ECStore>, ctx: &CancellationToken) -> IoResult<Vec<String>> {
|
||||
pub(crate) async fn init_embedded_bucket_metadata_runtime(store: Arc<ECStore>, ctx: &CancellationToken) -> IoResult<()> {
|
||||
let buckets_list = retry_metadata_startup(ctx, "list_buckets", || async {
|
||||
store.list_buckets_for_metadata_startup().await.map_err(IoError::other)
|
||||
})
|
||||
@@ -67,12 +68,12 @@ pub(crate) async fn init_embedded_bucket_metadata_runtime(store: Arc<ECStore>, c
|
||||
init_on_demand_migration_runtime();
|
||||
init_bucket_metadata_sys(store.clone(), buckets.clone()).await;
|
||||
retry_metadata_startup(ctx, "iam_migration", || try_migrate_iam_config(store.clone())).await?;
|
||||
spawn_bucket_resync_startup_reconcile(buckets.clone(), ctx.clone(), false);
|
||||
spawn_bucket_resync_startup_reconcile(buckets, ctx.clone(), false);
|
||||
|
||||
Ok(buckets)
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) async fn init_bucket_metadata_runtime(store: Arc<ECStore>, ctx: CancellationToken) -> IoResult<Vec<String>> {
|
||||
pub(crate) async fn init_bucket_metadata_runtime(store: Arc<ECStore>, ctx: CancellationToken) -> IoResult<()> {
|
||||
let buckets_list = retry_metadata_startup(&ctx, "list_buckets", || async {
|
||||
store.list_buckets_for_metadata_startup().await.map_err(IoError::other)
|
||||
})
|
||||
@@ -85,9 +86,9 @@ pub(crate) async fn init_bucket_metadata_runtime(store: Arc<ECStore>, ctx: Cance
|
||||
retry_metadata_startup(&ctx, "iam_migration", || try_migrate_iam_config(store.clone())).await?;
|
||||
init_on_demand_migration_runtime();
|
||||
init_bucket_metadata_sys(store, buckets.clone()).await;
|
||||
spawn_bucket_resync_startup_reconcile(buckets.clone(), ctx, true);
|
||||
spawn_bucket_resync_startup_reconcile(buckets, ctx, true);
|
||||
|
||||
Ok(buckets)
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn retry_metadata_startup<T, Operation, OperationFuture>(
|
||||
@@ -166,7 +167,16 @@ fn init_on_demand_migration_runtime() {
|
||||
let sys = OnDemandMigrationSys::get();
|
||||
sys.set_module_enabled(enabled);
|
||||
sys.set_write_back(Arc::new(OnDemandMigrationWriteBack::new()));
|
||||
let hook_registered = sys.register_config_hook();
|
||||
let hook_registered = BUCKET_CONFIG_PUBLISH_HOOK
|
||||
.set(Box::new(move |bucket, file, stored| {
|
||||
let stored = stored.map(|(bytes, _, incarnation_id)| (bytes, incarnation_id));
|
||||
match file {
|
||||
BUCKET_ON_DEMAND_MIGRATION_CONFIG => sys.publish_stored(bucket, stored),
|
||||
BUCKET_NOTIFICATION_CONFIG => crate::init::publish_bucket_notification_configuration(bucket, stored),
|
||||
_ => {}
|
||||
}
|
||||
}))
|
||||
.is_ok();
|
||||
tracing::info!(
|
||||
event = EVENT_ON_DEMAND_MIGRATION_RUNTIME_INITIALIZED,
|
||||
component = LOG_COMPONENT_STARTUP_BUCKET_METADATA,
|
||||
|
||||
@@ -12,7 +12,6 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::init::add_bucket_notification_configuration;
|
||||
use crate::storage_api::startup::notification::{EndpointServerPools, Result as StorageResult, new_global_notification_sys};
|
||||
use std::{
|
||||
future::Future,
|
||||
@@ -27,17 +26,13 @@ const LOG_SUBSYSTEM_STARTUP: &str = "startup";
|
||||
const EVENT_EMBEDDED_OPTIONAL_SERVICE_SKIPPED: &str = "embedded_optional_service_skipped";
|
||||
const EVENT_NOTIFICATION_SYSTEM_INITIALIZATION_FAILED: &str = "notification_system_initialization_failed";
|
||||
|
||||
pub(crate) async fn init_embedded_notification_runtime(endpoint_pools: EndpointServerPools, buckets: Vec<String>) {
|
||||
add_bucket_notification_configuration(buckets).await;
|
||||
|
||||
pub(crate) async fn init_embedded_notification_runtime(endpoint_pools: EndpointServerPools) {
|
||||
if let Err(err) = init_notification_system(endpoint_pools).await {
|
||||
log_embedded_optional_service_skipped("notification", err);
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn init_notification_runtime(endpoint_pools: EndpointServerPools, buckets: Vec<String>) -> Result<()> {
|
||||
add_bucket_notification_configuration(buckets).await;
|
||||
|
||||
pub(crate) async fn init_notification_runtime(endpoint_pools: EndpointServerPools) -> Result<()> {
|
||||
init_notification_system(endpoint_pools).await.map_err(|err| {
|
||||
error!(
|
||||
event = EVENT_NOTIFICATION_SYSTEM_INITIALIZATION_FAILED,
|
||||
|
||||
@@ -68,11 +68,11 @@ pub(crate) async fn init_embedded_startup_runtime_services(
|
||||
server_ctx: Arc<ServerContextSlot>,
|
||||
) -> Result<EmbeddedStartupServiceRuntime> {
|
||||
init_embedded_optional_service_runtime(config, store.clone()).await;
|
||||
let buckets = init_embedded_bucket_metadata_runtime(store.clone(), &ctx).await?;
|
||||
init_embedded_bucket_metadata_runtime(store.clone(), &ctx).await?;
|
||||
let iam_bootstrap = init_embedded_iam_runtime(store, ctx.clone(), readiness, server_ctx.clone())
|
||||
.await
|
||||
.map_err(|err| std::io::Error::other(format!("IAM bootstrap setup: {err}")))?;
|
||||
init_embedded_notification_runtime(endpoint_pools, buckets).await;
|
||||
init_embedded_notification_runtime(endpoint_pools).await;
|
||||
|
||||
Ok(EmbeddedStartupServiceRuntime {
|
||||
iam_bootstrap,
|
||||
@@ -108,7 +108,7 @@ pub(crate) async fn init_startup_runtime_services(
|
||||
init_buffer_profile_system(config);
|
||||
init_deadlock_detector_runtime();
|
||||
|
||||
let buckets = init_bucket_metadata_runtime(store.clone(), ctx.clone()).await?;
|
||||
init_bucket_metadata_runtime(store.clone(), ctx.clone()).await?;
|
||||
let iam_bootstrap = init_iam_runtime(store.clone(), ctx.clone(), readiness, state_manager, server_ctx.clone()).await?;
|
||||
|
||||
// Audit initialization requires the AppContext (server config + object store)
|
||||
@@ -120,7 +120,7 @@ pub(crate) async fn init_startup_runtime_services(
|
||||
// bucket metadata itself, and its first pass runs off this path.
|
||||
spawn_site_replication_reconcile_task(ctx.clone());
|
||||
let oidc_recovery = init_auth_integrations(ctx.child_token()).await?;
|
||||
init_notification_runtime(endpoint_pools, buckets).await?;
|
||||
init_notification_runtime(endpoint_pools).await?;
|
||||
let enable_scanner = init_background_service_runtime(store.clone()).await?;
|
||||
init_observability_runtime(store.clone(), ctx.clone()).await;
|
||||
let heartbeat = start_heartbeat_runtime(heartbeat_config.clone(), heartbeat_nodes, &ctx)?;
|
||||
|
||||
@@ -48,6 +48,7 @@ use tracing::{Instrument, info_span, warn};
|
||||
|
||||
/// The request fields used to build an audit entry, shared by both request edges.
|
||||
pub(crate) struct AuditRequestView<'a> {
|
||||
bucket_generation: Option<&'a super::authorization::BucketGenerationGuard>,
|
||||
headers: &'a HeaderMap,
|
||||
uri: &'a Uri,
|
||||
pub(crate) req_info: Option<&'a ReqInfo>,
|
||||
@@ -59,6 +60,7 @@ pub(crate) struct AuditRequestView<'a> {
|
||||
impl<'a> AuditRequestView<'a> {
|
||||
pub(crate) fn legacy<T>(req: &'a S3Request<T>) -> Self {
|
||||
Self {
|
||||
bucket_generation: req.extensions.get(),
|
||||
headers: &req.headers,
|
||||
uri: &req.uri,
|
||||
req_info: req.extensions.get::<ReqInfo>(),
|
||||
@@ -79,8 +81,10 @@ impl<'a> AuditRequestView<'a> {
|
||||
client_info: Option<&'a ClientInfo>,
|
||||
remote_addr: Option<SocketAddr>,
|
||||
request_context: Option<&'a RequestContext>,
|
||||
bucket_generation: Option<&'a super::authorization::BucketGenerationGuard>,
|
||||
) -> Self {
|
||||
Self {
|
||||
bucket_generation,
|
||||
headers,
|
||||
uri,
|
||||
req_info,
|
||||
@@ -495,7 +499,12 @@ impl OperationHelper {
|
||||
// initialize event builder
|
||||
// object is a placeholder that must be set later using the `object()` method.
|
||||
let event_builder = if notify_enabled {
|
||||
let incarnation_id = req
|
||||
.bucket_generation
|
||||
.filter(|guard| guard.bucket == bucket)
|
||||
.map(|guard| guard.incarnation_id);
|
||||
let mut event_builder = EventArgsBuilder::new(event, bucket, convert_ecstore_object_info(event_object))
|
||||
.bucket_incarnation_id(incarnation_id)
|
||||
.host(get_request_host(req.headers))
|
||||
.port(get_request_port(req.headers))
|
||||
.user_agent(get_request_user_agent(req.headers))
|
||||
@@ -545,6 +554,16 @@ impl OperationHelper {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn with_bucket_removed_targets(mut self, targets: rustfs_notify::rules::TargetIdSet) -> Self {
|
||||
if let Self::Enabled(state) = &mut self {
|
||||
state.event_builder = state
|
||||
.event_builder
|
||||
.take()
|
||||
.map(|builder| builder.bucket_removed_targets(targets));
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
/// Sets the ObjectInfo for event notification.
|
||||
pub fn object(mut self, object_info: ObjectInfo) -> Self {
|
||||
if let Self::Enabled(state) = &mut self
|
||||
|
||||
@@ -69,9 +69,9 @@ pub(crate) use storage_api::{
|
||||
ecfs_extend_consumer, ecstore_admin, ecstore_bucket, ecstore_capacity, ecstore_client, ecstore_cluster, ecstore_compression,
|
||||
ecstore_config, ecstore_data_usage, ecstore_disk, ecstore_error, ecstore_event, ecstore_layout, ecstore_metrics,
|
||||
ecstore_notification, ecstore_rebalance, ecstore_rio, ecstore_rpc, ecstore_set_disk, ecstore_storage, ecstore_tier,
|
||||
encode_tags, find_local_disk_by_ref, get_bucket_cors_config, get_bucket_metadata, get_bucket_notification_config,
|
||||
get_bucket_sse_config, get_bucket_website_config_for_store, get_local_server_property, get_lock_acquire_timeout,
|
||||
head_prefix_consumer, helper_consumer, init_background_replication, init_bucket_metadata_sys, init_ecstore_config,
|
||||
encode_tags, find_local_disk_by_ref, get_bucket_cors_config, get_bucket_metadata, get_bucket_sse_config,
|
||||
get_bucket_website_config_for_store, get_local_server_property, get_lock_acquire_timeout, head_prefix_consumer,
|
||||
helper_consumer, init_background_replication, init_bucket_metadata_sys, init_ecstore_config,
|
||||
init_local_disks_with_instance_ctx, init_lock_clients, is_err_bucket_not_found, is_err_object_not_found,
|
||||
is_err_version_not_found, is_valid_storage_class, options_consumer, prewarm_local_disk_id_map_with_instance_ctx, read_config,
|
||||
record_replication_proxy, rpc_consumer, runtime_sources_consumer, s3_api_consumer, table_catalog_path_hash, to_s3s_etag,
|
||||
|
||||
@@ -1334,12 +1334,6 @@ pub(crate) async fn all_local_disk() -> Vec<DiskStore> {
|
||||
ecstore_storage::all_local_disk().await
|
||||
}
|
||||
|
||||
pub(crate) async fn get_bucket_notification_config(
|
||||
bucket: &str,
|
||||
) -> Result<Option<rustfs_gateway_types::persistence::PersistedNotificationConfiguration>> {
|
||||
ecstore_bucket::metadata_sys::get_notification_config(bucket).await
|
||||
}
|
||||
|
||||
pub(crate) async fn init_bucket_metadata_sys(api: Arc<ECStore>, buckets: Vec<String>) {
|
||||
ecstore_bucket::metadata_sys::init_bucket_metadata_sys(api, buckets).await;
|
||||
}
|
||||
|
||||
@@ -432,12 +432,10 @@ pub(crate) mod startup {
|
||||
}
|
||||
|
||||
pub(crate) mod bucket_metadata {
|
||||
pub(crate) mod contract {
|
||||
pub(crate) mod bucket {
|
||||
pub(crate) use super::super::super::super::storage_contracts::{BucketOperations, BucketOptions};
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) use crate::storage::storage_api::ecstore_bucket::{
|
||||
metadata::{BUCKET_NOTIFICATION_CONFIG, BUCKET_ON_DEMAND_MIGRATION_CONFIG},
|
||||
metadata_sys::BUCKET_CONFIG_PUBLISH_HOOK,
|
||||
};
|
||||
pub(crate) use crate::storage::storage_api::{
|
||||
ECStore, Error, Result, get_global_replication_pool, init_bucket_metadata_sys,
|
||||
reconcile_bucket_resync_target_intents, try_migrate_bucket_metadata, try_migrate_iam_config,
|
||||
@@ -466,9 +464,9 @@ pub(crate) mod startup {
|
||||
}
|
||||
|
||||
pub(crate) use crate::storage::storage_api::ecstore_bucket::config_parse_mode::validate_bucket_config_parse_mode_env;
|
||||
pub(crate) use crate::storage::storage_api::ecstore_bucket::utils::deserialize;
|
||||
pub(crate) use crate::storage::storage_api::{
|
||||
get_bucket_notification_config, process_lambda_configurations, process_queue_configurations,
|
||||
process_topic_configurations,
|
||||
process_lambda_configurations, process_queue_configurations, process_topic_configurations,
|
||||
};
|
||||
|
||||
pub(crate) mod concurrency {
|
||||
@@ -589,9 +587,8 @@ pub(crate) mod on_demand_migration {
|
||||
pub(crate) use crate::storage::storage_api::contract::{
|
||||
namespace::NamespaceLocking, object::HTTPPreconditions, object::ObjectOperations, range::HTTPRangeSpec,
|
||||
};
|
||||
pub(crate) use crate::storage::storage_api::ecstore_bucket::metadata::BUCKET_ON_DEMAND_MIGRATION_CONFIG;
|
||||
pub(crate) use crate::storage::storage_api::ecstore_bucket::metadata_sys::{
|
||||
BUCKET_CONFIG_PUBLISH_HOOK, get_on_demand_migration_config, get_on_demand_migration_config_in,
|
||||
get_on_demand_migration_config, get_on_demand_migration_config_in,
|
||||
};
|
||||
pub(crate) use crate::storage::storage_api::ecstore_bucket::remote_s3_client;
|
||||
pub(crate) use crate::storage::storage_api::ecstore_config::com::{read_config_with_metadata, save_config_with_opts};
|
||||
|
||||
Reference in New Issue
Block a user