Compare commits

...

6 Commits

58 changed files with 925 additions and 639 deletions
Generated
+1
View File
@@ -10843,6 +10843,7 @@ dependencies = [
"tracing",
"tracing-subscriber",
"url",
"uuid",
]
[[package]]
@@ -864,6 +864,70 @@ async fn test_webhook_event_delivery_and_filtering() -> TestResult {
Ok(())
}
#[tokio::test]
async fn test_webhook_subscriptions_follow_bucket_incarnation() -> TestResult {
init_logging();
let (endpoint, mut rx, collector) = spawn_event_collector().await?;
let allowed_origin = endpoint_origin(&endpoint)?;
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server_with_env(vec![], &[(ENV_OUTBOUND_ALLOW_ORIGINS, allowed_origin.as_str())])
.await?;
enable_notify_module(&env).await?;
let target = "incarnation";
configure_webhook_target(&env, target, &endpoint).await?;
wait_for_target_registered(&env, target).await?;
let client = env.create_s3_client();
let bucket = "notify-incarnation";
let control_bucket = "notify-incarnation-control";
for name in [bucket, control_bucket] {
client.create_bucket().bucket(name).send().await?;
put_notification_config(&client, name, target, "events/", ".txt").await?;
}
let put = |bucket, key| {
client
.put_object()
.bucket(bucket)
.key(key)
.body(ByteStream::from_static(b"event fixture"))
.send()
};
let wait = Duration::from_secs(20);
let original_key = "events/original.txt";
put(bucket, original_key).await?;
wait_for_event(&mut rx, original_key, "s3:ObjectCreated:", wait).await?;
assert!(client.delete_bucket().bucket(bucket).send().await.is_err());
let after_failed_delete = "events/retained.txt";
put(bucket, after_failed_delete).await?;
wait_for_event(&mut rx, after_failed_delete, "s3:ObjectCreated:", wait).await?;
for key in [original_key, after_failed_delete] {
client.delete_object().bucket(bucket).key(key).send().await?;
}
client.delete_bucket().bucket(bucket).send().await?;
client.create_bucket().bucket(bucket).send().await?;
let config = client.get_bucket_notification_configuration().bucket(bucket).send().await?;
assert!(config.queue_configurations().is_empty());
let new_key = "events/fresh.txt";
let sentinel_key = "events/sentinel.txt";
put(bucket, new_key).await?;
put(control_bucket, sentinel_key).await?;
let events = collect_until(&mut rx, sentinel_key, wait, Duration::from_secs(2)).await;
assert!(events.iter().any(|event| event_key(event).as_deref() == Some(sentinel_key)));
assert!(events.iter().all(|event| event_key(event).as_deref() != Some(new_key)));
put_notification_config(&client, bucket, target, "events/", ".txt").await?;
let subscribed_key = "events/subscribed.txt";
put(bucket, subscribed_key).await?;
wait_for_event(&mut rx, subscribed_key, "s3:ObjectCreated:", wait).await?;
env.stop_server();
collector.abort();
Ok(())
}
/// An event queued while the target endpoint rejects delivery survives on the
/// durable store and is redelivered once the endpoint comes back.
#[tokio::test]
@@ -51,15 +51,20 @@ impl LcAuditEvent {
}
}
pub(crate) fn emit_transition_failed_event(object: ObjectInfo) {
emit_lifecycle_event(EventName::ObjectTransitionFailed, object, LIFECYCLE_TRANSITION_USER_AGENT);
pub(crate) fn emit_transition_failed_event(object: ObjectInfo, incarnation_id: Option<uuid::Uuid>) {
emit_lifecycle_event(EventName::ObjectTransitionFailed, object, LIFECYCLE_TRANSITION_USER_AGENT, incarnation_id);
}
pub(crate) fn emit_transition_complete_event(object: ObjectInfo) {
emit_lifecycle_event(EventName::ObjectTransitionComplete, object, LIFECYCLE_TRANSITION_USER_AGENT);
pub(crate) fn emit_transition_complete_event(object: ObjectInfo, incarnation_id: Option<uuid::Uuid>) {
emit_lifecycle_event(
EventName::ObjectTransitionComplete,
object,
LIFECYCLE_TRANSITION_USER_AGENT,
incarnation_id,
);
}
pub(crate) fn emit_transitioned_expiration_event(source: &ObjectInfo, deleted: &ObjectInfo) {
pub(crate) fn emit_transitioned_expiration_event(source: &ObjectInfo, deleted: &ObjectInfo, incarnation_id: uuid::Uuid) {
let event_name = transitioned_expiration_event_name(source.delete_marker, deleted.delete_marker);
let object = ObjectInfo {
bucket: source.bucket.clone(),
@@ -69,12 +74,17 @@ pub(crate) fn emit_transitioned_expiration_event(source: &ObjectInfo, deleted: &
delete_marker: source.delete_marker,
..Default::default()
};
emit_lifecycle_event(event_name, object, LIFECYCLE_EXPIRY_USER_AGENT);
emit_lifecycle_event(event_name, object, LIFECYCLE_EXPIRY_USER_AGENT, Some(incarnation_id));
}
pub(crate) fn emit_non_transitioned_expiration_event(action: IlmAction, source: &ObjectInfo, deleted: ObjectInfo) {
pub(crate) fn emit_non_transitioned_expiration_event(
action: IlmAction,
source: &ObjectInfo,
deleted: ObjectInfo,
incarnation_id: uuid::Uuid,
) {
let event_name = non_transitioned_expiration_event_name(action, source.delete_marker, deleted.delete_marker);
emit_lifecycle_event(event_name, deleted, LIFECYCLE_EXPIRY_USER_AGENT);
emit_lifecycle_event(event_name, deleted, LIFECYCLE_EXPIRY_USER_AGENT, Some(incarnation_id));
}
/// Emit the lifecycle expiration event for one version removed by the batch
@@ -83,9 +93,15 @@ pub(crate) fn emit_non_transitioned_expiration_event(action: IlmAction, source:
/// That path never sent events, so a successful noncurrent-version expiry was
/// invisible to notification subscribers even though the equivalent
/// current-version path emits one (backlog#2202).
pub(crate) fn emit_noncurrent_expiration_event(bucket: &str, target: &ObjectToDelete, deleted: &DeletedObject, failed: bool) {
pub(crate) fn emit_noncurrent_expiration_event(
bucket: &str,
target: &ObjectToDelete,
deleted: &DeletedObject,
failed: bool,
incarnation_id: uuid::Uuid,
) {
if let Some((event_name, object)) = noncurrent_expiration_event(bucket, target, deleted, failed) {
emit_lifecycle_event(event_name, object, LIFECYCLE_EXPIRY_USER_AGENT);
emit_lifecycle_event(event_name, object, LIFECYCLE_EXPIRY_USER_AGENT, Some(incarnation_id));
}
}
@@ -131,9 +147,10 @@ fn noncurrent_expiration_event(
Some((event_name, object))
}
fn emit_lifecycle_event(event_name: EventName, object: ObjectInfo, user_agent: &str) {
fn emit_lifecycle_event(event_name: EventName, object: ObjectInfo, user_agent: &str, incarnation_id: Option<uuid::Uuid>) {
send_event(EventArgs {
event_name: event_name.to_string(),
bucket_incarnation_id: incarnation_id,
bucket_name: object.bucket.clone(),
object,
user_agent: user_agent.to_string(),
@@ -2166,9 +2166,10 @@ impl TransitionState {
..Default::default()
};
if let Err(err) =
let (incarnation_id, transition_result) =
transition_object(api.clone(), &task.obj_info, LcAuditEvent::new(task.event.clone(), task.src.clone()))
.await
.await;
if let Err(err) = transition_result
{
if let (Some(job_id), Some(result_key)) = (task.manual_job_id, task.manual_result_key.as_deref()) {
record_manual_transition_worker_result_for_task_with_reason(
@@ -2195,7 +2196,7 @@ impl TransitionState {
"Lifecycle tier operation failed"
);
}
emit_transition_failed_event(obj_info_for_event);
emit_transition_failed_event(obj_info_for_event, incarnation_id);
} else {
if let (Some(job_id), Some(result_key)) = (task.manual_job_id, task.manual_result_key.as_deref()) {
record_manual_transition_worker_result_for_task(
@@ -2217,7 +2218,7 @@ impl TransitionState {
}
transition_state.add_lastday_stats(&task.event.storage_class, ts);
emit_transition_complete_event(obj_info_for_event);
emit_transition_complete_event(obj_info_for_event, incarnation_id);
}
// Release the dedup claim so a later lifecycle pass can
// re-transition this object if needed (rustfs/backlog#1268).
@@ -5121,7 +5122,7 @@ async fn expire_transitioned_object_with_lock_lost_signal(
}
};
schedule_lifecycle_replication_delete_if_needed(oi, &dobj).await;
schedule_lifecycle_replication_delete_if_needed(oi, &dobj, bucket_incarnation_id).await;
// The transitioned version is gone; evict any cached body for this object
// so it does not linger until TTL (ODC-26).
@@ -5134,7 +5135,7 @@ async fn expire_transitioned_object_with_lock_lost_signal(
//audit_log_lifecycle(oi, ILMExpiry, tags);
emit_transitioned_expiration_event(oi, &dobj);
emit_transitioned_expiration_event(oi, &dobj, bucket_incarnation_id);
Ok(dobj)
}
@@ -5152,12 +5153,12 @@ pub fn gen_transition_objname(bucket: &str) -> Result<String, Error> {
Ok(obj)
}
pub async fn transition_object(api: Arc<ECStore>, oi: &ObjectInfo, lae: LcAuditEvent) -> Result<(), Error> {
async fn transition_object(api: Arc<ECStore>, oi: &ObjectInfo, lae: LcAuditEvent) -> (Option<Uuid>, Result<(), Error>) {
let time_ilm = Metrics::time_ilm(lae.event.action);
let (publication_guard, bucket_incarnation_id) = lifecycle_transition_publication_guard(&api, oi)
.await
.ok_or_else(|| Error::other("lifecycle transition is not allowed for this bucket"))?;
let Some((publication_guard, bucket_incarnation_id)) = lifecycle_transition_publication_guard(&api, oi).await else {
return (None, Err(Error::other("lifecycle transition is not allowed for this bucket")));
};
let etag = if let Some(etag) = &oi.etag { etag } else { "" };
let etag = etag.to_string();
@@ -5180,7 +5181,7 @@ pub async fn transition_object(api: Arc<ECStore>, oi: &ObjectInfo, lae: LcAuditE
opts.add_namespace_lock_guard(&publication_guard);
let result = api.transition_object(&oi.bucket, &oi.name, &opts).await;
time_ilm(1)();
result
(Some(bucket_incarnation_id), result)
}
#[allow(
@@ -6034,7 +6035,7 @@ async fn apply_expiry_on_non_transitioned_objects_with_lock_lost_signal(
{
accounting.commit(size).await;
}
schedule_lifecycle_replication_delete_if_needed(oi, &dobj).await;
schedule_lifecycle_replication_delete_if_needed(oi, &dobj, bucket_incarnation_id).await;
// The object (or all its versions, for delete_all) was expired; evict any
// cached body so dead bytes do not sit resident until TTL (ODC-26). The
@@ -6050,7 +6051,7 @@ async fn apply_expiry_on_non_transitioned_objects_with_lock_lost_signal(
//let tags = LcAuditEvent::new(lc_event.clone(), src.clone()).tags();
//tags["version-id"] = dobj.version_id;
emit_non_transitioned_expiration_event(lc_event.action, oi, dobj);
emit_non_transitioned_expiration_event(lc_event.action, oi, dobj, bucket_incarnation_id);
if lc_event.action != IlmAction::NoneAction {
let mut num_versions = 1_u64;
@@ -6202,12 +6203,12 @@ fn lifecycle_deleted_object(oi: &ObjectInfo, dobj: &ObjectInfo) -> DeletedObject
}
}
async fn schedule_lifecycle_replication_delete_if_needed(oi: &ObjectInfo, dobj: &ObjectInfo) {
async fn schedule_lifecycle_replication_delete_if_needed(oi: &ObjectInfo, dobj: &ObjectInfo, bucket_incarnation_id: Uuid) {
let delete_object = lifecycle_deleted_object(oi, dobj);
if delete_object.replication_state.is_none() {
return;
}
replication_sink::schedule_delete(oi.bucket.clone(), delete_object).await;
replication_sink::schedule_delete(oi.bucket.clone(), delete_object, Some(bucket_incarnation_id)).await;
}
async fn lifecycle_delete_config_snapshot(api: &ECStore, oi: &ObjectInfo) -> Result<DeleteReplicationConfigSnapshot, Error> {
@@ -128,7 +128,7 @@ pub async fn delete_object_versions(
// order and admission; the event is derived from the committed
// result, and a send failure never rolls back a delete that
// already happened (backlog#2202).
emit_noncurrent_expiration_event(bucket, target, deleted_obj, false);
emit_noncurrent_expiration_event(bucket, target, deleted_obj, false, bucket_incarnation_id);
}
if !deleted_obj
.replication_state
@@ -137,7 +137,7 @@ pub async fn delete_object_versions(
{
continue;
}
replication_sink::schedule_delete(bucket.to_string(), deleted_obj.clone()).await;
replication_sink::schedule_delete(bucket.to_string(), deleted_obj.clone(), Some(bucket_incarnation_id)).await;
}
for (i, err) in errors.iter().enumerate() {
@@ -66,8 +66,12 @@ pub(crate) fn lifecycle_action_waits_for_replication(action: IlmAction) -> bool
)
}
pub(crate) async fn schedule_delete(bucket: String, delete_object: DeletedObject) {
ReplicationLifecycleBridge::schedule_delete(bucket, delete_object).await;
pub(crate) async fn schedule_delete(
bucket: String,
delete_object: DeletedObject,
notification_bucket_incarnation: Option<uuid::Uuid>,
) {
ReplicationLifecycleBridge::schedule_delete(bucket, delete_object, notification_bucket_incarnation).await;
}
#[cfg(test)]
+94 -62
View File
@@ -460,7 +460,7 @@ fn clear_bucket_overrides(bucket: &str) {
}
/// Publish application-owned bytes on every cache install path.
fn sync_on_demand_migration(bucket: &str, bm: &BucketMetadata) {
fn sync_application_bucket_configs(bucket: &str, bm: &BucketMetadata) {
if let Some(hook) = BUCKET_CONFIG_PUBLISH_HOOK.get() {
hook(
bucket,
@@ -468,12 +468,22 @@ fn sync_on_demand_migration(bucket: &str, bm: &BucketMetadata) {
bm.on_demand_migration_config()
.map(|(bytes, stamp)| (bytes, stamp, bm.bucket_incarnation_id)),
);
hook(
bucket,
super::metadata::BUCKET_NOTIFICATION_CONFIG,
(!bm.notification_config_xml.is_empty()).then_some((
bm.notification_config_xml.as_slice(),
bm.notification_config_updated_at,
bm.bucket_incarnation_id,
)),
);
}
}
fn clear_on_demand_migration(bucket: &str) {
fn clear_application_bucket_configs(bucket: &str) {
if let Some(hook) = BUCKET_CONFIG_PUBLISH_HOOK.get() {
hook(bucket, super::metadata::BUCKET_ON_DEMAND_MIGRATION_CONFIG, None);
hook(bucket, super::metadata::BUCKET_NOTIFICATION_CONFIG, None);
}
}
@@ -2253,6 +2263,7 @@ impl BucketMetadataSys {
.await?;
self.publish_if_unchanged(bucket, expected.as_ref(), metadata, persisted, &guard)
.await
.map(|_| ())
}
#[cfg(test)]
@@ -2486,7 +2497,7 @@ impl BucketMetadataSys {
if removed {
BucketTargetSys::get().delete(bucket).await;
clear_bucket_overrides(bucket);
clear_on_demand_migration(bucket);
clear_application_bucket_configs(bucket);
}
}
return Ok(());
@@ -2524,7 +2535,7 @@ impl BucketMetadataSys {
self.missing_buckets.invalidate(bucket).await;
sync_bucket_target_sys(bucket, &bm).await;
sync_bucket_overrides(bucket, &bm);
sync_on_demand_migration(bucket, &bm);
sync_application_bucket_configs(bucket, &bm);
}
MetadataLoadMode::Initial => {
let _publish_guard = self
@@ -2550,7 +2561,7 @@ impl BucketMetadataSys {
metadata: BucketMetadata,
persisted: bool,
namespace_guard: &rustfs_lock::NamespaceLockGuard,
) -> Result<()> {
) -> Result<bool> {
self.revoke_pending_bucket_creation(bucket).await;
if !persisted {
let _publish_guard = self
@@ -2563,7 +2574,7 @@ impl BucketMetadataSys {
_ => false,
};
if !unchanged {
return Ok(());
return Ok(false);
}
let removed = map.remove(bucket).is_some();
drop(map);
@@ -2572,9 +2583,9 @@ impl BucketMetadataSys {
if removed {
BucketTargetSys::get().delete(bucket).await;
clear_bucket_overrides(bucket);
clear_on_demand_migration(bucket);
clear_application_bucket_configs(bucket);
}
return Ok(());
return Ok(true);
}
let _publish_guard = self
.lock_metadata_publish(bucket, namespace_guard, "refreshed bucket metadata publish")
@@ -2587,7 +2598,7 @@ impl BucketMetadataSys {
_ => false,
};
if !unchanged {
return Ok(());
return Ok(false);
}
map.insert(bucket.to_string(), Arc::clone(&metadata));
drop(map);
@@ -2595,8 +2606,8 @@ impl BucketMetadataSys {
self.missing_buckets.invalidate(bucket).await;
sync_bucket_target_sys(bucket, &metadata).await;
sync_bucket_overrides(bucket, &metadata);
sync_on_demand_migration(bucket, &metadata);
Ok(())
sync_application_bucket_configs(bucket, &metadata);
Ok(true)
}
pub async fn get(&self, bucket: &str) -> Result<Arc<BucketMetadata>> {
@@ -2625,7 +2636,7 @@ impl BucketMetadataSys {
self.missing_buckets.invalidate(&bucket).await;
sync_bucket_target_sys(&bucket, &bm).await;
sync_bucket_overrides(&bucket, &bm);
sync_on_demand_migration(&bucket, &bm);
sync_application_bucket_configs(&bucket, &bm);
}
}
@@ -2649,7 +2660,7 @@ impl BucketMetadataSys {
if removed {
BucketTargetSys::get().delete(bucket).await;
clear_bucket_overrides(bucket);
clear_on_demand_migration(bucket);
clear_application_bucket_configs(bucket);
}
removed || removed_fabricated
}
@@ -2876,25 +2887,35 @@ impl BucketMetadataSys {
namespace_guard: &rustfs_lock::NamespaceLockGuard,
) -> Result<()> {
self.revoke_pending_bucket_creation(bucket).await;
let expected = self.metadata_map.read().await.get(bucket).cloned();
if !self
.bucket_exists(bucket, namespace_guard, "peer bucket metadata existence check")
.await?
{
return Err(Error::other(PEER_METADATA_NOT_PERSISTED));
}
let (metadata, persisted) = await_bucket_namespace_operation(
Some(namespace_guard),
bucket,
"peer bucket metadata load",
load_bucket_metadata_parse_with_presence(self.object_store(), bucket, true),
)
.await?;
if !persisted {
return Err(Error::other(PEER_METADATA_NOT_PERSISTED));
// A concurrent publication can reject the snapshot we read. Re-read
// from storage so a newer committed config does not lose its reload to
// an older one that happened to publish first.
for _ in 0..3 {
let expected = self.metadata_map.read().await.get(bucket).cloned();
let (metadata, persisted) = await_bucket_namespace_operation(
Some(namespace_guard),
bucket,
"peer bucket metadata load",
load_bucket_metadata_parse_with_presence(self.object_store(), bucket, true),
)
.await?;
if !persisted {
return Err(Error::other(PEER_METADATA_NOT_PERSISTED));
}
if self
.publish_if_unchanged(bucket, expected.as_ref(), metadata, true, namespace_guard)
.await?
{
return Ok(());
}
}
self.publish_if_unchanged(bucket, expected.as_ref(), metadata, true, namespace_guard)
.await
Err(Error::other("bucket metadata changed repeatedly during peer reload"))
}
pub async fn get_config(&self, bucket: &str) -> Result<(Arc<BucketMetadata>, bool)> {
@@ -2992,7 +3013,7 @@ impl BucketMetadataSys {
self.missing_buckets.invalidate(bucket).await;
sync_bucket_target_sys(bucket, &bm).await;
sync_bucket_overrides(bucket, &bm);
sync_on_demand_migration(bucket, &bm);
sync_application_bucket_configs(bucket, &bm);
return Ok((bm, true));
} else {
let exists = self
@@ -3443,7 +3464,7 @@ impl BucketMetadataSys {
self.missing_buckets.invalidate(bucket).await;
sync_bucket_target_sys(bucket, &metadata).await;
sync_bucket_overrides(bucket, &metadata);
sync_on_demand_migration(bucket, &metadata);
sync_application_bucket_configs(bucket, &metadata);
Ok(BucketMetadataAuthority::Authoritative(metadata))
}
@@ -6792,31 +6813,33 @@ mod tests {
const ODM_JSON: &[u8] = br#"{"source":{"provider":"minio","endpoint":"https://legacy.example.com:9000","region":"auto","bucket":"legacy-bucket","credentials":{"access_key":"AK","secret_key":"SK"}}}"#;
type RecordedOdmConfig = Option<(Vec<u8>, OffsetDateTime, Uuid)>;
type RecordedOdmHookCall = (String, RecordedOdmConfig);
const NOTIFY_XML: &[u8] = br#"<NotificationConfiguration><QueueConfiguration><Id>test</Id><Queue>arn:rustfs:sqs:us-east-1:primary:webhook</Queue><Event>s3:ObjectCreated:*</Event></QueueConfiguration></NotificationConfiguration>"#;
const PUBLISH_TEST_BUCKET: &str = "application-config-hook-paths";
/// Every `(bucket, config)` the recording hook has seen. Tests filter by
/// their own bucket name; the hook is process-wide and set once.
static ODM_HOOK_CALLS: std::sync::Mutex<Vec<RecordedOdmHookCall>> = std::sync::Mutex::new(Vec::new());
type RecordedConfig = Option<(Vec<u8>, OffsetDateTime, Uuid)>;
type RecordedConfigCall = (String, RecordedConfig);
fn install_recording_odm_hook() {
static CONFIG_HOOK_CALLS: std::sync::Mutex<Vec<RecordedConfigCall>> = std::sync::Mutex::new(Vec::new());
fn install_recording_config_hook() {
BUCKET_CONFIG_PUBLISH_HOOK.get_or_init(|| {
Box::new(|bucket, config_file, config| {
assert_eq!(config_file, super::super::metadata::BUCKET_ON_DEMAND_MIGRATION_CONFIG);
ODM_HOOK_CALLS.lock().unwrap().push((
bucket.to_string(),
config.map(|(bytes, stamp, incarnation)| (bytes.to_vec(), stamp, incarnation)),
));
if bucket == PUBLISH_TEST_BUCKET {
CONFIG_HOOK_CALLS.lock().unwrap().push((
config_file.to_string(),
config.map(|(bytes, stamp, incarnation)| (bytes.to_vec(), stamp, incarnation)),
));
}
})
});
}
fn odm_hook_calls(bucket: &str) -> Vec<RecordedOdmConfig> {
ODM_HOOK_CALLS
fn config_hook_calls(config_file: &str) -> Vec<RecordedConfig> {
CONFIG_HOOK_CALLS
.lock()
.unwrap()
.iter()
.filter(|(name, _)| name == bucket)
.filter(|(file, _)| file == config_file)
.map(|(_, config)| config.clone())
.collect()
}
@@ -6826,51 +6849,56 @@ mod tests {
/// reload, refresh loop, lazy load) and withdraws on removal, mirroring
/// `sync_bucket_durability`.
#[tokio::test]
async fn on_demand_migration_hook_fires_on_every_cache_install_path() {
install_recording_odm_hook();
async fn application_config_hook_fires_on_every_cache_install_path() {
use crate::bucket::metadata::{BUCKET_NOTIFICATION_CONFIG, BUCKET_ON_DEMAND_MIGRATION_CONFIG};
install_recording_config_hook();
let (dirs, ecstore) = isolated_store_over_temp_disks().await;
let bucket = "odm-hook-paths";
let bucket = PUBLISH_TEST_BUCKET;
for dir in &dirs {
std::fs::create_dir_all(dir.path().join(bucket)).expect("physical bucket should exist");
}
let incarnation = Uuid::new_v4();
let expect_publish = |before: usize, label: &str| {
let calls = odm_hook_calls(bucket);
assert_eq!(calls.len(), before + 1, "{label} must publish exactly once");
assert_eq!(
calls.last().unwrap().as_ref().map(|(bytes, _, _)| bytes.as_slice()),
Some(ODM_JSON),
"{label} must publish the stored bytes"
);
assert_eq!(calls.last().unwrap().as_ref().map(|(_, _, id)| *id), Some(incarnation));
for (file, bytes) in [
(BUCKET_ON_DEMAND_MIGRATION_CONFIG, ODM_JSON),
(BUCKET_NOTIFICATION_CONFIG, NOTIFY_XML),
] {
let calls = config_hook_calls(file);
assert_eq!(calls.len(), before + 1, "{label} must publish {file} exactly once");
let (stored, _, id) = calls.last().unwrap().as_ref().expect("stored configuration");
assert_eq!(stored, bytes, "{label} must publish the stored config");
assert_eq!(*id, incarnation);
}
};
// set (via persist_new_and_set, which installs through `set`).
let mut bm = BucketMetadata::new(bucket);
bm.bucket_incarnation_id = incarnation;
bm.update_config(crate::bucket::metadata::BUCKET_ON_DEMAND_MIGRATION_CONFIG, ODM_JSON.to_vec())
bm.update_config(BUCKET_ON_DEMAND_MIGRATION_CONFIG, ODM_JSON.to_vec())
.unwrap();
bm.update_config(BUCKET_NOTIFICATION_CONFIG, NOTIFY_XML.to_vec()).unwrap();
let writer = BucketMetadataSys::new(ecstore.clone());
let before = odm_hook_calls(bucket).len();
let before = config_hook_calls(BUCKET_ON_DEMAND_MIGRATION_CONFIG).len();
writer.persist_new_and_set(bm).await.expect("metadata should persist");
expect_publish(before, "set");
// init (initial load on a cold system).
let mut cold = BucketMetadataSys::new(ecstore.clone());
let before = odm_hook_calls(bucket).len();
let before = config_hook_calls(BUCKET_ON_DEMAND_MIGRATION_CONFIG).len();
cold.init(vec![bucket.to_string()]).await;
assert!(cold.get(bucket).await.is_ok(), "initial load must cache the bucket");
expect_publish(before, "init");
// peer reload.
let before = odm_hook_calls(bucket).len();
let before = config_hook_calls(BUCKET_ON_DEMAND_MIGRATION_CONFIG).len();
cold.reload_from_store(bucket).await.expect("peer reload should publish");
expect_publish(before, "peer reload");
// refresh loop.
let before = odm_hook_calls(bucket).len();
let before = config_hook_calls(BUCKET_ON_DEMAND_MIGRATION_CONFIG).len();
let mut failed = HashSet::new();
cold.concurrent_load(&[bucket.to_string()], &mut failed, MetadataLoadMode::Refresh)
.await;
@@ -6879,24 +6907,28 @@ mod tests {
// lazy load on another cold system.
let lazy = BucketMetadataSys::new(ecstore);
let before = odm_hook_calls(bucket).len();
let before = config_hook_calls(BUCKET_ON_DEMAND_MIGRATION_CONFIG).len();
let (_, loaded) = lazy.get_config(bucket).await.expect("lazy load should publish");
assert!(loaded, "the lazy path must have gone to disk");
expect_publish(before, "lazy load");
// Removal withdraws the config.
let before = odm_hook_calls(bucket).len();
let before = config_hook_calls(BUCKET_ON_DEMAND_MIGRATION_CONFIG).len();
assert!(lazy.remove(bucket).await);
let calls = odm_hook_calls(bucket);
let calls = config_hook_calls(BUCKET_ON_DEMAND_MIGRATION_CONFIG);
assert_eq!(calls.len(), before + 1, "remove must withdraw exactly once");
assert_eq!(calls.last().unwrap(), &None);
let notify_calls = config_hook_calls(BUCKET_NOTIFICATION_CONFIG);
assert_eq!(notify_calls.len(), before + 1);
assert_eq!(notify_calls.last(), Some(&None));
// Opaque bytes reach the application even if they are not valid JSON.
let mut corrupt = BucketMetadata::new(bucket);
corrupt.on_demand_migration_config_json = b"not-json".to_vec();
let before = odm_hook_calls(bucket).len();
let before = config_hook_calls(BUCKET_ON_DEMAND_MIGRATION_CONFIG).len();
lazy.set(bucket.to_string(), Arc::new(corrupt)).await;
let calls = odm_hook_calls(bucket);
let calls = config_hook_calls(BUCKET_ON_DEMAND_MIGRATION_CONFIG);
assert_eq!(calls.len(), before + 1);
assert_eq!(
calls.last().unwrap().as_ref().map(|(bytes, _, _)| bytes.as_slice()),
@@ -13,6 +13,8 @@
// limitations under the License.
pub(crate) use crate::services::event_notification::EventArgs;
#[cfg(test)]
pub(crate) use crate::services::event_notification::test_recorder;
use super::runtime_boundary;
@@ -89,7 +89,11 @@ impl ReplicationLifecycleBridge {
}
}
pub(crate) async fn schedule_delete(bucket: String, delete_object: DeletedObject) {
pub(crate) async fn schedule_delete(
bucket: String,
delete_object: DeletedObject,
notification_bucket_incarnation: Option<uuid::Uuid>,
) {
#[cfg(test)]
{
SCHEDULED_DELETE_OBJECTS
@@ -98,6 +102,7 @@ impl ReplicationLifecycleBridge {
.push(delete_object.clone());
}
super::replication_pool::schedule_replication_delete(DeletedObjectReplicationInfo {
notification_bucket_incarnation,
delete_object: deleted_object_for_replication(delete_object),
bucket,
event_type: REPLICATE_INCOMING_DELETE.to_string(),
@@ -22,6 +22,10 @@ use rustfs_utils::path::path_join_buf;
use time::OffsetDateTime;
use super::replication_error_boundary::{Error, Result};
#[cfg(test)]
pub(crate) use crate::bucket::metadata_sys::{
bucket_metadata_sys_of, init_bucket_metadata_sys, set_bucket_metadata_in, test_support,
};
pub(crate) type ReplicationInstanceContext = InstanceContext;
@@ -110,13 +110,16 @@ impl ReplicationObjectBridge {
check_replicate_delete_with_snapshot(object, source, opts, source_error, snapshot)
}
/// Admits replication with the bucket identity verified by the originating mutation.
/// An absent identity suppresses external notifications; workers never infer one by name.
pub async fn schedule_object<S: ReplicationStorage>(
object: ObjectInfo,
storage: Arc<S>,
decision: ReplicateDecision,
op_type: ReplicationType,
notification_bucket_incarnation: Option<uuid::Uuid>,
) {
schedule_replication(object, storage, decision, op_type).await;
schedule_replication(object, storage, decision, op_type, notification_bucket_incarnation).await;
}
pub async fn schedule_delete(delete_object: DeletedObjectReplicationInfo) {
@@ -148,8 +151,16 @@ impl ReplicationObjectBridge {
}
}
pub async fn schedule_storage_delete(delete_object: DeletedObject, bucket: String, event_type: String) {
/// Schedules a committed delete with its originating bucket identity for notifications.
/// An absent identity leaves external notifications unbound.
pub async fn schedule_storage_delete(
delete_object: DeletedObject,
bucket: String,
event_type: String,
notification_bucket_incarnation: Option<uuid::Uuid>,
) {
Self::schedule_delete(DeletedObjectReplicationInfo {
notification_bucket_incarnation,
delete_object: deleted_object_for_replication(delete_object),
bucket,
event_type,
@@ -158,10 +169,18 @@ impl ReplicationObjectBridge {
.await;
}
pub async fn schedule_storage_deletes(delete_objects: Vec<DeletedObject>, bucket: String, event_type: String) {
/// Schedules committed deletes from one bucket instance, retaining that notification identity.
/// An absent identity leaves external notifications unbound.
pub async fn schedule_storage_deletes(
delete_objects: Vec<DeletedObject>,
bucket: String,
event_type: String,
notification_bucket_incarnation: Option<uuid::Uuid>,
) {
let delete_objects = delete_objects
.into_iter()
.map(|delete_object| DeletedObjectReplicationInfo {
notification_bucket_incarnation,
delete_object: deleted_object_for_replication(delete_object),
bucket: bucket.clone(),
event_type: event_type.clone(),
@@ -1016,7 +1016,9 @@ async fn replay_mrf_metadata_entry<S: ReplicationStorage>(
/// admitted targets, carrying over the entry's retry count.
fn admitted_mrf_replicate_object(oi: ObjectInfo, entry: &MrfReplicateEntry, op_type: ReplicationType) -> ReplicateObjectInfo {
let dsc = replicate_decision_for_admitted_targets(&entry.target_arns);
let mut roi = replicate_object_info_from_object_info(oi, dsc, op_type);
// The journal does not bind this task to a bucket instance. Keep external
// notifications unbound instead of attaching an old task to a reused name.
let mut roi = replicate_object_info_from_object_info(oi, dsc, op_type, None);
roi.retry_count = entry.retry_count.max(0) as u32;
roi
}
@@ -3171,12 +3173,13 @@ pub(crate) async fn schedule_replication<S: ReplicationStorage>(
o: Arc<S>,
dsc: ReplicateDecision,
op_type: ReplicationType,
notification_bucket_incarnation: Option<uuid::Uuid>,
) {
let (synchronous, asynchronous) = dsc.partition_by_sync();
let mut async_oi = oi;
if synchronous.replicate_any() {
let ri = replicate_object_info_from_object_info(async_oi.clone(), synchronous, op_type);
let ri = replicate_object_info_from_object_info(async_oi.clone(), synchronous, op_type, notification_bucket_incarnation);
let state = replicate_object(ri, o.clone()).await;
async_oi.replication_status_internal = state.replication_status_internal;
async_oi.version_purge_status_internal = state.version_purge_status_internal;
@@ -3185,7 +3188,7 @@ pub(crate) async fn schedule_replication<S: ReplicationStorage>(
if asynchronous.replicate_any()
&& let Some(pool) = runtime_sources::replication_pool()
{
let ri = replicate_object_info_from_object_info(async_oi, asynchronous, op_type);
let ri = replicate_object_info_from_object_info(async_oi, asynchronous, op_type, notification_bucket_incarnation);
let _ = pool.queue_replica_task(ri).await;
}
}
@@ -3194,6 +3197,7 @@ fn replicate_object_info_from_object_info(
oi: ObjectInfo,
dsc: ReplicateDecision,
op_type: ReplicationType,
notification_bucket_incarnation: Option<uuid::Uuid>,
) -> ReplicateObjectInfo {
let tgt_statuses = replication_statuses_map(&oi.replication_status_internal.clone().unwrap_or_default());
let purge_statuses = version_purge_statuses_map(&oi.version_purge_status_internal.clone().unwrap_or_default());
@@ -3209,6 +3213,7 @@ fn replicate_object_info_from_object_info(
let checksum = if ssec { oi.checksum.clone() } else { None };
ReplicateObjectInfo {
notification_bucket_incarnation,
name: oi.name,
size: oi.size,
actual_size: asz,
@@ -3350,7 +3355,7 @@ pub async fn queue_replication_metadata(bucket: &str, oi: ObjectInfo, retry_coun
return ReplicationQueueAdmission::Skipped;
}
let mut roi = replicate_object_info_from_object_info(oi, dsc, ReplicationType::Metadata);
let mut roi = replicate_object_info_from_object_info(oi, dsc, ReplicationType::Metadata, None);
roi.retry_count = retry_count;
if let Some(pool) = runtime_sources::replication_pool() {
pool.queue_replica_task(roi).await
@@ -5205,6 +5210,117 @@ mod tests {
}
}
#[test]
fn replication_admission_keeps_original_incarnation_and_mrf_replay_does_not_guess() {
let incarnation = Uuid::new_v4();
let target = "arn:rustfs:replication:target-a";
for op_type in [ReplicationType::Object, ReplicationType::Metadata] {
let object = ObjectInfo {
bucket: "source".to_string(),
name: "object".to_string(),
version_id: Some(Uuid::new_v4()),
..Default::default()
};
let admitted = replicate_object_info_from_object_info(
object.clone(),
test_replicate_decision(&[target]),
op_type,
Some(incarnation),
);
assert_eq!(admitted.notification_bucket_incarnation, Some(incarnation));
let encoded = encode_mrf_file(&[admitted.to_mrf_entry()]).expect("encode queued replication");
let entries = decode_mrf_file(&encoded).expect("decode queued replication");
let replayed = admitted_mrf_replicate_object(object, &entries[0], op_type);
assert_eq!(
replayed.notification_bucket_incarnation, None,
"MRF replay must not infer a bucket instance from a reused name"
);
}
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[serial_test::serial]
async fn queued_delete_notifications_keep_origin_across_bucket_recreation() {
use super::super::replication_event_sink::test_recorder;
use super::super::replication_metadata_boundary::{
bucket_metadata_sys_of, init_bucket_metadata_sys, set_bucket_metadata_in,
test_support::isolated_store_over_temp_disks,
};
use super::super::replication_storage_boundary::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions};
let (_dirs, store) = isolated_store_over_temp_disks().await;
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
runtime_sources::set_test_object_layer(store.clone()).await;
test_recorder::install();
let bucket = "queued-delete-notification-origin";
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("create original bucket");
let original = store.bucket_incarnation_id(bucket).await.expect("original incarnation");
let queued = DeletedObjectReplicationInfo {
notification_bucket_incarnation: Some(original),
bucket: bucket.to_string(),
target_arn: "arn:rustfs:replication:target-a".to_string(),
delete_object: ReplicationDeletedObject {
object_name: "original-object".to_string(),
version_id: Some(Uuid::new_v4()),
replication_state: Some(ReplicationState {
replicate_decision_str: "malformed-decision".to_string(),
..Default::default()
}),
..Default::default()
},
..Default::default()
};
let encoded = encode_mrf_file(&[queued.to_mrf_entry()]).expect("persist old delete task");
let entries = decode_mrf_file(&encoded).expect("recover old delete task");
let mut replayed = reconstructed_heal_delete_info(
&entries[0],
&ObjectInfo::default(),
&replicate_decision_for_admitted_targets(&entries[0].target_arns),
);
replayed
.delete_object
.replication_state
.as_mut()
.expect("reconstructed replication state")
.replicate_decision_str = "malformed-decision".to_string();
store
.delete_bucket(bucket, &DeleteBucketOptions::default())
.await
.expect("delete original bucket");
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("create replacement bucket");
let replacement = store.bucket_incarnation_id(bucket).await.expect("replacement incarnation");
assert_ne!(original, replacement);
let sys = bucket_metadata_sys_of(&store.ctx).expect("metadata owner");
let mut metadata = (*sys.read().await.get(bucket).await.expect("replacement metadata")).clone();
metadata.notification_config_xml = br#"<NotificationConfiguration><QueueConfiguration><Id>replacement</Id><Queue>arn:rustfs:sqs:us-east-1:replacement:webhook</Queue><Event>s3:Replication:*</Event></QueueConfiguration></NotificationConfiguration>"#.to_vec();
set_bucket_metadata_in(&store.ctx, metadata)
.await
.expect("install replacement notification configuration");
assert!(!replicate_delete_with_outcome(queued, store.clone()).await);
assert!(!replicate_delete_with_outcome(replayed, store).await);
let events = test_recorder::recorded_for_bucket(bucket);
assert_eq!(events.len(), 2, "both worker failures must emit an observable local event");
assert!(
events
.iter()
.all(|event| event.object == "original-object" && event.event_name == "s3:Replication:OperationNotTracked")
);
assert_eq!(events[0].bucket_incarnation_id, Some(original));
assert_eq!(
events[1].bucket_incarnation_id, None,
"a replay without provenance must not acquire the replacement bucket's identity"
);
assert!(events.iter().all(|event| event.bucket_incarnation_id != Some(replacement)));
}
#[test]
fn replicate_object_info_from_object_info_preserves_ssec_checksum() {
let checksum = bytes::Bytes::from_static(b"ssec-checksum");
@@ -5219,7 +5335,7 @@ mod tests {
..Default::default()
};
let ri = replicate_object_info_from_object_info(oi, ReplicateDecision::default(), ReplicationType::Object);
let ri = replicate_object_info_from_object_info(oi, ReplicateDecision::default(), ReplicationType::Object, None);
assert!(ri.ssec);
assert_eq!(ri.checksum, Some(checksum));
@@ -5235,8 +5351,12 @@ mod tests {
user_tags: Arc::new("owner=a3".to_string()),
..Default::default()
};
let live =
replicate_object_info_from_object_info(object.clone(), test_replicate_decision(&[target]), ReplicationType::Metadata);
let live = replicate_object_info_from_object_info(
object.clone(),
test_replicate_decision(&[target]),
ReplicationType::Metadata,
None,
);
let persisted = live.to_mrf_entry();
let encoded = encode_mrf_file(std::slice::from_ref(&persisted)).expect("metadata MRF entry should encode");
let decoded = decode_mrf_file(&encoded).expect("metadata MRF entry should decode");
@@ -2275,6 +2275,7 @@ pub async fn get_heal_replicate_object_info(oi: &ObjectInfo, rcfg: &ReplicationC
let replication_generation = oi.replication_generation_snapshot();
Ok(ReplicateObjectInfo {
notification_bucket_incarnation: None,
name: oi.name.clone(),
size: oi.size,
actual_size,
@@ -2449,6 +2450,7 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
"Failed to parse replicate decision"
);
send_local_event(EventArgs {
bucket_incarnation_id: dobj.notification_bucket_incarnation,
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
object: ObjectInfo {
@@ -2530,6 +2532,7 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
"Failed to verify source delete-marker state before replication"
);
send_local_event(EventArgs {
bucket_incarnation_id: dobj.notification_bucket_incarnation,
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
object: ObjectInfo {
@@ -2564,6 +2567,7 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
"Skipping replication delete"
);
send_local_event(EventArgs {
bucket_incarnation_id: dobj.notification_bucket_incarnation,
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
object: ObjectInfo {
@@ -2594,6 +2598,7 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
"Skipping replication delete"
);
send_local_event(EventArgs {
bucket_incarnation_id: dobj.notification_bucket_incarnation,
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
object: ObjectInfo {
@@ -2693,6 +2698,7 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
"Replication resync task failed"
);
send_event(EventArgs {
bucket_incarnation_id: dobj.notification_bucket_incarnation,
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
object: ObjectInfo {
@@ -2801,6 +2807,7 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
{
Ok(object) => {
send_event(EventArgs {
bucket_incarnation_id: dobj.notification_bucket_incarnation,
event_name,
bucket_name: bucket.clone(),
object,
@@ -2826,6 +2833,7 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
);
}
send_event(EventArgs {
bucket_incarnation_id: dobj.notification_bucket_incarnation,
event_name,
bucket_name: bucket.clone(),
object: ObjectInfo {
@@ -3157,6 +3165,7 @@ async fn purge_stale_delete_marker_targets(bucket: &str, dobj: &DeletedObjectRep
}
async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &DeletedObjectReplicationInfo, storage: Arc<S>) -> bool {
let incarnation_id = dobj.notification_bucket_incarnation;
let bucket = &dobj.bucket;
let object_name = &dobj.delete_object.object_name;
let admitted_target_arns = dobj.admitted_target_arns();
@@ -3177,6 +3186,7 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
"Skipping replication force-delete because replication config is missing"
);
send_local_event(EventArgs {
bucket_incarnation_id: incarnation_id,
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
object: ObjectInfo {
@@ -3200,6 +3210,7 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
"Skipping replication force-delete because replication config lookup failed"
);
send_local_event(EventArgs {
bucket_incarnation_id: incarnation_id,
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
object: ObjectInfo {
@@ -3234,6 +3245,7 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
"Skipping replication force-delete"
);
send_local_event(EventArgs {
bucket_incarnation_id: incarnation_id,
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
object: ObjectInfo {
@@ -3262,6 +3274,7 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
"Skipping replication force-delete"
);
send_local_event(EventArgs {
bucket_incarnation_id: incarnation_id,
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
object: ObjectInfo {
@@ -3309,6 +3322,7 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
"Skipping replication force-delete because target client is unavailable"
);
send_local_event(EventArgs {
bucket_incarnation_id: incarnation_id,
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
object: ObjectInfo {
@@ -3339,6 +3353,7 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
"Skipping replication force-delete"
);
send_local_event(EventArgs {
bucket_incarnation_id: incarnation_id,
event_name: EventName::ObjectReplicationFailed.to_string(),
bucket_name: bucket.clone(),
object: ObjectInfo {
@@ -3375,6 +3390,7 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
"Replication target operation failed"
);
send_local_event(EventArgs {
bucket_incarnation_id: incarnation_id,
event_name: EventName::ObjectReplicationFailed.to_string(),
bucket_name: bucket.clone(),
object: ObjectInfo {
@@ -4239,6 +4255,7 @@ pub(crate) async fn replicate_object_with_outcome<S: ReplicationStorage>(
"Skipping replication object"
);
send_local_event(EventArgs {
bucket_incarnation_id: roi.notification_bucket_incarnation,
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
object: roi.to_object_info(),
@@ -4262,6 +4279,7 @@ pub(crate) async fn replicate_object_with_outcome<S: ReplicationStorage>(
"Skipping replication object"
);
send_local_event(EventArgs {
bucket_incarnation_id: roi.notification_bucket_incarnation,
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
object: roi.to_object_info(),
@@ -4328,6 +4346,7 @@ pub(crate) async fn replicate_object_with_outcome<S: ReplicationStorage>(
"Replication resync task failed"
);
send_local_event(EventArgs {
bucket_incarnation_id: roi.notification_bucket_incarnation,
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
object: roi.to_object_info(),
@@ -4472,6 +4491,7 @@ pub(crate) async fn replicate_object_with_outcome<S: ReplicationStorage>(
if publication.emit_event {
send_local_event(EventArgs {
bucket_incarnation_id: roi.notification_bucket_incarnation,
event_name,
bucket_name: bucket.clone(),
object: object_info,
@@ -4614,6 +4634,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
"Skipping replication object target"
);
send_local_event(EventArgs {
bucket_incarnation_id: self.notification_bucket_incarnation,
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
object: self.to_object_info(),
@@ -4629,6 +4650,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
let Some(ssec_audit_required) = resolve_ssec_passthrough_gate(self.ssec, &tgt_client, &bucket, &object, &mut rinfo).await
else {
send_local_event(EventArgs {
bucket_incarnation_id: self.notification_bucket_incarnation,
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
object: self.to_object_info(),
@@ -4672,6 +4694,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
);
send_local_event(EventArgs {
bucket_incarnation_id: self.notification_bucket_incarnation,
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
object: self.to_object_info(),
@@ -4703,6 +4726,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
"Skipping replication object target"
);
send_local_event(EventArgs {
bucket_incarnation_id: self.notification_bucket_incarnation,
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
object: object_info,
@@ -4728,6 +4752,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
"Skipping replication object target"
);
send_local_event(EventArgs {
bucket_incarnation_id: self.notification_bucket_incarnation,
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
object: object_info,
@@ -4862,6 +4887,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
"Replication target operation failed"
);
send_local_event(EventArgs {
bucket_incarnation_id: self.notification_bucket_incarnation,
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
object: object_info,
@@ -4890,6 +4916,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
"Replication target operation failed"
);
send_local_event(EventArgs {
bucket_incarnation_id: self.notification_bucket_incarnation,
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
object: object_info,
@@ -4987,6 +5014,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
let Some(ssec_audit_required) = resolve_ssec_passthrough_gate(self.ssec, &tgt_client, &bucket, &object, &mut rinfo).await
else {
send_local_event(EventArgs {
bucket_incarnation_id: self.notification_bucket_incarnation,
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
object: self.to_object_info(),
@@ -5020,7 +5048,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
let size = match object_info.get_actual_size() {
Ok(size) => size,
Err(e) => {
note_replicate_all_size_unavailable(&bucket, &tgt_client, object_info, &e);
note_replicate_all_size_unavailable(&bucket, &tgt_client, object_info, &e, self.notification_bucket_incarnation);
return rinfo;
}
};
@@ -5029,7 +5057,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
let transfer_size = if self.ssec { object_info.size } else { size };
if tgt_client.bucket.is_empty() {
note_replicate_all_target_bucket_empty(&bucket, &tgt_client, object_info);
note_replicate_all_target_bucket_empty(&bucket, &tgt_client, object_info, self.notification_bucket_incarnation);
return rinfo;
}
@@ -5075,7 +5103,15 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
let (put_opts, is_multipart) = match replication_put_object_options_for_target(&tgt_client, &object_info) {
Ok((put_opts, is_mp)) => (put_opts, is_mp),
Err(e) => {
fail_replicate_all_put_options(&mut rinfo, &tgt_client, &bucket, object_info, &e, start_time);
fail_replicate_all_put_options(
&mut rinfo,
&tgt_client,
&bucket,
object_info,
&e,
start_time,
self.notification_bucket_incarnation,
);
return rinfo;
}
};
@@ -5179,6 +5215,7 @@ fn note_replicate_all_target_offline(roi: &ReplicateObjectInfo, bucket: &str, tg
"Skipped replication because target is offline"
);
send_local_event(EventArgs {
bucket_incarnation_id: roi.notification_bucket_incarnation,
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.to_string(),
object: roi.to_object_info(),
@@ -5217,6 +5254,7 @@ fn note_replicate_all_reader_unavailable(roi: &ReplicateObjectInfo, bucket: &str
"Skipped replication because object reader is unavailable"
);
send_local_event(EventArgs {
bucket_incarnation_id: roi.notification_bucket_incarnation,
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.to_string(),
object: roi.to_object_info(),
@@ -5228,7 +5266,13 @@ fn note_replicate_all_reader_unavailable(roi: &ReplicateObjectInfo, bucket: &str
/// Log and notify that replication was skipped because the actual object size
/// is unavailable.
fn note_replicate_all_size_unavailable(bucket: &str, tgt_client: &TargetClient, object_info: ObjectInfo, e: &std::io::Error) {
fn note_replicate_all_size_unavailable(
bucket: &str,
tgt_client: &TargetClient,
object_info: ObjectInfo,
e: &std::io::Error,
incarnation_id: Option<uuid::Uuid>,
) {
debug!(
event = EVENT_RESYNC_RUNTIME_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
@@ -5241,6 +5285,7 @@ fn note_replicate_all_size_unavailable(bucket: &str, tgt_client: &TargetClient,
"Skipped replication because actual object size is unavailable"
);
send_local_event(EventArgs {
bucket_incarnation_id: incarnation_id,
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.to_string(),
object: object_info,
@@ -5251,7 +5296,12 @@ fn note_replicate_all_size_unavailable(bucket: &str, tgt_client: &TargetClient,
/// Log and notify that replication was skipped because the target bucket is
/// empty.
fn note_replicate_all_target_bucket_empty(bucket: &str, tgt_client: &TargetClient, object_info: ObjectInfo) {
fn note_replicate_all_target_bucket_empty(
bucket: &str,
tgt_client: &TargetClient,
object_info: ObjectInfo,
incarnation_id: Option<uuid::Uuid>,
) {
debug!(
event = EVENT_RESYNC_RUNTIME_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
@@ -5263,6 +5313,7 @@ fn note_replicate_all_target_bucket_empty(bucket: &str, tgt_client: &TargetClien
"Skipped replication because target bucket is empty"
);
send_local_event(EventArgs {
bucket_incarnation_id: incarnation_id,
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.to_string(),
object: object_info,
@@ -5376,6 +5427,7 @@ async fn resolve_replicate_all_action(
"Skipping replication because newer target version exists"
);
send_local_event(EventArgs {
bucket_incarnation_id: roi.notification_bucket_incarnation,
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.to_string(),
object: object_info.clone(),
@@ -5499,6 +5551,7 @@ async fn resolve_replicate_all_action(
"Failed replication head-object fallback"
);
send_local_event(EventArgs {
bucket_incarnation_id: roi.notification_bucket_incarnation,
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.to_string(),
object: object_info,
@@ -5531,6 +5584,7 @@ async fn resolve_replicate_all_action(
);
send_local_event(EventArgs {
bucket_incarnation_id: roi.notification_bucket_incarnation,
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.to_string(),
object: object_info,
@@ -5559,6 +5613,7 @@ fn fail_replicate_all_put_options(
object_info: ObjectInfo,
e: &Error,
start_time: OffsetDateTime,
incarnation_id: Option<uuid::Uuid>,
) {
rinfo.replication_status = ReplicationStatusType::Failed;
rinfo.error = Some(e.to_string());
@@ -5574,6 +5629,7 @@ fn fail_replicate_all_put_options(
"Replication target operation failed"
);
send_local_event(EventArgs {
bucket_incarnation_id: incarnation_id,
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.to_string(),
object: object_info,
@@ -24,6 +24,8 @@ pub(crate) use crate::object_api::{
};
#[cfg(test)]
pub(crate) use crate::object_api::{NamespaceLockFence, NamespaceLockSignalTestFence};
#[cfg(test)]
pub(crate) use crate::storage_api_contracts::bucket::{BucketOperations, DeleteBucketOptions, MakeBucketOptions};
pub(crate) use crate::storage_api_contracts::list::{
ListOperations, StorageListObjectVersionsInfo, StorageListObjectsV2Info, StorageObjectInfoOrErr, StorageWalkOptions,
};
@@ -18,6 +18,8 @@ use super::replication_bandwidth_boundary::ReplicationBucketMonitor;
use super::replication_pool::DynReplicationPool;
use super::replication_state::ReplicationStats;
use super::replication_storage_boundary::ReplicationObjectStore;
#[cfg(test)]
pub(crate) use crate::runtime::global::set_object_layer as set_test_object_layer;
use crate::runtime::sources;
pub use crate::runtime::sources::{
ScannerDirtyUsageMutationObserver, ScannerDirtyUsageMutationSource, set_scanner_dirty_usage_mutation_observer,
@@ -82,6 +82,8 @@ impl EventNotifier {
#[derive(Debug, Default)]
pub struct EventArgs {
/// Bucket instance verified by storage when producing this event.
pub bucket_incarnation_id: Option<uuid::Uuid>,
pub event_name: String,
pub bucket_name: String,
pub object: ObjectInfo,
@@ -135,6 +137,7 @@ pub(crate) mod test_recorder {
#[derive(Clone, Debug)]
pub(crate) struct RecordedEvent {
pub(crate) event_name: String,
pub(crate) bucket_incarnation_id: Option<Uuid>,
pub(crate) bucket: String,
pub(crate) object: String,
pub(crate) version_id: Option<Uuid>,
@@ -154,6 +157,7 @@ pub(crate) mod test_recorder {
register_event_dispatch_hook(|args| {
recorded().lock().unwrap_or_else(|err| err.into_inner()).push(RecordedEvent {
event_name: args.event_name,
bucket_incarnation_id: args.bucket_incarnation_id,
bucket: args.bucket_name,
object: args.object.name,
version_id: args.object.version_id,
+15 -2
View File
@@ -9294,7 +9294,12 @@ impl SetDisks {
)
.await?;
if let Some((_, deleted_object)) = replication_delete {
ReplicationLifecycleBridge::schedule_delete(bucket.to_string(), deleted_object).await;
ReplicationLifecycleBridge::schedule_delete(
bucket.to_string(),
deleted_object,
opts.expected_bucket_incarnation_id,
)
.await;
}
deleted += 1;
}
@@ -11563,7 +11568,12 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
)
.await?;
if let Some((_, deleted_object)) = replication_delete {
ReplicationLifecycleBridge::schedule_delete(bucket.to_string(), deleted_object).await;
ReplicationLifecycleBridge::schedule_delete(
bucket.to_string(),
deleted_object,
opts.expected_bucket_incarnation_id,
)
.await;
}
}
} else {
@@ -12732,6 +12742,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
let obj_info = ObjectInfo::from_file_info(&fi, bucket, object, opts.versioned || opts.version_suspended);
send_event(EventArgs {
event_name: event_name.to_string(),
bucket_incarnation_id: opts.expected_bucket_incarnation_id,
bucket_name: bucket.to_string(),
object: obj_info,
user_agent: "Internal: [ILM-Transition]".to_string(),
@@ -12878,6 +12889,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
let restored_info = self_.finalize_restore_metadata(bucket, object, &restored_info, &opts).await?;
send_event(EventArgs {
event_name: EventName::ObjectRestoreCompleted.as_str().to_string(),
bucket_incarnation_id: opts.expected_bucket_incarnation_id,
bucket_name: bucket.to_string(),
object: restored_info,
user_agent: "Internal: [Restore-Completed]".to_string(),
@@ -13014,6 +13026,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
let restored_info = self_.finalize_restore_metadata(bucket, object, &restored_info, opts).await?;
send_event(EventArgs {
event_name: EventName::ObjectRestoreCompleted.as_str().to_string(),
bucket_incarnation_id: opts.expected_bucket_incarnation_id,
bucket_name: bucket.to_string(),
object: restored_info,
user_agent: "Internal: [Restore-Completed]".to_string(),
+1
View File
@@ -74,6 +74,7 @@ jiff = { workspace = true, features = ["serde"] }
percent-encoding = { workspace = true }
rayon = { workspace = true }
rustc-hash = { workspace = true }
uuid = { workspace = true }
serde = { workspace = true, features = ["derive"] }
starshard = { workspace = true, features = ["rayon", "async", "serde"] }
thiserror = { workspace = true }
+19 -27
View File
@@ -13,9 +13,8 @@
// limitations under the License.
use crate::{
BucketNotificationConfig, NotificationError, config_manager::notify_configuration_hint,
notification_system_subscriber::NotificationSystemSubscriberView, notifier::EventNotifier, rule_engine::NotifyRuleEngine,
rules::ParseConfigError,
BucketNotificationConfig, NotificationError, config_manager::notify_configuration_hint, notifier::EventNotifier,
rule_engine::NotifyRuleEngine, rules::ParseConfigError,
};
use rustfs_s3_types::EventName;
use std::sync::Arc;
@@ -30,26 +29,14 @@ const EVENT_NOTIFY_BUCKET_CONFIG_LOADED: &str = "notify_bucket_config_loaded";
pub struct NotifyBucketConfigManager {
notifier: Arc<EventNotifier>,
rule_engine: NotifyRuleEngine,
subscriber_view: Arc<NotificationSystemSubscriberView>,
}
impl NotifyBucketConfigManager {
pub fn new(
notifier: Arc<EventNotifier>,
rule_engine: NotifyRuleEngine,
subscriber_view: Arc<NotificationSystemSubscriberView>,
) -> Self {
Self {
notifier,
rule_engine,
subscriber_view,
}
pub fn new(notifier: Arc<EventNotifier>, rule_engine: NotifyRuleEngine) -> Self {
Self { notifier, rule_engine }
}
pub async fn has_subscriber(&self, bucket: &str, event: &EventName) -> bool {
if !self.subscriber_view.has_subscriber(bucket, event) {
return false;
}
self.rule_engine.has_subscriber(bucket, event).await
}
@@ -105,7 +92,6 @@ impl NotifyBucketConfigManager {
/// Publishes an already validated `cfg` to the runtime rule state.
pub async fn apply_bucket_notification_config(&self, bucket: &str, cfg: &BucketNotificationConfig) {
self.subscriber_view.apply_bucket_config(bucket, cfg);
self.rule_engine.set_bucket_rules(bucket, cfg.get_rules_map().clone()).await;
info!(
event = EVENT_NOTIFY_BUCKET_CONFIG_LOADED,
@@ -128,8 +114,16 @@ impl NotifyBucketConfigManager {
Ok(())
}
pub(crate) fn publish_bucket_notification_config(&self, bucket: &str, cfg: Option<(uuid::Uuid, &BucketNotificationConfig)>) {
if let Some((incarnation_id, cfg)) = cfg {
self.rule_engine
.publish_bucket_rules(bucket, Some(incarnation_id), cfg.get_rules_map().clone());
} else {
self.rule_engine.withdraw_bucket_rules(bucket);
}
}
pub async fn remove_bucket_notification_config(&self, bucket: &str) {
self.subscriber_view.clear_bucket(bucket);
self.rule_engine.clear_bucket_rules(bucket).await;
}
}
@@ -138,8 +132,8 @@ impl NotifyBucketConfigManager {
mod tests {
use super::NotifyBucketConfigManager;
use crate::{
BucketNotificationConfig, NotificationError, integration::NotificationMetrics,
notification_system_subscriber::NotificationSystemSubscriberView, notifier::EventNotifier, rule_engine::NotifyRuleEngine,
BucketNotificationConfig, NotificationError, integration::NotificationMetrics, notifier::EventNotifier,
rule_engine::NotifyRuleEngine,
};
use rustfs_s3_types::EventName;
use rustfs_targets::arn::TargetID;
@@ -149,8 +143,7 @@ mod tests {
let metrics = Arc::new(NotificationMetrics::new());
let rule_engine = NotifyRuleEngine::new();
let notifier = Arc::new(EventNotifier::new(metrics, rule_engine.clone()));
let subscriber_view = Arc::new(NotificationSystemSubscriberView::new());
NotifyBucketConfigManager::new(notifier, rule_engine, subscriber_view)
NotifyBucketConfigManager::new(notifier, rule_engine)
}
#[tokio::test]
@@ -166,11 +159,11 @@ mod tests {
let mut cfg = BucketNotificationConfig::new("us-east-1");
cfg.add_rule(&[EventName::ObjectCreatedPut], "*".to_string(), target_id);
manager.subscriber_view.apply_bucket_config("bucket", &cfg);
assert!(manager.subscriber_view.has_subscriber("bucket", &EventName::ObjectCreatedPut));
manager.apply_bucket_notification_config("bucket", &cfg).await;
assert!(manager.has_subscriber("bucket", &EventName::ObjectCreatedPut).await);
manager.remove_bucket_notification_config("bucket").await;
assert!(!manager.subscriber_view.has_subscriber("bucket", &EventName::ObjectCreatedPut));
assert!(!manager.has_subscriber("bucket", &EventName::ObjectCreatedPut).await);
}
#[tokio::test]
@@ -186,7 +179,6 @@ mod tests {
.expect_err("validation must fail when the runtime has no notify targets");
assert!(matches!(err, NotificationError::Configuration(_)));
assert!(!manager.has_subscriber("bucket", &EventName::ObjectCreatedPut).await);
assert!(!manager.subscriber_view.has_subscriber("bucket", &EventName::ObjectCreatedPut));
}
#[tokio::test]
+31
View File
@@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::rules::TargetIdSet;
use hashbrown::HashMap;
use jiff::{Timestamp, tz::TimeZone};
use rustfs_s3_ops::is_object_removed_event;
@@ -170,6 +171,10 @@ pub struct Event {
pub glacier_event_data: Option<GlacierEventData>,
/// Information about the source of the event
pub source: Source,
#[serde(skip)]
pub(crate) bucket_incarnation_id: Option<uuid::Uuid>,
#[serde(skip)]
pub(crate) bucket_removed_targets: Option<TargetIdSet>,
}
impl Event {
@@ -195,6 +200,8 @@ impl Event {
user_metadata.insert("x-request-time".to_string(), format_timestamp_rfc3339_millis(&Timestamp::now()));
Event {
bucket_incarnation_id: None,
bucket_removed_targets: None,
event_version: event_schema_version(event_name).to_string(),
event_source: "rustfs:s3".to_string(),
aws_region: "us-east-1".to_string(),
@@ -320,6 +327,8 @@ impl Event {
};
Self {
bucket_incarnation_id: args.bucket_incarnation_id,
bucket_removed_targets: args.bucket_removed_targets,
event_version: event_schema_version(args.event_name).to_string(),
event_source: "rustfs:s3".to_string(),
aws_region: args.req_params.get("region").cloned().unwrap_or_default(),
@@ -387,6 +396,10 @@ fn initialize_response_elements(elements: &mut HashMap<String, String>, keys: &[
#[derive(Debug, Clone)]
pub struct EventArgs {
/// Bucket identity established by the producing operation.
pub bucket_incarnation_id: Option<uuid::Uuid>,
/// Delivery targets captured before a successful bucket removal withdraws its rules.
pub bucket_removed_targets: Option<TargetIdSet>,
pub event_name: EventName,
pub bucket_name: String,
pub object: NotifyObjectInfo,
@@ -436,6 +449,8 @@ impl EventArgs {
/// ```
#[derive(Debug, Clone, Default)]
pub struct EventArgsBuilder {
bucket_incarnation_id: Option<uuid::Uuid>,
bucket_removed_targets: Option<TargetIdSet>,
event_name: EventName,
bucket_name: String,
object: NotifyObjectInfo,
@@ -458,6 +473,18 @@ impl EventArgsBuilder {
}
}
/// Carries the operation's confirmed bucket identity to event dispatch.
pub fn bucket_incarnation_id(mut self, incarnation_id: Option<uuid::Uuid>) -> Self {
self.bucket_incarnation_id = incarnation_id;
self
}
/// Uses the subscription targets captured before deletion.
pub fn bucket_removed_targets(mut self, targets: TargetIdSet) -> Self {
self.bucket_removed_targets = Some(targets);
self
}
/// Sets the event name.
pub fn event_name(mut self, event_name: EventName) -> Self {
self.event_name = event_name;
@@ -529,6 +556,8 @@ impl EventArgsBuilder {
/// This method consumes the builder and returns the constructed `EventArgs`.
pub fn build(self) -> EventArgs {
EventArgs {
bucket_incarnation_id: self.bucket_incarnation_id,
bucket_removed_targets: self.bucket_removed_targets,
event_name: self.event_name,
bucket_name: self.bucket_name,
object: self.object,
@@ -823,6 +852,8 @@ mod event_args_tests {
req_params.insert((*k).to_string(), (*v).to_string());
}
EventArgs {
bucket_incarnation_id: None,
bucket_removed_targets: None,
event_name: EventName::ObjectRemovedDelete,
bucket_name: "b".to_string(),
object: ObjectInfo::default(),
+2 -1
View File
@@ -239,7 +239,8 @@ pub mod notifier_global {
.await
}
fn bucket_notification_config_for_rules(
/// Compiles event rules into the bucket configuration used by validation and publication.
pub fn bucket_notification_config_for_rules(
region: &str,
event_rules: &[(Vec<EventName>, String, String, Vec<TargetID>)],
) -> BucketNotificationConfig {
+7 -3
View File
@@ -12,7 +12,6 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::notification_system_subscriber::NotificationSystemSubscriberView;
use crate::notifier::{EventNotifier, TargetList};
use crate::services::NotifyServices;
use crate::{
@@ -182,7 +181,6 @@ impl NotificationSystem {
rustfs_utils::get_env_usize(ENV_NOTIFY_TARGET_STREAM_CONCURRENCY, DEFAULT_NOTIFY_TARGET_STREAM_CONCURRENCY);
let (live_event_sender, _) = broadcast::channel(1024);
let metrics = Arc::new(NotificationMetrics::new());
let subscriber_view = Arc::new(NotificationSystemSubscriberView::new());
let rule_engine = NotifyRuleEngine::new();
let notifier = Arc::new(EventNotifier::new(metrics.clone(), rule_engine.clone()));
let target_list = notifier.target_list();
@@ -200,7 +198,6 @@ impl NotificationSystem {
stream_cancellers,
concurrency_limiter,
metrics,
subscriber_view,
live_event_sender,
live_event_history,
);
@@ -413,6 +410,13 @@ impl NotificationSystem {
.await
}
/// Applies the current committed bucket rules while the caller holds the metadata publication fence.
pub fn publish_bucket_notification_config(&self, bucket: &str, cfg: Option<(uuid::Uuid, &BucketNotificationConfig)>) {
self.services
.bucket_config_manager
.publish_bucket_notification_config(bucket, cfg);
}
/// Sends an event
pub async fn send_event(&self, event: Arc<Event>) {
self.services.pipeline.send_event(event).await;
-1
View File
@@ -26,7 +26,6 @@ pub mod factory;
mod global;
pub mod integration;
mod lifecycle;
mod notification_system_subscriber;
pub mod notifier;
mod pipeline;
pub mod registry;
@@ -1,74 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::BucketNotificationConfig;
use crate::rules::{BucketRulesSnapshot, DynRulesContainer, SubscriberIndex};
use rustfs_s3_types::EventName;
/// NotificationSystemSubscriberView - Provides an interface to manage and query
/// the subscription status of buckets in the notification system.
#[derive(Debug)]
pub struct NotificationSystemSubscriberView {
index: SubscriberIndex,
}
impl NotificationSystemSubscriberView {
/// Creates a new NotificationSystemSubscriberView with an empty SubscriberIndex.
///
/// Returns a new instance of NotificationSystemSubscriberView.
pub fn new() -> Self {
Self {
index: SubscriberIndex::default(),
}
}
/// Checks if a bucket has any subscribers for a specific event.
/// This is a quick check using the event mask in the snapshot.
///
/// # Arguments
/// * `bucket` - The name of the bucket to check.
/// * `event` - The event name to check for subscriptions.
///
/// Returns `true` if there are subscribers for the event, `false` otherwise.
#[inline]
pub fn has_subscriber(&self, bucket: &str, event: &EventName) -> bool {
self.index.has_subscriber(bucket, event)
}
/// Builds and atomically replaces a bucket's subscription snapshot from the configuration.
///
/// Core principle: masks and rules are calculated and stored together in the same update.
///
/// # Arguments
/// * `bucket` - The name of the bucket to update.
/// * `cfg` - The bucket notification configuration to compile into a snapshot.
pub fn apply_bucket_config(&self, bucket: &str, cfg: &BucketNotificationConfig) {
// *It is recommended to merge compile into one function to ensure the same origin.
let snapshot: BucketRulesSnapshot<DynRulesContainer> = cfg.compile_snapshot();
// *debug to prevent inconsistencies from being introduced when modifying the compile logic in the future.
snapshot.debug_assert_mask_consistent();
self.index.store_snapshot(bucket, snapshot);
}
/// Clears a bucket's subscription snapshot.
///
/// #Arguments
/// * `bucket` - The name of the bucket to clear.
#[inline]
pub fn clear_bucket(&self, bucket: &str) {
self.index.clear_bucket(bucket);
}
}
+67 -2
View File
@@ -13,8 +13,9 @@
// limitations under the License.
use crate::error::NotificationError;
use crate::{event::Event, integration::NotificationMetrics, rule_engine::NotifyRuleEngine};
use crate::{event::Event, integration::NotificationMetrics, rule_engine::NotifyRuleEngine, rules::TargetIdSet};
use rustfs_config::notify::{DEFAULT_NOTIFY_SEND_CONCURRENCY, ENV_NOTIFY_SEND_CONCURRENCY};
use rustfs_s3_types::EventName;
use rustfs_targets::Target;
use rustfs_targets::arn::TargetID;
use rustfs_targets::target::EntityTarget;
@@ -235,6 +236,12 @@ impl EventNotifier {
);
}
/// Captures the deleted bucket's final delivery targets before its rules are withdrawn.
pub fn bucket_removed_targets(&self, bucket: &str, incarnation_id: uuid::Uuid) -> TargetIdSet {
self.rule_engine
.match_bucket_targets(bucket, Some(incarnation_id), EventName::BucketRemoved, "")
}
/// Sends an event to the appropriate targets based on the bucket rules
///
/// # Arguments
@@ -245,7 +252,25 @@ impl EventNotifier {
let object_key = &event.s3.object.key;
let event_name = event.event_name;
let target_ids = self.rule_engine.match_targets(bucket_name, event_name, object_key).await;
let target_ids = match &event.bucket_removed_targets {
Some(targets) if event_name == EventName::BucketRemoved => targets.clone(),
_ => {
let match_targets = || {
self.rule_engine
.match_bucket_targets(bucket_name, event.bucket_incarnation_id, event_name, object_key)
};
let targets = match_targets();
if targets.is_empty()
&& let Some(id) = event.bucket_incarnation_id
&& self.rule_engine.request_incarnation_refresh(bucket_name, id)
&& crate::storage_api::refresh_notify_bucket_metadata(bucket_name).await
{
match_targets()
} else {
targets
}
}
};
if target_ids.is_empty() {
debug!(
event = EVENT_NOTIFY_DISPATCH_SKIPPED,
@@ -1110,6 +1135,46 @@ mod tests {
assert_eq!(disabled_target.save_call_count(), 0);
}
#[tokio::test]
async fn bucket_removal_targets_survive_withdrawal_without_routing_recreated_objects() {
let engine = NotifyRuleEngine::new();
let notifier = EventNotifier::new(Arc::new(NotificationMetrics::new()), engine.clone());
let target = MockTarget::new("removed-bucket", "webhook");
let old_id = uuid::Uuid::new_v4();
let new_id = uuid::Uuid::new_v4();
let mut rules = RulesMap::new();
rules.add_rule_config(
&[EventName::BucketRemoved, EventName::ObjectCreatedPut],
"*".to_owned(),
target.target_id(),
);
engine.publish_bucket_rules("bucket", Some(old_id), rules);
notifier
.target_list()
.write()
.await
.add(Arc::new(target.clone()))
.expect("register target");
let targets = notifier.bucket_removed_targets("bucket", old_id);
assert!(targets.contains(&target.target_id()));
engine.withdraw_bucket_rules("bucket");
let mut removed = Event::new_test_event("bucket", "", EventName::BucketRemoved);
removed.bucket_incarnation_id = Some(old_id);
removed.bucket_removed_targets = Some(targets);
let encoded = serde_json::to_value(&removed).expect("serialize webhook event");
assert!(encoded.get("bucketIncarnationId").is_none());
assert!(encoded.get("bucketRemovedTargets").is_none());
notifier.send(Arc::new(removed)).await;
assert_eq!(target.save_call_count(), 1, "the deleted bucket earns its final removal event");
let mut created = Event::new_test_event("bucket", "fresh.txt", EventName::ObjectCreatedPut);
created.bucket_incarnation_id = Some(new_id);
notifier.send(Arc::new(created)).await;
assert_eq!(target.save_call_count(), 1, "the new bucket has no subscription");
}
#[tokio::test]
async fn send_event_respects_prefix_suffix_filters() {
let rule_engine = NotifyRuleEngine::new();
+110 -27
View File
@@ -16,9 +16,13 @@ use crate::rules::{RulesMap, TargetIdSet};
use percent_encoding::percent_decode_str;
use rustfs_s3_types::EventName;
use rustfs_targets::arn::TargetID;
use starshard::{AsyncShardedHashMap, DEFAULT_SHARDS, SnapshotMode};
use std::sync::Arc;
use starshard::{DEFAULT_SHARDS, ShardedHashMap, SnapshotMode};
use std::sync::{
Arc,
atomic::{AtomicBool, Ordering},
};
use tracing::info;
use uuid::Uuid;
const LOG_COMPONENT_NOTIFY: &str = "notify";
const LOG_SUBSYSTEM_RULE_ENGINE: &str = "rule_engine";
@@ -33,34 +37,56 @@ fn decoded_object_key_for_matching(object_key: &str) -> Option<String> {
(decoded != object_key).then(|| decoded.into_owned())
}
struct BucketRules {
incarnation_id: Option<Uuid>,
rules: RulesMap,
refresh_requested: Arc<AtomicBool>,
}
#[derive(Clone)]
pub struct NotifyRuleEngine {
bucket_rules_map: Arc<AsyncShardedHashMap<String, RulesMap, rustc_hash::FxBuildHasher>>,
bucket_rules_map: Arc<ShardedHashMap<String, Arc<BucketRules>, rustc_hash::FxBuildHasher>>,
}
impl NotifyRuleEngine {
pub fn new() -> Self {
Self {
bucket_rules_map: Arc::new(AsyncShardedHashMap::with_snapshot_mode(DEFAULT_SHARDS, SnapshotMode::Cached)),
bucket_rules_map: Arc::new(ShardedHashMap::with_snapshot_mode(DEFAULT_SHARDS, SnapshotMode::Cached)),
}
}
pub async fn is_target_bound_to_any_bucket(&self, target_id: &TargetID) -> bool {
let items = self.bucket_rules_map.iter().await;
for (_bucket, rules_map) in items {
if rules_map.contains_target_id(target_id) {
return true;
}
}
false
self.bucket_rules_map
.iter()
.any(|(_, snapshot)| snapshot.rules.contains_target_id(target_id))
}
pub async fn set_bucket_rules(&self, bucket: &str, rules_map: RulesMap) {
self.publish_bucket_rules(bucket, None, rules_map);
}
/// Atomically publishes the rules and the identity used by event dispatch.
pub(crate) fn publish_bucket_rules(&self, bucket: &str, incarnation_id: Option<Uuid>, rules_map: RulesMap) {
let event_count = rules_map.iter_events().count();
let bucket_key = bucket.to_owned();
if rules_map.is_empty() {
self.bucket_rules_map.remove(&bucket.to_string()).await;
self.bucket_rules_map.remove(&bucket_key);
} else {
self.bucket_rules_map.insert(bucket.to_string(), rules_map).await;
// A reload that confirms the current incarnation must not re-arm
// the refresh triggered by delayed events from an older bucket.
let refresh_requested = self
.bucket_rules_map
.get(&bucket_key)
.filter(|current| current.incarnation_id == incarnation_id)
.map_or_else(|| Arc::new(AtomicBool::new(false)), |current| current.refresh_requested.clone());
self.bucket_rules_map.insert(
bucket_key,
Arc::new(BucketRules {
incarnation_id,
rules: rules_map,
refresh_requested,
}),
);
}
info!(
event = EVENT_NOTIFY_RULES_UPDATED,
@@ -74,11 +100,17 @@ impl NotifyRuleEngine {
}
pub async fn get_bucket_rules(&self, bucket: &str) -> Option<RulesMap> {
self.bucket_rules_map.get(&bucket.to_string()).await
self.bucket_rules_map
.get(&bucket.to_string())
.map(|snapshot| snapshot.rules.clone())
}
pub async fn clear_bucket_rules(&self, bucket: &str) {
if self.bucket_rules_map.remove(&bucket.to_string()).await.is_some() {
self.withdraw_bucket_rules(bucket);
}
pub(crate) fn withdraw_bucket_rules(&self, bucket: &str) {
if self.bucket_rules_map.remove(&bucket.to_string()).is_some() {
info!(
event = EVENT_NOTIFY_RULES_UPDATED,
component = LOG_COMPONENT_NOTIFY,
@@ -91,21 +123,43 @@ impl NotifyRuleEngine {
}
pub async fn has_subscriber(&self, bucket: &str, event: &EventName) -> bool {
self.get_bucket_rules(bucket)
.await
.is_some_and(|rules_map| rules_map.has_subscriber(event))
self.bucket_rules_map
.get(&bucket.to_string())
.is_some_and(|snapshot| snapshot.rules.has_subscriber(event))
}
pub async fn match_targets(&self, bucket: &str, event_name: EventName, object_key: &str) -> TargetIdSet {
self.get_bucket_rules(bucket)
.await
.map_or_else(TargetIdSet::new, |rules_map| {
let mut target_ids = rules_map.match_rules(event_name, object_key);
if let Some(decoded_key) = decoded_object_key_for_matching(object_key) {
target_ids.extend(rules_map.match_rules(event_name, &decoded_key));
}
target_ids
})
self.match_bucket_targets(bucket, None, event_name, object_key)
}
/// Allow one targeted reload per installed bucket incarnation. The ordinary
/// metadata refresh loop can still recover a failed or interrupted reload.
pub(crate) fn request_incarnation_refresh(&self, bucket: &str, incarnation_id: Uuid) -> bool {
self.bucket_rules_map.get(&bucket.to_string()).is_some_and(|snapshot| {
snapshot.incarnation_id.is_some_and(|current| current != incarnation_id)
&& !snapshot.refresh_requested.swap(true, Ordering::Relaxed)
})
}
pub(crate) fn match_bucket_targets(
&self,
bucket: &str,
incarnation_id: Option<Uuid>,
event_name: EventName,
object_key: &str,
) -> TargetIdSet {
let Some(snapshot) = self
.bucket_rules_map
.get(&bucket.to_string())
.filter(|snapshot| snapshot.incarnation_id == incarnation_id)
else {
return TargetIdSet::new();
};
let mut targets = snapshot.rules.match_rules(event_name, object_key);
if let Some(decoded_key) = decoded_object_key_for_matching(object_key) {
targets.extend(snapshot.rules.match_rules(event_name, &decoded_key));
}
targets
}
}
@@ -122,6 +176,35 @@ mod tests {
use rustfs_s3_types::EventName;
use rustfs_targets::arn::TargetID;
#[test]
fn bucket_rules_route_events_only_with_the_matching_incarnation() {
let engine = NotifyRuleEngine::new();
let old_id = uuid::Uuid::new_v4();
let new_id = uuid::Uuid::new_v4();
let old_target = TargetID::new("old".to_string(), "webhook".to_string());
let new_target = TargetID::new("new".to_string(), "webhook".to_string());
let rules = |target: TargetID| {
let mut rules = RulesMap::new();
rules.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_owned(), target);
rules
};
let matched = |id| engine.match_bucket_targets("bucket", id, EventName::ObjectCreatedPut, "object");
engine.publish_bucket_rules("bucket", Some(old_id), rules(old_target.clone()));
assert!(matched(Some(old_id)).contains(&old_target));
assert!(matched(Some(new_id)).is_empty(), "a peer may still have the old rules");
assert!(engine.request_incarnation_refresh("bucket", new_id));
assert!(!engine.request_incarnation_refresh("bucket", new_id));
assert!(matched(None).is_empty());
engine.withdraw_bucket_rules("bucket");
assert!(matched(Some(old_id)).is_empty());
assert!(matched(Some(new_id)).is_empty());
engine.publish_bucket_rules("bucket", Some(new_id), rules(new_target.clone()));
assert!(matched(Some(old_id)).is_empty());
assert!(matched(Some(new_id)).contains(&new_target));
}
#[tokio::test]
async fn rule_engine_tracks_bucket_rule_lifecycle() {
let engine = NotifyRuleEngine::new();
+3 -7
View File
@@ -17,7 +17,6 @@ use crate::{
bucket_config_manager::NotifyBucketConfigManager,
config_manager::NotifyConfigManager,
integration::NotificationMetrics,
notification_system_subscriber::NotificationSystemSubscriberView,
notifier::{EventNotifier, SharedNotifyTargetList},
pipeline::{LiveEventHistory, NotifyPipeline},
registry::TargetRegistry,
@@ -52,7 +51,6 @@ impl NotifyServices {
stream_cancellers: Arc<RwLock<ReplayWorkerManager>>,
concurrency_limiter: Arc<Semaphore>,
metrics: Arc<NotificationMetrics>,
subscriber_view: Arc<NotificationSystemSubscriberView>,
live_event_sender: broadcast::Sender<Arc<Event>>,
live_event_history: Arc<RwLock<LiveEventHistory>>,
) -> Self {
@@ -65,7 +63,7 @@ impl NotifyServices {
metrics.clone(),
);
let config_manager = NotifyConfigManager::new(config, registry, rule_engine.clone(), runtime_facade.clone());
let bucket_config_manager = NotifyBucketConfigManager::new(notifier.clone(), rule_engine, subscriber_view);
let bucket_config_manager = NotifyBucketConfigManager::new(notifier.clone(), rule_engine);
let pipeline = NotifyPipeline::new(notifier, live_event_sender, live_event_history);
let status_view = NotifyStatusView::new(metrics);
@@ -84,8 +82,8 @@ impl NotifyServices {
mod tests {
use super::NotifyServices;
use crate::{
integration::NotificationMetrics, notification_system_subscriber::NotificationSystemSubscriberView,
notifier::EventNotifier, pipeline::LiveEventHistory, registry::TargetRegistry, rule_engine::NotifyRuleEngine,
integration::NotificationMetrics, notifier::EventNotifier, pipeline::LiveEventHistory, registry::TargetRegistry,
rule_engine::NotifyRuleEngine,
};
use rustfs_config::server_config::Config;
use rustfs_targets::ReplayWorkerManager;
@@ -102,7 +100,6 @@ mod tests {
let config = Arc::new(RwLock::new(Config::default()));
let stream_cancellers = Arc::new(RwLock::new(ReplayWorkerManager::new()));
let concurrency_limiter = Arc::new(Semaphore::new(4));
let subscriber_view = Arc::new(NotificationSystemSubscriberView::new());
let (live_event_sender, _) = broadcast::channel(16);
let live_event_history = Arc::new(RwLock::new(LiveEventHistory::default()));
@@ -115,7 +112,6 @@ mod tests {
stream_cancellers,
concurrency_limiter,
metrics,
subscriber_view,
live_event_sender,
live_event_history,
);
+9
View File
@@ -27,6 +27,15 @@ pub(crate) fn resolve_notify_object_store_handle() -> Option<Arc<NotifyStore>> {
resolve_notify_object_store_handle_from_backend()
}
pub(crate) async fn refresh_notify_bucket_metadata(bucket: &str) -> bool {
let Some(store) = resolve_notify_object_store_handle() else {
return false;
};
rustfs_ecstore::api::bucket::metadata_sys::reload_bucket_metadata(store, bucket)
.await
.is_ok()
}
pub(crate) type NotifyServerConfigSnapshot = rustfs_ecstore::api::config::com::ServerConfigSnapshot;
pub(crate) async fn read_notify_server_config_snapshot(store: Arc<NotifyStore>) -> Result<NotifyServerConfigSnapshot, String> {
+7
View File
@@ -24,6 +24,9 @@ use crate::{
#[derive(Debug, Clone, Default)]
pub struct DeletedObjectReplicationInfo {
/// The originating mutation's bucket instance, retained across in-process retries.
/// Replay without this identity leaves it absent and cannot deliver external notifications.
pub notification_bucket_incarnation: Option<Uuid>,
pub delete_object: DeletedObject,
pub bucket: String,
pub event_type: String,
@@ -182,6 +185,7 @@ pub fn resync_existing_delete_replication_info(roi: &ReplicateObjectInfo, target
};
DeletedObjectReplicationInfo {
notification_bucket_incarnation: roi.notification_bucket_incarnation,
delete_object: DeletedObject {
object_name: roi.name.clone(),
delete_marker_version_id: dm_version_id,
@@ -519,7 +523,9 @@ mod tests {
/// pin that this shape flows into purge — not marker-creation — semantics.
#[test]
fn resync_rebuilt_purge_entry_keeps_versioned_delete_semantics() {
let incarnation = Uuid::new_v4();
let roi = ReplicateObjectInfo {
notification_bucket_incarnation: Some(incarnation),
bucket: "bucket".to_string(),
name: "object".to_string(),
version_id: Some(Uuid::new_v4()),
@@ -530,6 +536,7 @@ mod tests {
let info = resync_existing_delete_replication_info(&roi, "arn:target-a");
assert_eq!(info.notification_bucket_incarnation, Some(incarnation));
assert!(is_version_delete_replication(&info.delete_object));
assert!(
!delete_replication_creates_marker(&info.delete_object),
+4
View File
@@ -891,6 +891,10 @@ pub struct ReplicationGenerationSnapshot {
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct ReplicateObjectInfo {
/// The originating mutation's bucket instance, retained across in-process retries.
/// Replay without this identity leaves it absent and cannot deliver external notifications.
#[serde(skip)]
pub notification_bucket_incarnation: Option<Uuid>,
pub name: String,
pub size: i64,
pub actual_size: i64,
+14 -2
View File
@@ -262,8 +262,16 @@ impl DefaultBucketUsecase {
// same-name make while it waits for repair coordination.
let operation_bucket = bucket.clone();
let operation_store = store.clone();
with_site_replication_bucket_mutation_lock(store, &bucket, move || async move {
let removed_targets = with_site_replication_bucket_mutation_lock(store, &bucket, move || async move {
let intent = prepare_site_replication_delete_bucket(&operation_bucket, force).await?;
let removed_targets = match rustfs_notify::notification_system() {
Some(system) => operation_store
.bucket_incarnation_id(&operation_bucket)
.await
.ok()
.map(|id| system.notifier.bucket_removed_targets(&operation_bucket, id)),
None => None,
};
let delete_result = operation_store
.delete_bucket(
&operation_bucket,
@@ -281,7 +289,7 @@ impl DefaultBucketUsecase {
{
warn!(bucket = %operation_bucket, error = ?err, "site replication delete bucket hook failed");
}
Ok::<(), S3Error>(())
Ok::<_, S3Error>(removed_targets)
}
Err(err) => {
if let Some(intent) = intent {
@@ -306,6 +314,10 @@ impl DefaultBucketUsecase {
// Notify peers to drop their cached metadata for the now-deleted bucket.
notify_bucket_metadata_delete(bucket, envelope.request_context().cloned());
let helper = match removed_targets {
Some(targets) => helper.with_bucket_removed_targets(targets),
None => helper,
};
let result = Ok(gateway_dto::DeleteBucketOutput::default());
let _ = helper.complete_owned(&result, StatusCode::OK, &http::HeaderMap::new());
result
+2 -12
View File
@@ -183,7 +183,7 @@ impl DefaultBucketUsecase {
})
}
/// Persists notification configuration, then updates the active rules for the bucket.
/// Persists notification configuration and publishes the committed metadata snapshot.
pub(crate) async fn execute_put_bucket_notification_configuration(
&self,
envelope: RequestEnvelope,
@@ -217,7 +217,7 @@ impl DefaultBucketUsecase {
let notify = current_notify_interface_for_context(self.context.as_deref());
let event_rules = build_notification_event_rules(destinations)?;
// Validate before the write. Clear and add still run after the durable update.
// The metadata publisher applies rules only after the durable update.
notify
.validate_event_specific_rules(bucket, &region, &event_rules)
.await
@@ -230,16 +230,6 @@ impl DefaultBucketUsecase {
notify_bucket_metadata_reload(bucket.to_owned(), "put bucket notification", request_context, false).await;
notify
.clear_bucket_notification_rules(bucket)
.await
.map_err(|e| s3_error!(InternalError, "Failed to clear rules: {e}"))?;
warn!("notify event rules: {:?}", &event_rules);
notify
.add_event_specific_rules(bucket, &region, &event_rules)
.await
.map_err(|e| s3_error!(InternalError, "Failed to add rules: {e}"))?;
Ok(gateway_dto::PutBucketNotificationConfigurationOutput::default())
},
)
@@ -51,22 +51,18 @@ enum Callback {
Validate,
ReloadStarted,
ReloadCompleted,
Clear,
Add,
}
#[derive(Debug, PartialEq, Eq)]
enum Observation {
Rules(Callback, String, Vec<RuntimeRule>, Vec<u8>),
Reload(Callback, &'static str, bool),
Clear(Vec<u8>),
}
struct RecordingNotify {
bucket: String,
fail_at: Option<Callback>,
calls: Arc<Mutex<Vec<Observation>>>,
active_rules: Mutex<Vec<RuntimeRule>>,
}
async fn read_notification_xml(bucket: &str) -> Vec<u8> {
@@ -113,22 +109,6 @@ impl NotifyInterface for RecordingNotify {
self.record_rules(Callback::Validate, bucket, region, rules).await;
self.maybe_fail(Callback::Validate)
}
async fn add_event_specific_rules(&self, bucket: &str, region: &str, rules: &[RuntimeRule]) -> Result<(), NotificationError> {
self.record_rules(Callback::Add, bucket, region, rules).await;
self.maybe_fail(Callback::Add)?;
*self.active_rules.lock().unwrap() = rules.to_vec();
Ok(())
}
async fn clear_bucket_notification_rules(&self, bucket: &str) -> Result<(), NotificationError> {
assert_eq!(bucket, self.bucket);
let xml = read_notification_xml(bucket).await;
self.calls.lock().unwrap().push(Observation::Clear(xml));
self.maybe_fail(Callback::Clear)?;
self.active_rules.lock().unwrap().clear();
Ok(())
}
}
fn configuration() -> NotificationConfiguration {
@@ -219,7 +199,7 @@ fn check_notification_write_order() {
run_large_stack_test("notification-write-order", || async {
let ambient = shared_gating_ambient().await;
let store = ambient.object_store();
for fail_at in [None, Some(Callback::Validate), Some(Callback::Clear), Some(Callback::Add)] {
for fail_at in [None, Some(Callback::Validate)] {
let bucket = format!("notify-order-{}", uuid::Uuid::new_v4().simple());
store.make_bucket(&bucket, &MakeBucketOptions::default()).await.unwrap();
store
@@ -241,9 +221,8 @@ fn check_notification_write_order() {
bucket: bucket.clone(),
fail_at,
calls: calls.clone(),
active_rules: Mutex::new(expected_rules()),
});
let context = Arc::new(app_context_from_current_environment(&ambient).with_test_notify_interface(recorder.clone()));
let context = Arc::new(app_context_from_current_environment(&ambient).with_test_notify_interface(recorder));
let region = context
.region()
.get()
@@ -261,18 +240,14 @@ fn check_notification_write_order() {
let mut timeline = vec![Observation::Rules(
Callback::Validate,
region.clone(),
expected.clone(),
expected,
OLD_XML.to_vec(),
)];
if fail_at != Some(Callback::Validate) {
timeline.extend([
Observation::Reload(Callback::ReloadStarted, "put bucket notification", false),
Observation::Reload(Callback::ReloadCompleted, "put bucket notification", false),
Observation::Clear(NEW_XML.to_vec()),
]);
if fail_at != Some(Callback::Clear) {
timeline.push(Observation::Rules(Callback::Add, region.clone(), expected.clone(), NEW_XML.to_vec()));
}
}
assert_eq!(*calls.lock().unwrap(), timeline);
let stored = if fail_at == Some(Callback::Validate) {
@@ -282,7 +257,6 @@ fn check_notification_write_order() {
};
assert_eq!(read_notification_xml(&bucket).await, stored);
let error_message = match fail_at {
Some(Callback::Clear) => Some("Failed to clear rules: Rule configuration error: fixture failure"),
Some(_) => Some("Failed to add rules: Rule configuration error: fixture failure"),
None => None,
};
@@ -304,7 +278,6 @@ fn check_notification_write_order() {
assert!(audit.events.lock().unwrap().is_empty());
if fail_at.is_none() {
assert_eq!(*recorder.active_rules.lock().unwrap(), expected);
let mut invalid = configuration();
invalid.lambda_function_configurations.as_mut().unwrap()[0].lambda_function_arn = "invalid-arn".into();
let mut invalid = notification_request(&bucket, invalid, &slot);
@@ -350,12 +323,9 @@ fn check_notification_write_order() {
Observation::Rules(Callback::Validate, region.clone(), Vec::new(), NEW_XML.to_vec()),
Observation::Reload(Callback::ReloadStarted, "put bucket notification", false),
Observation::Reload(Callback::ReloadCompleted, "put bucket notification", false),
Observation::Clear(EMPTY_XML.to_vec()),
Observation::Rules(Callback::Add, region, Vec::new(), EMPTY_XML.to_vec()),
]
);
assert_eq!(read_notification_xml(&bucket).await, EMPTY_XML);
assert!(recorder.active_rules.lock().unwrap().is_empty());
}
}
});
@@ -412,7 +382,6 @@ fn legacy_put_notification_does_not_write_after_the_bucket_is_recreated() {
bucket: bucket.clone(),
fail_at: None,
calls: calls.clone(),
active_rules: Mutex::new(Vec::new()),
});
let context = Arc::new(app_context_from_current_environment(&ambient).with_test_notify_interface(recorder));
let slot = ServerContextSlot::new();
+1
View File
@@ -849,6 +849,7 @@ impl RequestEnvelope {
self.client_info(),
self.remote_addr(),
self.request_context(),
self.bucket_generation(),
)
}
}
-13
View File
@@ -191,19 +191,6 @@ impl NotifyInterface for NotifyHandle {
) -> Result<(), NotificationError> {
runtime_sources::validate_event_specific_rules(bucket_name, region, event_rules).await
}
async fn add_event_specific_rules(
&self,
bucket_name: &str,
region: &str,
event_rules: &[(Vec<EventName>, String, String, Vec<TargetID>)],
) -> Result<(), NotificationError> {
runtime_sources::add_event_specific_rules(bucket_name, region, event_rules).await
}
async fn clear_bucket_notification_rules(&self, bucket_name: &str) -> Result<(), NotificationError> {
runtime_sources::clear_bucket_notification_rules(bucket_name).await
}
}
/// Default notification system handle adapter.
-9
View File
@@ -90,15 +90,6 @@ pub trait NotifyInterface: Send + Sync {
region: &str,
event_rules: &[(Vec<EventName>, String, String, Vec<TargetID>)],
) -> Result<(), NotificationError>;
async fn add_event_specific_rules(
&self,
bucket_name: &str,
region: &str,
event_rules: &[(Vec<EventName>, String, String, Vec<TargetID>)],
) -> Result<(), NotificationError>;
async fn clear_bucket_notification_rules(&self, bucket_name: &str) -> Result<(), NotificationError>;
}
/// Notification system handle interface for admin peer orchestration.
-12
View File
@@ -79,18 +79,6 @@ pub async fn validate_event_specific_rules(
notifier_global::validate_event_specific_rules(bucket_name, region, event_rules).await
}
pub async fn add_event_specific_rules(
bucket_name: &str,
region: &str,
event_rules: &[(Vec<EventName>, String, String, Vec<TargetID>)],
) -> Result<(), NotificationError> {
notifier_global::add_event_specific_rules(bucket_name, region, event_rules).await
}
pub async fn clear_bucket_notification_rules(bucket_name: &str) -> Result<(), NotificationError> {
notifier_global::clear_bucket_notification_rules(bucket_name).await
}
pub fn notification_system() -> Option<Arc<NotificationSys>> {
get_global_notification_sys()
}
+1 -1
View File
@@ -563,7 +563,7 @@ impl DefaultMultipartUsecase {
&& decision.replicate_any()
{
warn!("need multipart replication");
schedule_object_replication(obj_info.clone(), store, decision).await;
schedule_object_replication(obj_info.clone(), store, decision, opts.expected_bucket_incarnation_id).await;
}
rustfs_scanner::record_dirty_usage_object_from_producer(
+2 -1
View File
@@ -821,7 +821,8 @@ impl DefaultObjectUsecase {
// the persisted pending marker and the schedule always agree, mirroring
// the PUT path.
if dsc.replicate_any() {
schedule_object_replication(oi.clone(), Arc::clone(&store), dsc).await;
schedule_object_replication(oi.clone(), Arc::clone(&store), dsc, dst_opts.expected_bucket_incarnation_id)
.await;
}
maybe_enqueue_transition_immediate(&oi, LcEventSrc::S3CopyObject).await;
+24 -4
View File
@@ -945,6 +945,7 @@ impl DefaultObjectUsecase {
..Default::default()
};
apply_bucket_generation_guard(&envelope, &bucket, &mut storage_delete_opts)?;
let incarnation_id = storage_delete_opts.expected_bucket_incarnation_id;
let storage_started = std::time::Instant::now();
let (dobjs, errs, accounting) = store
.delete_objects_with_tier_delete_journal_and_accounting(&bucket, object_to_delete.clone(), storage_delete_opts)
@@ -1096,8 +1097,13 @@ impl DefaultObjectUsecase {
let bucket_for_replication = bucket.clone();
let replication_task = tokio::spawn(async move {
let _activity_guard = DeleteTailActivityGuard::new(DeleteTailStage::Replication);
schedule_replication_deletes(replication_deletes, bucket_for_replication, REPLICATE_INCOMING_DELETE.to_string())
.await;
schedule_replication_deletes(
replication_deletes,
bucket_for_replication,
REPLICATE_INCOMING_DELETE.to_string(),
incarnation_id,
)
.await;
});
// The spawned task owns every locally committed delete. Dropping the
// join handle on request cancellation therefore cannot lose the tail.
@@ -1158,6 +1164,7 @@ impl DefaultObjectUsecase {
..Default::default()
},
)
.bucket_incarnation_id(incarnation_id)
.version_id(version_id.unwrap_or_default())
.req_params(req_params.clone())
.resp_elements(resp_elements.clone())
@@ -1499,6 +1506,7 @@ impl DefaultObjectUsecase {
},
bucket.clone(),
REPLICATE_INCOMING_DELETE.to_string(),
opts.expected_bucket_incarnation_id,
)
.await;
} else if replicate_force_delete {
@@ -1521,7 +1529,13 @@ impl DefaultObjectUsecase {
) {
set_deleted_object_replication_state(&mut delete_object, &replication_state);
}
schedule_replication_delete(delete_object, bucket.clone(), REPLICATE_INCOMING_DELETE.to_string()).await;
schedule_replication_delete(
delete_object,
bucket.clone(),
REPLICATE_INCOMING_DELETE.to_string(),
opts.expected_bucket_incarnation_id,
)
.await;
}
// Prefix/force-delete returns empty ObjectInfo; still emit bucket notification so webhooks match S3 DELETE.
helper = helper
@@ -1602,7 +1616,13 @@ impl DefaultObjectUsecase {
set_deleted_object_replication_state(&mut deleted_object, &replication_state_source.replication_state());
enrich_delete_replication_state_if_needed(&delete_config_snapshot, &mut deleted_object, replication_state_source);
}
schedule_replication_delete(deleted_object, bucket.clone(), REPLICATE_INCOMING_DELETE.to_string()).await;
schedule_replication_delete(
deleted_object,
bucket.clone(),
REPLICATE_INCOMING_DELETE.to_string(),
opts.expected_bucket_incarnation_id,
)
.await;
}
let delete_marker = obj_info.delete_marker;
+4 -14
View File
@@ -740,13 +740,16 @@ async fn complete_extract_member_post_commit(
let _ = invalidate_object_data_cache_after_put_success(&context.cache_adapter, &context.bucket, &key).await;
if replication.replicate_any() {
schedule_object_replication(obj_info.clone(), context.store.clone(), replication).await;
schedule_object_replication(obj_info.clone(), context.store.clone(), replication, opts.expected_bucket_incarnation_id)
.await;
}
let actual_version_id = extract_notification_version_id(obj_info.version_id, opts.versioned, opts.version_suspended);
let mut event_object = convert_ecstore_object_info(obj_info);
event_object.version_id = (!actual_version_id.is_empty()).then_some(actual_version_id.clone());
let event = rustfs_notify::EventArgs {
bucket_incarnation_id: opts.expected_bucket_incarnation_id,
bucket_removed_targets: None,
event_name: put_event_name_for_post_object(false),
bucket_name: context.bucket.clone(),
object: event_object,
@@ -2858,19 +2861,6 @@ mod tests {
) -> Result<(), rustfs_notify::NotificationError> {
Ok(())
}
async fn add_event_specific_rules(
&self,
_bucket_name: &str,
_region: &str,
_event_rules: &[(Vec<rustfs_targets::EventName>, String, String, Vec<rustfs_targets::arn::TargetID>)],
) -> Result<(), rustfs_notify::NotificationError> {
Ok(())
}
async fn clear_bucket_notification_rules(&self, _bucket_name: &str) -> Result<(), rustfs_notify::NotificationError> {
Ok(())
}
}
async fn recording_extract_commit_context(
+13 -1
View File
@@ -187,6 +187,11 @@ impl InternalPutObjectEvent {
.req_param("principalId", principal_id)
}
pub(super) fn with_bucket_incarnation(mut self, incarnation_id: Option<Uuid>) -> Self {
self.builder = self.builder.bucket_incarnation_id(incarnation_id);
self
}
pub(super) fn object(self, obj_info: ObjectInfo) -> Self {
let Self {
builder,
@@ -709,6 +714,7 @@ impl DefaultObjectUsecase {
&key,
ctx.principal_id,
)
.map(|event| event.with_bucket_incarnation(opts.expected_bucket_incarnation_id))
});
// The spawned task owns the commit so a cancelled caller cannot leave
@@ -742,7 +748,13 @@ impl DefaultObjectUsecase {
enqueue_transition_immediate(&obj_info, LcEventSrc::S3CompleteMultipartUpload).await;
if completion_replication_decision.replicate_any() {
schedule_object_replication(obj_info.clone(), store, completion_replication_decision).await;
schedule_object_replication(
obj_info.clone(),
store,
completion_replication_decision,
opts.expected_bucket_incarnation_id,
)
.await;
}
rustfs_scanner::record_dirty_usage_object_from_producer(
+2 -1
View File
@@ -2023,6 +2023,7 @@ impl DefaultObjectUsecase {
&key,
principal_id,
)
.map(|event| event.with_bucket_incarnation(opts.expected_bucket_incarnation_id))
});
(PutObjectCompletion::Internal(event.flatten().map(Box::new)), Some(request_context))
}
@@ -2316,7 +2317,7 @@ impl DefaultObjectUsecase {
if dsc.replicate_any() {
let replication_schedule_stage_start = put_stage_metrics_enabled.then(Instant::now);
schedule_object_replication(obj_info.clone(), store, dsc).await;
schedule_object_replication(obj_info.clone(), store, dsc, opts.expected_bucket_incarnation_id).await;
rustfs_io_metrics::record_put_object_stage_duration_from(
"app_post_store_replication_schedule",
replication_schedule_stage_start,
+1 -1
View File
@@ -132,7 +132,7 @@ impl ObjectMetaUsecase {
// The current target transport carries the updated metadata in a full-object PUT.
if dsc.replicate_any() {
schedule_metadata_replication(info.clone(), store.clone(), dsc).await;
schedule_metadata_replication(info.clone(), store.clone(), dsc, popts.expected_bucket_incarnation_id).await;
}
let output = PutObjectLegalHoldOutput {
+1 -1
View File
@@ -151,7 +151,7 @@ impl ObjectMetaUsecase {
// The current target transport carries the updated metadata in a full-object PUT.
if dsc.replicate_any() {
schedule_metadata_replication(object_info.clone(), store.clone(), dsc).await;
schedule_metadata_replication(object_info.clone(), store.clone(), dsc, opts.expected_bucket_incarnation_id).await;
}
let output = PutObjectRetentionOutput {
+6 -2
View File
@@ -29,6 +29,7 @@ use crate::app::storage_api::object_meta_usecase::{
};
use crate::app::storage_api::object_usecase::ECStore;
use crate::app::storage_api::object_usecase::StorageObjectOptions as ObjectOptions;
use crate::app::storage_api::object_usecase::access::apply_bucket_generation_guard;
use crate::app::storage_api::object_usecase::bucket::replication::{must_replicate_metadata, schedule_metadata_replication};
use crate::app::storage_api::object_usecase::bucket::versioning_sys::BucketVersioningSys;
use crate::app::storage_api::object_usecase::error::{is_err_object_not_found, is_err_version_not_found};
@@ -208,6 +209,7 @@ impl ObjectMetaUsecase {
let mut opts = get_opts(bucket, object, version_id.clone(), None, envelope.headers())
.await
.map_err(ApiError::from)?;
apply_bucket_generation_guard(&envelope, bucket, &mut opts)?;
let existing = match store.get_object_info(bucket, object, &opts).await {
Ok(info) => info,
Err(error) => {
@@ -264,7 +266,7 @@ impl ObjectMetaUsecase {
ApiError::from(error)
})?;
if decision.replicate_any() {
schedule_metadata_replication(info.clone(), Arc::clone(&store), decision).await;
schedule_metadata_replication(info.clone(), Arc::clone(&store), decision, opts.expected_bucket_incarnation_id).await;
}
counter!("rustfs_put_object_tagging_success").increment(1);
@@ -328,6 +330,7 @@ impl ObjectMetaUsecase {
let mut opts = get_opts(bucket, object, version_id.clone(), None, envelope.headers())
.await
.map_err(ApiError::from)?;
apply_bucket_generation_guard(&envelope, bucket, &mut opts)?;
let existing_object_info = match store.get_object_info(bucket, object, &opts).await {
Ok(info) => info,
Err(e) => {
@@ -388,7 +391,8 @@ impl ObjectMetaUsecase {
.or_else(|| object_info.version_id.map(|version_id| version_id.to_string()))
.unwrap_or_default();
if dsc.replicate_any() {
schedule_metadata_replication(object_info.clone(), Arc::clone(&store), dsc).await;
schedule_metadata_replication(object_info.clone(), Arc::clone(&store), dsc, opts.expected_bucket_incarnation_id)
.await;
}
counter!("rustfs_delete_object_tagging_success").increment(1);
helper = helper.object(object_info).version_id(event_version_id);
+29 -4
View File
@@ -1385,42 +1385,67 @@ pub(crate) mod bucket {
oi: crate::storage::storage_api::StorageObjectInfo,
store: Arc<crate::storage::storage_api::ECStore>,
dsc: ReplicateDecision,
notification_bucket_incarnation: Option<Uuid>,
) {
ReplicationObjectBridge::schedule_object(oi, store, dsc, replication_contracts::ReplicationType::Object).await;
ReplicationObjectBridge::schedule_object(
oi,
store,
dsc,
replication_contracts::ReplicationType::Object,
notification_bucket_incarnation,
)
.await;
}
pub(crate) async fn schedule_metadata_replication(
oi: crate::storage::storage_api::StorageObjectInfo,
store: Arc<crate::storage::storage_api::ECStore>,
dsc: ReplicateDecision,
notification_bucket_incarnation: Option<Uuid>,
) {
ReplicationObjectBridge::schedule_object(oi, store, dsc, replication_contracts::ReplicationType::Metadata).await;
ReplicationObjectBridge::schedule_object(
oi,
store,
dsc,
replication_contracts::ReplicationType::Metadata,
notification_bucket_incarnation,
)
.await;
}
pub(crate) async fn schedule_replication_delete(
delete_object: crate::storage::storage_api::StorageDeletedObject,
bucket: String,
event_type: String,
notification_bucket_incarnation: Option<Uuid>,
) {
#[cfg(test)]
SCHEDULED_REPLICATION_DELETES
.lock()
.expect("scheduled replication delete lock should not be poisoned")
.push(delete_object.clone());
ReplicationObjectBridge::schedule_storage_delete(delete_object, bucket, event_type).await;
ReplicationObjectBridge::schedule_storage_delete(delete_object, bucket, event_type, notification_bucket_incarnation)
.await;
}
pub(crate) async fn schedule_replication_deletes(
delete_objects: Vec<crate::storage::storage_api::StorageDeletedObject>,
bucket: String,
event_type: String,
notification_bucket_incarnation: Option<Uuid>,
) {
#[cfg(test)]
SCHEDULED_REPLICATION_DELETES
.lock()
.expect("scheduled replication delete lock should not be poisoned")
.extend(delete_objects.iter().cloned());
ReplicationObjectBridge::schedule_storage_deletes(delete_objects, bucket, event_type).await;
ReplicationObjectBridge::schedule_storage_deletes(
delete_objects,
bucket,
event_type,
notification_bucket_incarnation,
)
.await;
}
pub(crate) fn set_deleted_object_replication_state(
+43 -162
View File
@@ -16,12 +16,9 @@
use crate::protocols::ProtocolStorageClient;
use crate::runtime_sources::current_region;
use crate::server::ShutdownHandle;
use crate::server::runtime_sources::current_notify_interface;
use crate::storage_api::error::{StorageError, is_unreadable_config_error};
use crate::storage_api::startup::bucket_metadata::contract::bucket::{BucketOperations, BucketOptions};
use crate::storage_api::startup::init::notification_config_to_s3s;
use crate::storage_api::startup::init::{
get_bucket_notification_config, process_lambda_configurations, process_queue_configurations, process_topic_configurations,
deserialize, process_lambda_configurations, process_queue_configurations, process_topic_configurations,
};
use crate::storage_api::startup::services::ECStore;
#[cfg(any(feature = "ftps", feature = "webdav", feature = "sftp"))]
@@ -32,8 +29,7 @@ use rustfs_config::{
DEFAULT_BUFFER_MAX_SIZE, DEFAULT_BUFFER_MIN_SIZE, DEFAULT_BUFFER_PROFILE, DEFAULT_BUFFER_UNKNOWN_SIZE, DEFAULT_UPDATE_CHECK,
ENV_RUSTFS_BUFFER_DEFAULT_SIZE, ENV_RUSTFS_BUFFER_MAX_SIZE, ENV_RUSTFS_BUFFER_MIN_SIZE, ENV_UPDATE_CHECK, RUSTFS_REGION,
};
use rustfs_notify::NotificationError;
use rustfs_s3_types::{EventName, s3_error};
use rustfs_s3_types::EventName;
use rustfs_targets::arn::{ARN, TargetID, TargetIDError};
use rustfs_utils::get_env_usize;
use std::env;
@@ -174,148 +170,44 @@ fn notification_config_to_event_rules(
Ok(event_rules)
}
/// One bucket's persisted notification configuration as startup sees it.
#[derive(Debug)]
enum BucketNotificationLookup {
Configured(rustfs_gateway_types::persistence::PersistedNotificationConfiguration),
Missing,
/// Stored bytes cannot be parsed. Deterministic, so a retry cannot help,
/// and reading it as missing would clear the bucket's rules.
Unreadable,
}
fn classify_bucket_notification_lookup(
bucket: &str,
lookup: Result<Option<rustfs_gateway_types::persistence::PersistedNotificationConfiguration>, StorageError>,
) -> Result<BucketNotificationLookup, NotificationError> {
match lookup {
Ok(Some(cfg)) => Ok(BucketNotificationLookup::Configured(cfg)),
Ok(None) => Ok(BucketNotificationLookup::Missing),
Err(err) if is_unreadable_config_error(&err) => {
/// Publishes the complete, current configuration under the bucket metadata publication lock.
pub(crate) fn publish_bucket_notification_configuration(bucket: &str, stored: Option<(&[u8], uuid::Uuid)>) {
let Some((bytes, incarnation_id)) = stored else {
if let Some(system) = rustfs_notify::notification_system() {
system.publish_bucket_notification_config(bucket, None);
}
return;
};
let parsed = (|| {
let config = deserialize::<rustfs_gateway_types::persistence::PersistedNotificationConfiguration>(bytes)
.map_err(|_| "invalid_xml")?;
let config = notification_config_to_s3s(&config).map_err(|_| "invalid_configuration")?;
notification_config_to_event_rules(&config).map_err(|_| "invalid_target_arn")
})();
match parsed {
Ok(rules) => {
let config = rustfs_notify::notifier_global::bucket_notification_config_for_rules(&notification_region(), &rules);
rustfs_notify::ensure_live_events().publish_bucket_notification_config(bucket, Some((incarnation_id, &config)));
}
Err(reason) => {
error!(
target: "rustfs::init",
event = "notification_config_unreadable",
component = LOG_COMPONENT_INIT,
subsystem = LOG_SUBSYSTEM_NOTIFICATION,
bucket = %bucket,
error = %err,
reason,
"Bucket notification configuration is unreadable; leaving its rules unchanged"
);
Ok(BucketNotificationLookup::Unreadable)
}
Err(err) => Err(NotificationError::StorageNotAvailable(format!(
"load bucket notification config for {bucket}: {err}"
))),
}
}
/// Apply one bucket's persisted notification rules. An unreadable config is
/// isolated to its bucket: it neither aborts setup for the other buckets nor
/// clears the bucket's rules as if it had none.
async fn apply_bucket_notification_configuration(bucket: &str, region: &str) -> Result<bool, NotificationError> {
match classify_bucket_notification_lookup(bucket, get_bucket_notification_config(bucket).await)? {
BucketNotificationLookup::Unreadable => Ok(false),
BucketNotificationLookup::Configured(cfg) => {
let cfg = notification_config_to_s3s(&cfg).map_err(|err| NotificationError::BucketNotification(err.to_string()))?;
info!(
target: "rustfs::init",
event = "notification_config_loaded",
component = LOG_COMPONENT_INIT,
subsystem = LOG_SUBSYSTEM_NOTIFICATION,
bucket = %bucket,
queue_configuration_count = cfg.queue_configurations.as_ref().map_or(0, Vec::len),
topic_configuration_count = cfg.topic_configurations.as_ref().map_or(0, Vec::len),
lambda_configuration_count = cfg.lambda_function_configurations.as_ref().map_or(0, Vec::len),
"Loaded bucket notification configuration"
);
let event_rules =
notification_config_to_event_rules(&cfg).map_err(|err| NotificationError::BucketNotification(err.to_string()))?;
current_notify_interface()
.add_event_specific_rules(bucket, region, &event_rules)
.await?;
Ok(true)
}
BucketNotificationLookup::Missing => {
info!(
target: "rustfs::init",
event = "notification_config_missing",
component = LOG_COMPONENT_INIT,
subsystem = LOG_SUBSYSTEM_NOTIFICATION,
bucket = %bucket,
"Bucket notification configuration not found"
);
current_notify_interface().clear_bucket_notification_rules(bucket).await?;
Ok(false)
}
}
}
/// Add existing bucket notification configurations to the global notifier system.
/// This function retrieves notification configurations for each bucket
/// and registers the corresponding event rules with the notifier system.
/// It processes queue, topic, and lambda configurations and maps them to event rules.
/// # Arguments
/// * `buckets` - A vector of bucket names to process
#[instrument(skip_all)]
pub async fn add_bucket_notification_configuration(buckets: Vec<String>) {
let region = notification_region();
for bucket in buckets.iter() {
if let Err(err) = apply_bucket_notification_configuration(bucket, region.as_str()).await {
let err = s3_error!(InternalError, "Failed to add rules: {err}");
error!(
target: "rustfs::init",
event = "notification_rules_registration_failed",
component = LOG_COMPONENT_INIT,
subsystem = LOG_SUBSYSTEM_NOTIFICATION,
bucket = %bucket,
region,
error = ?err,
"Failed to register bucket notification rules"
);
}
}
}
pub(crate) async fn reconcile_persisted_bucket_notification_configurations(
store: Arc<rustfs_notify::NotifyStore>,
) -> Result<usize, NotificationError> {
let bucket_infos = store
.list_bucket(&BucketOptions {
no_metadata: true,
..Default::default()
})
.await
.map_err(|err| NotificationError::StorageNotAvailable(format!("list buckets for notification reconciliation: {err}")))?;
let region = notification_region();
let mut configured_bucket_count = 0;
for bucket in bucket_infos {
if apply_bucket_notification_configuration(&bucket.name, region.as_str()).await? {
configured_bucket_count += 1;
}
}
Ok(configured_bucket_count)
}
fn notification_region() -> String {
let global_region = current_region();
global_region
current_region()
.as_ref()
.filter(|r| !r.as_str().is_empty())
.map(|r| r.to_string())
.unwrap_or_else(|| {
warn!(
target: "rustfs::init",
event = "notification_region_fallback",
component = LOG_COMPONENT_INIT,
subsystem = LOG_SUBSYSTEM_NOTIFICATION,
fallback_region = RUSTFS_REGION,
"Notification configuration falling back to default region"
);
RUSTFS_REGION.to_string()
})
.filter(|region| !region.as_str().is_empty())
.map_or_else(|| RUSTFS_REGION.to_owned(), ToString::to_string)
}
/// Build KMS configuration for local backend
@@ -1423,42 +1315,16 @@ pub async fn init_sftp_system(
#[cfg(test)]
mod tests {
use super::{
BucketNotificationLookup, build_aws_kms_config, build_vault_kms_config, build_vault_transit_kms_config,
classify_bucket_notification_lookup, notification_config_to_event_rules, resolve_buffer_profile_config,
build_aws_kms_config, build_vault_kms_config, build_vault_transit_kms_config, notification_config_to_event_rules,
resolve_buffer_profile_config,
};
use crate::config::{BufferConfig, WorkloadProfile};
use crate::storage_api::error::{StorageError, UnreadableBucketConfig};
use rustfs_config::KI_B;
use rustfs_s3_types::EventName;
use s3s::dto::{
FilterRule, FilterRuleName, NotificationConfiguration, NotificationConfigurationFilter, QueueConfiguration, S3KeyFilter,
};
/// rustfs/backlog#1734: an unreadable notification config is isolated to
/// its bucket. It must not abort setup for every other bucket (a storage
/// fault still does, so reconciliation retries) and must not read as
/// missing, which would clear the bucket's rules.
#[test]
fn unreadable_notification_config_is_isolated_to_its_bucket() {
let unreadable = StorageError::other(UnreadableBucketConfig {
bucket: "b".to_string(),
config_file: "notification.xml".to_string(),
raw_len: 5,
});
assert!(matches!(
classify_bucket_notification_lookup("b", Err(unreadable)),
Ok(BucketNotificationLookup::Unreadable)
));
assert!(matches!(
classify_bucket_notification_lookup("b", Ok(None)),
Ok(BucketNotificationLookup::Missing)
));
assert!(matches!(
classify_bucket_notification_lookup("b", Err(StorageError::ErasureReadQuorum)),
Err(rustfs_notify::NotificationError::StorageNotAvailable(_))
));
}
#[test]
fn resolve_buffer_profile_config_returns_fallback_when_primary_is_invalid() {
let invalid_primary = WorkloadProfile::Custom(BufferConfig {
@@ -1488,6 +1354,21 @@ mod tests {
assert!(resolved.is_none());
}
#[tokio::test]
async fn corrupt_notification_config_keeps_same_incarnation_rules_until_withdrawn() {
let bucket = "unreadable-notify-config";
let id = uuid::Uuid::new_v4();
let xml = br#"<NotificationConfiguration><QueueConfiguration><Id>test</Id><Queue>arn:rustfs:sqs:us-east-1:primary:webhook</Queue><Event>s3:ObjectCreated:*</Event></QueueConfiguration></NotificationConfiguration>"#;
super::publish_bucket_notification_configuration(bucket, Some((xml, id)));
let system = rustfs_notify::notification_system().expect("notification system created by metadata install");
assert!(system.has_subscriber(bucket, &EventName::ObjectCreatedPut).await);
super::publish_bucket_notification_configuration(bucket, Some((b"malformed xml", id)));
assert!(system.has_subscriber(bucket, &EventName::ObjectCreatedPut).await);
super::publish_bucket_notification_configuration(bucket, None);
assert!(!system.has_subscriber(bucket, &EventName::ObjectCreatedPut).await);
}
#[test]
fn notification_config_to_event_rules_preserves_target_and_filters() {
let cfg = NotificationConfiguration {
@@ -17,10 +17,9 @@
#[cfg(test)]
pub(super) use crate::storage_api::on_demand_migration::test_support;
pub(super) use crate::storage_api::on_demand_migration::{
BUCKET_CONFIG_PUBLISH_HOOK, BUCKET_META_PREFIX, BUCKET_ON_DEMAND_MIGRATION_CONFIG, ECStore, HTTPPreconditions, HTTPRangeSpec,
NamespaceLocking, ObjectOperations, ObjectOptions, RUSTFS_META_BUCKET, StorageError, WriteCompletion,
get_lock_acquire_timeout, get_on_demand_migration_config, get_on_demand_migration_config_in, read_config_with_metadata,
remote_s3_client, save_config_with_opts,
BUCKET_META_PREFIX, ECStore, HTTPPreconditions, HTTPRangeSpec, NamespaceLocking, ObjectOperations, ObjectOptions,
RUSTFS_META_BUCKET, StorageError, WriteCompletion, get_lock_acquire_timeout, get_on_demand_migration_config,
get_on_demand_migration_config_in, read_config_with_metadata, remote_s3_client, save_config_with_opts,
};
pub(super) async fn local_node_name() -> String {
+2 -15
View File
@@ -19,7 +19,7 @@
//! [`SourceClient`], a circuit breaker, a negative cache, a per-key
//! singleflight table, a pull concurrency limit and counters. Its lifecycle
//! follows the bucket metadata cache through the publish hook registered in
//! [`BUCKET_CONFIG_PUBLISH_HOOK`]; the hook fires on every cache install
//! `BUCKET_CONFIG_PUBLISH_HOOK`. It fires on every cache install
//! path (initial load, admin update, peer reload, refresh loop, lazy load).
//!
//! Change detection compares the config by value (`PartialEq`) rather than
@@ -52,7 +52,6 @@ use super::stats::{GaugeGuard, OdmStats, OdmStatsSnapshot, PullFailureReason};
use super::storage_api::remote_s3_client::{
PathStyle as ClientPathStyle, RemoteCredentials, RemoteS3ClientError, RemoteS3RetryPolicy,
};
use super::storage_api::{BUCKET_CONFIG_PUBLISH_HOOK, BUCKET_ON_DEMAND_MIGRATION_CONFIG};
use parking_lot::{Mutex, RwLock};
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
@@ -746,20 +745,8 @@ impl OnDemandMigrationSys {
self.write_back.read().clone()
}
/// Registers `publish` as the bucket-metadata publish hook. Returns
/// `false` when a hook was already registered.
pub fn register_config_hook(&'static self) -> bool {
BUCKET_CONFIG_PUBLISH_HOOK
.set(Box::new(move |bucket, config_file, stored| {
if config_file == BUCKET_ON_DEMAND_MIGRATION_CONFIG {
self.publish_stored(bucket, stored.map(|(bytes, _, incarnation)| (bytes, incarnation)));
}
}))
.is_ok()
}
/// Corrupt persisted bytes withdraw state synchronously, just like deletion.
fn publish_stored(&'static self, bucket: &str, stored: Option<(&[u8], uuid::Uuid)>) {
pub(crate) fn publish_stored(&'static self, bucket: &str, stored: Option<(&[u8], uuid::Uuid)>) {
let incarnation_id = stored.map(|(_, id)| id).unwrap_or_default();
match stored.map(|(bytes, _)| OnDemandMigrationConfig::from_json(bytes)).transpose() {
Ok(config) => self.publish_for_incarnation(bucket, incarnation_id, config.as_ref()),
+4 -55
View File
@@ -16,7 +16,6 @@ use super::{
module_switch::{resolve_notify_module_state, validate_notify_module_env, with_refreshed_notify_module_state_from},
refresh_persisted_module_switches_from, runtime_sources,
};
use crate::init::reconcile_persisted_bucket_notification_configurations;
use crate::storage_api::server::event::{
EventArgs as EcstoreEventArgs, read_existing_server_config_no_lock, register_event_dispatch_hook,
with_server_config_read_lock,
@@ -36,7 +35,6 @@ use tokio_util::sync::CancellationToken;
use tracing::{info, instrument, warn};
static NOTIFY_RUNTIME_RECONCILED: AtomicBool = AtomicBool::new(false);
static NOTIFY_BUCKET_RULES_RECONCILED: AtomicBool = AtomicBool::new(false);
static ECSTORE_EVENT_DISPATCH_HOOK: OnceLock<()> = OnceLock::new();
// Serialize local notification lifecycle publication with the periodic and
@@ -59,7 +57,6 @@ pub(crate) fn mark_event_notifier_reconciled() {
pub(crate) fn mark_event_notifier_unreconciled() {
NOTIFY_RUNTIME_RECONCILED.store(false, Ordering::Release);
NOTIFY_BUCKET_RULES_RECONCILED.store(false, Ordering::Release);
}
pub(crate) async fn with_notify_runtime_reconcile_lock<Fut, T>(operation: Fut) -> T
@@ -70,18 +67,6 @@ where
operation.await
}
fn are_bucket_notification_rules_reconciled() -> bool {
NOTIFY_BUCKET_RULES_RECONCILED.load(Ordering::Acquire)
}
fn mark_bucket_notification_rules_reconciled() {
NOTIFY_BUCKET_RULES_RECONCILED.store(true, Ordering::Release);
}
fn should_reconcile_bucket_notification_rules(runtime_changed: bool, notify_enabled: bool) -> bool {
notify_enabled && (runtime_changed || !are_bucket_notification_rules_reconciled())
}
pub fn refresh_notify_module_enabled() -> bool {
let enabled = resolve_notify_module_state().enabled;
crate::module_switches::set_notify_module_enabled(enabled);
@@ -111,6 +96,8 @@ fn convert_ecstore_event_args(args: EcstoreEventArgs) -> Option<NotifyEventArgs>
};
Some(NotifyEventArgs {
bucket_incarnation_id: args.bucket_incarnation_id,
bucket_removed_targets: None,
event_name,
bucket_name: args.bucket_name,
object: convert_ecstore_object_info(args.object),
@@ -216,24 +203,11 @@ async fn reconcile_event_notifier_from_store_unlocked(
.await
.map_err(|err| NotificationError::Initialization(format!("failed to refresh notify module switch: {err}")))??;
let runtime_changed = transition.is_some();
if let Some(transition) = transition {
transition.wait().await?;
}
ensure_event_notifier_converged(&system)?;
if should_reconcile_bucket_notification_rules(runtime_changed, is_notify_module_enabled()) {
let configured_bucket_count = reconcile_persisted_bucket_notification_configurations(store).await?;
mark_bucket_notification_rules_reconciled();
info!(
event = EVENT_NOTIFY_RUNTIME_RECONCILE,
component = "notify",
subsystem = "bucket_rules",
configured_bucket_count,
"Persisted bucket notification rules reconciled"
);
}
Ok(())
}
.await;
@@ -427,9 +401,8 @@ mod tests {
set_persisted_module_switches,
};
use super::{
convert_ecstore_object_info, init_event_notifier_with_store, mark_bucket_notification_rules_reconciled,
parse_host_and_port, run_persisted_event_notifier_reconciler, should_reconcile_bucket_notification_rules,
with_notify_runtime_reconcile_lock,
convert_ecstore_object_info, init_event_notifier_with_store, parse_host_and_port,
run_persisted_event_notifier_reconciler, with_notify_runtime_reconcile_lock,
};
use crate::server::is_event_notifier_reconciled;
use crate::storage_api::server::event::StorageObjectInfo;
@@ -594,30 +567,6 @@ mod tests {
assert_eq!(converted.transitioned_tier.as_deref(), Some("DEEP_ARCHIVE"));
}
#[test]
fn bucket_rule_reconcile_runs_once_and_after_runtime_change() {
super::mark_event_notifier_unreconciled();
assert!(
should_reconcile_bucket_notification_rules(false, true),
"enabled notify must restore bucket rules until the first successful replay"
);
mark_bucket_notification_rules_reconciled();
assert!(
!should_reconcile_bucket_notification_rules(false, true),
"steady-state reconcile must not rescan buckets every tick"
);
assert!(
should_reconcile_bucket_notification_rules(true, true),
"target runtime changes must replay persisted bucket rules"
);
assert!(
!should_reconcile_bucket_notification_rules(true, false),
"disabled notify must not load external target rules"
);
}
#[tokio::test(start_paused = true)]
async fn persisted_reconciler_converges_after_one_injected_tick() {
let persisted_generation = Arc::new(AtomicUsize::new(1));
+19 -9
View File
@@ -16,8 +16,9 @@ use crate::app::object::OnDemandMigrationWriteBack;
use crate::module_switches::{on_demand_migration_enabled_from_env, set_on_demand_migration_module_enabled};
use crate::on_demand_migration::OnDemandMigrationSys;
use crate::storage_api::startup::bucket_metadata::{
ECStore, Error as StorageError, Result as StorageResult, get_global_replication_pool, init_bucket_metadata_sys,
reconcile_bucket_resync_target_intents, try_migrate_bucket_metadata, try_migrate_iam_config,
BUCKET_CONFIG_PUBLISH_HOOK, BUCKET_NOTIFICATION_CONFIG, BUCKET_ON_DEMAND_MIGRATION_CONFIG, ECStore, Error as StorageError,
Result as StorageResult, get_global_replication_pool, init_bucket_metadata_sys, reconcile_bucket_resync_target_intents,
try_migrate_bucket_metadata, try_migrate_iam_config,
};
use std::{
future::Future,
@@ -55,7 +56,7 @@ const STARTUP_BACKGROUND_STATUS_SUCCEEDED: f64 = 1.0;
const STARTUP_BACKGROUND_STATUS_RUNNING: f64 = 2.0;
const STARTUP_BACKGROUND_STATUS_CANCELED: f64 = 3.0;
pub(crate) async fn init_embedded_bucket_metadata_runtime(store: Arc<ECStore>, ctx: &CancellationToken) -> IoResult<Vec<String>> {
pub(crate) async fn init_embedded_bucket_metadata_runtime(store: Arc<ECStore>, ctx: &CancellationToken) -> IoResult<()> {
let buckets_list = retry_metadata_startup(ctx, "list_buckets", || async {
store.list_buckets_for_metadata_startup().await.map_err(IoError::other)
})
@@ -67,12 +68,12 @@ pub(crate) async fn init_embedded_bucket_metadata_runtime(store: Arc<ECStore>, c
init_on_demand_migration_runtime();
init_bucket_metadata_sys(store.clone(), buckets.clone()).await;
retry_metadata_startup(ctx, "iam_migration", || try_migrate_iam_config(store.clone())).await?;
spawn_bucket_resync_startup_reconcile(buckets.clone(), ctx.clone(), false);
spawn_bucket_resync_startup_reconcile(buckets, ctx.clone(), false);
Ok(buckets)
Ok(())
}
pub(crate) async fn init_bucket_metadata_runtime(store: Arc<ECStore>, ctx: CancellationToken) -> IoResult<Vec<String>> {
pub(crate) async fn init_bucket_metadata_runtime(store: Arc<ECStore>, ctx: CancellationToken) -> IoResult<()> {
let buckets_list = retry_metadata_startup(&ctx, "list_buckets", || async {
store.list_buckets_for_metadata_startup().await.map_err(IoError::other)
})
@@ -85,9 +86,9 @@ pub(crate) async fn init_bucket_metadata_runtime(store: Arc<ECStore>, ctx: Cance
retry_metadata_startup(&ctx, "iam_migration", || try_migrate_iam_config(store.clone())).await?;
init_on_demand_migration_runtime();
init_bucket_metadata_sys(store, buckets.clone()).await;
spawn_bucket_resync_startup_reconcile(buckets.clone(), ctx, true);
spawn_bucket_resync_startup_reconcile(buckets, ctx, true);
Ok(buckets)
Ok(())
}
async fn retry_metadata_startup<T, Operation, OperationFuture>(
@@ -166,7 +167,16 @@ fn init_on_demand_migration_runtime() {
let sys = OnDemandMigrationSys::get();
sys.set_module_enabled(enabled);
sys.set_write_back(Arc::new(OnDemandMigrationWriteBack::new()));
let hook_registered = sys.register_config_hook();
let hook_registered = BUCKET_CONFIG_PUBLISH_HOOK
.set(Box::new(move |bucket, file, stored| {
let stored = stored.map(|(bytes, _, incarnation_id)| (bytes, incarnation_id));
match file {
BUCKET_ON_DEMAND_MIGRATION_CONFIG => sys.publish_stored(bucket, stored),
BUCKET_NOTIFICATION_CONFIG => crate::init::publish_bucket_notification_configuration(bucket, stored),
_ => {}
}
}))
.is_ok();
tracing::info!(
event = EVENT_ON_DEMAND_MIGRATION_RUNTIME_INITIALIZED,
component = LOG_COMPONENT_STARTUP_BUCKET_METADATA,
+2 -7
View File
@@ -12,7 +12,6 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::init::add_bucket_notification_configuration;
use crate::storage_api::startup::notification::{EndpointServerPools, Result as StorageResult, new_global_notification_sys};
use std::{
future::Future,
@@ -27,17 +26,13 @@ const LOG_SUBSYSTEM_STARTUP: &str = "startup";
const EVENT_EMBEDDED_OPTIONAL_SERVICE_SKIPPED: &str = "embedded_optional_service_skipped";
const EVENT_NOTIFICATION_SYSTEM_INITIALIZATION_FAILED: &str = "notification_system_initialization_failed";
pub(crate) async fn init_embedded_notification_runtime(endpoint_pools: EndpointServerPools, buckets: Vec<String>) {
add_bucket_notification_configuration(buckets).await;
pub(crate) async fn init_embedded_notification_runtime(endpoint_pools: EndpointServerPools) {
if let Err(err) = init_notification_system(endpoint_pools).await {
log_embedded_optional_service_skipped("notification", err);
}
}
pub(crate) async fn init_notification_runtime(endpoint_pools: EndpointServerPools, buckets: Vec<String>) -> Result<()> {
add_bucket_notification_configuration(buckets).await;
pub(crate) async fn init_notification_runtime(endpoint_pools: EndpointServerPools) -> Result<()> {
init_notification_system(endpoint_pools).await.map_err(|err| {
error!(
event = EVENT_NOTIFICATION_SYSTEM_INITIALIZATION_FAILED,
+4 -4
View File
@@ -68,11 +68,11 @@ pub(crate) async fn init_embedded_startup_runtime_services(
server_ctx: Arc<ServerContextSlot>,
) -> Result<EmbeddedStartupServiceRuntime> {
init_embedded_optional_service_runtime(config, store.clone()).await;
let buckets = init_embedded_bucket_metadata_runtime(store.clone(), &ctx).await?;
init_embedded_bucket_metadata_runtime(store.clone(), &ctx).await?;
let iam_bootstrap = init_embedded_iam_runtime(store, ctx.clone(), readiness, server_ctx.clone())
.await
.map_err(|err| std::io::Error::other(format!("IAM bootstrap setup: {err}")))?;
init_embedded_notification_runtime(endpoint_pools, buckets).await;
init_embedded_notification_runtime(endpoint_pools).await;
Ok(EmbeddedStartupServiceRuntime {
iam_bootstrap,
@@ -108,7 +108,7 @@ pub(crate) async fn init_startup_runtime_services(
init_buffer_profile_system(config);
init_deadlock_detector_runtime();
let buckets = init_bucket_metadata_runtime(store.clone(), ctx.clone()).await?;
init_bucket_metadata_runtime(store.clone(), ctx.clone()).await?;
let iam_bootstrap = init_iam_runtime(store.clone(), ctx.clone(), readiness, state_manager, server_ctx.clone()).await?;
// Audit initialization requires the AppContext (server config + object store)
@@ -120,7 +120,7 @@ pub(crate) async fn init_startup_runtime_services(
// bucket metadata itself, and its first pass runs off this path.
spawn_site_replication_reconcile_task(ctx.clone());
let oidc_recovery = init_auth_integrations(ctx.child_token()).await?;
init_notification_runtime(endpoint_pools, buckets).await?;
init_notification_runtime(endpoint_pools).await?;
let enable_scanner = init_background_service_runtime(store.clone()).await?;
init_observability_runtime(store.clone(), ctx.clone()).await;
let heartbeat = start_heartbeat_runtime(heartbeat_config.clone(), heartbeat_nodes, &ctx)?;
+19
View File
@@ -48,6 +48,7 @@ use tracing::{Instrument, info_span, warn};
/// The request fields used to build an audit entry, shared by both request edges.
pub(crate) struct AuditRequestView<'a> {
bucket_generation: Option<&'a super::authorization::BucketGenerationGuard>,
headers: &'a HeaderMap,
uri: &'a Uri,
pub(crate) req_info: Option<&'a ReqInfo>,
@@ -59,6 +60,7 @@ pub(crate) struct AuditRequestView<'a> {
impl<'a> AuditRequestView<'a> {
pub(crate) fn legacy<T>(req: &'a S3Request<T>) -> Self {
Self {
bucket_generation: req.extensions.get(),
headers: &req.headers,
uri: &req.uri,
req_info: req.extensions.get::<ReqInfo>(),
@@ -79,8 +81,10 @@ impl<'a> AuditRequestView<'a> {
client_info: Option<&'a ClientInfo>,
remote_addr: Option<SocketAddr>,
request_context: Option<&'a RequestContext>,
bucket_generation: Option<&'a super::authorization::BucketGenerationGuard>,
) -> Self {
Self {
bucket_generation,
headers,
uri,
req_info,
@@ -495,7 +499,12 @@ impl OperationHelper {
// initialize event builder
// object is a placeholder that must be set later using the `object()` method.
let event_builder = if notify_enabled {
let incarnation_id = req
.bucket_generation
.filter(|guard| guard.bucket == bucket)
.map(|guard| guard.incarnation_id);
let mut event_builder = EventArgsBuilder::new(event, bucket, convert_ecstore_object_info(event_object))
.bucket_incarnation_id(incarnation_id)
.host(get_request_host(req.headers))
.port(get_request_port(req.headers))
.user_agent(get_request_user_agent(req.headers))
@@ -545,6 +554,16 @@ impl OperationHelper {
}
}
pub(crate) fn with_bucket_removed_targets(mut self, targets: rustfs_notify::rules::TargetIdSet) -> Self {
if let Self::Enabled(state) = &mut self {
state.event_builder = state
.event_builder
.take()
.map(|builder| builder.bucket_removed_targets(targets));
}
self
}
/// Sets the ObjectInfo for event notification.
pub fn object(mut self, object_info: ObjectInfo) -> Self {
if let Self::Enabled(state) = &mut self
+3 -3
View File
@@ -69,9 +69,9 @@ pub(crate) use storage_api::{
ecfs_extend_consumer, ecstore_admin, ecstore_bucket, ecstore_capacity, ecstore_client, ecstore_cluster, ecstore_compression,
ecstore_config, ecstore_data_usage, ecstore_disk, ecstore_error, ecstore_event, ecstore_layout, ecstore_metrics,
ecstore_notification, ecstore_rebalance, ecstore_rio, ecstore_rpc, ecstore_set_disk, ecstore_storage, ecstore_tier,
encode_tags, find_local_disk_by_ref, get_bucket_cors_config, get_bucket_metadata, get_bucket_notification_config,
get_bucket_sse_config, get_bucket_website_config_for_store, get_local_server_property, get_lock_acquire_timeout,
head_prefix_consumer, helper_consumer, init_background_replication, init_bucket_metadata_sys, init_ecstore_config,
encode_tags, find_local_disk_by_ref, get_bucket_cors_config, get_bucket_metadata, get_bucket_sse_config,
get_bucket_website_config_for_store, get_local_server_property, get_lock_acquire_timeout, head_prefix_consumer,
helper_consumer, init_background_replication, init_bucket_metadata_sys, init_ecstore_config,
init_local_disks_with_instance_ctx, init_lock_clients, is_err_bucket_not_found, is_err_object_not_found,
is_err_version_not_found, is_valid_storage_class, options_consumer, prewarm_local_disk_id_map_with_instance_ctx, read_config,
record_replication_proxy, rpc_consumer, runtime_sources_consumer, s3_api_consumer, table_catalog_path_hash, to_s3s_etag,
-6
View File
@@ -1334,12 +1334,6 @@ pub(crate) async fn all_local_disk() -> Vec<DiskStore> {
ecstore_storage::all_local_disk().await
}
pub(crate) async fn get_bucket_notification_config(
bucket: &str,
) -> Result<Option<rustfs_gateway_types::persistence::PersistedNotificationConfiguration>> {
ecstore_bucket::metadata_sys::get_notification_config(bucket).await
}
pub(crate) async fn init_bucket_metadata_sys(api: Arc<ECStore>, buckets: Vec<String>) {
ecstore_bucket::metadata_sys::init_bucket_metadata_sys(api, buckets).await;
}
+7 -10
View File
@@ -432,12 +432,10 @@ pub(crate) mod startup {
}
pub(crate) mod bucket_metadata {
pub(crate) mod contract {
pub(crate) mod bucket {
pub(crate) use super::super::super::super::storage_contracts::{BucketOperations, BucketOptions};
}
}
pub(crate) use crate::storage::storage_api::ecstore_bucket::{
metadata::{BUCKET_NOTIFICATION_CONFIG, BUCKET_ON_DEMAND_MIGRATION_CONFIG},
metadata_sys::BUCKET_CONFIG_PUBLISH_HOOK,
};
pub(crate) use crate::storage::storage_api::{
ECStore, Error, Result, get_global_replication_pool, init_bucket_metadata_sys,
reconcile_bucket_resync_target_intents, try_migrate_bucket_metadata, try_migrate_iam_config,
@@ -466,9 +464,9 @@ pub(crate) mod startup {
}
pub(crate) use crate::storage::storage_api::ecstore_bucket::config_parse_mode::validate_bucket_config_parse_mode_env;
pub(crate) use crate::storage::storage_api::ecstore_bucket::utils::deserialize;
pub(crate) use crate::storage::storage_api::{
get_bucket_notification_config, process_lambda_configurations, process_queue_configurations,
process_topic_configurations,
process_lambda_configurations, process_queue_configurations, process_topic_configurations,
};
pub(crate) mod concurrency {
@@ -589,9 +587,8 @@ pub(crate) mod on_demand_migration {
pub(crate) use crate::storage::storage_api::contract::{
namespace::NamespaceLocking, object::HTTPPreconditions, object::ObjectOperations, range::HTTPRangeSpec,
};
pub(crate) use crate::storage::storage_api::ecstore_bucket::metadata::BUCKET_ON_DEMAND_MIGRATION_CONFIG;
pub(crate) use crate::storage::storage_api::ecstore_bucket::metadata_sys::{
BUCKET_CONFIG_PUBLISH_HOOK, get_on_demand_migration_config, get_on_demand_migration_config_in,
get_on_demand_migration_config, get_on_demand_migration_config_in,
};
pub(crate) use crate::storage::storage_api::ecstore_bucket::remote_s3_client;
pub(crate) use crate::storage::storage_api::ecstore_config::com::{read_config_with_metadata, save_config_with_opts};