mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-19 06:24:17 +00:00
Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 42a1417ed0 |
@@ -22,8 +22,6 @@ Identify the tested commit and any local changes. When testing a prebuilt binary
|
||||
List relevant checks not run and the remaining risk. Use the validation tier in AGENTS.md; do not run broader checks solely to fill this section. For documentation-only changes, list the applicable documentation checks. Use N/A only when verification is not applicable.
|
||||
-->
|
||||
|
||||
**auto-testing regression coverage** <!-- For each issue-backed bug fix, give the added or extended case ID or PR link, the existing case ID/path and assertion that catches the bug, or "Skipped" with a brief reason. Omit this line for other changes, including documentation-only fixes. -->
|
||||
|
||||
## Impact
|
||||
<!--
|
||||
Describe user-facing, compatibility, API, deployment, configuration, or documentation impact. Use N/A when there is no expected impact.
|
||||
|
||||
@@ -80,18 +80,6 @@ Do not commit one-shot plans, trackers, migration ledgers, benchmark snapshots,
|
||||
|
||||
Select checks from the final task-owned diff. Scoped `AGENTS.md` files may add a concrete path-specific check, but must not replace this tiering with a generic full-workspace gate.
|
||||
|
||||
### Issue Fixes and External Regression Coverage
|
||||
|
||||
After fixing a bug tracked by an issue and running the applicable local checks, ask the developer once per fixed issue whether to cover it in [rustfs/auto-testing](https://github.com/rustfs/auto-testing). Ask before opening or updating the fix PR, or before reporting completion if there is no PR. Reuse a choice already made for that fix. Investigation alone and documentation-only changes do not trigger this prompt.
|
||||
|
||||
When presenting the three choices, describe the failing scenario and the assertion that would catch it.
|
||||
|
||||
- **Add or extend a case.** Prefer extending an existing scenario when it can exercise the regression. Record the resulting case ID or auto-testing PR link.
|
||||
- **Already covered.** Cite the existing auto-testing case ID or path and the assertion that would catch this bug.
|
||||
- **Skip.** Record the developer's one-line reason and continue with the fix.
|
||||
|
||||
Record the choice in the PR's Verification section or in the completion report for work without a PR. All three choices retain the applicable RustFS verification requirements below.
|
||||
|
||||
### Documentation and Instructions
|
||||
|
||||
For prose, comments, agent instructions, and skill metadata that cannot affect runtime/build output:
|
||||
|
||||
@@ -1405,7 +1405,7 @@ impl BucketMetadata {
|
||||
/// | public access block | Fails closed in `get_public_access_block_config`: degrading grants the anonymous access the operator asked to block. |
|
||||
/// | quota | Fails closed in `get_quota_config`; the enforcement path in `quota::checker` already re-parses the raw JSON and refuses on error. |
|
||||
/// | lifecycle | `get_lifecycle_config` fails closed, so GetBucketLifecycle reports the fault instead of NoSuchLifecycleConfiguration. ILM, scanner and expiry-header consumers still degrade to "no rules": nothing is deleted or moved on the strength of an unreadable rule set, and the bucket keeps serving reads and writes. |
|
||||
/// | notification | `get_notification_config` fails closed, so GetBucketNotificationConfiguration reports the fault and startup leaves that one bucket's rules unchanged instead of clearing them; other buckets are unaffected. |
|
||||
/// | notification | `get_notification_config` fails closed, so GetBucketNotificationConfiguration reports the fault and startup skips that bucket's rule registration; metadata-bound server dispatch suppresses external delivery for the unreadable snapshot. Other buckets are unaffected. |
|
||||
/// | tagging, CORS, logging, website, accelerate, request payment | The getter fails closed so the matching GET API reports the fault instead of "not configured". Per-request consumers (CORS response headers) still degrade to "not configured", which is the restrictive direction. |
|
||||
/// | bucket ACL | Safe to degrade: it only shapes an optional response. |
|
||||
///
|
||||
|
||||
@@ -2612,6 +2612,10 @@ impl BucketMetadataSys {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn cached_metadata(&self) -> Vec<Arc<BucketMetadata>> {
|
||||
self.metadata_map.read().await.values().cloned().collect()
|
||||
}
|
||||
|
||||
pub async fn set(&self, bucket: String, bm: Arc<BucketMetadata>) {
|
||||
self.revoke_pending_bucket_creation(&bucket).await;
|
||||
bump_local_meta_mutation_gen();
|
||||
@@ -5760,6 +5764,64 @@ mod tests {
|
||||
assert_eq!(tags.tag_set.len(), WRITERS, "every concurrent rewrite must survive: {tags:?}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn notification_snapshots_follow_periodic_refresh_after_lost_peer_signals() {
|
||||
let (dirs, store) = isolated_store_over_temp_disks().await;
|
||||
let writer = BucketMetadataSys::new(store.clone());
|
||||
let peer = Arc::new(RwLock::new(BucketMetadataSys::new(store.clone())));
|
||||
let bucket = "notification-periodic-snapshots";
|
||||
for dir in &dirs {
|
||||
std::fs::create_dir_all(dir.path().join(bucket)).expect("create empty fixture bucket");
|
||||
}
|
||||
let mut original = BucketMetadata::new(bucket);
|
||||
original.notification_config_xml = concat!(
|
||||
"<NotificationConfiguration><QueueConfiguration><Event>s3:ObjectCreated:Put</Event>",
|
||||
"<Queue>arn:rustfs:sqs:us-east-1:inert:webhook</Queue></QueueConfiguration></NotificationConfiguration>"
|
||||
)
|
||||
.as_bytes()
|
||||
.to_vec();
|
||||
original.parse_all_configs().expect("parse inert subscription");
|
||||
writer
|
||||
.persist_new_and_set(original)
|
||||
.await
|
||||
.expect("persist initial subscription");
|
||||
peer.read().await.reload_from_store(bucket).await.expect("prime peer cache");
|
||||
let old = peer.read().await.get(bucket).await.expect("cached original snapshot");
|
||||
|
||||
let mut removed = (*old).clone();
|
||||
removed.notification_config_xml = b"<NotificationConfiguration/>".to_vec();
|
||||
removed.parse_all_configs().expect("parse removed subscription");
|
||||
writer
|
||||
.persist_new_and_set(removed)
|
||||
.await
|
||||
.expect("persist removal without peer callback");
|
||||
assert!(Arc::ptr_eq(&old, &peer.read().await.get(bucket).await.expect("peer remains stale")));
|
||||
|
||||
// Exercise the timer's actual one-pass entry point, not the peer RPC.
|
||||
refresh_buckets_metadata_once(peer.clone()).await;
|
||||
let snapshots = peer.read().await.cached_metadata().await;
|
||||
assert_eq!(snapshots.len(), 1);
|
||||
assert!(!Arc::ptr_eq(&old, &snapshots[0]));
|
||||
assert_eq!(old.bucket_incarnation_id, snapshots[0].bucket_incarnation_id);
|
||||
assert_eq!(snapshots[0].notification_config_xml, b"<NotificationConfiguration/>");
|
||||
assert!(
|
||||
snapshots[0]
|
||||
.notification_config
|
||||
.as_ref()
|
||||
.expect("parsed removal")
|
||||
.behavior()
|
||||
.queue_routes
|
||||
.is_empty()
|
||||
);
|
||||
|
||||
for dir in &dirs {
|
||||
std::fs::remove_dir(dir.path().join(bucket)).expect("remove empty fixture bucket without peer callback");
|
||||
}
|
||||
refresh_buckets_metadata_once(peer.clone()).await;
|
||||
assert!(peer.read().await.cached_metadata().await.is_empty());
|
||||
assert!(peer.read().await.get(bucket).await.is_err());
|
||||
}
|
||||
|
||||
/// Pins the peer reload-notification contract (`reload_from_store`, the
|
||||
/// LoadBucketMetadata RPC path): only metadata actually read from
|
||||
/// persisted storage enters the cache. A load miss errors out and leaves
|
||||
|
||||
@@ -775,6 +775,15 @@ impl ECStore {
|
||||
sys.read().await.get(bucket).await
|
||||
}
|
||||
|
||||
/// Snapshots this instance's cached bucket metadata without disk IO.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns an error when this instance's metadata system is not initialized.
|
||||
pub async fn cached_bucket_metadata(&self) -> Result<Vec<Arc<BucketMetadata>>> {
|
||||
let sys = metadata_sys::require_bucket_metadata_sys_in(&self.ctx)?;
|
||||
Ok(sys.read().await.cached_metadata().await)
|
||||
}
|
||||
|
||||
/// Resolve stored tags for authorization. Only confirmed absence returns an
|
||||
/// empty map; unavailable, fabricated or malformed metadata must not grant access.
|
||||
pub async fn get_bucket_tags_for_policy(&self, bucket: &str) -> Result<HashMap<String, String>> {
|
||||
|
||||
@@ -47,12 +47,18 @@ impl NotifyBucketConfigManager {
|
||||
}
|
||||
|
||||
pub async fn has_subscriber(&self, bucket: &str, event: &EventName) -> bool {
|
||||
if !self.subscriber_view.has_subscriber(bucket, event) {
|
||||
// The standalone index is only a fast negative check for name-only
|
||||
// rules. Metadata-bound rules can change without an index callback.
|
||||
if !self.rule_engine.uses_bucket_metadata_source() && !self.subscriber_view.has_subscriber(bucket, event) {
|
||||
return false;
|
||||
}
|
||||
self.rule_engine.has_subscriber(bucket, event).await
|
||||
}
|
||||
|
||||
pub(crate) fn bind_bucket_metadata_source(&self, store: &Arc<crate::NotifyStore>) {
|
||||
self.rule_engine.bind_bucket_metadata_source(store);
|
||||
}
|
||||
|
||||
/// Validates that the runtime can accept `cfg` for `bucket` without
|
||||
/// mutating any state.
|
||||
pub async fn validate_bucket_notification_config(
|
||||
|
||||
@@ -249,6 +249,16 @@ impl NotificationSystem {
|
||||
self.services.bucket_config_manager.has_subscriber(bucket, event).await
|
||||
}
|
||||
|
||||
/// Binds server dispatch to the store-owned metadata cache. Local writes,
|
||||
/// peer reloads, deletes and periodic metadata refreshes then select rules
|
||||
/// from the same snapshot; delayed name-only rule replacements are ignored
|
||||
/// and clears only evict compiled snapshots. Missing/unreadable metadata or
|
||||
/// a dropped store suppresses external dispatch without falling back to the
|
||||
/// standalone rules. Already queued or in-flight deliveries are not recalled.
|
||||
pub fn bind_bucket_metadata_source(&self, store: &Arc<crate::NotifyStore>) {
|
||||
self.services.bucket_config_manager.bind_bucket_metadata_source(store);
|
||||
}
|
||||
|
||||
/// Returns true when at least one in-process consumer is subscribed to live events.
|
||||
pub fn has_live_listeners(&self) -> bool {
|
||||
self.services.pipeline.has_live_listeners()
|
||||
|
||||
@@ -12,12 +12,14 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::rules::xml_config::{FilterRule, S3KeyFilter};
|
||||
use crate::rules::{RulesMap, TargetIdSet};
|
||||
use crate::storage_api::{BucketMetadata, NotifyStore};
|
||||
use percent_encoding::percent_decode_str;
|
||||
use rustfs_s3_types::EventName;
|
||||
use rustfs_targets::arn::TargetID;
|
||||
use rustfs_targets::arn::{ARN, TargetID};
|
||||
use starshard::{AsyncShardedHashMap, DEFAULT_SHARDS, SnapshotMode};
|
||||
use std::sync::Arc;
|
||||
use std::sync::{Arc, RwLock, Weak};
|
||||
use tracing::info;
|
||||
|
||||
const LOG_COMPONENT_NOTIFY: &str = "notify";
|
||||
@@ -35,20 +37,82 @@ fn decoded_object_key_for_matching(object_key: &str) -> Option<String> {
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct NotifyRuleEngine {
|
||||
bucket_rules_map: Arc<AsyncShardedHashMap<String, RulesMap, rustc_hash::FxBuildHasher>>,
|
||||
bucket_rules_map: Arc<AsyncShardedHashMap<String, CachedBucketRules, rustc_hash::FxBuildHasher>>,
|
||||
// Never hold this short synchronous guard across an await. Metadata cache
|
||||
// reads precede the compiled-rules cache, so there is no reverse lock order.
|
||||
metadata_source: Arc<RwLock<Option<Weak<NotifyStore>>>>,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct CachedBucketRules {
|
||||
metadata: Option<Weak<BucketMetadata>>,
|
||||
// Cache rejected snapshots too, so corrupt persisted routes are not
|
||||
// repeatedly compiled on the event hot path.
|
||||
rules: Option<RulesMap>,
|
||||
}
|
||||
|
||||
fn rules_from_metadata(metadata: &BucketMetadata) -> Option<RulesMap> {
|
||||
let mut rules = RulesMap::new();
|
||||
let Some(config) = metadata.notification_config.as_ref() else {
|
||||
// Unreadable metadata is not authority to keep an old subscription.
|
||||
return metadata.notification_config_xml.is_empty().then_some(rules);
|
||||
};
|
||||
let config = config.behavior();
|
||||
for route in config
|
||||
.queue_routes
|
||||
.into_iter()
|
||||
.chain(config.topic_routes)
|
||||
.chain(config.lambda_routes)
|
||||
{
|
||||
let target = ARN::parse(&route.destination).ok()?.target_id;
|
||||
let events: Vec<_> = route.events.iter().filter_map(|event| EventName::parse(event).ok()).collect();
|
||||
let mut filter = S3KeyFilter::default();
|
||||
for (name, value) in route.filter_rules.into_iter().flatten() {
|
||||
filter.filter_rule_list.push(FilterRule {
|
||||
name: name?,
|
||||
value: value?,
|
||||
});
|
||||
}
|
||||
filter.validate().ok()?;
|
||||
rules.add_rule_config(&events, filter.pattern(), target);
|
||||
}
|
||||
Some(rules)
|
||||
}
|
||||
|
||||
impl NotifyRuleEngine {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
bucket_rules_map: Arc::new(AsyncShardedHashMap::with_snapshot_mode(DEFAULT_SHARDS, SnapshotMode::Cached)),
|
||||
metadata_source: Arc::new(RwLock::new(None)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Uses this store's immutable metadata snapshots as rule authority. Once
|
||||
/// bound, missing metadata or a dropped store fails closed, never falling
|
||||
/// back to the standalone name-only table. The weak handle avoids cycles.
|
||||
pub fn bind_bucket_metadata_source(&self, store: &Arc<NotifyStore>) {
|
||||
*self.metadata_source.write().unwrap_or_else(|err| err.into_inner()) = Some(Arc::downgrade(store));
|
||||
}
|
||||
|
||||
fn metadata_source(&self) -> Option<Weak<NotifyStore>> {
|
||||
self.metadata_source.read().unwrap_or_else(|err| err.into_inner()).clone()
|
||||
}
|
||||
|
||||
pub(crate) fn uses_bucket_metadata_source(&self) -> bool {
|
||||
self.metadata_source().is_some()
|
||||
}
|
||||
|
||||
pub async fn is_target_bound_to_any_bucket(&self, target_id: &TargetID) -> bool {
|
||||
if let Some(source) = self.metadata_source() {
|
||||
let Some(store) = source.upgrade() else { return true };
|
||||
let Ok(metadata) = store.cached_bucket_metadata().await else { return true };
|
||||
return metadata
|
||||
.iter()
|
||||
.any(|metadata| rules_from_metadata(metadata).is_none_or(|rules| rules.contains_target_id(target_id)));
|
||||
}
|
||||
let items = self.bucket_rules_map.iter().await;
|
||||
for (_bucket, rules_map) in items {
|
||||
if rules_map.contains_target_id(target_id) {
|
||||
if rules_map.rules.is_some_and(|rules| rules.contains_target_id(target_id)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -56,11 +120,24 @@ impl NotifyRuleEngine {
|
||||
}
|
||||
|
||||
pub async fn set_bucket_rules(&self, bucket: &str, rules_map: RulesMap) {
|
||||
if self.uses_bucket_metadata_source() {
|
||||
// Delayed PUT/startup callbacks cannot override a newer revision or
|
||||
// a same-name replacement. Only published metadata is authoritative.
|
||||
return;
|
||||
}
|
||||
let event_count = rules_map.iter_events().count();
|
||||
if rules_map.is_empty() {
|
||||
self.bucket_rules_map.remove(&bucket.to_string()).await;
|
||||
} else {
|
||||
self.bucket_rules_map.insert(bucket.to_string(), rules_map).await;
|
||||
self.bucket_rules_map
|
||||
.insert(
|
||||
bucket.to_string(),
|
||||
CachedBucketRules {
|
||||
metadata: None,
|
||||
rules: Some(rules_map),
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
info!(
|
||||
event = EVENT_NOTIFY_RULES_UPDATED,
|
||||
@@ -74,7 +151,47 @@ impl NotifyRuleEngine {
|
||||
}
|
||||
|
||||
pub async fn get_bucket_rules(&self, bucket: &str) -> Option<RulesMap> {
|
||||
self.bucket_rules_map.get(&bucket.to_string()).await
|
||||
if let Some(source) = self.metadata_source() {
|
||||
let metadata = match source.upgrade() {
|
||||
Some(store) => store.get_bucket_metadata(bucket).await.ok(),
|
||||
None => None,
|
||||
};
|
||||
return self.rules_for_metadata(bucket, metadata).await;
|
||||
}
|
||||
self.bucket_rules_map
|
||||
.get(&bucket.to_string())
|
||||
.await
|
||||
.and_then(|cached| cached.rules)
|
||||
}
|
||||
|
||||
async fn rules_for_metadata(&self, bucket: &str, metadata: Option<Arc<BucketMetadata>>) -> Option<RulesMap> {
|
||||
let key = bucket.to_owned();
|
||||
let Some(metadata) = metadata else {
|
||||
self.bucket_rules_map.remove(&key).await;
|
||||
return None;
|
||||
};
|
||||
if let Some(cached) = self.bucket_rules_map.get(&key).await
|
||||
&& cached
|
||||
.metadata
|
||||
.as_ref()
|
||||
.is_some_and(|snapshot| Weak::ptr_eq(snapshot, &Arc::downgrade(&metadata)))
|
||||
{
|
||||
return cached.rules;
|
||||
}
|
||||
// Snapshot identity covers incarnation, revision AND bytes, without
|
||||
// trusting wall-clock ordering. A late compilation may evict a newer
|
||||
// cache entry, but cannot be reused by a reader of a different snapshot.
|
||||
let rules = rules_from_metadata(&metadata);
|
||||
self.bucket_rules_map
|
||||
.insert(
|
||||
key,
|
||||
CachedBucketRules {
|
||||
metadata: Some(Arc::downgrade(&metadata)),
|
||||
rules: rules.clone(),
|
||||
},
|
||||
)
|
||||
.await;
|
||||
rules
|
||||
}
|
||||
|
||||
pub async fn clear_bucket_rules(&self, bucket: &str) {
|
||||
@@ -119,8 +236,179 @@ impl Default for NotifyRuleEngine {
|
||||
mod tests {
|
||||
use super::NotifyRuleEngine;
|
||||
use crate::rules::RulesMap;
|
||||
use crate::storage_api::BucketMetadata;
|
||||
use rustfs_s3_types::EventName;
|
||||
use rustfs_targets::arn::TargetID;
|
||||
use std::sync::{Arc, Weak};
|
||||
|
||||
fn metadata(target: &str) -> Arc<BucketMetadata> {
|
||||
let mut metadata = BucketMetadata::new("bucket");
|
||||
metadata.notification_config_xml = format!(
|
||||
"<NotificationConfiguration><QueueConfiguration><Event>s3:ObjectCreated:Put</Event>\
|
||||
<Queue>arn:rustfs:sqs:us-east-1:{target}:webhook</Queue></QueueConfiguration></NotificationConfiguration>"
|
||||
)
|
||||
.into_bytes();
|
||||
metadata.notification_config = Some(
|
||||
rustfs_ecstore::api::bucket::utils::deserialize(&metadata.notification_config_xml)
|
||||
.expect("parse inert notification fixture"),
|
||||
);
|
||||
Arc::new(metadata)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn notification_snapshot_cache_never_reuses_replaced_incarnation_or_revision() {
|
||||
let engine = NotifyRuleEngine::new();
|
||||
let old = metadata("old");
|
||||
let mut replacement = (*metadata("new")).clone();
|
||||
// Identical timestamps must not make different configurations equal.
|
||||
replacement.notification_config_updated_at = old.notification_config_updated_at;
|
||||
let replacement = Arc::new(replacement);
|
||||
assert_ne!(old.bucket_incarnation_id, replacement.bucket_incarnation_id);
|
||||
let old_target = TargetID::new("old".into(), "webhook".into());
|
||||
let new_target = TargetID::new("new".into(), "webhook".into());
|
||||
|
||||
for (snapshot, wanted, unwanted) in [
|
||||
(old.clone(), &old_target, &new_target),
|
||||
(replacement.clone(), &new_target, &old_target),
|
||||
// Simulate a late compilation of a captured, older snapshot.
|
||||
(old.clone(), &old_target, &new_target),
|
||||
(replacement.clone(), &new_target, &old_target),
|
||||
] {
|
||||
let rules = engine
|
||||
.rules_for_metadata("bucket", Some(snapshot))
|
||||
.await
|
||||
.expect("readable snapshot");
|
||||
let targets = rules.match_rules(EventName::ObjectCreatedPut, "object");
|
||||
assert!(targets.contains(wanted));
|
||||
assert!(!targets.contains(unwanted));
|
||||
}
|
||||
|
||||
let mut next_revision = (*old).clone();
|
||||
next_revision.notification_config = replacement.notification_config.clone();
|
||||
next_revision.notification_config_xml = replacement.notification_config_xml.clone();
|
||||
let rules = engine
|
||||
.rules_for_metadata("bucket", Some(Arc::new(next_revision)))
|
||||
.await
|
||||
.expect("same-incarnation revision");
|
||||
assert!(!rules.contains_target_id(&old_target));
|
||||
assert!(rules.contains_target_id(&new_target));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn notification_snapshot_absence_and_corruption_never_fall_back_to_old_rules() {
|
||||
let engine = NotifyRuleEngine::new();
|
||||
let old = metadata("old");
|
||||
assert!(engine.rules_for_metadata("bucket", Some(old.clone())).await.is_some());
|
||||
|
||||
let mut unreadable = (*old).clone();
|
||||
unreadable.notification_config = None;
|
||||
unreadable.notification_config_xml = b"<broken>".to_vec();
|
||||
assert!(
|
||||
engine
|
||||
.rules_for_metadata("bucket", Some(Arc::new(unreadable)))
|
||||
.await
|
||||
.is_none()
|
||||
);
|
||||
assert!(
|
||||
engine
|
||||
.bucket_rules_map
|
||||
.get(&"bucket".to_owned())
|
||||
.await
|
||||
.expect("cache rejected snapshot")
|
||||
.rules
|
||||
.is_none()
|
||||
);
|
||||
assert!(engine.rules_for_metadata("bucket", None).await.is_none());
|
||||
assert!(engine.bucket_rules_map.get(&"bucket".to_owned()).await.is_none());
|
||||
|
||||
// A recreated bucket without notifications has no inherited targets.
|
||||
let replacement = Arc::new(BucketMetadata::new("bucket"));
|
||||
assert!(
|
||||
engine
|
||||
.rules_for_metadata("bucket", Some(replacement))
|
||||
.await
|
||||
.expect("empty snapshot")
|
||||
.is_empty()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn notification_snapshot_binding_never_falls_back_after_store_loss() {
|
||||
let engine = NotifyRuleEngine::new();
|
||||
let target = TargetID::new("old".into(), "webhook".into());
|
||||
let mut rules = RulesMap::new();
|
||||
rules.add_rule_config(&[EventName::ObjectCreatedPut], "*".into(), target.clone());
|
||||
engine.set_bucket_rules("bucket", rules.clone()).await;
|
||||
assert!(engine.has_subscriber("bucket", &EventName::ObjectCreatedPut).await);
|
||||
|
||||
*engine.metadata_source.write().expect("metadata source lock") = Some(Weak::new());
|
||||
// A delayed validated PUT must not revive the standalone table, even
|
||||
// when its owning store is gone. Target removal also fails closed.
|
||||
engine.set_bucket_rules("bucket", rules).await;
|
||||
assert!(!engine.has_subscriber("bucket", &EventName::ObjectCreatedPut).await);
|
||||
assert!(
|
||||
engine
|
||||
.match_targets("bucket", EventName::ObjectCreatedPut, "object")
|
||||
.await
|
||||
.is_empty()
|
||||
);
|
||||
assert!(engine.is_target_bound_to_any_bucket(&target).await);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn notification_snapshot_compilation_preserves_destinations_events_and_filters() {
|
||||
let mut metadata = (*metadata("queue")).clone();
|
||||
metadata.notification_config_xml = concat!(
|
||||
"<NotificationConfiguration>",
|
||||
"<QueueConfiguration><Event>s3:ObjectCreated:*</Event><Event>s3:Future:Unrecognized</Event>",
|
||||
"<Queue>arn:rustfs:sqs:us-east-1:queue:webhook</Queue>",
|
||||
"<Filter><S3Key><FilterRule><Name>Prefix</Name><Value>logs/</Value></FilterRule>",
|
||||
"<FilterRule><Name>Suffix</Name><Value>.csv</Value></FilterRule></S3Key></Filter></QueueConfiguration>",
|
||||
"<TopicConfiguration><Event>s3:ObjectRemoved:*</Event><Topic>arn:rustfs:sqs:us-east-1:topic:webhook</Topic></TopicConfiguration>",
|
||||
"<CloudFunctionConfiguration><Event>s3:ObjectCreated:Put</Event>",
|
||||
"<CloudFunction>arn:rustfs:sqs:us-east-1:lambda:webhook</CloudFunction></CloudFunctionConfiguration>",
|
||||
"</NotificationConfiguration>"
|
||||
).as_bytes().to_vec();
|
||||
metadata.notification_config =
|
||||
Some(rustfs_ecstore::api::bucket::utils::deserialize(&metadata.notification_config_xml).expect("parse destinations"));
|
||||
let rules = super::rules_from_metadata(&metadata).expect("compile destinations");
|
||||
assert_eq!(rules.match_rules(EventName::ObjectCreatedPut, "logs/data.csv").len(), 2);
|
||||
assert_eq!(rules.match_rules(EventName::ObjectCreatedPut, "other/data.csv").len(), 1);
|
||||
assert_eq!(rules.match_rules(EventName::ObjectCreatedPut, "logs/data.txt").len(), 1);
|
||||
let removed = rules.match_rules(EventName::ObjectRemovedDelete, "object");
|
||||
assert_eq!(removed.len(), 1);
|
||||
assert!(removed.contains(&TargetID::new("topic".into(), "webhook".into())));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn notification_snapshot_compilation_rejects_invalid_persisted_routes() {
|
||||
for (destination, filter) in [
|
||||
("not-an-arn", ""),
|
||||
(
|
||||
"arn:rustfs:sqs:us-east-1:inert:webhook",
|
||||
"<Filter><S3Key><FilterRule><Value>logs/</Value></FilterRule></S3Key></Filter>",
|
||||
),
|
||||
(
|
||||
"arn:rustfs:sqs:us-east-1:inert:webhook",
|
||||
concat!(
|
||||
"<Filter><S3Key><FilterRule><Name>prefix</Name><Value>logs/</Value></FilterRule>",
|
||||
"<FilterRule><Name>Prefix</Name><Value>private/</Value></FilterRule></S3Key></Filter>"
|
||||
),
|
||||
),
|
||||
] {
|
||||
let mut metadata = BucketMetadata::new("bucket");
|
||||
metadata.notification_config_xml = format!(
|
||||
"<NotificationConfiguration><QueueConfiguration><Event>s3:ObjectCreated:Put</Event>\
|
||||
<Queue>{destination}</Queue>{filter}</QueueConfiguration></NotificationConfiguration>"
|
||||
)
|
||||
.into_bytes();
|
||||
metadata.notification_config = Some(
|
||||
rustfs_ecstore::api::bucket::utils::deserialize(&metadata.notification_config_xml)
|
||||
.expect("parse invalid but well-formed persisted route"),
|
||||
);
|
||||
assert!(super::rules_from_metadata(&metadata).is_none(), "invalid route must not dispatch");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rule_engine_tracks_bucket_rule_lifecycle() {
|
||||
|
||||
@@ -14,6 +14,8 @@
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
pub(crate) use rustfs_ecstore::api::bucket::metadata::BucketMetadata;
|
||||
|
||||
use rustfs_ecstore::api::config::com::{
|
||||
read_existing_server_config_no_lock as read_existing_notify_config_from_backend_no_lock,
|
||||
read_server_config_snapshot as read_notify_server_config_snapshot_from_backend,
|
||||
|
||||
@@ -457,3 +457,130 @@ fn legacy_put_notification_does_not_write_after_the_bucket_is_recreated() {
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn notification_dispatch_follows_peer_reload_delete_and_recreation() {
|
||||
run_large_stack_test("notification-metadata-authority", || async {
|
||||
let ambient = shared_gating_ambient().await;
|
||||
let store = ambient.object_store();
|
||||
let bucket = format!("notify-authority-{}", uuid::Uuid::new_v4().simple());
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
let xml = |event: &str| {
|
||||
format!(
|
||||
"<NotificationConfiguration><QueueConfiguration><Event>{event}</Event>\
|
||||
<Queue>arn:rustfs:sqs:us-east-1:inert:webhook</Queue></QueueConfiguration></NotificationConfiguration>"
|
||||
)
|
||||
.into_bytes()
|
||||
};
|
||||
store
|
||||
.update_bucket_metadata_config(&bucket, BUCKET_NOTIFICATION_CONFIG, xml("s3:ObjectCreated:Put"))
|
||||
.await
|
||||
.expect("persist initial subscription");
|
||||
let old = store.get_bucket_metadata(&bucket).await.expect("initial cached metadata");
|
||||
let system = rustfs_notify::NotificationSystem::new(Default::default());
|
||||
system.bind_bucket_metadata_source(&store);
|
||||
let engine = rustfs_notify::NotifyRuleEngine::new();
|
||||
engine.bind_bucket_metadata_source(&store);
|
||||
let inert_target = TargetID::new("inert".into(), "webhook".into());
|
||||
assert!(system.has_subscriber(&bucket, &EventName::ObjectCreatedPut).await);
|
||||
assert!(engine.is_target_bound_to_any_bucket(&inert_target).await);
|
||||
|
||||
store
|
||||
.update_bucket_metadata_config(&bucket, BUCKET_NOTIFICATION_CONFIG, EMPTY_XML.to_vec())
|
||||
.await
|
||||
.expect("remove subscription durably");
|
||||
// Stand in for a peer whose cache has not received the reload yet.
|
||||
crate::storage::storage_api::set_bucket_metadata(bucket.clone(), (*old).clone())
|
||||
.await
|
||||
.expect("install stale peer cache");
|
||||
assert!(system.has_subscriber(&bucket, &EventName::ObjectCreatedPut).await);
|
||||
crate::storage::storage_api::reload_bucket_metadata(store.clone(), &bucket)
|
||||
.await
|
||||
.expect("peer reload");
|
||||
assert!(!system.has_subscriber(&bucket, &EventName::ObjectCreatedPut).await);
|
||||
assert!(
|
||||
engine
|
||||
.match_targets(&bucket, EventName::ObjectCreatedPut, "object")
|
||||
.await
|
||||
.is_empty()
|
||||
);
|
||||
assert!(!engine.is_target_bound_to_any_bucket(&inert_target).await);
|
||||
|
||||
store
|
||||
.update_bucket_metadata_config(&bucket, BUCKET_NOTIFICATION_CONFIG, xml("s3:ObjectCreated:Put"))
|
||||
.await
|
||||
.expect("restore initial subscription");
|
||||
assert!(system.has_subscriber(&bucket, &EventName::ObjectCreatedPut).await);
|
||||
store
|
||||
.delete_bucket(&bucket, &DeleteBucketOptions::default())
|
||||
.await
|
||||
.expect("delete bucket");
|
||||
assert!(!system.has_subscriber(&bucket, &EventName::ObjectCreatedPut).await);
|
||||
// A peer may still have the deleted incarnation cached until its delete
|
||||
// callback (or the periodic metadata refresh) establishes absence.
|
||||
crate::storage::storage_api::set_bucket_metadata(bucket.clone(), (*old).clone())
|
||||
.await
|
||||
.expect("install deleted peer snapshot");
|
||||
assert!(system.has_subscriber(&bucket, &EventName::ObjectCreatedPut).await);
|
||||
assert!(
|
||||
crate::storage::storage_api::remove_bucket_metadata_if_absent(store.clone(), &bucket)
|
||||
.await
|
||||
.expect("peer deletion")
|
||||
);
|
||||
assert!(!system.has_subscriber(&bucket, &EventName::ObjectCreatedPut).await);
|
||||
assert!(
|
||||
engine
|
||||
.match_targets(&bucket, EventName::ObjectCreatedPut, "object")
|
||||
.await
|
||||
.is_empty()
|
||||
);
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("recreate bucket");
|
||||
assert_ne!(
|
||||
old.bucket_incarnation_id,
|
||||
store.bucket_incarnation_id(&bucket).await.expect("replacement identity")
|
||||
);
|
||||
assert!(!system.has_subscriber(&bucket, &EventName::ObjectCreatedPut).await);
|
||||
store
|
||||
.update_bucket_metadata_config(&bucket, BUCKET_NOTIFICATION_CONFIG, xml("s3:ObjectRemoved:Delete"))
|
||||
.await
|
||||
.expect("replacement subscription");
|
||||
assert!(system.has_subscriber(&bucket, &EventName::ObjectRemovedDelete).await);
|
||||
assert!(engine.has_subscriber(&bucket, &EventName::ObjectRemovedDelete).await);
|
||||
|
||||
assert!(
|
||||
!crate::storage::storage_api::remove_bucket_metadata_if_absent(store.clone(), &bucket)
|
||||
.await
|
||||
.expect("delayed peer deletion")
|
||||
);
|
||||
let mut stale = rustfs_notify::BucketNotificationConfig::new("us-east-1");
|
||||
stale.add_rule(&[EventName::ObjectCreatedPut], "*".into(), TargetID::new("old".into(), "webhook".into()));
|
||||
// No runtime targets are activated: all assertions inspect inert rules.
|
||||
system.remove_bucket_notification_config(&bucket).await;
|
||||
// Applying already-validated rules models a delayed PUT completion.
|
||||
engine.set_bucket_rules(&bucket, stale.get_rules_map().clone()).await;
|
||||
assert!(engine.has_subscriber(&bucket, &EventName::ObjectRemovedDelete).await);
|
||||
assert!(!engine.has_subscriber(&bucket, &EventName::ObjectCreatedPut).await);
|
||||
assert!(
|
||||
engine
|
||||
.match_targets(&bucket, EventName::ObjectCreatedPut, "object")
|
||||
.await
|
||||
.is_empty()
|
||||
);
|
||||
assert_eq!(
|
||||
engine
|
||||
.match_targets(&bucket, EventName::ObjectRemovedDelete, "object")
|
||||
.await
|
||||
.into_iter()
|
||||
.collect::<Vec<_>>(),
|
||||
vec![inert_target]
|
||||
);
|
||||
assert!(system.has_subscriber(&bucket, &EventName::ObjectRemovedDelete).await);
|
||||
assert!(!system.has_subscriber(&bucket, &EventName::ObjectCreatedPut).await);
|
||||
});
|
||||
}
|
||||
|
||||
+5
-4
@@ -180,7 +180,7 @@ enum BucketNotificationLookup {
|
||||
Configured(rustfs_gateway_types::persistence::PersistedNotificationConfiguration),
|
||||
Missing,
|
||||
/// Stored bytes cannot be parsed. Deterministic, so a retry cannot help,
|
||||
/// and reading it as missing would clear the bucket's rules.
|
||||
/// and registration must not treat it as a readable empty configuration.
|
||||
Unreadable,
|
||||
}
|
||||
|
||||
@@ -199,7 +199,7 @@ fn classify_bucket_notification_lookup(
|
||||
subsystem = LOG_SUBSYSTEM_NOTIFICATION,
|
||||
bucket = %bucket,
|
||||
error = %err,
|
||||
"Bucket notification configuration is unreadable; leaving its rules unchanged"
|
||||
"Bucket notification configuration is unreadable; skipping rule registration"
|
||||
);
|
||||
Ok(BucketNotificationLookup::Unreadable)
|
||||
}
|
||||
@@ -210,8 +210,9 @@ fn classify_bucket_notification_lookup(
|
||||
}
|
||||
|
||||
/// Apply one bucket's persisted notification rules. An unreadable config is
|
||||
/// isolated to its bucket: it neither aborts setup for the other buckets nor
|
||||
/// clears the bucket's rules as if it had none.
|
||||
/// isolated to its bucket and does not abort setup for the other buckets.
|
||||
/// Metadata-bound server dispatch independently suppresses external delivery
|
||||
/// for unreadable snapshots rather than retaining an old subscription.
|
||||
async fn apply_bucket_notification_configuration(bucket: &str, region: &str) -> Result<bool, NotificationError> {
|
||||
match classify_bucket_notification_lookup(bucket, get_bucket_notification_config(bucket).await)? {
|
||||
BucketNotificationLookup::Unreadable => Ok(false),
|
||||
|
||||
@@ -79,6 +79,9 @@ fn mark_bucket_notification_rules_reconciled() {
|
||||
}
|
||||
|
||||
fn should_reconcile_bucket_notification_rules(runtime_changed: bool, notify_enabled: bool) -> bool {
|
||||
// Dispatch follows store-owned metadata snapshots, including the periodic
|
||||
// metadata refresh that repairs lost peer signals. Startup validation need
|
||||
// not rescan all buckets on every five-second target-runtime tick.
|
||||
notify_enabled && (runtime_changed || !are_bucket_notification_rules_reconciled())
|
||||
}
|
||||
|
||||
@@ -186,6 +189,7 @@ async fn reconcile_event_notifier_from_store_unlocked(
|
||||
let result = async {
|
||||
validate_notify_module_env().map_err(NotificationError::Initialization)?;
|
||||
let system = ensure_live_events_initialized();
|
||||
system.bind_bucket_metadata_source(&store);
|
||||
let transition_system = system.clone();
|
||||
let transition_store = store.clone();
|
||||
let transition = with_refreshed_notify_module_state_from(store.clone(), move |resolution| async move {
|
||||
@@ -353,6 +357,8 @@ where
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
system.bind_bucket_metadata_source(&store);
|
||||
|
||||
refresh_persisted_module_switches_from(store.clone())
|
||||
.await
|
||||
.map_err(|err| NotificationError::Initialization(format!("failed to refresh notify module switch: {err}")))?;
|
||||
|
||||
Reference in New Issue
Block a user