Files
rustfs/crates/notify/src/rules/xml_config.rs
T
houseme c9dba2c6c2 fix(notify): close core notify correctness and safety gaps (#4502)
Land the remaining notify-crate audit fixes.

backlog#979(b): remove_target now enforces the same bucket-binding guard as
remove_target_config, refusing to delete a target still referenced by a bucket
rule so notification rules are not left orphaned.

backlog#984:
- event.rs: an unversioned object omits versionId entirely instead of
  serializing versionId:"" (empty object/request versions treated as "no
  version").
- notifier.rs: RUSTFS_NOTIFY_SEND_CONCURRENCY=0 coerces back to the default
  instead of building a zero-permit semaphore that deadlocks every dispatch;
  init_bucket_targets_shared closes the replaced targets instead of dropping
  them without close() (connection leak).
- subscriber_index.rs: store_snapshot uses an atomic compute_if_absent upsert,
  removing the get-then-insert TOCTOU that could clobber a concurrent
  first-writer's snapshot cell.
- pipeline.rs: send_event assigns the history sequence and broadcasts to live
  subscribers under one critical section so broadcast order matches recorded
  sequence order.
- xml_config.rs: filter value length is bounded by character count, not byte
  length, so valid multi-byte keys are no longer wrongly rejected.
- global.rs: a losing initialize() race shuts the just-initialized system down
  instead of leaking its targets/replay workers.

backlog#970 (notify part): reload_config stops the running replay workers
before activating the new ones, so old and new workers do not concurrently
drain the same persisted stores. The full signal+join shutdown lives in the
targets crate under the same issue.

Tests: added regression coverage for each fix.
cargo build -p rustfs-notify, cargo test -p rustfs-notify --lib (98 passed),
cargo clippy -p rustfs-notify --all-targets (clean).

Relates to rustfs/backlog#979
Relates to rustfs/backlog#984
Relates to rustfs/backlog#970

Co-authored-by: heihutu <heihutu@gmail.com>
2026-07-08 16:20:02 +00:00

422 lines
16 KiB
Rust

// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::rules::pattern;
use hashbrown::HashSet;
use rustfs_s3_types::EventName;
use rustfs_targets::arn::{ARN, ArnError, TargetIDError};
use serde::{Deserialize, Serialize};
use std::io::Read;
use thiserror::Error;
#[derive(Debug, Error)]
pub enum ParseConfigError {
#[error("XML parsing error:{0}")]
XmlError(#[from] quick_xml::errors::serialize::DeError),
#[error("Invalid filter value:{0}")]
InvalidFilterValue(String),
#[error("Invalid filter name: {0}, only 'prefix' or 'suffix' is allowed")]
InvalidFilterName(String),
#[error("There can only be one 'prefix' in the filter rule")]
DuplicatePrefixFilter,
#[error("There can only be one 'suffix' in the filter rule")]
DuplicateSuffixFilter,
#[error("Missing event name")]
MissingEventName,
#[error("Duplicate event name:{0}")]
DuplicateEventName(String), // EventName is usually an enum, and here String is used to represent its text
#[error("Repeated queue configuration: ID={0:?}, ARN={1}")]
DuplicateQueueConfiguration(Option<String>, String),
#[error("Unsupported configuration types (e.g. Lambda, Topic)")]
UnsupportedConfiguration,
#[error("ARN not found:{0}")]
ArnNotFound(String),
#[error("Unknown area:{0}")]
UnknownRegion(String),
#[error("ARN parsing error:{0}")]
ArnParseError(#[from] ArnError),
#[error("TargetID parsing error:{0}")]
TargetIDParseError(#[from] TargetIDError),
#[error("IO Error:{0}")]
IoError(#[from] std::io::Error),
#[error("Region mismatch: Configure region {config_region}, current region {current_region}")]
RegionMismatch { config_region: String, current_region: String },
#[error("ARN {0} Not found in the provided list")]
ArnValidation(String),
}
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
pub struct FilterRule {
#[serde(rename = "Name")]
pub name: String,
#[serde(rename = "Value")]
pub value: String,
}
impl FilterRule {
fn validate(&self) -> Result<(), ParseConfigError> {
if !self.name.eq_ignore_ascii_case("prefix") && !self.name.eq_ignore_ascii_case("suffix") {
return Err(ParseConfigError::InvalidFilterName(self.name.clone()));
}
// ValidateFilterRuleValue from Go:
// no "." or ".." path segments, <= 1024 characters, no '\'.
for segment in self.value.split('/') {
if segment == "." || segment == ".." {
return Err(ParseConfigError::InvalidFilterValue(self.value.clone()));
}
}
// The limit is 1024 *characters* (runes), matching S3/Go semantics. Using the
// byte length (`str::len`) would wrongly reject valid keys whose multi-byte
// UTF-8 encoding exceeds 1024 bytes while staying under 1024 characters
// (backlog#984). `self.value` is a `String`, so it is already valid UTF-8 —
// no separate UTF-8 check is needed.
if self.value.chars().count() > 1024 || self.value.contains('\\') {
return Err(ParseConfigError::InvalidFilterValue(self.value.clone()));
}
Ok(())
}
}
#[derive(Debug, Serialize, Clone, Default, PartialEq, Eq)]
pub struct S3KeyFilter {
#[serde(rename = "FilterRule", default, skip_serializing_if = "Vec::is_empty")]
pub filter_rule_list: Vec<FilterRule>,
}
impl S3KeyFilter {
/// Validate filter rules for duplicates.
/// According to AWS S3 documentation, there can be at most one prefix
/// and one suffix filter rule per queue configuration.
pub fn validate(&self) -> Result<(), ParseConfigError> {
let mut has_prefix = false;
let mut has_suffix = false;
for rule in &self.filter_rule_list {
rule.validate()?;
if rule.name.eq_ignore_ascii_case("prefix") {
if has_prefix {
return Err(ParseConfigError::DuplicatePrefixFilter);
}
has_prefix = true;
} else if rule.name.eq_ignore_ascii_case("suffix") {
if has_suffix {
return Err(ParseConfigError::DuplicateSuffixFilter);
}
has_suffix = true;
}
}
Ok(())
}
/// Check if filter rule list is empty.
pub fn is_empty(&self) -> bool {
self.filter_rule_list.is_empty()
}
/// Generate pattern string from filter rules.
/// This method extracts prefix and suffix values from filter rules
/// and generates a wildcard pattern string for matching object keys.
pub fn pattern(&self) -> String {
let mut prefix_val: Option<&str> = None;
let mut suffix_val: Option<&str> = None;
for rule in &self.filter_rule_list {
if rule.name.eq_ignore_ascii_case("prefix") {
prefix_val = Some(&rule.value);
} else if rule.name.eq_ignore_ascii_case("suffix") {
suffix_val = Some(&rule.value);
}
}
pattern::new_pattern(prefix_val, suffix_val)
}
}
#[derive(Debug, serde::Deserialize)]
struct S3KeyContent {
#[serde(rename = "FilterRule", default)]
filter_rule_list: Vec<FilterRule>,
#[serde(rename = "FilterRuleList", default)]
filter_rule_list_wrapper: Option<S3KeyFilterRuleList>,
}
#[derive(Debug, serde::Deserialize)]
struct S3KeyFilterRuleList {
#[serde(rename = "FilterRule", default)]
filter_rule_list: Vec<FilterRule>,
}
impl S3KeyContent {
/// Get all filter rules from this S3Key content, handling both direct FilterRule
/// and FilterRuleList wrapper structures
fn get_filter_rules(&self) -> Vec<FilterRule> {
// If we have a FilterRuleList wrapper, use that
if let Some(wrapper) = &self.filter_rule_list_wrapper
&& !wrapper.filter_rule_list.is_empty()
{
return wrapper.filter_rule_list.clone();
}
// Otherwise use direct FilterRule list
self.filter_rule_list.clone()
}
}
/// Custom deserializer for S3KeyFilter to handle Filter element correctly.
/// AWS S3 XML structure: <Filter><S3Key><FilterRule>...</FilterRule></S3Key></Filter>
impl<'de> Deserialize<'de> for S3KeyFilter {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: serde::Deserializer<'de>,
{
struct S3KeyFilterVisitor {
filter_rules: Vec<FilterRule>,
}
impl S3KeyFilterVisitor {
fn new() -> Self {
Self {
filter_rules: Vec::new(),
}
}
}
impl<'de> serde::de::Visitor<'de> for S3KeyFilterVisitor {
type Value = S3KeyFilter;
fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
write!(
formatter,
"an S3Key filter map with an `S3Key` element containing one or more \
`FilterRule` children (e.g. <Filter><S3Key><FilterRule>...</FilterRule></S3Key></Filter>)"
)
}
fn visit_map<V>(mut self, mut map: V) -> Result<Self::Value, V::Error>
where
V: serde::de::MapAccess<'de>,
{
while let Some(key) = map.next_key::<String>()? {
match key.as_str() {
"S3Key" => {
// Parse S3Key content which contains FilterRule(s)
let s3key_content: S3KeyContent = map.next_value()?;
self.filter_rules = s3key_content.get_filter_rules();
}
_ => {
return Err(serde::de::Error::unknown_field(&key, &["S3Key"]));
}
}
}
Ok(S3KeyFilter {
filter_rule_list: self.filter_rules,
})
}
}
deserializer.deserialize_map(S3KeyFilterVisitor::new())
}
}
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
pub struct QueueConfig {
#[serde(rename = "Id", skip_serializing_if = "Option::is_none")]
pub id: Option<String>,
#[serde(rename = "Queue")] // This is ARN in XML
pub arn: ARN,
#[serde(rename = "Event", default)] // XML has multiple <Event> tags
pub events: Vec<EventName>, // EventName needs to handle XML (de)serialization if not string
#[serde(rename = "Filter", default, skip_serializing_if = "s3key_filter_is_empty")]
pub filter: S3KeyFilter,
}
fn s3key_filter_is_empty(f: &S3KeyFilter) -> bool {
f.filter_rule_list.is_empty()
}
impl QueueConfig {
pub fn validate(&self, region: &str, arn_list: &[String]) -> Result<(), ParseConfigError> {
if self.events.is_empty() {
return Err(ParseConfigError::MissingEventName);
}
let mut event_set = HashSet::new();
for event in &self.events {
// EventName::to_string() or similar for uniqueness check
if !event_set.insert(event.to_string()) {
return Err(ParseConfigError::DuplicateEventName(event.to_string()));
}
}
self.filter.validate()?;
// Validate ARN (similar to Go's Queue.Validate)
// The Go code checks targetList.Exists(q.ARN.TargetID)
// Here we check against a provided arn_list
let _config_arn_str = self.arn.to_string();
if !self.arn.region.is_empty() && self.arn.region != region {
return Err(ParseConfigError::UnknownRegion(self.arn.region.clone()));
}
// Construct the ARN string that would be in arn_list
// The arn_list contains ARNs like "arn:rustfs:sqs:REGION:ID:NAME"
// We need to ensure self.arn (potentially with region adjusted) is in arn_list
let effective_arn = ARN {
target_id: self.arn.target_id.clone(),
region: if self.arn.region.is_empty() {
region.to_string()
} else {
self.arn.region.clone()
},
service: self.arn.service.clone(), // or default "sqs"
partition: self.arn.partition.clone(), // or default "rustfs"
};
if !arn_list.contains(&effective_arn.to_string()) {
return Err(ParseConfigError::ArnNotFound(effective_arn.to_string()));
}
Ok(())
}
/// Sets the region if it's not already set in the ARN.
pub fn set_region_if_empty(&mut self, region: &str) {
if self.arn.region.is_empty() {
self.arn.region = region.to_string();
}
}
}
/// Corresponding to the `lambda` structure in the Go code.
/// Used to parse <CloudFunction> ARN from inside the <CloudFunctionConfiguration> tag.
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq, Default)]
pub struct LambdaConfigDetail {
#[serde(rename = "CloudFunction")]
pub arn: String,
// According to AWS S3 documentation, <CloudFunctionConfiguration> usually also contains Id, Event, Filter
// But in order to strictly correspond to the Go `lambda` structure provided, only ARN is included here.
// If full support is required, additional fields can be added.
// For example:
// #[serde(rename = "Id", skip_serializing_if = "Option::is_none")]
// pub id: Option<String>,
// #[serde(rename = "Event", default, skip_serializing_if = "Vec::is_empty")]
// pub events: Vec<EventName>,
// #[serde(rename = "Filter", default, skip_serializing_if = "S3KeyFilterIsEmpty")]
// pub filter: S3KeyFilter,
}
/// Corresponding to the `topic` structure in the Go code.
/// Used to parse <Topic> ARN from inside the <TopicConfiguration> tag.
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq, Default)]
pub struct TopicConfigDetail {
#[serde(rename = "Topic")]
pub arn: String,
// Similar to LambdaConfigDetail, it can be extended to include fields such as Id, Event, Filter, etc.
}
#[derive(Debug, Serialize, Deserialize, Clone, Default, PartialEq, Eq)]
#[serde(rename = "NotificationConfiguration")]
pub struct NotificationConfiguration {
#[serde(rename = "xmlns", skip_serializing_if = "Option::is_none")]
pub xmlns: Option<String>,
#[serde(rename = "QueueConfiguration", default, skip_serializing_if = "Vec::is_empty")]
pub queue_list: Vec<QueueConfig>,
#[serde(
rename = "CloudFunctionConfiguration", // Tags for each lambda configuration item in XML
default,
skip_serializing_if = "Vec::is_empty"
)]
pub lambda_list: Vec<LambdaConfigDetail>, // Modify: Use a new structure
#[serde(
rename = "TopicConfiguration", // Tags for each topic configuration item in XML
default,
skip_serializing_if = "Vec::is_empty"
)]
pub topic_list: Vec<TopicConfigDetail>, // Modify: Use a new structure
}
impl NotificationConfiguration {
pub fn from_reader<R: Read + std::io::BufRead>(reader: R) -> Result<Self, ParseConfigError> {
let config: NotificationConfiguration = quick_xml::de::from_reader(reader)?;
Ok(config)
}
pub fn validate(&self, current_region: &str, arn_list: &[String]) -> Result<(), ParseConfigError> {
// Verification logic remains the same: if lambda_list or topic_list is not empty, it is considered an unsupported configuration
if !self.lambda_list.is_empty() || !self.topic_list.is_empty() {
return Err(ParseConfigError::UnsupportedConfiguration);
}
let mut unique_queues = HashSet::new();
for queue_config in &self.queue_list {
queue_config.validate(current_region, arn_list)?;
let queue_key = (
queue_config.id.clone(),
queue_config.arn.to_string(), // Assuming that the ARN structure implements Display or ToString
);
if !unique_queues.insert(queue_key.clone()) {
return Err(ParseConfigError::DuplicateQueueConfiguration(queue_key.0, queue_key.1));
}
}
Ok(())
}
pub fn set_defaults(&mut self, region: &str) {
for queue_config in &mut self.queue_list {
queue_config.set_region_if_empty(region);
}
if self.xmlns.is_none() {
self.xmlns = Some("http://s3.amazonaws.com/doc/2006-03-01/".to_string());
}
// Note: If LambdaConfigDetail and TopicConfigDetail contain information such as regions in the future,
// You may also need to set the default value here. But according to the current definition, they only contain ARN strings.
}
}
#[cfg(test)]
mod filter_rule_length_tests {
use super::{FilterRule, ParseConfigError};
fn prefix(value: String) -> FilterRule {
FilterRule {
name: "prefix".to_string(),
value,
}
}
#[test]
fn length_limit_counts_characters_not_bytes() {
// 1024 multi-byte characters: 3072 bytes but exactly 1024 chars — must pass.
// The old byte-length check (`str::len`) would have rejected this valid key
// (backlog#984).
let multibyte = "".repeat(1024);
assert_eq!(multibyte.chars().count(), 1024);
assert!(multibyte.len() > 1024, "test setup: byte length must exceed the char limit");
prefix(multibyte).validate().expect("1024-character value must be accepted");
// 1025 characters must be rejected.
let too_long = "a".repeat(1025);
assert!(matches!(prefix(too_long).validate(), Err(ParseConfigError::InvalidFilterValue(_))));
// Exactly 1024 ASCII characters must pass.
prefix("a".repeat(1024))
.validate()
.expect("1024 ASCII chars must be accepted");
}
#[test]
fn backslash_is_still_rejected() {
assert!(matches!(
prefix("bad\\path".to_string()).validate(),
Err(ParseConfigError::InvalidFilterValue(_))
));
}
}