mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-26 08:18:18 +00:00
feat: add an opt-in NATS JetStream publish path for the notify and audit targets (#4634)
feat(targets): add an opt-in NATS JetStream publish path for the notify and audit targets The NATS notify and audit targets publish through NATS Core, which returns before the server has durably accepted the message. A broker restart or a connection drop between the publish and the flush loses the event, even though the send queue has already cleared it, and no acknowledgement gates that clear. An opt-in JetStream publish path clears a queued event only after the server returns a durable PublishAck, so delivery is at-least-once across a broker restart or a reconnect. It applies to both the notify and audit NATS targets, is off by default, and is byte-identical to the NATS Core path when disabled. The path includes durable store-and-forward, a stable dedup id sent as the Nats-Msg-Id header so a replayed event is collapsed by the stream duplicate window, pre-flight stream validation, and a bounded failed-events store for terminally-failed and retry-exhausted events. Three configuration keys per target select it: JETSTREAM_ENABLE, JETSTREAM_STREAM_NAME, and JETSTREAM_ACK_TIMEOUT_SECS, under the RUSTFS_NOTIFY_NATS_ and RUSTFS_AUDIT_NATS_ prefixes. The on-disk batch filename separator changes from colon to underscore so batch names are valid on Windows filesystems, with transparent read-back of files written under the previous separator. The migration affects the shared queue store for every target type and lands with this feature because the store gains its first Windows-exercised paths here. Co-authored-by: houseme <housemecn@gmail.com>
This commit is contained in:
escapecode
@@ -34,6 +34,7 @@ rustls-native-certs = { workspace = true }
|
||||
s3s = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
sha2 = { workspace = true }
|
||||
snap = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
tokio = { workspace = true, features = ["fs", "rt-multi-thread", "sync", "time"] }
|
||||
@@ -56,6 +57,7 @@ metrics = { workspace = true }
|
||||
criterion = { workspace = true }
|
||||
tempfile = { workspace = true }
|
||||
rcgen = { workspace = true }
|
||||
tracing-subscriber = { workspace = true }
|
||||
|
||||
[[bench]]
|
||||
name = "queue_store_benchmark"
|
||||
|
||||
@@ -100,11 +100,27 @@ pub async fn check_nats_server_available(args: &crate::target::nats::NATSArgs) -
|
||||
.flush()
|
||||
.await
|
||||
.map_err(|e| crate::TargetError::Network(format!("NATS connection check failed: {e}")))?;
|
||||
// Validate the configured stream on the live connection before the drain closes it, so a
|
||||
// missing or non-writable stream is rejected here rather than at first publish. The lookup is
|
||||
// read-only and never creates a stream. The drain runs regardless of the validation outcome so
|
||||
// the check connection is closed gracefully, and the validation error then propagates.
|
||||
let validation = if args.jetstream_enable.unwrap_or(false) {
|
||||
let mut context = async_nats::jetstream::new(client.clone());
|
||||
// Match every other context construction site: the ack timeout is the per-operation await
|
||||
// bound, so the validation get_stream lookup uses it rather than the library default.
|
||||
context.set_timeout(std::time::Duration::from_secs(
|
||||
args.jetstream_ack_timeout_secs
|
||||
.unwrap_or(rustfs_config::NATS_JETSTREAM_ACK_TIMEOUT_DEFAULT_SECS),
|
||||
));
|
||||
crate::target::nats::validate_jetstream_stream(&context, args, "nats-check", None).await
|
||||
} else {
|
||||
Ok(())
|
||||
};
|
||||
client
|
||||
.drain()
|
||||
.await
|
||||
.map_err(|e| crate::TargetError::Network(format!("Failed to close NATS check connection: {e}")))?;
|
||||
Ok(())
|
||||
validation
|
||||
})
|
||||
.await
|
||||
.unwrap_or_else(|_| Err(crate::TargetError::Timeout("NATS connection timed out".to_string())))
|
||||
|
||||
@@ -13,13 +13,15 @@
|
||||
// limitations under the License.
|
||||
|
||||
use crate::TargetError;
|
||||
use crate::target::nats::validate_jetstream_settings;
|
||||
use crate::target::pulsar::validate_pulsar_broker;
|
||||
use async_nats::ServerAddr;
|
||||
use rustfs_config::server_config::KVS;
|
||||
use rustfs_config::{
|
||||
DEFAULT_DELIMITER, ENABLE_KEY, EnableState, NATS_CREDENTIALS_FILE, NATS_PASSWORD, NATS_QUEUE_DIR, NATS_SUBJECT, NATS_TLS_CA,
|
||||
NATS_TLS_CLIENT_CERT, NATS_TLS_CLIENT_KEY, NATS_TOKEN, NATS_USERNAME, PULSAR_AUTH_TOKEN, PULSAR_PASSWORD, PULSAR_QUEUE_DIR,
|
||||
PULSAR_TLS_CA, PULSAR_TOPIC, PULSAR_USERNAME,
|
||||
DEFAULT_DELIMITER, ENABLE_KEY, EnableState, NATS_CREDENTIALS_FILE, NATS_JETSTREAM_ACK_TIMEOUT_SECS, NATS_JETSTREAM_ENABLE,
|
||||
NATS_JETSTREAM_STREAM_NAME, NATS_PASSWORD, NATS_QUEUE_DIR, NATS_SUBJECT, NATS_TLS_CA, NATS_TLS_CLIENT_CERT,
|
||||
NATS_TLS_CLIENT_KEY, NATS_TOKEN, NATS_USERNAME, PULSAR_AUTH_TOKEN, PULSAR_PASSWORD, PULSAR_QUEUE_DIR, PULSAR_TLS_CA,
|
||||
PULSAR_TOPIC, PULSAR_USERNAME,
|
||||
};
|
||||
use rustfs_utils::egress::validate_outbound_url;
|
||||
use std::collections::HashSet;
|
||||
@@ -130,10 +132,42 @@ pub(super) fn validate_nats_server_config(server: &ServerAddr, config: &KVS, def
|
||||
return Err(TargetError::Configuration("NATS queue directory must be an absolute path".to_string()));
|
||||
}
|
||||
|
||||
let jetstream_enable = parse_jetstream_enable(config)?.unwrap_or(false);
|
||||
let jetstream_stream_name = config.lookup(NATS_JETSTREAM_STREAM_NAME).unwrap_or_default();
|
||||
let jetstream_ack_timeout_secs = parse_jetstream_ack_timeout_secs(config)?;
|
||||
validate_jetstream_settings(jetstream_enable, &jetstream_stream_name, &queue_dir, jetstream_ack_timeout_secs)?;
|
||||
|
||||
let _ = server;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Parses the jetstream_enable flag, distinguishing an absent or blank key (None, disabled) from a
|
||||
/// key that is present but carries an unparsable value, which is a configuration error rather than
|
||||
/// a silent disable. Unwrapping an unparsable value to disabled would drop the durability guarantee
|
||||
/// the operator asked for with no signal.
|
||||
pub(super) fn parse_jetstream_enable(config: &KVS) -> Result<Option<bool>, TargetError> {
|
||||
match config.lookup(NATS_JETSTREAM_ENABLE) {
|
||||
Some(value) if !value.trim().is_empty() => parse_target_bool(Some(value.as_str())).map(Some).ok_or_else(|| {
|
||||
TargetError::Configuration(format!(
|
||||
"Invalid {NATS_JETSTREAM_ENABLE} boolean value: {value} (accepted values include on, off, true, false, yes, no, 1, 0)"
|
||||
))
|
||||
}),
|
||||
_ => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn parse_jetstream_ack_timeout_secs(config: &KVS) -> Result<Option<u64>, TargetError> {
|
||||
match config.lookup(NATS_JETSTREAM_ACK_TIMEOUT_SECS) {
|
||||
Some(value) if !value.trim().is_empty() => {
|
||||
let secs = value.trim().parse::<u64>().map_err(|_| {
|
||||
TargetError::Configuration(format!("{NATS_JETSTREAM_ACK_TIMEOUT_SECS} must be a whole number of seconds"))
|
||||
})?;
|
||||
Ok(Some(secs))
|
||||
}
|
||||
_ => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn validate_pulsar_broker_config(broker: &str, config: &KVS, default_queue_dir: &str) -> Result<(), TargetError> {
|
||||
let url = validate_pulsar_broker(broker)?;
|
||||
|
||||
@@ -193,15 +227,26 @@ pub(super) fn validate_outbound_http_url(value: &Url, field_label: &str) -> Resu
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{validate_nats_server_config, validate_pulsar_broker_config};
|
||||
use super::{parse_jetstream_enable, validate_nats_server_config, validate_pulsar_broker_config};
|
||||
use async_nats::ServerAddr;
|
||||
use rustfs_config::server_config::KVS;
|
||||
use rustfs_config::{
|
||||
NATS_PASSWORD, NATS_QUEUE_DIR, NATS_SUBJECT, NATS_TOKEN, NATS_USERNAME, PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA,
|
||||
PULSAR_TLS_HOSTNAME_VERIFICATION, PULSAR_TOPIC,
|
||||
NATS_JETSTREAM_ACK_TIMEOUT_SECS, NATS_JETSTREAM_ENABLE, NATS_JETSTREAM_STREAM_NAME, NATS_PASSWORD, NATS_QUEUE_DIR,
|
||||
NATS_SUBJECT, NATS_TOKEN, NATS_USERNAME, PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA, PULSAR_TLS_HOSTNAME_VERIFICATION,
|
||||
PULSAR_TOPIC,
|
||||
};
|
||||
use std::str::FromStr;
|
||||
|
||||
fn nats_server() -> ServerAddr {
|
||||
ServerAddr::from_str("nats://127.0.0.1:4222").expect("valid nats address")
|
||||
}
|
||||
|
||||
// Absolute on Linux, macOS, and Windows. temp_dir needs no filesystem to exist for a
|
||||
// validation-only test, and Path::is_absolute stays true across platforms.
|
||||
fn nats_queue_dir() -> String {
|
||||
std::env::temp_dir().join("rustfs-nats-queue").to_string_lossy().into_owned()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_nats_server_config_rejects_multiple_auth_methods() {
|
||||
let server = ServerAddr::from_str("nats://127.0.0.1:4222").expect("valid nats address");
|
||||
@@ -228,6 +273,131 @@ mod tests {
|
||||
assert!(err.to_string().contains("absolute path"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_nats_server_config_rejects_jetstream_without_stream() {
|
||||
let mut config = KVS::new();
|
||||
config.insert(NATS_SUBJECT.to_string(), "events".to_string());
|
||||
config.insert(NATS_QUEUE_DIR.to_string(), nats_queue_dir());
|
||||
config.insert(NATS_JETSTREAM_ENABLE.to_string(), "on".to_string());
|
||||
|
||||
let err = validate_nats_server_config(&nats_server(), &config, "")
|
||||
.expect_err("enabled jetstream without a stream should be rejected");
|
||||
assert!(err.to_string().contains(NATS_JETSTREAM_STREAM_NAME));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_nats_server_config_rejects_jetstream_without_queue_dir() {
|
||||
let mut config = KVS::new();
|
||||
config.insert(NATS_SUBJECT.to_string(), "events".to_string());
|
||||
config.insert(NATS_JETSTREAM_ENABLE.to_string(), "on".to_string());
|
||||
config.insert(NATS_JETSTREAM_STREAM_NAME.to_string(), "RUSTFS_EVENTS".to_string());
|
||||
|
||||
let err = validate_nats_server_config(&nats_server(), &config, "")
|
||||
.expect_err("enabled jetstream without a queue_dir should be rejected");
|
||||
assert!(err.to_string().contains(NATS_QUEUE_DIR));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_nats_server_config_rejects_out_of_range_ack_timeout() {
|
||||
let mut config = KVS::new();
|
||||
config.insert(NATS_SUBJECT.to_string(), "events".to_string());
|
||||
config.insert(NATS_JETSTREAM_ACK_TIMEOUT_SECS.to_string(), "5".to_string());
|
||||
|
||||
let err =
|
||||
validate_nats_server_config(&nats_server(), &config, "").expect_err("out-of-range ack timeout should be rejected");
|
||||
assert!(err.to_string().contains("between"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_nats_server_config_accepts_enabled_jetstream_with_stream_and_queue() {
|
||||
let mut config = KVS::new();
|
||||
config.insert(NATS_SUBJECT.to_string(), "events".to_string());
|
||||
config.insert(NATS_QUEUE_DIR.to_string(), nats_queue_dir());
|
||||
config.insert(NATS_JETSTREAM_ENABLE.to_string(), "on".to_string());
|
||||
config.insert(NATS_JETSTREAM_STREAM_NAME.to_string(), "RUSTFS_EVENTS".to_string());
|
||||
|
||||
validate_nats_server_config(&nats_server(), &config, "").expect("a stream name and queue dir accept enabling jetstream");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_nats_server_config_audit_parity_matches_notify() {
|
||||
let mut config = KVS::new();
|
||||
config.insert(NATS_SUBJECT.to_string(), "events".to_string());
|
||||
config.insert(NATS_QUEUE_DIR.to_string(), nats_queue_dir());
|
||||
config.insert(NATS_JETSTREAM_ENABLE.to_string(), "on".to_string());
|
||||
|
||||
// The same invalid combination (enabled, stream name missing) is rejected identically on the
|
||||
// notify and the audit path, proving one shared validator with no drift.
|
||||
let notify = validate_nats_server_config(&nats_server(), &config, "/opt/rustfs/events").expect_err("notify rejects");
|
||||
let audit = validate_nats_server_config(&nats_server(), &config, "/opt/rustfs/audit").expect_err("audit rejects");
|
||||
assert_eq!(notify.to_string(), audit.to_string());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_nats_server_config_accepts_jetstream_absent() {
|
||||
let mut config = KVS::new();
|
||||
config.insert(NATS_SUBJECT.to_string(), "events".to_string());
|
||||
config.insert(NATS_QUEUE_DIR.to_string(), nats_queue_dir());
|
||||
|
||||
validate_nats_server_config(&nats_server(), &config, "").expect("baseline config validates");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_jetstream_enable_accepts_the_standard_boolean_forms() {
|
||||
// An absent or blank key is None, legitimately disabled.
|
||||
assert_eq!(parse_jetstream_enable(&KVS::new()).unwrap(), None, "an absent key parses as unset");
|
||||
let mut blank = KVS::new();
|
||||
blank.insert(NATS_JETSTREAM_ENABLE.to_string(), " ".to_string());
|
||||
assert_eq!(parse_jetstream_enable(&blank).unwrap(), None, "a blank value parses as unset");
|
||||
|
||||
for (value, expected) in [
|
||||
("on", true),
|
||||
("off", false),
|
||||
("true", true),
|
||||
("false", false),
|
||||
("yes", true),
|
||||
("no", false),
|
||||
("1", true),
|
||||
("0", false),
|
||||
("enabled", true),
|
||||
("disabled", false),
|
||||
] {
|
||||
let mut config = KVS::new();
|
||||
config.insert(NATS_JETSTREAM_ENABLE.to_string(), value.to_string());
|
||||
assert_eq!(
|
||||
parse_jetstream_enable(&config).unwrap(),
|
||||
Some(expected),
|
||||
"value {value} parses as {expected}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_jetstream_enable_rejects_an_unparsable_value_naming_the_key() {
|
||||
// A typo must fail validation rather than silently disabling the durability guarantee.
|
||||
for value in ["y", "t", "enabl", "10"] {
|
||||
let mut config = KVS::new();
|
||||
config.insert(NATS_JETSTREAM_ENABLE.to_string(), value.to_string());
|
||||
let err = parse_jetstream_enable(&config).expect_err("an unparsable value is rejected");
|
||||
assert!(
|
||||
err.to_string().contains(NATS_JETSTREAM_ENABLE),
|
||||
"the error names the key for value {value}: {err}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_nats_server_config_rejects_an_invalid_jetstream_enable_value() {
|
||||
let mut config = KVS::new();
|
||||
config.insert(NATS_SUBJECT.to_string(), "events".to_string());
|
||||
config.insert(NATS_QUEUE_DIR.to_string(), nats_queue_dir());
|
||||
config.insert(NATS_JETSTREAM_ENABLE.to_string(), "y".to_string());
|
||||
|
||||
let err = validate_nats_server_config(&nats_server(), &config, "")
|
||||
.expect_err("an unparsable enable value fails validation instead of passing as disabled");
|
||||
assert!(err.to_string().contains(NATS_JETSTREAM_ENABLE), "the error names the key: {err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_pulsar_broker_config_rejects_tls_ca_without_tls_scheme() {
|
||||
let mut config = KVS::new();
|
||||
|
||||
@@ -13,7 +13,8 @@
|
||||
// limitations under the License.
|
||||
|
||||
use super::common::{
|
||||
parse_target_bool, parse_url, validate_nats_server_config, validate_outbound_http_url, validate_pulsar_broker_config,
|
||||
parse_jetstream_ack_timeout_secs, parse_jetstream_enable, parse_target_bool, parse_url, validate_nats_server_config,
|
||||
validate_outbound_http_url, validate_pulsar_broker_config,
|
||||
};
|
||||
use crate::error::TargetError;
|
||||
use crate::target::{
|
||||
@@ -38,17 +39,17 @@ use rustfs_config::{
|
||||
MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA,
|
||||
MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY, MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME,
|
||||
MQTT_WS_PATH_ALLOWLIST, MYSQL_DSN_STRING, MYSQL_FORMAT, MYSQL_MAX_OPEN_CONNECTIONS, MYSQL_QUEUE_DIR, MYSQL_QUEUE_LIMIT,
|
||||
MYSQL_TABLE, MYSQL_TLS_CA, MYSQL_TLS_CLIENT_CERT, MYSQL_TLS_CLIENT_KEY, NATS_ADDRESS, NATS_CREDENTIALS_FILE, NATS_PASSWORD,
|
||||
NATS_QUEUE_DIR, NATS_QUEUE_LIMIT, NATS_SUBJECT, NATS_TLS_CA, NATS_TLS_CLIENT_CERT, NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED,
|
||||
NATS_TOKEN, NATS_USERNAME, POSTGRES_DSN_STRING, POSTGRES_FORMAT, POSTGRES_QUEUE_DIR, POSTGRES_QUEUE_LIMIT, POSTGRES_TABLE,
|
||||
POSTGRES_TLS_CA, POSTGRES_TLS_CLIENT_CERT, POSTGRES_TLS_CLIENT_KEY, POSTGRES_TLS_REQUIRED, PULSAR_AUTH_TOKEN, PULSAR_BROKER,
|
||||
PULSAR_PASSWORD, PULSAR_QUEUE_DIR, PULSAR_QUEUE_LIMIT, PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA,
|
||||
PULSAR_TLS_HOSTNAME_VERIFICATION, PULSAR_TOPIC, PULSAR_USERNAME, REDIS_CHANNEL, REDIS_CONNECTION_TIMEOUT,
|
||||
REDIS_KEEP_ALIVE_INTERVAL, REDIS_MAX_RETRY_ATTEMPTS, REDIS_MAX_RETRY_DELAY, REDIS_MIN_RETRY_DELAY, REDIS_PASSWORD,
|
||||
REDIS_PIPELINE_BUFFER_SIZE, REDIS_QUEUE_DIR, REDIS_QUEUE_LIMIT, REDIS_RECONNECT_RETRY_ATTEMPTS, REDIS_RESPONSE_TIMEOUT,
|
||||
REDIS_TLS_ALLOW_INSECURE, REDIS_TLS_CA, REDIS_TLS_CLIENT_CERT, REDIS_TLS_CLIENT_KEY, REDIS_TLS_POLICY, REDIS_URL,
|
||||
REDIS_USERNAME, RUSTFS_WEBHOOK_SKIP_TLS_VERIFY_DEFAULT, WEBHOOK_AUTH_TOKEN, WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT,
|
||||
WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT, WEBHOOK_SKIP_TLS_VERIFY,
|
||||
MYSQL_TABLE, MYSQL_TLS_CA, MYSQL_TLS_CLIENT_CERT, MYSQL_TLS_CLIENT_KEY, NATS_ADDRESS, NATS_CREDENTIALS_FILE,
|
||||
NATS_JETSTREAM_STREAM_NAME, NATS_PASSWORD, NATS_QUEUE_DIR, NATS_QUEUE_LIMIT, NATS_SUBJECT, NATS_TLS_CA, NATS_TLS_CLIENT_CERT,
|
||||
NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED, NATS_TOKEN, NATS_USERNAME, POSTGRES_DSN_STRING, POSTGRES_FORMAT, POSTGRES_QUEUE_DIR,
|
||||
POSTGRES_QUEUE_LIMIT, POSTGRES_TABLE, POSTGRES_TLS_CA, POSTGRES_TLS_CLIENT_CERT, POSTGRES_TLS_CLIENT_KEY,
|
||||
POSTGRES_TLS_REQUIRED, PULSAR_AUTH_TOKEN, PULSAR_BROKER, PULSAR_PASSWORD, PULSAR_QUEUE_DIR, PULSAR_QUEUE_LIMIT,
|
||||
PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA, PULSAR_TLS_HOSTNAME_VERIFICATION, PULSAR_TOPIC, PULSAR_USERNAME, REDIS_CHANNEL,
|
||||
REDIS_CONNECTION_TIMEOUT, REDIS_KEEP_ALIVE_INTERVAL, REDIS_MAX_RETRY_ATTEMPTS, REDIS_MAX_RETRY_DELAY, REDIS_MIN_RETRY_DELAY,
|
||||
REDIS_PASSWORD, REDIS_PIPELINE_BUFFER_SIZE, REDIS_QUEUE_DIR, REDIS_QUEUE_LIMIT, REDIS_RECONNECT_RETRY_ATTEMPTS,
|
||||
REDIS_RESPONSE_TIMEOUT, REDIS_TLS_ALLOW_INSECURE, REDIS_TLS_CA, REDIS_TLS_CLIENT_CERT, REDIS_TLS_CLIENT_KEY,
|
||||
REDIS_TLS_POLICY, REDIS_URL, REDIS_USERNAME, RUSTFS_WEBHOOK_SKIP_TLS_VERIFY_DEFAULT, WEBHOOK_AUTH_TOKEN, WEBHOOK_CLIENT_CA,
|
||||
WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT, WEBHOOK_SKIP_TLS_VERIFY,
|
||||
};
|
||||
use std::path::Path;
|
||||
use std::time::Duration;
|
||||
@@ -315,6 +316,11 @@ pub fn build_nats_args(config: &KVS, default_queue_dir: &str, target_type: Targe
|
||||
.lookup(NATS_QUEUE_LIMIT)
|
||||
.and_then(|v| v.parse::<u64>().ok())
|
||||
.unwrap_or(DEFAULT_LIMIT),
|
||||
// Strict parse: an unparsable value is a configuration error, never a silent disable, so the
|
||||
// durability guarantee the operator asked for cannot vanish over a typo.
|
||||
jetstream_enable: parse_jetstream_enable(config)?,
|
||||
jetstream_stream_name: config.lookup(NATS_JETSTREAM_STREAM_NAME).filter(|name| !name.is_empty()),
|
||||
jetstream_ack_timeout_secs: parse_jetstream_ack_timeout_secs(config)?,
|
||||
target_type,
|
||||
})
|
||||
}
|
||||
@@ -598,9 +604,9 @@ pub fn validate_mysql_config(config: &KVS, default_queue_dir: &str) -> Result<()
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
build_amqp_args, build_kafka_args, build_mysql_args, build_postgres_args, build_redis_args, build_webhook_args,
|
||||
validate_amqp_config, validate_kafka_config, validate_mysql_config, validate_postgres_config, validate_redis_config,
|
||||
validate_webhook_config,
|
||||
build_amqp_args, build_kafka_args, build_mysql_args, build_nats_args, build_postgres_args, build_redis_args,
|
||||
build_webhook_args, validate_amqp_config, validate_kafka_config, validate_mysql_config, validate_postgres_config,
|
||||
validate_redis_config, validate_webhook_config,
|
||||
};
|
||||
use crate::target::{
|
||||
TargetType,
|
||||
@@ -612,7 +618,8 @@ mod tests {
|
||||
AMQP_EXCHANGE, AMQP_MANDATORY, AMQP_PASSWORD, AMQP_PERSISTENT, AMQP_QUEUE_DIR, AMQP_ROUTING_KEY, AMQP_TLS_CLIENT_CERT,
|
||||
AMQP_TLS_CLIENT_KEY, AMQP_URL, AMQP_USERNAME, KAFKA_ACKS, KAFKA_BROKERS, KAFKA_SASL_ENABLE, KAFKA_SASL_MECHANISM,
|
||||
KAFKA_SASL_PASSWORD, KAFKA_SASL_USERNAME, KAFKA_TLS_ENABLE, KAFKA_TOPIC, MYSQL_DSN_STRING, MYSQL_MAX_OPEN_CONNECTIONS,
|
||||
MYSQL_QUEUE_DIR, MYSQL_TABLE, MYSQL_TLS_CA, MYSQL_TLS_CLIENT_CERT, MYSQL_TLS_CLIENT_KEY, POSTGRES_DSN_STRING,
|
||||
MYSQL_QUEUE_DIR, MYSQL_TABLE, MYSQL_TLS_CA, MYSQL_TLS_CLIENT_CERT, MYSQL_TLS_CLIENT_KEY, NATS_ADDRESS,
|
||||
NATS_JETSTREAM_ACK_TIMEOUT_SECS, NATS_JETSTREAM_ENABLE, NATS_JETSTREAM_STREAM_NAME, NATS_SUBJECT, POSTGRES_DSN_STRING,
|
||||
POSTGRES_FORMAT, POSTGRES_QUEUE_DIR, POSTGRES_TABLE, POSTGRES_TLS_CA, POSTGRES_TLS_CLIENT_CERT, POSTGRES_TLS_CLIENT_KEY,
|
||||
REDIS_CHANNEL, REDIS_CONNECTION_TIMEOUT, REDIS_MAX_RETRY_DELAY, REDIS_MIN_RETRY_DELAY, REDIS_PIPELINE_BUFFER_SIZE,
|
||||
REDIS_RECONNECT_RETRY_ATTEMPTS, REDIS_RESPONSE_TIMEOUT, REDIS_TLS_ALLOW_INSECURE, REDIS_URL, WEBHOOK_ENDPOINT,
|
||||
@@ -1157,4 +1164,68 @@ mod tests {
|
||||
let err = validate_postgres_config(&config, "").expect_err("relative tls_client_cert should fail");
|
||||
assert!(err.to_string().contains("must be an absolute path"));
|
||||
}
|
||||
|
||||
fn nats_base_config() -> KVS {
|
||||
let mut config = KVS::new();
|
||||
config.insert(NATS_ADDRESS.to_string(), "nats://127.0.0.1:4222".to_string());
|
||||
config.insert(NATS_SUBJECT.to_string(), "rustfs.events".to_string());
|
||||
config
|
||||
}
|
||||
|
||||
// Absolute on Linux, macOS, and Windows. temp_dir needs no filesystem to exist for a
|
||||
// validation-only test, and Path::is_absolute stays true across platforms.
|
||||
fn nats_queue_dir() -> String {
|
||||
std::env::temp_dir().join("rustfs-nats-queue").to_string_lossy().into_owned()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_nats_args_parses_jetstream_keys() {
|
||||
let mut config = nats_base_config();
|
||||
config.insert(NATS_JETSTREAM_ENABLE.to_string(), "on".to_string());
|
||||
config.insert(NATS_JETSTREAM_STREAM_NAME.to_string(), "RUSTFS_EVENTS".to_string());
|
||||
config.insert(NATS_JETSTREAM_ACK_TIMEOUT_SECS.to_string(), "45".to_string());
|
||||
|
||||
for target_type in [TargetType::NotifyEvent, TargetType::AuditLog] {
|
||||
let args = build_nats_args(&config, &nats_queue_dir(), target_type).expect("jetstream keys parse");
|
||||
assert_eq!(args.jetstream_enable, Some(true));
|
||||
assert_eq!(args.jetstream_stream_name.as_deref(), Some("RUSTFS_EVENTS"));
|
||||
assert_eq!(args.jetstream_ack_timeout_secs, Some(45));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_nats_args_rejects_an_invalid_jetstream_enable_value() {
|
||||
// A typo in the enable flag is a configuration error naming the key, never a silent
|
||||
// disable that drops the durability guarantee the operator asked for.
|
||||
let mut config = nats_base_config();
|
||||
config.insert(NATS_JETSTREAM_ENABLE.to_string(), "t".to_string());
|
||||
|
||||
let err = build_nats_args(&config, &nats_queue_dir(), TargetType::NotifyEvent)
|
||||
.expect_err("an unparsable enable value fails the build");
|
||||
assert!(err.to_string().contains(NATS_JETSTREAM_ENABLE), "the error names the key: {err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_nats_args_without_jetstream_keys_is_flag_off() {
|
||||
let config = nats_base_config();
|
||||
let args = build_nats_args(&config, &nats_queue_dir(), TargetType::NotifyEvent).expect("baseline config parses");
|
||||
|
||||
assert_eq!(args.jetstream_enable, None);
|
||||
assert_eq!(args.jetstream_stream_name, None);
|
||||
assert_eq!(args.jetstream_ack_timeout_secs, None);
|
||||
assert_eq!(args.address, "nats://127.0.0.1:4222");
|
||||
assert_eq!(args.subject, "rustfs.events");
|
||||
assert_eq!(args.queue_dir, nats_queue_dir());
|
||||
args.validate().expect("flag-off config validates");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_nats_args_rejects_non_integer_ack_timeout() {
|
||||
let mut config = nats_base_config();
|
||||
config.insert(NATS_JETSTREAM_ACK_TIMEOUT_SECS.to_string(), "soon".to_string());
|
||||
|
||||
let err = build_nats_args(&config, &nats_queue_dir(), TargetType::NotifyEvent)
|
||||
.expect_err("non-integer ack timeout should be rejected");
|
||||
assert!(err.to_string().contains("whole number of seconds"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -52,6 +52,9 @@ pub enum TargetError {
|
||||
#[error("Request error: {0}")]
|
||||
Request(String),
|
||||
|
||||
#[error("JetStream publish error ({}): {detail}", if *.retryable { "retryable" } else { "terminal" })]
|
||||
JetStreamPublish { retryable: bool, detail: String },
|
||||
|
||||
#[error("Timeout error: {0}")]
|
||||
Timeout(String),
|
||||
|
||||
|
||||
+1228
-35
File diff suppressed because it is too large
Load Diff
+993
-43
File diff suppressed because it is too large
Load Diff
@@ -657,8 +657,11 @@ where
|
||||
}
|
||||
|
||||
fn delivery_snapshot(&self) -> TargetDeliverySnapshot {
|
||||
self.delivery_counters
|
||||
.snapshot(self.store.as_deref().map_or(0, |store| store.len() as u64))
|
||||
self.delivery_counters.snapshot(
|
||||
self.store.as_deref().map_or(0, |store| store.len() as u64),
|
||||
// AMQP targets record no terminal failures and keep no failed store.
|
||||
0,
|
||||
)
|
||||
}
|
||||
|
||||
fn record_final_failure(&self) {
|
||||
|
||||
@@ -503,8 +503,11 @@ where
|
||||
}
|
||||
|
||||
fn delivery_snapshot(&self) -> TargetDeliverySnapshot {
|
||||
self.delivery_counters
|
||||
.snapshot(self.store.as_deref().map_or(0, |store| store.len() as u64))
|
||||
self.delivery_counters.snapshot(
|
||||
self.store.as_deref().map_or(0, |store| store.len() as u64),
|
||||
// Kafka targets record no terminal failures and keep no failed store.
|
||||
0,
|
||||
)
|
||||
}
|
||||
|
||||
fn record_final_failure(&self) {
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
use crate::arn::TargetID;
|
||||
use crate::plugin::PluginEvent;
|
||||
use crate::store::{Key, QueueStore, Store};
|
||||
use crate::store::{FailedEventStore, Key, QueueStore, Store};
|
||||
use crate::{StoreError, TargetError, TargetLog};
|
||||
use async_trait::async_trait;
|
||||
use rustfs_s3_types::EventName;
|
||||
@@ -58,6 +58,7 @@ pub(crate) fn redacted_optional_secret(value: Option<&str>) -> &'static str {
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct TargetDeliverySnapshot {
|
||||
pub failed_messages: u64,
|
||||
pub failed_store_length: u64,
|
||||
pub queue_length: u64,
|
||||
pub total_messages: u64,
|
||||
}
|
||||
@@ -83,9 +84,10 @@ impl TargetDeliveryCounters {
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn snapshot(&self, queue_length: u64) -> TargetDeliverySnapshot {
|
||||
pub fn snapshot(&self, queue_length: u64, failed_store_length: u64) -> TargetDeliverySnapshot {
|
||||
TargetDeliverySnapshot {
|
||||
failed_messages: self.failed_messages.load(Ordering::Relaxed),
|
||||
failed_store_length,
|
||||
queue_length,
|
||||
total_messages: self.total_messages.load(Ordering::Relaxed),
|
||||
}
|
||||
@@ -159,6 +161,25 @@ where
|
||||
/// Returns the store associated with the target (if any)
|
||||
fn store(&self) -> Option<&(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync)>;
|
||||
|
||||
/// Returns the failed-events store capability when the target records terminal failures.
|
||||
///
|
||||
/// The default is no capability, so a target that never parks a terminal entry runs no failed-store
|
||||
/// maintenance and reports zero failed-store depth.
|
||||
fn failed_store(&self) -> Option<&dyn FailedEventStore> {
|
||||
None
|
||||
}
|
||||
|
||||
/// Moves a terminally failed entry to the target's failed-events store, returning true when the entry was handled. A target without a terminal-failure store declines the move and the entry stays live.
|
||||
async fn handle_terminal_failure(
|
||||
&self,
|
||||
_store: &(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send),
|
||||
_key: &Key,
|
||||
_error: &TargetError,
|
||||
_retry_count: u32,
|
||||
) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
/// Returns the type of the target
|
||||
fn clone_dyn(&self) -> Box<dyn Target<E> + Send + Sync>;
|
||||
|
||||
@@ -174,6 +195,7 @@ where
|
||||
/// Returns a read-only delivery snapshot for metrics collection.
|
||||
fn delivery_snapshot(&self) -> TargetDeliverySnapshot {
|
||||
TargetDeliverySnapshot {
|
||||
failed_store_length: self.failed_store().map_or(0, |failed_store| failed_store.failed_len() as u64),
|
||||
queue_length: self.store().map_or(0, |store| store.len() as u64),
|
||||
..TargetDeliverySnapshot::default()
|
||||
}
|
||||
@@ -202,6 +224,55 @@ pub struct QueuedPayloadMeta {
|
||||
pub content_type: String,
|
||||
pub queued_at_unix_ms: u64,
|
||||
pub payload_len: usize,
|
||||
/// Stable per-entry deduplication identifier sent as the NATS JetStream Nats-Msg-Id header.
|
||||
/// Empty for every non-JetStream target and for entries queued before JetStream was enabled, so
|
||||
/// it is skipped on serialization and absent stored entries decode to an empty value. This keeps
|
||||
/// the stored bytes identical to entries written without the field.
|
||||
#[serde(default, skip_serializing_if = "String::is_empty")]
|
||||
pub dedup_id: String,
|
||||
|
||||
/// Set only on an entry written to the failed-events store. None on every live-queue entry, so it
|
||||
/// is skipped on serialization and a live entry decodes with no failure metadata, keeping live
|
||||
/// stored bytes identical to entries written without the field.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub failure: Option<FailedEntryMeta>,
|
||||
}
|
||||
|
||||
/// The error class recorded on a failed-store entry. Only a non-retryable publish error reaches the
|
||||
/// failed store, so the class confirms a terminal cause for an operator.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub enum FailedErrorClass {
|
||||
/// A non-retryable publish error. Replaying it without fixing the underlying configuration repeats
|
||||
/// the failure.
|
||||
Terminal,
|
||||
}
|
||||
|
||||
impl FailedErrorClass {
|
||||
/// Stable lowercase tag used in structured logs and operator tooling.
|
||||
pub fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
FailedErrorClass::Terminal => "terminal",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Failure metadata added to a queued payload when it is moved to the failed-events store, carried
|
||||
/// inside the existing QueuedPayload meta so the failed entry decodes through the same reader.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct FailedEntryMeta {
|
||||
/// The failure class recorded for operator triage. A failed-store entry is always terminal.
|
||||
pub error_class: FailedErrorClass,
|
||||
/// An allowlisted, credential-free summary of the failure for operator diagnosis. Never the raw
|
||||
/// error rendering.
|
||||
pub error_detail: String,
|
||||
/// The stored dedup identifier, recording the id the publish attempted. A failed record is never
|
||||
/// republished.
|
||||
pub nats_msg_id: String,
|
||||
/// The instant the entry entered the failed store, a diagnostic field for operator triage. Expiry
|
||||
/// uses the filesystem modification time, not this value.
|
||||
pub failed_at_unix_ms: u64,
|
||||
/// The replay attempt count recorded at the terminal failure.
|
||||
pub retry_count: u32,
|
||||
}
|
||||
|
||||
impl QueuedPayloadMeta {
|
||||
@@ -219,6 +290,8 @@ impl QueuedPayloadMeta {
|
||||
content_type: content_type.into(),
|
||||
queued_at_unix_ms: SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_millis() as u64,
|
||||
payload_len,
|
||||
dedup_id: String::new(),
|
||||
failure: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -420,14 +493,13 @@ fn fnv1a_hash(bytes: &[u8]) -> u64 {
|
||||
|
||||
/// Maps a target-id component to a filesystem-safe queue directory name.
|
||||
///
|
||||
/// Path-unsafe characters are replaced with `_`. Because that replacement is
|
||||
/// lossy, two distinct ids (e.g. `a/b` and `a_b`) could otherwise collapse to
|
||||
/// the same directory and interleave their persisted events. To prevent that,
|
||||
/// whenever any character had to be replaced we append a short hash of the
|
||||
/// original component, guaranteeing distinct ids map to distinct directories.
|
||||
/// Path-unsafe characters are replaced with an underscore. That replacement is lossy, so two
|
||||
/// distinct ids (for example a/b and a_b) could otherwise collapse to the same directory and
|
||||
/// interleave their persisted events. A short hash of the original component is appended whenever any
|
||||
/// character was replaced, so distinct ids map to distinct directories.
|
||||
///
|
||||
/// Ids that are already path-safe are returned unchanged, preserving the
|
||||
/// on-disk directory layout for existing deployments (no queue migration).
|
||||
/// Ids that are already path-safe are returned unchanged, preserving the on-disk directory layout for
|
||||
/// existing deployments.
|
||||
pub(crate) fn sanitize_queue_dir_component(component: &str) -> String {
|
||||
let mut sanitized = String::with_capacity(component.len());
|
||||
let mut lossy = false;
|
||||
@@ -441,7 +513,7 @@ pub(crate) fn sanitize_queue_dir_component(component: &str) -> String {
|
||||
}
|
||||
|
||||
if sanitized.is_empty() {
|
||||
// An entirely non-safe id would otherwise all collapse to "_"; key it by
|
||||
// An entirely non-safe id would otherwise collapse to a single underscore. Key it by
|
||||
// the original bytes so distinct ids stay distinct.
|
||||
return format!("_{:016x}", fnv1a_hash(component.as_bytes()));
|
||||
}
|
||||
@@ -529,10 +601,20 @@ pub(crate) fn open_target_queue_store(
|
||||
target_id: &TargetID,
|
||||
open_context: &str,
|
||||
) -> Result<Option<BoxedQueuedStore>, TargetError> {
|
||||
fn boxed_queue_store(store: QueueStore<QueuedPayload>) -> BoxedQueuedStore {
|
||||
Box::new(store)
|
||||
}
|
||||
let store = open_target_queue_store_typed(queue_dir, queue_limit, target_type, target_type_label, target_id, open_context)?;
|
||||
Ok(store.map(|store| Box::new(store) as BoxedQueuedStore))
|
||||
}
|
||||
|
||||
/// Opens the queue store and returns the concrete QueueStore, so a target that needs its typed
|
||||
/// failed-store capability holds it directly rather than through the type-erased Store handle.
|
||||
pub(crate) fn open_target_queue_store_typed(
|
||||
queue_dir: &str,
|
||||
queue_limit: u64,
|
||||
target_type: TargetType,
|
||||
target_type_label: &str,
|
||||
target_id: &TargetID,
|
||||
open_context: &str,
|
||||
) -> Result<Option<QueueStore<QueuedPayload>>, TargetError> {
|
||||
if queue_dir.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
@@ -547,7 +629,7 @@ pub(crate) fn open_target_queue_store(
|
||||
.open()
|
||||
.map_err(|err| TargetError::Storage(format!("{open_context}: {err}")))?;
|
||||
|
||||
Ok(Some(boxed_queue_store(store)))
|
||||
Ok(Some(store))
|
||||
}
|
||||
|
||||
pub(crate) fn persist_queued_payload_to_store(
|
||||
@@ -584,7 +666,7 @@ pub(crate) fn mark_target_disconnected_on_connectivity_error(connected: &AtomicB
|
||||
}
|
||||
|
||||
pub(crate) fn delete_stored_payload(
|
||||
store: &(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync),
|
||||
store: &(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send),
|
||||
key: &Key,
|
||||
) -> Result<(), TargetError> {
|
||||
match store.del(key) {
|
||||
@@ -593,6 +675,96 @@ pub(crate) fn delete_stored_payload(
|
||||
}
|
||||
}
|
||||
|
||||
/// Upper bound on the characters retained from a classified error so a failed-store entry and its
|
||||
/// alarm carry a diagnosable summary without an unbounded message.
|
||||
const FAILED_ERROR_DETAIL_MAX_LEN: usize = 256;
|
||||
|
||||
/// Fallback label substituted for a JetStreamPublish detail that falls outside the fixed vocabulary.
|
||||
const UNRECOGNIZED_DETAIL_LABEL: &str = "unrecognized detail";
|
||||
|
||||
/// Restricts a JetStreamPublish detail to the fixed vocabulary at the persistence boundary. A detail
|
||||
/// of lowercase letters, digits, space, underscore, and colon passes verbatim, any other content is
|
||||
/// replaced with a fixed fallback label.
|
||||
fn sanitize_failed_detail(detail: &str) -> &str {
|
||||
let in_vocabulary = !detail.is_empty()
|
||||
&& detail.chars().all(|character| {
|
||||
character.is_ascii_lowercase() || character.is_ascii_digit() || matches!(character, ' ' | '_' | ':')
|
||||
});
|
||||
if in_vocabulary { detail } else { UNRECOGNIZED_DETAIL_LABEL }
|
||||
}
|
||||
|
||||
/// Builds a credential-free diagnostic string for a failed entry from a classified error. The
|
||||
/// publish-error detail passes through the persistence-boundary sanitizer, a non-publish error
|
||||
/// contributes only its variant category, and any embedded value is redacted. The result is
|
||||
/// length-bounded at a character boundary.
|
||||
pub(crate) fn build_failed_error_detail(error: &TargetError) -> String {
|
||||
let summary = match error {
|
||||
// The publish detail is sanitized to the fixed vocabulary at the persistence boundary.
|
||||
TargetError::JetStreamPublish { detail, .. } => format!("jetstream_publish: {}", sanitize_failed_detail(detail)),
|
||||
TargetError::Dropped(reason) => format!("dropped: {}", redacted_secret(reason)),
|
||||
// Every other variant carries a free-form message that may name a host, path, or credential.
|
||||
// Only the variant category is recorded, with any embedded value redacted.
|
||||
TargetError::Network(value) => format!("network: {}", redacted_secret(value)),
|
||||
TargetError::Request(value) => format!("request: {}", redacted_secret(value)),
|
||||
TargetError::Timeout(value) => format!("timeout: {}", redacted_secret(value)),
|
||||
TargetError::Storage(value) => format!("storage: {}", redacted_secret(value)),
|
||||
TargetError::Authentication(_) => "authentication".to_string(),
|
||||
TargetError::Configuration(_) => "configuration".to_string(),
|
||||
other => format!("error: {}", redacted_secret(&other_error_category(other))),
|
||||
};
|
||||
|
||||
let mut detail = summary;
|
||||
truncate_to_char_boundary(&mut detail, FAILED_ERROR_DETAIL_MAX_LEN);
|
||||
detail
|
||||
}
|
||||
|
||||
/// Truncates the string to at most max_len bytes, stepping the cut down to the nearest character
|
||||
/// boundary so a multi-byte character straddling the cap is dropped whole rather than split.
|
||||
fn truncate_to_char_boundary(value: &mut String, max_len: usize) {
|
||||
if value.len() <= max_len {
|
||||
return;
|
||||
}
|
||||
let mut cut = max_len;
|
||||
while !value.is_char_boundary(cut) {
|
||||
cut -= 1;
|
||||
}
|
||||
value.truncate(cut);
|
||||
}
|
||||
|
||||
/// Names the category of an otherwise free-form error without revealing its message.
|
||||
fn other_error_category(error: &TargetError) -> String {
|
||||
match error {
|
||||
TargetError::Encoding(_) => "encoding".to_string(),
|
||||
TargetError::Serialization(_) => "serialization".to_string(),
|
||||
TargetError::Initialization(_) => "initialization".to_string(),
|
||||
TargetError::Unknown(_) => "unknown".to_string(),
|
||||
_ => "other".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Encodes a queued payload as a failed-store entry, extending its meta with the failure fields.
|
||||
///
|
||||
/// Reuses the QueuedPayload format so the failed entry decodes through the same reader, preserving the
|
||||
/// routing meta. The error_detail is the credential-free summary. The nats_msg_id is the resolved
|
||||
/// dedup id, so an operator sees the id the server saw even for a pre-enable entry.
|
||||
pub(crate) fn encode_failed_entry(
|
||||
mut queued: QueuedPayload,
|
||||
error_class: FailedErrorClass,
|
||||
error: &TargetError,
|
||||
retry_count: u32,
|
||||
resolved_dedup_id: &str,
|
||||
) -> Result<Vec<u8>, TargetError> {
|
||||
let failed_at_unix_ms = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_millis() as u64;
|
||||
queued.meta.failure = Some(FailedEntryMeta {
|
||||
error_class,
|
||||
error_detail: build_failed_error_detail(error),
|
||||
nats_msg_id: resolved_dedup_id.to_string(),
|
||||
failed_at_unix_ms,
|
||||
retry_count,
|
||||
});
|
||||
queued.encode()
|
||||
}
|
||||
|
||||
/// Ensures a rustls crypto provider is installed before any TLS operation.
|
||||
///
|
||||
/// Multiple target modules (MySQL, Redis, Postgres, MQTT) need this because
|
||||
@@ -607,6 +779,97 @@ pub(crate) fn ensure_rustls_provider_installed() {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) mod test_support {
|
||||
use super::{EntityTarget, QueuedPayload, QueuedPayloadMeta};
|
||||
use crate::arn::TargetID;
|
||||
use crate::store::{FailedEventStore, Key, QueueStore, Store};
|
||||
use crate::{StoreError, Target, TargetError};
|
||||
use async_trait::async_trait;
|
||||
use rustfs_s3_types::EventName;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use uuid::Uuid;
|
||||
|
||||
/// A minimal target for failed-store move tests: every delivery method succeeds, the optional
|
||||
/// store backs the store and failed-store accessors, and final failures land on a shared counter.
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct MoveTestTarget {
|
||||
pub(crate) id: TargetID,
|
||||
pub(crate) store: Option<Arc<QueueStore<QueuedPayload>>>,
|
||||
pub(crate) failed: Arc<AtomicU64>,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Target<String> for MoveTestTarget {
|
||||
fn id(&self) -> TargetID {
|
||||
self.id.clone()
|
||||
}
|
||||
async fn is_active(&self) -> Result<bool, TargetError> {
|
||||
Ok(true)
|
||||
}
|
||||
async fn save(&self, _event: Arc<EntityTarget<String>>) -> Result<(), TargetError> {
|
||||
Ok(())
|
||||
}
|
||||
async fn send_raw_from_store(&self, _key: Key, _body: Vec<u8>, _meta: QueuedPayloadMeta) -> Result<(), TargetError> {
|
||||
Ok(())
|
||||
}
|
||||
async fn close(&self) -> Result<(), TargetError> {
|
||||
Ok(())
|
||||
}
|
||||
fn store(&self) -> Option<&(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync)> {
|
||||
self.store
|
||||
.as_deref()
|
||||
.map(|store| store as &(dyn Store<_, Error = StoreError, Key = Key> + Send + Sync))
|
||||
}
|
||||
fn failed_store(&self) -> Option<&dyn FailedEventStore> {
|
||||
self.store.as_deref().map(|store| store as &dyn FailedEventStore)
|
||||
}
|
||||
fn clone_dyn(&self) -> Box<dyn Target<String> + Send + Sync> {
|
||||
Box::new(self.clone())
|
||||
}
|
||||
fn is_enabled(&self) -> bool {
|
||||
true
|
||||
}
|
||||
fn record_final_failure(&self) {
|
||||
self.failed.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn move_test_target() -> Arc<dyn Target<String> + Send + Sync> {
|
||||
Arc::new(MoveTestTarget {
|
||||
id: TargetID::new("target-a".to_string(), "nats".to_string()),
|
||||
store: None,
|
||||
failed: Arc::new(AtomicU64::new(0)),
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn move_test_target_with_store(store: Arc<QueueStore<QueuedPayload>>) -> Arc<dyn Target<String> + Send + Sync> {
|
||||
Arc::new(MoveTestTarget {
|
||||
id: TargetID::new("target-a".to_string(), "nats".to_string()),
|
||||
store: Some(store),
|
||||
failed: Arc::new(AtomicU64::new(0)),
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn failed_store_dir(name: &str) -> PathBuf {
|
||||
std::env::temp_dir().join(format!("rustfs-failed-{name}-{}", Uuid::new_v4()))
|
||||
}
|
||||
|
||||
pub(crate) fn sample_queued(dedup_id: &str) -> QueuedPayload {
|
||||
let mut meta = QueuedPayloadMeta::new(
|
||||
EventName::ObjectCreatedPut,
|
||||
"bucket-a".to_string(),
|
||||
"obj.txt".to_string(),
|
||||
"application/json",
|
||||
7,
|
||||
);
|
||||
meta.dedup_id = dedup_id.to_string();
|
||||
QueuedPayload::new(meta, br#"{"x":1}"#.to_vec())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tls_state_tests {
|
||||
use super::{TargetTlsFingerprintState, TargetTlsGeneration, TargetTlsState};
|
||||
@@ -743,6 +1006,45 @@ mod tests {
|
||||
assert_eq!(ChannelTargetType::Amqp.to_string(), "amqp");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn queued_payload_meta_omits_empty_dedup_id_on_serialization() {
|
||||
let meta = QueuedPayloadMeta::new(
|
||||
EventName::ObjectCreatedPut,
|
||||
"bucket-a".to_string(),
|
||||
"obj.txt".to_string(),
|
||||
"application/json",
|
||||
7,
|
||||
);
|
||||
assert!(meta.dedup_id.is_empty());
|
||||
|
||||
let json = serde_json::to_string(&meta).unwrap();
|
||||
assert!(
|
||||
!json.contains("dedup_id"),
|
||||
"an empty dedup id is skipped so stored bytes match the pre-feature format"
|
||||
);
|
||||
|
||||
// An entry written without the field decodes to an empty dedup id.
|
||||
let decoded: QueuedPayloadMeta = serde_json::from_str(&json).unwrap();
|
||||
assert!(decoded.dedup_id.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn queued_payload_meta_round_trips_a_populated_dedup_id() {
|
||||
let mut meta = QueuedPayloadMeta::new(
|
||||
EventName::ObjectCreatedPut,
|
||||
"bucket-a".to_string(),
|
||||
"obj.txt".to_string(),
|
||||
"application/json",
|
||||
7,
|
||||
);
|
||||
meta.dedup_id = "minted-id".to_string();
|
||||
|
||||
let json = serde_json::to_string(&meta).unwrap();
|
||||
assert!(json.contains("dedup_id"));
|
||||
let decoded: QueuedPayloadMeta = serde_json::from_str(&json).unwrap();
|
||||
assert_eq!(decoded.dedup_id, "minted-id");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn queued_payload_round_trips_meta_and_body() {
|
||||
let body = br#"{"ok":true}"#.to_vec();
|
||||
@@ -1056,4 +1358,96 @@ mod tests {
|
||||
let err = QueuedPayload::decode(&encoded).unwrap_err();
|
||||
assert!(err.to_string().contains("body length mismatch"), "unexpected error: {err}");
|
||||
}
|
||||
|
||||
use super::test_support::sample_queued;
|
||||
|
||||
// The error_detail recorded on a failed entry is built from an allowlist of error categories and
|
||||
// the fixed publish-error vocabulary, never the raw error rendering, so a credential embedded in
|
||||
// a free-form error message is absent from the detail.
|
||||
#[test]
|
||||
fn build_failed_error_detail_redacts_credential_bearing_messages() {
|
||||
let secret = "nats://user:supersecret@broker:4222";
|
||||
let cases = [
|
||||
TargetError::Network(secret.to_string()),
|
||||
TargetError::Request(secret.to_string()),
|
||||
TargetError::Timeout(secret.to_string()),
|
||||
TargetError::Storage(secret.to_string()),
|
||||
TargetError::Authentication(secret.to_string()),
|
||||
TargetError::Configuration(secret.to_string()),
|
||||
TargetError::Dropped(secret.to_string()),
|
||||
TargetError::Unknown(secret.to_string()),
|
||||
];
|
||||
for error in cases {
|
||||
let detail = build_failed_error_detail(&error);
|
||||
assert!(!detail.contains("supersecret"), "detail leaked a credential: {detail}");
|
||||
assert!(!detail.contains("broker:4222"), "detail leaked a connection string: {detail}");
|
||||
}
|
||||
}
|
||||
|
||||
// The publish-error classifier sets the detail from its fixed vocabulary of kind labels and
|
||||
// numeric codes, so the failed-entry detail names the cause without leaking a server-side
|
||||
// message.
|
||||
#[test]
|
||||
fn build_failed_error_detail_uses_the_classified_publish_kind() {
|
||||
let error = TargetError::JetStreamPublish {
|
||||
retryable: false,
|
||||
detail: "max payload exceeded".to_string(),
|
||||
};
|
||||
let detail = build_failed_error_detail(&error);
|
||||
assert_eq!(detail, "jetstream_publish: max payload exceeded");
|
||||
}
|
||||
|
||||
// The persistence-boundary sanitizer replaces a JetStreamPublish detail that carries anything
|
||||
// outside the fixed vocabulary with the fallback label, while a vocabulary detail passes through
|
||||
// verbatim.
|
||||
#[test]
|
||||
fn build_failed_error_detail_sanitizes_a_hostile_jetstream_detail() {
|
||||
let hostile = TargetError::JetStreamPublish {
|
||||
retryable: false,
|
||||
detail: "Boom! nats://user:pass@host/DROP".to_string(),
|
||||
};
|
||||
assert_eq!(build_failed_error_detail(&hostile), "jetstream_publish: unrecognized detail");
|
||||
|
||||
let vocabulary = TargetError::JetStreamPublish {
|
||||
retryable: false,
|
||||
detail: "wrong last sequence".to_string(),
|
||||
};
|
||||
assert_eq!(build_failed_error_detail(&vocabulary), "jetstream_publish: wrong last sequence");
|
||||
}
|
||||
|
||||
// A summary whose cap lands inside a multi-byte character truncates at the preceding character
|
||||
// boundary instead of panicking on a split character. A three-byte character does not divide
|
||||
// the 256-byte cap, so one character straddles the cap by construction.
|
||||
#[test]
|
||||
fn truncate_to_char_boundary_handles_multi_byte_characters_at_the_cap() {
|
||||
let mut value = "\u{4e2d}".repeat(FAILED_ERROR_DETAIL_MAX_LEN);
|
||||
assert!(!value.is_char_boundary(FAILED_ERROR_DETAIL_MAX_LEN), "a character straddles the cap");
|
||||
truncate_to_char_boundary(&mut value, FAILED_ERROR_DETAIL_MAX_LEN);
|
||||
assert!(value.len() <= FAILED_ERROR_DETAIL_MAX_LEN, "the result stays within the cap");
|
||||
assert!(value.is_char_boundary(value.len()), "the result ends on a character boundary");
|
||||
}
|
||||
|
||||
// A terminal move writes a failed entry carrying the full failure meta extension and preserves the
|
||||
// original routing metadata and dedup id through the reused QueuedPayload format.
|
||||
#[test]
|
||||
fn encode_failed_entry_carries_full_failure_meta() {
|
||||
let queued = sample_queued("minted-id");
|
||||
let error = TargetError::JetStreamPublish {
|
||||
retryable: false,
|
||||
detail: "wrong last sequence".to_string(),
|
||||
};
|
||||
let encoded = encode_failed_entry(queued, FailedErrorClass::Terminal, &error, 0, "minted-id").unwrap();
|
||||
let decoded = QueuedPayload::decode(&encoded).unwrap();
|
||||
|
||||
let failure = decoded.meta.failure.expect("a failed entry carries failure meta");
|
||||
assert_eq!(failure.error_class, FailedErrorClass::Terminal);
|
||||
assert_eq!(failure.error_detail, "jetstream_publish: wrong last sequence");
|
||||
assert_eq!(failure.nats_msg_id, "minted-id");
|
||||
assert_eq!(failure.retry_count, 0);
|
||||
assert!(failure.failed_at_unix_ms > 0);
|
||||
// The original routing metadata survives the move.
|
||||
assert_eq!(decoded.meta.bucket_name, "bucket-a");
|
||||
assert_eq!(decoded.meta.object_name, "obj.txt");
|
||||
assert_eq!(decoded.body, br#"{"x":1}"#);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1693,8 +1693,11 @@ where
|
||||
}
|
||||
|
||||
fn delivery_snapshot(&self) -> TargetDeliverySnapshot {
|
||||
self.delivery_counters
|
||||
.snapshot(self.store.as_deref().map_or(0, |store| store.len() as u64))
|
||||
self.delivery_counters.snapshot(
|
||||
self.store.as_deref().map_or(0, |store| store.len() as u64),
|
||||
// MQTT targets record no terminal failures and keep no failed store.
|
||||
0,
|
||||
)
|
||||
}
|
||||
|
||||
fn record_final_failure(&self) {
|
||||
|
||||
@@ -1082,8 +1082,11 @@ where
|
||||
}
|
||||
|
||||
fn delivery_snapshot(&self) -> TargetDeliverySnapshot {
|
||||
self.delivery_counters
|
||||
.snapshot(self.store.as_deref().map_or(0, |store| store.len() as u64))
|
||||
self.delivery_counters.snapshot(
|
||||
self.store.as_deref().map_or(0, |store| store.len() as u64),
|
||||
// MySQL targets record no terminal failures and keep no failed store.
|
||||
0,
|
||||
)
|
||||
}
|
||||
|
||||
fn record_final_failure(&self) {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -21,22 +21,36 @@ use crate::{
|
||||
ReloadableTargetTls, TargetTlsGeneration, TargetTlsInputSet, TlsReloadAdapter, config::ReloadApplyMode,
|
||||
validate_tls_material,
|
||||
},
|
||||
store::{Key, Store},
|
||||
store::{FailedEventStore, Key, QueueStore, Store},
|
||||
target::{
|
||||
ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
|
||||
TargetTlsState, TargetType, build_queued_payload_with_records, build_target_tls_fingerprint, is_connectivity_error,
|
||||
open_target_queue_store, persist_queued_payload_to_store, redacted_secret,
|
||||
open_target_queue_store_typed, persist_queued_payload_to_store, redacted_secret,
|
||||
},
|
||||
};
|
||||
use async_trait::async_trait;
|
||||
use rustfs_config::{NATS_CREDENTIALS_FILE, NATS_TLS_CA, NATS_TLS_CLIENT_CERT, NATS_TLS_CLIENT_KEY};
|
||||
use rustfs_config::{
|
||||
NATS_CREDENTIALS_FILE, NATS_JETSTREAM_ACK_TIMEOUT_DEFAULT_SECS, NATS_TLS_CA, NATS_TLS_CLIENT_CERT, NATS_TLS_CLIENT_KEY,
|
||||
};
|
||||
use std::fmt;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::str::FromStr;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::time::Duration;
|
||||
use tokio::sync::Mutex;
|
||||
use tracing::{info, instrument, warn};
|
||||
use tracing::{error, info, instrument, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
mod jetstream;
|
||||
mod publish_error;
|
||||
mod validation;
|
||||
|
||||
use jetstream::{CachedJetStreamContext, drain_jetstream_context};
|
||||
use publish_error::{classify_nats_flush_error, classify_nats_publish_error};
|
||||
|
||||
pub(crate) use jetstream::resolve_dedup_id;
|
||||
pub(crate) use validation::{validate_jetstream_settings, validate_jetstream_stream};
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct NATSArgs {
|
||||
@@ -53,6 +67,10 @@ pub struct NATSArgs {
|
||||
pub tls_required: bool,
|
||||
pub queue_dir: String,
|
||||
pub queue_limit: u64,
|
||||
// JetStream publish settings. Absent maps to off and the defaults.
|
||||
pub jetstream_enable: Option<bool>,
|
||||
pub jetstream_stream_name: Option<String>,
|
||||
pub jetstream_ack_timeout_secs: Option<u64>,
|
||||
pub target_type: TargetType,
|
||||
}
|
||||
|
||||
@@ -72,6 +90,9 @@ impl fmt::Debug for NATSArgs {
|
||||
.field("tls_required", &self.tls_required)
|
||||
.field("queue_dir", &self.queue_dir)
|
||||
.field("queue_limit", &self.queue_limit)
|
||||
.field("jetstream_enable", &self.jetstream_enable)
|
||||
.field("jetstream_stream_name", &self.jetstream_stream_name)
|
||||
.field("jetstream_ack_timeout_secs", &self.jetstream_ack_timeout_secs)
|
||||
.field("target_type", &self.target_type)
|
||||
.finish()
|
||||
}
|
||||
@@ -114,6 +135,13 @@ impl NATSArgs {
|
||||
return Err(TargetError::Configuration("NATS queue directory must be an absolute path".to_string()));
|
||||
}
|
||||
|
||||
validate_jetstream_settings(
|
||||
self.jetstream_enable.unwrap_or(false),
|
||||
self.jetstream_stream_name.as_deref().unwrap_or_default(),
|
||||
&self.queue_dir,
|
||||
self.jetstream_ack_timeout_secs,
|
||||
)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -160,12 +188,9 @@ fn validate_nats_auth(args: &NATSArgs) -> Result<(), TargetError> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Returns true when the target is configured to send credentials (token,
|
||||
/// username/password, or a credentials file) over a connection that does not
|
||||
/// require TLS, which would transmit the secrets in cleartext (backlog#983).
|
||||
///
|
||||
/// TLS is considered active when `tls_required` is set or the address uses the
|
||||
/// `tls://` scheme.
|
||||
/// Returns true when the target sends credentials over a connection that does not require TLS, which
|
||||
/// would transmit the secrets in cleartext (backlog#983). TLS is active when tls_required is set or
|
||||
/// the address uses the tls:// scheme.
|
||||
fn nats_sends_credentials_without_tls(args: &NATSArgs) -> bool {
|
||||
let has_auth = !args.token.is_empty() || !args.credentials_file.is_empty() || !args.username.is_empty();
|
||||
if !has_auth {
|
||||
@@ -211,12 +236,16 @@ where
|
||||
id: TargetID,
|
||||
args: NATSArgs,
|
||||
client: Arc<Mutex<Option<async_nats::Client>>>,
|
||||
/// Cached JetStream context, one per target shared across clones so a single acker and semaphore
|
||||
/// serve every clone. Read out under the lock before each publish await, never held across it.
|
||||
/// Carries the stream-validation verdict for the bound connection.
|
||||
jetstream_context: Arc<Mutex<Option<CachedJetStreamContext>>>,
|
||||
tls_state: Arc<parking_lot::Mutex<TargetTlsState>>,
|
||||
/// Adapter that bridges this target to the TLS reload coordinator.
|
||||
/// When `Some`, the target uses coordinator-managed material; when `None`,
|
||||
/// it falls back to inline fingerprint-based change detection.
|
||||
/// When set, the coordinator drives TLS reload, otherwise inline fingerprint change detection.
|
||||
tls_adapter: Option<TlsReloadAdapter<async_nats::Client>>,
|
||||
store: Option<Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>>,
|
||||
/// The concrete queue store, held typed so the target projects both the generic Store handle and
|
||||
/// its own failed-events capability from a single store. Shared across clones through the Arc.
|
||||
store: Option<Arc<QueueStore<QueuedPayload>>>,
|
||||
connected: AtomicBool,
|
||||
delivery_counters: Arc<TargetDeliveryCounters>,
|
||||
_phantom: std::marker::PhantomData<E>,
|
||||
@@ -231,9 +260,10 @@ where
|
||||
id: self.id.clone(),
|
||||
args: self.args.clone(),
|
||||
client: Arc::clone(&self.client),
|
||||
jetstream_context: Arc::clone(&self.jetstream_context),
|
||||
tls_state: Arc::clone(&self.tls_state),
|
||||
tls_adapter: self.tls_adapter.clone(),
|
||||
store: self.store.as_ref().map(|s| s.boxed_clone()),
|
||||
store: self.store.clone(),
|
||||
connected: AtomicBool::new(self.connected.load(Ordering::SeqCst)),
|
||||
delivery_counters: Arc::clone(&self.delivery_counters),
|
||||
_phantom: std::marker::PhantomData,
|
||||
@@ -251,19 +281,21 @@ where
|
||||
);
|
||||
}
|
||||
let target_id = TargetID::new(id, ChannelTargetType::Nats.as_str().to_string());
|
||||
let queue_store = open_target_queue_store(
|
||||
let queue_store = open_target_queue_store_typed(
|
||||
&args.queue_dir,
|
||||
args.queue_limit,
|
||||
args.target_type,
|
||||
ChannelTargetType::Nats.as_str(),
|
||||
&target_id,
|
||||
"Failed to open store for NATS target",
|
||||
)?;
|
||||
)?
|
||||
.map(Arc::new);
|
||||
|
||||
Ok(Self {
|
||||
id: target_id,
|
||||
args,
|
||||
client: Arc::new(Mutex::new(None)),
|
||||
jetstream_context: Arc::new(Mutex::new(None)),
|
||||
tls_state: Arc::new(parking_lot::Mutex::new(TargetTlsState::default())),
|
||||
tls_adapter: None,
|
||||
store: queue_store,
|
||||
@@ -299,7 +331,6 @@ where
|
||||
};
|
||||
if tls_changed {
|
||||
self.invalidate_cached_client_connection().await;
|
||||
self.tls_state.lock().refresh(next_fingerprint);
|
||||
}
|
||||
|
||||
{
|
||||
@@ -316,13 +347,53 @@ where
|
||||
.map_err(|e| TargetError::Network(format!("Failed to flush NATS connection: {e}")))?;
|
||||
self.connected.store(true, Ordering::SeqCst);
|
||||
|
||||
let mut guard = self.client.lock().await;
|
||||
let shared = guard.get_or_insert_with(|| client.clone()).clone();
|
||||
let mut client_guard = self.client.lock().await;
|
||||
let shared = client_guard.get_or_insert_with(|| client.clone()).clone();
|
||||
|
||||
// Swap in a context built from the winning client while the client lock is held, so client
|
||||
// and context swap as a unit and no clone reads a context bound to the old client. Only the
|
||||
// JetStream path caches a context. The previous context is drained after the locks release.
|
||||
let previous_context = if tls_changed && self.jetstream_enabled() {
|
||||
let ack_timeout = self.ack_timeout();
|
||||
let mut context_guard = self.jetstream_context.lock().await;
|
||||
let mut context = async_nats::jetstream::new(shared.clone());
|
||||
context.set_timeout(ack_timeout);
|
||||
context_guard.replace(CachedJetStreamContext::new(context))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
drop(client_guard);
|
||||
|
||||
// Advance the recorded fingerprint only after the reconnect and flush succeed, so a rotation
|
||||
// whose first reconnect fails stays detected and the next success still rebuilds the context.
|
||||
if tls_changed {
|
||||
self.tls_state.lock().refresh(next_fingerprint);
|
||||
}
|
||||
|
||||
drain_jetstream_context(previous_context.map(|cached| cached.context)).await;
|
||||
Ok(shared)
|
||||
}
|
||||
|
||||
fn jetstream_enabled(&self) -> bool {
|
||||
self.args.jetstream_enable.unwrap_or(false)
|
||||
}
|
||||
|
||||
fn ack_timeout(&self) -> Duration {
|
||||
Duration::from_secs(
|
||||
self.args
|
||||
.jetstream_ack_timeout_secs
|
||||
.unwrap_or(NATS_JETSTREAM_ACK_TIMEOUT_DEFAULT_SECS),
|
||||
)
|
||||
}
|
||||
|
||||
fn build_queued_payload(&self, event: &EntityTarget<E>) -> Result<QueuedPayload, TargetError> {
|
||||
build_queued_payload_with_records(event, vec![event.clone()])
|
||||
let mut queued = build_queued_payload_with_records(event, vec![event.clone()])?;
|
||||
if self.jetstream_enabled() {
|
||||
// Mint the dedup id once at enqueue so it is identical across every retry and replay and
|
||||
// distinct across entries. The body-hash fallback only covers entries queued before enable.
|
||||
queued.meta.dedup_id = Uuid::new_v4().to_string();
|
||||
}
|
||||
Ok(queued)
|
||||
}
|
||||
|
||||
async fn send_body(&self, body: Vec<u8>) -> Result<(), TargetError> {
|
||||
@@ -336,10 +407,8 @@ where
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
// `publish` only enqueues the message on the client's outbound channel;
|
||||
// it does not guarantee the broker received it. Flush to confirm the
|
||||
// message reached the server before the caller treats delivery as
|
||||
// successful and deletes the durable copy (backlog#971).
|
||||
// publish only enqueues the message on the client's outbound channel. Flush to confirm the
|
||||
// message reached the server before delivery is treated as successful (backlog#971).
|
||||
if let Err(e) = client.flush().await {
|
||||
let err = classify_nats_flush_error(&e);
|
||||
self.invalidate_cached_client_connection().await;
|
||||
@@ -352,30 +421,6 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
/// Classifies a NATS publish failure using the typed error kind rather than a
|
||||
/// substring match. `Send` means the outbound channel is closed (the connection
|
||||
/// is gone) and is retriable; payload/subject violations are permanent
|
||||
/// request-level errors (backlog#971, backlog#973).
|
||||
fn classify_nats_publish_error(err: &async_nats::PublishError) -> TargetError {
|
||||
use async_nats::PublishErrorKind;
|
||||
match err.kind() {
|
||||
PublishErrorKind::Send => TargetError::NotConnected,
|
||||
PublishErrorKind::MaxPayloadExceeded | PublishErrorKind::InvalidSubject => {
|
||||
TargetError::Request(format!("Failed to publish NATS message: {err}"))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Classifies a NATS flush failure. Both flush error kinds indicate the message
|
||||
/// was not confirmed on the wire (a connectivity problem), so the event is kept
|
||||
/// for replay (backlog#971, backlog#973).
|
||||
fn classify_nats_flush_error(err: &async_nats::client::FlushError) -> TargetError {
|
||||
use async_nats::client::FlushErrorKind;
|
||||
match err.kind() {
|
||||
FlushErrorKind::SendError | FlushErrorKind::FlushError => TargetError::NotConnected,
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl<E> Target<E> for NATSTarget<E>
|
||||
where
|
||||
@@ -386,6 +431,13 @@ where
|
||||
}
|
||||
|
||||
async fn is_active(&self) -> Result<bool, TargetError> {
|
||||
// With JetStream enabled the health answer covers the stream as well as the connection, so a
|
||||
// reachable broker with a failing stream reports the validation error. The verdict is cached
|
||||
// and reset on a reconnect, TLS rotation, wrong-stream ack, or stream-not-found outcome, so a
|
||||
// reset forces a live lookup on the next check.
|
||||
if self.jetstream_enabled() {
|
||||
self.validated_jetstream_context().await?;
|
||||
}
|
||||
let client = self.get_or_connect().await?;
|
||||
client
|
||||
.flush()
|
||||
@@ -418,11 +470,31 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
async fn send_raw_from_store(&self, _key: Key, body: Vec<u8>, _meta: QueuedPayloadMeta) -> Result<(), TargetError> {
|
||||
self.send_body(body).await
|
||||
async fn send_raw_from_store(&self, key: Key, body: Vec<u8>, meta: QueuedPayloadMeta) -> Result<(), TargetError> {
|
||||
if self.jetstream_enabled() {
|
||||
let dedup_id = resolve_dedup_id(&meta.dedup_id, &key);
|
||||
self.publish_jetstream(body, &dedup_id).await
|
||||
} else {
|
||||
self.send_body(body).await
|
||||
}
|
||||
}
|
||||
|
||||
async fn close(&self) -> Result<(), TargetError> {
|
||||
// Drain the cached context's in-flight acks before dropping it so the async-nats acker exits.
|
||||
// Drain the context before the client, since the acker rides the client connection.
|
||||
let context = {
|
||||
let mut guard = self.jetstream_context.lock().await;
|
||||
guard.take()
|
||||
};
|
||||
// Bound the drain by the ack timeout so an unresponsive broker cannot stall close. On elapse
|
||||
// the client drain below still runs.
|
||||
if tokio::time::timeout(self.ack_timeout(), drain_jetstream_context(context.map(|cached| cached.context)))
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
warn!(target_id = %self.id, "Timed out draining JetStream acks on close, proceeding to close the client");
|
||||
}
|
||||
|
||||
let client = {
|
||||
let mut guard = self.client.lock().await;
|
||||
guard.take()
|
||||
@@ -435,12 +507,49 @@ where
|
||||
.await
|
||||
.map_err(|e| TargetError::Network(format!("Failed to drain NATS client: {e}")))?;
|
||||
}
|
||||
// The durable queue store stays on disk, so a queued entry survives close and replays. Close
|
||||
// releases the cached handles, never the entries.
|
||||
info!(target_id = %self.id, "NATS target closed");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn store(&self) -> Option<&(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync)> {
|
||||
self.store.as_deref()
|
||||
self.store
|
||||
.as_deref()
|
||||
.map(|store| store as &(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync))
|
||||
}
|
||||
|
||||
fn failed_store(&self) -> Option<&dyn FailedEventStore> {
|
||||
self.store.as_deref().map(|store| store as &dyn FailedEventStore)
|
||||
}
|
||||
|
||||
async fn handle_terminal_failure(
|
||||
&self,
|
||||
store: &(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send),
|
||||
key: &Key,
|
||||
error: &TargetError,
|
||||
retry_count: u32,
|
||||
) -> bool {
|
||||
let Some(failed_store) = self.failed_store() else {
|
||||
error!(
|
||||
target_id = %self.id,
|
||||
replay_key = %key,
|
||||
"NATS target has no failed-events store for the terminal move"
|
||||
);
|
||||
return false;
|
||||
};
|
||||
match jetstream::move_entry_to_failed_store(store, failed_store, &self.id, key, error, retry_count).await {
|
||||
Ok(()) => true,
|
||||
Err(move_err) => {
|
||||
error!(
|
||||
target_id = %self.id,
|
||||
error = %move_err,
|
||||
replay_key = %key,
|
||||
"Failed to move event to the failed-events store"
|
||||
);
|
||||
false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn clone_dyn(&self) -> Box<dyn Target<E> + Send + Sync> {
|
||||
@@ -452,6 +561,13 @@ where
|
||||
return Ok(());
|
||||
}
|
||||
let _ = self.get_or_connect().await?;
|
||||
if self.jetstream_enabled() {
|
||||
let cached = self.jetstream_context().await?;
|
||||
validate_jetstream_stream(&cached.context, &self.args, &self.id.to_string(), Some(&cached.validation_logged)).await?;
|
||||
// Record the verdict on the context so the first publish does not repeat the stream
|
||||
// lookup that just passed.
|
||||
cached.stream_validated.store(true, Ordering::Release);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -460,8 +576,10 @@ where
|
||||
}
|
||||
|
||||
fn delivery_snapshot(&self) -> TargetDeliverySnapshot {
|
||||
self.delivery_counters
|
||||
.snapshot(self.store.as_deref().map_or(0, |store| store.len() as u64))
|
||||
self.delivery_counters.snapshot(
|
||||
self.store.as_deref().map_or(0, |store| store.len() as u64),
|
||||
self.failed_store().map_or(0, |failed_store| failed_store.failed_len() as u64),
|
||||
)
|
||||
}
|
||||
|
||||
fn record_final_failure(&self) {
|
||||
@@ -510,11 +628,18 @@ where
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
pub(crate) mod test_support {
|
||||
use super::*;
|
||||
use crate::target::REDACTED_SECRET;
|
||||
use async_nats::jetstream;
|
||||
use rustfs_s3_types::EventName;
|
||||
|
||||
fn base_args() -> NATSArgs {
|
||||
// Absolute on Linux, macOS, and Windows. temp_dir needs no filesystem to exist for a
|
||||
// validation-only test, and Path::is_absolute stays true across platforms.
|
||||
pub(crate) fn nats_queue_dir() -> String {
|
||||
std::env::temp_dir().join("rustfs-nats-queue").to_string_lossy().into_owned()
|
||||
}
|
||||
|
||||
pub(crate) fn base_args() -> NATSArgs {
|
||||
NATSArgs {
|
||||
enable: true,
|
||||
address: "nats://127.0.0.1:4222".to_string(),
|
||||
@@ -529,10 +654,123 @@ mod tests {
|
||||
tls_required: false,
|
||||
queue_dir: String::new(),
|
||||
queue_limit: 0,
|
||||
jetstream_enable: None,
|
||||
jetstream_stream_name: None,
|
||||
jetstream_ack_timeout_secs: None,
|
||||
target_type: TargetType::NotifyEvent,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn jetstream_args(target_type: TargetType) -> NATSArgs {
|
||||
NATSArgs {
|
||||
jetstream_enable: Some(true),
|
||||
jetstream_stream_name: Some("RUSTFS_EVENTS".to_string()),
|
||||
jetstream_ack_timeout_secs: Some(30),
|
||||
target_type,
|
||||
..base_args()
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn nats_target(args: NATSArgs) -> NATSTarget<String> {
|
||||
NATSTarget::<String> {
|
||||
id: TargetID::new("test-target".to_string(), ChannelTargetType::Nats.as_str().to_string()),
|
||||
args,
|
||||
client: Arc::new(Mutex::new(None)),
|
||||
jetstream_context: Arc::new(Mutex::new(None)),
|
||||
tls_state: Arc::new(parking_lot::Mutex::new(TargetTlsState::default())),
|
||||
tls_adapter: None,
|
||||
store: None,
|
||||
connected: AtomicBool::new(false),
|
||||
delivery_counters: Arc::new(TargetDeliveryCounters::default()),
|
||||
_phantom: std::marker::PhantomData,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn sample_event() -> EntityTarget<String> {
|
||||
EntityTarget {
|
||||
object_name: "folder/object.txt".to_string(),
|
||||
bucket_name: "bucket-a".to_string(),
|
||||
event_name: EventName::ObjectCreatedPut,
|
||||
data: "payload-data".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn sample_stored_key(name: &str) -> Key {
|
||||
Key {
|
||||
name: name.to_string(),
|
||||
extension: ".event".to_string(),
|
||||
item_count: 1,
|
||||
compress: false,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn nats_target_with_store(args: NATSArgs, queue_dir: &str) -> NATSTarget<String> {
|
||||
let mut configured = args;
|
||||
configured.queue_dir = queue_dir.to_string();
|
||||
NATSTarget::<String>::new("store-target".to_string(), configured).expect("target with store builds")
|
||||
}
|
||||
|
||||
/// A stream configuration that passes every assertion: it captures the configured subject, returns
|
||||
/// acks, accepts writes, and deduplicates well beyond the retry lifetime.
|
||||
pub(crate) fn writable_stream_config(subject: &str) -> jetstream::stream::Config {
|
||||
jetstream::stream::Config {
|
||||
name: "RUSTFS_EVENTS".to_string(),
|
||||
subjects: vec![subject.to_string()],
|
||||
no_ack: false,
|
||||
sealed: false,
|
||||
duplicate_window: Duration::from_secs(600),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
// Live-broker behaviour tests. They require a NATS server with JetStream enabled and are ignored by
|
||||
// default. To run locally:
|
||||
//
|
||||
// docker run -d --name rustfs-nats-test -p 4222:4222 nats:2 -js
|
||||
// cargo test -p rustfs-targets --lib -- --ignored nats::tests::tls_change
|
||||
//
|
||||
// Override the server URL with RUSTFS_TEST_NATS_URL.
|
||||
pub(crate) fn broker_url() -> String {
|
||||
std::env::var("RUSTFS_TEST_NATS_URL").unwrap_or_else(|_| "nats://127.0.0.1:4222".to_string())
|
||||
}
|
||||
|
||||
/// Log sink for asserting a specific line is written. The subscriber writes into a shared
|
||||
/// buffer the assertion reads back.
|
||||
#[derive(Clone, Default)]
|
||||
pub(crate) struct CapturedLog(Arc<parking_lot::Mutex<Vec<u8>>>);
|
||||
|
||||
impl CapturedLog {
|
||||
pub(crate) fn contents(&self) -> String {
|
||||
String::from_utf8_lossy(&self.0.lock()).into_owned()
|
||||
}
|
||||
}
|
||||
|
||||
impl std::io::Write for CapturedLog {
|
||||
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
|
||||
self.0.lock().extend_from_slice(buf);
|
||||
Ok(buf.len())
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> std::io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for CapturedLog {
|
||||
type Writer = CapturedLog;
|
||||
|
||||
fn make_writer(&'a self) -> Self::Writer {
|
||||
self.clone()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::target::REDACTED_SECRET;
|
||||
use crate::target::nats::test_support::*;
|
||||
|
||||
#[test]
|
||||
fn debug_redacts_nats_secret_fields() {
|
||||
let args = NATSArgs {
|
||||
@@ -575,8 +813,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn nats_credentials_without_tls_is_detected() {
|
||||
// Token auth over a plaintext nats:// address without tls_required leaks
|
||||
// the credential in cleartext (backlog#983).
|
||||
// Token auth over a plaintext nats:// address without tls_required leaks the credential (backlog#983).
|
||||
let insecure = NATSArgs {
|
||||
token: "secret-token".to_string(),
|
||||
tls_required: false,
|
||||
@@ -603,25 +840,17 @@ mod tests {
|
||||
assert!(!nats_sends_credentials_without_tls(&no_auth));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nats_publish_error_classification() {
|
||||
use async_nats::PublishErrorKind;
|
||||
// `Send` means the outbound channel is gone: retriable connectivity error.
|
||||
let send_err = async_nats::PublishError::new(PublishErrorKind::Send);
|
||||
assert!(matches!(classify_nats_publish_error(&send_err), TargetError::NotConnected));
|
||||
#[tokio::test]
|
||||
async fn flag_off_stored_meta_is_byte_identical_to_pre_feature() {
|
||||
// With the flag off the payload carries no dedup id, so its encoded meta matches a pre-feature entry.
|
||||
let disabled = nats_target(base_args());
|
||||
let off_payload = disabled.build_queued_payload(&sample_event()).expect("payload builds");
|
||||
assert!(off_payload.meta.dedup_id.is_empty(), "no dedup id is minted with the flag off");
|
||||
|
||||
// Payload/subject violations are permanent request-level errors.
|
||||
let payload_err = async_nats::PublishError::new(PublishErrorKind::MaxPayloadExceeded);
|
||||
assert!(matches!(classify_nats_publish_error(&payload_err), TargetError::Request(_)));
|
||||
let subject_err = async_nats::PublishError::new(PublishErrorKind::InvalidSubject);
|
||||
assert!(matches!(classify_nats_publish_error(&subject_err), TargetError::Request(_)));
|
||||
}
|
||||
let meta_json = serde_json::to_string(&off_payload.meta).expect("meta serializes");
|
||||
assert!(!meta_json.contains("dedup_id"), "the absent dedup id is skipped on serialization");
|
||||
|
||||
#[test]
|
||||
fn nats_flush_error_classification() {
|
||||
use async_nats::client::{FlushError, FlushErrorKind};
|
||||
for kind in [FlushErrorKind::SendError, FlushErrorKind::FlushError] {
|
||||
assert!(matches!(classify_nats_flush_error(&FlushError::new(kind)), TargetError::NotConnected));
|
||||
}
|
||||
// No JetStream context is built on the flag-off path.
|
||||
assert!(disabled.jetstream_context.lock().await.is_none(), "no context is built with the flag off");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,346 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::jetstream::STREAM_NOT_FOUND_DETAIL;
|
||||
use crate::error::TargetError;
|
||||
use async_nats::jetstream::context::PublishErrorKind;
|
||||
|
||||
/// Maps an async-nats publish error to a typed TargetError with the retryable-versus-terminal
|
||||
/// classification, so the replay loop retries recoverable errors instead of dropping them. The
|
||||
/// detail comes from the fixed vocabulary in publish_error_detail, never from raw error text.
|
||||
pub(crate) fn classify_publish_error(err: &async_nats::jetstream::context::PublishError) -> TargetError {
|
||||
let retryable = match err.kind() {
|
||||
PublishErrorKind::TimedOut
|
||||
| PublishErrorKind::BrokenPipe
|
||||
| PublishErrorKind::MaxAckPending
|
||||
| PublishErrorKind::StreamNotFound => true,
|
||||
PublishErrorKind::MaxPayloadExceeded | PublishErrorKind::WrongLastMessageId | PublishErrorKind::WrongLastSequence => {
|
||||
false
|
||||
}
|
||||
PublishErrorKind::Other => !other_error_is_terminal(err),
|
||||
};
|
||||
TargetError::JetStreamPublish {
|
||||
retryable,
|
||||
detail: publish_error_detail(err),
|
||||
}
|
||||
}
|
||||
|
||||
/// Fixed diagnostic detail for a publish failure. Each error kind maps to a constant label, and a
|
||||
/// server rejection carries the numeric JetStream error code and status. Never the library rendering
|
||||
/// or any server-returned text, so the detail is safe to persist in a failed-store entry.
|
||||
fn publish_error_detail(err: &async_nats::jetstream::context::PublishError) -> String {
|
||||
use async_nats::jetstream::Error as JetStreamApiError;
|
||||
match err.kind() {
|
||||
PublishErrorKind::StreamNotFound => STREAM_NOT_FOUND_DETAIL.to_string(),
|
||||
PublishErrorKind::TimedOut => "publish timed out".to_string(),
|
||||
PublishErrorKind::BrokenPipe => "broken pipe".to_string(),
|
||||
PublishErrorKind::MaxAckPending => "max ack pending reached".to_string(),
|
||||
PublishErrorKind::MaxPayloadExceeded => "max payload exceeded".to_string(),
|
||||
PublishErrorKind::WrongLastMessageId => "wrong last message id".to_string(),
|
||||
PublishErrorKind::WrongLastSequence => "wrong last sequence".to_string(),
|
||||
PublishErrorKind::Other => {
|
||||
match std::error::Error::source(err).and_then(|source| source.downcast_ref::<JetStreamApiError>()) {
|
||||
Some(api_error) => format!("server error code {} status {}", api_error.error_code().0, api_error.code()),
|
||||
None => "publish failed".to_string(),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// HTTP-style 5xx status range. A rejection reporting a 5xx status is a server-side condition and
|
||||
/// stays retryable regardless of its error code.
|
||||
const SERVER_ERROR_STATUS_RANGE: std::ops::Range<usize> = 500..600;
|
||||
|
||||
/// Classifies an Other publish error against a terminal allowlist. Terminal only for an explicit
|
||||
/// permanent error code, STREAM_SEALED being the sole current member. Everything else is retryable,
|
||||
/// so only a recognized permanent rejection enters the failed store.
|
||||
fn other_error_is_terminal(err: &async_nats::jetstream::context::PublishError) -> bool {
|
||||
use async_nats::jetstream::{Error as JetStreamApiError, ErrorCode};
|
||||
|
||||
let Some(api_error) = std::error::Error::source(err).and_then(|source| source.downcast_ref::<JetStreamApiError>()) else {
|
||||
return false;
|
||||
};
|
||||
|
||||
// A 5xx status is a server-side condition, so it stays retryable even when the error code is a
|
||||
// permanent one.
|
||||
if SERVER_ERROR_STATUS_RANGE.contains(&api_error.code()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
matches!(api_error.error_code(), ErrorCode::STREAM_SEALED)
|
||||
}
|
||||
|
||||
/// Classifies a NATS publish failure by its typed error kind. Send means the outbound channel is
|
||||
/// closed and is retriable. Payload and subject violations are permanent request-level errors
|
||||
/// (backlog#971, backlog#973).
|
||||
pub(crate) fn classify_nats_publish_error(err: &async_nats::PublishError) -> TargetError {
|
||||
use async_nats::PublishErrorKind;
|
||||
match err.kind() {
|
||||
PublishErrorKind::Send => TargetError::NotConnected,
|
||||
PublishErrorKind::MaxPayloadExceeded | PublishErrorKind::InvalidSubject => {
|
||||
TargetError::Request(format!("Failed to publish NATS message: {err}"))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Classifies a NATS flush failure. Both kinds mean the message was not confirmed on the wire, so
|
||||
/// the event is kept for replay (backlog#971, backlog#973).
|
||||
pub(crate) fn classify_nats_flush_error(err: &async_nats::client::FlushError) -> TargetError {
|
||||
use async_nats::client::FlushErrorKind;
|
||||
match err.kind() {
|
||||
FlushErrorKind::SendError | FlushErrorKind::FlushError => TargetError::NotConnected,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::target::nats::jetstream::{ACK_STREAM_MISMATCH_DETAIL, ATTEMPT_DEADLINE_DETAIL};
|
||||
use crate::target::nats::validation::gate_publish_on_stream_validation;
|
||||
use async_nats::jetstream::context::{PublishError, PublishErrorKind};
|
||||
use std::sync::atomic::AtomicBool;
|
||||
|
||||
#[test]
|
||||
fn nats_publish_error_classification() {
|
||||
use async_nats::PublishErrorKind;
|
||||
// Send means the outbound channel is gone: retriable connectivity error.
|
||||
let send_err = async_nats::PublishError::new(PublishErrorKind::Send);
|
||||
assert!(matches!(classify_nats_publish_error(&send_err), TargetError::NotConnected));
|
||||
|
||||
// Payload and subject violations are permanent request-level errors.
|
||||
let payload_err = async_nats::PublishError::new(PublishErrorKind::MaxPayloadExceeded);
|
||||
assert!(matches!(classify_nats_publish_error(&payload_err), TargetError::Request(_)));
|
||||
let subject_err = async_nats::PublishError::new(PublishErrorKind::InvalidSubject);
|
||||
assert!(matches!(classify_nats_publish_error(&subject_err), TargetError::Request(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nats_flush_error_classification() {
|
||||
use async_nats::client::{FlushError, FlushErrorKind};
|
||||
for kind in [FlushErrorKind::SendError, FlushErrorKind::FlushError] {
|
||||
assert!(matches!(classify_nats_flush_error(&FlushError::new(kind)), TargetError::NotConnected));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classify_publish_error_marks_recoverable_kinds_retryable() {
|
||||
for kind in [
|
||||
PublishErrorKind::TimedOut,
|
||||
PublishErrorKind::BrokenPipe,
|
||||
PublishErrorKind::MaxAckPending,
|
||||
PublishErrorKind::StreamNotFound,
|
||||
] {
|
||||
let classified = classify_publish_error(&PublishError::new(kind));
|
||||
match classified {
|
||||
TargetError::JetStreamPublish { retryable, .. } => assert!(retryable, "{kind} is retryable"),
|
||||
other => panic!("expected JetStreamPublish, got {other:?}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classify_publish_error_marks_terminal_kinds_terminal() {
|
||||
for kind in [
|
||||
PublishErrorKind::MaxPayloadExceeded,
|
||||
PublishErrorKind::WrongLastMessageId,
|
||||
PublishErrorKind::WrongLastSequence,
|
||||
] {
|
||||
let classified = classify_publish_error(&PublishError::new(kind));
|
||||
match classified {
|
||||
TargetError::JetStreamPublish { retryable, .. } => assert!(!retryable, "{kind} is terminal"),
|
||||
other => panic!("expected JetStreamPublish, got {other:?}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classify_publish_error_other_without_source_is_retryable() {
|
||||
// An undecodable Other carries no code to recognize as permanent, so it stays on the retry path.
|
||||
let classified = classify_publish_error(&PublishError::new(PublishErrorKind::Other));
|
||||
match classified {
|
||||
TargetError::JetStreamPublish { retryable, .. } => assert!(retryable, "an undecodable Other is retryable"),
|
||||
other => panic!("expected JetStreamPublish, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classify_publish_error_unknown_code_is_retryable() {
|
||||
// A decoded rejection carrying a code outside the terminal allowlist is retryable.
|
||||
let api_error: async_nats::jetstream::Error =
|
||||
serde_json::from_value(serde_json::json!({"code": 400, "err_code": 10999, "description": "an unrecognized code"}))
|
||||
.expect("api error deserializes");
|
||||
let publish_error = PublishError::with_source(PublishErrorKind::Other, api_error);
|
||||
match classify_publish_error(&publish_error) {
|
||||
TargetError::JetStreamPublish { retryable, .. } => assert!(retryable, "an unknown error code is retryable"),
|
||||
other => panic!("expected JetStreamPublish, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classify_publish_error_other_with_transient_code_is_retryable() {
|
||||
// Server conditions that each clear on their own. None is in the terminal allowlist, so each
|
||||
// is retryable. The status is 400 in every case, so retryability comes from the allowlist
|
||||
// policy rather than the 5xx status range.
|
||||
for (err_code, condition) in [
|
||||
(10002, "account resources exceeded"),
|
||||
(10008, "cluster not available"),
|
||||
(10009, "cluster not leader"),
|
||||
(10028, "memory resources exceeded"),
|
||||
(10039, "jetstream not enabled for account"),
|
||||
(10040, "cluster peer not a member"),
|
||||
(10041, "raft general error"),
|
||||
(10076, "jetstream not enabled"),
|
||||
(10118, "stream offline"),
|
||||
(10194, "stream offline with a reason"),
|
||||
(10202, "server member change in flight"),
|
||||
] {
|
||||
let api_error: async_nats::jetstream::Error =
|
||||
serde_json::from_value(serde_json::json!({"code": 400, "err_code": err_code, "description": condition}))
|
||||
.expect("api error deserializes");
|
||||
let publish_error = PublishError::with_source(PublishErrorKind::Other, api_error);
|
||||
let classified = classify_publish_error(&publish_error);
|
||||
match classified {
|
||||
TargetError::JetStreamPublish { retryable, .. } => {
|
||||
assert!(retryable, "transient code {err_code} ({condition}) is retryable")
|
||||
}
|
||||
other => panic!("expected JetStreamPublish for code {err_code}, got {other:?}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classify_publish_error_uncoded_server_error_status_is_retryable() {
|
||||
// A rejection with no err_code and a 5xx status is a server-side condition, kept on the retry path.
|
||||
for status in [500, 503] {
|
||||
let api_error: async_nats::jetstream::Error =
|
||||
serde_json::from_value(serde_json::json!({"code": status, "description": "insufficient resources"}))
|
||||
.expect("api error deserializes");
|
||||
let publish_error = PublishError::with_source(PublishErrorKind::Other, api_error);
|
||||
let classified = classify_publish_error(&publish_error);
|
||||
match classified {
|
||||
TargetError::JetStreamPublish { retryable, .. } => {
|
||||
assert!(retryable, "an uncoded {status} status is retryable")
|
||||
}
|
||||
other => panic!("expected JetStreamPublish for status {status}, got {other:?}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classify_publish_error_other_with_terminal_code_is_terminal() {
|
||||
// STREAM_SEALED (10109) is the sole terminal-allowlist member and its 400 status keeps it terminal.
|
||||
let api_error: async_nats::jetstream::Error =
|
||||
serde_json::from_value(serde_json::json!({"code": 400, "err_code": 10109, "description": "stream sealed"}))
|
||||
.expect("api error deserializes");
|
||||
let publish_error = PublishError::with_source(PublishErrorKind::Other, api_error);
|
||||
match classify_publish_error(&publish_error) {
|
||||
TargetError::JetStreamPublish { retryable, .. } => assert!(!retryable, "a sealed stream is terminal"),
|
||||
other => panic!("expected JetStreamPublish, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classify_publish_error_uncoded_4xx_is_retryable() {
|
||||
// An uncoded 4xx rejection carries no permanent code, so the allowlist policy leaves it on the retry path.
|
||||
let api_error: async_nats::jetstream::Error =
|
||||
serde_json::from_value(serde_json::json!({"code": 400, "description": "bad request without an error code"}))
|
||||
.expect("api error deserializes");
|
||||
let publish_error = PublishError::with_source(PublishErrorKind::Other, api_error);
|
||||
match classify_publish_error(&publish_error) {
|
||||
TargetError::JetStreamPublish { retryable, .. } => assert!(retryable, "an uncoded 4xx rejection is retryable"),
|
||||
other => panic!("expected JetStreamPublish, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classify_publish_error_terminal_code_with_server_error_status_is_retryable() {
|
||||
// Deliberate policy: a 5xx status classifies retryable even when the error code is terminal.
|
||||
let api_error: async_nats::jetstream::Error =
|
||||
serde_json::from_value(serde_json::json!({"code": 503, "err_code": 10109, "description": "stream sealed"}))
|
||||
.expect("api error deserializes");
|
||||
let publish_error = PublishError::with_source(PublishErrorKind::Other, api_error);
|
||||
let classified = classify_publish_error(&publish_error);
|
||||
match classified {
|
||||
TargetError::JetStreamPublish { retryable, .. } => {
|
||||
assert!(retryable, "a 5xx status keeps the STREAM_SEALED code on the retry path")
|
||||
}
|
||||
other => panic!("expected JetStreamPublish, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classify_publish_error_detail_omits_server_body() {
|
||||
// A server rejection contributes its numeric code and status, never the server-returned description.
|
||||
let api_error: async_nats::jetstream::Error = serde_json::from_value(
|
||||
serde_json::json!({"code": 503, "err_code": 10008, "description": "secret-token-abc123 in description"}),
|
||||
)
|
||||
.expect("api error deserializes");
|
||||
let classified = classify_publish_error(&PublishError::with_source(PublishErrorKind::Other, api_error));
|
||||
match classified {
|
||||
TargetError::JetStreamPublish { detail, .. } => {
|
||||
assert_eq!(detail, "server error code 10008 status 503");
|
||||
}
|
||||
other => panic!("expected JetStreamPublish, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classified_publish_details_use_the_fixed_vocabulary() {
|
||||
// Every classification path draws its detail from fixed labels and numeric fields. The pattern
|
||||
// (lowercase letters, digits, spaces, underscores, colons) rejects raw error text.
|
||||
let is_fixed = |detail: &str| {
|
||||
!detail.is_empty()
|
||||
&& detail
|
||||
.chars()
|
||||
.all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || matches!(c, ' ' | '_' | ':'))
|
||||
};
|
||||
|
||||
let mut details = vec![ATTEMPT_DEADLINE_DETAIL.to_string(), ACK_STREAM_MISMATCH_DETAIL.to_string()];
|
||||
for kind in [
|
||||
PublishErrorKind::StreamNotFound,
|
||||
PublishErrorKind::TimedOut,
|
||||
PublishErrorKind::BrokenPipe,
|
||||
PublishErrorKind::MaxAckPending,
|
||||
PublishErrorKind::MaxPayloadExceeded,
|
||||
PublishErrorKind::WrongLastMessageId,
|
||||
PublishErrorKind::WrongLastSequence,
|
||||
PublishErrorKind::Other,
|
||||
] {
|
||||
match classify_publish_error(&PublishError::new(kind)) {
|
||||
TargetError::JetStreamPublish { detail, .. } => details.push(detail),
|
||||
other => panic!("expected JetStreamPublish, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
let api_error: async_nats::jetstream::Error =
|
||||
serde_json::from_value(serde_json::json!({"code": 503, "err_code": 10077, "description": "raw server text"}))
|
||||
.expect("api error deserializes");
|
||||
match classify_publish_error(&PublishError::with_source(PublishErrorKind::Other, api_error)) {
|
||||
TargetError::JetStreamPublish { detail, .. } => details.push(detail),
|
||||
other => panic!("expected JetStreamPublish, got {other:?}"),
|
||||
}
|
||||
|
||||
// The validation gate detail must stay fixed even when the verdict error carries text outside the vocabulary.
|
||||
let stream_validated = AtomicBool::new(false);
|
||||
let verdict = Err(TargetError::Configuration("Raw Display TEXT with (punctuation)".to_string()));
|
||||
match gate_publish_on_stream_validation(verdict, &stream_validated) {
|
||||
Err(TargetError::JetStreamPublish { detail, .. }) => details.push(detail),
|
||||
other => panic!("expected a JetStreamPublish gate error, got {other:?}"),
|
||||
}
|
||||
|
||||
for detail in details {
|
||||
assert!(is_fixed(&detail), "detail falls outside the fixed vocabulary: {detail:?}");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,405 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::NATSArgs;
|
||||
use super::jetstream::retry_lifetime;
|
||||
use crate::error::TargetError;
|
||||
use async_nats::jetstream;
|
||||
use rustfs_config::{
|
||||
NATS_JETSTREAM_ACK_TIMEOUT_DEFAULT_SECS, NATS_JETSTREAM_ACK_TIMEOUT_MAX_SECS, NATS_JETSTREAM_ACK_TIMEOUT_MIN_SECS,
|
||||
NATS_JETSTREAM_ACK_TIMEOUT_SECS, NATS_JETSTREAM_ENABLE, NATS_JETSTREAM_STREAM_NAME, NATS_QUEUE_DIR,
|
||||
};
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::time::Duration;
|
||||
use tracing::{debug, error, info};
|
||||
|
||||
/// Validates the JetStream publish settings shared by the config-time and connect-time validators,
|
||||
/// so both reject the same combinations.
|
||||
pub(crate) fn validate_jetstream_settings(
|
||||
enable: bool,
|
||||
stream_name: &str,
|
||||
queue_dir: &str,
|
||||
ack_timeout_secs: Option<u64>,
|
||||
) -> Result<(), TargetError> {
|
||||
if let Some(secs) = ack_timeout_secs
|
||||
&& !(NATS_JETSTREAM_ACK_TIMEOUT_MIN_SECS..=NATS_JETSTREAM_ACK_TIMEOUT_MAX_SECS).contains(&secs)
|
||||
{
|
||||
return Err(TargetError::Configuration(format!(
|
||||
"{NATS_JETSTREAM_ACK_TIMEOUT_SECS} must be between {NATS_JETSTREAM_ACK_TIMEOUT_MIN_SECS} and {NATS_JETSTREAM_ACK_TIMEOUT_MAX_SECS} seconds"
|
||||
)));
|
||||
}
|
||||
|
||||
if !enable {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if stream_name.trim().is_empty() {
|
||||
return Err(TargetError::Configuration(format!(
|
||||
"{NATS_JETSTREAM_STREAM_NAME} is required when {NATS_JETSTREAM_ENABLE} is on"
|
||||
)));
|
||||
}
|
||||
|
||||
if queue_dir.trim().is_empty() {
|
||||
return Err(TargetError::Configuration(format!(
|
||||
"{NATS_QUEUE_DIR} is required when {NATS_JETSTREAM_ENABLE} is on"
|
||||
)));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Detail carried by the retryable publish error raised when the stream-validation gate closes. The
|
||||
/// full validation cause is logged at error level inside the validator.
|
||||
pub(crate) const STREAM_VALIDATION_FAILED_DETAIL: &str = "stream validation failed";
|
||||
|
||||
/// Applies a stream-validation verdict on the publish path. A pass records the verdict so later
|
||||
/// publishes skip re-validation. A failure returns a retryable JetStream publish error carrying the
|
||||
/// fixed label and leaves the verdict unvalidated.
|
||||
pub(crate) fn gate_publish_on_stream_validation(
|
||||
verdict: Result<(), TargetError>,
|
||||
stream_validated: &AtomicBool,
|
||||
) -> Result<(), TargetError> {
|
||||
match verdict {
|
||||
Ok(()) => {
|
||||
stream_validated.store(true, Ordering::Release);
|
||||
Ok(())
|
||||
}
|
||||
Err(_) => Err(TargetError::JetStreamPublish {
|
||||
retryable: true,
|
||||
detail: STREAM_VALIDATION_FAILED_DETAIL.to_string(),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
/// Reports whether a publish subject is captured by a stream subject filter, honouring the NATS
|
||||
/// wildcard tokens: a single-token wildcard matches one subject token, a trailing multi-token
|
||||
/// wildcard matches one or more remaining tokens, any other token matches literally.
|
||||
fn subject_filter_captures(filter: &str, subject: &str) -> bool {
|
||||
let mut filter_tokens = filter.split('.');
|
||||
let mut subject_tokens = subject.split('.');
|
||||
|
||||
loop {
|
||||
match (filter_tokens.next(), subject_tokens.next()) {
|
||||
(Some(">"), Some(_)) => return true,
|
||||
(Some("*"), Some(_)) => continue,
|
||||
(Some(filter_token), Some(subject_token)) if filter_token == subject_token => continue,
|
||||
(None, None) => return true,
|
||||
_ => return false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Validates the configured JetStream stream from a single get_stream lookup when enabled. Asserts
|
||||
/// the stream exists, captures the subject, returns acks, accepts writes, and keeps a duplicate
|
||||
/// window that covers one retry cycle. The client-facing error is generic and every concrete cause
|
||||
/// is logged server-side. The stream is never created, a missing stream is an error.
|
||||
///
|
||||
/// When validation_logged is set, the first passing validation logs at info and later revalidations
|
||||
/// of the same context log at debug, so a persistent wrong-stream loop does not flood the log.
|
||||
pub(crate) async fn validate_jetstream_stream(
|
||||
context: &jetstream::Context,
|
||||
args: &NATSArgs,
|
||||
target_label: &str,
|
||||
validation_logged: Option<&AtomicBool>,
|
||||
) -> Result<(), TargetError> {
|
||||
if !args.jetstream_enable.unwrap_or(false) {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let stream_name = args.jetstream_stream_name.as_deref().unwrap_or_default();
|
||||
if stream_name.is_empty() {
|
||||
error!(target = %target_label, "JetStream enabled without a configured stream name");
|
||||
return Err(TargetError::Configuration("JetStream stream is not configured".to_string()));
|
||||
}
|
||||
|
||||
let stream = context.get_stream(stream_name).await.map_err(|err| {
|
||||
error!(
|
||||
target = %target_label,
|
||||
stream = %stream_name,
|
||||
error = %err,
|
||||
"JetStream stream lookup failed, the stream must be pre-provisioned"
|
||||
);
|
||||
TargetError::Configuration("configured JetStream stream is unavailable".to_string())
|
||||
})?;
|
||||
|
||||
assert_stream_writable(&stream.cached_info().config, args, target_label)?;
|
||||
// Info on a context's first passing validation, debug on later revalidations of the same context.
|
||||
let first_pass = validation_logged
|
||||
.map(|logged| !logged.swap(true, Ordering::AcqRel))
|
||||
.unwrap_or(true);
|
||||
if first_pass {
|
||||
info!(
|
||||
target = %target_label,
|
||||
stream = %stream_name,
|
||||
"JetStream stream validation succeeded"
|
||||
);
|
||||
} else {
|
||||
debug!(
|
||||
target = %target_label,
|
||||
stream = %stream_name,
|
||||
"JetStream stream revalidation succeeded"
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Asserts the retrieved stream configuration captures the subject, returns acks, accepts writes, and
|
||||
/// keeps a duplicate window that covers one retry cycle. The client-facing error is generic, the
|
||||
/// concrete cause logged server-side.
|
||||
fn assert_stream_writable(config: &jetstream::stream::Config, args: &NATSArgs, target_label: &str) -> Result<(), TargetError> {
|
||||
let stream_name = args.jetstream_stream_name.as_deref().unwrap_or_default();
|
||||
let subject = &args.subject;
|
||||
let subject_bound = config.subjects.iter().any(|filter| subject_filter_captures(filter, subject));
|
||||
if !subject_bound {
|
||||
error!(
|
||||
target = %target_label,
|
||||
stream = %stream_name,
|
||||
subject = %subject,
|
||||
"configured subject is not captured by the JetStream stream subjects"
|
||||
);
|
||||
return Err(TargetError::Configuration(
|
||||
"configured JetStream stream does not capture the subject".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
if config.no_ack {
|
||||
error!(
|
||||
target = %target_label,
|
||||
stream = %stream_name,
|
||||
"JetStream stream has no_ack set, publishes would never acknowledge"
|
||||
);
|
||||
return Err(TargetError::Configuration(
|
||||
"configured JetStream stream does not acknowledge writes".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
if config.sealed {
|
||||
error!(
|
||||
target = %target_label,
|
||||
stream = %stream_name,
|
||||
"JetStream stream is sealed, writes are rejected"
|
||||
);
|
||||
return Err(TargetError::Configuration("configured JetStream stream is sealed".to_string()));
|
||||
}
|
||||
|
||||
let ack_timeout = Duration::from_secs(
|
||||
args.jetstream_ack_timeout_secs
|
||||
.unwrap_or(NATS_JETSTREAM_ACK_TIMEOUT_DEFAULT_SECS),
|
||||
);
|
||||
let required_window = retry_lifetime(ack_timeout);
|
||||
if config.duplicate_window < required_window {
|
||||
error!(
|
||||
target = %target_label,
|
||||
stream = %stream_name,
|
||||
configured_window_secs = config.duplicate_window.as_secs(),
|
||||
required_window_secs = required_window.as_secs(),
|
||||
"JetStream stream duplicate_window is below the retry lifetime, a late retry would be delivered twice"
|
||||
);
|
||||
return Err(TargetError::Configuration(
|
||||
"configured JetStream stream duplicate window is too small for the retry lifetime".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::target::TargetType;
|
||||
use crate::target::nats::test_support::*;
|
||||
|
||||
#[test]
|
||||
fn jetstream_settings_disabled_passes() {
|
||||
validate_jetstream_settings(false, "", "", None).expect("disabled jetstream validates");
|
||||
validate_jetstream_settings(false, "", "", Some(30)).expect("disabled jetstream ignores stream and queue");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn jetstream_settings_ack_timeout_range() {
|
||||
for secs in [
|
||||
NATS_JETSTREAM_ACK_TIMEOUT_MIN_SECS,
|
||||
NATS_JETSTREAM_ACK_TIMEOUT_DEFAULT_SECS,
|
||||
NATS_JETSTREAM_ACK_TIMEOUT_MAX_SECS,
|
||||
] {
|
||||
validate_jetstream_settings(false, "", "", Some(secs)).expect("in-range ack timeout accepted");
|
||||
}
|
||||
for secs in [
|
||||
NATS_JETSTREAM_ACK_TIMEOUT_MIN_SECS - 1,
|
||||
NATS_JETSTREAM_ACK_TIMEOUT_MAX_SECS + 1,
|
||||
] {
|
||||
let err = validate_jetstream_settings(false, "", "", Some(secs)).expect_err("out-of-range ack timeout rejected");
|
||||
assert!(err.to_string().contains("between"));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn jetstream_settings_requires_stream_name() {
|
||||
let err = validate_jetstream_settings(true, " ", &nats_queue_dir(), Some(30))
|
||||
.expect_err("empty stream name rejected when enabled");
|
||||
assert!(err.to_string().contains(NATS_JETSTREAM_STREAM_NAME));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn jetstream_settings_requires_queue_dir() {
|
||||
let err =
|
||||
validate_jetstream_settings(true, "RUSTFS_EVENTS", "", Some(30)).expect_err("empty queue_dir rejected when enabled");
|
||||
assert!(err.to_string().contains(NATS_QUEUE_DIR));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn jetstream_settings_enabled_with_stream_and_queue_passes() {
|
||||
validate_jetstream_settings(true, "RUSTFS_EVENTS", &nats_queue_dir(), Some(30))
|
||||
.expect("a stream name and queue dir accept enabling jetstream");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn natsargs_validate_rejects_enabled_jetstream_without_stream() {
|
||||
let args = NATSArgs {
|
||||
queue_dir: nats_queue_dir(),
|
||||
jetstream_enable: Some(true),
|
||||
jetstream_stream_name: None,
|
||||
..base_args()
|
||||
};
|
||||
let err = args.validate().expect_err("enabled jetstream without a stream is rejected");
|
||||
assert!(err.to_string().contains(NATS_JETSTREAM_STREAM_NAME));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn subject_filter_matches_literal_and_wildcards() {
|
||||
assert!(subject_filter_captures("rustfs.events", "rustfs.events"));
|
||||
assert!(subject_filter_captures("rustfs.*", "rustfs.events"));
|
||||
assert!(subject_filter_captures("rustfs.>", "rustfs.events.created"));
|
||||
assert!(subject_filter_captures("rustfs.*.created", "rustfs.events.created"));
|
||||
assert!(!subject_filter_captures("rustfs.events", "rustfs.audit"));
|
||||
assert!(!subject_filter_captures("rustfs.*", "rustfs.events.created"));
|
||||
assert!(!subject_filter_captures("rustfs.events", "rustfs.events.created"));
|
||||
assert!(!subject_filter_captures("rustfs.events.created", "rustfs.events"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn assert_stream_writable_enforces_the_worst_case_window_at_the_default_ack_timeout() {
|
||||
// The default 30s ack timeout requires a 274s window. One second below is rejected, exactly at it is accepted.
|
||||
let args = jetstream_args(TargetType::NotifyEvent);
|
||||
assert_eq!(retry_lifetime(Duration::from_secs(30)), Duration::from_secs(274));
|
||||
|
||||
let mut below = writable_stream_config(&args.subject);
|
||||
below.duplicate_window = Duration::from_secs(273);
|
||||
let err = assert_stream_writable(&below, &args, "test").expect_err("a window below the worst case is rejected");
|
||||
assert!(matches!(err, TargetError::Configuration(_)));
|
||||
|
||||
let mut at = writable_stream_config(&args.subject);
|
||||
at.duplicate_window = Duration::from_secs(274);
|
||||
assert_stream_writable(&at, &args, "test").expect("a window at the worst case is accepted");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn assert_stream_writable_accepts_a_valid_stream() {
|
||||
for target_type in [TargetType::NotifyEvent, TargetType::AuditLog] {
|
||||
let args = jetstream_args(target_type);
|
||||
let config = writable_stream_config(&args.subject);
|
||||
assert_stream_writable(&config, &args, "test")
|
||||
.unwrap_or_else(|err| panic!("a writable stream passes for {target_type:?}: {err}"));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn assert_stream_writable_passes_when_subject_is_bound_by_wildcard() {
|
||||
let args = jetstream_args(TargetType::NotifyEvent);
|
||||
let mut config = writable_stream_config(&args.subject);
|
||||
config.subjects = vec!["rustfs.>".to_string()];
|
||||
// The configured subject rustfs.events is captured by the rustfs.> filter.
|
||||
assert_stream_writable(&config, &args, "test").expect("a wildcard-bound subject passes");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn assert_stream_writable_rejects_a_subject_not_bound() {
|
||||
let args = jetstream_args(TargetType::NotifyEvent);
|
||||
let mut config = writable_stream_config(&args.subject);
|
||||
config.subjects = vec!["some.other.subject".to_string()];
|
||||
let err = assert_stream_writable(&config, &args, "test").expect_err("an unbound subject is rejected");
|
||||
assert!(matches!(err, TargetError::Configuration(_)), "the error is a generic configuration error");
|
||||
assert!(
|
||||
!err.to_string().contains("some.other.subject"),
|
||||
"the client message omits the stream subjects"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn assert_stream_writable_rejects_no_ack() {
|
||||
let args = jetstream_args(TargetType::NotifyEvent);
|
||||
let mut config = writable_stream_config(&args.subject);
|
||||
config.no_ack = true;
|
||||
let err = assert_stream_writable(&config, &args, "test").expect_err("a no_ack stream is rejected");
|
||||
assert!(matches!(err, TargetError::Configuration(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn assert_stream_writable_rejects_sealed() {
|
||||
let args = jetstream_args(TargetType::NotifyEvent);
|
||||
let mut config = writable_stream_config(&args.subject);
|
||||
config.sealed = true;
|
||||
let err = assert_stream_writable(&config, &args, "test").expect_err("a sealed stream is rejected");
|
||||
assert!(matches!(err, TargetError::Configuration(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn assert_stream_writable_rejects_a_too_small_duplicate_window() {
|
||||
let args = jetstream_args(TargetType::NotifyEvent);
|
||||
let mut config = writable_stream_config(&args.subject);
|
||||
// Below the worst-case retry lifetime at the default ack timeout, so a late retry would deliver twice.
|
||||
config.duplicate_window = Duration::from_secs(90);
|
||||
let err = assert_stream_writable(&config, &args, "test").expect_err("a too-small window is rejected");
|
||||
assert!(matches!(err, TargetError::Configuration(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn assert_stream_writable_rejects_a_zero_duplicate_window() {
|
||||
let args = jetstream_args(TargetType::NotifyEvent);
|
||||
let mut config = writable_stream_config(&args.subject);
|
||||
// The window defaults to zero (dedup disabled) when the operator leaves it unset.
|
||||
config.duplicate_window = Duration::ZERO;
|
||||
let err = assert_stream_writable(&config, &args, "test").expect_err("a zero window is rejected");
|
||||
assert!(matches!(err, TargetError::Configuration(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stream_validation_gate_blocks_the_publish_and_classifies_retryable() {
|
||||
// A failed validation returns before any publish call, classified retryable so the entry is never lost.
|
||||
let stream_validated = AtomicBool::new(false);
|
||||
let verdict = Err(TargetError::Configuration("configured JetStream stream is unavailable".to_string()));
|
||||
let err = gate_publish_on_stream_validation(verdict, &stream_validated).expect_err("a failed validation closes the gate");
|
||||
match err {
|
||||
TargetError::JetStreamPublish { retryable, detail } => {
|
||||
assert!(retryable, "a validation failure is retryable");
|
||||
assert_eq!(detail, STREAM_VALIDATION_FAILED_DETAIL, "the fixed label is the detail");
|
||||
}
|
||||
other => panic!("expected a retryable JetStreamPublish, got {other:?}"),
|
||||
}
|
||||
assert!(
|
||||
!stream_validated.load(Ordering::SeqCst),
|
||||
"the verdict stays closed so the next attempt re-validates"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stream_validation_gate_pass_is_recorded_with_the_context() {
|
||||
let stream_validated = AtomicBool::new(false);
|
||||
gate_publish_on_stream_validation(Ok(()), &stream_validated).expect("a passing validation opens the gate");
|
||||
assert!(
|
||||
stream_validated.load(Ordering::SeqCst),
|
||||
"the recorded verdict lets later publishes skip re-validation"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -904,8 +904,11 @@ where
|
||||
}
|
||||
|
||||
fn delivery_snapshot(&self) -> TargetDeliverySnapshot {
|
||||
self.delivery_counters
|
||||
.snapshot(self.store.as_deref().map_or(0, |store| store.len() as u64))
|
||||
self.delivery_counters.snapshot(
|
||||
self.store.as_deref().map_or(0, |store| store.len() as u64),
|
||||
// Postgres targets record no terminal failures and keep no failed store.
|
||||
0,
|
||||
)
|
||||
}
|
||||
|
||||
fn record_final_failure(&self) {
|
||||
|
||||
@@ -481,8 +481,11 @@ where
|
||||
}
|
||||
|
||||
fn delivery_snapshot(&self) -> TargetDeliverySnapshot {
|
||||
self.delivery_counters
|
||||
.snapshot(self.store.as_deref().map_or(0, |store| store.len() as u64))
|
||||
self.delivery_counters.snapshot(
|
||||
self.store.as_deref().map_or(0, |store| store.len() as u64),
|
||||
// Pulsar targets record no terminal failures and keep no failed store.
|
||||
0,
|
||||
)
|
||||
}
|
||||
|
||||
fn record_final_failure(&self) {
|
||||
|
||||
@@ -658,8 +658,11 @@ where
|
||||
}
|
||||
|
||||
fn delivery_snapshot(&self) -> TargetDeliverySnapshot {
|
||||
self.delivery_counters
|
||||
.snapshot(self.store.as_deref().map_or(0, |store| store.len() as u64))
|
||||
self.delivery_counters.snapshot(
|
||||
self.store.as_deref().map_or(0, |store| store.len() as u64),
|
||||
// Redis targets record no terminal failures and keep no failed store.
|
||||
0,
|
||||
)
|
||||
}
|
||||
|
||||
fn record_final_failure(&self) {
|
||||
|
||||
@@ -644,8 +644,11 @@ where
|
||||
}
|
||||
|
||||
fn delivery_snapshot(&self) -> TargetDeliverySnapshot {
|
||||
self.delivery_counters
|
||||
.snapshot(self.store.as_deref().map_or(0, |store| store.len() as u64))
|
||||
self.delivery_counters.snapshot(
|
||||
self.store.as_deref().map_or(0, |store| store.len() as u64),
|
||||
// Webhook targets record no terminal failures and keep no failed store.
|
||||
0,
|
||||
)
|
||||
}
|
||||
|
||||
fn record_final_failure(&self) {
|
||||
|
||||
@@ -0,0 +1,259 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Regression guards for the NATS JetStream publish feature.
|
||||
//!
|
||||
//! The JetStream surface can be removed and the build still succeed unless something fails when it
|
||||
//! disappears. These layered guards make a silent removal or a silent classification drift a build or
|
||||
//! test failure:
|
||||
//!
|
||||
//! - a compile-time reference to the JetStream config surface and the target entry point,
|
||||
//! - a module-presence test naming the public entry type and the publish-error variant,
|
||||
//! - completeness tests over the publish-error-class enums with no wildcard arm,
|
||||
//! - a server-backed end-to-end publish-and-ack test, ignored by default so the offline gate stays
|
||||
//! green without a NATS server.
|
||||
//!
|
||||
//! The end-to-end guard requires a running NATS server with JetStream enabled. To run it locally:
|
||||
//!
|
||||
//! docker run -d --name rustfs-nats-test -p 4222:4222 nats:2 -js
|
||||
//! cargo test -p rustfs-targets --test nats_jetstream_regression_guards -- --ignored
|
||||
//!
|
||||
//! Override the server URL with RUSTFS_TEST_NATS_URL.
|
||||
|
||||
use async_nats::jetstream::context::PublishErrorKind;
|
||||
use rustfs_targets::check_nats_server_available;
|
||||
use rustfs_targets::error::TargetError;
|
||||
use rustfs_targets::target::nats::{NATSArgs, NATSTarget};
|
||||
use rustfs_targets::target::{FailedErrorClass, TargetType};
|
||||
|
||||
// Compile-time module-presence assertion.
|
||||
//
|
||||
// A const initializer names the JetStream config fields on the public target args type and the public
|
||||
// target constructor by path, referencing the JetStream wiring at compile time without running
|
||||
// anything. Removing the jetstream_enable, jetstream_stream_name, or jetstream_ack_timeout_secs fields
|
||||
// from NATSArgs, or removing the NATSTarget constructor, while the config flag keys remain, fails to
|
||||
// compile here. A static reference rather than a runtime check, so it fires at build time even when no
|
||||
// test is run.
|
||||
const _: fn(&NATSArgs) = |args: &NATSArgs| {
|
||||
// Each field is read by name, so removing any of the three from NATSArgs fails to compile here.
|
||||
let _enable: Option<bool> = args.jetstream_enable;
|
||||
let _stream_name: &Option<String> = &args.jetstream_stream_name;
|
||||
let _ack_timeout: Option<u64> = args.jetstream_ack_timeout_secs;
|
||||
|
||||
// The publish target entry point. Naming it by path ties this assertion to the constructor that
|
||||
// wires the JetStream context, so deleting the entry point breaks the build.
|
||||
let _entry: fn(String, NATSArgs) -> Result<NATSTarget<String>, TargetError> = NATSTarget::<String>::new;
|
||||
};
|
||||
|
||||
/// Module-presence test naming the public JetStream entry type, the health-check entry, and the
|
||||
/// publish-error variant by path. Deleting the NATSTarget constructor, the check_nats_server_available
|
||||
/// entry, or the JetStreamPublish error variant fails to compile this test.
|
||||
#[test]
|
||||
fn jetstream_public_entry_points_are_present() {
|
||||
let _entry: fn(String, NATSArgs) -> Result<NATSTarget<String>, TargetError> = NATSTarget::<String>::new;
|
||||
|
||||
let _health_check = check_nats_server_available;
|
||||
|
||||
// The publish path classifies a publish failure into this variant. Constructing it by name binds
|
||||
// the guard to the variant, so removing it from TargetError breaks the build.
|
||||
let publish_error = TargetError::JetStreamPublish {
|
||||
retryable: true,
|
||||
detail: "guard".to_string(),
|
||||
};
|
||||
match publish_error {
|
||||
TargetError::JetStreamPublish { retryable, detail } => {
|
||||
assert!(retryable);
|
||||
assert_eq!(detail, "guard");
|
||||
}
|
||||
other => panic!("expected the JetStream publish variant, found {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Cross-module completeness over the publish-error class.
|
||||
///
|
||||
/// Lists every async-nats PublishErrorKind variant by name with no wildcard arm. The publish path
|
||||
/// classifies each kind into retryable or terminal. A new variant added upstream, or a renamed or
|
||||
/// removed one, fails to compile this match, forcing the classification to be revisited rather than
|
||||
/// silently folding the new kind into a catch-all.
|
||||
#[test]
|
||||
fn publish_error_kind_is_exhaustively_enumerated() {
|
||||
fn name_of(kind: PublishErrorKind) -> &'static str {
|
||||
match kind {
|
||||
PublishErrorKind::StreamNotFound => "StreamNotFound",
|
||||
PublishErrorKind::WrongLastMessageId => "WrongLastMessageId",
|
||||
PublishErrorKind::WrongLastSequence => "WrongLastSequence",
|
||||
PublishErrorKind::TimedOut => "TimedOut",
|
||||
PublishErrorKind::BrokenPipe => "BrokenPipe",
|
||||
PublishErrorKind::MaxAckPending => "MaxAckPending",
|
||||
PublishErrorKind::MaxPayloadExceeded => "MaxPayloadExceeded",
|
||||
PublishErrorKind::Other => "Other",
|
||||
}
|
||||
}
|
||||
|
||||
let all = [
|
||||
PublishErrorKind::StreamNotFound,
|
||||
PublishErrorKind::WrongLastMessageId,
|
||||
PublishErrorKind::WrongLastSequence,
|
||||
PublishErrorKind::TimedOut,
|
||||
PublishErrorKind::BrokenPipe,
|
||||
PublishErrorKind::MaxAckPending,
|
||||
PublishErrorKind::MaxPayloadExceeded,
|
||||
PublishErrorKind::Other,
|
||||
];
|
||||
assert_eq!(
|
||||
all.len(),
|
||||
8,
|
||||
"the publish-error class has eight kinds, update the classification on a change"
|
||||
);
|
||||
for kind in all {
|
||||
assert!(!name_of(kind).is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
/// Cross-module completeness over the failed-store error class.
|
||||
///
|
||||
/// Lists every FailedErrorClass variant by name with no wildcard arm. The replay worker tags a failed
|
||||
/// entry with one of these classes. Adding or removing a class fails to compile this match, so the
|
||||
/// tagging and the operator-facing tag stay in step.
|
||||
#[test]
|
||||
fn failed_error_class_is_exhaustively_enumerated() {
|
||||
fn tag_of(class: FailedErrorClass) -> &'static str {
|
||||
match class {
|
||||
FailedErrorClass::Terminal => "terminal",
|
||||
}
|
||||
}
|
||||
|
||||
let all = [FailedErrorClass::Terminal];
|
||||
assert_eq!(
|
||||
all.len(),
|
||||
1,
|
||||
"the failed-store error class has one variant, update the tagging on a change"
|
||||
);
|
||||
for class in all {
|
||||
assert_eq!(tag_of(class), class.as_str(), "the guard tag matches the operator-facing tag");
|
||||
}
|
||||
}
|
||||
|
||||
fn server_url() -> String {
|
||||
std::env::var("RUSTFS_TEST_NATS_URL").unwrap_or_else(|_| "nats://127.0.0.1:4222".to_string())
|
||||
}
|
||||
|
||||
fn jetstream_args(subject: &str, stream_name: &str, queue_dir: &str) -> NATSArgs {
|
||||
NATSArgs {
|
||||
enable: true,
|
||||
address: server_url(),
|
||||
subject: subject.to_string(),
|
||||
username: String::new(),
|
||||
password: String::new(),
|
||||
token: String::new(),
|
||||
credentials_file: String::new(),
|
||||
tls_ca: String::new(),
|
||||
tls_client_cert: String::new(),
|
||||
tls_client_key: String::new(),
|
||||
tls_required: false,
|
||||
queue_dir: queue_dir.to_string(),
|
||||
queue_limit: 0,
|
||||
jetstream_enable: Some(true),
|
||||
jetstream_stream_name: Some(stream_name.to_string()),
|
||||
jetstream_ack_timeout_secs: Some(30),
|
||||
target_type: TargetType::NotifyEvent,
|
||||
}
|
||||
}
|
||||
|
||||
/// Server-backed end-to-end publish-and-ack guard.
|
||||
///
|
||||
/// Provisions a writable stream as an operator would, then drives the real target publish path with
|
||||
/// the JetStream flag on: init connects and validates the stream, save persists the event to the
|
||||
/// on-disk queue, send_from_store publishes and awaits the real PublishAck and clears the entry on it.
|
||||
/// The stream message count rising and the queue draining proves the message was published and
|
||||
/// acknowledged through the production code, the user-visible signal a build-only check cannot see.
|
||||
///
|
||||
/// Ignored by default because it needs a running NATS server with JetStream.
|
||||
#[tokio::test]
|
||||
#[ignore]
|
||||
async fn end_to_end_publish_is_acked_on_the_stream() {
|
||||
use rustfs_targets::EventName;
|
||||
use rustfs_targets::Target;
|
||||
use rustfs_targets::target::EntityTarget;
|
||||
use std::sync::Arc;
|
||||
use uuid::Uuid;
|
||||
|
||||
let suffix = Uuid::new_v4().simple().to_string();
|
||||
let stream_name = format!("RUSTFS_TEST_{suffix}");
|
||||
let subject = format!("rustfs.guard.{suffix}");
|
||||
let queue_dir = std::env::temp_dir().join(format!("rustfs-nats-guard-{suffix}"));
|
||||
let queue_dir = queue_dir.to_string_lossy().to_string();
|
||||
|
||||
// Provision the stream as the operator would. RustFS validates it, never creates it.
|
||||
let client = async_nats::connect(server_url()).await.expect("connect to NATS");
|
||||
let context = async_nats::jetstream::new(client);
|
||||
let _ = context.delete_stream(&stream_name).await;
|
||||
context
|
||||
.create_stream(async_nats::jetstream::stream::Config {
|
||||
name: stream_name.clone(),
|
||||
subjects: vec![subject.clone()],
|
||||
duplicate_window: std::time::Duration::from_secs(300),
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.expect("provision the stream");
|
||||
|
||||
let args = jetstream_args(&subject, &stream_name, &queue_dir);
|
||||
let target = NATSTarget::<String>::new("guard-target".to_string(), args).expect("build the flag-on target");
|
||||
|
||||
target
|
||||
.init()
|
||||
.await
|
||||
.expect("init connects and validates the operator-provisioned stream");
|
||||
|
||||
let count_before = stream_message_count(&context, &stream_name).await;
|
||||
|
||||
let event = EntityTarget {
|
||||
object_name: "guard/object.txt".to_string(),
|
||||
bucket_name: "guard-bucket".to_string(),
|
||||
event_name: EventName::ObjectCreatedPut,
|
||||
data: "guard-payload".to_string(),
|
||||
};
|
||||
target
|
||||
.save(Arc::new(event))
|
||||
.await
|
||||
.expect("save persists the event to the queue");
|
||||
|
||||
let keys = target.store().expect("the flag-on target has a queue store").list();
|
||||
assert_eq!(keys.len(), 1, "the saved event is queued exactly once");
|
||||
target
|
||||
.send_from_store(keys[0].clone())
|
||||
.await
|
||||
.expect("the queued event publishes and is acked");
|
||||
|
||||
let count_after = stream_message_count(&context, &stream_name).await;
|
||||
assert_eq!(
|
||||
count_after,
|
||||
count_before + 1,
|
||||
"the published event is durably on the stream after its ack"
|
||||
);
|
||||
|
||||
let remaining = target.store().expect("the flag-on target has a queue store").list();
|
||||
assert!(remaining.is_empty(), "the queue entry is cleared on the ack");
|
||||
|
||||
target.close().await.expect("close drains the target");
|
||||
let _ = context.delete_stream(&stream_name).await;
|
||||
let _ = std::fs::remove_dir_all(&queue_dir);
|
||||
}
|
||||
|
||||
async fn stream_message_count(context: &async_nats::jetstream::Context, stream_name: &str) -> u64 {
|
||||
let mut stream = context.get_stream(stream_name).await.expect("look up the stream");
|
||||
let info = stream.info().await.expect("read the stream info");
|
||||
info.state.messages
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! JetStream stream validation integration tests for the NATS target.
|
||||
//!
|
||||
//! These tests are ignored because they require a running NATS server with JetStream enabled. To
|
||||
//! run locally:
|
||||
//!
|
||||
//! docker run -d --name rustfs-nats-test -p 4222:4222 nats:2 -js
|
||||
//! cargo test -p rustfs-targets --test nats_jetstream_validation_integration -- --ignored
|
||||
//!
|
||||
//! Override the server URL with RUSTFS_TEST_NATS_URL. Each test provisions and removes its own
|
||||
//! stream. RustFS never provisions the stream, the test harness does, mirroring the operator role.
|
||||
|
||||
use async_nats::jetstream::stream::Config;
|
||||
use rustfs_targets::check_nats_server_available;
|
||||
use rustfs_targets::target::TargetType;
|
||||
use rustfs_targets::target::nats::NATSArgs;
|
||||
use std::time::Duration;
|
||||
use uuid::Uuid;
|
||||
|
||||
fn server_url() -> String {
|
||||
std::env::var("RUSTFS_TEST_NATS_URL").unwrap_or_else(|_| "nats://127.0.0.1:4222".to_string())
|
||||
}
|
||||
|
||||
fn jetstream_args(subject: &str, stream_name: &str) -> NATSArgs {
|
||||
NATSArgs {
|
||||
enable: true,
|
||||
address: server_url(),
|
||||
subject: subject.to_string(),
|
||||
username: String::new(),
|
||||
password: String::new(),
|
||||
token: String::new(),
|
||||
credentials_file: String::new(),
|
||||
tls_ca: String::new(),
|
||||
tls_client_cert: String::new(),
|
||||
tls_client_key: String::new(),
|
||||
tls_required: false,
|
||||
queue_dir: "/tmp/rustfs-nats-jetstream-test".to_string(),
|
||||
queue_limit: 0,
|
||||
jetstream_enable: Some(true),
|
||||
jetstream_stream_name: Some(stream_name.to_string()),
|
||||
jetstream_ack_timeout_secs: Some(30),
|
||||
target_type: TargetType::NotifyEvent,
|
||||
}
|
||||
}
|
||||
|
||||
/// A pre-provisioned writable stream: it captures the subject, returns acks, accepts writes, and its
|
||||
/// duplicate window exceeds the retry lifetime. The test harness creates it, never RustFS.
|
||||
async fn provision_writable_stream(stream_name: &str, subject: &str) {
|
||||
let client = async_nats::connect(server_url()).await.expect("connect to NATS");
|
||||
let context = async_nats::jetstream::new(client);
|
||||
// Clear any leftover from a prior aborted run. Absent is fine, so the result is dropped.
|
||||
let _ = context.delete_stream(stream_name).await;
|
||||
context
|
||||
.create_stream(Config {
|
||||
name: stream_name.to_string(),
|
||||
subjects: vec![subject.to_string()],
|
||||
duplicate_window: Duration::from_secs(300),
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.expect("provision the stream");
|
||||
}
|
||||
|
||||
async fn remove_stream(stream_name: &str) {
|
||||
let client = async_nats::connect(server_url()).await.expect("connect to NATS");
|
||||
let context = async_nats::jetstream::new(client);
|
||||
// Best-effort teardown. A failure to delete does not fail the test, so the result is dropped.
|
||||
let _ = context.delete_stream(stream_name).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore]
|
||||
async fn missing_stream_fails_the_health_check() {
|
||||
let stream_name = format!("RUSTFS_TEST_{}", Uuid::new_v4().simple());
|
||||
let args = jetstream_args("rustfs.events", &stream_name);
|
||||
// No stream is provisioned, so the lookup must fail.
|
||||
let err = check_nats_server_available(&args)
|
||||
.await
|
||||
.expect_err("a missing stream fails the health check");
|
||||
assert!(!err.to_string().is_empty(), "the failure carries a generic message");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore]
|
||||
async fn valid_stream_passes_the_health_check() {
|
||||
let stream_name = format!("RUSTFS_TEST_{}", Uuid::new_v4().simple());
|
||||
let subject = "rustfs.events";
|
||||
provision_writable_stream(&stream_name, subject).await;
|
||||
let args = jetstream_args(subject, &stream_name);
|
||||
let result = check_nats_server_available(&args).await;
|
||||
remove_stream(&stream_name).await;
|
||||
result.expect("a pre-provisioned writable stream passes the health check");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore]
|
||||
async fn stream_not_capturing_the_subject_fails_the_health_check() {
|
||||
let stream_name = format!("RUSTFS_TEST_{}", Uuid::new_v4().simple());
|
||||
// The stream binds a different subject than the target publishes to.
|
||||
provision_writable_stream(&stream_name, "some.other.subject").await;
|
||||
let args = jetstream_args("rustfs.events", &stream_name);
|
||||
let result = check_nats_server_available(&args).await;
|
||||
remove_stream(&stream_name).await;
|
||||
result.expect_err("a stream that does not capture the subject fails the health check");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore]
|
||||
async fn too_small_duplicate_window_fails_the_health_check() {
|
||||
let stream_name = format!("RUSTFS_TEST_{}", Uuid::new_v4().simple());
|
||||
let subject = "rustfs.events";
|
||||
let client = async_nats::connect(server_url()).await.expect("connect to NATS");
|
||||
let context = async_nats::jetstream::new(client);
|
||||
// Clear any leftover from a prior aborted run. Absent is fine, so the result is dropped.
|
||||
let _ = context.delete_stream(&stream_name).await;
|
||||
context
|
||||
.create_stream(Config {
|
||||
name: stream_name.clone(),
|
||||
subjects: vec![subject.to_string()],
|
||||
// Below the worst-case retry lifetime at the default ack timeout.
|
||||
duplicate_window: Duration::from_secs(60),
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.expect("provision the stream");
|
||||
let args = jetstream_args(subject, &stream_name);
|
||||
let result = check_nats_server_available(&args).await;
|
||||
remove_stream(&stream_name).await;
|
||||
result.expect_err("a too-small duplicate window fails the health check");
|
||||
}
|
||||
Reference in New Issue
Block a user