mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-22 16:02:46 +00:00
Compare commits
15 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| bc1581eaee | |||
| 52795bc3f3 | |||
| 54a8b7b2b9 | |||
| 18352bafb5 | |||
| 71d5d21695 | |||
| df4465985f | |||
| 3792a92fc3 | |||
| 539ea7d978 | |||
| 6a4d04ea3c | |||
| 342207e3be | |||
| 7852b0cfde | |||
| efa8bc6d4b | |||
| fefd238d57 | |||
| a8312c1157 | |||
| 323f308d57 |
@@ -51,7 +51,7 @@ hotpath-cpu = [
|
||||
]
|
||||
|
||||
[dependencies]
|
||||
hotpath.workspace = true
|
||||
hotpath = { workspace = true, optional = true }
|
||||
rustfs-targets = { workspace = true }
|
||||
rustfs-config = { workspace = true, features = ["audit", "server-config-model"] }
|
||||
rustfs-s3-types = { workspace = true }
|
||||
|
||||
@@ -20,7 +20,7 @@ hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc", "rustfs-io-core/hotpath-all
|
||||
hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu", "rustfs-io-core/hotpath-cpu"]
|
||||
|
||||
[dependencies]
|
||||
hotpath.workspace = true
|
||||
hotpath = { workspace = true, optional = true }
|
||||
# Internal crates
|
||||
rustfs-io-core = { workspace = true }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
|
||||
@@ -77,7 +77,7 @@ ftps = []
|
||||
sftp = []
|
||||
|
||||
[dependencies]
|
||||
hotpath.workspace = true
|
||||
hotpath = { workspace = true, optional = true }
|
||||
rustfs-config = { workspace = true, features = ["constants"] }
|
||||
rustfs-credentials.workspace = true
|
||||
rustfs-ecstore.workspace = true
|
||||
|
||||
@@ -34,7 +34,7 @@ hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc", "rustfs-io-metrics/hotpath-
|
||||
hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu", "rustfs-io-metrics/hotpath-cpu"]
|
||||
|
||||
[dependencies]
|
||||
hotpath.workspace = true
|
||||
hotpath = { workspace = true, optional = true }
|
||||
bytes = { workspace = true, features = ["serde"] }
|
||||
thiserror = { workspace = true }
|
||||
tokio = { workspace = true, features = ["io-util", "fs", "sync", "rt-multi-thread"] }
|
||||
|
||||
@@ -50,7 +50,7 @@ hotpath-cpu = [
|
||||
]
|
||||
|
||||
[dependencies]
|
||||
hotpath.workspace = true
|
||||
hotpath = { workspace = true, optional = true }
|
||||
async-trait.workspace = true
|
||||
metrics.workspace = true
|
||||
rustfs-scanner-metrics.workspace = true
|
||||
|
||||
@@ -59,7 +59,7 @@ hotpath-cpu = [
|
||||
demo-examples = []
|
||||
|
||||
[dependencies]
|
||||
hotpath.workspace = true
|
||||
hotpath = { workspace = true, optional = true }
|
||||
rustfs-config = { workspace = true, features = ["notify", "server-config-model"] }
|
||||
rustfs-ecstore = { workspace = true }
|
||||
rustfs-s3-types = { workspace = true }
|
||||
|
||||
@@ -77,7 +77,18 @@ pub async fn initialize(config: Config) -> Result<(), NotificationError> {
|
||||
|
||||
let task_system = system.clone();
|
||||
tokio::spawn(async move {
|
||||
let result = task_system.set_targets_enabled(true, Some(config)).await;
|
||||
// The one-shot API keeps its strict contract: report an activation that
|
||||
// left quarantined targets so the caller can retry with a fixed config.
|
||||
// The reconciliation path quarantines instead and stays admin-usable.
|
||||
let result = task_system.set_targets_enabled(true, Some(config)).await.and_then(|()| {
|
||||
if task_system.runtime_lifecycle_all_targets_activated() {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(NotificationError::Initialization(
|
||||
"one or more notification targets failed to activate".to_string(),
|
||||
))
|
||||
}
|
||||
});
|
||||
let mut legacy = LEGACY_INITIALIZATION.lock().unwrap_or_else(|err| err.into_inner());
|
||||
if matches!(
|
||||
legacy.as_ref(),
|
||||
|
||||
@@ -393,11 +393,20 @@ impl NotificationSystem {
|
||||
self.services.config_manager.lifecycle().state()
|
||||
}
|
||||
|
||||
/// Returns whether the latest accepted lifecycle generation is active.
|
||||
/// Returns whether the latest accepted lifecycle generation has been handed
|
||||
/// to the runtime. A published generation serves admin operations even when
|
||||
/// some targets are quarantined; per-target health is reported by the target
|
||||
/// list, and the reconciler must not rebuild a published generation.
|
||||
pub fn runtime_lifecycle_is_converged(&self) -> bool {
|
||||
self.services.config_manager.lifecycle().is_converged()
|
||||
}
|
||||
|
||||
/// Returns whether the last published generation activated every configured
|
||||
/// target. Used by the one-shot `initialize` API's strict contract.
|
||||
pub fn runtime_lifecycle_all_targets_activated(&self) -> bool {
|
||||
self.services.config_manager.lifecycle().all_targets_activated()
|
||||
}
|
||||
|
||||
/// Validates a bucket notification configuration against the runtime targets
|
||||
/// without publishing it, so callers can reject a request before persisting.
|
||||
pub async fn validate_bucket_notification_config(
|
||||
|
||||
@@ -21,12 +21,12 @@ use rustfs_config::server_config::Config;
|
||||
use rustfs_targets::Target;
|
||||
use std::future::Future;
|
||||
use std::pin::Pin;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
||||
use std::sync::{Arc, Mutex as StdMutex};
|
||||
use tokio::sync::{Mutex, RwLock};
|
||||
use tokio::task::JoinHandle;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::{debug, info};
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
type CurrentGeneration = Arc<dyn Fn() -> bool + Send + Sync>;
|
||||
type CommitGeneration = Box<dyn FnOnce(bool) + Send>;
|
||||
@@ -238,11 +238,18 @@ impl LifecycleRuntime for DefaultLifecycleRuntime {
|
||||
close_old?;
|
||||
close_rejected?;
|
||||
close_activation_rejected?;
|
||||
// A target that cannot activate is quarantined and reported by the
|
||||
// admin target list. Failing the whole generation here would leave
|
||||
// the lifecycle permanently unconverged and block every admin call.
|
||||
if !activation_failures.is_empty() {
|
||||
return Err(NotificationError::Initialization(format!(
|
||||
"one or more notification targets failed to activate: {}",
|
||||
activation_failures.join("; ")
|
||||
)));
|
||||
warn!(
|
||||
event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
|
||||
component = LOG_COMPONENT_NOTIFY,
|
||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||
state = "degraded",
|
||||
failures = %activation_failures.join("; "),
|
||||
"notify runtime lifecycle"
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
})))
|
||||
@@ -281,6 +288,7 @@ impl LifecycleRuntime for DefaultLifecycleRuntime {
|
||||
struct LifecycleCoordinatorInner {
|
||||
applied_generation: AtomicU64,
|
||||
apply_lock: Mutex<()>,
|
||||
all_targets_activated: AtomicBool,
|
||||
desired: StdMutex<DesiredState>,
|
||||
publication_gate: Arc<StdMutex<()>>,
|
||||
runtime: Arc<dyn LifecycleRuntime>,
|
||||
@@ -309,6 +317,7 @@ impl NotifyLifecycleCoordinator {
|
||||
inner: Arc::new(LifecycleCoordinatorInner {
|
||||
applied_generation: AtomicU64::new(0),
|
||||
apply_lock: Mutex::new(()),
|
||||
all_targets_activated: AtomicBool::new(true),
|
||||
desired: StdMutex::new(DesiredState {
|
||||
cancellation: CancellationToken::new(),
|
||||
config: initial_config,
|
||||
@@ -357,11 +366,22 @@ impl NotifyLifecycleCoordinator {
|
||||
*self.inner.state.lock().unwrap_or_else(|err| err.into_inner())
|
||||
}
|
||||
|
||||
/// Whether the latest accepted generation has been handed to the runtime.
|
||||
/// This is the rebuild trigger and the admin availability gate: a published
|
||||
/// generation stays published until the configuration changes, so one
|
||||
/// unreachable target cannot keep the runtime churning through rebuilds.
|
||||
pub(crate) fn is_converged(&self) -> bool {
|
||||
let desired_generation = self.inner.desired.lock().unwrap_or_else(|err| err.into_inner()).generation;
|
||||
self.inner.applied_generation.load(Ordering::Acquire) >= desired_generation
|
||||
}
|
||||
|
||||
/// Whether the last published generation activated every configured target.
|
||||
/// Per-target health is reported by the admin target list; this only drives
|
||||
/// the one-shot `initialize` API's strict contract.
|
||||
pub(crate) fn all_targets_activated(&self) -> bool {
|
||||
self.inner.all_targets_activated.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
fn accept<F>(&self, allow_terminated: bool, update: F) -> NotificationLifecycleTransition
|
||||
where
|
||||
F: FnOnce(&mut DesiredState) -> Result<bool, NotificationError>,
|
||||
@@ -454,9 +474,8 @@ impl LifecycleCoordinatorInner {
|
||||
|
||||
fn mark_published(&self, generation: u64, state: NotificationRuntimeState, complete: bool) {
|
||||
*self.state.lock().unwrap_or_else(|err| err.into_inner()) = state;
|
||||
if complete {
|
||||
self.applied_generation.store(generation, Ordering::Release);
|
||||
}
|
||||
self.all_targets_activated.store(complete, Ordering::Release);
|
||||
self.applied_generation.store(generation, Ordering::Release);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -979,7 +998,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn partial_activation_reports_error_and_same_config_can_retry() {
|
||||
async fn partial_activation_quarantines_failed_target_without_rebuild_churn() {
|
||||
// The template's single-failure init budget is shared by every clone the factory hands
|
||||
// out, so the "bad" instance fails once in the first generation and recovers on retry.
|
||||
let bad_template = MockTarget::new("bad", RETRY_INIT_TARGET_TYPE).with_init_failures(1);
|
||||
@@ -1006,14 +1025,25 @@ mod tests {
|
||||
.insert("bad".to_string(), bad);
|
||||
let (coordinator, runtime_view, _) = real_runtime(config.clone(), Arc::new(TargetRegistry::with_plugins(plugins)), None);
|
||||
|
||||
// A quarantined target is a per-target health outcome, so the mutation
|
||||
// itself succeeds and admin callers are not locked out.
|
||||
coordinator
|
||||
.set_mode(true, Some(config.clone()))
|
||||
.wait()
|
||||
.await
|
||||
.expect_err("the originating transition must report the failed target");
|
||||
.expect("a quarantined target must not fail the lifecycle transition");
|
||||
let partial = runtime_view.runtime_status_snapshot().await;
|
||||
assert_eq!(partial.target_count, 1, "the healthy target should remain fault-isolated and active");
|
||||
assert!(!coordinator.is_converged(), "a partial generation must remain retryable");
|
||||
assert_eq!(partial.target_count, 2, "a quarantined target stays in the runtime beside healthy ones");
|
||||
// A published generation must not look stale: otherwise the reconciler
|
||||
// tears the whole runtime down and rebuilds it on every tick.
|
||||
assert!(
|
||||
coordinator.is_converged(),
|
||||
"a quarantined target must not leave the generation unpublished"
|
||||
);
|
||||
assert!(
|
||||
!coordinator.all_targets_activated(),
|
||||
"the quarantined target is still reported as not fully activated"
|
||||
);
|
||||
assert_eq!(coordinator.state(), NotificationRuntimeState::TargetsEnabled { generation: 1 });
|
||||
|
||||
coordinator
|
||||
@@ -1024,6 +1054,7 @@ mod tests {
|
||||
let recovered = runtime_view.runtime_status_snapshot().await;
|
||||
assert_eq!(recovered.target_count, 2);
|
||||
assert!(coordinator.is_converged());
|
||||
assert!(coordinator.all_targets_activated(), "the transient failure recovers on the next apply");
|
||||
assert_eq!(coordinator.state(), NotificationRuntimeState::TargetsEnabled { generation: 2 });
|
||||
|
||||
coordinator
|
||||
|
||||
@@ -263,6 +263,16 @@ impl EventNotifier {
|
||||
let mut deferred_handles = Vec::new();
|
||||
let mut direct_handles = Vec::new();
|
||||
|
||||
// The entity carries only event-level fields and every target reads it
|
||||
// without mutation, so build it once and share it across the matched
|
||||
// targets instead of cloning the whole event per target.
|
||||
let entity_target: Arc<EntityTarget<Event>> = Arc::new(EntityTarget {
|
||||
object_name: object_key.to_string(),
|
||||
bucket_name: bucket_name.to_string(),
|
||||
event_name,
|
||||
data: event.as_ref().clone(),
|
||||
});
|
||||
|
||||
// A lifecycle writer holds this gate only while handing queue-store
|
||||
// ownership from one runtime generation to the next. Taking the read
|
||||
// guard before cloning targets means the writer both blocks new sends
|
||||
@@ -286,12 +296,15 @@ impl EventNotifier {
|
||||
// Clone an Arc<Box<dyn Target>> (which is where target_list is stored) to move into an asynchronous task
|
||||
// target_arc is already Arc, clone it for the async task
|
||||
let target_for_task = target_arc.clone();
|
||||
// Resolve the identifier once: `name()`/`id()` each clone the
|
||||
// two Strings inside `TargetID`, and this runs per target per event.
|
||||
let target_id = target_for_task.id();
|
||||
if !target_for_task.is_enabled() {
|
||||
debug!(
|
||||
event = EVENT_NOTIFY_DISPATCH_SKIPPED,
|
||||
component = LOG_COMPONENT_NOTIFY,
|
||||
subsystem = LOG_SUBSYSTEM_DISPATCH,
|
||||
target_id = %target_for_task.id(),
|
||||
target_id = %target_id,
|
||||
reason = "target_disabled",
|
||||
"Skipped notify dispatch target"
|
||||
);
|
||||
@@ -307,24 +320,15 @@ impl EventNotifier {
|
||||
self.send_limiter.clone()
|
||||
};
|
||||
let metrics = self.metrics.clone();
|
||||
let event_clone = event.clone();
|
||||
let target_name_for_task = target_for_task.name(); // Get the name before generating the task
|
||||
debug!(
|
||||
event = EVENT_NOTIFY_DISPATCH_STARTED,
|
||||
component = LOG_COMPONENT_NOTIFY,
|
||||
subsystem = LOG_SUBSYSTEM_DISPATCH,
|
||||
target_id = %target_for_task.id(),
|
||||
target_id = %target_id,
|
||||
deferred = is_deferred,
|
||||
"notify dispatch"
|
||||
);
|
||||
// Use cloned data in closures to avoid borrowing conflicts
|
||||
// Create an EntityTarget from the event
|
||||
let entity_target: Arc<EntityTarget<Event>> = Arc::new(EntityTarget {
|
||||
object_name: object_key.to_string(),
|
||||
bucket_name: bucket_name.to_string(),
|
||||
event_name,
|
||||
data: event_clone.as_ref().clone(),
|
||||
});
|
||||
let entity_target = Arc::clone(&entity_target);
|
||||
let handle = tokio::spawn(async move {
|
||||
let _direct_dispatch_lease = direct_dispatch_lease;
|
||||
let _deferred_dispatch_guard = deferred_dispatch_guard;
|
||||
@@ -339,7 +343,7 @@ impl EventNotifier {
|
||||
event = EVENT_NOTIFY_DISPATCH_SKIPPED,
|
||||
component = LOG_COMPONENT_NOTIFY,
|
||||
subsystem = LOG_SUBSYSTEM_DISPATCH,
|
||||
target_id = %target_name_for_task,
|
||||
target_id = %target_id,
|
||||
reason = "runtime_generation_replaced",
|
||||
"Skipped pending direct notify dispatch"
|
||||
);
|
||||
@@ -357,7 +361,7 @@ impl EventNotifier {
|
||||
event = EVENT_NOTIFY_DISPATCH_FAILED,
|
||||
component = LOG_COMPONENT_NOTIFY,
|
||||
subsystem = LOG_SUBSYSTEM_DISPATCH,
|
||||
target_id = %target_name_for_task,
|
||||
target_id = %target_id,
|
||||
error = %e,
|
||||
reason = "permit_acquire_failed",
|
||||
"Failed to acquire notify dispatch permit"
|
||||
@@ -366,13 +370,13 @@ impl EventNotifier {
|
||||
return;
|
||||
}
|
||||
};
|
||||
if let Err(e) = target_for_task.save(entity_target.clone()).await {
|
||||
if let Err(e) = target_for_task.save(entity_target).await {
|
||||
metrics.increment_failed();
|
||||
error!(
|
||||
event = EVENT_NOTIFY_DISPATCH_FAILED,
|
||||
component = LOG_COMPONENT_NOTIFY,
|
||||
subsystem = LOG_SUBSYSTEM_DISPATCH,
|
||||
target_id = %target_name_for_task,
|
||||
target_id = %target_id,
|
||||
error = %e,
|
||||
reason = "send_failed",
|
||||
"Failed to dispatch notify event"
|
||||
@@ -396,7 +400,7 @@ impl EventNotifier {
|
||||
event = EVENT_NOTIFY_DISPATCH_COMPLETED,
|
||||
component = LOG_COMPONENT_NOTIFY,
|
||||
subsystem = LOG_SUBSYSTEM_DISPATCH,
|
||||
target_id = %target_name_for_task,
|
||||
target_id = %target_id,
|
||||
deferred = is_deferred,
|
||||
"notify dispatch"
|
||||
);
|
||||
|
||||
@@ -47,8 +47,9 @@ pub struct NotifyRuleEngine {
|
||||
struct CachedBucketRules {
|
||||
metadata: Option<Weak<BucketMetadata>>,
|
||||
// Cache rejected snapshots too, so corrupt persisted routes are not
|
||||
// repeatedly compiled on the event hot path.
|
||||
rules: Option<RulesMap>,
|
||||
// repeatedly compiled on the event hot path. Shared by `Arc` so matching an
|
||||
// event does not clone the whole rule set.
|
||||
rules: Option<Arc<RulesMap>>,
|
||||
}
|
||||
|
||||
fn rules_from_metadata(metadata: &BucketMetadata) -> Option<RulesMap> {
|
||||
@@ -134,7 +135,7 @@ impl NotifyRuleEngine {
|
||||
bucket.to_string(),
|
||||
CachedBucketRules {
|
||||
metadata: None,
|
||||
rules: Some(rules_map),
|
||||
rules: Some(Arc::new(rules_map)),
|
||||
},
|
||||
)
|
||||
.await;
|
||||
@@ -150,7 +151,7 @@ impl NotifyRuleEngine {
|
||||
);
|
||||
}
|
||||
|
||||
pub async fn get_bucket_rules(&self, bucket: &str) -> Option<RulesMap> {
|
||||
pub async fn get_bucket_rules(&self, bucket: &str) -> Option<Arc<RulesMap>> {
|
||||
if let Some(source) = self.metadata_source() {
|
||||
let metadata = match source.upgrade() {
|
||||
Some(store) => store.get_bucket_metadata(bucket).await.ok(),
|
||||
@@ -158,13 +159,17 @@ impl NotifyRuleEngine {
|
||||
};
|
||||
return self.rules_for_metadata(bucket, metadata).await;
|
||||
}
|
||||
self.cached_rules(bucket).await
|
||||
}
|
||||
|
||||
async fn cached_rules(&self, bucket: &str) -> Option<Arc<RulesMap>> {
|
||||
self.bucket_rules_map
|
||||
.get(&bucket.to_string())
|
||||
.get(&bucket.to_owned())
|
||||
.await
|
||||
.and_then(|cached| cached.rules)
|
||||
}
|
||||
|
||||
async fn rules_for_metadata(&self, bucket: &str, metadata: Option<Arc<BucketMetadata>>) -> Option<RulesMap> {
|
||||
async fn rules_for_metadata(&self, bucket: &str, metadata: Option<Arc<BucketMetadata>>) -> Option<Arc<RulesMap>> {
|
||||
let key = bucket.to_owned();
|
||||
let Some(metadata) = metadata else {
|
||||
self.bucket_rules_map.remove(&key).await;
|
||||
@@ -181,7 +186,7 @@ impl NotifyRuleEngine {
|
||||
// Snapshot identity covers incarnation, revision AND bytes, without
|
||||
// trusting wall-clock ordering. A late compilation may evict a newer
|
||||
// cache entry, but cannot be reused by a reader of a different snapshot.
|
||||
let rules = rules_from_metadata(&metadata);
|
||||
let rules = rules_from_metadata(&metadata).map(Arc::new);
|
||||
self.bucket_rules_map
|
||||
.insert(
|
||||
key,
|
||||
@@ -428,7 +433,7 @@ mod tests {
|
||||
.await
|
||||
.into_iter()
|
||||
.collect::<Vec<_>>(),
|
||||
vec![target_id.clone()]
|
||||
vec![std::sync::Arc::new(target_id.clone())]
|
||||
);
|
||||
|
||||
engine.clear_bucket_rules("bucket").await;
|
||||
|
||||
@@ -18,6 +18,7 @@ use hashbrown::HashMap;
|
||||
use rayon::prelude::*;
|
||||
use rustfs_targets::arn::TargetID;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::sync::Arc;
|
||||
|
||||
/// PatternRules - Event rule that maps object name patterns to TargetID collections.
|
||||
/// `event.Rules` (map[string]TargetIDSet) in the Go code
|
||||
@@ -39,7 +40,7 @@ impl PatternRules {
|
||||
/// * `pattern` - The object name pattern.
|
||||
/// * `target_id` - The TargetID to associate with the pattern.
|
||||
pub fn add(&mut self, pattern: String, target_id: TargetID) {
|
||||
self.rules.entry(pattern).or_default().insert(target_id);
|
||||
self.rules.entry(pattern).or_default().insert(Arc::new(target_id));
|
||||
}
|
||||
|
||||
/// Checks if there are any rules that match the given object name.
|
||||
|
||||
@@ -14,9 +14,13 @@
|
||||
|
||||
use hashbrown::HashSet;
|
||||
use rustfs_targets::arn::TargetID;
|
||||
use std::sync::Arc;
|
||||
|
||||
/// TargetIDSet - A collection representation of TargetID.
|
||||
pub type TargetIdSet = HashSet<TargetID>;
|
||||
///
|
||||
/// Shared by `Arc` so matching an object key collects reference-count bumps
|
||||
/// instead of cloning the two Strings inside every `TargetID`.
|
||||
pub type TargetIdSet = HashSet<Arc<TargetID>>;
|
||||
|
||||
// HashSet has built-in clone, union, difference and other operations.
|
||||
// But the Go version of the method returns a new Set, and the HashSet method is usually iterator or modify itself.
|
||||
|
||||
@@ -107,7 +107,7 @@ required-features = ["dial9"]
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
hotpath.workspace = true
|
||||
hotpath = { workspace = true, optional = true }
|
||||
rustfs-audit = { workspace = true }
|
||||
rustfs-common = { workspace = true }
|
||||
rustfs-heal-contracts = { workspace = true }
|
||||
|
||||
@@ -111,7 +111,7 @@ webdav = ["dep:dav-server", "dep:hyper", "dep:hyper-util", "dep:http-body-util",
|
||||
sftp = ["dep:russh", "dep:russh-sftp", "dep:uuid", "dep:subtle", "dep:tokio-util", "dep:socket2", "dep:urlencoding"]
|
||||
|
||||
[dependencies]
|
||||
hotpath.workspace = true
|
||||
hotpath = { workspace = true, optional = true }
|
||||
# Core RustFS dependencies
|
||||
rustfs-iam = { workspace = true }
|
||||
rustfs-credentials = { workspace = true }
|
||||
|
||||
@@ -66,7 +66,7 @@ hotpath-cpu = [
|
||||
]
|
||||
|
||||
[dependencies]
|
||||
hotpath.workspace = true
|
||||
hotpath = { workspace = true, optional = true }
|
||||
rustfs-common.workspace = true
|
||||
rustfs-heal-contracts.workspace = true
|
||||
rustfs-io-metrics.workspace = true
|
||||
|
||||
@@ -53,7 +53,7 @@ hotpath-cpu = [
|
||||
]
|
||||
|
||||
[dependencies]
|
||||
hotpath.workspace = true
|
||||
hotpath = { workspace = true, optional = true }
|
||||
aes-gcm = { workspace = true, features = ["rand_core"] }
|
||||
bytes = { workspace = true, features = ["serde"] }
|
||||
chacha20poly1305.workspace = true
|
||||
|
||||
@@ -59,7 +59,7 @@ hotpath-cpu = [
|
||||
]
|
||||
|
||||
[dependencies]
|
||||
hotpath.workspace = true
|
||||
hotpath = { workspace = true, optional = true }
|
||||
arc-swap.workspace = true
|
||||
tokio = { workspace = true, features = ["io-util", "macros", "net", "rt-multi-thread", "sync", "time"] }
|
||||
rand = { workspace = true, features = ["serde"] }
|
||||
|
||||
@@ -58,7 +58,7 @@ hotpath-cpu = [
|
||||
]
|
||||
|
||||
[dependencies]
|
||||
hotpath.workspace = true
|
||||
hotpath = { workspace = true, optional = true }
|
||||
metrics = { workspace = true }
|
||||
async-compression = { workspace = true, features = ["tokio", "gzip", "bzip2"] }
|
||||
async-trait.workspace = true
|
||||
|
||||
@@ -41,7 +41,7 @@ hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc", "rustfs-s3select-api/hotpat
|
||||
hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu", "rustfs-s3select-api/hotpath-cpu"]
|
||||
|
||||
[dependencies]
|
||||
hotpath.workspace = true
|
||||
hotpath = { workspace = true, optional = true }
|
||||
rustfs-s3select-api = { workspace = true }
|
||||
async-trait.workspace = true
|
||||
bytes.workspace = true
|
||||
|
||||
@@ -34,7 +34,7 @@ hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc", "rustfs-filemeta/hotpath-al
|
||||
hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu", "rustfs-filemeta/hotpath-cpu"]
|
||||
|
||||
[dependencies]
|
||||
hotpath.workspace = true
|
||||
hotpath = { workspace = true, optional = true }
|
||||
async-trait.workspace = true
|
||||
# Storage-facing replication contracts are isolated in src/replication.rs until
|
||||
# the underlying wire types can move without creating a replication/storage-api cycle.
|
||||
|
||||
@@ -162,7 +162,6 @@ where
|
||||
) -> PreparedActivation<E> {
|
||||
let mut prepared = Vec::with_capacity(targets.len());
|
||||
let mut failures = Vec::new();
|
||||
let mut rejected_targets = Vec::new();
|
||||
let mut targets = targets.into_iter();
|
||||
while let Some(target) = targets.next() {
|
||||
match prepare_target(target, cancellation).await {
|
||||
@@ -179,19 +178,6 @@ where
|
||||
});
|
||||
prepared.push(target);
|
||||
}
|
||||
PrepareTargetResult::Failed { error, target } => {
|
||||
drop(error);
|
||||
let target_id = target.id().to_string();
|
||||
tracing::warn!(
|
||||
target_id,
|
||||
reason = "initialization_failed",
|
||||
"Target initialization failed during lifecycle preparation"
|
||||
);
|
||||
failures.push(TargetActivationFailure {
|
||||
detail: format!("{target_id}: initialization failed"),
|
||||
});
|
||||
rejected_targets.push(Arc::from(target));
|
||||
}
|
||||
PrepareTargetResult::Cancelled(target) => {
|
||||
prepared.push(Arc::from(target));
|
||||
prepared.extend(targets.map(Arc::from));
|
||||
@@ -201,7 +187,7 @@ where
|
||||
}
|
||||
PreparedActivation {
|
||||
failures,
|
||||
rejected_targets,
|
||||
rejected_targets: Vec::new(),
|
||||
targets: prepared,
|
||||
}
|
||||
}
|
||||
@@ -615,13 +601,15 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn builtin_adapter_skips_non_store_target_when_init_fails() {
|
||||
async fn builtin_adapter_keeps_non_store_target_when_init_fails() {
|
||||
let adapter = builtin_adapter();
|
||||
let target = MockTarget::new("primary", "webhook").with_init_failures(usize::MAX);
|
||||
|
||||
// A target without a queue store has nothing to replay, but it stays in
|
||||
// the runtime so it self-heals and remains visible to the target list.
|
||||
let activation = adapter.activate_with_replay(vec![Box::new(target)]).await;
|
||||
|
||||
assert!(activation.targets.is_empty());
|
||||
assert_eq!(activation.targets.len(), 1);
|
||||
assert!(activation.replay_workers.is_empty());
|
||||
}
|
||||
|
||||
|
||||
@@ -87,14 +87,7 @@ where
|
||||
E: PluginEvent,
|
||||
{
|
||||
Ready(SharedTarget<E>),
|
||||
Degraded {
|
||||
error: TargetError,
|
||||
target: SharedTarget<E>,
|
||||
},
|
||||
Failed {
|
||||
error: TargetError,
|
||||
target: Box<dyn Target<E> + Send + Sync>,
|
||||
},
|
||||
Degraded { error: TargetError, target: SharedTarget<E> },
|
||||
Cancelled(Box<dyn Target<E> + Send + Sync>),
|
||||
}
|
||||
|
||||
@@ -537,12 +530,7 @@ where
|
||||
) -> (mpsc::Sender<()>, JoinHandle<()>),
|
||||
{
|
||||
let shared = match prepare_target(target, None).await {
|
||||
PrepareTargetResult::Ready(target) => target,
|
||||
PrepareTargetResult::Degraded { target, .. } => target,
|
||||
PrepareTargetResult::Failed { target, .. } => {
|
||||
let _ = target.close().await;
|
||||
return None;
|
||||
}
|
||||
PrepareTargetResult::Ready(target) | PrepareTargetResult::Degraded { target, .. } => target,
|
||||
PrepareTargetResult::Cancelled(_) => unreachable!("preparation without a cancellation token cannot be cancelled"),
|
||||
};
|
||||
let target_id = shared.id().to_string();
|
||||
@@ -567,7 +555,6 @@ where
|
||||
E: PluginEvent,
|
||||
{
|
||||
let target_id = target.id().to_string();
|
||||
let has_store = target.store().is_some();
|
||||
|
||||
let init_result = match cancellation {
|
||||
Some(cancellation) => {
|
||||
@@ -582,13 +569,9 @@ where
|
||||
|
||||
if let Err(err) = init_result {
|
||||
tracing::error!(target_id = %target_id, reason = "initialization_failed", "Failed to initialize target");
|
||||
if !has_store {
|
||||
return PrepareTargetResult::Failed { error: err, target };
|
||||
}
|
||||
tracing::warn!(
|
||||
target_id = %target_id,
|
||||
"Proceeding with store-backed target despite init failure"
|
||||
);
|
||||
// Keep the target in the runtime whether or not it has a queue store: a
|
||||
// quarantined target self-heals through delivery retries and is reported
|
||||
// by the target list, so recovery never needs a runtime rebuild.
|
||||
return PrepareTargetResult::Degraded {
|
||||
error: err,
|
||||
target: Arc::from(target),
|
||||
|
||||
@@ -722,7 +722,7 @@ pub(crate) fn build_queued_payload<E>(event: &EntityTarget<E>) -> Result<QueuedP
|
||||
where
|
||||
E: PluginEvent,
|
||||
{
|
||||
build_queued_payload_with_records(event, vec![event.data.clone()])
|
||||
build_queued_payload_with_records(event, vec![&event.data])
|
||||
}
|
||||
|
||||
pub(crate) fn build_queued_payload_with_records<E, R>(
|
||||
|
||||
@@ -51,7 +51,7 @@ hotpath-cpu = [
|
||||
]
|
||||
|
||||
[dependencies]
|
||||
hotpath.workspace = true
|
||||
hotpath = { workspace = true, optional = true }
|
||||
rustfs-ecstore.workspace = true
|
||||
rustfs-storage-api = { workspace = true }
|
||||
tokio = { workspace = true, features = ["fs", "rt-multi-thread"] }
|
||||
|
||||
@@ -35,7 +35,7 @@ hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc", "rustfs-rio/hotpath-alloc"]
|
||||
hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu", "rustfs-rio/hotpath-cpu"]
|
||||
|
||||
[dependencies]
|
||||
hotpath.workspace = true
|
||||
hotpath = { workspace = true, optional = true }
|
||||
async-compression = { workspace = true, features = [
|
||||
"tokio",
|
||||
"bzip2",
|
||||
|
||||
@@ -221,6 +221,13 @@ pub fn register_notification_target_route(r: &mut S3Router<AdminOperation>) -> s
|
||||
AdminOperation(&NotificationTarget {}),
|
||||
)?;
|
||||
|
||||
r.insert(
|
||||
Method::DELETE,
|
||||
format!("{}{}", ADMIN_PREFIX, "/v3/target/{target_type}/{target_name}").as_str(),
|
||||
AdminOperation(&RemoveNotificationTarget {}),
|
||||
)?;
|
||||
|
||||
// Legacy spelling kept for consoles that still call the reset suffix.
|
||||
r.insert(
|
||||
Method::DELETE,
|
||||
format!("{}{}", ADMIN_PREFIX, "/v3/target/{target_type}/{target_name}/reset").as_str(),
|
||||
|
||||
@@ -22,7 +22,7 @@ use rustfs_trusted_proxies::ClientInfo;
|
||||
use std::net::SocketAddr;
|
||||
|
||||
/// An admin request whose application-facing state is owned by RustFS.
|
||||
pub(crate) struct AdminRequest {
|
||||
pub struct AdminRequest {
|
||||
envelope: RequestEnvelope,
|
||||
body: Body,
|
||||
}
|
||||
@@ -129,7 +129,7 @@ impl AdminRequest {
|
||||
|
||||
/// The raw HTTP response produced by an owned admin operation.
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct AdminResponse {
|
||||
pub struct AdminResponse {
|
||||
status: StatusCode,
|
||||
headers: HeaderMap,
|
||||
body: Body,
|
||||
@@ -156,7 +156,7 @@ impl AdminResponse {
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
pub(crate) trait Operation: Send + Sync + 'static {
|
||||
pub trait Operation: Send + Sync + 'static {
|
||||
async fn call(&self, req: AdminRequest, params: Params<'_, '_>) -> S3Result<AdminResponse>;
|
||||
}
|
||||
|
||||
|
||||
@@ -305,6 +305,12 @@ pub const ADMIN_ROUTE_POLICY_SPECS: &[AdminRouteSpec] = &[
|
||||
SET_BUCKET_TARGET,
|
||||
RouteRiskLevel::High,
|
||||
),
|
||||
admin(
|
||||
HttpMethod::Delete,
|
||||
"/rustfs/admin/v3/target/{target_type}/{target_name}",
|
||||
SET_BUCKET_TARGET,
|
||||
RouteRiskLevel::High,
|
||||
),
|
||||
admin(
|
||||
HttpMethod::Delete,
|
||||
"/rustfs/admin/v3/target/{target_type}/{target_name}/reset",
|
||||
|
||||
@@ -212,6 +212,7 @@ fn expected_admin_route_matrix() -> Vec<RouteMatrixEntry> {
|
||||
admin_route(Method::GET, "/v3/idp/builtin/policy-entities"),
|
||||
admin_route(Method::GET, "/v3/target/list"),
|
||||
admin_route_sample(Method::PUT, "/v3/target/{target_type}/{target_name}", "/v3/target/webhook/test-target"),
|
||||
admin_route_sample(Method::DELETE, "/v3/target/{target_type}/{target_name}", "/v3/target/webhook/test-target"),
|
||||
admin_route_sample(
|
||||
Method::DELETE,
|
||||
"/v3/target/{target_type}/{target_name}/reset",
|
||||
|
||||
@@ -651,7 +651,7 @@ fn notification_dispatch_follows_peer_reload_delete_and_recreation() {
|
||||
.await
|
||||
.into_iter()
|
||||
.collect::<Vec<_>>(),
|
||||
vec![inert_target]
|
||||
vec![std::sync::Arc::new(inert_target)]
|
||||
);
|
||||
assert!(system.has_subscriber(&bucket, &EventName::ObjectRemovedDelete).await);
|
||||
assert!(!system.has_subscriber(&bucket, &EventName::ObjectCreatedPut).await);
|
||||
|
||||
@@ -167,6 +167,10 @@ fn ensure_live_events_initialized() -> std::sync::Arc<NotificationSystem> {
|
||||
system
|
||||
}
|
||||
|
||||
/// A published lifecycle generation is usable even when some targets are
|
||||
/// quarantined: healthy targets already carry events and per-target health is
|
||||
/// reported by the admin target list. Only an unpublished generation means the
|
||||
/// runtime is not available yet.
|
||||
fn ensure_event_notifier_converged(system: &NotificationSystem) -> Result<(), NotificationError> {
|
||||
if system.runtime_lifecycle_is_converged() {
|
||||
Ok(())
|
||||
|
||||
Reference in New Issue
Block a user