Compare commits

..

5 Commits

Author SHA1 Message Date
Hauser Zhi 7852b0cfde style(notify): match the lifecycle module's log field shape
The quarantined-target warning carried its own message where every
sibling entry in this module labels itself "notify runtime lifecycle"
and lets the state field distinguish the outcome. Conforming keeps the
event stream uniform and drops the second copy of the sentence.

Co-Authored-By: qunjiazhu <qunjiazhu@gmail.com>
2026-11-21 14:54:27 +08:00
Hauser Zhi efa8bc6d4b merge: sync with origin/main 2026-11-21 14:47:08 +08:00
Hauser Zhi fefd238d57 perf(notify): publish quarantined generations instead of rebuilding forever
is_converged() answered "did every target activate?" but was used as
"does the runtime need rebuilding?". One unreachable target therefore
left the generation permanently unpublished, and the 5s reconciler tore
the whole runtime down and rebuilt it on every tick: 9 full rebuilds
(measured: create + init probe + queue reopen + replay restart + close)
in 30s, which is where the delayed first events after a restart come from.

mark_published now always advances the applied generation and records
per-target completeness separately, so is_converged() means "published".
The reconciler's rebuild trigger and the admin availability gate are
correct with no call-site changes, and runtime_lifecycle_is_published()
is gone again. the one-shot initialize() API keeps its strict contract
through the new all_targets_activated() signal.

prepare_target no longer drops store-less targets whose init failed:
they stay quarantined like store-backed ones, so recovery never needs a
runtime rebuild and PrepareTargetResult::Failed disappears.

Measured on the issue #9545 scenario: 0 rebuilds in the 30s window that
previously saw 9, with the admin surface unchanged.

Co-Authored-By: qunjiazhu <qunjiazhu@gmail.com>
2026-11-21 14:45:03 +08:00
Hauser Zhi a8312c1157 merge: sync with origin/main 2026-11-21 14:20:51 +08:00
Hauser Zhi 323f308d57 fix(notify): quarantine unreachable targets instead of failing the runtime
A target that cannot activate is a per-target health outcome, not a
lifecycle transition failure. Turning it into a transition error left
the generation permanently unconverged, so every notification admin call
re-entered the failing reconcile and answered 500, and the broken target
could not be removed because only the /reset DELETE route existed.

Publish the generation with the failed target quarantined, gate admin
access on the shared runtime_lifecycle_is_published() predicate, and
register the bare DELETE /v3/target/{target_type}/{target_name} route.
The one-shot initialize() API keeps its strict retry contract.

Co-Authored-By: qunjiazhu <qunjiazhu@gmail.com>
2026-11-21 14:16:44 +08:00
11 changed files with 114 additions and 95 deletions
+12 -1
View File
@@ -77,7 +77,18 @@ pub async fn initialize(config: Config) -> Result<(), NotificationError> {
let task_system = system.clone();
tokio::spawn(async move {
let result = task_system.set_targets_enabled(true, Some(config)).await;
// The one-shot API keeps its strict contract: report an activation that
// left quarantined targets so the caller can retry with a fixed config.
// The reconciliation path quarantines instead and stays admin-usable.
let result = task_system.set_targets_enabled(true, Some(config)).await.and_then(|()| {
if task_system.runtime_lifecycle_all_targets_activated() {
Ok(())
} else {
Err(NotificationError::Initialization(
"one or more notification targets failed to activate".to_string(),
))
}
});
let mut legacy = LEGACY_INITIALIZATION.lock().unwrap_or_else(|err| err.into_inner());
if matches!(
legacy.as_ref(),
+10 -1
View File
@@ -393,11 +393,20 @@ impl NotificationSystem {
self.services.config_manager.lifecycle().state()
}
/// Returns whether the latest accepted lifecycle generation is active.
/// Returns whether the latest accepted lifecycle generation has been handed
/// to the runtime. A published generation serves admin operations even when
/// some targets are quarantined; per-target health is reported by the target
/// list, and the reconciler must not rebuild a published generation.
pub fn runtime_lifecycle_is_converged(&self) -> bool {
self.services.config_manager.lifecycle().is_converged()
}
/// Returns whether the last published generation activated every configured
/// target. Used by the one-shot `initialize` API's strict contract.
pub fn runtime_lifecycle_all_targets_activated(&self) -> bool {
self.services.config_manager.lifecycle().all_targets_activated()
}
/// Validates a bucket notification configuration against the runtime targets
/// without publishing it, so callers can reject a request before persisting.
pub async fn validate_bucket_notification_config(
+44 -13
View File
@@ -21,12 +21,12 @@ use rustfs_config::server_config::Config;
use rustfs_targets::Target;
use std::future::Future;
use std::pin::Pin;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::{Arc, Mutex as StdMutex};
use tokio::sync::{Mutex, RwLock};
use tokio::task::JoinHandle;
use tokio_util::sync::CancellationToken;
use tracing::{debug, info};
use tracing::{debug, info, warn};
type CurrentGeneration = Arc<dyn Fn() -> bool + Send + Sync>;
type CommitGeneration = Box<dyn FnOnce(bool) + Send>;
@@ -238,11 +238,18 @@ impl LifecycleRuntime for DefaultLifecycleRuntime {
close_old?;
close_rejected?;
close_activation_rejected?;
// A target that cannot activate is quarantined and reported by the
// admin target list. Failing the whole generation here would leave
// the lifecycle permanently unconverged and block every admin call.
if !activation_failures.is_empty() {
return Err(NotificationError::Initialization(format!(
"one or more notification targets failed to activate: {}",
activation_failures.join("; ")
)));
warn!(
event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
component = LOG_COMPONENT_NOTIFY,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
state = "degraded",
failures = %activation_failures.join("; "),
"notify runtime lifecycle"
);
}
Ok(())
})))
@@ -281,6 +288,7 @@ impl LifecycleRuntime for DefaultLifecycleRuntime {
struct LifecycleCoordinatorInner {
applied_generation: AtomicU64,
apply_lock: Mutex<()>,
all_targets_activated: AtomicBool,
desired: StdMutex<DesiredState>,
publication_gate: Arc<StdMutex<()>>,
runtime: Arc<dyn LifecycleRuntime>,
@@ -309,6 +317,7 @@ impl NotifyLifecycleCoordinator {
inner: Arc::new(LifecycleCoordinatorInner {
applied_generation: AtomicU64::new(0),
apply_lock: Mutex::new(()),
all_targets_activated: AtomicBool::new(true),
desired: StdMutex::new(DesiredState {
cancellation: CancellationToken::new(),
config: initial_config,
@@ -357,11 +366,22 @@ impl NotifyLifecycleCoordinator {
*self.inner.state.lock().unwrap_or_else(|err| err.into_inner())
}
/// Whether the latest accepted generation has been handed to the runtime.
/// This is the rebuild trigger and the admin availability gate: a published
/// generation stays published until the configuration changes, so one
/// unreachable target cannot keep the runtime churning through rebuilds.
pub(crate) fn is_converged(&self) -> bool {
let desired_generation = self.inner.desired.lock().unwrap_or_else(|err| err.into_inner()).generation;
self.inner.applied_generation.load(Ordering::Acquire) >= desired_generation
}
/// Whether the last published generation activated every configured target.
/// Per-target health is reported by the admin target list; this only drives
/// the one-shot `initialize` API's strict contract.
pub(crate) fn all_targets_activated(&self) -> bool {
self.inner.all_targets_activated.load(Ordering::Acquire)
}
fn accept<F>(&self, allow_terminated: bool, update: F) -> NotificationLifecycleTransition
where
F: FnOnce(&mut DesiredState) -> Result<bool, NotificationError>,
@@ -454,9 +474,8 @@ impl LifecycleCoordinatorInner {
fn mark_published(&self, generation: u64, state: NotificationRuntimeState, complete: bool) {
*self.state.lock().unwrap_or_else(|err| err.into_inner()) = state;
if complete {
self.applied_generation.store(generation, Ordering::Release);
}
self.all_targets_activated.store(complete, Ordering::Release);
self.applied_generation.store(generation, Ordering::Release);
}
}
@@ -979,7 +998,7 @@ mod tests {
}
#[tokio::test]
async fn partial_activation_reports_error_and_same_config_can_retry() {
async fn partial_activation_quarantines_failed_target_without_rebuild_churn() {
// The template's single-failure init budget is shared by every clone the factory hands
// out, so the "bad" instance fails once in the first generation and recovers on retry.
let bad_template = MockTarget::new("bad", RETRY_INIT_TARGET_TYPE).with_init_failures(1);
@@ -1006,14 +1025,25 @@ mod tests {
.insert("bad".to_string(), bad);
let (coordinator, runtime_view, _) = real_runtime(config.clone(), Arc::new(TargetRegistry::with_plugins(plugins)), None);
// A quarantined target is a per-target health outcome, so the mutation
// itself succeeds and admin callers are not locked out.
coordinator
.set_mode(true, Some(config.clone()))
.wait()
.await
.expect_err("the originating transition must report the failed target");
.expect("a quarantined target must not fail the lifecycle transition");
let partial = runtime_view.runtime_status_snapshot().await;
assert_eq!(partial.target_count, 1, "the healthy target should remain fault-isolated and active");
assert!(!coordinator.is_converged(), "a partial generation must remain retryable");
assert_eq!(partial.target_count, 2, "a quarantined target stays in the runtime beside healthy ones");
// A published generation must not look stale: otherwise the reconciler
// tears the whole runtime down and rebuilds it on every tick.
assert!(
coordinator.is_converged(),
"a quarantined target must not leave the generation unpublished"
);
assert!(
!coordinator.all_targets_activated(),
"the quarantined target is still reported as not fully activated"
);
assert_eq!(coordinator.state(), NotificationRuntimeState::TargetsEnabled { generation: 1 });
coordinator
@@ -1024,6 +1054,7 @@ mod tests {
let recovered = runtime_view.runtime_status_snapshot().await;
assert_eq!(recovered.target_count, 2);
assert!(coordinator.is_converged());
assert!(coordinator.all_targets_activated(), "the transient failure recovers on the next apply");
assert_eq!(coordinator.state(), NotificationRuntimeState::TargetsEnabled { generation: 2 });
coordinator
+5 -17
View File
@@ -162,7 +162,6 @@ where
) -> PreparedActivation<E> {
let mut prepared = Vec::with_capacity(targets.len());
let mut failures = Vec::new();
let mut rejected_targets = Vec::new();
let mut targets = targets.into_iter();
while let Some(target) = targets.next() {
match prepare_target(target, cancellation).await {
@@ -179,19 +178,6 @@ where
});
prepared.push(target);
}
PrepareTargetResult::Failed { error, target } => {
drop(error);
let target_id = target.id().to_string();
tracing::warn!(
target_id,
reason = "initialization_failed",
"Target initialization failed during lifecycle preparation"
);
failures.push(TargetActivationFailure {
detail: format!("{target_id}: initialization failed"),
});
rejected_targets.push(Arc::from(target));
}
PrepareTargetResult::Cancelled(target) => {
prepared.push(Arc::from(target));
prepared.extend(targets.map(Arc::from));
@@ -201,7 +187,7 @@ where
}
PreparedActivation {
failures,
rejected_targets,
rejected_targets: Vec::new(),
targets: prepared,
}
}
@@ -615,13 +601,15 @@ mod tests {
}
#[tokio::test]
async fn builtin_adapter_skips_non_store_target_when_init_fails() {
async fn builtin_adapter_keeps_non_store_target_when_init_fails() {
let adapter = builtin_adapter();
let target = MockTarget::new("primary", "webhook").with_init_failures(usize::MAX);
// A target without a queue store has nothing to replay, but it stays in
// the runtime so it self-heals and remains visible to the target list.
let activation = adapter.activate_with_replay(vec![Box::new(target)]).await;
assert!(activation.targets.is_empty());
assert_eq!(activation.targets.len(), 1);
assert!(activation.replay_workers.is_empty());
}
+5 -22
View File
@@ -87,14 +87,7 @@ where
E: PluginEvent,
{
Ready(SharedTarget<E>),
Degraded {
error: TargetError,
target: SharedTarget<E>,
},
Failed {
error: TargetError,
target: Box<dyn Target<E> + Send + Sync>,
},
Degraded { error: TargetError, target: SharedTarget<E> },
Cancelled(Box<dyn Target<E> + Send + Sync>),
}
@@ -537,12 +530,7 @@ where
) -> (mpsc::Sender<()>, JoinHandle<()>),
{
let shared = match prepare_target(target, None).await {
PrepareTargetResult::Ready(target) => target,
PrepareTargetResult::Degraded { target, .. } => target,
PrepareTargetResult::Failed { target, .. } => {
let _ = target.close().await;
return None;
}
PrepareTargetResult::Ready(target) | PrepareTargetResult::Degraded { target, .. } => target,
PrepareTargetResult::Cancelled(_) => unreachable!("preparation without a cancellation token cannot be cancelled"),
};
let target_id = shared.id().to_string();
@@ -567,7 +555,6 @@ where
E: PluginEvent,
{
let target_id = target.id().to_string();
let has_store = target.store().is_some();
let init_result = match cancellation {
Some(cancellation) => {
@@ -582,13 +569,9 @@ where
if let Err(err) = init_result {
tracing::error!(target_id = %target_id, reason = "initialization_failed", "Failed to initialize target");
if !has_store {
return PrepareTargetResult::Failed { error: err, target };
}
tracing::warn!(
target_id = %target_id,
"Proceeding with store-backed target despite init failure"
);
// Keep the target in the runtime whether or not it has a queue store: a
// quarantined target self-heals through delivery retries and is reported
// by the target list, so recovery never needs a runtime rebuild.
return PrepareTargetResult::Degraded {
error: err,
target: Arc::from(target),
+7
View File
@@ -221,6 +221,13 @@ pub fn register_notification_target_route(r: &mut S3Router<AdminOperation>) -> s
AdminOperation(&NotificationTarget {}),
)?;
r.insert(
Method::DELETE,
format!("{}{}", ADMIN_PREFIX, "/v3/target/{target_type}/{target_name}").as_str(),
AdminOperation(&RemoveNotificationTarget {}),
)?;
// Legacy spelling kept for consoles that still call the reset suffix.
r.insert(
Method::DELETE,
format!("{}{}", ADMIN_PREFIX, "/v3/target/{target_type}/{target_name}/reset").as_str(),
+6
View File
@@ -305,6 +305,12 @@ pub const ADMIN_ROUTE_POLICY_SPECS: &[AdminRouteSpec] = &[
SET_BUCKET_TARGET,
RouteRiskLevel::High,
),
admin(
HttpMethod::Delete,
"/rustfs/admin/v3/target/{target_type}/{target_name}",
SET_BUCKET_TARGET,
RouteRiskLevel::High,
),
admin(
HttpMethod::Delete,
"/rustfs/admin/v3/target/{target_type}/{target_name}/reset",
@@ -212,6 +212,7 @@ fn expected_admin_route_matrix() -> Vec<RouteMatrixEntry> {
admin_route(Method::GET, "/v3/idp/builtin/policy-entities"),
admin_route(Method::GET, "/v3/target/list"),
admin_route_sample(Method::PUT, "/v3/target/{target_type}/{target_name}", "/v3/target/webhook/test-target"),
admin_route_sample(Method::DELETE, "/v3/target/{target_type}/{target_name}", "/v3/target/webhook/test-target"),
admin_route_sample(
Method::DELETE,
"/v3/target/{target_type}/{target_name}/reset",
-27
View File
@@ -1021,33 +1021,6 @@ impl<T> S3Router<T> {
}
}
#[cfg(test)]
impl<T: OwnedOperation> S3Router<T> {
/// Dispatches one legacy admin request. Callers keep their assertions and do not name s3s.
pub(crate) async fn call_legacy(
&self,
method: Method,
uri: &str,
extensions: http::Extensions,
) -> s3s::S3Result<S3Response<Body>> {
S3Route::call(
self,
S3Request {
input: Body::empty(),
method,
uri: uri.parse().expect("test URI"),
headers: HeaderMap::new(),
extensions,
credentials: None,
region: None,
service: None,
trailing_headers: None,
},
)
.await
}
}
impl<T> Default for S3Router<T> {
fn default() -> Self {
Self::new(false)
+20 -14
View File
@@ -142,12 +142,14 @@ mod tests {
};
use crate::admin::operation::{AdminRequest, AdminResponse, Operation};
use crate::admin::router::S3Router;
use crate::app::{RequestEnvelope, RequestHead};
use crate::app::RequestEnvelope;
use hyper::{Method, StatusCode};
use matchit::Params;
use rustfs_iam::{store::object::ObjectStore, sys::IamSys};
use rustfs_kms::KmsServiceManager;
use rustfs_s3_types::{S3ErrorCode, s3_error};
use s3s::route::S3Route;
use s3s::{Body, S3Request};
use std::sync::Arc;
struct UnreadyIam;
@@ -190,18 +192,22 @@ mod tests {
Arc::new(AppContext::new(context.object_store(), Arc::new(UnreadyIam), Arc::new(TestKms)))
}
fn envelope(extensions: http::Extensions) -> RequestEnvelope {
RequestEnvelope::from_parts(
RequestHead {
method: Method::GET,
uri: "/context-probe".parse().expect("test URI"),
headers: http::HeaderMap::new(),
credentials: None,
region: None,
service: None,
},
fn request(extensions: http::Extensions) -> S3Request<Body> {
S3Request {
input: Body::empty(),
method: Method::GET,
uri: "/context-probe".parse().expect("test URI"),
headers: http::HeaderMap::new(),
extensions,
)
credentials: None,
region: None,
service: None,
trailing_headers: None,
}
}
fn envelope(extensions: http::Extensions) -> RequestEnvelope {
RequestEnvelope::from_s3s(&request(extensions))
}
#[tokio::test]
@@ -324,14 +330,14 @@ mod tests {
.expect("register test route");
let err = router
.call_legacy(Method::GET, "/context-probe", http::Extensions::new())
.call(request(http::Extensions::new()))
.await
.expect_err("empty slot must fail closed");
assert_eq!(err.code(), &S3ErrorCode::ServiceUnavailable.into());
assert!(slot.install(distinct_context(&ambient)));
let response = router
.call_legacy(Method::GET, "/context-probe", http::Extensions::new())
.call(request(http::Extensions::new()))
.await
.expect("installed slot must serve request");
assert_eq!(response.status, Some(StatusCode::NO_CONTENT));
+4
View File
@@ -167,6 +167,10 @@ fn ensure_live_events_initialized() -> std::sync::Arc<NotificationSystem> {
system
}
/// A published lifecycle generation is usable even when some targets are
/// quarantined: healthy targets already carry events and per-target health is
/// reported by the admin target list. Only an unpublished generation means the
/// runtime is not available yet.
fn ensure_event_notifier_converged(system: &NotificationSystem) -> Result<(), NotificationError> {
if system.runtime_lifecycle_is_converged() {
Ok(())