Compare commits

..

1 Commits

Author SHA1 Message Date
overtrue 357e78c523 test(admin): build runtime-context probes on the owned envelope
The router production path is unchanged. A test-only caller builds the empty request.
2026-11-21 15:02:19 +08:00
11 changed files with 95 additions and 114 deletions
+1 -12
View File
@@ -77,18 +77,7 @@ pub async fn initialize(config: Config) -> Result<(), NotificationError> {
let task_system = system.clone();
tokio::spawn(async move {
// The one-shot API keeps its strict contract: report an activation that
// left quarantined targets so the caller can retry with a fixed config.
// The reconciliation path quarantines instead and stays admin-usable.
let result = task_system.set_targets_enabled(true, Some(config)).await.and_then(|()| {
if task_system.runtime_lifecycle_all_targets_activated() {
Ok(())
} else {
Err(NotificationError::Initialization(
"one or more notification targets failed to activate".to_string(),
))
}
});
let result = task_system.set_targets_enabled(true, Some(config)).await;
let mut legacy = LEGACY_INITIALIZATION.lock().unwrap_or_else(|err| err.into_inner());
if matches!(
legacy.as_ref(),
+1 -10
View File
@@ -393,20 +393,11 @@ impl NotificationSystem {
self.services.config_manager.lifecycle().state()
}
/// Returns whether the latest accepted lifecycle generation has been handed
/// to the runtime. A published generation serves admin operations even when
/// some targets are quarantined; per-target health is reported by the target
/// list, and the reconciler must not rebuild a published generation.
/// Returns whether the latest accepted lifecycle generation is active.
pub fn runtime_lifecycle_is_converged(&self) -> bool {
self.services.config_manager.lifecycle().is_converged()
}
/// Returns whether the last published generation activated every configured
/// target. Used by the one-shot `initialize` API's strict contract.
pub fn runtime_lifecycle_all_targets_activated(&self) -> bool {
self.services.config_manager.lifecycle().all_targets_activated()
}
/// Validates a bucket notification configuration against the runtime targets
/// without publishing it, so callers can reject a request before persisting.
pub async fn validate_bucket_notification_config(
+13 -44
View File
@@ -21,12 +21,12 @@ use rustfs_config::server_config::Config;
use rustfs_targets::Target;
use std::future::Future;
use std::pin::Pin;
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, Mutex as StdMutex};
use tokio::sync::{Mutex, RwLock};
use tokio::task::JoinHandle;
use tokio_util::sync::CancellationToken;
use tracing::{debug, info, warn};
use tracing::{debug, info};
type CurrentGeneration = Arc<dyn Fn() -> bool + Send + Sync>;
type CommitGeneration = Box<dyn FnOnce(bool) + Send>;
@@ -238,18 +238,11 @@ impl LifecycleRuntime for DefaultLifecycleRuntime {
close_old?;
close_rejected?;
close_activation_rejected?;
// A target that cannot activate is quarantined and reported by the
// admin target list. Failing the whole generation here would leave
// the lifecycle permanently unconverged and block every admin call.
if !activation_failures.is_empty() {
warn!(
event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
component = LOG_COMPONENT_NOTIFY,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
state = "degraded",
failures = %activation_failures.join("; "),
"notify runtime lifecycle"
);
return Err(NotificationError::Initialization(format!(
"one or more notification targets failed to activate: {}",
activation_failures.join("; ")
)));
}
Ok(())
})))
@@ -288,7 +281,6 @@ impl LifecycleRuntime for DefaultLifecycleRuntime {
struct LifecycleCoordinatorInner {
applied_generation: AtomicU64,
apply_lock: Mutex<()>,
all_targets_activated: AtomicBool,
desired: StdMutex<DesiredState>,
publication_gate: Arc<StdMutex<()>>,
runtime: Arc<dyn LifecycleRuntime>,
@@ -317,7 +309,6 @@ impl NotifyLifecycleCoordinator {
inner: Arc::new(LifecycleCoordinatorInner {
applied_generation: AtomicU64::new(0),
apply_lock: Mutex::new(()),
all_targets_activated: AtomicBool::new(true),
desired: StdMutex::new(DesiredState {
cancellation: CancellationToken::new(),
config: initial_config,
@@ -366,22 +357,11 @@ impl NotifyLifecycleCoordinator {
*self.inner.state.lock().unwrap_or_else(|err| err.into_inner())
}
/// Whether the latest accepted generation has been handed to the runtime.
/// This is the rebuild trigger and the admin availability gate: a published
/// generation stays published until the configuration changes, so one
/// unreachable target cannot keep the runtime churning through rebuilds.
pub(crate) fn is_converged(&self) -> bool {
let desired_generation = self.inner.desired.lock().unwrap_or_else(|err| err.into_inner()).generation;
self.inner.applied_generation.load(Ordering::Acquire) >= desired_generation
}
/// Whether the last published generation activated every configured target.
/// Per-target health is reported by the admin target list; this only drives
/// the one-shot `initialize` API's strict contract.
pub(crate) fn all_targets_activated(&self) -> bool {
self.inner.all_targets_activated.load(Ordering::Acquire)
}
fn accept<F>(&self, allow_terminated: bool, update: F) -> NotificationLifecycleTransition
where
F: FnOnce(&mut DesiredState) -> Result<bool, NotificationError>,
@@ -474,8 +454,9 @@ impl LifecycleCoordinatorInner {
fn mark_published(&self, generation: u64, state: NotificationRuntimeState, complete: bool) {
*self.state.lock().unwrap_or_else(|err| err.into_inner()) = state;
self.all_targets_activated.store(complete, Ordering::Release);
self.applied_generation.store(generation, Ordering::Release);
if complete {
self.applied_generation.store(generation, Ordering::Release);
}
}
}
@@ -998,7 +979,7 @@ mod tests {
}
#[tokio::test]
async fn partial_activation_quarantines_failed_target_without_rebuild_churn() {
async fn partial_activation_reports_error_and_same_config_can_retry() {
// The template's single-failure init budget is shared by every clone the factory hands
// out, so the "bad" instance fails once in the first generation and recovers on retry.
let bad_template = MockTarget::new("bad", RETRY_INIT_TARGET_TYPE).with_init_failures(1);
@@ -1025,25 +1006,14 @@ mod tests {
.insert("bad".to_string(), bad);
let (coordinator, runtime_view, _) = real_runtime(config.clone(), Arc::new(TargetRegistry::with_plugins(plugins)), None);
// A quarantined target is a per-target health outcome, so the mutation
// itself succeeds and admin callers are not locked out.
coordinator
.set_mode(true, Some(config.clone()))
.wait()
.await
.expect("a quarantined target must not fail the lifecycle transition");
.expect_err("the originating transition must report the failed target");
let partial = runtime_view.runtime_status_snapshot().await;
assert_eq!(partial.target_count, 2, "a quarantined target stays in the runtime beside healthy ones");
// A published generation must not look stale: otherwise the reconciler
// tears the whole runtime down and rebuilds it on every tick.
assert!(
coordinator.is_converged(),
"a quarantined target must not leave the generation unpublished"
);
assert!(
!coordinator.all_targets_activated(),
"the quarantined target is still reported as not fully activated"
);
assert_eq!(partial.target_count, 1, "the healthy target should remain fault-isolated and active");
assert!(!coordinator.is_converged(), "a partial generation must remain retryable");
assert_eq!(coordinator.state(), NotificationRuntimeState::TargetsEnabled { generation: 1 });
coordinator
@@ -1054,7 +1024,6 @@ mod tests {
let recovered = runtime_view.runtime_status_snapshot().await;
assert_eq!(recovered.target_count, 2);
assert!(coordinator.is_converged());
assert!(coordinator.all_targets_activated(), "the transient failure recovers on the next apply");
assert_eq!(coordinator.state(), NotificationRuntimeState::TargetsEnabled { generation: 2 });
coordinator
+17 -5
View File
@@ -162,6 +162,7 @@ where
) -> PreparedActivation<E> {
let mut prepared = Vec::with_capacity(targets.len());
let mut failures = Vec::new();
let mut rejected_targets = Vec::new();
let mut targets = targets.into_iter();
while let Some(target) = targets.next() {
match prepare_target(target, cancellation).await {
@@ -178,6 +179,19 @@ where
});
prepared.push(target);
}
PrepareTargetResult::Failed { error, target } => {
drop(error);
let target_id = target.id().to_string();
tracing::warn!(
target_id,
reason = "initialization_failed",
"Target initialization failed during lifecycle preparation"
);
failures.push(TargetActivationFailure {
detail: format!("{target_id}: initialization failed"),
});
rejected_targets.push(Arc::from(target));
}
PrepareTargetResult::Cancelled(target) => {
prepared.push(Arc::from(target));
prepared.extend(targets.map(Arc::from));
@@ -187,7 +201,7 @@ where
}
PreparedActivation {
failures,
rejected_targets: Vec::new(),
rejected_targets,
targets: prepared,
}
}
@@ -601,15 +615,13 @@ mod tests {
}
#[tokio::test]
async fn builtin_adapter_keeps_non_store_target_when_init_fails() {
async fn builtin_adapter_skips_non_store_target_when_init_fails() {
let adapter = builtin_adapter();
let target = MockTarget::new("primary", "webhook").with_init_failures(usize::MAX);
// A target without a queue store has nothing to replay, but it stays in
// the runtime so it self-heals and remains visible to the target list.
let activation = adapter.activate_with_replay(vec![Box::new(target)]).await;
assert_eq!(activation.targets.len(), 1);
assert!(activation.targets.is_empty());
assert!(activation.replay_workers.is_empty());
}
+22 -5
View File
@@ -87,7 +87,14 @@ where
E: PluginEvent,
{
Ready(SharedTarget<E>),
Degraded { error: TargetError, target: SharedTarget<E> },
Degraded {
error: TargetError,
target: SharedTarget<E>,
},
Failed {
error: TargetError,
target: Box<dyn Target<E> + Send + Sync>,
},
Cancelled(Box<dyn Target<E> + Send + Sync>),
}
@@ -530,7 +537,12 @@ where
) -> (mpsc::Sender<()>, JoinHandle<()>),
{
let shared = match prepare_target(target, None).await {
PrepareTargetResult::Ready(target) | PrepareTargetResult::Degraded { target, .. } => target,
PrepareTargetResult::Ready(target) => target,
PrepareTargetResult::Degraded { target, .. } => target,
PrepareTargetResult::Failed { target, .. } => {
let _ = target.close().await;
return None;
}
PrepareTargetResult::Cancelled(_) => unreachable!("preparation without a cancellation token cannot be cancelled"),
};
let target_id = shared.id().to_string();
@@ -555,6 +567,7 @@ where
E: PluginEvent,
{
let target_id = target.id().to_string();
let has_store = target.store().is_some();
let init_result = match cancellation {
Some(cancellation) => {
@@ -569,9 +582,13 @@ where
if let Err(err) = init_result {
tracing::error!(target_id = %target_id, reason = "initialization_failed", "Failed to initialize target");
// Keep the target in the runtime whether or not it has a queue store: a
// quarantined target self-heals through delivery retries and is reported
// by the target list, so recovery never needs a runtime rebuild.
if !has_store {
return PrepareTargetResult::Failed { error: err, target };
}
tracing::warn!(
target_id = %target_id,
"Proceeding with store-backed target despite init failure"
);
return PrepareTargetResult::Degraded {
error: err,
target: Arc::from(target),
-7
View File
@@ -221,13 +221,6 @@ pub fn register_notification_target_route(r: &mut S3Router<AdminOperation>) -> s
AdminOperation(&NotificationTarget {}),
)?;
r.insert(
Method::DELETE,
format!("{}{}", ADMIN_PREFIX, "/v3/target/{target_type}/{target_name}").as_str(),
AdminOperation(&RemoveNotificationTarget {}),
)?;
// Legacy spelling kept for consoles that still call the reset suffix.
r.insert(
Method::DELETE,
format!("{}{}", ADMIN_PREFIX, "/v3/target/{target_type}/{target_name}/reset").as_str(),
-6
View File
@@ -305,12 +305,6 @@ pub const ADMIN_ROUTE_POLICY_SPECS: &[AdminRouteSpec] = &[
SET_BUCKET_TARGET,
RouteRiskLevel::High,
),
admin(
HttpMethod::Delete,
"/rustfs/admin/v3/target/{target_type}/{target_name}",
SET_BUCKET_TARGET,
RouteRiskLevel::High,
),
admin(
HttpMethod::Delete,
"/rustfs/admin/v3/target/{target_type}/{target_name}/reset",
@@ -212,7 +212,6 @@ fn expected_admin_route_matrix() -> Vec<RouteMatrixEntry> {
admin_route(Method::GET, "/v3/idp/builtin/policy-entities"),
admin_route(Method::GET, "/v3/target/list"),
admin_route_sample(Method::PUT, "/v3/target/{target_type}/{target_name}", "/v3/target/webhook/test-target"),
admin_route_sample(Method::DELETE, "/v3/target/{target_type}/{target_name}", "/v3/target/webhook/test-target"),
admin_route_sample(
Method::DELETE,
"/v3/target/{target_type}/{target_name}/reset",
+27
View File
@@ -1021,6 +1021,33 @@ impl<T> S3Router<T> {
}
}
#[cfg(test)]
impl<T: OwnedOperation> S3Router<T> {
/// Dispatches one legacy admin request. Callers keep their assertions and do not name s3s.
pub(crate) async fn call_legacy(
&self,
method: Method,
uri: &str,
extensions: http::Extensions,
) -> s3s::S3Result<S3Response<Body>> {
S3Route::call(
self,
S3Request {
input: Body::empty(),
method,
uri: uri.parse().expect("test URI"),
headers: HeaderMap::new(),
extensions,
credentials: None,
region: None,
service: None,
trailing_headers: None,
},
)
.await
}
}
impl<T> Default for S3Router<T> {
fn default() -> Self {
Self::new(false)
+14 -20
View File
@@ -142,14 +142,12 @@ mod tests {
};
use crate::admin::operation::{AdminRequest, AdminResponse, Operation};
use crate::admin::router::S3Router;
use crate::app::RequestEnvelope;
use crate::app::{RequestEnvelope, RequestHead};
use hyper::{Method, StatusCode};
use matchit::Params;
use rustfs_iam::{store::object::ObjectStore, sys::IamSys};
use rustfs_kms::KmsServiceManager;
use rustfs_s3_types::{S3ErrorCode, s3_error};
use s3s::route::S3Route;
use s3s::{Body, S3Request};
use std::sync::Arc;
struct UnreadyIam;
@@ -192,22 +190,18 @@ mod tests {
Arc::new(AppContext::new(context.object_store(), Arc::new(UnreadyIam), Arc::new(TestKms)))
}
fn request(extensions: http::Extensions) -> S3Request<Body> {
S3Request {
input: Body::empty(),
method: Method::GET,
uri: "/context-probe".parse().expect("test URI"),
headers: http::HeaderMap::new(),
extensions,
credentials: None,
region: None,
service: None,
trailing_headers: None,
}
}
fn envelope(extensions: http::Extensions) -> RequestEnvelope {
RequestEnvelope::from_s3s(&request(extensions))
RequestEnvelope::from_parts(
RequestHead {
method: Method::GET,
uri: "/context-probe".parse().expect("test URI"),
headers: http::HeaderMap::new(),
credentials: None,
region: None,
service: None,
},
extensions,
)
}
#[tokio::test]
@@ -330,14 +324,14 @@ mod tests {
.expect("register test route");
let err = router
.call(request(http::Extensions::new()))
.call_legacy(Method::GET, "/context-probe", http::Extensions::new())
.await
.expect_err("empty slot must fail closed");
assert_eq!(err.code(), &S3ErrorCode::ServiceUnavailable.into());
assert!(slot.install(distinct_context(&ambient)));
let response = router
.call(request(http::Extensions::new()))
.call_legacy(Method::GET, "/context-probe", http::Extensions::new())
.await
.expect("installed slot must serve request");
assert_eq!(response.status, Some(StatusCode::NO_CONTENT));
-4
View File
@@ -167,10 +167,6 @@ fn ensure_live_events_initialized() -> std::sync::Arc<NotificationSystem> {
system
}
/// A published lifecycle generation is usable even when some targets are
/// quarantined: healthy targets already carry events and per-target health is
/// reported by the admin target list. Only an unpublished generation means the
/// runtime is not available yet.
fn ensure_event_notifier_converged(system: &NotificationSystem) -> Result<(), NotificationError> {
if system.runtime_lifecycle_is_converged() {
Ok(())