mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-21 07:26:37 +00:00
Compare commits
5 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 7852b0cfde | |||
| efa8bc6d4b | |||
| fefd238d57 | |||
| a8312c1157 | |||
| 323f308d57 |
@@ -77,7 +77,18 @@ pub async fn initialize(config: Config) -> Result<(), NotificationError> {
|
||||
|
||||
let task_system = system.clone();
|
||||
tokio::spawn(async move {
|
||||
let result = task_system.set_targets_enabled(true, Some(config)).await;
|
||||
// The one-shot API keeps its strict contract: report an activation that
|
||||
// left quarantined targets so the caller can retry with a fixed config.
|
||||
// The reconciliation path quarantines instead and stays admin-usable.
|
||||
let result = task_system.set_targets_enabled(true, Some(config)).await.and_then(|()| {
|
||||
if task_system.runtime_lifecycle_all_targets_activated() {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(NotificationError::Initialization(
|
||||
"one or more notification targets failed to activate".to_string(),
|
||||
))
|
||||
}
|
||||
});
|
||||
let mut legacy = LEGACY_INITIALIZATION.lock().unwrap_or_else(|err| err.into_inner());
|
||||
if matches!(
|
||||
legacy.as_ref(),
|
||||
|
||||
@@ -393,11 +393,20 @@ impl NotificationSystem {
|
||||
self.services.config_manager.lifecycle().state()
|
||||
}
|
||||
|
||||
/// Returns whether the latest accepted lifecycle generation is active.
|
||||
/// Returns whether the latest accepted lifecycle generation has been handed
|
||||
/// to the runtime. A published generation serves admin operations even when
|
||||
/// some targets are quarantined; per-target health is reported by the target
|
||||
/// list, and the reconciler must not rebuild a published generation.
|
||||
pub fn runtime_lifecycle_is_converged(&self) -> bool {
|
||||
self.services.config_manager.lifecycle().is_converged()
|
||||
}
|
||||
|
||||
/// Returns whether the last published generation activated every configured
|
||||
/// target. Used by the one-shot `initialize` API's strict contract.
|
||||
pub fn runtime_lifecycle_all_targets_activated(&self) -> bool {
|
||||
self.services.config_manager.lifecycle().all_targets_activated()
|
||||
}
|
||||
|
||||
/// Validates a bucket notification configuration against the runtime targets
|
||||
/// without publishing it, so callers can reject a request before persisting.
|
||||
pub async fn validate_bucket_notification_config(
|
||||
|
||||
@@ -21,12 +21,12 @@ use rustfs_config::server_config::Config;
|
||||
use rustfs_targets::Target;
|
||||
use std::future::Future;
|
||||
use std::pin::Pin;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
||||
use std::sync::{Arc, Mutex as StdMutex};
|
||||
use tokio::sync::{Mutex, RwLock};
|
||||
use tokio::task::JoinHandle;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::{debug, info};
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
type CurrentGeneration = Arc<dyn Fn() -> bool + Send + Sync>;
|
||||
type CommitGeneration = Box<dyn FnOnce(bool) + Send>;
|
||||
@@ -238,11 +238,18 @@ impl LifecycleRuntime for DefaultLifecycleRuntime {
|
||||
close_old?;
|
||||
close_rejected?;
|
||||
close_activation_rejected?;
|
||||
// A target that cannot activate is quarantined and reported by the
|
||||
// admin target list. Failing the whole generation here would leave
|
||||
// the lifecycle permanently unconverged and block every admin call.
|
||||
if !activation_failures.is_empty() {
|
||||
return Err(NotificationError::Initialization(format!(
|
||||
"one or more notification targets failed to activate: {}",
|
||||
activation_failures.join("; ")
|
||||
)));
|
||||
warn!(
|
||||
event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
|
||||
component = LOG_COMPONENT_NOTIFY,
|
||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||
state = "degraded",
|
||||
failures = %activation_failures.join("; "),
|
||||
"notify runtime lifecycle"
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
})))
|
||||
@@ -281,6 +288,7 @@ impl LifecycleRuntime for DefaultLifecycleRuntime {
|
||||
struct LifecycleCoordinatorInner {
|
||||
applied_generation: AtomicU64,
|
||||
apply_lock: Mutex<()>,
|
||||
all_targets_activated: AtomicBool,
|
||||
desired: StdMutex<DesiredState>,
|
||||
publication_gate: Arc<StdMutex<()>>,
|
||||
runtime: Arc<dyn LifecycleRuntime>,
|
||||
@@ -309,6 +317,7 @@ impl NotifyLifecycleCoordinator {
|
||||
inner: Arc::new(LifecycleCoordinatorInner {
|
||||
applied_generation: AtomicU64::new(0),
|
||||
apply_lock: Mutex::new(()),
|
||||
all_targets_activated: AtomicBool::new(true),
|
||||
desired: StdMutex::new(DesiredState {
|
||||
cancellation: CancellationToken::new(),
|
||||
config: initial_config,
|
||||
@@ -357,11 +366,22 @@ impl NotifyLifecycleCoordinator {
|
||||
*self.inner.state.lock().unwrap_or_else(|err| err.into_inner())
|
||||
}
|
||||
|
||||
/// Whether the latest accepted generation has been handed to the runtime.
|
||||
/// This is the rebuild trigger and the admin availability gate: a published
|
||||
/// generation stays published until the configuration changes, so one
|
||||
/// unreachable target cannot keep the runtime churning through rebuilds.
|
||||
pub(crate) fn is_converged(&self) -> bool {
|
||||
let desired_generation = self.inner.desired.lock().unwrap_or_else(|err| err.into_inner()).generation;
|
||||
self.inner.applied_generation.load(Ordering::Acquire) >= desired_generation
|
||||
}
|
||||
|
||||
/// Whether the last published generation activated every configured target.
|
||||
/// Per-target health is reported by the admin target list; this only drives
|
||||
/// the one-shot `initialize` API's strict contract.
|
||||
pub(crate) fn all_targets_activated(&self) -> bool {
|
||||
self.inner.all_targets_activated.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
fn accept<F>(&self, allow_terminated: bool, update: F) -> NotificationLifecycleTransition
|
||||
where
|
||||
F: FnOnce(&mut DesiredState) -> Result<bool, NotificationError>,
|
||||
@@ -454,9 +474,8 @@ impl LifecycleCoordinatorInner {
|
||||
|
||||
fn mark_published(&self, generation: u64, state: NotificationRuntimeState, complete: bool) {
|
||||
*self.state.lock().unwrap_or_else(|err| err.into_inner()) = state;
|
||||
if complete {
|
||||
self.applied_generation.store(generation, Ordering::Release);
|
||||
}
|
||||
self.all_targets_activated.store(complete, Ordering::Release);
|
||||
self.applied_generation.store(generation, Ordering::Release);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -979,7 +998,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn partial_activation_reports_error_and_same_config_can_retry() {
|
||||
async fn partial_activation_quarantines_failed_target_without_rebuild_churn() {
|
||||
// The template's single-failure init budget is shared by every clone the factory hands
|
||||
// out, so the "bad" instance fails once in the first generation and recovers on retry.
|
||||
let bad_template = MockTarget::new("bad", RETRY_INIT_TARGET_TYPE).with_init_failures(1);
|
||||
@@ -1006,14 +1025,25 @@ mod tests {
|
||||
.insert("bad".to_string(), bad);
|
||||
let (coordinator, runtime_view, _) = real_runtime(config.clone(), Arc::new(TargetRegistry::with_plugins(plugins)), None);
|
||||
|
||||
// A quarantined target is a per-target health outcome, so the mutation
|
||||
// itself succeeds and admin callers are not locked out.
|
||||
coordinator
|
||||
.set_mode(true, Some(config.clone()))
|
||||
.wait()
|
||||
.await
|
||||
.expect_err("the originating transition must report the failed target");
|
||||
.expect("a quarantined target must not fail the lifecycle transition");
|
||||
let partial = runtime_view.runtime_status_snapshot().await;
|
||||
assert_eq!(partial.target_count, 1, "the healthy target should remain fault-isolated and active");
|
||||
assert!(!coordinator.is_converged(), "a partial generation must remain retryable");
|
||||
assert_eq!(partial.target_count, 2, "a quarantined target stays in the runtime beside healthy ones");
|
||||
// A published generation must not look stale: otherwise the reconciler
|
||||
// tears the whole runtime down and rebuilds it on every tick.
|
||||
assert!(
|
||||
coordinator.is_converged(),
|
||||
"a quarantined target must not leave the generation unpublished"
|
||||
);
|
||||
assert!(
|
||||
!coordinator.all_targets_activated(),
|
||||
"the quarantined target is still reported as not fully activated"
|
||||
);
|
||||
assert_eq!(coordinator.state(), NotificationRuntimeState::TargetsEnabled { generation: 1 });
|
||||
|
||||
coordinator
|
||||
@@ -1024,6 +1054,7 @@ mod tests {
|
||||
let recovered = runtime_view.runtime_status_snapshot().await;
|
||||
assert_eq!(recovered.target_count, 2);
|
||||
assert!(coordinator.is_converged());
|
||||
assert!(coordinator.all_targets_activated(), "the transient failure recovers on the next apply");
|
||||
assert_eq!(coordinator.state(), NotificationRuntimeState::TargetsEnabled { generation: 2 });
|
||||
|
||||
coordinator
|
||||
|
||||
@@ -162,7 +162,6 @@ where
|
||||
) -> PreparedActivation<E> {
|
||||
let mut prepared = Vec::with_capacity(targets.len());
|
||||
let mut failures = Vec::new();
|
||||
let mut rejected_targets = Vec::new();
|
||||
let mut targets = targets.into_iter();
|
||||
while let Some(target) = targets.next() {
|
||||
match prepare_target(target, cancellation).await {
|
||||
@@ -179,19 +178,6 @@ where
|
||||
});
|
||||
prepared.push(target);
|
||||
}
|
||||
PrepareTargetResult::Failed { error, target } => {
|
||||
drop(error);
|
||||
let target_id = target.id().to_string();
|
||||
tracing::warn!(
|
||||
target_id,
|
||||
reason = "initialization_failed",
|
||||
"Target initialization failed during lifecycle preparation"
|
||||
);
|
||||
failures.push(TargetActivationFailure {
|
||||
detail: format!("{target_id}: initialization failed"),
|
||||
});
|
||||
rejected_targets.push(Arc::from(target));
|
||||
}
|
||||
PrepareTargetResult::Cancelled(target) => {
|
||||
prepared.push(Arc::from(target));
|
||||
prepared.extend(targets.map(Arc::from));
|
||||
@@ -201,7 +187,7 @@ where
|
||||
}
|
||||
PreparedActivation {
|
||||
failures,
|
||||
rejected_targets,
|
||||
rejected_targets: Vec::new(),
|
||||
targets: prepared,
|
||||
}
|
||||
}
|
||||
@@ -615,13 +601,15 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn builtin_adapter_skips_non_store_target_when_init_fails() {
|
||||
async fn builtin_adapter_keeps_non_store_target_when_init_fails() {
|
||||
let adapter = builtin_adapter();
|
||||
let target = MockTarget::new("primary", "webhook").with_init_failures(usize::MAX);
|
||||
|
||||
// A target without a queue store has nothing to replay, but it stays in
|
||||
// the runtime so it self-heals and remains visible to the target list.
|
||||
let activation = adapter.activate_with_replay(vec![Box::new(target)]).await;
|
||||
|
||||
assert!(activation.targets.is_empty());
|
||||
assert_eq!(activation.targets.len(), 1);
|
||||
assert!(activation.replay_workers.is_empty());
|
||||
}
|
||||
|
||||
|
||||
@@ -87,14 +87,7 @@ where
|
||||
E: PluginEvent,
|
||||
{
|
||||
Ready(SharedTarget<E>),
|
||||
Degraded {
|
||||
error: TargetError,
|
||||
target: SharedTarget<E>,
|
||||
},
|
||||
Failed {
|
||||
error: TargetError,
|
||||
target: Box<dyn Target<E> + Send + Sync>,
|
||||
},
|
||||
Degraded { error: TargetError, target: SharedTarget<E> },
|
||||
Cancelled(Box<dyn Target<E> + Send + Sync>),
|
||||
}
|
||||
|
||||
@@ -537,12 +530,7 @@ where
|
||||
) -> (mpsc::Sender<()>, JoinHandle<()>),
|
||||
{
|
||||
let shared = match prepare_target(target, None).await {
|
||||
PrepareTargetResult::Ready(target) => target,
|
||||
PrepareTargetResult::Degraded { target, .. } => target,
|
||||
PrepareTargetResult::Failed { target, .. } => {
|
||||
let _ = target.close().await;
|
||||
return None;
|
||||
}
|
||||
PrepareTargetResult::Ready(target) | PrepareTargetResult::Degraded { target, .. } => target,
|
||||
PrepareTargetResult::Cancelled(_) => unreachable!("preparation without a cancellation token cannot be cancelled"),
|
||||
};
|
||||
let target_id = shared.id().to_string();
|
||||
@@ -567,7 +555,6 @@ where
|
||||
E: PluginEvent,
|
||||
{
|
||||
let target_id = target.id().to_string();
|
||||
let has_store = target.store().is_some();
|
||||
|
||||
let init_result = match cancellation {
|
||||
Some(cancellation) => {
|
||||
@@ -582,13 +569,9 @@ where
|
||||
|
||||
if let Err(err) = init_result {
|
||||
tracing::error!(target_id = %target_id, reason = "initialization_failed", "Failed to initialize target");
|
||||
if !has_store {
|
||||
return PrepareTargetResult::Failed { error: err, target };
|
||||
}
|
||||
tracing::warn!(
|
||||
target_id = %target_id,
|
||||
"Proceeding with store-backed target despite init failure"
|
||||
);
|
||||
// Keep the target in the runtime whether or not it has a queue store: a
|
||||
// quarantined target self-heals through delivery retries and is reported
|
||||
// by the target list, so recovery never needs a runtime rebuild.
|
||||
return PrepareTargetResult::Degraded {
|
||||
error: err,
|
||||
target: Arc::from(target),
|
||||
|
||||
@@ -221,6 +221,13 @@ pub fn register_notification_target_route(r: &mut S3Router<AdminOperation>) -> s
|
||||
AdminOperation(&NotificationTarget {}),
|
||||
)?;
|
||||
|
||||
r.insert(
|
||||
Method::DELETE,
|
||||
format!("{}{}", ADMIN_PREFIX, "/v3/target/{target_type}/{target_name}").as_str(),
|
||||
AdminOperation(&RemoveNotificationTarget {}),
|
||||
)?;
|
||||
|
||||
// Legacy spelling kept for consoles that still call the reset suffix.
|
||||
r.insert(
|
||||
Method::DELETE,
|
||||
format!("{}{}", ADMIN_PREFIX, "/v3/target/{target_type}/{target_name}/reset").as_str(),
|
||||
|
||||
@@ -305,6 +305,12 @@ pub const ADMIN_ROUTE_POLICY_SPECS: &[AdminRouteSpec] = &[
|
||||
SET_BUCKET_TARGET,
|
||||
RouteRiskLevel::High,
|
||||
),
|
||||
admin(
|
||||
HttpMethod::Delete,
|
||||
"/rustfs/admin/v3/target/{target_type}/{target_name}",
|
||||
SET_BUCKET_TARGET,
|
||||
RouteRiskLevel::High,
|
||||
),
|
||||
admin(
|
||||
HttpMethod::Delete,
|
||||
"/rustfs/admin/v3/target/{target_type}/{target_name}/reset",
|
||||
|
||||
@@ -212,6 +212,7 @@ fn expected_admin_route_matrix() -> Vec<RouteMatrixEntry> {
|
||||
admin_route(Method::GET, "/v3/idp/builtin/policy-entities"),
|
||||
admin_route(Method::GET, "/v3/target/list"),
|
||||
admin_route_sample(Method::PUT, "/v3/target/{target_type}/{target_name}", "/v3/target/webhook/test-target"),
|
||||
admin_route_sample(Method::DELETE, "/v3/target/{target_type}/{target_name}", "/v3/target/webhook/test-target"),
|
||||
admin_route_sample(
|
||||
Method::DELETE,
|
||||
"/v3/target/{target_type}/{target_name}/reset",
|
||||
|
||||
@@ -1021,33 +1021,6 @@ impl<T> S3Router<T> {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl<T: OwnedOperation> S3Router<T> {
|
||||
/// Dispatches one legacy admin request. Callers keep their assertions and do not name s3s.
|
||||
pub(crate) async fn call_legacy(
|
||||
&self,
|
||||
method: Method,
|
||||
uri: &str,
|
||||
extensions: http::Extensions,
|
||||
) -> s3s::S3Result<S3Response<Body>> {
|
||||
S3Route::call(
|
||||
self,
|
||||
S3Request {
|
||||
input: Body::empty(),
|
||||
method,
|
||||
uri: uri.parse().expect("test URI"),
|
||||
headers: HeaderMap::new(),
|
||||
extensions,
|
||||
credentials: None,
|
||||
region: None,
|
||||
service: None,
|
||||
trailing_headers: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> Default for S3Router<T> {
|
||||
fn default() -> Self {
|
||||
Self::new(false)
|
||||
|
||||
@@ -142,12 +142,14 @@ mod tests {
|
||||
};
|
||||
use crate::admin::operation::{AdminRequest, AdminResponse, Operation};
|
||||
use crate::admin::router::S3Router;
|
||||
use crate::app::{RequestEnvelope, RequestHead};
|
||||
use crate::app::RequestEnvelope;
|
||||
use hyper::{Method, StatusCode};
|
||||
use matchit::Params;
|
||||
use rustfs_iam::{store::object::ObjectStore, sys::IamSys};
|
||||
use rustfs_kms::KmsServiceManager;
|
||||
use rustfs_s3_types::{S3ErrorCode, s3_error};
|
||||
use s3s::route::S3Route;
|
||||
use s3s::{Body, S3Request};
|
||||
use std::sync::Arc;
|
||||
|
||||
struct UnreadyIam;
|
||||
@@ -190,18 +192,22 @@ mod tests {
|
||||
Arc::new(AppContext::new(context.object_store(), Arc::new(UnreadyIam), Arc::new(TestKms)))
|
||||
}
|
||||
|
||||
fn envelope(extensions: http::Extensions) -> RequestEnvelope {
|
||||
RequestEnvelope::from_parts(
|
||||
RequestHead {
|
||||
method: Method::GET,
|
||||
uri: "/context-probe".parse().expect("test URI"),
|
||||
headers: http::HeaderMap::new(),
|
||||
credentials: None,
|
||||
region: None,
|
||||
service: None,
|
||||
},
|
||||
fn request(extensions: http::Extensions) -> S3Request<Body> {
|
||||
S3Request {
|
||||
input: Body::empty(),
|
||||
method: Method::GET,
|
||||
uri: "/context-probe".parse().expect("test URI"),
|
||||
headers: http::HeaderMap::new(),
|
||||
extensions,
|
||||
)
|
||||
credentials: None,
|
||||
region: None,
|
||||
service: None,
|
||||
trailing_headers: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn envelope(extensions: http::Extensions) -> RequestEnvelope {
|
||||
RequestEnvelope::from_s3s(&request(extensions))
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -324,14 +330,14 @@ mod tests {
|
||||
.expect("register test route");
|
||||
|
||||
let err = router
|
||||
.call_legacy(Method::GET, "/context-probe", http::Extensions::new())
|
||||
.call(request(http::Extensions::new()))
|
||||
.await
|
||||
.expect_err("empty slot must fail closed");
|
||||
assert_eq!(err.code(), &S3ErrorCode::ServiceUnavailable.into());
|
||||
|
||||
assert!(slot.install(distinct_context(&ambient)));
|
||||
let response = router
|
||||
.call_legacy(Method::GET, "/context-probe", http::Extensions::new())
|
||||
.call(request(http::Extensions::new()))
|
||||
.await
|
||||
.expect("installed slot must serve request");
|
||||
assert_eq!(response.status, Some(StatusCode::NO_CONTENT));
|
||||
|
||||
@@ -167,6 +167,10 @@ fn ensure_live_events_initialized() -> std::sync::Arc<NotificationSystem> {
|
||||
system
|
||||
}
|
||||
|
||||
/// A published lifecycle generation is usable even when some targets are
|
||||
/// quarantined: healthy targets already carry events and per-target health is
|
||||
/// reported by the admin target list. Only an unpublished generation means the
|
||||
/// runtime is not available yet.
|
||||
fn ensure_event_notifier_converged(system: &NotificationSystem) -> Result<(), NotificationError> {
|
||||
if system.runtime_lifecycle_is_converged() {
|
||||
Ok(())
|
||||
|
||||
Reference in New Issue
Block a user