gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-08 22:33:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(notify): unify runtime lifecycle coordination (#5088)

Browse Source 
* fix(notify): unify runtime lifecycle coordination

* fix(notify): repair lifecycle convergence checks

* fix(admin): expose effective notify state (#5097)
This commit is contained in:
cxymds
2026-07-22 13:01:15 +08:00
committed by GitHub
parent 0adb3c5ea1
commit 1655f3192e
66 changed files with 8091 additions and 1370 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rustfs/src/admin/handlers/audit.rs
+47 -31
View File
@@ -221,7 +221,7 @@ async fn authorize_audit_admin_request(req: &S3Request<Body>, action: AdminActio
validate_admin_request(&req.headers, &cred, owner, false, vec![Action::AdminAction(action)], remote_addr).await
}
fn audit_target_mutation_block_reason(config: &Config, target_type: &str, target_name: &str) -> Option<String> {
fn audit_target_mutation_block_reason(config: &Config, target_type: &str, target_name: &str) -> S3Result<Option<String>> {
shared_target_mutation_block_reason(
audit_target_specs(),
AUDIT_ROUTE_PREFIX,
@@ -248,16 +248,18 @@ async fn audit_target_operation_block_reason(action: &str) -> Option<String> {
target_module_disabled_reason("audit", rustfs_config::ENV_AUDIT_ENABLE, is_audit_module_enabled(), action)
}
fn merge_audit_endpoints(config: &Config, runtime_statuses: HashMap<EndpointKey, String>) -> Vec<AuditEndpoint> {
shared_merge_target_endpoints(audit_target_specs(), AUDIT_ROUTE_PREFIX, config, runtime_statuses)
.into_iter()
.map(|endpoint| AuditEndpoint {
account_id: endpoint.account_id,
service: endpoint.service,
status: endpoint.status,
source: endpoint.source,
})
.collect()
fn merge_audit_endpoints(config: &Config, runtime_statuses: HashMap<EndpointKey, String>) -> S3Result<Vec<AuditEndpoint>> {
Ok(
shared_merge_target_endpoints(audit_target_specs(), AUDIT_ROUTE_PREFIX, config, runtime_statuses)?
.into_iter()
.map(|endpoint| AuditEndpoint {
account_id: endpoint.account_id,
service: endpoint.service,
status: endpoint.status,
source: endpoint.source,
})
.collect(),
)
}
fn extract_target_params<'a>(params: &'a Params<'_, '_>) -> S3Result<(&'a str, &'a str)> {
@@ -279,7 +281,7 @@ impl Operation for AuditTargetConfig {
return Err(s3_error!(InvalidRequest, "{reason}"));
}
let config_snapshot = load_server_config_from_store().await?;
if let Some(reason) = audit_target_mutation_block_reason(&config_snapshot, target_type, target_name) {
if let Some(reason) = audit_target_mutation_block_reason(&config_snapshot, target_type, target_name)? {
log_audit_target_operation_blocked!("set_audit_target_config", Some(target_type), Some(target_name), &reason);
return Err(s3_error!(InvalidRequest, "{reason}"));
}
@@ -305,12 +307,14 @@ impl Operation for AuditTargetConfig {
)
.await?;
update_audit_config_and_reload(audit_target_specs(), |config| {
let mutation_target_type = target_type.to_lowercase();
let mutation_target_name = target_name.to_lowercase();
update_audit_config_and_reload(audit_target_specs(), move |config| {
config
.0
.entry(target_type.to_lowercase())
.entry(mutation_target_type.clone())
.or_default()
.insert(target_name.to_lowercase(), kvs.clone());
.insert(mutation_target_name.clone(), kvs.clone());
true
})
.await
@@ -344,7 +348,7 @@ impl Operation for ListAuditTargets {
}
let config = load_server_config_from_store().await?;
let audit_endpoints = merge_audit_endpoints(&config, runtime_statuses);
let audit_endpoints = merge_audit_endpoints(&config, runtime_statuses)?;
let data = serde_json::to_vec(&AuditEndpointsResponse { audit_endpoints }).map_err(|e| {
log_audit_target_request_failed!("list_audit_targets", "serialize_audit_targets_failed", None, None, e);
s3_error!(InternalError, "failed to serialize audit targets: {}", e)
@@ -369,19 +373,21 @@ impl Operation for RemoveAuditTarget {
return Err(s3_error!(InvalidRequest, "{reason}"));
}
let config_snapshot = load_server_config_from_store().await?;
if let Some(reason) = audit_target_mutation_block_reason(&config_snapshot, target_type, target_name) {
if let Some(reason) = audit_target_mutation_block_reason(&config_snapshot, target_type, target_name)? {
log_audit_target_operation_blocked!("remove_audit_target_config", Some(target_type), Some(target_name), &reason);
return Err(s3_error!(InvalidRequest, "{reason}"));
}
update_audit_config_and_reload(audit_target_specs(), |config| {
let mutation_target_type = target_type.to_lowercase();
let mutation_target_name = target_name.to_lowercase();
update_audit_config_and_reload(audit_target_specs(), move |config| {
let mut changed = false;
if let Some(targets) = config.0.get_mut(&target_type.to_lowercase()) {
if targets.remove(&target_name.to_lowercase()).is_some() {
if let Some(targets) = config.0.get_mut(&mutation_target_type) {
if targets.remove(&mutation_target_name).is_some() {
changed = true;
}
if targets.is_empty() {
config.0.remove(&target_type.to_lowercase());
config.0.remove(&mutation_target_type);
}
}
changed
@@ -469,7 +475,7 @@ mod tests {
(("mixed-target".to_string(), "webhook".to_string()), "online".to_string()),
(("env-only".to_string(), "webhook".to_string()), "online".to_string()),
]);
let merged = merge_audit_endpoints(&config, runtime);
let merged = merge_audit_endpoints(&config, runtime).expect("merge audit endpoints");
let mixed = merged
.iter()
@@ -512,7 +518,7 @@ mod tests {
(("mixed-kafka".to_string(), "kafka".to_string()), "online".to_string()),
(("env-kafka".to_string(), "kafka".to_string()), "online".to_string()),
]);
let merged = merge_audit_endpoints(&config, runtime);
let merged = merge_audit_endpoints(&config, runtime).expect("merge audit endpoints");
let mixed = merged
.iter()
@@ -549,7 +555,7 @@ mod tests {
(("mixed-amqp".to_string(), "amqp".to_string()), "online".to_string()),
(("env-amqp".to_string(), "amqp".to_string()), "online".to_string()),
]);
let merged = merge_audit_endpoints(&config, runtime);
let merged = merge_audit_endpoints(&config, runtime).expect("merge audit endpoints");
let mixed = merged
.iter()
@@ -576,7 +582,8 @@ mod tests {
],
|| {
let config = Config(HashMap::new());
let reason = audit_target_mutation_block_reason(&config, AUDIT_WEBHOOK_SUB_SYS, "primary");
let reason = audit_target_mutation_block_reason(&config, AUDIT_WEBHOOK_SUB_SYS, "primary")
.expect("audit target mutation block reason");
assert!(reason.is_some());
assert!(reason.unwrap().contains("managed by environment variables"));
},
@@ -613,7 +620,8 @@ mod tests {
AUDIT_WEBHOOK_SUB_SYS.to_string(),
HashMap::from([("primary".to_string(), enabled_kvs("on"))]),
)]));
let reason = audit_target_mutation_block_reason(&config, AUDIT_WEBHOOK_SUB_SYS, "primary");
let reason = audit_target_mutation_block_reason(&config, AUDIT_WEBHOOK_SUB_SYS, "primary")
.expect("audit target mutation block reason");
assert!(reason.is_some());
assert!(reason.unwrap().contains("both persisted config and environment variables"));
});
@@ -633,7 +641,7 @@ mod tests {
("RUSTFS_AUDIT_WEBHOOK_ENDPOINT_MIXED-DISABLED", Some("https://example.com/hook")),
],
|| {
let merged = merge_audit_endpoints(&config, HashMap::new());
let merged = merge_audit_endpoints(&config, HashMap::new()).expect("merge audit endpoints");
let mixed = merged
.iter()
.find(|entry| entry.account_id == "mixed-disabled")
@@ -655,7 +663,7 @@ mod tests {
("RUSTFS_AUDIT_WEBHOOK_ENDPOINT_ENV-ONLY", Some("https://example.com/env")),
],
|| {
let merged = merge_audit_endpoints(&config, HashMap::new());
let merged = merge_audit_endpoints(&config, HashMap::new()).expect("merge audit endpoints");
let env_only = merged
.iter()
.find(|entry| entry.account_id == "env-only")
@@ -768,7 +776,7 @@ mod tests {
],
|| {
let runtime = HashMap::from([(("PrimaryCase".to_string(), "webhook".to_string()), "online".to_string())]);
let merged = merge_audit_endpoints(&config, runtime);
let merged = merge_audit_endpoints(&config, runtime).expect("merge audit endpoints");
let mixed = merged
.iter()
.find(|entry| entry.account_id == "PrimaryCase" && entry.service == "webhook")
@@ -787,7 +795,11 @@ mod tests {
)]));
with_audit_webhook_target_env_cleared("primarycase", || {
assert!(audit_target_mutation_block_reason(&config, AUDIT_WEBHOOK_SUB_SYS, "primarycase").is_none());
assert!(
audit_target_mutation_block_reason(&config, AUDIT_WEBHOOK_SUB_SYS, "primarycase")
.expect("audit target mutation block reason")
.is_none()
);
});
}
@@ -795,7 +807,11 @@ mod tests {
fn audit_target_mutation_block_reason_allows_runtime_only_target() {
with_audit_webhook_target_env_cleared("primary", || {
let config = Config(HashMap::new());
assert!(audit_target_mutation_block_reason(&config, AUDIT_WEBHOOK_SUB_SYS, "primary").is_none());
assert!(
audit_target_mutation_block_reason(&config, AUDIT_WEBHOOK_SUB_SYS, "primary")
.expect("audit target mutation block reason")
.is_none()
);
});
}
rustfs/src/admin/handlers/audit_runtime_config.rs
+54 -35
View File
@@ -14,11 +14,15 @@
use crate::admin::handlers::target_descriptor::AdminTargetSpec;
use crate::admin::runtime_sources::{AppContext, current_app_context, current_object_store_handle_for_context};
use crate::admin::storage_api::config::{read_admin_config_without_migrate, save_admin_server_config};
use crate::admin::storage_api::config::{
read_admin_config_without_migrate, read_admin_config_without_migrate_no_lock, save_admin_server_config_no_lock,
with_admin_server_config_write_lock,
};
use rustfs_audit::{audit_system, start_audit_system as start_global_audit_system, system::AuditSystemState};
use rustfs_config::DEFAULT_DELIMITER;
use rustfs_config::server_config::Config;
use s3s::{S3Result, s3_error};
use tracing::warn;
pub(crate) async fn load_server_config_from_store_for_context(context: Option<&AppContext>) -> S3Result<Config> {
let Some(store) = current_object_store_handle_for_context(context) else {
@@ -51,30 +55,31 @@ pub(crate) async fn apply_audit_runtime_config(specs: &[AdminTargetSpec], config
match system.get_state().await {
AuditSystemState::Running | AuditSystemState::Paused | AuditSystemState::Starting => {
if has_targets {
system
.reload_config(config)
.await
.map_err(|e| s3_error!(InternalError, "failed to reload audit config: {}", e))?;
system.reload_config(config).await.map_err(|_| {
warn!(reason = "reload_failed", "Failed to reload local audit runtime");
s3_error!(InternalError, "failed to reload audit config")
})?;
} else {
system
.close()
.await
.map_err(|e| s3_error!(InternalError, "failed to stop audit system: {}", e))?;
system.close().await.map_err(|_| {
warn!(reason = "stop_failed", "Failed to stop local audit runtime");
s3_error!(InternalError, "failed to stop audit system")
})?;
}
}
AuditSystemState::Stopped | AuditSystemState::Stopping => {
if has_targets {
system
.start(config)
.await
.map_err(|e| s3_error!(InternalError, "failed to start audit system: {}", e))?;
system.start(config).await.map_err(|_| {
warn!(reason = "start_failed", "Failed to start local audit runtime");
s3_error!(InternalError, "failed to start audit system")
})?;
}
}
}
} else if has_targets {
start_global_audit_system(config)
.await
.map_err(|e| s3_error!(InternalError, "failed to start audit system: {}", e))?;
start_global_audit_system(config).await.map_err(|_| {
warn!(reason = "start_failed", "Failed to start global audit runtime");
s3_error!(InternalError, "failed to start audit system")
})?;
}
Ok(())
@@ -86,30 +91,40 @@ async fn update_audit_config_and_reload_for_context<F>(
mut modifier: F,
) -> S3Result<()>
where
F: FnMut(&mut Config) -> bool,
F: FnMut(&mut Config) -> bool + Send + 'static,
{
let Some(store) = current_object_store_handle_for_context(context) else {
return Err(s3_error!(InternalError, "server storage not initialized"));
};
let mut config = read_admin_config_without_migrate(store.clone())
.await
.map_err(|e| s3_error!(InternalError, "failed to read server config: {}", e))?;
let specs = specs.to_vec();
let lock_store = store.clone();
with_admin_server_config_write_lock(lock_store, move || async move {
let mut config = read_admin_config_without_migrate_no_lock(store.clone())
.await
.map_err(|e| s3_error!(InternalError, "failed to read server config: {}", e))?;
if !modifier(&mut config) {
return Ok(());
}
if !modifier(&mut config) {
return Ok(());
}
save_admin_server_config(store, &config)
.await
.map_err(|e| s3_error!(InternalError, "failed to save audit config: {}", e))?;
save_admin_server_config_no_lock(store, &config)
.await
.map_err(|e| s3_error!(InternalError, "failed to save audit config: {}", e))?;
apply_audit_runtime_config(specs, config).await
// Keep persistence and runtime publication in one detached, serialized
// mutation. Otherwise a cancelled caller or two concurrent updates can
// leave the persisted config and active audit generation disagreeing.
apply_audit_runtime_config(&specs, config).await
})
.await
.map_err(|err| s3_error!(InternalError, "failed to lock server config update: {}", err))??;
Ok(())
}
pub(crate) async fn update_audit_config_and_reload<F>(specs: &[AdminTargetSpec], modifier: F) -> S3Result<()>
where
F: FnMut(&mut Config) -> bool,
F: FnMut(&mut Config) -> bool + Send + 'static,
{
let context = current_app_context();
update_audit_config_and_reload_for_context(context.as_deref(), specs, modifier).await
@@ -121,26 +136,30 @@ pub(crate) async fn set_audit_target_config(
target_name: &str,
kvs: rustfs_config::server_config::KVS,
) -> S3Result<()> {
update_audit_config_and_reload(specs, |config| {
let subsystem = subsystem.to_lowercase();
let target_name = target_name.to_lowercase();
update_audit_config_and_reload(specs, move |config| {
config
.0
.entry(subsystem.to_lowercase())
.entry(subsystem.clone())
.or_default()
.insert(target_name.to_lowercase(), kvs.clone());
.insert(target_name.clone(), kvs.clone());
true
})
.await
}
pub(crate) async fn remove_audit_target_config(specs: &[AdminTargetSpec], subsystem: &str, target_name: &str) -> S3Result<()> {
update_audit_config_and_reload(specs, |config| {
let subsystem = subsystem.to_lowercase();
let target_name = target_name.to_lowercase();
update_audit_config_and_reload(specs, move |config| {
let mut changed = false;
if let Some(targets) = config.0.get_mut(&subsystem.to_lowercase()) {
if targets.remove(&target_name.to_lowercase()).is_some() {
if let Some(targets) = config.0.get_mut(&subsystem) {
if targets.remove(&target_name).is_some() {
changed = true;
}
if targets.is_empty() {
config.0.remove(&subsystem.to_lowercase());
config.0.remove(&subsystem);
}
}
changed
rustfs/src/admin/handlers/config_admin.rs
+248 -175
View File
@@ -13,6 +13,7 @@
// limitations under the License.
use crate::admin::auth::validate_admin_request;
use crate::admin::handlers::supervise_admin_mutation;
use crate::admin::router::{AdminOperation, Operation, S3Router};
use crate::admin::runtime_sources::{
current_app_context, current_object_store_handle_for_context, current_server_config_for_context, publish_server_config,
@@ -20,12 +21,14 @@ use crate::admin::runtime_sources::{
use crate::admin::service::config::{
CONFIG_WORKER_RELOAD_FAILURE_STATE, EVENT_CONFIG_WORKER_RELOAD_FAILED, FULL_CONFIG_WORKER_SUBSYSTEMS, LOG_COMPONENT_ADMIN,
LOG_SUBSYSTEM_CONFIG, PreparedRuntimeConfig, apply_dynamic_config_for_subsystem, is_dynamic_config_subsystem,
prepare_server_config, signal_config_snapshot_reload, signal_dynamic_config_reload,
preflight_dynamic_config_reload, prepare_server_config, signal_config_snapshot_reload_checked,
signal_dynamic_config_reload_checked,
};
use crate::admin::storage_api::config::storageclass::{INLINE_BLOCK_ENV, OPTIMIZE_ENV, RRS_ENV, STANDARD_ENV};
use crate::admin::storage_api::config::{
RUSTFS_META_BUCKET, STORAGE_CLASS_SUB_SYS, delete_admin_config, read_admin_config, read_admin_config_without_migrate,
save_admin_config, save_admin_server_config,
read_admin_config_without_migrate_no_lock, save_admin_config, save_admin_server_config_no_lock,
with_admin_server_config_write_lock,
};
use crate::admin::storage_api::contract::list::ListOperations as _;
use crate::admin::utils::{encode_compatible_admin_payload, is_compat_admin_request, read_compatible_admin_body};
@@ -47,7 +50,7 @@ use rustfs_config::notify::{
ENV_NOTIFY_MQTT_QOS, ENV_NOTIFY_MQTT_QUEUE_DIR, ENV_NOTIFY_MQTT_QUEUE_LIMIT, ENV_NOTIFY_MQTT_RECONNECT_INTERVAL,
ENV_NOTIFY_MQTT_TOPIC, ENV_NOTIFY_MQTT_USERNAME, ENV_NOTIFY_WEBHOOK_AUTH_TOKEN, ENV_NOTIFY_WEBHOOK_CLIENT_CERT,
ENV_NOTIFY_WEBHOOK_CLIENT_KEY, ENV_NOTIFY_WEBHOOK_ENABLE, ENV_NOTIFY_WEBHOOK_ENDPOINT, ENV_NOTIFY_WEBHOOK_QUEUE_DIR,
ENV_NOTIFY_WEBHOOK_QUEUE_LIMIT, NOTIFY_MQTT_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS,
ENV_NOTIFY_WEBHOOK_QUEUE_LIMIT, NOTIFY_MQTT_SUB_SYS, NOTIFY_SUB_SYSTEMS, NOTIFY_WEBHOOK_SUB_SYS,
};
use rustfs_config::oidc::{
ENV_IDENTITY_OPENID_CLAIM_NAME, ENV_IDENTITY_OPENID_CLAIM_PREFIX, ENV_IDENTITY_OPENID_CLIENT_ID,
@@ -726,6 +729,14 @@ async fn load_server_config_from_store() -> S3Result<ServerConfig> {
.map_err(Into::into)
}
async fn load_server_config_from_store_locked() -> S3Result<ServerConfig> {
let store = object_store()?;
read_admin_config_without_migrate_no_lock(store)
.await
.map_err(ApiError::from)
.map_err(Into::into)
}
async fn load_active_server_config() -> S3Result<ServerConfig> {
if let Ok(config) = load_server_config_from_store().await {
return Ok(config);
@@ -736,9 +747,9 @@ async fn load_active_server_config() -> S3Result<ServerConfig> {
.ok_or_else(|| s3_error!(InternalError, "server config is not initialized"))
}
async fn save_server_config_to_store(config: &ServerConfig) -> S3Result<()> {
async fn save_server_config_to_store_locked(config: &ServerConfig) -> S3Result<()> {
let store = object_store()?;
save_admin_server_config(store, config)
save_admin_server_config_no_lock(store, config)
.await
.map_err(ApiError::from)
.map_err(Into::into)
@@ -1563,20 +1574,144 @@ async fn commit_prepared_config(
/// Re-apply local mutable worker families after a full-config replacement.
/// Peers receive one full-snapshot signal after this returns; signaling each
/// family here as well would recreate audit/scanner targets twice per peer.
async fn apply_dynamic_subsystems(config: &ServerConfig) {
fn publish_notify_config_intent(
config: &ServerConfig,
sub_system: Option<&str>,
) -> Option<rustfs_notify::NotificationLifecycleTransition> {
(sub_system.is_none() || sub_system.is_some_and(|sub_system| NOTIFY_SUB_SYSTEMS.contains(&sub_system)))
.then(|| rustfs_notify::ensure_live_events().publish_config(config.clone()))
}
async fn preflight_notify_config_intent(sub_system: Option<&str>) -> S3Result<()> {
if sub_system.is_none() || sub_system.is_some_and(|sub_system| NOTIFY_SUB_SYSTEMS.contains(&sub_system)) {
preflight_dynamic_config_reload(sub_system.unwrap_or(NOTIFY_WEBHOOK_SUB_SYS)).await?;
}
Ok(())
}
async fn wait_notify_config_intent(transition: Option<rustfs_notify::NotificationLifecycleTransition>) -> S3Result<bool> {
let Some(transition) = transition else {
return Ok(false);
};
transition.wait().await.map_err(|err| {
warn!(error = %err, "Failed to apply local notification config");
s3_error!(InternalError, "failed to apply notification config")
})?;
Ok(true)
}
async fn apply_non_notify_dynamic_subsystems(config: &ServerConfig) -> Vec<String> {
let mut failures = Vec::new();
for sub_system in FULL_CONFIG_WORKER_SUBSYSTEMS {
if let Err(err) = apply_dynamic_config_for_subsystem(config, sub_system).await {
if NOTIFY_SUB_SYSTEMS.contains(&sub_system) {
continue;
}
if apply_dynamic_config_for_subsystem(config, sub_system).await.is_err() {
failures.push(format!("local {sub_system}"));
warn!(
event = EVENT_CONFIG_WORKER_RELOAD_FAILED,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_CONFIG,
config_subsystem = sub_system,
state = CONFIG_WORKER_RELOAD_FAILURE_STATE,
error = %err,
reason = "apply_failed",
"Published server config but failed to reload a local worker subsystem"
);
}
}
failures
}
fn finish_config_reconciliation(errors: Vec<String>) -> S3Result<()> {
if errors.is_empty() {
Ok(())
} else {
Err(s3_error!(
InternalError,
"server config persisted but runtime convergence failed: {}",
errors.join("; ")
))
}
}
async fn reconcile_targeted_config(
config: ServerConfig,
sub_system: Option<String>,
storage_class_applied: bool,
notify_transition: Option<rustfs_notify::NotificationLifecycleTransition>,
) -> S3Result<bool> {
let mut errors = Vec::new();
let notify_applied = notify_transition.is_some();
if let Err(err) = wait_notify_config_intent(notify_transition).await {
warn!(error = %err, "Local notification config failed to converge");
errors.push("local notify".to_string());
}
let config_applied = if notify_applied {
if let Some(sub_system) = sub_system.as_deref()
&& let Err(err) = signal_dynamic_config_reload_checked(sub_system).await
{
warn!(config_subsystem = sub_system, error = %err, "Peer config reload failed");
errors.push(format!("peer {sub_system}"));
}
true
} else if storage_class_applied {
if let Err(err) = signal_dynamic_config_reload_checked(STORAGE_CLASS_SUB_SYS).await {
warn!(error = %err, "Peer storage-class reload failed");
errors.push(format!("peer {STORAGE_CLASS_SUB_SYS}"));
}
true
} else if let Some(sub_system) = sub_system.as_deref()
&& is_dynamic_config_subsystem(sub_system)
{
let config_applied = match apply_dynamic_config_for_subsystem(&config, sub_system).await {
Ok(applied) => applied,
Err(_) => {
warn!(config_subsystem = sub_system, reason = "apply_failed", "Local config reload failed");
errors.push(format!("local {sub_system}"));
false
}
};
if let Err(err) = signal_dynamic_config_reload_checked(sub_system).await {
warn!(config_subsystem = sub_system, error = %err, "Peer config reload failed");
errors.push(format!("peer {sub_system}"));
}
config_applied
} else {
if let Err(err) = signal_config_snapshot_reload_checked().await {
warn!(error = %err, "Peer config snapshot reload failed");
errors.push("peer config snapshot".to_string());
}
false
};
finish_config_reconciliation(errors)?;
Ok(config_applied)
}
async fn reconcile_full_config(
config: ServerConfig,
notify_transition: Option<rustfs_notify::NotificationLifecycleTransition>,
) -> S3Result<()> {
let mut errors = Vec::new();
if let Err(err) = wait_notify_config_intent(notify_transition).await {
warn!(error = %err, "Local notification config failed to converge");
errors.push("local notify".to_string());
}
if let Err(err) = signal_dynamic_config_reload_checked(STORAGE_CLASS_SUB_SYS).await {
warn!(error = %err, "Peer storage-class reload failed");
errors.push(format!("peer {STORAGE_CLASS_SUB_SYS}"));
}
errors.extend(apply_non_notify_dynamic_subsystems(&config).await);
if let Err(err) = signal_dynamic_config_reload_checked(NOTIFY_WEBHOOK_SUB_SYS).await {
warn!(error = %err, "Peer notification config reload failed");
errors.push("peer notify".to_string());
}
if let Err(err) = signal_config_snapshot_reload_checked().await {
warn!(error = %err, "Peer config snapshot reload failed");
errors.push("peer config snapshot".to_string());
}
finish_config_reconciliation(errors)
}
pub struct GetConfigKVHandler {}
@@ -1611,37 +1746,39 @@ impl Operation for SetConfigKVHandler {
}
validate_config_directives(&directives)?;
let sub_system = config_update_sub_system(&directives)?;
let mut config = load_server_config_from_store().await?;
apply_set_directives(&mut config, &directives)?;
let prepared = prepare_server_config(&config, sub_system).await?;
save_server_config_history(&body).await?;
let config_applied = if sub_system == Some(STORAGE_CLASS_SUB_SYS) {
commit_prepared_config(
config.clone(),
prepared,
save_server_config_to_store(&config),
publish_prepared_config_snapshots,
)
.await?;
signal_dynamic_config_reload(STORAGE_CLASS_SUB_SYS).await;
true
} else {
save_server_config_to_store(&config).await?;
publish_server_config(config.clone());
if let Some(sub_system) = sub_system
&& is_dynamic_config_subsystem(sub_system)
{
let config_applied = apply_dynamic_config_for_subsystem(&config, sub_system).await?;
if config_applied {
signal_dynamic_config_reload(sub_system).await;
}
config_applied
} else {
signal_config_snapshot_reload().await;
false
}
};
let sub_system = config_update_sub_system(&directives)?.map(str::to_owned);
let transaction_sub_system = sub_system.clone();
let config_store = object_store()?;
let config_applied = supervise_admin_mutation("config mutation", async move {
preflight_notify_config_intent(sub_system.as_deref()).await?;
let (config, storage_class_applied, notify_transition) =
with_admin_server_config_write_lock(config_store, move || async move {
let sub_system = transaction_sub_system.as_deref();
let mut config = load_server_config_from_store_locked().await?;
apply_set_directives(&mut config, &directives)?;
let prepared = prepare_server_config(&config, sub_system).await?;
save_server_config_history(&body).await?;
if sub_system == Some(STORAGE_CLASS_SUB_SYS) {
commit_prepared_config(
config.clone(),
prepared,
save_server_config_to_store_locked(&config),
publish_prepared_config_snapshots,
)
.await?;
} else {
save_server_config_to_store_locked(&config).await?;
publish_server_config(config.clone());
}
let notify_transition = publish_notify_config_intent(&config, sub_system);
Ok::<_, S3Error>((config, sub_system == Some(STORAGE_CLASS_SUB_SYS), notify_transition))
})
.await
.map_err(|err| s3_error!(InternalError, "failed to lock server config update: {}", err))??;
reconcile_targeted_config(config, sub_system, storage_class_applied, notify_transition).await
})
.await?;
success_response(config_applied)
}
@@ -1660,37 +1797,39 @@ impl Operation for DelConfigKVHandler {
}
validate_config_directives(&directives)?;
let sub_system = config_update_sub_system(&directives)?;
let mut config = load_server_config_from_store().await?;
apply_delete_directives(&mut config, &directives);
let prepared = prepare_server_config(&config, sub_system).await?;
save_server_config_history(&body).await?;
let config_applied = if sub_system == Some(STORAGE_CLASS_SUB_SYS) {
commit_prepared_config(
config.clone(),
prepared,
save_server_config_to_store(&config),
publish_prepared_config_snapshots,
)
.await?;
signal_dynamic_config_reload(STORAGE_CLASS_SUB_SYS).await;
true
} else {
save_server_config_to_store(&config).await?;
publish_server_config(config.clone());
if let Some(sub_system) = sub_system
&& is_dynamic_config_subsystem(sub_system)
{
let config_applied = apply_dynamic_config_for_subsystem(&config, sub_system).await?;
if config_applied {
signal_dynamic_config_reload(sub_system).await;
}
config_applied
} else {
signal_config_snapshot_reload().await;
false
}
};
let sub_system = config_update_sub_system(&directives)?.map(str::to_owned);
let transaction_sub_system = sub_system.clone();
let config_store = object_store()?;
let config_applied = supervise_admin_mutation("config mutation", async move {
preflight_notify_config_intent(sub_system.as_deref()).await?;
let (config, storage_class_applied, notify_transition) =
with_admin_server_config_write_lock(config_store, move || async move {
let sub_system = transaction_sub_system.as_deref();
let mut config = load_server_config_from_store_locked().await?;
apply_delete_directives(&mut config, &directives);
let prepared = prepare_server_config(&config, sub_system).await?;
save_server_config_history(&body).await?;
if sub_system == Some(STORAGE_CLASS_SUB_SYS) {
commit_prepared_config(
config.clone(),
prepared,
save_server_config_to_store_locked(&config),
publish_prepared_config_snapshots,
)
.await?;
} else {
save_server_config_to_store_locked(&config).await?;
publish_server_config(config.clone());
}
let notify_transition = publish_notify_config_intent(&config, sub_system);
Ok::<_, S3Error>((config, sub_system == Some(STORAGE_CLASS_SUB_SYS), notify_transition))
})
.await
.map_err(|err| s3_error!(InternalError, "failed to lock server config update: {}", err))??;
reconcile_targeted_config(config, sub_system, storage_class_applied, notify_transition).await
})
.await?;
success_response(config_applied)
}
@@ -1776,16 +1915,26 @@ impl Operation for RestoreConfigHistoryKVHandler {
let mut config = ServerConfig::new();
apply_set_directives(&mut config, &directives)?;
let prepared = prepare_server_config(&config, None).await?;
commit_prepared_config(
config.clone(),
prepared,
save_server_config_to_store(&config),
publish_prepared_config_snapshots,
)
let config_store = object_store()?;
supervise_admin_mutation("config mutation", async move {
preflight_notify_config_intent(None).await?;
let persisted_config = config.clone();
let notify_transition = with_admin_server_config_write_lock(config_store, move || async move {
commit_prepared_config(
persisted_config.clone(),
prepared,
save_server_config_to_store_locked(&persisted_config),
publish_prepared_config_snapshots,
)
.await?;
Ok::<_, S3Error>(publish_notify_config_intent(&persisted_config, None))
})
.await
.map_err(|err| s3_error!(InternalError, "failed to lock server config restore: {}", err))??;
reconcile_full_config(config, notify_transition).await
})
.await?;
signal_dynamic_config_reload(STORAGE_CLASS_SUB_SYS).await;
apply_dynamic_subsystems(&config).await;
signal_config_snapshot_reload().await;
success_response(false)
}
@@ -1820,17 +1969,27 @@ impl Operation for SetConfigHandler {
let mut config = ServerConfig::new();
apply_set_directives(&mut config, &directives)?;
let prepared = prepare_server_config(&config, None).await?;
save_server_config_history(&body).await?;
commit_prepared_config(
config.clone(),
prepared,
save_server_config_to_store(&config),
publish_prepared_config_snapshots,
)
let config_store = object_store()?;
supervise_admin_mutation("config mutation", async move {
preflight_notify_config_intent(None).await?;
save_server_config_history(&body).await?;
let persisted_config = config.clone();
let notify_transition = with_admin_server_config_write_lock(config_store, move || async move {
commit_prepared_config(
persisted_config.clone(),
prepared,
save_server_config_to_store_locked(&persisted_config),
publish_prepared_config_snapshots,
)
.await?;
Ok::<_, S3Error>(publish_notify_config_intent(&persisted_config, None))
})
.await
.map_err(|err| s3_error!(InternalError, "failed to lock full server config update: {}", err))??;
reconcile_full_config(config, notify_transition).await
})
.await?;
signal_dynamic_config_reload(STORAGE_CLASS_SUB_SYS).await;
apply_dynamic_subsystems(&config).await;
signal_config_snapshot_reload().await;
success_response(false)
}
@@ -1890,92 +2049,6 @@ mod tests {
assert_eq!(*events.lock().expect("result events lock"), ["persist"]);
}
#[test]
fn storage_config_write_handlers_persist_before_publishing() {
const SOURCE: &str = include_str!("config_admin.rs");
for (handler, next_handler, follow_up) in [
(
"SetConfigKVHandler",
"DelConfigKVHandler",
"signal_dynamic_config_reload(STORAGE_CLASS_SUB_SYS).await",
),
(
"DelConfigKVHandler",
"HelpConfigKVHandler",
"signal_dynamic_config_reload(STORAGE_CLASS_SUB_SYS).await",
),
(
"RestoreConfigHistoryKVHandler",
"GetConfigHandler",
"signal_dynamic_config_reload(STORAGE_CLASS_SUB_SYS).await",
),
(
"SetConfigHandler",
"#[cfg(test)]",
"signal_dynamic_config_reload(STORAGE_CLASS_SUB_SYS).await",
),
] {
let start_marker = format!("impl Operation for {handler}");
let start = SOURCE
.find(&start_marker)
.unwrap_or_else(|| panic!("missing {handler} implementation"));
let tail = &SOURCE[start..];
let end = tail
.find(next_handler)
.unwrap_or_else(|| panic!("missing {next_handler} after {handler}"));
let implementation = &tail[..end];
let prepared_commit_path = if matches!(handler, "SetConfigKVHandler" | "DelConfigKVHandler") {
let branch_start = implementation
.find("if sub_system == Some(STORAGE_CLASS_SUB_SYS)")
.unwrap_or_else(|| panic!("missing storage-class branch in {handler}"));
let branch = &implementation[branch_start..];
let branch_end = branch
.find("} else {")
.unwrap_or_else(|| panic!("missing non-storage branch in {handler}"));
&branch[..branch_end]
} else {
implementation
};
assert_eq!(prepared_commit_path.matches("commit_prepared_config(").count(), 1, "{handler}");
let commit_start = prepared_commit_path.find("commit_prepared_config(").expect("commit call");
let commit_end = prepared_commit_path[commit_start..].find(';').expect("commit terminator") + commit_start;
let commit_statement = &prepared_commit_path[commit_start..=commit_end];
assert!(commit_statement.contains(".await?;"), "{handler} must propagate commit failure");
let follow_up_start = prepared_commit_path
.find(follow_up)
.unwrap_or_else(|| panic!("missing follow-up in {handler}"));
assert!(follow_up_start > commit_end, "{handler} must run follow-up only after commit");
assert!(
!prepared_commit_path.contains("publish_server_config("),
"{handler} must not publish directly"
);
assert!(
!prepared_commit_path.contains(".publish_storage_class("),
"{handler} must not publish directly"
);
if matches!(handler, "RestoreConfigHistoryKVHandler" | "SetConfigHandler") {
let worker_start = prepared_commit_path
.find("apply_dynamic_subsystems(&config).await")
.unwrap_or_else(|| panic!("missing local worker apply in {handler}"));
let signal_start = prepared_commit_path
.find("signal_config_snapshot_reload().await")
.unwrap_or_else(|| panic!("missing snapshot signal in {handler}"));
assert!(
worker_start > follow_up_start,
"{handler} must converge peer parity before local worker apply"
);
assert!(
signal_start > worker_start,
"{handler} must signal the full snapshot after local worker apply"
);
}
}
}
#[test]
fn tokenize_config_line_handles_quotes_and_escapes() {
let tokens = tokenize_config_line(r#"identity_openid client_id="console app" client_secret="s3cr\"et" enable=on"#)
rustfs/src/admin/handlers/event.rs
+176 -31
View File
@@ -15,6 +15,7 @@
use crate::admin::{
auth::validate_admin_request,
handlers::notify_runtime_access::{get_notification_system, load_notification_config_snapshot},
handlers::supervise_admin_mutation,
handlers::target_descriptor::{
AdminTargetSpec, EndpointKey, TargetEndpointSource, admin_target_spec_from_builtin, build_enabled_target_kvs,
build_json_response, collect_runtime_statuses, extract_supported_target_params,
@@ -22,6 +23,8 @@ use crate::admin::{
target_mutation_block_reason as shared_target_mutation_block_reason,
},
router::{AdminOperation, Operation, S3Router},
runtime_sources::{AppContext, app_context_from_req},
service::config::{preflight_dynamic_config_reload_for_context, signal_dynamic_config_reload_checked_for_context},
};
use crate::auth::{check_key_valid, get_session_token};
use crate::server::{
@@ -43,6 +46,45 @@ use std::sync::LazyLock;
use tracing::{Span, error, info, warn};
const LOG_COMPONENT_ADMIN_API: &str = "admin_api";
async fn converge_target_mutation_on_cluster(
context: Option<&AppContext>,
target_type: &str,
local_result: Result<(), rustfs_notify::NotificationError>,
) -> S3Result<()> {
let subsystem = notification_target_subsystem(target_type)?;
let peer_result = signal_dynamic_config_reload_checked_for_context(context, subsystem).await;
match (local_result, peer_result) {
(Ok(()), Ok(())) => Ok(()),
(Err(local), Ok(())) => {
warn!(target_type, error = %local, "Local notification runtime failed to converge");
Err(s3_error!(InternalError, "local notification runtime failed to converge"))
}
(Ok(()), Err(peer)) => Err(peer),
(Err(local), Err(peer)) => {
warn!(target_type, error = %local, "Local notification runtime failed while peer convergence also failed");
Err(s3_error!(
InternalError,
"local notification runtime and peer convergence failed: {}",
peer
))
}
}
}
fn notification_target_subsystem(target_type: &str) -> S3Result<&'static str> {
notification_target_specs()
.iter()
.find(|spec| spec.subsystem == target_type)
.map(|spec| spec.subsystem)
.ok_or_else(|| s3_error!(InvalidArgument, "unsupported notification target type: {}", target_type))
}
async fn preflight_target_mutation_on_cluster(context: Option<&AppContext>, target_type: &str) -> S3Result<()> {
preflight_dynamic_config_reload_for_context(context, notification_target_subsystem(target_type)?).await
}
const LOG_SUBSYSTEM_NOTIFICATION_TARGET: &str = "notification_target";
const EVENT_ADMIN_REQUEST_REJECTED: &str = "admin_request_rejected";
const EVENT_ADMIN_REQUEST_FAILED: &str = "admin_request_failed";
@@ -203,6 +245,7 @@ struct NotificationEndpoint {
#[derive(Serialize, Debug)]
struct NotificationEndpointsResponse {
notify_enabled: bool,
notification_endpoints: Vec<NotificationEndpoint>,
}
@@ -229,7 +272,7 @@ async fn authorize_notification_admin_request(req: &S3Request<Body>, action: Adm
validate_admin_request(&req.headers, &cred, owner, false, vec![Action::AdminAction(action)], remote_addr).await
}
fn target_mutation_block_reason(config: &Config, target_type: &str, target_name: &str) -> Option<String> {
fn target_mutation_block_reason(config: &Config, target_type: &str, target_name: &str) -> S3Result<Option<String>> {
shared_target_mutation_block_reason(
notification_target_specs(),
NOTIFY_ROUTE_PREFIX,
@@ -256,16 +299,21 @@ async fn notification_target_operation_block_reason(action: &str) -> Option<Stri
target_module_disabled_reason("notify", rustfs_config::ENV_NOTIFY_ENABLE, is_notify_module_enabled(), action)
}
fn merge_notification_endpoints(config: &Config, runtime_statuses: HashMap<EndpointKey, String>) -> Vec<NotificationEndpoint> {
shared_merge_target_endpoints(notification_target_specs(), NOTIFY_ROUTE_PREFIX, config, runtime_statuses)
.into_iter()
.map(|endpoint| NotificationEndpoint {
account_id: endpoint.account_id,
service: endpoint.service,
status: endpoint.status,
source: endpoint.source,
})
.collect()
fn merge_notification_endpoints(
config: &Config,
runtime_statuses: HashMap<EndpointKey, String>,
) -> S3Result<Vec<NotificationEndpoint>> {
Ok(
shared_merge_target_endpoints(notification_target_specs(), NOTIFY_ROUTE_PREFIX, config, runtime_statuses)?
.into_iter()
.map(|endpoint| NotificationEndpoint {
account_id: endpoint.account_id,
service: endpoint.service,
status: endpoint.status,
source: endpoint.source,
})
.collect(),
)
}
fn collect_online_target_arns(region: &str, target_statuses: Vec<(rustfs_targets::arn::TargetID, String)>) -> Vec<String> {
@@ -284,6 +332,7 @@ impl Operation for NotificationTarget {
let span = Span::current();
let _enter = span.enter();
let (target_type, target_name) = extract_target_params(&params)?;
let context = app_context_from_req(&req);
authorize_notification_admin_request(&req, AdminAction::SetBucketTargetAction).await?;
if let Some(reason) = notification_target_operation_block_reason("managing notification targets from the console").await {
@@ -291,7 +340,7 @@ impl Operation for NotificationTarget {
return Err(s3_error!(InvalidRequest, "{reason}"));
}
let (ns, config_snapshot) = load_notification_config_snapshot().await?;
if let Some(reason) = target_mutation_block_reason(&config_snapshot, target_type, target_name) {
if let Some(reason) = target_mutation_block_reason(&config_snapshot, target_type, target_name)? {
log_notification_target_operation_blocked!("set_target_config", Some(target_type), Some(target_name), &reason);
return Err(s3_error!(InvalidRequest, "{reason}"));
}
@@ -320,7 +369,15 @@ impl Operation for NotificationTarget {
)
.await?;
ns.set_target_config(target_type, target_name, kvs).await.map_err(|e| {
let mutation_target_type = target_type.to_owned();
let mutation_target_name = target_name.to_owned();
supervise_admin_mutation("notification target mutation", async move {
preflight_target_mutation_on_cluster(context.as_deref(), &mutation_target_type).await?;
let local_result = ns.set_target_config(&mutation_target_type, &mutation_target_name, kvs).await;
converge_target_mutation_on_cluster(context.as_deref(), &mutation_target_type, local_result).await
})
.await
.map_err(|e| {
log_notification_target_request_failed!(
"set_target_config",
"set_target_config_failed",
@@ -328,7 +385,7 @@ impl Operation for NotificationTarget {
Some(target_name),
e
);
s3_error!(InternalError, "failed to set target config: {}", e)
e
})?;
log_notification_target_config_updated!("set_target_config", target_type, target_name);
@@ -343,11 +400,29 @@ impl Operation for ListNotificationTargets {
let span = Span::current();
let _enter = span.enter();
authorize_notification_admin_request(&req, AdminAction::GetBucketTargetAction).await?;
refresh_persisted_module_switches_from_store().await.map_err(|err| {
warn!(
event = EVENT_ADMIN_REQUEST_FAILED,
component = LOG_COMPONENT_ADMIN_API,
subsystem = LOG_SUBSYSTEM_NOTIFICATION_TARGET,
operation = "list_targets",
result = "failed",
reason = "module_switch_refresh_failed",
error = %err,
"admin request failed"
);
s3_error!(InternalError, "failed to refresh notification module state")
})?;
let notify_enabled = refresh_notify_module_enabled();
let (ns, config) = load_notification_config_snapshot().await?;
let runtime_statuses = collect_runtime_statuses(ns.get_target_values().await).await;
let notification_endpoints = merge_notification_endpoints(&config, runtime_statuses);
let notification_endpoints = merge_notification_endpoints(&config, runtime_statuses)?;
let data = serde_json::to_vec(&NotificationEndpointsResponse { notification_endpoints }).map_err(|e| {
let data = serde_json::to_vec(&NotificationEndpointsResponse {
notify_enabled,
notification_endpoints,
})
.map_err(|e| {
log_notification_target_request_failed!("list_targets", "serialize_targets_failed", None, None, e);
s3_error!(InternalError, "failed to serialize targets: {}", e)
})?;
@@ -409,6 +484,7 @@ impl Operation for RemoveNotificationTarget {
let span = Span::current();
let _enter = span.enter();
let (target_type, target_name) = extract_target_params(&params)?;
let context = app_context_from_req(&req);
authorize_notification_admin_request(&req, AdminAction::SetBucketTargetAction).await?;
if let Some(reason) = notification_target_operation_block_reason("managing notification targets from the console").await {
@@ -416,12 +492,20 @@ impl Operation for RemoveNotificationTarget {
return Err(s3_error!(InvalidRequest, "{reason}"));
}
let (ns, config_snapshot) = load_notification_config_snapshot().await?;
if let Some(reason) = target_mutation_block_reason(&config_snapshot, target_type, target_name) {
if let Some(reason) = target_mutation_block_reason(&config_snapshot, target_type, target_name)? {
log_notification_target_operation_blocked!("remove_target_config", Some(target_type), Some(target_name), &reason);
return Err(s3_error!(InvalidRequest, "{reason}"));
}
ns.remove_target_config(target_type, target_name).await.map_err(|e| {
let mutation_target_type = target_type.to_owned();
let mutation_target_name = target_name.to_owned();
supervise_admin_mutation("notification target mutation", async move {
preflight_target_mutation_on_cluster(context.as_deref(), &mutation_target_type).await?;
let local_result = ns.remove_target_config(&mutation_target_type, &mutation_target_name).await;
converge_target_mutation_on_cluster(context.as_deref(), &mutation_target_type, local_result).await
})
.await
.map_err(|e| {
log_notification_target_request_failed!(
"remove_target_config",
"remove_target_config_failed",
@@ -429,7 +513,7 @@ impl Operation for RemoveNotificationTarget {
Some(target_name),
e
);
s3_error!(InternalError, "failed to remove target config: {}", e)
e
})?;
log_notification_target_config_updated!("remove_target_config", target_type, target_name);
@@ -464,6 +548,41 @@ mod tests {
}])
}
#[test]
fn notification_target_subsystem_resolves_admin_route_type() {
assert_eq!(
notification_target_subsystem(NOTIFY_WEBHOOK_SUB_SYS).expect("webhook subsystem should resolve"),
NOTIFY_WEBHOOK_SUB_SYS
);
assert!(notification_target_subsystem("webhook").is_err());
}
#[test]
fn notification_endpoints_response_includes_required_module_state() {
let response = NotificationEndpointsResponse {
notify_enabled: true,
notification_endpoints: vec![NotificationEndpoint {
account_id: "primary".to_string(),
service: "webhook".to_string(),
status: "online".to_string(),
source: TargetEndpointSource::Config,
}],
};
assert_eq!(
serde_json::to_value(response).expect("notification target response should serialize"),
serde_json::json!({
"notify_enabled": true,
"notification_endpoints": [{
"account_id": "primary",
"service": "webhook",
"status": "online",
"source": "config"
}]
})
);
}
#[test]
fn merge_notification_endpoints_keeps_configured_targets_after_runtime_loss() {
let mut cfg_map = HashMap::new();
@@ -478,7 +597,7 @@ mod tests {
let config = Config(cfg_map);
let runtime = HashMap::from([(("webhook-a".to_string(), "webhook".to_string()), "online".to_string())]);
let merged = merge_notification_endpoints(&config, runtime);
let merged = merge_notification_endpoints(&config, runtime).expect("merge notification endpoints");
let mqtt = merged
.iter()
@@ -507,7 +626,7 @@ mod tests {
(("webhook-enabled".to_string(), "webhook".to_string()), "online".to_string()),
(("env-only".to_string(), "mqtt".to_string()), "offline".to_string()),
]);
let merged = merge_notification_endpoints(&config, runtime);
let merged = merge_notification_endpoints(&config, runtime).expect("merge notification endpoints");
let env_only = merged
.iter()
@@ -549,7 +668,7 @@ mod tests {
(("mixed-target".to_string(), "webhook".to_string()), "online".to_string()),
(("env-only".to_string(), "webhook".to_string()), "online".to_string()),
]);
let merged = merge_notification_endpoints(&config, runtime);
let merged = merge_notification_endpoints(&config, runtime).expect("merge notification endpoints");
let mixed = merged
.iter()
@@ -592,7 +711,7 @@ mod tests {
(("mixed-kafka".to_string(), "kafka".to_string()), "online".to_string()),
(("env-kafka".to_string(), "kafka".to_string()), "online".to_string()),
]);
let merged = merge_notification_endpoints(&config, runtime);
let merged = merge_notification_endpoints(&config, runtime).expect("merge notification endpoints");
let mixed = merged
.iter()
@@ -629,7 +748,7 @@ mod tests {
(("mixed-amqp".to_string(), "amqp".to_string()), "online".to_string()),
(("env-amqp".to_string(), "amqp".to_string()), "online".to_string()),
]);
let merged = merge_notification_endpoints(&config, runtime);
let merged = merge_notification_endpoints(&config, runtime).expect("merge notification endpoints");
let mixed = merged
.iter()
@@ -656,7 +775,8 @@ mod tests {
],
|| {
let config = Config(HashMap::new());
let reason = target_mutation_block_reason(&config, NOTIFY_WEBHOOK_SUB_SYS, "primary");
let reason = target_mutation_block_reason(&config, NOTIFY_WEBHOOK_SUB_SYS, "primary")
.expect("target mutation block reason");
assert!(reason.is_some());
assert!(reason.unwrap().contains("managed by environment variables"));
},
@@ -696,7 +816,8 @@ mod tests {
NOTIFY_WEBHOOK_SUB_SYS.to_string(),
HashMap::from([("primary".to_string(), enabled_kvs("on"))]),
)]));
let reason = target_mutation_block_reason(&config, NOTIFY_WEBHOOK_SUB_SYS, "primary");
let reason =
target_mutation_block_reason(&config, NOTIFY_WEBHOOK_SUB_SYS, "primary").expect("target mutation block reason");
assert!(reason.is_some());
assert!(reason.unwrap().contains("both persisted config and environment variables"));
});
@@ -709,7 +830,11 @@ mod tests {
NOTIFY_WEBHOOK_SUB_SYS.to_string(),
HashMap::from([(target_name.to_string(), enabled_kvs("on"))]),
)]));
assert!(target_mutation_block_reason(&config, NOTIFY_WEBHOOK_SUB_SYS, target_name).is_none());
assert!(
target_mutation_block_reason(&config, NOTIFY_WEBHOOK_SUB_SYS, target_name)
.expect("target mutation block reason")
.is_none()
);
}
#[test]
@@ -726,7 +851,7 @@ mod tests {
("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_MIXED-DISABLED", Some("https://example.com/hook")),
],
|| {
let merged = merge_notification_endpoints(&config, HashMap::new());
let merged = merge_notification_endpoints(&config, HashMap::new()).expect("merge notification endpoints");
let mixed = merged
.iter()
.find(|entry| entry.account_id == "mixed-disabled")
@@ -748,7 +873,7 @@ mod tests {
("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_ENV-ONLY", Some("https://example.com/env")),
],
|| {
let merged = merge_notification_endpoints(&config, HashMap::new());
let merged = merge_notification_endpoints(&config, HashMap::new()).expect("merge notification endpoints");
let env_only = merged
.iter()
.find(|entry| entry.account_id == "env-only")
@@ -798,7 +923,7 @@ mod tests {
],
|| {
let runtime = HashMap::from([(("PrimaryCase".to_string(), "webhook".to_string()), "online".to_string())]);
let merged = merge_notification_endpoints(&config, runtime);
let merged = merge_notification_endpoints(&config, runtime).expect("merge notification endpoints");
let mixed = merged
.iter()
.find(|entry| entry.account_id == "PrimaryCase" && entry.service == "webhook")
@@ -835,7 +960,11 @@ mod tests {
("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_PRIMARYCASE", None::<&str>),
],
|| {
assert!(target_mutation_block_reason(&config, NOTIFY_WEBHOOK_SUB_SYS, "primarycase").is_none());
assert!(
target_mutation_block_reason(&config, NOTIFY_WEBHOOK_SUB_SYS, "primarycase")
.expect("target mutation block reason")
.is_none()
);
},
);
}
@@ -864,6 +993,22 @@ mod tests {
list_block.contains("authorize_notification_admin_request(&req, AdminAction::GetBucketTargetAction).await?;"),
"notification target list should require GetBucketTargetAction"
);
let authorize_index = list_block
.find("authorize_notification_admin_request")
.expect("target list should authorize the request");
let refresh_index = list_block
.find("refresh_persisted_module_switches_from_store().await.map_err")
.expect("target list should fail when persisted module state cannot be refreshed");
let effective_state_index = list_block
.find("let notify_enabled = refresh_notify_module_enabled();")
.expect("target list should resolve the effective env and persisted module state");
let load_index = list_block
.find("load_notification_config_snapshot().await?")
.expect("target list should load notification config");
assert!(
authorize_index < refresh_index && refresh_index < effective_state_index && effective_state_index < load_index,
"target list should authorize, refresh persisted state, resolve effective state, then load targets"
);
assert!(
arns_block.contains("authorize_notification_admin_request(&req, AdminAction::GetBucketTargetAction).await?;"),
"notification target arn listing should require GetBucketTargetAction"
rustfs/src/admin/handlers/mod.rs
+51
View File
@@ -63,6 +63,19 @@ pub mod user_iam;
pub mod user_lifecycle;
pub mod user_policy_binding;
pub(crate) async fn supervise_admin_mutation<T>(
operation: &'static str,
mutation: impl std::future::Future<Output = s3s::S3Result<T>> + Send + 'static,
) -> s3s::S3Result<T>
where
T: Send + 'static,
{
tokio::spawn(mutation).await.map_err(|err| {
let outcome = if err.is_cancelled() { "cancelled" } else { "panicked" };
s3s::s3_error!(InternalError, "{} task {}", operation, outcome)
})?
}
#[cfg(test)]
mod tests {
use super::*;
@@ -114,6 +127,44 @@ mod tests {
// Test passes if we reach this point without panicking
}
#[tokio::test]
async fn supervised_admin_mutation_survives_waiter_cancellation() {
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
let (release_tx, release_rx) = tokio::sync::oneshot::channel();
let (completed_tx, completed_rx) = tokio::sync::oneshot::channel();
let waiter = tokio::spawn(async move {
supervise_admin_mutation("test mutation", async move {
let _ = started_tx.send(());
let _ = release_rx.await;
let _ = completed_tx.send(());
Ok(())
})
.await
});
started_rx.await.expect("mutation started");
waiter.abort();
release_tx.send(()).expect("release mutation");
tokio::time::timeout(std::time::Duration::from_secs(1), completed_rx)
.await
.expect("detached mutation should complete")
.expect("completion signal");
}
#[tokio::test]
async fn supervised_admin_mutation_does_not_expose_panic_payload() {
let error = supervise_admin_mutation::<()>("test mutation", async {
panic!("do-not-expose-payload");
})
.await
.expect_err("panicking mutation should fail");
let rendered = error.to_string();
assert!(rendered.contains("panicked"));
assert!(!rendered.contains("do-not-expose-payload"));
}
// Note: Testing the actual async handler implementations requires:
// 1. S3Request setup with proper headers, URI, and credentials
// 2. Global object store initialization
rustfs/src/admin/handlers/module_switch.rs
+89 -29
View File
@@ -12,26 +12,31 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::admin::runtime_sources::default_admin_usecase;
use crate::admin::runtime_sources::{AppContext, app_context_from_req, default_admin_usecase};
use crate::admin::service::config::{
preflight_dynamic_config_reload_for_context, signal_dynamic_config_reload_checked_for_context,
};
use crate::admin::{
auth::validate_admin_request,
handlers::supervise_admin_mutation,
router::{AdminOperation, Operation, S3Router},
};
use crate::auth::{check_key_valid, get_session_token};
use crate::server::{
ADMIN_PREFIX, ModuleSwitchSnapshot, ModuleSwitchSource, PersistedModuleSwitches, RemoteAddr, current_module_switch_snapshot,
init_event_notifier, refresh_audit_module_enabled, refresh_notify_module_enabled,
refresh_persisted_module_switches_from_store, save_persisted_module_switches_to_store, shutdown_event_notifier,
start_audit_system, stop_audit_system, validate_module_switch_update,
ADMIN_PREFIX, MODULE_SWITCHES_SIGNAL_SUBSYSTEM, ModuleSwitchSnapshot, ModuleSwitchSource, PersistedModuleSwitches,
RemoteAddr, apply_audit_module_switch_for_context, current_module_switch_snapshot, mark_event_notifier_reconciled,
mark_event_notifier_unreconciled, refresh_audit_module_enabled, refresh_notify_module_enabled,
refresh_persisted_module_switches_from, refresh_persisted_module_switches_from_store, save_persisted_module_switches_to,
validate_module_switch_update,
};
use http::{HeaderMap, StatusCode};
use hyper::Method;
use matchit::Params;
use rustfs_audit::AuditError;
use rustfs_config::MAX_ADMIN_REQUEST_BODY_SIZE;
use rustfs_policy::policy::action::{Action, AdminAction};
use s3s::{Body, S3Request, S3Response, S3Result, header::CONTENT_TYPE, s3_error};
use serde::{Deserialize, Serialize};
use std::sync::Arc;
pub fn register_module_switch_route(r: &mut S3Router<AdminOperation>) -> std::io::Result<()> {
r.insert(
@@ -139,6 +144,74 @@ async fn refresh_module_switch_snapshot() -> S3Result<ModuleSwitchSnapshot> {
Ok(current_module_switch_snapshot())
}
async fn apply_module_switch_update(context: Arc<AppContext>, switches: PersistedModuleSwitches) -> S3Result<()> {
preflight_dynamic_config_reload_for_context(Some(context.as_ref()), MODULE_SWITCHES_SIGNAL_SUBSYSTEM).await?;
let store = context.object_store();
mark_event_notifier_unreconciled();
let notification_system = rustfs_notify::ensure_live_events();
if switches.notify_enabled {
notification_system
.reload_persisted_config_from_store(store.clone())
.await
.map_err(|err| {
tracing::warn!(error = %err, "Failed to load notification config for module switch update");
s3_error!(InternalError, "failed to load notification config")
})?;
}
let transition_system = notification_system.clone();
let notify_transition = save_persisted_module_switches_to(store.clone(), switches, move || {
let enabled = refresh_notify_module_enabled();
transition_system.publish_targets_enabled(enabled, None)
})
.await
.map_err(|err| {
tracing::warn!(error = %err, "Failed to save module switches");
s3_error!(InternalError, "failed to save module switches")
})?;
let mut failures = Vec::new();
let mut notify_converged = true;
if let Err(err) = notify_transition.wait().await {
tracing::warn!(error = %err, "Local notification runtime failed to apply module switch update");
notify_converged = false;
failures.push("local notify");
}
if !switches.notify_enabled
&& let Err(err) = notification_system.reload_persisted_config_from_store(store).await
{
tracing::warn!(error = %err, "Local notification config cache failed to reload after module disable");
notify_converged = false;
failures.push("local notify config cache");
}
if notify_converged && notification_system.runtime_lifecycle_is_converged() {
mark_event_notifier_reconciled();
} else if notify_converged {
failures.push("local notify convergence");
}
if apply_audit_module_switch_for_context(Some(context.as_ref())).await.is_err() {
tracing::warn!(reason = "apply_failed", "Local audit runtime failed to apply module switch update");
failures.push("local audit");
}
if let Err(err) =
signal_dynamic_config_reload_checked_for_context(Some(context.as_ref()), MODULE_SWITCHES_SIGNAL_SUBSYSTEM).await
{
tracing::warn!(error = %err, "Peer nodes failed to apply module switch update");
failures.push("peer module switches");
}
if failures.is_empty() {
Ok(())
} else {
Err(s3_error!(
InternalError,
"module switches persisted but runtime convergence failed: {}",
failures.join("; ")
))
}
}
pub struct GetModuleSwitchesHandler {}
#[async_trait::async_trait]
@@ -156,7 +229,9 @@ pub struct UpdateModuleSwitchesHandler {}
impl Operation for UpdateModuleSwitchesHandler {
async fn call(&self, mut req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
authorize_module_switch_request(&req, AdminAction::ConfigUpdateAdminAction).await?;
refresh_persisted_module_switches_from_store()
let context = app_context_from_req(&req).ok_or_else(|| s3_error!(InternalError, "storage layer not initialized"))?;
let store = context.object_store();
refresh_persisted_module_switches_from(store.clone())
.await
.map_err(|e| s3_error!(InternalError, "failed to reload persisted module switches: {}", e))?;
@@ -183,28 +258,7 @@ impl Operation for UpdateModuleSwitchesHandler {
return Err(s3_error!(InvalidRequest, "{err}"));
}
save_persisted_module_switches_to_store(switches)
.await
.map_err(|e| s3_error!(InternalError, "failed to save module switches: {}", e))?;
// Apply the new effective values immediately on this node so the console
// response reflects the runtime state after to write completes.
if refresh_notify_module_enabled() {
init_event_notifier().await;
} else {
shutdown_event_notifier().await;
}
if refresh_audit_module_enabled() {
match start_audit_system().await {
Ok(()) | Err(AuditError::AlreadyInitialized) => {}
Err(e) => return Err(s3_error!(InternalError, "failed to apply audit module switch: {}", e)),
}
} else {
stop_audit_system()
.await
.map_err(|e| s3_error!(InternalError, "failed to stop audit module after switch update: {}", e))?;
}
supervise_admin_mutation("module switch update", apply_module_switch_update(context, switches)).await?;
let snapshot = current_module_switch_snapshot();
build_response(StatusCode::OK, &ModuleSwitchesResponse::from(snapshot), req.headers.get("x-request-id"))
@@ -233,6 +287,12 @@ mod tests {
put_block.contains("authorize_module_switch_request(&req, AdminAction::ConfigUpdateAdminAction).await?;"),
"module switch PUT should require ConfigUpdateAdminAction"
);
assert!(
put_block.contains(
"supervise_admin_mutation(\"module switch update\", apply_module_switch_update(context, switches)).await?;"
),
"module switch PUT must delegate the complete mutation to its supervisor"
);
}
#[test]
rustfs/src/admin/handlers/notify_runtime_access.rs
+14 -12
View File
@@ -12,31 +12,33 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::server::init_event_notifier;
use crate::server::{init_event_notifier, is_event_notifier_reconciled};
use rustfs_config::server_config::Config;
use s3s::{S3Result, s3_error};
use std::sync::Arc;
use tokio::sync::Mutex;
static NOTIFICATION_SYSTEM_INIT_LOCK: Mutex<()> = Mutex::const_new(());
pub(crate) async fn get_notification_system() -> S3Result<Arc<rustfs_notify::NotificationSystem>> {
if let Some(system) = rustfs_notify::notification_system() {
if is_event_notifier_reconciled()
&& let Some(system) = rustfs_notify::notification_system()
&& system.runtime_lifecycle_is_converged()
{
return Ok(system);
}
let _guard = NOTIFICATION_SYSTEM_INIT_LOCK.lock().await;
if let Some(system) = rustfs_notify::notification_system() {
return Ok(system);
init_event_notifier()
.await
.map_err(|err| s3_error!(InternalError, "failed to reconcile notification runtime: {}", err))?;
let system =
rustfs_notify::notification_system().ok_or_else(|| s3_error!(InternalError, "notification system not initialized"))?;
if !system.runtime_lifecycle_is_converged() {
return Err(s3_error!(InternalError, "latest notification lifecycle generation has not converged"));
}
init_event_notifier().await;
rustfs_notify::notification_system().ok_or_else(|| s3_error!(InternalError, "notification system not initialized"))
Ok(system)
}
pub(crate) async fn load_notification_config_snapshot() -> S3Result<(Arc<rustfs_notify::NotificationSystem>, Config)> {
let system = get_notification_system().await?;
let config = system.config.read().await.clone();
let config = system.config_snapshot().await;
Ok((system, config))
}
rustfs/src/admin/handlers/oidc.rs
+44 -18
View File
@@ -19,7 +19,10 @@ use crate::admin::runtime_sources::{
current_server_config_for_context,
};
use crate::admin::service::federated_identity::DefaultFederatedSessionBinding;
use crate::admin::storage_api::config::{read_admin_config_without_migrate, save_admin_server_config};
use crate::admin::storage_api::config::{
read_admin_config_without_migrate, read_admin_config_without_migrate_no_lock, save_admin_server_config_no_lock,
with_admin_server_config_write_lock,
};
use crate::auth::{check_key_valid, get_session_token};
use crate::server::{ADMIN_PREFIX, CONSOLE_PREFIX, MINIO_ADMIN_PREFIX, RemoteAddr};
use http::StatusCode;
@@ -368,13 +371,20 @@ impl Operation for PutOidcConfigHandler {
if is_env_managed_provider(provider_id) {
return Err(s3_error!(AccessDenied, "provider is managed by environment variables"));
}
let provider_id = provider_id.to_owned();
let request: OidcConfigUpsertRequest = parse_json_body(&mut req).await?;
let mut config = load_server_config_from_store().await?;
let existing_secret = persisted_provider_secret(&config, provider_id);
let provider_config = build_provider_config_from_upsert(provider_id, request, existing_secret)?;
upsert_persisted_provider_config(&mut config, &provider_config);
save_server_config_to_store(&config).await?;
let store = oidc_config_store()?;
let lock_store = store.clone();
with_admin_server_config_write_lock(lock_store, move || async move {
let mut config = load_server_config_from_store_locked(store.clone()).await?;
let existing_secret = persisted_provider_secret(&config, &provider_id);
let provider_config = build_provider_config_from_upsert(&provider_id, request, existing_secret)?;
upsert_persisted_provider_config(&mut config, &provider_config);
save_server_config_to_store_locked(store, &config).await
})
.await
.map_err(|err| s3_error!(InternalError, "failed to lock server config update: {}", err))??;
json_response(
StatusCode::OK,
@@ -403,10 +413,17 @@ impl Operation for DeleteOidcConfigHandler {
if is_env_managed_provider(provider_id) {
return Err(s3_error!(AccessDenied, "provider is managed by environment variables"));
}
let provider_id = provider_id.to_owned();
let mut config = load_server_config_from_store().await?;
delete_persisted_provider_config(&mut config, provider_id)?;
save_server_config_to_store(&config).await?;
let store = oidc_config_store()?;
let lock_store = store.clone();
with_admin_server_config_write_lock(lock_store, move || async move {
let mut config = load_server_config_from_store_locked(store.clone()).await?;
delete_persisted_provider_config(&mut config, &provider_id)?;
save_server_config_to_store_locked(store, &config).await
})
.await
.map_err(|err| s3_error!(InternalError, "failed to lock server config update: {}", err))??;
json_response(
StatusCode::OK,
@@ -881,23 +898,32 @@ fn json_response<T: Serialize>(status: StatusCode, payload: &T) -> S3Result<S3Re
}
async fn load_server_config_from_store() -> S3Result<ServerConfig> {
let context = current_app_context();
let Some(store) = current_object_store_handle_for_context(context.as_deref()) else {
return Err(s3_error!(InternalError, "storage layer not initialized"));
};
let store = oidc_config_store()?;
read_admin_config_without_migrate(store)
.await
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("failed to load server config: {e}")))
}
async fn save_server_config_to_store(config: &ServerConfig) -> S3Result<()> {
fn oidc_config_store() -> S3Result<std::sync::Arc<crate::admin::storage_api::runtime::ECStore>> {
let context = current_app_context();
let Some(store) = current_object_store_handle_for_context(context.as_deref()) else {
return Err(s3_error!(InternalError, "storage layer not initialized"));
};
current_object_store_handle_for_context(context.as_deref())
.ok_or_else(|| s3_error!(InternalError, "storage layer not initialized"))
}
save_admin_server_config(store, config)
async fn load_server_config_from_store_locked(
store: std::sync::Arc<crate::admin::storage_api::runtime::ECStore>,
) -> S3Result<ServerConfig> {
read_admin_config_without_migrate_no_lock(store)
.await
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("failed to load server config: {e}")))
}
async fn save_server_config_to_store_locked(
store: std::sync::Arc<crate::admin::storage_api::runtime::ECStore>,
config: &ServerConfig,
) -> S3Result<()> {
save_admin_server_config_no_lock(store, config)
.await
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("failed to save server config: {e}")))
}
rustfs/src/admin/handlers/plugins_instances.rs
+11 -14
View File
@@ -605,7 +605,7 @@ fn plugin_instance_mutation_block_reason(
target_type: &str,
target_name: &str,
target_label: &str,
) -> Option<String> {
) -> S3Result<Option<String>> {
shared_target_mutation_block_reason(context.specs, context.route_prefix, config, target_type, target_name, target_label)
}
@@ -667,7 +667,7 @@ async fn collect_domain_instances(context: PluginInstanceDomainContext) -> S3Res
let config = plugin_instance_config_snapshot(context).await?;
let module_disabled_reason = module_disabled_block_reason(context.domain, "listing plugin instances");
let mut entries = Vec::new();
for instance in collect_target_instances(context.specs, context.route_prefix, &config, runtime_statuses) {
for instance in collect_target_instances(context.specs, context.route_prefix, &config, runtime_statuses)? {
entries.push(plugin_instance_list_entry(instance, module_disabled_reason.clone()));
}
Ok(entries)
@@ -688,13 +688,7 @@ async fn find_plugin_instance(instance_id: &str) -> S3Result<Option<TargetInstan
let context = plugin_instance_domain_context(parse_plugin_instance_id(instance_id)?.1);
let runtime_statuses = plugin_instance_runtime_statuses(context).await?;
let config = plugin_instance_config_snapshot(context).await?;
Ok(find_target_instance(
context.specs,
context.route_prefix,
&config,
runtime_statuses,
instance_id,
))
find_target_instance(context.specs, context.route_prefix, &config, runtime_statuses, instance_id)
}
async fn set_plugin_instance_config(
@@ -800,7 +794,7 @@ impl Operation for PutPluginInstanceHandler {
resolved.target_spec.subsystem,
&resolved.target_name,
"plugin instance",
) {
)? {
return Err(s3_error!(InvalidRequest, "{reason}"));
}
@@ -852,7 +846,7 @@ impl Operation for DeletePluginInstanceHandler {
resolved.target_spec.subsystem,
&resolved.target_name,
"plugin instance",
) {
)? {
return Err(s3_error!(InvalidRequest, "{reason}"));
}
@@ -965,7 +959,8 @@ mod tests {
)]));
let instances =
collect_target_instances(super::notification_target_specs(), NOTIFY_ROUTE_PREFIX, &config, HashMap::new());
collect_target_instances(super::notification_target_specs(), NOTIFY_ROUTE_PREFIX, &config, HashMap::new())
.expect("collect target instances");
let primary = instances
.into_iter()
.find(|instance| instance.account_id == "primary" && instance.service == "webhook")
@@ -988,7 +983,8 @@ mod tests {
NOTIFY_ROUTE_PREFIX,
&Config(HashMap::new()),
HashMap::new(),
);
)
.expect("collect target instances");
let env_only = instances
.into_iter()
.find(|instance| instance.account_id == "env-only")
@@ -1008,7 +1004,8 @@ mod tests {
NOTIFY_ROUTE_PREFIX,
&Config(HashMap::new()),
runtime_statuses,
);
)
.expect("collect target instances");
let runtime_only = instances
.into_iter()
rustfs/src/admin/handlers/target_descriptor.rs
+29 -27
View File
@@ -30,7 +30,7 @@ use rustfs_targets::{
check_redis_server_available,
config::{
TargetPluginInstanceCompatDescriptor, TargetPluginInstanceRecord, build_amqp_args, build_kafka_args, build_mysql_args,
build_nats_args, build_postgres_args, build_pulsar_args, build_redis_args, normalize_target_plugin_instances,
build_nats_args, build_postgres_args, build_pulsar_args, build_redis_args, try_normalize_target_plugin_instances,
validate_redis_config,
},
manifest::builtin_target_manifest,
@@ -223,11 +223,11 @@ pub(crate) fn endpoint_source(
config: &Config,
target_type: &str,
target_name: &str,
) -> TargetEndpointSource {
let snapshot = collect_endpoint_snapshot(specs, route_prefix, config);
) -> S3Result<TargetEndpointSource> {
let snapshot = collect_endpoint_snapshot(specs, route_prefix, config)?;
let service = target_service_name(specs, target_type).unwrap_or_default();
let key = normalized_endpoint_key(target_name, service);
classify_endpoint_source(&snapshot.config_targets, &snapshot.env_targets, &key)
Ok(classify_endpoint_source(&snapshot.config_targets, &snapshot.env_targets, &key))
}
pub(crate) fn target_mutation_block_reason(
@@ -237,8 +237,8 @@ pub(crate) fn target_mutation_block_reason(
target_type: &str,
target_name: &str,
target_label: &str,
) -> Option<String> {
match endpoint_source(specs, route_prefix, config, target_type, target_name) {
) -> S3Result<Option<String>> {
Ok(match endpoint_source(specs, route_prefix, config, target_type, target_name)? {
TargetEndpointSource::Env => Some(format!(
"{} '{}' is managed by environment variables and cannot be modified from the console",
target_label, target_name
@@ -248,7 +248,7 @@ pub(crate) fn target_mutation_block_reason(
target_label, target_name
)),
TargetEndpointSource::Config | TargetEndpointSource::Runtime => None,
}
})
}
pub(crate) fn target_module_disabled_reason(module_name: &str, env_key: &str, enabled: bool, action: &str) -> Option<String> {
@@ -304,10 +304,10 @@ pub(crate) fn merge_target_endpoints(
route_prefix: &str,
config: &Config,
runtime_statuses: HashMap<EndpointKey, String>,
) -> Vec<MergedTargetEndpoint> {
) -> S3Result<Vec<MergedTargetEndpoint>> {
let mut endpoints = Vec::new();
let mut seen = HashSet::new();
let snapshot = collect_endpoint_snapshot(specs, route_prefix, config);
let snapshot = collect_endpoint_snapshot(specs, route_prefix, config)?;
let mut normalized_runtime_statuses: HashMap<EndpointKey, (String, String, String)> = HashMap::new();
for ((account_id, service), status) in runtime_statuses {
@@ -361,7 +361,7 @@ pub(crate) fn merge_target_endpoints(
}
endpoints.sort_by(|a, b| a.service.cmp(&b.service).then_with(|| a.account_id.cmp(&b.account_id)));
endpoints
Ok(endpoints)
}
pub(crate) fn canonical_target_instance_id(plugin_id: &str, domain: TargetDomain, instance_id: &str) -> String {
@@ -373,12 +373,12 @@ pub(crate) fn collect_target_instances(
route_prefix: &str,
config: &Config,
runtime_statuses: HashMap<EndpointKey, String>,
) -> Vec<TargetInstanceReadModel> {
) -> S3Result<Vec<TargetInstanceReadModel>> {
let mut instances = Vec::new();
let mut seen = HashSet::new();
let mut normalized_runtime_statuses: HashMap<EndpointKey, (String, String, String)> = HashMap::new();
let domain = inferred_target_domain(route_prefix);
let snapshot = collect_endpoint_snapshot(specs, route_prefix, config);
let snapshot = collect_endpoint_snapshot(specs, route_prefix, config)?;
for ((account_id, service), status) in runtime_statuses {
let normalized = normalized_endpoint_key(&account_id, &service);
@@ -439,7 +439,7 @@ pub(crate) fn collect_target_instances(
}
instances.sort_by(|a, b| a.service.cmp(&b.service).then_with(|| a.account_id.cmp(&b.account_id)));
instances
Ok(instances)
}
pub(crate) fn find_target_instance(
@@ -448,10 +448,10 @@ pub(crate) fn find_target_instance(
config: &Config,
runtime_statuses: HashMap<EndpointKey, String>,
canonical_id: &str,
) -> Option<TargetInstanceReadModel> {
collect_target_instances(specs, route_prefix, config, runtime_statuses)
) -> S3Result<Option<TargetInstanceReadModel>> {
Ok(collect_target_instances(specs, route_prefix, config, runtime_statuses)?
.into_iter()
.find(|instance| instance.canonical_id == canonical_id)
.find(|instance| instance.canonical_id == canonical_id))
}
pub(crate) fn allowed_target_keys(specs: &[AdminTargetSpec], target_type: &str) -> HashSet<&'static str> {
@@ -559,11 +559,11 @@ fn normalized_target_instances(
specs: &[AdminTargetSpec],
route_prefix: &str,
config: &Config,
) -> Vec<TargetPluginInstanceRecord> {
specs
.iter()
.flat_map(|spec| {
normalize_target_plugin_instances(
) -> S3Result<Vec<TargetPluginInstanceRecord>> {
let mut instances = Vec::new();
for spec in specs {
instances.extend(
try_normalize_target_plugin_instances(
config,
&TargetPluginInstanceCompatDescriptor {
domain: inferred_target_domain(route_prefix),
@@ -574,8 +574,10 @@ fn normalized_target_instances(
valid_fields: spec.valid_keys,
},
)
})
.collect()
.map_err(|err| s3_error!(InvalidRequest, "invalid {} target environment: {}", spec.service, err))?,
);
}
Ok(instances)
}
fn inferred_target_domain(route_prefix: &str) -> TargetDomain {
@@ -597,8 +599,8 @@ fn target_spec_by_service<'a>(specs: &'a [AdminTargetSpec], service: &str) -> Op
specs.iter().find(|spec| spec.service == service)
}
fn collect_endpoint_snapshot(specs: &[AdminTargetSpec], route_prefix: &str, config: &Config) -> TargetEndpointSnapshot {
let normalized_instances = normalized_target_instances(specs, route_prefix, config);
fn collect_endpoint_snapshot(specs: &[AdminTargetSpec], route_prefix: &str, config: &Config) -> S3Result<TargetEndpointSnapshot> {
let normalized_instances = normalized_target_instances(specs, route_prefix, config)?;
let mut configured_keys = Vec::new();
let mut config_targets = HbHashSet::new();
let mut env_targets = HbHashSet::new();
@@ -618,12 +620,12 @@ fn collect_endpoint_snapshot(specs: &[AdminTargetSpec], route_prefix: &str, conf
}
}
TargetEndpointSnapshot {
Ok(TargetEndpointSnapshot {
normalized_instances,
configured_keys,
config_targets,
env_targets,
}
})
}
async fn retry_with_backoff<F, Fut, T>(mut operation: F, max_attempts: usize, base_delay: Duration) -> Result<T, Error>
rustfs/src/admin/router.rs
+1 -1
View File
@@ -628,7 +628,7 @@ fn resolve_object_lambda_webhook_config_from_server_config(
async fn load_current_server_config() -> S3Result<Config> {
if let Some(system) = notification_system() {
return Ok(system.config.read().await.clone());
return Ok(system.config_snapshot().await);
}
if let Some(store) = current_object_store_handle() {
rustfs/src/admin/service/config.rs
+416 -64
View File
@@ -19,18 +19,21 @@ use crate::admin::runtime_sources::{
use crate::admin::storage_api::config::{STORAGE_CLASS_SUB_SYS, read_admin_config_without_migrate, storageclass};
use crate::admin::storage_api::contract::admin::StorageAdminApi;
use crate::admin::storage_api::runtime::ECStore;
use crate::server::{
MODULE_SWITCHES_SIGNAL_SUBSYSTEM, apply_audit_module_switch_for_context, reconcile_event_notifier_from_store,
};
use rustfs_audit::reload_audit_config;
use rustfs_config::AUDIT_DEFAULT_DIR;
use rustfs_config::audit::{AUDIT_MQTT_SUB_SYS, AUDIT_REDIS_DEFAULT_CHANNEL, AUDIT_WEBHOOK_SUB_SYS};
use rustfs_config::notify::{NOTIFY_MQTT_SUB_SYS, NOTIFY_REDIS_DEFAULT_CHANNEL, NOTIFY_WEBHOOK_SUB_SYS};
use rustfs_config::notify::{NOTIFY_ROUTE_PREFIX, NOTIFY_SUB_SYSTEMS};
use rustfs_config::oidc::IDENTITY_OPENID_SUB_SYS;
use rustfs_config::server_config::{Config as ServerConfig, KVS};
use rustfs_config::{AUDIT_DEFAULT_DIR, EVENT_DEFAULT_DIR};
use rustfs_config::{DEFAULT_DELIMITER, ENABLE_KEY, EnableState};
use rustfs_config::{HEAL_SUB_SYS, SCANNER_SUB_SYS};
use rustfs_iam::oidc::load_oidc_provider_configs_from_server_config;
use rustfs_targets::config::{
validate_amqp_config, validate_kafka_config, validate_mqtt_config, validate_mysql_config, validate_nats_config,
validate_postgres_config, validate_pulsar_config, validate_redis_config, validate_webhook_config,
try_collect_target_configs, validate_amqp_config, validate_kafka_config, validate_mqtt_config, validate_mysql_config,
validate_nats_config, validate_postgres_config, validate_pulsar_config, validate_redis_config, validate_webhook_config,
};
use s3s::{S3Error, S3ErrorCode, S3Result};
use std::future::Future;
@@ -38,10 +41,11 @@ use tracing::warn;
use url::Url;
pub fn is_dynamic_config_subsystem(sub_system: &str) -> bool {
matches!(
sub_system,
STORAGE_CLASS_SUB_SYS | AUDIT_WEBHOOK_SUB_SYS | AUDIT_MQTT_SUB_SYS | SCANNER_SUB_SYS | HEAL_SUB_SYS
)
NOTIFY_SUB_SYSTEMS.contains(&sub_system)
|| matches!(
sub_system,
STORAGE_CLASS_SUB_SYS | AUDIT_WEBHOOK_SUB_SYS | AUDIT_MQTT_SUB_SYS | SCANNER_SUB_SYS | HEAL_SUB_SYS
)
}
pub(crate) const FULL_CONFIG_WORKER_SUBSYSTEMS: [&str; 2] = [AUDIT_WEBHOOK_SUB_SYS, SCANNER_SUB_SYS];
@@ -179,30 +183,20 @@ fn target_kvs(config: &ServerConfig, sub_system: &str, target: &str) -> KVS {
}
fn validate_notify_subsystem_config(config: &ServerConfig, sub_system: &str) -> S3Result<()> {
let Some(targets) = config.0.get(sub_system) else {
let Some(descriptor) = rustfs_notify::factory::builtin_target_descriptors()
.into_iter()
.find(|descriptor| descriptor.subsystem() == sub_system)
else {
return Ok(());
};
let plugin = descriptor.plugin();
for target in targets.keys() {
let kvs = target_kvs(config, sub_system, target);
if !target_enabled(&kvs) {
continue;
}
let result = match sub_system {
"notify_webhook" => validate_webhook_config(&kvs, EVENT_DEFAULT_DIR),
"notify_amqp" => validate_amqp_config(&kvs, EVENT_DEFAULT_DIR),
"notify_kafka" => validate_kafka_config(&kvs, EVENT_DEFAULT_DIR),
"notify_mqtt" => validate_mqtt_config(&kvs),
"notify_mysql" => validate_mysql_config(&kvs, EVENT_DEFAULT_DIR),
"notify_nats" => validate_nats_config(&kvs, EVENT_DEFAULT_DIR),
"notify_postgres" => validate_postgres_config(&kvs, EVENT_DEFAULT_DIR),
"notify_pulsar" => validate_pulsar_config(&kvs, EVENT_DEFAULT_DIR),
"notify_redis" => validate_redis_config(&kvs, EVENT_DEFAULT_DIR, NOTIFY_REDIS_DEFAULT_CHANNEL),
_ => return Ok(()),
};
result.map_err(|err| invalid_request(format!("invalid {sub_system} config for target '{target}': {err}")))?;
for (target, kvs) in try_collect_target_configs(config, NOTIFY_ROUTE_PREFIX, plugin.target_type(), plugin.valid_fields_set())
.map_err(|err| invalid_request(format!("invalid {sub_system} config: {err}")))?
{
plugin
.validate_config(&kvs)
.map_err(|err| invalid_request(format!("invalid {sub_system} config for target '{target}': {err}")))?;
}
Ok(())
@@ -314,8 +308,7 @@ pub(crate) async fn prepare_server_config_for_context(
Some(STORAGE_CLASS_SUB_SYS) => {
prepared.storage_class = Some(prepare_storage_class_runtime_config_for_context(context, config).await?);
}
Some(NOTIFY_WEBHOOK_SUB_SYS) => validate_notify_subsystem_config(config, NOTIFY_WEBHOOK_SUB_SYS)?,
Some(NOTIFY_MQTT_SUB_SYS) => validate_notify_subsystem_config(config, NOTIFY_MQTT_SUB_SYS)?,
Some(sub_system) if NOTIFY_SUB_SYSTEMS.contains(&sub_system) => validate_notify_subsystem_config(config, sub_system)?,
Some(AUDIT_WEBHOOK_SUB_SYS) => validate_audit_subsystem_config(config, AUDIT_WEBHOOK_SUB_SYS)?,
Some(AUDIT_MQTT_SUB_SYS) => validate_audit_subsystem_config(config, AUDIT_MQTT_SUB_SYS)?,
Some(IDENTITY_OPENID_SUB_SYS) => validate_identity_openid_config(config)?,
@@ -324,8 +317,9 @@ pub(crate) async fn prepare_server_config_for_context(
Some(_) => {}
None => {
prepared.storage_class = Some(prepare_storage_class_runtime_config_for_context(context, config).await?);
validate_notify_subsystem_config(config, NOTIFY_WEBHOOK_SUB_SYS)?;
validate_notify_subsystem_config(config, NOTIFY_MQTT_SUB_SYS)?;
for sub_system in NOTIFY_SUB_SYSTEMS {
validate_notify_subsystem_config(config, sub_system)?;
}
validate_audit_subsystem_config(config, AUDIT_WEBHOOK_SUB_SYS)?;
validate_audit_subsystem_config(config, AUDIT_MQTT_SUB_SYS)?;
validate_identity_openid_config(config)?;
@@ -371,6 +365,18 @@ pub async fn apply_dynamic_config_for_subsystem_for_context(
AUDIT_WEBHOOK_SUB_SYS | AUDIT_MQTT_SUB_SYS => reload_audit_config(config.clone())
.await
.map_err(|err| internal_error(format!("failed to reload audit config: {err}")))?,
sub_system if NOTIFY_SUB_SYSTEMS.contains(&sub_system) => {
// The notify lifecycle is intentionally process-wide. The explicit
// store keeps the persisted snapshot bound to the request context;
// the ECStore NotificationSys resolved below is the peer-broadcast
// service, not the notify target runtime.
let system = rustfs_notify::ensure_live_events();
let store = resolve_runtime_config_store_for_context(context)?;
system
.reload_persisted_config_from_store(store)
.await
.map_err(|err| internal_error(format!("failed to reload notification config: {err}")))?;
}
SCANNER_SUB_SYS => rustfs_scanner::apply_scanner_runtime_config(config)
.map_err(|err| internal_error(format!("failed to reload scanner config: {err}")))?,
HEAL_SUB_SYS => rustfs_scanner::apply_scanner_runtime_config(config)
@@ -387,6 +393,18 @@ pub async fn apply_dynamic_config_for_subsystem(config: &ServerConfig, sub_syste
}
pub async fn reload_dynamic_config_runtime_state_for_context(context: Option<&AppContext>, sub_system: &str) -> S3Result<()> {
if sub_system == MODULE_SWITCHES_SIGNAL_SUBSYSTEM {
let store = resolve_runtime_config_store_for_context(context)?;
let notify_result = reconcile_event_notifier_from_store(store).await;
let audit_result = apply_audit_module_switch_for_context(context).await;
return match (notify_result, audit_result) {
(Ok(()), Ok(())) => Ok(()),
(Err(_), Ok(())) => Err(internal_error("failed to reconcile notification module switch")),
(Ok(()), Err(_)) => Err(internal_error("failed to reconcile audit module switch")),
(Err(_), Err(_)) => Err(internal_error("failed to reconcile notification and audit module switches")),
};
}
if !is_dynamic_config_subsystem(sub_system) {
return Err(internal_error(format!("unsupported dynamic config subsystem: {sub_system}")));
}
@@ -398,9 +416,8 @@ pub async fn reload_dynamic_config_runtime_state_for_context(context: Option<&Ap
})?;
apply_dynamic_config_for_subsystem_for_context(context, &config, sub_system)
.await
.map_err(|err| {
warn!("peer reload_dynamic_config: failed to apply {sub_system}: {err}");
err
.inspect_err(|_| {
warn!(config_subsystem = sub_system, reason = "apply_failed", "Peer dynamic config apply failed");
})?;
Ok(())
}
@@ -428,19 +445,19 @@ where
let (config, prepared) = prepare(config).await?;
publish(&config, prepared)?;
// Worker reloads mutate live state and have no rollback contract. They are
// therefore best-effort after the validated storage/server snapshots are
// published; a transient worker failure must not leave this peer on stale
// erasure geometry.
// Worker reloads mutate live state and have no rollback contract, so the
// validated snapshots stay published. The RPC still reports convergence
// failure explicitly so the originating Admin request cannot claim success.
if let Err(err) = apply_workers(config).await {
warn!(
event = EVENT_CONFIG_WORKER_RELOAD_FAILED,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_CONFIG,
state = CONFIG_WORKER_RELOAD_FAILURE_STATE,
error = ?err,
reason = "apply_failed",
"Runtime config snapshot was published but a worker reload failed"
);
return Err(err);
}
Ok(())
}
@@ -468,20 +485,29 @@ pub async fn reload_runtime_config_snapshot_for_context(context: Option<&AppCont
Ok(())
},
|config| async move {
let mut failures = Vec::new();
for sub_system in FULL_CONFIG_WORKER_SUBSYSTEMS {
if let Err(err) = apply_dynamic_config_for_subsystem_for_context(context, &config, sub_system).await {
if apply_dynamic_config_for_subsystem_for_context(context, &config, sub_system)
.await
.is_err()
{
failures.push(sub_system);
warn!(
event = EVENT_CONFIG_WORKER_RELOAD_FAILED,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_CONFIG,
config_subsystem = sub_system,
state = CONFIG_WORKER_RELOAD_FAILURE_STATE,
error = ?err,
reason = "apply_failed",
"Peer runtime config snapshot was published but a subsystem worker reload failed"
);
}
}
Ok(())
if failures.is_empty() {
Ok(())
} else {
Err(internal_error(format!("runtime worker reload failed: {}", failures.join("; "))))
}
},
)
.await
@@ -492,20 +518,77 @@ pub async fn reload_runtime_config_snapshot() -> S3Result<()> {
reload_runtime_config_snapshot_for_context(context.as_deref()).await
}
pub async fn signal_dynamic_config_reload_for_context(context: Option<&AppContext>, sub_system: &str) {
if !is_dynamic_config_subsystem(sub_system) {
return;
pub async fn preflight_dynamic_config_reload_for_context(context: Option<&AppContext>, sub_system: &str) -> S3Result<()> {
if sub_system != MODULE_SWITCHES_SIGNAL_SUBSYSTEM && !is_dynamic_config_subsystem(sub_system) {
return Err(internal_error(format!("unsupported dynamic config subsystem: {sub_system}")));
}
let Some(notification_sys) = current_notification_system_for_context(context) else {
return;
return Ok(());
};
for failure in notification_sys.reload_dynamic_config(sub_system).await {
if let Some(err) = failure.err {
tracing::warn!("peer {} dynamic config reload for {} failed: {}", failure.host, sub_system, err);
let mut failed = 0usize;
for failure in notification_sys.preflight_dynamic_config(sub_system).await {
if failure.err.is_some() {
failed += 1;
let host = if failure.host.is_empty() { "<unknown>" } else { &failure.host };
warn!(
peer = host,
config_subsystem = sub_system,
reason = "unsupported",
"Peer does not support dynamic config convergence"
);
}
}
if failed == 0 {
Ok(())
} else {
Err(internal_error(format!(
"{failed} peer(s) do not support dynamic config convergence for {sub_system}"
)))
}
}
pub async fn preflight_dynamic_config_reload(sub_system: &str) -> S3Result<()> {
let context = current_app_context();
preflight_dynamic_config_reload_for_context(context.as_deref(), sub_system).await
}
pub async fn signal_dynamic_config_reload_checked_for_context(context: Option<&AppContext>, sub_system: &str) -> S3Result<()> {
if sub_system != MODULE_SWITCHES_SIGNAL_SUBSYSTEM && !is_dynamic_config_subsystem(sub_system) {
return Err(internal_error(format!("unsupported dynamic config subsystem: {sub_system}")));
}
let Some(notification_sys) = current_notification_system_for_context(context) else {
return Ok(());
};
let mut failed = 0usize;
for failure in notification_sys.reload_dynamic_config(sub_system).await {
if failure.err.is_some() {
failed += 1;
let host = if failure.host.is_empty() { "<unknown>" } else { &failure.host };
warn!(
peer = host,
config_subsystem = sub_system,
reason = "reload_failed",
"Peer dynamic config convergence failed"
);
}
}
if failed == 0 {
Ok(())
} else {
Err(internal_error(format!("{failed} peer(s) failed dynamic config reload for {sub_system}")))
}
}
pub async fn signal_dynamic_config_reload_for_context(context: Option<&AppContext>, sub_system: &str) {
if let Err(err) = signal_dynamic_config_reload_checked_for_context(context, sub_system).await {
tracing::warn!("peer dynamic config reload for {} failed: {}", sub_system, err);
}
}
pub async fn signal_dynamic_config_reload(sub_system: &str) {
@@ -513,16 +596,36 @@ pub async fn signal_dynamic_config_reload(sub_system: &str) {
signal_dynamic_config_reload_for_context(context.as_deref(), sub_system).await;
}
pub async fn signal_config_snapshot_reload_for_context(context: Option<&AppContext>) {
pub async fn signal_dynamic_config_reload_checked(sub_system: &str) -> S3Result<()> {
let context = current_app_context();
signal_dynamic_config_reload_checked_for_context(context.as_deref(), sub_system).await
}
pub async fn signal_config_snapshot_reload_checked_for_context(context: Option<&AppContext>) -> S3Result<()> {
let Some(notification_sys) = current_notification_system_for_context(context) else {
return;
return Ok(());
};
let mut failed = 0usize;
for failure in notification_sys.refresh_config_snapshot().await {
if let Some(err) = failure.err {
tracing::warn!("peer config snapshot refresh failed for {}: {}", failure.host, err);
if failure.err.is_some() {
failed += 1;
let host = if failure.host.is_empty() { "<unknown>" } else { &failure.host };
warn!(peer = host, reason = "reload_failed", "Peer config snapshot refresh failed");
}
}
if failed == 0 {
Ok(())
} else {
Err(internal_error(format!("{failed} peer(s) failed config snapshot reload")))
}
}
pub async fn signal_config_snapshot_reload_for_context(context: Option<&AppContext>) {
if let Err(err) = signal_config_snapshot_reload_checked_for_context(context).await {
tracing::warn!("peer config snapshot refresh failed: {err}");
}
}
pub async fn signal_config_snapshot_reload() {
@@ -530,30 +633,46 @@ pub async fn signal_config_snapshot_reload() {
signal_config_snapshot_reload_for_context(context.as_deref()).await;
}
pub async fn signal_config_snapshot_reload_checked() -> S3Result<()> {
let context = current_app_context();
signal_config_snapshot_reload_checked_for_context(context.as_deref()).await
}
#[cfg(test)]
mod tests {
use super::*;
use crate::admin::runtime_sources::{IamInterface, KmsInterface, ServerConfigInterface, StorageClassInterface};
use crate::admin::storage_api::bucket::metadata::{BUCKET_LIFECYCLE_CONFIG, BUCKET_REPLICATION_CONFIG};
use crate::admin::storage_api::config::save_admin_server_config;
use crate::admin::storage_api::config::{
read_admin_config_without_migrate, read_admin_config_without_migrate_no_lock, save_admin_server_config,
save_admin_server_config_no_lock, with_admin_server_config_write_lock,
};
use crate::admin::storage_api::error::StorageError;
use crate::server::{
ModuleSwitchSource, PersistedModuleSwitches, current_module_switch_snapshot, is_event_notifier_reconciled,
refresh_persisted_module_switches_from, save_persisted_module_switches_to,
};
use crate::storage_api::cluster::{Endpoint, EndpointServerPools, Endpoints, PoolEndpoints};
use crate::storage_api::startup::storage::{init_local_disks_with_instance_ctx, new_instance_ctx};
use rustfs_config::notify::NOTIFY_WEBHOOK_SUB_SYS;
use rustfs_config::notify::{ENV_NOTIFY_WEBHOOK_ENABLE, ENV_NOTIFY_WEBHOOK_ENDPOINT, NOTIFY_WEBHOOK_SUB_SYS};
use rustfs_config::oidc::{OIDC_CLIENT_ID, OIDC_CONFIG_URL, OIDC_SCOPES};
use rustfs_config::{HEAL_SUB_SYS, SCANNER_SUB_SYS};
use rustfs_config::{MQTT_BROKER, MQTT_QUEUE_DIR, MQTT_TOPIC, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR};
use rustfs_iam::{store::object::ObjectStore, sys::IamSys};
use rustfs_kms::KmsServiceManager;
use std::collections::HashMap;
use std::future::{Future, poll_fn};
use std::path::Path;
use std::sync::{
Arc, Mutex,
atomic::{AtomicUsize, Ordering},
};
use std::task::Poll;
use tempfile::TempDir;
use tokio_util::sync::CancellationToken;
const LIFECYCLE_RELOAD_LABEL: &str = "lifecycle";
const REAL_STORE_TEST_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30);
const REPLICATION_RELOAD_LABEL: &str = "replication";
fn without_storage_class_env<R>(f: impl FnOnce() -> R) -> R {
@@ -748,6 +867,139 @@ mod tests {
}
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn concurrent_real_store_config_rmw_preserves_notify_and_oidc_updates() {
let temp_dir = TempDir::new().expect("server config RMW temp dir");
let store = build_isolated_heterogeneous_store(temp_dir.path()).await;
save_admin_server_config(store.clone(), &ServerConfig::new())
.await
.expect("persist baseline server config");
let notify_entered = Arc::new(tokio::sync::Notify::new());
let release_notify = Arc::new(tokio::sync::Notify::new());
let (oidc_polled_tx, oidc_polled_rx) = tokio::sync::oneshot::channel();
let (oidc_entered_tx, mut oidc_entered_rx) = tokio::sync::oneshot::channel();
let transaction_order = Arc::new(Mutex::new(Vec::new()));
let notify_task = {
let store = store.clone();
let notify_entered = notify_entered.clone();
let release_notify = release_notify.clone();
let transaction_order = transaction_order.clone();
tokio::spawn(async move {
let transaction_store = store.clone();
with_admin_server_config_write_lock(store, move || async move {
let mut config = read_admin_config_without_migrate_no_lock(transaction_store.clone()).await?;
transaction_order.lock().expect("transaction order lock").push("notify-read");
notify_entered.notify_one();
release_notify.notified().await;
let mut target = KVS::new();
target.insert(ENABLE_KEY.to_string(), EnableState::On.to_string());
target.insert(WEBHOOK_ENDPOINT.to_string(), "https://notify.example.test/hook".to_string());
config
.0
.entry(NOTIFY_WEBHOOK_SUB_SYS.to_string())
.or_default()
.insert("concurrent-notify".to_string(), target);
save_admin_server_config_no_lock(transaction_store, &config).await?;
transaction_order.lock().expect("transaction order lock").push("notify-save");
Ok::<(), StorageError>(())
})
.await
.expect("notify transaction should acquire config locks")
.expect("notify transaction should persist");
})
};
tokio::time::timeout(REAL_STORE_TEST_TIMEOUT, notify_entered.notified())
.await
.expect("notify transaction should enter");
let oidc_task = {
let store = store.clone();
let transaction_order = transaction_order.clone();
tokio::spawn(async move {
let transaction_store = store.clone();
let transaction = with_admin_server_config_write_lock(store, move || async move {
let _ = oidc_entered_tx.send(());
let mut config = read_admin_config_without_migrate_no_lock(transaction_store.clone()).await?;
transaction_order.lock().expect("transaction order lock").push("oidc-read");
let mut provider = KVS::new();
provider.insert(
OIDC_CONFIG_URL.to_string(),
"https://identity.example.test/.well-known/openid-configuration".to_string(),
);
provider.insert(OIDC_CLIENT_ID.to_string(), "console".to_string());
config
.0
.entry(IDENTITY_OPENID_SUB_SYS.to_string())
.or_default()
.insert("concurrent-oidc".to_string(), provider);
save_admin_server_config_no_lock(transaction_store, &config).await?;
transaction_order.lock().expect("transaction order lock").push("oidc-save");
Ok::<(), StorageError>(())
});
tokio::pin!(transaction);
let mut oidc_polled_tx = Some(oidc_polled_tx);
poll_fn(|cx| match transaction.as_mut().poll(cx) {
Poll::Pending => {
if let Some(tx) = oidc_polled_tx.take() {
let _ = tx.send(());
}
Poll::Ready(())
}
Poll::Ready(_) => panic!("OIDC transaction entered while notify held the config lock"),
})
.await;
transaction
.await
.expect("OIDC transaction should acquire config locks")
.expect("OIDC transaction should persist");
})
};
tokio::time::timeout(REAL_STORE_TEST_TIMEOUT, oidc_polled_rx)
.await
.expect("OIDC transaction should be polled")
.expect("OIDC transaction poll signal should be delivered");
assert!(
matches!(oidc_entered_rx.try_recv(), Err(tokio::sync::oneshot::error::TryRecvError::Empty)),
"OIDC RMW must not enter while the notify RMW owns the server-config transaction"
);
release_notify.notify_one();
tokio::time::timeout(REAL_STORE_TEST_TIMEOUT, async {
notify_task.await.expect("notify task should not panic");
oidc_task.await.expect("OIDC task should not panic");
})
.await
.expect("both server-config transactions should finish");
let persisted = read_admin_config_without_migrate(store)
.await
.expect("read final server config");
assert_eq!(
persisted
.get_value(NOTIFY_WEBHOOK_SUB_SYS, "concurrent-notify")
.expect("notify update should survive")
.get(WEBHOOK_ENDPOINT),
"https://notify.example.test/hook"
);
assert_eq!(
persisted
.get_value(IDENTITY_OPENID_SUB_SYS, "concurrent-oidc")
.expect("OIDC update should survive")
.get(OIDC_CLIENT_ID),
"console"
);
assert_eq!(
*transaction_order.lock().expect("transaction order lock"),
vec!["notify-read", "notify-save", "oidc-read", "oidc-save"]
);
}
#[tokio::test]
#[serial_test::serial(storage_class_env)]
async fn peer_dynamic_reload_rejects_later_pool_without_publishing() {
@@ -811,6 +1063,67 @@ mod tests {
.await;
}
#[tokio::test]
#[serial_test::serial]
async fn peer_module_switch_reload_uses_its_explicit_context_store() {
temp_env::async_with_vars(
[
(rustfs_config::ENV_NOTIFY_ENABLE, None::<&str>),
(rustfs_config::ENV_AUDIT_ENABLE, None::<&str>),
],
async {
let selected = runtime_config_reload_fixture().await;
let fallback = runtime_config_reload_fixture().await;
save_persisted_module_switches_to(
selected.context.object_store(),
PersistedModuleSwitches {
notify_enabled: true,
audit_enabled: true,
},
|| (),
)
.await
.expect("persist selected module switches");
refresh_persisted_module_switches_from(fallback.context.object_store())
.await
.expect("refresh absent fallback module switches");
assert!(
!current_module_switch_snapshot().notify_enabled,
"fallback store should resolve the default"
);
reload_dynamic_config_runtime_state_for_context(Some(&selected.context), MODULE_SWITCHES_SIGNAL_SUBSYSTEM)
.await
.expect("peer module switch reload should converge");
let selected_snapshot = current_module_switch_snapshot();
assert!(selected_snapshot.notify_enabled);
assert!(selected_snapshot.persisted_audit_enabled);
assert_eq!(selected_snapshot.notify_source, ModuleSwitchSource::Console);
assert!(matches!(
rustfs_notify::notification_system()
.expect("notification singleton should exist")
.runtime_lifecycle_state(),
rustfs_notify::NotificationRuntimeState::TargetsEnabled { .. }
));
assert!(is_event_notifier_reconciled());
reload_dynamic_config_runtime_state_for_context(Some(&fallback.context), MODULE_SWITCHES_SIGNAL_SUBSYSTEM)
.await
.expect("restore default module switch state");
assert!(!current_module_switch_snapshot().notify_enabled);
assert_eq!(
rustfs_notify::notification_system()
.expect("notification singleton should remain stable")
.runtime_lifecycle_state(),
rustfs_notify::NotificationRuntimeState::LiveOnly
);
},
)
.await;
}
#[tokio::test]
#[serial_test::serial(storage_class_env)]
async fn peer_full_reload_rejects_later_pool_without_publishing() {
@@ -835,12 +1148,12 @@ mod tests {
}
#[tokio::test]
async fn full_reload_publishes_snapshots_before_best_effort_worker_failure() {
async fn full_reload_publishes_snapshots_before_reporting_worker_failure() {
let events = Arc::new(Mutex::new(Vec::new()));
let publish_events = events.clone();
let worker_events = events.clone();
reload_runtime_config_snapshot_with(
let err = reload_runtime_config_snapshot_with(
async { Ok(ServerConfig::new()) },
|config| async { Ok((config, PreparedRuntimeConfig::default())) },
move |_config, _prepared| {
@@ -855,8 +1168,9 @@ mod tests {
},
)
.await
.expect("worker failure must not roll back validated storage/server snapshots");
.expect_err("worker failure must be reported after publishing validated snapshots");
assert_eq!(err.message(), Some("injected worker reload failure"));
assert_eq!(
*events.lock().expect("reload result lock"),
["publish", "worker-1-applied", "worker-2-failed"]
@@ -871,7 +1185,9 @@ mod tests {
assert!(is_dynamic_config_subsystem(HEAL_SUB_SYS));
assert!(is_dynamic_config_subsystem(STORAGE_CLASS_SUB_SYS));
assert!(!is_dynamic_config_subsystem("identity_openid"));
assert!(!is_dynamic_config_subsystem("notify_webhook"));
for sub_system in NOTIFY_SUB_SYSTEMS {
assert!(is_dynamic_config_subsystem(sub_system));
}
}
#[test]
@@ -885,6 +1201,7 @@ mod tests {
STORAGE_CLASS_SUB_SYS,
AUDIT_WEBHOOK_SUB_SYS,
AUDIT_MQTT_SUB_SYS,
NOTIFY_WEBHOOK_SUB_SYS,
SCANNER_SUB_SYS,
HEAL_SUB_SYS,
] {
@@ -1014,17 +1331,52 @@ mod tests {
}
#[test]
#[serial_test::serial(notify_config_env)]
fn validate_notify_subsystem_config_rejects_invalid_webhook_endpoint() {
crate::admin::storage_api::config::init_admin_config_defaults();
let mut config = ServerConfig::new();
let targets = config.0.get_mut(NOTIFY_WEBHOOK_SUB_SYS).expect("notify webhook defaults");
let kvs = targets.get_mut(DEFAULT_DELIMITER).expect("default target");
let kvs = targets.entry("primary".to_string()).or_default();
kvs.insert(ENABLE_KEY.to_string(), EnableState::On.to_string());
kvs.insert(WEBHOOK_ENDPOINT.to_string(), "not-a-url".to_string());
kvs.insert(WEBHOOK_QUEUE_DIR.to_string(), "/tmp/rustfs-notify".to_string());
let err = validate_notify_subsystem_config(&config, NOTIFY_WEBHOOK_SUB_SYS).expect_err("invalid endpoint should fail");
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
temp_env::with_vars_unset(
[
format!("{ENV_NOTIFY_WEBHOOK_ENABLE}_PRIMARY"),
format!("{ENV_NOTIFY_WEBHOOK_ENDPOINT}_PRIMARY"),
],
|| {
let err =
validate_notify_subsystem_config(&config, NOTIFY_WEBHOOK_SUB_SYS).expect_err("invalid endpoint should fail");
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
},
);
}
#[test]
#[serial_test::serial(notify_config_env)]
fn validate_notify_subsystem_config_uses_enabled_env_overlay() {
crate::admin::storage_api::config::init_admin_config_defaults();
let mut config = ServerConfig::new();
let targets = config.0.get_mut(NOTIFY_WEBHOOK_SUB_SYS).expect("notify webhook defaults");
let kvs = targets.entry("primary".to_string()).or_default();
kvs.insert(ENABLE_KEY.to_string(), EnableState::Off.to_string());
kvs.insert(WEBHOOK_ENDPOINT.to_string(), "https://example.com/hook".to_string());
kvs.insert(WEBHOOK_QUEUE_DIR.to_string(), "/tmp/rustfs-notify".to_string());
temp_env::with_vars(
[
(format!("{ENV_NOTIFY_WEBHOOK_ENABLE}_PRIMARY"), Some("on")),
(format!("{ENV_NOTIFY_WEBHOOK_ENDPOINT}_PRIMARY"), Some("not-a-url")),
],
|| {
let err = validate_notify_subsystem_config(&config, NOTIFY_WEBHOOK_SUB_SYS)
.expect_err("invalid environment endpoint should fail");
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
assert!(err.message().is_some_and(|message| message.contains("target 'primary'")));
},
);
}
#[test]
rustfs/src/admin/storage_api.rs
+25 -1
View File
@@ -412,6 +412,10 @@ pub(crate) async fn read_admin_config_without_migrate(api: Arc<ECStore>) -> Resu
ecstore_config::com::read_config_without_migrate(api).await
}
pub(crate) async fn read_admin_config_without_migrate_no_lock(api: Arc<ECStore>) -> Result<rustfs_config::server_config::Config> {
ecstore_config::com::read_config_without_migrate_no_lock(api).await
}
pub(crate) async fn save_admin_config(api: Arc<ECStore>, file: &str, data: Vec<u8>) -> Result<()> {
ecstore_config::com::save_config(api, file, data).await
}
@@ -420,10 +424,27 @@ pub(crate) async fn delete_admin_config(api: Arc<ECStore>, file: &str) -> Result
ecstore_config::com::delete_config(api, file).await
}
#[cfg(test)]
pub(crate) async fn save_admin_server_config(api: Arc<ECStore>, cfg: &rustfs_config::server_config::Config) -> Result<()> {
ecstore_config::com::save_server_config(api, cfg).await
}
pub(crate) async fn save_admin_server_config_no_lock(
api: Arc<ECStore>,
cfg: &rustfs_config::server_config::Config,
) -> Result<()> {
ecstore_config::com::save_server_config_no_lock(api, cfg).await
}
pub(crate) async fn with_admin_server_config_write_lock<F, Fut, T>(api: Arc<ECStore>, operation: F) -> Result<T>
where
F: FnOnce() -> Fut + Send + 'static,
Fut: std::future::Future<Output = T> + Send + 'static,
T: Send + 'static,
{
ecstore_config::com::with_server_config_write_lock(api, operation).await
}
pub(crate) fn init_admin_config_defaults() {
ecstore_config::init();
}
@@ -521,10 +542,13 @@ pub(crate) mod cluster {
}
pub(crate) mod config {
#[cfg(test)]
pub(crate) use super::save_admin_server_config;
pub(crate) use super::storageclass;
pub(crate) use super::{
RUSTFS_META_BUCKET, STORAGE_CLASS_SUB_SYS, delete_admin_config, init_admin_config_defaults, read_admin_config,
read_admin_config_without_migrate, save_admin_config, save_admin_server_config,
read_admin_config_without_migrate, read_admin_config_without_migrate_no_lock, save_admin_config,
save_admin_server_config_no_lock, with_admin_server_config_write_lock,
};
}
rustfs/src/embedded.rs
+41 -8
View File
@@ -52,7 +52,10 @@ use crate::server::ShutdownHandle;
use crate::startup_embedded::{EmbeddedStartedServer, EmbeddedStartupArgs, EmbeddedStartupError, run_embedded_startup};
use crate::startup_lifecycle::embedded_endpoint_address;
use crate::startup_server::find_embedded_available_port;
use crate::startup_shutdown::{run_embedded_server_drop_cleanup, run_embedded_server_shutdown};
use crate::startup_shutdown::{
EmbeddedRuntimeOwner, register_embedded_runtime_owner, run_embedded_server_drop_cleanup, run_embedded_server_shutdown,
run_embedded_shutdown_cleanup,
};
use std::net::SocketAddr;
use std::path::PathBuf;
use tokio_util::sync::CancellationToken;
@@ -215,14 +218,21 @@ impl RustFSServerBuilder {
}
async fn do_build(self) -> Result<RustFSServer, ServerError> {
let started = run_embedded_startup(self.startup_args).await?;
Ok(started.into())
let mut runtime_owner = register_embedded_runtime_owner().await;
match run_embedded_startup(self.startup_args).await {
Ok(started) => Ok(RustFSServer::from_started(started, runtime_owner)),
Err(err) => {
if let Some(cleanup) = runtime_owner.release() {
run_embedded_shutdown_cleanup(cleanup).await;
}
Err(err.into())
}
}
}
}
impl From<EmbeddedStartedServer> for RustFSServer {
fn from(started: EmbeddedStartedServer) -> Self {
impl RustFSServer {
fn from_started(started: EmbeddedStartedServer, runtime_owner: EmbeddedRuntimeOwner) -> Self {
Self {
address: started.bound_addr,
access_key: started.access_key,
@@ -231,6 +241,7 @@ impl From<EmbeddedStartedServer> for RustFSServer {
shutdown_handle: Some(started.shutdown_handle),
cancel_token: started.cancel_token,
temp_dir: started.temp_dir,
runtime_owner: Some(runtime_owner),
}
}
}
@@ -250,6 +261,7 @@ pub struct RustFSServer {
shutdown_handle: Option<ShutdownHandle>,
cancel_token: CancellationToken,
temp_dir: Option<PathBuf>,
runtime_owner: Option<EmbeddedRuntimeOwner>,
}
impl RustFSServer {
@@ -290,13 +302,34 @@ impl RustFSServer {
}
async fn do_shutdown(&mut self) {
run_embedded_server_shutdown(&self.cancel_token, &mut self.shutdown_handle, self.temp_dir.as_deref()).await;
let Some(runtime_owner) = self.runtime_owner.take() else {
return;
};
let runtime = runtime_owner.cleanup_runtime_handle();
run_embedded_server_shutdown(
&runtime,
&self.cancel_token,
&mut self.shutdown_handle,
&mut self.temp_dir,
Some(runtime_owner),
)
.await;
}
}
impl Drop for RustFSServer {
fn drop(&mut self) {
run_embedded_server_drop_cleanup(&self.cancel_token, &mut self.shutdown_handle, self.temp_dir.as_deref());
let Some(runtime_owner) = self.runtime_owner.take() else {
return;
};
let runtime = runtime_owner.cleanup_runtime_handle();
run_embedded_server_drop_cleanup(
&runtime,
&self.cancel_token,
&mut self.shutdown_handle,
&mut self.temp_dir,
Some(runtime_owner),
);
}
}
rustfs/src/server/audit.rs
+33 -3
View File
@@ -12,7 +12,11 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use super::{module_switch::resolve_audit_module_state, refresh_persisted_module_switches_from_store, runtime_sources};
use super::{
module_switch::resolve_audit_module_state, refresh_persisted_module_switches_from,
refresh_persisted_module_switches_from_store, runtime_sources,
};
use crate::runtime_sources::AppContext;
use rustfs_audit::{AuditError, AuditResult, audit_system, init_audit_system, system::AuditSystemState};
use std::sync::atomic::{AtomicBool, Ordering};
use tracing::{info, warn};
@@ -23,6 +27,13 @@ fn server_config_from_context() -> Option<rustfs_config::server_config::Config>
runtime_sources::current_server_config()
}
fn server_config_for_context(context: Option<&AppContext>) -> Option<rustfs_config::server_config::Config> {
match context {
Some(context) => context.server_config().get(),
None => server_config_from_context(),
}
}
pub fn refresh_audit_module_enabled() -> bool {
let enabled = resolve_audit_module_state().enabled;
AUDIT_MODULE_ENABLED.store(enabled, Ordering::Relaxed);
@@ -51,7 +62,15 @@ fn has_any_audit_targets(config: &rustfs_config::server_config::Config) -> bool
/// If not configured, it skips the initialization.
/// It also handles cases where the audit system is already running or if the global configuration is not loaded.
pub async fn start_audit_system() -> AuditResult<()> {
if let Err(err) = refresh_persisted_module_switches_from_store().await {
start_audit_system_for_context(None).await
}
pub(crate) async fn start_audit_system_for_context(context: Option<&AppContext>) -> AuditResult<()> {
let refresh_result = match context {
Some(context) => refresh_persisted_module_switches_from(context.object_store()).await,
None => refresh_persisted_module_switches_from_store().await,
};
if let Err(err) = refresh_result {
warn!("Failed to refresh persisted audit module switch from store: {}", err);
}
@@ -70,7 +89,7 @@ pub async fn start_audit_system() -> AuditResult<()> {
);
// 1. Get the global configuration loaded by ecstore
let server_config = match server_config_from_context() {
let server_config = match server_config_for_context(context) {
Some(config) => config,
None => {
warn!(
@@ -165,3 +184,14 @@ pub async fn stop_audit_system() -> AuditResult<()> {
Ok(())
}
}
pub(crate) async fn apply_audit_module_switch_for_context(context: Option<&AppContext>) -> AuditResult<()> {
if refresh_audit_module_enabled() {
match start_audit_system_for_context(context).await {
Ok(()) | Err(AuditError::AlreadyInitialized) => Ok(()),
Err(err) => Err(err),
}
} else {
stop_audit_system().await
}
}
rustfs/src/server/event.rs
+308 -83
View File
@@ -12,20 +12,48 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use super::{module_switch::resolve_notify_module_state, refresh_persisted_module_switches_from_store, runtime_sources};
use crate::storage_api::server::event::{EventArgs as EcstoreEventArgs, StorageObjectInfo, register_event_dispatch_hook};
use super::{
module_switch::{resolve_notify_module_state, validate_notify_module_env, with_refreshed_notify_module_state_from},
refresh_persisted_module_switches_from_store, runtime_sources,
};
use crate::storage_api::server::event::{
EventArgs as EcstoreEventArgs, StorageObjectInfo, read_existing_server_config_no_lock, register_event_dispatch_hook,
with_server_config_read_lock,
};
use chrono::{DateTime, Utc};
use rustfs_notify::{EventArgs as NotifyEventArgs, NotifyObjectInfo};
use rustfs_notify::{
EventArgs as NotifyEventArgs, NotificationError, NotificationRuntimeState, NotificationSystem, NotifyObjectInfo,
};
use rustfs_s3_types::EventName;
use std::future::Future;
use std::net::SocketAddr;
use std::sync::OnceLock;
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::Duration;
use tokio::spawn;
use tracing::{error, info, instrument, warn};
use tokio::task::JoinHandle;
use tokio::time::{Instant, MissedTickBehavior};
use tokio_util::sync::CancellationToken;
use tracing::{info, instrument, warn};
static NOTIFY_MODULE_ENABLED: AtomicBool = AtomicBool::new(rustfs_config::DEFAULT_NOTIFY_ENABLE);
static NOTIFY_RUNTIME_RECONCILED: AtomicBool = AtomicBool::new(false);
static ECSTORE_EVENT_DISPATCH_HOOK: OnceLock<()> = OnceLock::new();
fn server_config_from_context() -> Option<rustfs_config::server_config::Config> {
runtime_sources::current_server_config()
const EVENT_NOTIFIER_RECONCILE_INTERVAL: Duration = Duration::from_secs(5);
const EVENT_NOTIFIER_RECONCILE_ATTEMPT_TIMEOUT: Duration = Duration::from_secs(120);
const EVENT_NOTIFY_RUNTIME_RECONCILE: &str = "notify_runtime_reconcile";
pub(crate) fn is_event_notifier_reconciled() -> bool {
NOTIFY_RUNTIME_RECONCILED.load(Ordering::Acquire)
}
pub(crate) fn mark_event_notifier_reconciled() {
NOTIFY_RUNTIME_RECONCILED.store(true, Ordering::Release);
}
pub(crate) fn mark_event_notifier_unreconciled() {
NOTIFY_RUNTIME_RECONCILED.store(false, Ordering::Release);
}
pub fn refresh_notify_module_enabled() -> bool {
@@ -112,79 +140,201 @@ fn parse_host_and_port(host: String) -> (String, u16) {
}
fn install_ecstore_event_dispatch_hook() {
let installed = register_event_dispatch_hook(|args| {
let Some(notify_args) = convert_ecstore_event_args(args) else {
return;
};
spawn(async move {
runtime_sources::current_notify_interface().notify(notify_args).await;
ECSTORE_EVENT_DISPATCH_HOOK.get_or_init(|| {
let installed = register_event_dispatch_hook(|args| {
let Some(notify_args) = convert_ecstore_event_args(args) else {
return;
};
spawn(async move {
runtime_sources::current_notify_interface().notify(notify_args).await;
});
});
if !installed {
warn!("ECStore event dispatch hook was already registered");
}
});
}
if !installed {
warn!("ECStore event dispatch hook was already registered");
fn ensure_live_events_initialized() -> std::sync::Arc<NotificationSystem> {
let system = rustfs_notify::ensure_live_events();
install_ecstore_event_dispatch_hook();
system
}
fn ensure_event_notifier_converged(system: &NotificationSystem) -> Result<(), NotificationError> {
if system.runtime_lifecycle_is_converged() {
Ok(())
} else {
Err(NotificationError::Initialization(
"Latest notification lifecycle generation has not converged".to_string(),
))
}
}
fn ensure_live_events_initialized() -> bool {
if rustfs_notify::notification_system().is_some() {
return true;
}
pub(crate) async fn reconcile_event_notifier_from_store(
store: std::sync::Arc<rustfs_notify::NotifyStore>,
) -> Result<(), NotificationError> {
let result = async {
validate_notify_module_env().map_err(NotificationError::Initialization)?;
let system = ensure_live_events_initialized();
let transition_system = system.clone();
let transition_store = store.clone();
let transition = with_refreshed_notify_module_state_from(store, move |resolution| async move {
NOTIFY_MODULE_ENABLED.store(resolution.enabled, Ordering::Relaxed);
let read_store = transition_store.clone();
let config_system = transition_system.clone();
with_server_config_read_lock(transition_store, move || async move {
let config = read_existing_server_config_no_lock(read_store)
.await
.map_err(|err| NotificationError::ReadConfig(err.to_string()))?;
let mode_matches = match config_system.runtime_lifecycle_state() {
NotificationRuntimeState::LiveOnly => !resolution.enabled,
NotificationRuntimeState::TargetsEnabled { .. } => resolution.enabled,
NotificationRuntimeState::Terminated => false,
};
if config_system.config_snapshot().await == config
&& mode_matches
&& config_system.runtime_lifecycle_is_converged()
{
Ok::<_, NotificationError>(None)
} else {
Ok(Some(config_system.publish_targets_enabled(resolution.enabled, Some(config))))
}
})
.await
.map_err(|err| NotificationError::StorageNotAvailable(err.to_string()))?
})
.await
.map_err(|err| NotificationError::Initialization(format!("failed to refresh notify module switch: {err}")))??;
match rustfs_notify::initialize_live_events() {
Ok(()) => {
install_ecstore_event_dispatch_hook();
true
if let Some(transition) = transition {
transition.wait().await?;
}
Err(e) => {
error!("Failed to initialize live event stream support: {}", e);
false
ensure_event_notifier_converged(&system)
}
.await;
if result.is_ok() {
mark_event_notifier_reconciled();
} else {
mark_event_notifier_unreconciled();
}
result
}
pub(crate) fn start_persisted_event_notifier_reconciler(
store: std::sync::Arc<rustfs_notify::NotifyStore>,
cancellation: CancellationToken,
) -> JoinHandle<()> {
spawn(run_persisted_event_notifier_reconciler(
cancellation,
EVENT_NOTIFIER_RECONCILE_INTERVAL,
move || {
let store = store.clone();
async move { reconcile_event_notifier_from_store(store).await }
},
))
}
async fn run_persisted_event_notifier_reconciler<Reconcile, ReconcileFuture>(
cancellation: CancellationToken,
reconcile_interval: Duration,
mut reconcile: Reconcile,
) where
Reconcile: FnMut() -> ReconcileFuture,
ReconcileFuture: Future<Output = Result<(), NotificationError>>,
{
let first_tick = Instant::now() + reconcile_interval;
let mut ticker = tokio::time::interval_at(first_tick, reconcile_interval);
ticker.set_missed_tick_behavior(MissedTickBehavior::Skip);
let mut failure_reported = false;
loop {
tokio::select! {
biased;
_ = cancellation.cancelled() => break,
_ = ticker.tick() => {}
}
let result = tokio::select! {
biased;
_ = cancellation.cancelled() => break,
result = tokio::time::timeout(EVENT_NOTIFIER_RECONCILE_ATTEMPT_TIMEOUT, reconcile()) => result,
};
match result {
Ok(Ok(())) => {
if failure_reported {
info!(
event = EVENT_NOTIFY_RUNTIME_RECONCILE,
component = "notify",
subsystem = "lifecycle",
state = "recovered",
"Persisted notification runtime reconciliation recovered"
);
}
failure_reported = false;
}
Ok(Err(_)) | Err(_) => {
if !failure_reported {
warn!(
event = EVENT_NOTIFY_RUNTIME_RECONCILE,
component = "notify",
subsystem = "lifecycle",
state = "degraded",
reason = "reconcile_failed_or_timed_out",
"Persisted notification runtime reconciliation failed"
);
}
failure_reported = true;
}
}
}
}
/// Shuts down the event notifier system gracefully
pub async fn shutdown_event_notifier() {
/// Irreversibly shuts down the event notifier target runtime for process exit.
pub async fn shutdown_event_notifier() -> Result<(), NotificationError> {
info!("Shutting down event notifier system...");
if !rustfs_notify::is_notification_system_initialized() {
let Some(system) = rustfs_notify::notification_system() else {
info!("Event notifier system is not initialized, nothing to shut down.");
return;
}
let system = match rustfs_notify::notification_system() {
Some(sys) => sys,
None => {
info!("Event notifier system is not initialized.");
return;
}
return Ok(());
};
// Call the shutdown function from the rustfs_notify module
system.shutdown().await;
system.shutdown_checked().await?;
info!("Event notifier system shut down successfully.");
Ok(())
}
#[instrument]
pub async fn init_event_notifier() {
if let Err(err) = refresh_persisted_module_switches_from_store().await {
warn!("Failed to refresh persisted notify module switch from store: {}", err);
}
pub async fn init_event_notifier() -> Result<(), NotificationError> {
mark_event_notifier_unreconciled();
validate_notify_module_env().map_err(NotificationError::Initialization)?;
let system = ensure_live_events_initialized();
refresh_persisted_module_switches_from_store()
.await
.map_err(|err| NotificationError::Initialization(format!("failed to refresh notify module switch: {err}")))?;
let enabled = refresh_notify_module_enabled();
if !enabled {
info!(
target: "rustfs::main::init_event_notifier",
"Notify module is disabled, initializing live event stream support only. Set {}=true to enable notification targets.",
rustfs_config::ENV_NOTIFY_ENABLE
);
if ensure_live_events_initialized() {
info!(
target: "rustfs::main::init_event_notifier",
"Live event stream support initialized successfully."
);
if system.runtime_lifecycle_state() != NotificationRuntimeState::LiveOnly {
system.set_targets_enabled(false, None).await?;
}
return;
system.reload_persisted_config().await?;
info!(
target: "rustfs::main::init_event_notifier",
"Live event stream support initialized successfully."
);
ensure_event_notifier_converged(&system)?;
mark_event_notifier_reconciled();
return Ok(());
}
info!(
@@ -192,53 +342,44 @@ pub async fn init_event_notifier() {
"Initializing event notifier..."
);
// 1. Get the global configuration loaded by ecstore
let server_config = match server_config_from_context() {
Some(config) => config,
None => {
warn!("Event notifier initialization failed: Global server config not loaded.");
return;
}
};
info!(
target: "rustfs::main::init_event_notifier",
"Event notifier configuration found, proceeding with initialization."
);
if let Some(system) = rustfs_notify::notification_system() {
// Reuse the existing global system on re-enable so bucket rules, metrics,
// and stream lifecycle stay aligned with the current process singleton.
if let Err(e) = system.reload_config(server_config).await {
error!("Failed to reload event notifier system: {}", e);
} else {
info!(
target: "rustfs::main::init_event_notifier",
"Event notifier system reloaded successfully."
);
}
} else {
match rustfs_notify::initialize(server_config).await {
Ok(()) => {
install_ecstore_event_dispatch_hook();
info!(
target: "rustfs::main::init_event_notifier",
"Event notifier system initialized successfully."
);
}
Err(e) => error!("Failed to initialize event notifier system: {}", e),
}
system.reload_persisted_config().await?;
let runtime_state = system.runtime_lifecycle_state();
if !matches!(runtime_state, NotificationRuntimeState::TargetsEnabled { .. }) {
system.set_targets_enabled(true, None).await?;
}
info!(
target: "rustfs::main::init_event_notifier",
"Event notifier system initialized successfully."
);
ensure_event_notifier_converged(&system)?;
mark_event_notifier_reconciled();
Ok(())
}
#[cfg(test)]
mod tests {
use super::{convert_ecstore_object_info, parse_host_and_port};
use super::{convert_ecstore_object_info, parse_host_and_port, run_persisted_event_notifier_reconciler};
use crate::storage_api::server::event::StorageObjectInfo;
use crate::storage_api::server::event::contract::lifecycle::TransitionedObject;
use chrono::{DateTime, Utc};
use std::{collections::HashMap, sync::Arc};
use rustfs_notify::NotificationError;
use std::{
collections::HashMap,
future::pending,
sync::{
Arc,
atomic::{AtomicBool, AtomicUsize, Ordering},
},
time::Duration as StdDuration,
};
use time::{Duration, OffsetDateTime};
use tokio::sync::Notify;
use tokio_util::sync::CancellationToken;
#[test]
fn parse_host_and_port_with_ipv4_and_port() {
@@ -303,4 +444,88 @@ mod tests {
assert_eq!(converted.storage_class.as_deref(), Some("GLACIER"));
assert_eq!(converted.transitioned_tier.as_deref(), Some("DEEP_ARCHIVE"));
}
#[tokio::test(start_paused = true)]
async fn persisted_reconciler_converges_after_one_injected_tick() {
let persisted_generation = Arc::new(AtomicUsize::new(1));
let runtime_generation = Arc::new(AtomicUsize::new(1));
let runtime_converged = Arc::new(AtomicBool::new(true));
let reconcile_calls = Arc::new(AtomicUsize::new(0));
let reconciled = Arc::new(Notify::new());
let cancellation = CancellationToken::new();
let task = tokio::spawn(run_persisted_event_notifier_reconciler(
cancellation.clone(),
StdDuration::from_secs(5),
{
let persisted_generation = persisted_generation.clone();
let runtime_generation = runtime_generation.clone();
let runtime_converged = runtime_converged.clone();
let reconcile_calls = reconcile_calls.clone();
let reconciled = reconciled.clone();
move || {
let persisted_generation = persisted_generation.clone();
let runtime_generation = runtime_generation.clone();
let runtime_converged = runtime_converged.clone();
let reconcile_calls = reconcile_calls.clone();
let reconciled = reconciled.clone();
async move {
runtime_generation.store(persisted_generation.load(Ordering::SeqCst), Ordering::SeqCst);
runtime_converged.store(true, Ordering::SeqCst);
reconcile_calls.fetch_add(1, Ordering::SeqCst);
reconciled.notify_one();
Ok(())
}
}
},
));
tokio::task::yield_now().await;
persisted_generation.store(2, Ordering::SeqCst);
runtime_converged.store(false, Ordering::SeqCst);
assert_eq!(
reconcile_calls.load(Ordering::SeqCst),
0,
"the first tick must wait for the configured interval"
);
tokio::time::advance(StdDuration::from_secs(5)).await;
reconciled.notified().await;
assert_eq!(reconcile_calls.load(Ordering::SeqCst), 1);
assert_eq!(runtime_generation.load(Ordering::SeqCst), 2);
assert!(runtime_converged.load(Ordering::SeqCst));
cancellation.cancel();
task.await.expect("persisted reconciler should stop after cancellation");
}
#[tokio::test(start_paused = true)]
async fn persisted_reconciler_cancellation_interrupts_an_inflight_attempt() {
let entered = Arc::new(Notify::new());
let cancellation = CancellationToken::new();
let task = tokio::spawn(run_persisted_event_notifier_reconciler(
cancellation.clone(),
StdDuration::from_secs(5),
{
let entered = entered.clone();
move || {
let entered = entered.clone();
async move {
entered.notify_one();
pending::<Result<(), NotificationError>>().await
}
}
},
));
tokio::task::yield_now().await;
tokio::time::advance(StdDuration::from_secs(5)).await;
entered.notified().await;
cancellation.cancel();
tokio::time::timeout(StdDuration::from_secs(1), task)
.await
.expect("cancellation should stop an in-flight reconciliation attempt")
.expect("persisted reconciler should not panic");
}
}
rustfs/src/server/mod.rs
+8 -2
View File
@@ -32,6 +32,7 @@ pub mod tls_material;
use tracing::warn;
// Items used by main.rs (binary crate) and/or embedded.rs — must be fully pub.
pub(crate) use audit::apply_audit_module_switch_for_context;
pub use audit::{is_audit_module_enabled, refresh_audit_module_enabled, start_audit_system, stop_audit_system};
pub use event::{init_event_notifier, is_notify_module_enabled, refresh_notify_module_enabled, shutdown_event_notifier};
pub use http::start_http_server;
@@ -45,6 +46,10 @@ pub use service_state::wait_for_shutdown;
// Items only used within the library crate (admin handlers, server/http.rs, etc.).
pub(crate) use event::convert_ecstore_object_info;
pub(crate) use event::{
is_event_notifier_reconciled, mark_event_notifier_reconciled, mark_event_notifier_unreconciled,
reconcile_event_notifier_from_store, start_persisted_event_notifier_reconciler,
};
#[cfg(test)]
pub(crate) use health::{
HealthPayloadContext, HealthReadinessSource, build_component_details, build_health_payload, health_check_state,
@@ -55,8 +60,9 @@ pub(crate) use http::HeaderMapCarrier;
pub(crate) use http::active_http_requests;
pub(crate) use layer::RequestContextLayer;
pub(crate) use module_switch::{
ModuleSwitchSnapshot, ModuleSwitchSource, PersistedModuleSwitches, current_module_switch_snapshot,
refresh_persisted_module_switches_from_store, save_persisted_module_switches_to_store, validate_module_switch_update,
MODULE_SWITCHES_SIGNAL_SUBSYSTEM, ModuleSwitchSnapshot, ModuleSwitchSource, PersistedModuleSwitches,
current_module_switch_snapshot, refresh_persisted_module_switches_from, refresh_persisted_module_switches_from_store,
save_persisted_module_switches_to, validate_module_switch_update,
};
pub(crate) use prefix::{
ADMIN_PREFIX, CONSOLE_PREFIX, FAVICON_PATH, HEALTH_COMPAT_LIVE_PATH, HEALTH_PREFIX, HEALTH_READY_PATH, LICENSE,
rustfs/src/server/module_switch.rs
+366 -20
View File
@@ -13,17 +13,35 @@
// limitations under the License.
use super::runtime_sources;
use crate::storage_api::server::module_switch::{Error as StorageError, read_config, save_config};
use crate::storage_api::server::module_switch::{
Error as StorageError, read_config, read_config_no_lock, save_config_no_lock, with_config_object_read_lock,
with_config_object_write_lock,
};
use crate::storage_api::server::runtime_sources::ECStore;
use serde::{Deserialize, Serialize};
use std::sync::Arc;
use std::sync::LazyLock;
use std::sync::atomic::{AtomicBool, Ordering};
use tokio::sync::Mutex;
const MODULE_SWITCH_CONFIG_PATH: &str = "config/module_switches.json";
pub(crate) const MODULE_SWITCHES_SIGNAL_SUBSYSTEM: &str = "module_switches";
// Keep a cheap in-process snapshot so hot-path checks do not need to read
// cluster metadata after startup or console-triggered refresh.
static PERSISTED_NOTIFY_MODULE_ENABLED: AtomicBool = AtomicBool::new(rustfs_config::DEFAULT_NOTIFY_ENABLE);
static PERSISTED_AUDIT_MODULE_ENABLED: AtomicBool = AtomicBool::new(rustfs_config::DEFAULT_AUDIT_ENABLE);
static PERSISTED_MODULE_SWITCH_CONFIGURED: AtomicBool = AtomicBool::new(false);
static MODULE_SWITCH_RMW_LOCK: LazyLock<Mutex<()>> = LazyLock::new(|| Mutex::new(()));
async fn serialize_module_switch_rmw<F, Fut, T>(operation: F) -> T
where
F: FnOnce() -> Fut,
Fut: std::future::Future<Output = T>,
{
let _rmw_guard = MODULE_SWITCH_RMW_LOCK.lock().await;
operation().await
}
#[derive(Clone, Copy, Debug, Default, Deserialize, PartialEq, Eq, Serialize)]
pub(crate) struct PersistedModuleSwitches {
@@ -80,6 +98,14 @@ fn env_override_value(key: &str) -> Option<bool> {
rustfs_utils::get_env_opt_bool(key)
}
pub(crate) fn validate_notify_module_env() -> Result<(), String> {
let key = rustfs_config::ENV_NOTIFY_ENABLE;
if env_override_exists(key) && env_override_value(key).is_none() {
return Err(format!("{key} is not a valid boolean"));
}
Ok(())
}
fn effective_module_switch_state(env_key: &str, persisted_enabled: bool, default_enabled: bool) -> ModuleSwitchResolution {
// Explicit env remains the highest-priority source so process-level bootstrap
// cannot be silently overridden by a later console write.
@@ -162,42 +188,157 @@ pub(crate) async fn refresh_persisted_module_switches_from_store() -> Result<Per
let Some(store) = runtime_sources::current_object_store_handle() else {
return Err("storage layer not initialized".to_string());
};
refresh_persisted_module_switches_from(store).await
}
let (config, configured) = match read_config(store, MODULE_SWITCH_CONFIG_PATH).await {
Ok(data) => (
pub(crate) async fn refresh_persisted_module_switches_from(store: Arc<ECStore>) -> Result<PersistedModuleSwitches, String> {
refresh_persisted_module_switches_with(|| async move {
match read_config(store, MODULE_SWITCH_CONFIG_PATH).await {
Ok(data) => Ok(Some(data)),
Err(StorageError::ConfigNotFound) => Ok(None),
Err(err) => Err(format!("failed to load module switch config: {err}")),
}
})
.await
}
pub(crate) async fn with_refreshed_notify_module_state_from<T, Publish, PublishFuture>(
store: Arc<ECStore>,
publish: Publish,
) -> Result<T, String>
where
Publish: FnOnce(ModuleSwitchResolution) -> PublishFuture + Send + 'static,
PublishFuture: std::future::Future<Output = T> + Send + 'static,
T: Send + 'static,
{
// Lock order: local module RMW -> distributed module-object read -> the
// callback's server-config read lock. Server-config writers never acquire
// the module-object lock, and module writers release any server-config
// read before taking the module-object write lock.
serialize_module_switch_rmw(|| async move {
let read_store = store.clone();
with_config_object_read_lock(store, MODULE_SWITCH_CONFIG_PATH.to_string(), move || async move {
let persisted = match read_config_no_lock(read_store, MODULE_SWITCH_CONFIG_PATH).await {
Ok(data) => Some(data),
Err(StorageError::ConfigNotFound) => None,
Err(err) => return Err(format!("failed to load module switch config: {err}")),
};
let config = decode_persisted_module_switches(persisted)?;
set_persisted_module_switches(config.0, config.1);
Ok(publish(resolve_notify_module_state()).await)
})
.await
.map_err(|err| format!("failed to lock module switch refresh: {err}"))?
})
.await
}
fn decode_persisted_module_switches(data: Option<Vec<u8>>) -> Result<(PersistedModuleSwitches, bool), String> {
match data {
Some(data) => Ok((
serde_json::from_slice::<PersistedModuleSwitches>(&data)
.map_err(|e| format!("failed to deserialize module switch config: {e}"))?,
true,
),
Err(StorageError::ConfigNotFound) => (PersistedModuleSwitches::default(), false),
Err(err) => return Err(format!("failed to load module switch config: {err}")),
};
// Track whether the persisted file exists so the effective state can
// distinguish "console configured false" from "never configured, use default".
set_persisted_module_switches(config, configured);
Ok(config)
)),
None => Ok((PersistedModuleSwitches::default(), false)),
}
}
pub(crate) async fn save_persisted_module_switches_to_store(config: PersistedModuleSwitches) -> Result<(), String> {
let Some(store) = runtime_sources::current_object_store_handle() else {
return Err("storage layer not initialized".to_string());
};
async fn refresh_persisted_module_switches_with<F, Fut>(read: F) -> Result<PersistedModuleSwitches, String>
where
F: FnOnce() -> Fut,
Fut: std::future::Future<Output = Result<Option<Vec<u8>>, String>>,
{
serialize_module_switch_rmw(|| async move {
let (config, configured) = decode_persisted_module_switches(read().await?)?;
let data = serde_json::to_vec(&config).map_err(|e| format!("failed to serialize module switch config: {e}"))?;
save_config(store, MODULE_SWITCH_CONFIG_PATH, data)
// Track whether the persisted file exists so the effective state can
// distinguish "console configured false" from "never configured, use default".
set_persisted_module_switches(config, configured);
Ok(config)
})
.await
}
pub(crate) async fn save_persisted_module_switches_to<T, F>(
store: Arc<ECStore>,
config: PersistedModuleSwitches,
publish: F,
) -> Result<T, String>
where
F: FnOnce() -> T + Send + 'static,
T: Send + 'static,
{
// Lock order matches refresh_persisted_module_switches_from: local RMW
// guard before the namespace lock. Taking these in the opposite order lets
// a reader hold the local guard while waiting for a writer that is itself
// waiting for the local guard.
serialize_module_switch_rmw(|| async move {
let save_store = store.clone();
with_config_object_write_lock(store, MODULE_SWITCH_CONFIG_PATH.to_string(), move || async move {
save_persisted_module_switches_inner(
config,
move |data| async move {
save_config_no_lock(save_store, MODULE_SWITCH_CONFIG_PATH, data)
.await
.map_err(|e| format!("failed to save module switch config: {e}"))
},
publish,
)
.await
})
.await
.map_err(|e| format!("failed to save module switch config: {e}"))?;
.map_err(|err| format!("failed to lock module switch update: {err}"))?
})
.await
}
#[cfg(test)]
async fn save_persisted_module_switches_with<T, F, Save, SaveFuture>(
config: PersistedModuleSwitches,
save: Save,
publish: F,
) -> Result<T, String>
where
F: FnOnce() -> T,
Save: FnOnce(Vec<u8>) -> SaveFuture,
SaveFuture: std::future::Future<Output = Result<(), String>>,
{
serialize_module_switch_rmw(|| save_persisted_module_switches_inner(config, save, publish)).await
}
async fn save_persisted_module_switches_inner<T, F, Save, SaveFuture>(
config: PersistedModuleSwitches,
save: Save,
publish: F,
) -> Result<T, String>
where
F: FnOnce() -> T,
Save: FnOnce(Vec<u8>) -> SaveFuture,
SaveFuture: std::future::Future<Output = Result<(), String>>,
{
let data = serde_json::to_vec(&config).map_err(|e| format!("failed to serialize module switch config: {e}"))?;
save(data).await?;
set_persisted_module_switches(config, true);
Ok(())
Ok(publish())
}
#[cfg(test)]
mod tests {
use super::*;
use crate::storage_api::startup::storage::{EndpointServerPools, init_local_disks_with_instance_ctx, new_instance_ctx};
use serial_test::serial;
use std::future::{Future, poll_fn};
use std::sync::{
Arc, Mutex as StdMutex,
atomic::{AtomicBool, Ordering as AtomicOrdering},
};
use std::task::Poll;
use temp_env::{with_var, with_vars};
use tempfile::TempDir;
use tokio::sync::{Notify, oneshot};
use tokio_util::sync::CancellationToken;
#[test]
#[serial]
@@ -299,6 +440,15 @@ mod tests {
});
}
#[test]
#[serial]
fn invalid_notify_env_is_rejected_before_runtime_reconcile() {
with_var(rustfs_config::ENV_NOTIFY_ENABLE, Some("invalid"), || {
let err = validate_notify_module_env().expect_err("invalid notify env must fail closed");
assert!(err.contains(rustfs_config::ENV_NOTIFY_ENABLE));
});
}
#[test]
#[serial]
fn validate_module_switch_update_allows_matching_env_override() {
@@ -330,4 +480,200 @@ mod tests {
assert!(err.contains("not a valid boolean"));
});
}
#[tokio::test]
#[serial]
async fn save_publish_and_refresh_share_the_rmw_lock() {
let previous = current_persisted_module_switches();
let previous_configured = persisted_module_switches_configured();
save_persisted_module_switches_with(
PersistedModuleSwitches {
notify_enabled: false,
audit_enabled: true,
},
|_| async { Ok(()) },
|| {
assert!(MODULE_SWITCH_RMW_LOCK.try_lock().is_err(), "publish must run while the RMW lock is held");
},
)
.await
.expect("save and publish should succeed");
let events = Arc::new(StdMutex::new(Vec::new()));
let release_save = Arc::new(Notify::new());
let (save_started_tx, save_started_rx) = oneshot::channel();
let save_events = events.clone();
let publish_events = events.clone();
let save_release = release_save.clone();
let save_task = tokio::spawn(async move {
save_persisted_module_switches_with(
PersistedModuleSwitches {
notify_enabled: true,
audit_enabled: false,
},
move |_| async move {
save_events.lock().expect("lock event log").push("save");
save_started_tx.send(()).expect("signal save start");
save_release.notified().await;
Ok(())
},
move || {
publish_events.lock().expect("lock event log").push("publish");
},
)
.await
});
save_started_rx.await.expect("save should reach its persistence step");
let refresh_events = events.clone();
let (refresh_waiting_tx, refresh_waiting_rx) = oneshot::channel();
let refresh_task = tokio::spawn(async move {
let mut refresh = Box::pin(refresh_persisted_module_switches_with(move || async move {
refresh_events.lock().expect("lock event log").push("refresh");
Ok(None)
}));
poll_fn(|cx| match refresh.as_mut().poll(cx) {
Poll::Pending => Poll::Ready(()),
Poll::Ready(_) => panic!("refresh must wait for an in-flight save"),
})
.await;
refresh_waiting_tx.send(()).expect("signal refresh wait");
refresh.await
});
refresh_waiting_rx.await.expect("refresh should be waiting on the RMW lock");
release_save.notify_one();
save_task.await.expect("join save task").expect("save should succeed");
refresh_task
.await
.expect("join refresh task")
.expect("refresh should succeed");
assert_eq!(*events.lock().expect("lock event log"), ["save", "publish", "refresh"]);
set_persisted_module_switches(previous, previous_configured);
}
#[tokio::test]
#[serial]
async fn distributed_refresh_orders_old_true_before_new_false_publish() {
temp_env::async_with_vars([(rustfs_config::ENV_NOTIFY_ENABLE, None::<&str>)], async {
let previous = current_persisted_module_switches();
let previous_configured = persisted_module_switches_configured();
let temp_dir = TempDir::new().expect("module switch ordering temp dir");
let volume = temp_dir.path().join("disk");
tokio::fs::create_dir_all(&volume)
.await
.expect("create module switch test disk");
let (endpoint_pools, _) =
EndpointServerPools::from_volumes("127.0.0.1:29131", vec![volume.to_string_lossy().into_owned()])
.await
.expect("create module switch test endpoints");
let instance_ctx = new_instance_ctx();
init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone())
.await
.expect("initialize module switch test disk");
let shutdown = CancellationToken::new();
let store = ECStore::new_with_instance_ctx(
"127.0.0.1:29131".parse().expect("module switch test address"),
endpoint_pools,
shutdown.clone(),
instance_ctx,
)
.await
.expect("create module switch test store");
save_persisted_module_switches_to(
store.clone(),
PersistedModuleSwitches {
notify_enabled: true,
audit_enabled: false,
},
|| (),
)
.await
.expect("persist initial enabled module state");
let publish_order = Arc::new(StdMutex::new(Vec::new()));
let runtime_enabled = Arc::new(AtomicBool::new(false));
let reader_entered = Arc::new(Notify::new());
let release_reader = Arc::new(Notify::new());
let reader = tokio::spawn({
let store = store.clone();
let publish_order = publish_order.clone();
let runtime_enabled = runtime_enabled.clone();
let reader_entered = reader_entered.clone();
let release_reader = release_reader.clone();
async move {
with_refreshed_notify_module_state_from(store, move |resolution| async move {
assert!(resolution.enabled, "the old persisted snapshot should be enabled");
reader_entered.notify_one();
release_reader.notified().await;
publish_order.lock().expect("module publish order lock").push("old-true");
runtime_enabled.store(true, AtomicOrdering::SeqCst);
})
.await
}
});
reader_entered.notified().await;
let writer_entered = Arc::new(AtomicBool::new(false));
let (writer_started_tx, writer_started_rx) = oneshot::channel();
let writer = tokio::spawn({
let store = store.clone();
let publish_order = publish_order.clone();
let runtime_enabled = runtime_enabled.clone();
let writer_entered = writer_entered.clone();
async move {
writer_started_tx.send(()).expect("signal remote writer start");
let save_store = store.clone();
with_config_object_write_lock(store, MODULE_SWITCH_CONFIG_PATH.to_string(), move || async move {
writer_entered.store(true, AtomicOrdering::SeqCst);
let data = serde_json::to_vec(&PersistedModuleSwitches {
notify_enabled: false,
audit_enabled: false,
})
.expect("serialize disabled module state");
save_config_no_lock(save_store, MODULE_SWITCH_CONFIG_PATH, data)
.await
.map_err(|err| err.to_string())?;
publish_order.lock().expect("module publish order lock").push("new-false");
runtime_enabled.store(false, AtomicOrdering::SeqCst);
Ok::<(), String>(())
})
.await
.expect("remote writer should acquire module object lock")
.expect("remote writer should persist disabled state");
}
});
writer_started_rx.await.expect("remote writer should start");
for _ in 0..10 {
tokio::task::yield_now().await;
}
assert!(
!writer_entered.load(AtomicOrdering::SeqCst),
"the remote writer must wait until the old read has synchronously published"
);
release_reader.notify_one();
reader
.await
.expect("join module reader")
.expect("module reader should succeed");
writer.await.expect("join remote module writer");
assert_eq!(
*publish_order.lock().expect("module publish order result lock"),
["old-true", "new-false"]
);
assert!(!runtime_enabled.load(AtomicOrdering::SeqCst), "the latest persisted false state must win");
shutdown.cancel();
set_persisted_module_switches(previous, previous_configured);
})
.await;
}
}
rustfs/src/startup_audit.rs
+39 -3
View File
@@ -20,6 +20,7 @@ use tracing::{error, info};
const LOG_COMPONENT_MAIN: &str = "main";
const LOG_SUBSYSTEM_STARTUP: &str = "startup";
const EVENT_AUDIT_SYSTEM_STATE: &str = "audit_system_state";
const EVENT_NOTIFY_SYSTEM_STATE: &str = "notify_system_state";
pub(crate) async fn init_audit_runtime() {
match init_event_notifier_and_audit().await {
@@ -47,17 +48,28 @@ pub(crate) async fn init_event_notifier_and_audit() -> AuditResult<()> {
init_event_notifier_and_audit_with(init_event_notifier, start_audit_system).await
}
async fn init_event_notifier_and_audit_with<NotifyFn, NotifyFuture, AuditFn, AuditFuture>(
async fn init_event_notifier_and_audit_with<NotifyFn, NotifyFuture, NotifyError, AuditFn, AuditFuture>(
notify: NotifyFn,
start_audit: AuditFn,
) -> AuditResult<()>
where
NotifyFn: FnOnce() -> NotifyFuture,
NotifyFuture: Future<Output = ()>,
NotifyFuture: Future<Output = Result<(), NotifyError>>,
NotifyError: std::fmt::Display,
AuditFn: FnOnce() -> AuditFuture,
AuditFuture: Future<Output = AuditResult<()>>,
{
notify().await;
if let Err(err) = notify().await {
error!(
target: "rustfs::main::run",
event = EVENT_NOTIFY_SYSTEM_STATE,
component = LOG_COMPONENT_MAIN,
subsystem = LOG_SUBSYSTEM_STARTUP,
state = "degraded",
error = %err,
"Notification runtime failed to start; continuing in degraded mode"
);
}
start_audit().await
}
@@ -75,6 +87,7 @@ mod tests {
let result = init_event_notifier_and_audit_with(
move || async move {
notify_events.lock().unwrap_or_else(|err| err.into_inner()).push("notify");
Ok::<(), &'static str>(())
},
move || async move {
audit_events.lock().unwrap_or_else(|err| err.into_inner()).push("audit");
@@ -97,6 +110,7 @@ mod tests {
let result = init_event_notifier_and_audit_with(
move || async move {
notify_events.lock().unwrap_or_else(|err| err.into_inner()).push("notify");
Ok::<(), &'static str>(())
},
move || async move {
audit_events.lock().unwrap_or_else(|err| err.into_inner()).push("audit");
@@ -109,4 +123,26 @@ mod tests {
let events = events.lock().unwrap_or_else(|err| err.into_inner()).clone();
assert_eq!(events, ["notify", "audit"]);
}
#[tokio::test]
async fn notify_failure_still_starts_audit() {
let events = Arc::new(Mutex::new(Vec::new()));
let notify_events = events.clone();
let audit_events = events.clone();
let result = init_event_notifier_and_audit_with(
move || async move {
notify_events.lock().unwrap_or_else(|err| err.into_inner()).push("notify");
Err::<(), &'static str>("notify failed")
},
move || async move {
audit_events.lock().unwrap_or_else(|err| err.into_inner()).push("audit");
Ok(())
},
)
.await;
assert!(result.is_ok());
assert_eq!(*events.lock().unwrap_or_else(|err| err.into_inner()), ["notify", "audit"]);
}
}
rustfs/src/startup_lifecycle.rs
+13 -1
View File
@@ -14,7 +14,7 @@
use crate::storage_api::startup::lifecycle::ECStore;
use crate::{
server::{ServiceStateManager, ShutdownHandle, wait_for_shutdown},
server::{ServiceStateManager, ShutdownHandle, start_persisted_event_notifier_reconciler, wait_for_shutdown},
startup_iam::{IamBootstrapDisposition, publish_ready_for_iam_bootstrap},
startup_runtime_sources,
startup_services::StartupServiceRuntime,
@@ -142,6 +142,7 @@ pub(crate) async fn run_startup_runtime_lifecycle(lifecycle: StartupRuntimeLifec
);
publish_ready_for_iam_bootstrap(iam_bootstrap, readiness.as_ref(), Some(state_manager.as_ref())).await?;
startup_runtime_sources::publish_init_time_now().await;
let event_notifier_reconciler = start_persisted_event_notifier_reconciler(store.clone(), shutdown_token.clone());
if enable_scanner {
init_data_scanner(shutdown_token.clone(), store).await;
@@ -157,6 +158,17 @@ pub(crate) async fn run_startup_runtime_lifecycle(lifecycle: StartupRuntimeLifec
shutdown_token,
)
.await;
if let Err(err) = event_notifier_reconciler.await {
tracing::warn!(
target: "rustfs::main::run",
event = "notify_runtime_reconcile",
component = LOG_COMPONENT_MAIN,
subsystem = LOG_SUBSYSTEM_STARTUP,
state = "join_failed",
reason = if err.is_cancelled() { "task_cancelled" } else { "task_panicked" },
"Persisted notification runtime reconciler task failed to join"
);
}
info!(
target: "rustfs::main::run",
rustfs/src/startup_shutdown.rs
+389 -33
View File
@@ -21,8 +21,11 @@ use crate::{
startup_runtime_sources,
};
use rustfs_heal::shutdown_ahm_services;
use rustfs_notify::NotificationLifecycleTransition;
use rustfs_utils::get_env_bool_with_aliases;
use std::path::Path;
use std::future::Future;
use std::path::PathBuf;
use std::sync::Mutex;
use tokio_util::sync::CancellationToken;
use tracing::{error, info, warn};
@@ -43,6 +46,137 @@ const EVENT_EVENT_NOTIFIER_SHUTDOWN: &str = "event_notifier_shutdown";
const EVENT_PROFILING_SHUTDOWN: &str = "profiling_shutdown";
const EVENT_SERVER_SHUTDOWN_STATE: &str = "server_shutdown_state";
fn join_failure_reason(error: &tokio::task::JoinError) -> &'static str {
if error.is_cancelled() {
"join_cancelled"
} else {
"join_panicked"
}
}
struct EmbeddedRuntimeOwnerState {
owners: usize,
pending_cleanup: Option<CancellationToken>,
}
struct EmbeddedRuntimeOwners {
state: Mutex<EmbeddedRuntimeOwnerState>,
}
impl EmbeddedRuntimeOwners {
const fn new() -> Self {
Self {
state: Mutex::new(EmbeddedRuntimeOwnerState {
owners: 0,
pending_cleanup: None,
}),
}
}
fn register(&self) -> Option<CancellationToken> {
let mut state = self.state.lock().unwrap_or_else(|err| err.into_inner());
state.owners += 1;
state
.pending_cleanup
.as_ref()
.filter(|cleanup| !cleanup.is_cancelled())
.cloned()
}
fn release_with<T>(&self, prepare_cleanup: impl FnOnce(CancellationToken) -> T) -> Option<T> {
let mut state = self.state.lock().unwrap_or_else(|err| err.into_inner());
if state.owners == 0 {
return None;
}
state.owners -= 1;
if state.owners != 0 {
return None;
}
if state.pending_cleanup.as_ref().is_some_and(|cleanup| !cleanup.is_cancelled()) {
return None;
}
let completion = CancellationToken::new();
// The disable generation must be accepted while registration is excluded;
// otherwise a concurrently starting owner can enable targets first and be
// overwritten by this last-owner release.
let cleanup = prepare_cleanup(completion.clone());
state.pending_cleanup = Some(completion);
Some(cleanup)
}
#[cfg(test)]
fn owner_count(&self) -> usize {
self.state.lock().unwrap_or_else(|err| err.into_inner()).owners
}
}
static EMBEDDED_RUNTIME_OWNERS: EmbeddedRuntimeOwners = EmbeddedRuntimeOwners::new();
pub(crate) struct EmbeddedRuntimeOwner {
active: bool,
runtime: tokio::runtime::Handle,
}
pub(crate) struct EmbeddedRuntimeCleanup {
completion: CancellationToken,
notification: Option<NotificationLifecycleTransition>,
runtime: tokio::runtime::Handle,
}
impl EmbeddedRuntimeCleanup {
fn prepare(completion: CancellationToken, runtime: tokio::runtime::Handle) -> Self {
let _runtime_guard = runtime.enter();
let system = rustfs_notify::ensure_live_events();
Self {
completion,
notification: Some(system.publish_targets_enabled(false, None)),
runtime,
}
}
}
impl Drop for EmbeddedRuntimeCleanup {
fn drop(&mut self) {
self.completion.cancel();
}
}
impl EmbeddedRuntimeOwner {
pub(crate) fn cleanup_runtime_handle(&self) -> tokio::runtime::Handle {
tokio::runtime::Handle::try_current().unwrap_or_else(|_| self.runtime.clone())
}
pub(crate) fn release(&mut self) -> Option<EmbeddedRuntimeCleanup> {
if !self.active {
return None;
}
self.active = false;
let runtime = self.cleanup_runtime_handle();
EMBEDDED_RUNTIME_OWNERS.release_with(move |completion| EmbeddedRuntimeCleanup::prepare(completion, runtime))
}
}
impl Drop for EmbeddedRuntimeOwner {
fn drop(&mut self) {
if let Some(cleanup) = self.release() {
schedule_embedded_runtime_cleanup(cleanup);
}
}
}
pub(crate) async fn register_embedded_runtime_owner() -> EmbeddedRuntimeOwner {
let runtime = tokio::runtime::Handle::current();
let pending_cleanup = EMBEDDED_RUNTIME_OWNERS.register();
let owner = EmbeddedRuntimeOwner { active: true, runtime };
// Reserve ownership before waiting so another release cannot start a second
// process-runtime cleanup while this startup is queued behind the first.
if let Some(cleanup) = pending_cleanup {
cleanup.cancelled().await;
}
owner
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum BackgroundShutdownStep {
DataScanner,
@@ -153,7 +287,17 @@ pub(crate) async fn run_startup_shutdown_sequence(
state = "stopping",
"Event notifier shutdown started"
);
shutdown_event_notifier().await;
if let Err(err) = shutdown_event_notifier().await {
error!(
target: "rustfs::main::handle_shutdown",
event = EVENT_EVENT_NOTIFIER_SHUTDOWN,
component = LOG_COMPONENT_MAIN,
subsystem = LOG_SUBSYSTEM_STARTUP,
state = "stop_failed",
error = %err,
"Event notifier shutdown failed"
);
}
info!(
target: "rustfs::main::handle_shutdown",
@@ -222,9 +366,19 @@ pub(crate) async fn run_startup_shutdown_sequence(
);
}
pub(crate) async fn run_embedded_shutdown_cleanup() {
shutdown_event_notifier().await;
async fn run_embedded_runtime_cleanup(mut cleanup: EmbeddedRuntimeCleanup) {
if let Some(notification) = cleanup.notification.take()
&& let Err(err) = notification.wait().await
{
warn!(
component = LOG_COMPONENT_EMBEDDED,
subsystem = LOG_SUBSYSTEM_EMBEDDED,
event = EVENT_EMBEDDED_SHUTDOWN_CLEANUP_FAILED,
service = "notification",
error = %err,
"Embedded shutdown cleanup failed"
);
}
if let Err(err) = stop_audit_system().await {
warn!(
component = LOG_COMPONENT_EMBEDDED,
@@ -235,6 +389,38 @@ pub(crate) async fn run_embedded_shutdown_cleanup() {
"Embedded shutdown cleanup failed"
);
}
if let Err(err) = startup_runtime_sources::shutdown_observability_guard() {
warn!(
component = LOG_COMPONENT_EMBEDDED,
subsystem = LOG_SUBSYSTEM_EMBEDDED,
event = EVENT_EMBEDDED_SHUTDOWN_CLEANUP_FAILED,
service = "observability",
error = %err,
"Embedded shutdown cleanup failed"
);
}
cleanup.completion.cancel();
}
pub(crate) async fn run_embedded_shutdown_cleanup(cleanup: EmbeddedRuntimeCleanup) {
let completion = cleanup.completion.clone();
let runtime = cleanup.runtime.clone();
if let Err(err) = runtime.spawn(run_embedded_runtime_cleanup(cleanup)).await {
completion.cancel();
warn!(
component = LOG_COMPONENT_EMBEDDED,
subsystem = LOG_SUBSYSTEM_EMBEDDED,
event = EVENT_EMBEDDED_SHUTDOWN_CLEANUP_FAILED,
service = "process_runtime",
reason = join_failure_reason(&err),
"Embedded shutdown cleanup failed"
);
}
}
fn schedule_embedded_runtime_cleanup(cleanup: EmbeddedRuntimeCleanup) {
let runtime = cleanup.runtime.clone();
runtime.spawn(run_embedded_shutdown_cleanup(cleanup));
}
pub(crate) fn signal_embedded_startup_shutdown(shutdown_handle: &ShutdownHandle, ctx: &CancellationToken) {
@@ -242,24 +428,66 @@ pub(crate) fn signal_embedded_startup_shutdown(shutdown_handle: &ShutdownHandle,
ctx.cancel();
}
async fn release_embedded_runtime_after_drain(mut runtime_owner: Option<EmbeddedRuntimeOwner>) {
if let Some(runtime_owner) = runtime_owner.as_mut()
&& let Some(cleanup) = runtime_owner.release()
{
run_embedded_shutdown_cleanup(cleanup).await;
}
}
pub(crate) fn run_embedded_server_drop_cleanup(
runtime: &tokio::runtime::Handle,
ctx: &CancellationToken,
shutdown_handle: &mut Option<ShutdownHandle>,
temp_dir: Option<&Path>,
temp_dir: &mut Option<PathBuf>,
runtime_owner: Option<EmbeddedRuntimeOwner>,
) {
ctx.cancel();
if let Some(shutdown_handle) = shutdown_handle.take() {
if let Some(shutdown_handle) = shutdown_handle.as_ref() {
shutdown_handle.signal();
}
if let Some(dir) = temp_dir {
let _ = std::fs::remove_dir_all(dir);
let shutdown_handle = shutdown_handle.take();
let temp_dir = temp_dir.take();
runtime.spawn(finish_embedded_server_cleanup(
shutdown_handle,
temp_dir,
release_embedded_runtime_after_drain(runtime_owner),
));
}
async fn finish_embedded_server_cleanup<F>(shutdown_handle: Option<ShutdownHandle>, temp_dir: Option<PathBuf>, process_cleanup: F)
where
F: Future<Output = ()>,
{
if let Some(shutdown_handle) = shutdown_handle {
shutdown_handle.shutdown().await;
}
process_cleanup.await;
if let Some(dir) = temp_dir
&& let Err(err) = tokio::fs::remove_dir_all(&dir).await
{
warn!(
component = LOG_COMPONENT_EMBEDDED,
subsystem = LOG_SUBSYSTEM_EMBEDDED,
event = EVENT_EMBEDDED_SHUTDOWN_CLEANUP_FAILED,
service = "temp_dir",
path = %dir.display(),
error = %err,
"Embedded shutdown cleanup failed"
);
}
}
pub(crate) async fn run_embedded_server_shutdown(
runtime: &tokio::runtime::Handle,
ctx: &CancellationToken,
shutdown_handle: &mut Option<ShutdownHandle>,
temp_dir: Option<&Path>,
temp_dir: &mut Option<PathBuf>,
runtime_owner: Option<EmbeddedRuntimeOwner>,
) {
info!(
target: "rustfs::embedded",
@@ -272,22 +500,22 @@ pub(crate) async fn run_embedded_server_shutdown(
ctx.cancel();
run_embedded_shutdown_cleanup().await;
if let Some(shutdown_handle) = shutdown_handle.take() {
shutdown_handle.shutdown().await;
if let Some(shutdown_handle) = shutdown_handle.as_ref() {
shutdown_handle.signal();
}
if let Some(dir) = temp_dir
&& let Err(err) = tokio::fs::remove_dir_all(dir).await
{
let cleanup_task = runtime.spawn(finish_embedded_server_cleanup(
shutdown_handle.take(),
temp_dir.take(),
release_embedded_runtime_after_drain(runtime_owner),
));
if let Err(err) = cleanup_task.await {
warn!(
component = LOG_COMPONENT_EMBEDDED,
subsystem = LOG_SUBSYSTEM_EMBEDDED,
event = EVENT_EMBEDDED_SHUTDOWN_CLEANUP_FAILED,
service = "temp_dir",
path = %dir.display(),
error = %err,
service = "server_cleanup",
reason = join_failure_reason(&err),
"Embedded shutdown cleanup failed"
);
}
@@ -300,25 +528,16 @@ pub(crate) async fn run_embedded_server_shutdown(
state = "stopped",
"Embedded server state changed"
);
if let Err(err) = startup_runtime_sources::shutdown_observability_guard() {
warn!(
component = LOG_COMPONENT_EMBEDDED,
subsystem = LOG_SUBSYSTEM_EMBEDDED,
event = EVENT_EMBEDDED_SHUTDOWN_CLEANUP_FAILED,
service = "observability",
error = %err,
"Embedded shutdown cleanup failed"
);
}
}
#[cfg(test)]
mod tests {
use super::{
BackgroundShutdownStep, background_shutdown_steps, run_embedded_server_drop_cleanup, signal_embedded_startup_shutdown,
BackgroundShutdownStep, EmbeddedRuntimeOwners, background_shutdown_steps, finish_embedded_server_cleanup,
run_embedded_server_drop_cleanup, signal_embedded_startup_shutdown,
};
use crate::server::ShutdownHandle;
use std::sync::{Arc, mpsc};
use std::time::Duration;
use tokio::sync::broadcast;
use tokio_util::sync::CancellationToken;
@@ -367,15 +586,152 @@ mod tests {
let cancel_token = CancellationToken::new();
let temp_dir = tempfile::tempdir().expect("temp dir should create");
let temp_path = temp_dir.path().to_path_buf();
let mut owned_temp_path = Some(temp_path.clone());
run_embedded_server_drop_cleanup(&cancel_token, &mut shutdown_handle, Some(temp_dir.path()));
run_embedded_server_drop_cleanup(
&tokio::runtime::Handle::current(),
&cancel_token,
&mut shutdown_handle,
&mut owned_temp_path,
None,
);
tokio::time::timeout(Duration::from_secs(1), observed_rx)
.await
.expect("drop cleanup should signal shutdown")
.expect("shutdown signal should be delivered");
tokio::time::timeout(Duration::from_secs(1), async {
while temp_path.exists() {
tokio::task::yield_now().await;
}
})
.await
.expect("drop cleanup should remove the temporary directory");
assert!(cancel_token.is_cancelled());
assert!(shutdown_handle.is_none());
assert!(owned_temp_path.is_none());
assert!(!temp_path.exists());
}
#[test]
fn only_the_last_embedded_owner_cleans_process_runtime() {
let owners = EmbeddedRuntimeOwners::new();
assert!(owners.register().is_none());
assert!(owners.register().is_none());
assert!(owners.release_with(|_| ()).is_none());
assert!(owners.release_with(|_| ()).is_some());
assert!(owners.release_with(|_| ()).is_none());
}
#[test]
fn last_owner_cleanup_intent_is_serialized_before_new_registration() {
let owners = Arc::new(EmbeddedRuntimeOwners::new());
assert!(owners.register().is_none());
let (cleanup_entered_tx, cleanup_entered_rx) = mpsc::channel();
let (allow_cleanup_tx, allow_cleanup_rx) = mpsc::channel();
let release_owners = owners.clone();
let release_task = std::thread::spawn(move || {
release_owners.release_with(|completion| {
cleanup_entered_tx.send(()).expect("cleanup preparation should be observed");
allow_cleanup_rx.recv().expect("cleanup preparation should be released");
completion.cancel();
})
});
cleanup_entered_rx
.recv_timeout(Duration::from_secs(1))
.expect("last-owner cleanup should enter preparation");
let (register_started_tx, register_started_rx) = mpsc::channel();
let (register_done_tx, register_done_rx) = mpsc::channel();
let register_owners = owners.clone();
let register_task = std::thread::spawn(move || {
register_started_tx.send(()).expect("registration should start");
let pending_cleanup = register_owners.register();
register_done_tx
.send(pending_cleanup)
.expect("registration result should be observed");
});
register_started_rx
.recv_timeout(Duration::from_secs(1))
.expect("registration should start");
let early_registration = match register_done_rx.try_recv() {
Ok(result) => Some(result),
Err(mpsc::TryRecvError::Empty) => None,
Err(mpsc::TryRecvError::Disconnected) => panic!("registration thread disconnected"),
};
let registered_before_intent_published = early_registration.is_some();
allow_cleanup_tx.send(()).expect("cleanup preparation should finish");
assert!(release_task.join().expect("release thread should not panic").is_some());
let registration = match early_registration {
Some(result) => result,
None => register_done_rx
.recv_timeout(Duration::from_secs(1))
.expect("registration should complete"),
};
register_task.join().expect("register thread should not panic");
assert!(!registered_before_intent_published);
assert!(registration.is_none());
assert_eq!(owners.owner_count(), 1);
}
#[tokio::test]
async fn new_owner_waits_for_prior_last_owner_cleanup() {
let owners = EmbeddedRuntimeOwners::new();
assert!(owners.register().is_none());
let cleanup_finished = owners
.release_with(|completion| completion)
.expect("last owner should publish a cleanup barrier");
let pending_cleanup = owners.register().expect("new owner should observe unfinished cleanup");
let wait_task = tokio::spawn(async move {
pending_cleanup.cancelled().await;
});
tokio::task::yield_now().await;
assert!(!wait_task.is_finished());
cleanup_finished.cancel();
wait_task.await.expect("cleanup waiter should not panic");
}
#[tokio::test]
async fn server_drain_and_process_runtime_cleanup_finish_before_temp_dir_removal() {
let temp_dir = tempfile::tempdir().expect("temp dir should create");
let temp_path = temp_dir.path().to_path_buf();
let (shutdown_tx, mut shutdown_rx) = broadcast::channel(1);
let (shutdown_entered_tx, shutdown_entered_rx) = tokio::sync::oneshot::channel();
let (allow_shutdown_tx, allow_shutdown_rx) = tokio::sync::oneshot::channel();
let shutdown_task = tokio::spawn(async move {
let _ = shutdown_rx.recv().await;
shutdown_entered_tx.send(()).expect("shutdown should be observed");
allow_shutdown_rx.await.expect("shutdown should be released");
});
let shutdown_handle = ShutdownHandle::new(shutdown_tx, shutdown_task);
let (cleanup_entered_tx, mut cleanup_entered_rx) = tokio::sync::oneshot::channel();
let (allow_cleanup_tx, allow_cleanup_rx) = tokio::sync::oneshot::channel();
let cleanup_task = tokio::spawn(finish_embedded_server_cleanup(
Some(shutdown_handle),
Some(temp_path.clone()),
async move {
cleanup_entered_tx.send(()).expect("cleanup should be observed");
allow_cleanup_rx.await.expect("cleanup should be released");
},
));
shutdown_entered_rx.await.expect("shutdown should start");
assert!(
matches!(cleanup_entered_rx.try_recv(), Err(tokio::sync::oneshot::error::TryRecvError::Empty)),
"process runtime cleanup must wait for the server to drain"
);
assert!(temp_path.exists(), "temporary data must remain available while the server drains");
allow_shutdown_tx.send(()).expect("shutdown should finish");
cleanup_entered_rx.await.expect("cleanup should start");
assert!(temp_path.exists(), "temporary data must remain available during runtime cleanup");
allow_cleanup_tx.send(()).expect("cleanup should finish");
cleanup_task.await.expect("cleanup task should not panic");
assert!(!temp_path.exists(), "temporary data should be removed only after runtime cleanup");
}
}
rustfs/src/storage/mod.rs
+3 -3
View File
@@ -70,9 +70,9 @@ pub(crate) use storage_api::{
init_bucket_metadata_sys, init_ecstore_config, init_local_disks_with_instance_ctx, init_lock_clients,
is_all_buckets_not_found, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found, is_valid_storage_class,
load_bucket_metadata, options_consumer, prewarm_local_disk_id_map_with_instance_ctx, read_config, record_replication_proxy,
rpc_consumer, runtime_sources_consumer, s3_api_consumer, save_config, serialize, set_bucket_metadata,
table_catalog_path_hash, to_s3s_etag, topology_snapshot_from_endpoint_pools_with_capabilities, try_migrate_bucket_metadata,
try_migrate_iam_config, try_migrate_server_config, update_bucket_metadata_config, verify_rpc_signature, wrap_reader,
rpc_consumer, runtime_sources_consumer, s3_api_consumer, serialize, set_bucket_metadata, table_catalog_path_hash,
to_s3s_etag, topology_snapshot_from_endpoint_pools_with_capabilities, try_migrate_bucket_metadata, try_migrate_iam_config,
try_migrate_server_config, update_bucket_metadata_config, verify_rpc_signature, wrap_reader,
};
#[cfg(test)]
rustfs/src/storage/rpc/node_service.rs
+100 -24
View File
@@ -16,15 +16,15 @@ use crate::admin::service::{
config::{reload_dynamic_config_runtime_state, reload_runtime_config_snapshot},
site_replication::reload_site_replication_runtime_state,
};
use crate::server::MODULE_SWITCHES_SIGNAL_SUBSYSTEM;
use crate::storage::storage_api::ecstore_tier::tier_mutation_peer::{self, TierMutationPeerState as EcTierMutationPeerState};
#[cfg(test)]
use crate::storage::storage_api::rpc_consumer::node_service::STORAGE_CLASS_SUB_SYS;
#[cfg(test)]
use crate::storage::storage_api::rpc_consumer::node_service::{CollectMetricsOpts, MetricType};
use crate::storage::storage_api::rpc_consumer::node_service::{
DiskStore, ECStore, Error, LocalPeerS3Client, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, SERVICE_SIGNAL_REFRESH_CONFIG,
SERVICE_SIGNAL_RELOAD_DYNAMIC, StorageDiskRpcExt as _, StorageResult, all_local_disk_path, find_local_disk_by_ref,
reload_transition_tier_config,
DiskStore, ECStore, Error, LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS,
SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, StorageDiskRpcExt as _, StorageResult, all_local_disk_path,
find_local_disk_by_ref, reload_transition_tier_config,
};
use crate::storage::storage_api::runtime_sources_consumer::{EndpointServerPools, runtime_sources};
use crate::storage::storage_api::{sign_tonic_rpc_response_proof, verify_tonic_canonical_body_digest};
@@ -32,6 +32,9 @@ use bytes::Bytes;
use futures::Stream;
use futures_util::future::join_all;
use rmp_serde::Deserializer;
use rustfs_config::audit::{AUDIT_MQTT_SUB_SYS, AUDIT_WEBHOOK_SUB_SYS};
use rustfs_config::notify::NOTIFY_SUB_SYSTEMS;
use rustfs_config::{HEAL_SUB_SYS, SCANNER_SUB_SYS};
use rustfs_filemeta::MetacacheReader;
use rustfs_iam::store::UserType;
use rustfs_lock::LockClient;
@@ -75,6 +78,14 @@ const TIER_MUTATION_PEER_STATE_PREPARED_WIRE: i32 = 1;
const TIER_MUTATION_PEER_STATE_COMMITTED_WIRE: i32 = 2;
const TIER_MUTATION_PEER_STATE_ABORTED_WIRE: i32 = 3;
fn supports_dynamic_config_rpc(sub_system: &str) -> bool {
NOTIFY_SUB_SYSTEMS.contains(&sub_system)
|| matches!(
sub_system,
STORAGE_CLASS_SUB_SYS | AUDIT_WEBHOOK_SUB_SYS | AUDIT_MQTT_SUB_SYS | SCANNER_SUB_SYS | HEAL_SUB_SYS
)
}
#[derive(Debug)]
struct HealControlReplayEntry {
command_digest: [u8; 32],
@@ -1517,6 +1528,18 @@ impl Node for NodeService {
let raw_signal = vars.get(PEER_RESTSIGNAL).map(String::as_str);
let signal = raw_signal.and_then(|value| value.parse::<u64>().ok());
let sub_system = vars.get(PEER_RESTSUB_SYS).map(String::as_str).unwrap_or_default();
let dry_run = match vars.get(PEER_RESTDRY_RUN).map(String::as_str) {
None => false,
Some(value) => match value.parse::<bool>() {
Ok(value) => value,
Err(_) => {
return Ok(Response::new(SignalServiceResponse {
success: false,
error_info: Some(format!("invalid dry-run value: {value}")),
}));
}
},
};
match signal {
Some(SERVICE_SIGNAL_REFRESH_CONFIG) => match reload_runtime_config_snapshot().await {
@@ -1524,21 +1547,36 @@ impl Node for NodeService {
success: true,
error_info: None,
})),
Err(err) => Ok(Response::new(SignalServiceResponse {
Err(_) => Ok(Response::new(SignalServiceResponse {
success: false,
error_info: Some(err.to_string()),
})),
},
Some(SERVICE_SIGNAL_RELOAD_DYNAMIC) => match reload_dynamic_config_runtime_state(sub_system).await {
Ok(()) => Ok(Response::new(SignalServiceResponse {
success: true,
error_info: None,
})),
Err(err) => Ok(Response::new(SignalServiceResponse {
success: false,
error_info: Some(err.to_string()),
error_info: Some("runtime config snapshot reload failed".to_string()),
})),
},
Some(SERVICE_SIGNAL_RELOAD_DYNAMIC) => {
let supported = sub_system == MODULE_SWITCHES_SIGNAL_SUBSYSTEM || supports_dynamic_config_rpc(sub_system);
if !supported {
return Ok(Response::new(SignalServiceResponse {
success: false,
error_info: Some(format!("unsupported dynamic config subsystem: {sub_system}")),
}));
}
if dry_run {
return Ok(Response::new(SignalServiceResponse {
success: true,
error_info: None,
}));
}
match reload_dynamic_config_runtime_state(sub_system).await {
Ok(()) => Ok(Response::new(SignalServiceResponse {
success: true,
error_info: None,
})),
Err(_) => Ok(Response::new(SignalServiceResponse {
success: false,
error_info: Some(format!("dynamic config reload failed for {sub_system}")),
})),
}
}
Some(other) => Ok(Response::new(SignalServiceResponse {
success: false,
error_info: Some(format!("unsupported service signal: {other}")),
@@ -1817,12 +1855,12 @@ impl Node for NodeService {
#[allow(unused_imports)]
mod tests {
use super::{
CollectMetricsOpts, DiskStore, Error, HEAL_CONTROL_PAYLOAD_MAX_SIZE, MetricType, Node as _, NodeService, PEER_RESTSIGNAL,
PEER_RESTSUB_SYS, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, STORAGE_CLASS_SUB_SYS,
admit_heal_control_replay, background_rebalance_start_error_message, execute_heal_control_envelope_with_manager,
initialize_heal_topology_fingerprint, make_heal_control_server, make_heal_control_server_with_cache, make_server,
make_tier_mutation_control_server_for_context, remove_heal_control_replay, scanner_activity_response,
stop_rebalance_response,
CollectMetricsOpts, DiskStore, Error, HEAL_CONTROL_PAYLOAD_MAX_SIZE, MetricType, Node as _, NodeService,
PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC,
STORAGE_CLASS_SUB_SYS, admit_heal_control_replay, background_rebalance_start_error_message,
execute_heal_control_envelope_with_manager, initialize_heal_topology_fingerprint, make_heal_control_server,
make_heal_control_server_with_cache, make_server, make_tier_mutation_control_server_for_context,
remove_heal_control_replay, scanner_activity_response, stop_rebalance_response,
};
use crate::storage::rpc::node_service::heal::heal_topology_fingerprint;
use crate::storage::storage_api::rpc_consumer::node_service::{HealBucketInfo, HealEndpoint};
@@ -4185,6 +4223,44 @@ mod tests {
assert!(error_info.contains("unsupported dynamic config subsystem: identity_openid"));
}
#[test]
fn dynamic_config_rpc_allowlist_matches_supported_subsystems() {
for sub_system in rustfs_config::notify::NOTIFY_SUB_SYSTEMS {
assert!(super::supports_dynamic_config_rpc(sub_system));
}
for sub_system in [
STORAGE_CLASS_SUB_SYS,
rustfs_config::audit::AUDIT_WEBHOOK_SUB_SYS,
rustfs_config::audit::AUDIT_MQTT_SUB_SYS,
rustfs_config::SCANNER_SUB_SYS,
rustfs_config::HEAL_SUB_SYS,
] {
assert!(super::supports_dynamic_config_rpc(sub_system));
}
assert!(!super::supports_dynamic_config_rpc("identity_openid"));
}
#[tokio::test]
async fn test_signal_service_dry_run_accepts_notify_without_runtime_mutation() {
let service = create_test_node_service();
let mut vars = HashMap::new();
vars.insert(PEER_RESTSIGNAL.to_string(), SERVICE_SIGNAL_RELOAD_DYNAMIC.to_string());
vars.insert(PEER_RESTSUB_SYS.to_string(), rustfs_config::notify::NOTIFY_WEBHOOK_SUB_SYS.to_string());
vars.insert(PEER_RESTDRY_RUN.to_string(), true.to_string());
let response = service
.signal_service(Request::new(SignalServiceRequest {
vars: Some(Mss { value: vars }),
}))
.await
.expect("notify capability probe should return a response")
.into_inner();
assert!(response.success, "new nodes must advertise notify lifecycle reload support");
assert!(response.error_info.is_none());
}
#[tokio::test]
#[ignore = "requires isolated global object layer state"]
#[serial_test::serial]
@@ -4204,7 +4280,7 @@ mod tests {
let signal_response = response.unwrap().into_inner();
assert!(!signal_response.success);
let error_info = signal_response.error_info.expect("expected error info");
assert!(error_info.contains("storage layer not initialized"));
assert_eq!(error_info, "runtime config snapshot reload failed");
}
#[tokio::test]
@@ -4227,7 +4303,7 @@ mod tests {
let signal_response = response.unwrap().into_inner();
assert!(!signal_response.success);
let error_info = signal_response.error_info.expect("expected error info");
assert!(error_info.contains("storage layer not initialized"));
assert_eq!(error_info, format!("dynamic config reload failed for {STORAGE_CLASS_SUB_SYS}"));
}
fn assert_unimplemented_status<T>(response: Result<Response<T>, Status>, method: &str) {
rustfs/src/storage/storage_api.rs
+46 -12
View File
@@ -220,11 +220,11 @@ pub(crate) mod rpc_consumer {
pub(crate) mod node_service {
pub(crate) use super::super::{
BatchReadVersionReq, BatchReadVersionResp, CollectMetricsOpts, DeleteOptions, DiskError, DiskInfoOptions, DiskStore,
ECStore, Error, FileInfoVersions, LocalPeerS3Client, MetricType, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, ReadMultipleReq,
ReadMultipleResp, ReadOptions, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, StorageDiskRpcExt,
StoragePeerS3ClientExt, UpdateMetadataOpts, all_local_disk_path, collect_local_metrics, find_local_disk_by_ref,
get_local_server_property, load_bucket_metadata, reload_transition_tier_config, remove_bucket_metadata,
set_bucket_metadata, validate_batch_read_version_item_count,
ECStore, Error, FileInfoVersions, LocalPeerS3Client, MetricType, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS,
ReadMultipleReq, ReadMultipleResp, ReadOptions, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC,
StorageDiskRpcExt, StoragePeerS3ClientExt, UpdateMetadataOpts, all_local_disk_path, collect_local_metrics,
find_local_disk_by_ref, get_local_server_property, load_bucket_metadata, reload_transition_tier_config,
remove_bucket_metadata, set_bucket_metadata, validate_batch_read_version_item_count,
};
pub(crate) type StorageResult<T> = super::super::Result<T>;
@@ -233,7 +233,6 @@ pub(crate) mod rpc_consumer {
#[cfg(test)]
pub(crate) type HealBucketInfo = super::super::contract::bucket::BucketInfo;
#[cfg(test)]
pub(crate) const STORAGE_CLASS_SUB_SYS: &str = super::super::STORAGE_CLASS_SUB_SYS;
pub(crate) mod contract {
@@ -479,9 +478,9 @@ pub(crate) mod ecstore_rio {
pub(crate) mod ecstore_rpc {
pub(crate) use rustfs_ecstore::api::rpc::{
LocalPeerS3Client, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, PeerS3Client, SERVICE_SIGNAL_REFRESH_CONFIG,
SERVICE_SIGNAL_RELOAD_DYNAMIC, TONIC_RPC_PREFIX, normalize_tonic_rpc_audience, sign_tonic_rpc_response_proof,
verify_rpc_signature, verify_tonic_canonical_body_digest, verify_tonic_rpc_signature,
LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, PeerS3Client,
SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, TONIC_RPC_PREFIX, normalize_tonic_rpc_audience,
sign_tonic_rpc_response_proof, verify_rpc_signature, verify_tonic_canonical_body_digest, verify_tonic_rpc_signature,
};
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::rpc::{
@@ -533,6 +532,7 @@ pub(crate) const BUCKET_VERSIONING_CONFIG: &str = ecstore_bucket::metadata::BUCK
pub(crate) const BUCKET_WEBSITE_CONFIG: &str = ecstore_bucket::metadata::BUCKET_WEBSITE_CONFIG;
pub(crate) const DEFAULT_READ_BUFFER_SIZE: usize = ecstore_set_disk::DEFAULT_READ_BUFFER_SIZE;
pub(crate) const OBJECT_LOCK_CONFIG: &str = ecstore_bucket::metadata::OBJECT_LOCK_CONFIG;
pub(crate) const PEER_RESTDRY_RUN: &str = ecstore_rpc::PEER_RESTDRY_RUN;
pub(crate) const PEER_RESTSIGNAL: &str = ecstore_rpc::PEER_RESTSIGNAL;
pub(crate) const PEER_RESTSUB_SYS: &str = ecstore_rpc::PEER_RESTSUB_SYS;
pub(crate) const SERVICE_SIGNAL_REFRESH_CONFIG: u64 = ecstore_rpc::SERVICE_SIGNAL_REFRESH_CONFIG;
@@ -554,7 +554,6 @@ pub(crate) use ecstore_rpc::sign_tonic_rpc_response_proof;
#[cfg(test)]
pub(crate) use ecstore_rpc::verify_tonic_rpc_response_proof;
#[cfg(test)]
pub(crate) const STORAGE_CLASS_SUB_SYS: &str = ecstore_config::com::STORAGE_CLASS_SUB_SYS;
pub(crate) type BucketMetadata = ecstore_bucket::metadata::BucketMetadata;
@@ -873,6 +872,14 @@ pub(crate) async fn read_config(api: Arc<ECStore>, file: &str) -> Result<Vec<u8>
ecstore_config::com::read_config(api, file).await
}
pub(crate) async fn read_config_no_lock(api: Arc<ECStore>, file: &str) -> Result<Vec<u8>> {
ecstore_config::com::read_config_no_lock(api, file).await
}
pub(crate) async fn read_existing_server_config_no_lock(api: Arc<ECStore>) -> Result<rustfs_config::server_config::Config> {
ecstore_config::com::read_existing_server_config_no_lock(api).await
}
pub(crate) async fn prewarm_local_disk_id_map_with_instance_ctx(instance_ctx: &Arc<InstanceContext>) {
ecstore_storage::prewarm_local_disk_id_map_with_instance_ctx(instance_ctx).await;
}
@@ -881,8 +888,35 @@ pub(crate) fn replication_queue_current_count() -> Option<i64> {
get_global_replication_stats().and_then(|stats| stats.queue_current_count())
}
pub(crate) async fn save_config(api: Arc<ECStore>, file: &str, data: Vec<u8>) -> Result<()> {
ecstore_config::com::save_config(api, file, data).await
pub(crate) async fn save_config_no_lock(api: Arc<ECStore>, file: &str, data: Vec<u8>) -> Result<()> {
ecstore_config::com::save_config_no_lock(api, file, data).await
}
pub(crate) async fn with_config_object_write_lock<F, Fut, T>(api: Arc<ECStore>, object: String, operation: F) -> Result<T>
where
F: FnOnce() -> Fut + Send + 'static,
Fut: std::future::Future<Output = T> + Send + 'static,
T: Send + 'static,
{
ecstore_config::com::with_config_object_write_lock(api, object, operation).await
}
pub(crate) async fn with_config_object_read_lock<F, Fut, T>(api: Arc<ECStore>, object: String, operation: F) -> Result<T>
where
F: FnOnce() -> Fut + Send + 'static,
Fut: std::future::Future<Output = T> + Send + 'static,
T: Send + 'static,
{
ecstore_config::com::with_config_object_read_lock(api, object, operation).await
}
pub(crate) async fn with_server_config_read_lock<F, Fut, T>(api: Arc<ECStore>, operation: F) -> Result<T>
where
F: FnOnce() -> Fut + Send + 'static,
Fut: std::future::Future<Output = T> + Send + 'static,
T: Send + 'static,
{
ecstore_config::com::with_server_config_read_lock(api, operation).await
}
pub(crate) fn shutdown_background_services() {
rustfs/src/storage_api.rs
+8 -2
View File
@@ -89,7 +89,10 @@ pub(crate) mod server {
}
}
pub(crate) use crate::storage::storage_api::{EventArgs, StorageObjectInfo, register_event_dispatch_hook};
pub(crate) use crate::storage::storage_api::{
EventArgs, StorageObjectInfo, read_existing_server_config_no_lock, register_event_dispatch_hook,
with_server_config_read_lock,
};
}
pub(crate) mod http {
@@ -151,7 +154,10 @@ pub(crate) mod server {
}
pub(crate) mod module_switch {
pub(crate) use crate::storage::storage_api::{Error, read_config, save_config};
pub(crate) use crate::storage::storage_api::{
Error, read_config, read_config_no_lock, save_config_no_lock, with_config_object_read_lock,
with_config_object_write_lock,
};
}
pub(crate) mod readiness {
rustfs/tests/embedded_notification_lifecycle_test.rs
+102
View File
@@ -0,0 +1,102 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use aws_sdk_s3::config::{Credentials, Region};
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::{Client, Config};
use rustfs::embedded::{RustFSServerBuilder, find_available_port};
use rustfs_notify::{NotificationRuntimeState, notification_system};
fn s3_client(endpoint: &str, access_key: &str, secret_key: &str) -> Client {
let credentials = Credentials::new(access_key, secret_key, None, None, "test");
let config = Config::builder()
.credentials_provider(credentials)
.region(Region::new("us-east-1"))
.endpoint_url(endpoint)
.force_path_style(true)
.behavior_version_latest()
.build();
Client::from_conf(config)
}
#[tokio::test]
async fn notification_runtime_stays_enabled_until_the_last_embedded_owner_drains() {
temp_env::async_with_vars([(rustfs_config::ENV_NOTIFY_ENABLE, Some("true"))], async {
let port_a = match find_available_port() {
Ok(port) => port,
Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return,
Err(err) => panic!("find free port for server A: {err}"),
};
let server_a = RustFSServerBuilder::new()
.address(format!("127.0.0.1:{port_a}"))
.access_key("shared-access")
.secret_key("shared-secret")
.build()
.await
.expect("start embedded server A");
let port_b = match find_available_port() {
Ok(port) => port,
Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => {
server_a.shutdown().await;
return;
}
Err(err) => {
server_a.shutdown().await;
panic!("find free port for server B: {err}");
}
};
let server_b = RustFSServerBuilder::new()
.address(format!("127.0.0.1:{port_b}"))
.access_key("shared-access")
.secret_key("shared-secret")
.build()
.await
.expect("start embedded server B");
let notification = notification_system().expect("embedded startup should initialize notification runtime");
let active_state = notification.runtime_lifecycle_state();
assert!(matches!(active_state, NotificationRuntimeState::TargetsEnabled { .. }));
assert!(notification.runtime_lifecycle_is_converged());
server_b.shutdown().await;
assert_eq!(
notification.runtime_lifecycle_state(),
active_state,
"one owner must not suspend the shared notification runtime"
);
assert!(notification.runtime_lifecycle_is_converged());
let client_a = s3_client(&server_a.endpoint(), server_a.access_key(), server_a.secret_key());
client_a
.create_bucket()
.bucket("survives-notify-owner-shutdown")
.send()
.await
.expect("server A should remain usable after server B shuts down");
client_a
.put_object()
.bucket("survives-notify-owner-shutdown")
.key("marker.txt")
.body(ByteStream::from_static(b"still here"))
.send()
.await
.expect("server A should still write after server B shuts down");
server_a.shutdown().await;
assert_eq!(notification.runtime_lifecycle_state(), NotificationRuntimeState::LiveOnly);
assert!(notification.runtime_lifecycle_is_converged());
})
.await;
}