fix(scanner): make distributed usage convergence authoritative (#5151)

* fix(scanner): make distributed usage cycles authoritative

* fix(scanner): close distributed refresh races

* fix(config): align scanner reload integration

* fix(admin): scope config test helpers

* fix(scanner): harden distributed usage convergence

* fix(scanner): preserve rolling activity compatibility

* fix(admin): expose non-secret optional config values

* fix(scanner): acknowledge distributed dirty usage

* fix(ecstore): make bucket mutations cancellation safe

* fix(scanner): preserve pending dirty acknowledgements

* test(obs): account for superseded scanner metric

* fix(api): reject excess detached bucket mutations

* test: close scanner convergence coverage gaps

* fix(scanner): make path tracking cleanup one-shot

---------

Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
Co-authored-by: houseme <housemecn@gmail.com>
This commit is contained in:
Henry Guo
2026-07-25 18:45:16 +08:00
committed by GitHub
parent 0364523dad
commit a63b79004c
69 changed files with 18073 additions and 1585 deletions
+4 -2
View File
@@ -259,8 +259,10 @@ impl Operation for AccountInfoHandler {
.await
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, e.to_string()))?;
// Serve the last persisted scanner snapshot plus the in-memory overlay.
// This request path must never trigger a live full-version listing
// Serve the last persisted scanner snapshot plus an authoritative
// single-process overlay. Distributed nodes must not promote their
// process-local absolute counters to cluster-wide bucket totals.
// This path never triggers a live full-version listing
// (rustfs/backlog#1306); freshness is owned by the scanner.
let mut data_usage_info = map_data_usage_result(load_data_usage_from_backend_cached(store.clone()).await)?;
apply_bucket_usage_memory_overlay(&mut data_usage_info).await;
@@ -12,11 +12,11 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::admin::handlers::supervise_admin_mutation;
use crate::admin::handlers::target_descriptor::AdminTargetSpec;
use crate::admin::runtime_sources::{AppContext, current_app_context, current_object_store_handle_for_context};
use crate::admin::storage_api::config::{
read_admin_config_without_migrate, read_admin_config_without_migrate_no_lock, save_admin_server_config_no_lock,
with_admin_server_config_write_lock,
read_admin_config_without_migrate, read_admin_server_config_snapshot, save_admin_server_config_snapshot,
};
use rustfs_audit::{audit_system, start_audit_system as start_global_audit_system, system::AuditSystemState};
use rustfs_config::DEFAULT_DELIMITER;
@@ -96,20 +96,20 @@ where
let Some(store) = current_object_store_handle_for_context(context) else {
return Err(s3_error!(InternalError, "server storage not initialized"));
};
let specs = specs.to_vec();
let lock_store = store.clone();
with_admin_server_config_write_lock(lock_store, move || async move {
let mut config = read_admin_config_without_migrate_no_lock(store.clone())
supervise_admin_mutation("audit config update", async move {
let snapshot = read_admin_server_config_snapshot(store.clone())
.await
.map_err(|e| s3_error!(InternalError, "failed to read server config: {}", e))?;
let mut config = snapshot.config.clone();
if !modifier(&mut config) {
return Ok(());
}
save_admin_server_config_no_lock(store, &config)
save_admin_server_config_snapshot(store, &config, &snapshot)
.await
.map(|_| ())
.map_err(|e| s3_error!(InternalError, "failed to save audit config: {}", e))?;
// Keep persistence and runtime publication in one detached, serialized
@@ -118,8 +118,6 @@ where
apply_audit_runtime_config(&specs, config).await
})
.await
.map_err(|err| s3_error!(InternalError, "failed to lock server config update: {}", err))??;
Ok(())
}
pub(crate) async fn update_audit_config_and_reload<F>(specs: &[AdminTargetSpec], modifier: F) -> S3Result<()>
+303 -238
View File
@@ -21,15 +21,14 @@ use crate::admin::runtime_sources::{
};
use crate::admin::service::config::{
CONFIG_WORKER_RELOAD_FAILURE_STATE, EVENT_CONFIG_WORKER_RELOAD_FAILED, FULL_CONFIG_WORKER_SUBSYSTEMS, LOG_COMPONENT_ADMIN,
LOG_SUBSYSTEM_CONFIG, PreparedRuntimeConfig, apply_dynamic_config_for_subsystem, is_dynamic_config_subsystem,
preflight_dynamic_config_reload, prepare_server_config, signal_config_snapshot_reload_checked,
signal_dynamic_config_reload_checked,
LOG_SUBSYSTEM_CONFIG, PreparedRuntimeConfig, is_dynamic_config_subsystem, preflight_dynamic_config_reload,
prepare_server_config, reload_dynamic_config_runtime_state, reload_runtime_config_snapshot, signal_config_snapshot_reload,
signal_config_snapshot_reload_checked, signal_dynamic_config_reload_checked,
};
use crate::admin::storage_api::config::storageclass::{INLINE_BLOCK_ENV, OPTIMIZE_ENV, RRS_ENV, STANDARD_ENV};
use crate::admin::storage_api::config::{
RUSTFS_META_BUCKET, STORAGE_CLASS_SUB_SYS, delete_admin_config, read_admin_config, read_admin_config_without_migrate,
read_admin_config_without_migrate_no_lock, save_admin_config, save_admin_server_config_no_lock,
with_admin_server_config_write_lock,
AdminServerConfigSnapshot, RUSTFS_META_BUCKET, STORAGE_CLASS_SUB_SYS, delete_admin_config, read_admin_config,
read_admin_config_without_migrate, read_admin_server_config_snapshot, save_admin_config, save_admin_server_config_snapshot,
};
use crate::admin::storage_api::contract::list::ListOperations as _;
use crate::admin::utils::{encode_compatible_admin_payload, is_compat_admin_request, read_compatible_admin_body};
@@ -64,7 +63,7 @@ use rustfs_config::oidc::{
};
use rustfs_config::server_config::{Config as ServerConfig, DEFAULT_KVS, KV, KVS};
use rustfs_config::{
COMMENT_KEY, DEFAULT_DELIMITER, ENABLE_KEY, ENV_PREFIX, ENV_SCANNER_ALERT_EXCESS_FOLDERS,
BASE_DSN_STRING, COMMENT_KEY, DEFAULT_DELIMITER, ENABLE_KEY, ENV_PREFIX, ENV_SCANNER_ALERT_EXCESS_FOLDERS,
ENV_SCANNER_ALERT_EXCESS_VERSION_SIZE, ENV_SCANNER_ALERT_EXCESS_VERSIONS, ENV_SCANNER_BITROT_CYCLE_SECS,
ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS, ENV_SCANNER_CYCLE, ENV_SCANNER_CYCLE_MAX_DIRECTORIES,
ENV_SCANNER_CYCLE_MAX_DURATION_SECS, ENV_SCANNER_CYCLE_MAX_OBJECTS, ENV_SCANNER_DELAY, ENV_SCANNER_IDLE_MODE,
@@ -86,7 +85,6 @@ use s3s::{Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error}
use serde::Serialize;
use std::collections::{BTreeSet, HashMap};
use std::env;
use std::future::Future;
use std::mem::size_of;
use time::OffsetDateTime;
use tracing::warn;
@@ -750,14 +748,6 @@ async fn load_server_config_from_store() -> S3Result<ServerConfig> {
.map_err(Into::into)
}
async fn load_server_config_from_store_locked() -> S3Result<ServerConfig> {
let store = object_store()?;
read_admin_config_without_migrate_no_lock(store)
.await
.map_err(ApiError::from)
.map_err(Into::into)
}
async fn load_active_server_config() -> S3Result<ServerConfig> {
if let Ok(config) = load_server_config_from_store().await {
return Ok(config);
@@ -768,9 +758,17 @@ async fn load_active_server_config() -> S3Result<ServerConfig> {
.ok_or_else(|| s3_error!(InternalError, "server config is not initialized"))
}
async fn save_server_config_to_store_locked(config: &ServerConfig) -> S3Result<()> {
async fn load_server_config_snapshot_from_store() -> S3Result<AdminServerConfigSnapshot> {
let store = object_store()?;
save_admin_server_config_no_lock(store, config)
read_admin_server_config_snapshot(store)
.await
.map_err(ApiError::from)
.map_err(Into::into)
}
async fn save_server_config_to_store(config: &ServerConfig, snapshot: &AdminServerConfigSnapshot) -> S3Result<bool> {
let store = object_store()?;
save_admin_server_config_snapshot(store, config, snapshot)
.await
.map_err(ApiError::from)
.map_err(Into::into)
@@ -864,7 +862,17 @@ fn encode_config_history_recovery(config: &ServerConfig) -> Vec<u8> {
fn encode_config_history_snapshot(config: &ServerConfig) -> S3Result<Vec<u8>> {
let recovery = encode_config_history_recovery(config);
let display = render_full_config(config, true);
let mut display_config = config.clone();
for targets in display_config.0.values_mut() {
for kvs in targets.values_mut() {
for entry in &mut kvs.0 {
if (entry.hidden_if_empty || is_sensitive_key_name(&entry.key)) && !entry.value.trim().is_empty() {
entry.value = REDACTED_VALUE.to_string();
}
}
}
}
let display = render_full_config(&display_config, false);
let recovery_len =
u64::try_from(recovery.len()).map_err(|_| s3_error!(InternalError, "config history recovery snapshot is too large"))?;
let mut snapshot =
@@ -1267,6 +1275,7 @@ fn is_sensitive_key_name(key: &str) -> bool {
|| normalized.ends_with("_tls_client_key")
|| normalized == "private_key"
|| normalized.ends_with("_private_key")
|| normalized == BASE_DSN_STRING
}
fn is_sensitive_env_key_name(key: &str) -> bool {
@@ -1415,7 +1424,7 @@ fn apply_delete_directives(config: &mut ServerConfig, directives: &[ConfigDirect
}
fn render_entry_value(entry: &KV, redact_secrets: bool) -> String {
if redact_secrets && (entry.hidden_if_empty || is_sensitive_key_name(&entry.key)) && !entry.value.trim().is_empty() {
if redact_secrets && is_sensitive_key_name(&entry.key) && !entry.value.trim().is_empty() {
REDACTED_VALUE.to_string()
} else {
entry.value.clone()
@@ -1824,22 +1833,22 @@ fn build_help_response(sub_system: Option<&str>, key: Option<&str>, env_only: bo
})
}
async fn reject_lost_config_transaction<T>(snapshot: AdminServerConfigSnapshot, phase: &'static str) -> S3Result<T> {
drop(snapshot);
reload_runtime_config_snapshot().await?;
signal_config_snapshot_reload().await;
Err(s3_error!(
InternalError,
"server config transaction lock was lost {phase}; runtime state was reloaded from durable config"
))
}
fn publish_prepared_config_snapshots(config: ServerConfig, prepared: PreparedRuntimeConfig) -> S3Result<()> {
prepared.publish_storage_class()?;
publish_server_config(config);
Ok(())
}
async fn commit_prepared_config(
config: ServerConfig,
prepared: PreparedRuntimeConfig,
persist: impl Future<Output = S3Result<()>>,
publish: impl FnOnce(ServerConfig, PreparedRuntimeConfig) -> S3Result<()>,
) -> S3Result<()> {
persist.await?;
publish(config, prepared)
}
/// Re-apply local mutable worker families after a full-config replacement.
/// Peers receive one full-snapshot signal after this returns; signaling each
/// family here as well would recreate audit/scanner targets twice per peer.
@@ -1851,9 +1860,26 @@ fn publish_notify_config_intent(
.then(|| rustfs_notify::ensure_live_events().publish_config(config.clone()))
}
async fn preflight_notify_config_intent(sub_system: Option<&str>) -> S3Result<()> {
if sub_system.is_none() || sub_system.is_some_and(|sub_system| NOTIFY_SUB_SYSTEMS.contains(&sub_system)) {
preflight_dynamic_config_reload(sub_system.unwrap_or(NOTIFY_WEBHOOK_SUB_SYS)).await?;
fn config_preflight_subsystems(sub_system: Option<&str>) -> Vec<&str> {
if let Some(sub_system) = sub_system {
return is_dynamic_config_subsystem(sub_system)
.then_some(sub_system)
.into_iter()
.collect();
}
let mut sub_systems = vec![NOTIFY_WEBHOOK_SUB_SYS];
for sub_system in FULL_CONFIG_WORKER_SUBSYSTEMS {
if !NOTIFY_SUB_SYSTEMS.contains(&sub_system) {
sub_systems.push(sub_system);
}
}
sub_systems
}
async fn preflight_config_intent(sub_system: Option<&str>) -> S3Result<()> {
for sub_system in config_preflight_subsystems(sub_system) {
preflight_dynamic_config_reload(sub_system).await?;
}
Ok(())
}
@@ -1869,13 +1895,13 @@ async fn wait_notify_config_intent(transition: Option<rustfs_notify::Notificatio
Ok(true)
}
async fn apply_non_notify_dynamic_subsystems(config: &ServerConfig) -> Vec<String> {
async fn reload_non_notify_dynamic_subsystems() -> Vec<String> {
let mut failures = Vec::new();
for sub_system in FULL_CONFIG_WORKER_SUBSYSTEMS {
if NOTIFY_SUB_SYSTEMS.contains(&sub_system) {
continue;
}
if apply_dynamic_config_for_subsystem(config, sub_system).await.is_err() {
if reload_dynamic_config_runtime_state(sub_system).await.is_err() {
failures.push(format!("local {sub_system}"));
warn!(
event = EVENT_CONFIG_WORKER_RELOAD_FAILED,
@@ -1904,7 +1930,6 @@ fn finish_config_reconciliation(errors: Vec<String>) -> S3Result<()> {
}
async fn reconcile_targeted_config(
config: ServerConfig,
sub_system: Option<String>,
storage_class_applied: bool,
notify_transition: Option<rustfs_notify::NotificationLifecycleTransition>,
@@ -1933,8 +1958,8 @@ async fn reconcile_targeted_config(
} else if let Some(sub_system) = sub_system.as_deref()
&& is_dynamic_config_subsystem(sub_system)
{
let config_applied = match apply_dynamic_config_for_subsystem(&config, sub_system).await {
Ok(applied) => applied,
let config_applied = match reload_dynamic_config_runtime_state(sub_system).await {
Ok(()) => true,
Err(_) => {
warn!(config_subsystem = sub_system, reason = "apply_failed", "Local config reload failed");
errors.push(format!("local {sub_system}"));
@@ -1958,10 +1983,7 @@ async fn reconcile_targeted_config(
Ok(config_applied)
}
async fn reconcile_full_config(
config: ServerConfig,
notify_transition: Option<rustfs_notify::NotificationLifecycleTransition>,
) -> S3Result<()> {
async fn reconcile_full_config(notify_transition: Option<rustfs_notify::NotificationLifecycleTransition>) -> S3Result<()> {
let mut errors = Vec::new();
if let Err(err) = wait_notify_config_intent(notify_transition).await {
warn!(error = %err, "Local notification config failed to converge");
@@ -1971,7 +1993,7 @@ async fn reconcile_full_config(
warn!(error = %err, "Peer storage-class reload failed");
errors.push(format!("peer {STORAGE_CLASS_SUB_SYS}"));
}
errors.extend(apply_non_notify_dynamic_subsystems(&config).await);
errors.extend(reload_non_notify_dynamic_subsystems().await);
if let Err(err) = signal_dynamic_config_reload_checked(NOTIFY_WEBHOOK_SUB_SYS).await {
warn!(error = %err, "Peer notification config reload failed");
errors.push("peer notify".to_string());
@@ -1983,6 +2005,90 @@ async fn reconcile_full_config(
finish_config_reconciliation(errors)
}
struct PersistedConfigTransaction {
previous_config: ServerConfig,
history_restore_id: Option<String>,
storage_class_applied: bool,
notify_transition: Option<rustfs_notify::NotificationLifecycleTransition>,
}
async fn persist_server_config_transaction(
config: ServerConfig,
prepared: PreparedRuntimeConfig,
snapshot: AdminServerConfigSnapshot,
sub_system: Option<&str>,
) -> S3Result<PersistedConfigTransaction> {
snapshot.ensure_lock_held().map_err(ApiError::from).map_err(S3Error::from)?;
let previous_config = snapshot.config.clone();
let history_restore_id = save_server_config_history_snapshot(&previous_config).await?;
if snapshot.is_lock_lost() {
cleanup_failed_config_history_snapshot(&history_restore_id).await;
return reject_lost_config_transaction(snapshot, "before persistence").await;
}
let persisted = match save_server_config_to_store(&config, &snapshot).await {
Ok(persisted) => persisted,
Err(err) => {
cleanup_failed_config_history_snapshot(&history_restore_id).await;
return Err(err);
}
};
let history_restore_id = if persisted {
Some(history_restore_id)
} else {
cleanup_failed_config_history_snapshot(&history_restore_id).await;
None
};
if snapshot.is_lock_lost() {
return reject_lost_config_transaction(snapshot, "after persistence").await;
}
let storage_class_applied = sub_system.is_none_or(|value| value == STORAGE_CLASS_SUB_SYS);
if storage_class_applied {
if let Err(err) = publish_prepared_config_snapshots(config.clone(), prepared) {
return Err(match history_restore_id.as_deref() {
Some(restore_id) => s3_error!(
InternalError,
"config persisted but runtime publish failed; recovery snapshot restoreId={}: {}",
restore_id,
err
),
None => err,
});
}
} else {
publish_server_config(config.clone());
}
let notify_transition = publish_notify_config_intent(&config, sub_system);
if snapshot.is_lock_lost() {
return reject_lost_config_transaction(snapshot, "while publishing runtime snapshots").await;
}
drop(snapshot);
Ok(PersistedConfigTransaction {
previous_config,
history_restore_id,
storage_class_applied,
notify_transition,
})
}
async fn commit_server_config_transaction(
config: ServerConfig,
prepared: PreparedRuntimeConfig,
snapshot: AdminServerConfigSnapshot,
sub_system: Option<String>,
) -> S3Result<bool> {
let transaction = persist_server_config_transaction(config, prepared, snapshot, sub_system.as_deref()).await?;
if sub_system.is_none() {
reconcile_full_config(transaction.notify_transition).await?;
return Ok(false);
}
reconcile_targeted_config(sub_system, transaction.storage_class_applied, transaction.notify_transition).await
}
pub struct GetConfigKVHandler {}
#[async_trait::async_trait]
@@ -2016,41 +2122,13 @@ impl Operation for SetConfigKVHandler {
validate_config_directives(&directives)?;
let sub_system = config_update_sub_system(&directives)?.map(str::to_owned);
let transaction_sub_system = sub_system.clone();
let config_store = object_store()?;
let config_applied = supervise_admin_mutation("config mutation", async move {
preflight_notify_config_intent(sub_system.as_deref()).await?;
let (config, storage_class_applied, notify_transition) =
with_admin_server_config_write_lock(config_store, move || async move {
let sub_system = transaction_sub_system.as_deref();
let previous_config = load_server_config_from_store_locked().await?;
let mut config = previous_config.clone();
apply_set_directives(&mut config, &directives)?;
let prepared = prepare_server_config(&config, sub_system).await?;
let history_restore_id = save_server_config_history_snapshot(&previous_config).await?;
if let Err(err) = save_server_config_to_store_locked(&config).await {
cleanup_failed_config_history_snapshot(&history_restore_id).await;
return Err(err);
}
if sub_system == Some(STORAGE_CLASS_SUB_SYS) {
publish_prepared_config_snapshots(config.clone(), prepared).map_err(|err| {
s3_error!(
InternalError,
"config persisted but runtime publish failed; recovery snapshot restoreId={}: {}",
history_restore_id,
err
)
})?;
} else {
publish_server_config(config.clone());
}
let notify_transition = publish_notify_config_intent(&config, sub_system);
Ok::<_, S3Error>((config, sub_system == Some(STORAGE_CLASS_SUB_SYS), notify_transition))
})
.await
.map_err(|err| s3_error!(InternalError, "failed to lock server config update: {}", err))??;
reconcile_targeted_config(config, sub_system, storage_class_applied, notify_transition).await
preflight_config_intent(sub_system.as_deref()).await?;
let snapshot = load_server_config_snapshot_from_store().await?;
let mut config = snapshot.config.clone();
apply_set_directives(&mut config, &directives)?;
let prepared = prepare_server_config(&config, sub_system.as_deref()).await?;
commit_server_config_transaction(config, prepared, snapshot, sub_system).await
})
.await?;
@@ -2072,41 +2150,13 @@ impl Operation for DelConfigKVHandler {
validate_config_directives(&directives)?;
let sub_system = config_update_sub_system(&directives)?.map(str::to_owned);
let transaction_sub_system = sub_system.clone();
let config_store = object_store()?;
let config_applied = supervise_admin_mutation("config mutation", async move {
preflight_notify_config_intent(sub_system.as_deref()).await?;
let (config, storage_class_applied, notify_transition) =
with_admin_server_config_write_lock(config_store, move || async move {
let sub_system = transaction_sub_system.as_deref();
let previous_config = load_server_config_from_store_locked().await?;
let mut config = previous_config.clone();
apply_delete_directives(&mut config, &directives);
let prepared = prepare_server_config(&config, sub_system).await?;
let history_restore_id = save_server_config_history_snapshot(&previous_config).await?;
if let Err(err) = save_server_config_to_store_locked(&config).await {
cleanup_failed_config_history_snapshot(&history_restore_id).await;
return Err(err);
}
if sub_system == Some(STORAGE_CLASS_SUB_SYS) {
publish_prepared_config_snapshots(config.clone(), prepared).map_err(|err| {
s3_error!(
InternalError,
"config persisted but runtime publish failed; recovery snapshot restoreId={}: {}",
history_restore_id,
err
)
})?;
} else {
publish_server_config(config.clone());
}
let notify_transition = publish_notify_config_intent(&config, sub_system);
Ok::<_, S3Error>((config, sub_system == Some(STORAGE_CLASS_SUB_SYS), notify_transition))
})
.await
.map_err(|err| s3_error!(InternalError, "failed to lock server config update: {}", err))??;
reconcile_targeted_config(config, sub_system, storage_class_applied, notify_transition).await
preflight_config_intent(sub_system.as_deref()).await?;
let snapshot = load_server_config_snapshot_from_store().await?;
let mut config = snapshot.config.clone();
apply_delete_directives(&mut config, &directives);
let prepared = prepare_server_config(&config, sub_system.as_deref()).await?;
commit_server_config_transaction(config, prepared, snapshot, sub_system).await
})
.await?;
@@ -2186,79 +2236,108 @@ impl Operation for RestoreConfigHistoryKVHandler {
.ok_or_else(|| s3_error!(InvalidRequest, "missing restoreId query parameter"))?;
let history = read_server_config_history(restore_id).await?;
let config = decode_config_history_snapshot(&history)?;
let prepared = prepare_server_config(&config, None).await?;
let config_store = object_store()?;
supervise_admin_mutation("config mutation", async move {
preflight_notify_config_intent(None).await?;
let persisted_config = config.clone();
preflight_config_intent(None).await?;
let snapshot = load_server_config_snapshot_from_store().await?;
let prepared = prepare_server_config(&config, None).await?;
let restored_config = config.clone();
let rollback_store = config_store.clone();
let (previous_config, recovery_restore_id, notify_transition) =
with_admin_server_config_write_lock(config_store, move || async move {
let previous_config = load_server_config_from_store_locked().await?;
let recovery_restore_id = save_server_config_history_snapshot(&previous_config).await?;
if let Err(err) = save_server_config_to_store_locked(&persisted_config).await {
cleanup_failed_config_history_snapshot(&recovery_restore_id).await;
return Err(err);
}
publish_prepared_config_snapshots(persisted_config.clone(), prepared).map_err(|err| {
s3_error!(
InternalError,
"config restore persisted but runtime publish failed; recovery snapshot restoreId={}: {}",
recovery_restore_id,
err
)
})?;
let notify_transition = publish_notify_config_intent(&persisted_config, None);
Ok::<_, S3Error>((previous_config, recovery_restore_id, notify_transition))
})
.await
.map_err(|err| s3_error!(InternalError, "failed to lock server config restore: {}", err))??;
let Err(restore_error) = reconcile_full_config(config, notify_transition).await else {
let transaction = persist_server_config_transaction(config, prepared, snapshot, None).await?;
let Err(restore_error) = reconcile_full_config(transaction.notify_transition).await else {
return Ok(());
};
let rollback_config = previous_config.clone();
let rollback_expected_config = restored_config.clone();
let rollback_transition = with_admin_server_config_write_lock(rollback_store, move || async move {
let current_config = load_server_config_from_store_locked().await?;
validate_restore_rollback_generation(&current_config, &rollback_expected_config)?;
let rollback_prepared = prepare_server_config(&rollback_config, None).await?;
commit_prepared_config(
rollback_config.clone(),
rollback_prepared,
save_server_config_to_store_locked(&rollback_config),
publish_prepared_config_snapshots,
)
.await?;
Ok::<_, S3Error>(publish_notify_config_intent(&rollback_config, None))
})
.await
.map_err(|err| s3_error!(InternalError, "failed to lock server config rollback: {}", err))?;
let rollback_transition = match rollback_transition {
Ok(transition) => transition,
let previous_config = transaction.previous_config;
let recovery_restore_id = transaction.history_restore_id;
let recovery_reference = recovery_restore_id.as_deref().unwrap_or("not-created");
let rollback_snapshot = match load_server_config_snapshot_from_store().await {
Ok(snapshot) => snapshot,
Err(rollback_error) => {
return Err(s3_error!(
InternalError,
"config restore failed: {}; automatic rollback failed: {}; recovery snapshot restoreId={}",
restore_error,
rollback_error,
recovery_restore_id
recovery_reference
));
}
};
if let Err(rollback_error) = reconcile_full_config(previous_config, rollback_transition).await {
if let Err(rollback_error) = validate_restore_rollback_generation(&rollback_snapshot.config, &restored_config) {
return Err(s3_error!(
InternalError,
"config restore failed: {}; automatic rollback failed: {}; recovery snapshot restoreId={}",
restore_error,
rollback_error,
recovery_reference
));
}
let rollback_prepared = prepare_server_config(&previous_config, None).await?;
rollback_snapshot
.ensure_lock_held()
.map_err(ApiError::from)
.map_err(S3Error::from)?;
if let Err(rollback_error) = save_server_config_to_store(&previous_config, &rollback_snapshot).await {
return Err(s3_error!(
InternalError,
"config restore failed: {}; automatic rollback failed: {}; recovery snapshot restoreId={}",
restore_error,
rollback_error,
recovery_reference
));
}
if rollback_snapshot.is_lock_lost() {
if let Err(rollback_error) =
reject_lost_config_transaction::<()>(rollback_snapshot, "after restore rollback persistence").await
{
return Err(s3_error!(
InternalError,
"config restore failed: {}; automatic rollback failed: {}; recovery snapshot restoreId={}",
restore_error,
rollback_error,
recovery_reference
));
}
return Err(s3_error!(InternalError, "restore rollback lock-loss handling returned unexpectedly"));
}
if let Err(rollback_error) = publish_prepared_config_snapshots(previous_config.clone(), rollback_prepared) {
return Err(s3_error!(
InternalError,
"config restore failed: {}; automatic rollback publish failed: {}; recovery snapshot restoreId={}",
restore_error,
rollback_error,
recovery_reference
));
}
let rollback_transition = publish_notify_config_intent(&previous_config, None);
if rollback_snapshot.is_lock_lost() {
if let Err(rollback_error) =
reject_lost_config_transaction::<()>(rollback_snapshot, "while publishing restore rollback").await
{
return Err(s3_error!(
InternalError,
"config restore failed: {}; automatic rollback failed: {}; recovery snapshot restoreId={}",
restore_error,
rollback_error,
recovery_reference
));
}
return Err(s3_error!(InternalError, "restore rollback lock-loss handling returned unexpectedly"));
}
drop(rollback_snapshot);
if let Err(rollback_error) = reconcile_full_config(rollback_transition).await {
return Err(s3_error!(
InternalError,
"config restore failed: {}; persisted rollback did not converge: {}; recovery snapshot restoreId={}",
restore_error,
rollback_error,
recovery_restore_id
recovery_reference
));
}
cleanup_failed_config_history_snapshot(&recovery_restore_id).await;
if let Some(recovery_restore_id) = recovery_restore_id.as_deref() {
cleanup_failed_config_history_snapshot(recovery_restore_id).await;
}
Err(s3_error!(InternalError, "config restore failed and was rolled back: {}", restore_error))
})
.await?;
@@ -2293,34 +2372,14 @@ impl Operation for SetConfigHandler {
}
validate_config_directives(&directives)?;
let mut config = ServerConfig::new();
apply_set_directives(&mut config, &directives)?;
let prepared = prepare_server_config(&config, None).await?;
let config_store = object_store()?;
supervise_admin_mutation("config mutation", async move {
preflight_notify_config_intent(None).await?;
let persisted_config = config.clone();
let notify_transition = with_admin_server_config_write_lock(config_store, move || async move {
let previous_config = load_server_config_from_store_locked().await?;
let history_restore_id = save_server_config_history_snapshot(&previous_config).await?;
if let Err(err) = save_server_config_to_store_locked(&persisted_config).await {
cleanup_failed_config_history_snapshot(&history_restore_id).await;
return Err(err);
}
publish_prepared_config_snapshots(persisted_config.clone(), prepared).map_err(|err| {
s3_error!(
InternalError,
"full config persisted but runtime publish failed; recovery snapshot restoreId={}: {}",
history_restore_id,
err
)
})?;
Ok::<_, S3Error>(publish_notify_config_intent(&persisted_config, None))
})
.await
.map_err(|err| s3_error!(InternalError, "failed to lock full server config update: {}", err))??;
reconcile_full_config(config, notify_transition).await
preflight_config_intent(None).await?;
let snapshot = load_server_config_snapshot_from_store().await?;
let mut config = ServerConfig::new();
apply_set_directives(&mut config, &directives)?;
let prepared = prepare_server_config(&config, None).await?;
commit_server_config_transaction(config, prepared, snapshot, None).await?;
Ok(())
})
.await?;
@@ -2332,56 +2391,21 @@ impl Operation for SetConfigHandler {
mod tests {
use super::*;
use serial_test::serial;
use std::sync::{Arc, Mutex};
use temp_env::with_vars;
#[tokio::test]
async fn prepared_config_commit_persists_before_publish() {
let events = Arc::new(Mutex::new(Vec::new()));
let persist_events = events.clone();
let publish_events = events.clone();
commit_prepared_config(
ServerConfig::new(),
PreparedRuntimeConfig::default(),
async move {
persist_events.lock().expect("persist events lock").push("persist");
Ok(())
},
move |_, _| {
publish_events.lock().expect("publish events lock").push("publish");
Ok(())
},
)
.await
.expect("prepared config commit");
assert_eq!(*events.lock().expect("result events lock"), ["persist", "publish"]);
}
#[tokio::test]
async fn prepared_config_commit_does_not_publish_after_persist_failure() {
let events = Arc::new(Mutex::new(Vec::new()));
let persist_events = events.clone();
let publish_events = events.clone();
let err = commit_prepared_config(
ServerConfig::new(),
PreparedRuntimeConfig::default(),
async move {
persist_events.lock().expect("persist events lock").push("persist");
Err(s3_error!(InternalError, "injected persistence failure"))
},
move |_, _| {
publish_events.lock().expect("publish events lock").push("publish");
Ok(())
},
)
.await
.expect_err("persistence failure must propagate");
assert_eq!(err.code(), &S3ErrorCode::InternalError);
assert_eq!(*events.lock().expect("result events lock"), ["persist"]);
#[test]
fn config_preflight_covers_each_runtime_worker_family() {
assert_eq!(config_preflight_subsystems(Some(SCANNER_SUB_SYS)), [SCANNER_SUB_SYS]);
assert_eq!(config_preflight_subsystems(Some(HEAL_SUB_SYS)), [HEAL_SUB_SYS]);
assert!(config_preflight_subsystems(Some("identity_openid")).is_empty());
assert_eq!(
config_preflight_subsystems(None),
[
NOTIFY_WEBHOOK_SUB_SYS,
rustfs_config::audit::AUDIT_WEBHOOK_SUB_SYS,
SCANNER_SUB_SYS
]
);
}
#[test]
@@ -2499,6 +2523,17 @@ mod tests {
assert!(!rendered_after_delete.contains("client_secret="));
}
#[test]
fn config_rendering_redacts_database_dsn() {
let entry = KV {
key: BASE_DSN_STRING.to_string(),
value: "postgres://user:password@db.example/database".to_string(),
hidden_if_empty: true,
};
assert_eq!(render_entry_value(&entry, true), REDACTED_VALUE);
}
#[test]
fn full_config_export_can_be_reapplied() {
crate::admin::storage_api::config::init_admin_config_defaults();
@@ -2621,6 +2656,36 @@ identity_openid config_url="https://issuer.example" client_id="console""#,
validate_config_directives(&directives).expect("scanner pacing keys should be supported");
}
#[test]
fn scanner_cycle_config_set_and_get_round_trip() {
let mut config = ServerConfig::new();
let directives = parse_config_directives("scanner cycle=61", false).expect("parse scanner cycle directive");
validate_config_directives(&directives).expect("scanner cycle should be supported");
apply_set_directives(&mut config, &directives).expect("apply scanner cycle directive");
assert_eq!(
config
.get_value(SCANNER_SUB_SYS, DEFAULT_DELIMITER)
.expect("scanner KVS should exist")
.lookup(SCANNER_CYCLE)
.as_deref(),
Some("61")
);
let rendered = String::from_utf8(
render_selected_config(
&config,
&ConfigSelector {
sub_system: SCANNER_SUB_SYS.to_string(),
target: Some(DEFAULT_DELIMITER.to_string()),
},
true,
)
.expect("render scanner cycle config"),
)
.expect("scanner config should be UTF-8");
assert!(rendered.contains("cycle=\"61\""));
}
#[test]
fn build_help_response_reports_scanner_delay() {
let response = build_help_response(Some(SCANNER_SUB_SYS), Some(SCANNER_DELAY), false).expect("scanner help response");
+28 -34
View File
@@ -13,6 +13,7 @@
// limitations under the License.
use crate::admin::auth::validate_admin_request;
use crate::admin::handlers::supervise_admin_mutation;
use crate::admin::router::{AdminOperation, Operation, S3Router};
use crate::admin::runtime_sources::{
current_app_context, current_federated_identity_service, current_object_store_handle_for_context,
@@ -20,8 +21,7 @@ use crate::admin::runtime_sources::{
};
use crate::admin::service::federated_identity::DefaultFederatedSessionBinding;
use crate::admin::storage_api::config::{
read_admin_config_without_migrate, read_admin_config_without_migrate_no_lock, save_admin_server_config_no_lock,
with_admin_server_config_write_lock,
read_admin_config_without_migrate, read_admin_server_config_snapshot, save_admin_server_config_snapshot,
};
use crate::auth::{check_key_valid, get_session_token};
use crate::server::{ADMIN_PREFIX, CONSOLE_PREFIX, MINIO_ADMIN_PREFIX, RemoteAddr};
@@ -381,17 +381,13 @@ impl Operation for PutOidcConfigHandler {
let provider_id = provider_id.to_owned();
let request: OidcConfigUpsertRequest = parse_json_body(&mut req).await?;
let store = oidc_config_store()?;
let lock_store = store.clone();
with_admin_server_config_write_lock(lock_store, move || async move {
let mut config = load_server_config_from_store_locked(store.clone()).await?;
let existing_secret = persisted_provider_secret(&config, &provider_id);
update_oidc_server_config(move |config| {
let existing_secret = persisted_provider_secret(config, &provider_id);
let provider_config = build_provider_config_from_upsert(&provider_id, request, existing_secret)?;
upsert_persisted_provider_config(&mut config, &provider_config);
save_server_config_to_store_locked(store, &config).await
upsert_persisted_provider_config(config, &provider_config);
Ok(())
})
.await
.map_err(|err| s3_error!(InternalError, "failed to lock server config update: {}", err))??;
.await?;
json_response(
StatusCode::OK,
@@ -422,15 +418,11 @@ impl Operation for DeleteOidcConfigHandler {
}
let provider_id = provider_id.to_owned();
let store = oidc_config_store()?;
let lock_store = store.clone();
with_admin_server_config_write_lock(lock_store, move || async move {
let mut config = load_server_config_from_store_locked(store.clone()).await?;
delete_persisted_provider_config(&mut config, &provider_id)?;
save_server_config_to_store_locked(store, &config).await
update_oidc_server_config(move |config| {
delete_persisted_provider_config(config, &provider_id)?;
Ok(())
})
.await
.map_err(|err| s3_error!(InternalError, "failed to lock server config update: {}", err))??;
.await?;
json_response(
StatusCode::OK,
@@ -916,21 +908,23 @@ fn oidc_config_store() -> S3Result<std::sync::Arc<crate::admin::storage_api::run
.ok_or_else(|| s3_error!(InternalError, "storage layer not initialized"))
}
async fn load_server_config_from_store_locked(
store: std::sync::Arc<crate::admin::storage_api::runtime::ECStore>,
) -> S3Result<ServerConfig> {
read_admin_config_without_migrate_no_lock(store)
.await
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("failed to load server config: {e}")))
}
async fn save_server_config_to_store_locked(
store: std::sync::Arc<crate::admin::storage_api::runtime::ECStore>,
config: &ServerConfig,
) -> S3Result<()> {
save_admin_server_config_no_lock(store, config)
.await
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("failed to save server config: {e}")))
async fn update_oidc_server_config<F>(modifier: F) -> S3Result<()>
where
F: FnOnce(&mut ServerConfig) -> S3Result<()> + Send + 'static,
{
let store = oidc_config_store()?;
supervise_admin_mutation("OIDC config update", async move {
let snapshot = read_admin_server_config_snapshot(store.clone())
.await
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("failed to load server config: {e}")))?;
let mut config = snapshot.config.clone();
modifier(&mut config)?;
save_admin_server_config_snapshot(store, &config, &snapshot)
.await
.map(|_| ())
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("failed to save server config: {e}")))
})
.await
}
fn is_env_managed_provider(provider_id: &str) -> bool {
+3 -1
View File
@@ -30,7 +30,9 @@ pub(crate) use crate::runtime_sources::{
current_replication_stats_handle_for_context, current_server_config_for_context, current_token_signing_key,
};
#[cfg(test)]
pub(crate) use crate::runtime_sources::{IamInterface, KmsInterface, ServerConfigInterface, StorageClassInterface};
pub(crate) use crate::runtime_sources::{
IamInterface, KmsInterface, NotificationSystemInterface, ServerConfigInterface, StorageClassInterface,
};
use rustfs_config::server_config::Config;
use rustfs_kms::KmsServiceManager;
use rustfs_tls_runtime::{GlobalPublishedOutboundTlsState, TlsGeneration};
+99 -2
View File
@@ -37,9 +37,12 @@ use rustfs_targets::config::{
};
use s3s::{S3Error, S3ErrorCode, S3Result};
use std::future::Future;
use tokio::sync::Mutex as AsyncMutex;
use tracing::warn;
use url::Url;
static RUNTIME_CONFIG_RELOAD_MUTEX: AsyncMutex<()> = AsyncMutex::const_new(());
pub fn is_dynamic_config_subsystem(sub_system: &str) -> bool {
NOTIFY_SUB_SYSTEMS.contains(&sub_system)
|| matches!(
@@ -393,6 +396,7 @@ pub async fn apply_dynamic_config_for_subsystem(config: &ServerConfig, sub_syste
}
pub async fn reload_dynamic_config_runtime_state_for_context(context: Option<&AppContext>, sub_system: &str) -> S3Result<()> {
let _reload_guard = RUNTIME_CONFIG_RELOAD_MUTEX.lock().await;
if sub_system == MODULE_SWITCHES_SIGNAL_SUBSYSTEM {
let store = resolve_runtime_config_store_for_context(context)?;
let notify_result = reconcile_event_notifier_from_store(store).await;
@@ -414,6 +418,14 @@ pub async fn reload_dynamic_config_runtime_state_for_context(context: Option<&Ap
warn!("peer reload_dynamic_config: failed to load server config for {sub_system}: {err}");
internal_error(format!("failed to load server config: {err}"))
})?;
if matches!(sub_system, SCANNER_SUB_SYS | HEAL_SUB_SYS) {
validate_server_config_for_context(context, &config, Some(sub_system)).await?;
// Scanner cycles refresh from the process-wide server config before
// each pass. Publish the same validated snapshot first so that refresh
// cannot overwrite this peer's dynamic scanner update with stale data.
publish_server_config_for_context(context, config.clone());
}
apply_dynamic_config_for_subsystem_for_context(context, &config, sub_system)
.await
.inspect_err(|_| {
@@ -463,6 +475,7 @@ where
}
pub async fn reload_runtime_config_snapshot_for_context(context: Option<&AppContext>) -> S3Result<()> {
let _reload_guard = RUNTIME_CONFIG_RELOAD_MUTEX.lock().await;
let store = resolve_runtime_config_store_for_context(context)?;
reload_runtime_config_snapshot_with(
@@ -641,13 +654,16 @@ pub async fn signal_config_snapshot_reload_checked() -> S3Result<()> {
#[cfg(test)]
mod tests {
use super::*;
use crate::admin::runtime_sources::{IamInterface, KmsInterface, ServerConfigInterface, StorageClassInterface};
use crate::admin::runtime_sources::{
IamInterface, KmsInterface, NotificationSystemInterface, ServerConfigInterface, StorageClassInterface,
};
use crate::admin::storage_api::bucket::metadata::{BUCKET_LIFECYCLE_CONFIG, BUCKET_REPLICATION_CONFIG};
use crate::admin::storage_api::config::{
read_admin_config_without_migrate, read_admin_config_without_migrate_no_lock, save_admin_server_config,
save_admin_server_config_no_lock, with_admin_server_config_write_lock,
};
use crate::admin::storage_api::error::StorageError;
use crate::admin::storage_api::runtime_sources::NotificationSys;
use crate::server::{
ModuleSwitchSource, PersistedModuleSwitches, current_module_switch_snapshot, is_event_notifier_reconciled,
refresh_persisted_module_switches_from, save_persisted_module_switches_to,
@@ -656,7 +672,7 @@ mod tests {
use crate::storage_api::startup::storage::{init_local_disks_with_instance_ctx, new_instance_ctx};
use rustfs_config::notify::{ENV_NOTIFY_WEBHOOK_ENABLE, ENV_NOTIFY_WEBHOOK_ENDPOINT, NOTIFY_WEBHOOK_SUB_SYS};
use rustfs_config::oidc::{OIDC_CLIENT_ID, OIDC_CONFIG_URL, OIDC_SCOPES};
use rustfs_config::{HEAL_SUB_SYS, SCANNER_SUB_SYS};
use rustfs_config::{ENV_SCANNER_CYCLE, HEAL_SUB_SYS, SCANNER_CYCLE, SCANNER_SUB_SYS};
use rustfs_config::{MQTT_BROKER, MQTT_QUEUE_DIR, MQTT_TOPIC, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR};
use rustfs_iam::{store::object::ObjectStore, sys::IamSys};
use rustfs_kms::KmsServiceManager;
@@ -675,6 +691,36 @@ mod tests {
const REAL_STORE_TEST_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30);
const REPLICATION_RELOAD_LABEL: &str = "replication";
struct TestNotificationSystemInterface(Arc<NotificationSys>);
impl NotificationSystemInterface for TestNotificationSystemInterface {
fn handle(&self) -> Option<Arc<NotificationSys>> {
Some(self.0.clone())
}
}
#[tokio::test]
async fn checked_scanner_reload_reports_unreachable_peer() {
let temp_dir = TempDir::new().expect("scanner reload temp dir");
let store = build_isolated_heterogeneous_store(temp_dir.path()).await;
let mut notification_system = NotificationSys::new(EndpointServerPools::default()).await;
notification_system.peer_clients.push(None);
let context = AppContext::new(
store,
Arc::new(TestIamInterface),
Arc::new(TestKmsInterface {
manager: Arc::new(KmsServiceManager::new()),
}),
)
.with_test_notification_system_interface(Arc::new(TestNotificationSystemInterface(Arc::new(notification_system))));
let err = signal_dynamic_config_reload_checked_for_context(Some(&context), SCANNER_SUB_SYS)
.await
.expect_err("scanner config must not report success when a peer did not reload it");
assert_eq!(err.code(), &S3ErrorCode::InternalError);
assert_eq!(err.message(), Some("1 peer(s) failed dynamic config reload for scanner"));
}
fn without_storage_class_env<R>(f: impl FnOnce() -> R) -> R {
temp_env::with_vars_unset(
[
@@ -778,6 +824,16 @@ mod tests {
config
}
fn scanner_server_config(cycle: &str) -> ServerConfig {
let mut config = ServerConfig::new();
let mut kvs = KVS::new();
kvs.insert(SCANNER_CYCLE.to_string(), cycle.to_string());
config
.0
.insert(SCANNER_SUB_SYS.to_string(), HashMap::from([(DEFAULT_DELIMITER.to_string(), kvs)]));
config
}
async fn build_isolated_heterogeneous_store(temp_dir: &Path) -> Arc<ECStore> {
let mut pools = Vec::new();
for (pool_index, drives_per_set) in [4, 2].into_iter().enumerate() {
@@ -1124,6 +1180,47 @@ mod tests {
.await;
}
#[tokio::test]
#[serial_test::serial(scanner_env)]
async fn peer_scanner_reload_publishes_server_snapshot_used_by_cycle_refresh() {
temp_env::async_with_vars([(ENV_SCANNER_CYCLE, None::<&str>)], async {
let fixture = runtime_config_reload_fixture().await;
let candidate = scanner_server_config("61");
save_admin_server_config(fixture.context.object_store(), &candidate)
.await
.expect("persist scanner config");
reload_dynamic_config_runtime_state_for_context(Some(&fixture.context), SCANNER_SUB_SYS)
.await
.expect("scanner peer reload should apply persisted config");
assert_eq!(fixture.server_set_calls.load(Ordering::SeqCst), 1);
let snapshot = fixture
.server_snapshot
.lock()
.expect("server config result lock")
.clone()
.expect("scanner peer reload should publish a server config snapshot");
assert_eq!(
snapshot
.get_value(SCANNER_SUB_SYS, DEFAULT_DELIMITER)
.expect("scanner defaults should be present")
.get(SCANNER_CYCLE),
"61",
"scanner peer reload must keep the process-wide config source current"
);
let runtime = rustfs_scanner::scanner_runtime_config_status();
assert_eq!(runtime.cycle_interval_seconds.value, 61);
assert_eq!(
runtime.cycle_interval_seconds.source,
rustfs_scanner::runtime_config::ScannerRuntimeConfigSource::Config
);
rustfs_scanner::apply_scanner_runtime_config(&ServerConfig::new()).expect("restore scanner runtime defaults");
})
.await;
}
#[tokio::test]
#[serial_test::serial(storage_class_env)]
async fn peer_full_reload_rejects_later_pool_without_publishing() {
+25 -5
View File
@@ -639,10 +639,13 @@ pub(crate) async fn read_admin_config_without_migrate(api: Arc<ECStore>) -> Resu
ecstore_config::com::read_config_without_migrate(api).await
}
#[cfg(test)]
pub(crate) async fn read_admin_config_without_migrate_no_lock(api: Arc<ECStore>) -> Result<rustfs_config::server_config::Config> {
ecstore_config::com::read_config_without_migrate_no_lock(api).await
}
pub(crate) type AdminServerConfigSnapshot = ecstore_config::com::ServerConfigSnapshot;
pub(crate) async fn save_admin_config(api: Arc<ECStore>, file: &str, data: Vec<u8>) -> Result<()> {
ecstore_config::com::save_config(api, file, data).await
}
@@ -656,6 +659,7 @@ pub(crate) async fn save_admin_server_config(api: Arc<ECStore>, cfg: &rustfs_con
ecstore_config::com::save_server_config(api, cfg).await
}
#[cfg(test)]
pub(crate) async fn save_admin_server_config_no_lock(
api: Arc<ECStore>,
cfg: &rustfs_config::server_config::Config,
@@ -663,6 +667,7 @@ pub(crate) async fn save_admin_server_config_no_lock(
ecstore_config::com::save_server_config_no_lock(api, cfg).await
}
#[cfg(test)]
pub(crate) async fn with_admin_server_config_write_lock<F, Fut, T>(api: Arc<ECStore>, operation: F) -> Result<T>
where
F: FnOnce() -> Fut + Send + 'static,
@@ -672,6 +677,18 @@ where
ecstore_config::com::with_server_config_write_lock(api, operation).await
}
pub(crate) async fn read_admin_server_config_snapshot(api: Arc<ECStore>) -> Result<AdminServerConfigSnapshot> {
ecstore_config::com::read_server_config_snapshot(api).await
}
pub(crate) async fn save_admin_server_config_snapshot(
api: Arc<ECStore>,
cfg: &rustfs_config::server_config::Config,
snapshot: &AdminServerConfigSnapshot,
) -> Result<bool> {
ecstore_config::com::save_server_config_snapshot(api, cfg, snapshot).await
}
pub(crate) fn init_admin_config_defaults() {
ecstore_config::init();
}
@@ -769,13 +786,16 @@ pub(crate) mod cluster {
}
pub(crate) mod config {
#[cfg(test)]
pub(crate) use super::save_admin_server_config;
pub(crate) use super::storageclass;
pub(crate) use super::{
RUSTFS_META_BUCKET, STORAGE_CLASS_SUB_SYS, delete_admin_config, init_admin_config_defaults, read_admin_config,
read_admin_config_without_migrate, read_admin_config_without_migrate_no_lock, save_admin_config,
save_admin_server_config_no_lock, with_admin_server_config_write_lock,
AdminServerConfigSnapshot, RUSTFS_META_BUCKET, STORAGE_CLASS_SUB_SYS, delete_admin_config, init_admin_config_defaults,
read_admin_config, read_admin_config_without_migrate, read_admin_server_config_snapshot, save_admin_config,
save_admin_server_config_snapshot,
};
#[cfg(test)]
pub(crate) use super::{
read_admin_config_without_migrate_no_lock, save_admin_server_config, save_admin_server_config_no_lock,
with_admin_server_config_write_lock,
};
}
+4 -2
View File
@@ -247,8 +247,10 @@ impl DefaultAdminUsecase {
Self::query_data_usage_info_with_store(store).await
}
/// Serve the last persisted scanner snapshot plus the in-memory overlay.
/// This request path must never trigger a live full-version listing
/// Serve the last persisted scanner snapshot plus an authoritative
/// single-process overlay. Distributed nodes must not promote their
/// process-local absolute counters to cluster-wide bucket totals.
/// This path never triggers a live full-version listing
/// (rustfs/backlog#1306); freshness is owned by the scanner.
pub(crate) async fn query_data_usage_info_with_store(store: Arc<ECStore>) -> AdminUsecaseResult<DataUsageInfo> {
let mut info = Self::map_data_usage_load_result(load_data_usage_from_backend_cached(store.clone()).await)?;
+224 -9
View File
@@ -47,7 +47,6 @@ use super::storage_api::bucket_usecase::contract::bucket::{
use super::storage_api::bucket_usecase::contract::list::{
ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsV2Info as StorageListObjectsV2Info, ListOperations as _,
};
use super::storage_api::bucket_usecase::data_usage::remove_bucket_usage_from_backend;
use super::storage_api::bucket_usecase::error::StorageError;
use super::storage_api::bucket_usecase::helper::{OperationHelper, spawn_background_with_context};
use super::storage_api::bucket_usecase::object_utils::to_s3s_etag;
@@ -120,13 +119,18 @@ use std::{
borrow::Cow,
collections::{HashMap, HashSet},
fmt::Display,
future::Future,
io::Write,
sync::Arc,
sync::{Arc, LazyLock},
};
use tracing::{debug, error, info, instrument, warn};
use tokio::sync::{Semaphore, TryAcquireError};
use tracing::{Instrument as _, debug, error, info, instrument, warn};
const LOG_COMPONENT_APP: &str = "app";
const LOG_SUBSYSTEM_BUCKET: &str = "bucket";
const BUCKET_OPERATION_CONCURRENCY: usize = 8;
static BUCKET_OPERATION_ADMISSION: LazyLock<Arc<Semaphore>> =
LazyLock::new(|| Arc::new(Semaphore::new(BUCKET_OPERATION_CONCURRENCY)));
use urlencoding::encode;
type ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
@@ -1087,6 +1091,40 @@ pub struct DefaultBucketUsecase {
context: Option<Arc<AppContext>>,
}
async fn await_bucket_usecase_on_fresh_task<T, F>(operation: &'static str, future: F) -> S3Result<T>
where
T: Send + 'static,
F: Future<Output = S3Result<T>> + Send + 'static,
{
await_bucket_usecase_on_fresh_task_with_admission(operation, Arc::clone(&BUCKET_OPERATION_ADMISSION), future).await
}
async fn await_bucket_usecase_on_fresh_task_with_admission<T, F>(
operation: &'static str,
admission: Arc<Semaphore>,
future: F,
) -> S3Result<T>
where
T: Send + 'static,
F: Future<Output = S3Result<T>> + Send + 'static,
{
let permit = admission.try_acquire_owned().map_err(|err| match err {
TryAcquireError::NoPermits => {
S3Error::with_message(S3ErrorCode::SlowDown, format!("{operation} concurrency limit reached; retry later"))
}
TryAcquireError::Closed => S3Error::with_message(S3ErrorCode::InternalError, format!("{operation} admission closed")),
})?;
tokio::spawn(
async move {
let _permit = permit;
future.await
}
.in_current_span(),
)
.await
.map_err(|err| S3Error::with_message(S3ErrorCode::InternalError, format!("{operation} task failed: {err}")))?
}
impl DefaultBucketUsecase {
#[cfg(test)]
pub fn without_context() -> Self {
@@ -1121,6 +1159,11 @@ impl DefaultBucketUsecase {
fields(start_time=?time::OffsetDateTime::now_utc())
)]
pub async fn execute_create_bucket(&self, req: S3Request<CreateBucketInput>) -> S3Result<S3Response<CreateBucketOutput>> {
let usecase = self.clone();
await_bucket_usecase_on_fresh_task("bucket creation", async move { usecase.execute_create_bucket_inner(req).await }).await
}
async fn execute_create_bucket_inner(&self, req: S3Request<CreateBucketInput>) -> S3Result<S3Response<CreateBucketOutput>> {
let helper = OperationHelper::new(&req, EventName::BucketCreated, S3Operation::CreateBucket);
let requester_is_owner = match req_info_ref(&req) {
Ok(r) => r.is_owner,
@@ -1179,7 +1222,15 @@ impl DefaultBucketUsecase {
}
#[instrument(level = "debug", skip(self, req))]
pub async fn execute_delete_bucket(&self, mut req: S3Request<DeleteBucketInput>) -> S3Result<S3Response<DeleteBucketOutput>> {
pub async fn execute_delete_bucket(&self, req: S3Request<DeleteBucketInput>) -> S3Result<S3Response<DeleteBucketOutput>> {
let usecase = self.clone();
await_bucket_usecase_on_fresh_task("bucket deletion", async move { usecase.execute_delete_bucket_inner(req).await }).await
}
async fn execute_delete_bucket_inner(
&self,
mut req: S3Request<DeleteBucketInput>,
) -> S3Result<S3Response<DeleteBucketOutput>> {
let helper = OperationHelper::new(&req, EventName::BucketRemoved, S3Operation::DeleteBucket);
let input = req.input.clone();
@@ -1218,12 +1269,8 @@ impl DefaultBucketUsecase {
// Invalidate bucket validation cache
crate::storage::invalidate_bucket_validation_cache(&input.bucket);
// Re-evaluate lifecycle/replication after bucket removal and keep an
// absent-bucket dirty marker until a complete usage snapshot is durable.
// Re-evaluate lifecycle and replication after bucket removal.
rustfs_scanner::record_scanner_maintenance_change(&input.bucket);
if let Err(err) = remove_bucket_usage_from_backend(store.clone(), &input.bucket).await {
warn!(bucket = %input.bucket, error = ?err, "failed to remove deleted bucket from data usage");
}
if let Err(err) = site_replication_delete_bucket_hook(&input.bucket, force).await {
warn!(bucket = %input.bucket, error = ?err, "site replication delete bucket hook failed");
@@ -2633,6 +2680,155 @@ mod tests {
ServerSideEncryptionConfiguration, Tag, Transition, TransitionStorageClass,
};
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::time::Duration;
use tokio::sync::Notify;
#[tokio::test]
async fn bucket_usecase_task_finishes_post_commit_hooks_after_parent_cancellation() {
let admission = Arc::new(Semaphore::new(1));
let committed = Arc::new(Notify::new());
let committed_wait = committed.notified();
let release_hook = Arc::new(Notify::new());
let finished = Arc::new(Notify::new());
let finished_wait = finished.notified();
let hook_ran = Arc::new(AtomicBool::new(false));
let committed_for_task = committed.clone();
let release_hook_for_task = release_hook.clone();
let finished_for_task = finished.clone();
let hook_ran_for_task = hook_ran.clone();
let parent = tokio::spawn(await_bucket_usecase_on_fresh_task_with_admission(
"test bucket operation",
admission.clone(),
async move {
committed_for_task.notify_one();
release_hook_for_task.notified().await;
hook_ran_for_task.store(true, Ordering::SeqCst);
finished_for_task.notify_one();
Ok(())
},
));
committed_wait.await;
parent.abort();
let _ = parent.await;
release_hook.notify_waiters();
tokio::time::timeout(Duration::from_secs(1), finished_wait)
.await
.expect("the post-commit hook should finish after request cancellation");
assert!(
hook_ran.load(Ordering::SeqCst),
"the fresh task must own both the storage mutation and its post-commit hooks"
);
assert_eq!(admission.available_permits(), 1);
}
#[tokio::test]
async fn saturated_bucket_usecase_admission_does_not_start_work() {
let admission = Arc::new(Semaphore::new(1));
let held = admission
.clone()
.acquire_owned()
.await
.expect("test admission should remain open");
let started = Arc::new(AtomicBool::new(false));
let started_for_task = started.clone();
let result = tokio::time::timeout(
Duration::from_secs(1),
await_bucket_usecase_on_fresh_task_with_admission("test bucket operation", admission, async move {
started_for_task.store(true, Ordering::SeqCst);
Ok(())
}),
)
.await
.expect("saturated admission must fail within the bounded timeout");
drop(held);
assert!(
!started.load(Ordering::SeqCst),
"saturated admission must not start a detached bucket operation"
);
assert_eq!(result.expect_err("saturated admission must fail fast").code(), &S3ErrorCode::SlowDown);
}
#[tokio::test]
async fn bucket_usecase_admission_rejects_excess_detached_tasks() {
let admission = Arc::new(Semaphore::new(BUCKET_OPERATION_CONCURRENCY));
let release = Arc::new(Semaphore::new(0));
let started = Arc::new(AtomicUsize::new(0));
let mut tasks = Vec::with_capacity(BUCKET_OPERATION_CONCURRENCY);
for _ in 0..BUCKET_OPERATION_CONCURRENCY {
let admission_for_task = admission.clone();
let release_for_task = release.clone();
let started_for_task = started.clone();
tasks.push(tokio::spawn(await_bucket_usecase_on_fresh_task_with_admission(
"test bucket operation",
admission_for_task,
async move {
started_for_task.fetch_add(1, Ordering::SeqCst);
let _release = release_for_task
.acquire()
.await
.expect("test release gate should remain open");
Ok(())
},
)));
}
tokio::time::timeout(Duration::from_secs(1), async {
while started.load(Ordering::SeqCst) != BUCKET_OPERATION_CONCURRENCY {
tokio::task::yield_now().await;
}
})
.await
.expect("all admitted operations should start");
let ninth_started = Arc::new(AtomicBool::new(false));
let ninth_started_for_task = ninth_started.clone();
let ninth_result = tokio::time::timeout(
Duration::from_secs(1),
await_bucket_usecase_on_fresh_task_with_admission("test bucket operation", admission.clone(), async move {
ninth_started_for_task.store(true, Ordering::SeqCst);
Ok(())
}),
)
.await
.expect("an excess bucket transaction must fail within the bounded timeout");
assert!(
!ninth_started.load(Ordering::SeqCst),
"the ninth bucket transaction must not start when admission is saturated"
);
assert_eq!(
ninth_result.expect_err("the ninth bucket transaction must fail fast").code(),
&S3ErrorCode::SlowDown
);
release.add_permits(BUCKET_OPERATION_CONCURRENCY);
for task in tasks {
task.await
.expect("bucket transaction parent should join")
.expect("bucket transaction should succeed");
}
await_bucket_usecase_on_fresh_task_with_admission("test bucket operation", admission, async { Ok(()) })
.await
.expect("admission must recover after active transactions finish");
}
#[tokio::test]
async fn bucket_usecase_task_maps_panics_to_internal_errors() {
async fn panicking_bucket_operation() -> S3Result<()> {
panic!("injected bucket task panic");
}
let result = await_bucket_usecase_on_fresh_task("test bucket operation", panicking_bucket_operation()).await;
let err = result.expect_err("a bucket task panic must be returned as an error");
assert_eq!(err.code(), &S3ErrorCode::InternalError);
assert!(err.message().is_some_and(|message| message.contains("task failed")));
}
fn build_request<T>(input: T, method: Method) -> S3Request<T> {
S3Request {
@@ -2662,6 +2858,25 @@ mod tests {
&rest[..end]
}
#[test]
fn bucket_create_and_delete_use_admitted_fresh_tasks() {
let source = include_str!("bucket_usecase.rs");
for (method, operation, inner_method) in [
("execute_create_bucket", "bucket creation", "execute_create_bucket_inner"),
("execute_delete_bucket", "bucket deletion", "execute_delete_bucket_inner"),
] {
let body = usecase_method_source(source, method);
assert!(
body.contains(&format!("await_bucket_usecase_on_fresh_task(\"{operation}\"")),
"{method} must use bounded detached-task admission"
);
assert!(
body.contains(&format!("{inner_method}(req).await")),
"{method} must run its mutation inside the admitted task"
);
}
}
#[test]
fn bucket_metadata_config_changes_notify_peer_metadata_reload() {
let source = include_str!("bucket_usecase.rs");
+8
View File
@@ -354,6 +354,14 @@ impl AppContext {
self.storage_class = storage_class;
self
}
pub(crate) fn with_test_notification_system_interface(
mut self,
notification_system: Arc<dyn NotificationSystemInterface>,
) -> Self {
self.notification_system = notification_system;
self
}
}
static APP_CONTEXT_SINGLETON: OnceLock<Arc<AppContext>> = OnceLock::new();
+1 -8
View File
@@ -110,13 +110,6 @@ pub(crate) mod data_usage {
)
.await;
}
pub(crate) async fn remove_bucket_usage_from_backend(
store: Arc<crate::storage::storage_api::ECStore>,
bucket: &str,
) -> Result<(), crate::storage::storage_api::StorageError> {
crate::storage::storage_api::ecstore_data_usage::remove_bucket_usage_from_backend(store, bucket).await
}
}
pub(crate) mod runtime {
@@ -956,7 +949,7 @@ pub(crate) mod bucket_usecase {
}
}
pub(crate) use super::{access, bucket, data_usage, error, helper, object_utils, request_context, s3_api};
pub(crate) use super::{access, bucket, error, helper, object_utils, request_context, s3_api};
pub(crate) use crate::storage::storage_api::{
ECStore, StorageObjectInfo, get_validated_store, process_lambda_configurations, process_queue_configurations,
process_topic_configurations, validate_list_object_unordered_with_delimiter,
+1 -1
View File
@@ -61,7 +61,7 @@ pub(crate) fn set_test_outbound_tls_generation(generation: u64) {
pub(crate) use context::install_test_app_context;
#[cfg(test)]
pub(crate) use context::{IamInterface, KmsInterface, ServerConfigInterface, StorageClassInterface};
pub(crate) use context::{IamInterface, KmsInterface, NotificationSystemInterface, ServerConfigInterface, StorageClassInterface};
pub(crate) fn current_app_context() -> Option<Arc<AppContext>> {
context::get_global_app_context()
+577 -17
View File
@@ -14,22 +14,26 @@
use crate::server::RPC_PREFIX;
use crate::storage::request_context::spawn_traced;
#[cfg(test)]
use crate::storage::storage_api::rpc_consumer::http_service::WALK_DIR_BODY_SHA256_QUERY;
use crate::storage::storage_api::rpc_consumer::http_service::{
DEFAULT_READ_BUFFER_SIZE, StorageDiskRpcExt as _, WALK_DIR_STREAM_COMPLETION_V1, WalkDirOptions, find_local_disk_by_ref,
verify_rpc_signature,
DEFAULT_READ_BUFFER_SIZE, NS_SCANNER_PROTOCOL_VERSION, NsScannerCapabilityResponse, StorageDiskRpcExt as _,
WALK_DIR_STREAM_COMPLETION_V1, WalkDirOptions, find_local_disk_by_ref, sign_ns_scanner_capability, verify_rpc_signature,
};
#[cfg(test)]
use crate::storage::storage_api::rpc_consumer::http_service::{
NS_SCANNER_BODY_SHA256_QUERY, NS_SCANNER_CAPABILITY_CHALLENGE_QUERY, NS_SCANNER_CYCLE_QUERY, NS_SCANNER_LEADER_EPOCH_QUERY,
NS_SCANNER_REQUEST_ID_QUERY, NS_SCANNER_SERVER_EPOCH_QUERY, NS_SCANNER_SESSION_ID_QUERY, NS_SCANNER_SESSION_SEQUENCE_QUERY,
WALK_DIR_BODY_SHA256_QUERY,
};
use crate::storage::storage_api::runtime_sources_consumer::runtime_sources;
use bytes::{Bytes, BytesMut};
use futures_util::{Stream, StreamExt, TryStreamExt, stream};
use http::{HeaderMap, Method, Request, Response, StatusCode, Uri};
use http::{HeaderMap, HeaderValue, Method, Request, Response, StatusCode, Uri};
use http_body_util::{BodyExt, Limited};
use hyper::body::Incoming;
use rustfs_config::MAX_ADMIN_REQUEST_BODY_SIZE;
use rustfs_io_metrics::internode_metrics::{
INTERNODE_OPERATION_PUT_FILE_STREAM, INTERNODE_OPERATION_READ_FILE_STREAM, INTERNODE_OPERATION_WALK_DIR,
INTERNODE_TRANSPORT_BACKEND_TCP_HTTP,
INTERNODE_OPERATION_NS_SCANNER, INTERNODE_OPERATION_PUT_FILE_STREAM, INTERNODE_OPERATION_READ_FILE_STREAM,
INTERNODE_OPERATION_WALK_DIR, INTERNODE_TRANSPORT_BACKEND_TCP_HTTP,
};
use rustfs_utils::net::bytes_stream;
use s3s::Body;
@@ -39,11 +43,12 @@ use serde_urlencoded::from_bytes;
use sha2::{Digest, Sha256};
use std::future::Future;
use std::pin::Pin;
use std::sync::LazyLock;
use std::task::{Context, Poll};
use std::time::Instant;
use std::time::{Duration, Instant};
use tokio::io::{self, AsyncWriteExt};
use tokio::sync::oneshot;
use tokio_util::io::ReaderStream;
use tokio_util::{io::ReaderStream, sync::CancellationToken};
use tower::Service;
use tracing::{error, warn};
@@ -52,6 +57,7 @@ type RpcErrorResponse = Box<Response<Body>>;
const LOG_COMPONENT_INTERNODE_RPC: &str = "internode_rpc";
const LOG_SUBSYSTEM_FILE_TRANSFER: &str = "file_transfer";
const LOG_SUBSYSTEM_DIRECTORY_WALK: &str = "directory_walk";
const LOG_SUBSYSTEM_NAMESPACE_SCANNER: &str = "namespace_scanner";
const LOG_SUBSYSTEM_ROUTING: &str = "routing";
const EVENT_RPC_REQUEST_REJECTED: &str = "rpc_request_rejected";
const EVENT_RPC_REQUEST_FAILED: &str = "rpc_request_failed";
@@ -60,6 +66,10 @@ const RPC_OPERATION_UNKNOWN: &str = "unknown";
const READ_FILE_STREAM_PATH: &str = "/rustfs/rpc/read_file_stream";
const PUT_FILE_STREAM_PATH: &str = "/rustfs/rpc/put_file_stream";
const WALK_DIR_PATH: &str = "/rustfs/rpc/walk_dir";
const NS_SCANNER_PATH: &str = "/rustfs/rpc/ns_scanner";
const NS_SCANNER_REQUEST_BODY_TIMEOUT: Duration = Duration::from_secs(15);
const NS_SCANNER_STREAM_BUFFER_SIZE: usize = 64 * 1024;
static NS_SCANNER_SERVER_EPOCH: LazyLock<uuid::Uuid> = LazyLock::new(uuid::Uuid::new_v4);
macro_rules! log_internode_rpc_response_failure {
($status:expr, $rpc_path:expr, $method:expr, $operation:expr, $reason:expr, $result:expr, Some(($context_key:expr, $context_value:expr)), Some($error_text:expr)) => {{
@@ -247,6 +257,34 @@ struct WalkDirQuery {
walk_dir_body_sha256: Option<String>,
}
#[derive(Debug, Default, serde::Deserialize)]
#[serde(deny_unknown_fields)]
struct NsScannerQuery {
disk: String,
ns_scanner_request_id: uuid::Uuid,
ns_scanner_server_epoch: uuid::Uuid,
ns_scanner_session_id: uuid::Uuid,
ns_scanner_session_sequence: u64,
ns_scanner_cycle: u64,
ns_scanner_leader_epoch: u64,
ns_scanner_body_sha256: Option<String>,
}
#[derive(Debug, Default, serde::Deserialize)]
#[serde(deny_unknown_fields)]
struct NsScannerCapabilityQuery {
ns_scanner_protocol: Option<u16>,
ns_scanner_challenge: Option<uuid::Uuid>,
}
fn verify_ns_scanner_body_digest(query: &NsScannerQuery, body: &[u8]) -> bool {
let Some(expected) = query.ns_scanner_body_sha256.as_deref() else {
return false;
};
let actual = hex_simd::encode_to_string(Sha256::digest(body), hex_simd::AsciiCase::Lower);
expected == actual
}
fn supports_walk_dir_stream_completion(query: &WalkDirQuery) -> bool {
query.walk_dir_stream_completion.as_deref() == Some(WALK_DIR_STREAM_COMPLETION_V1)
}
@@ -322,6 +360,15 @@ async fn handle_internode_rpc(req: Request<Incoming>) -> Response<Body> {
let response = match (method, path) {
(Method::GET, READ_FILE_STREAM_PATH) | (Method::HEAD, READ_FILE_STREAM_PATH) => handle_read_file(req).await,
(Method::GET, WALK_DIR_PATH) | (Method::HEAD, WALK_DIR_PATH) => handle_walk_dir(req).await,
(Method::GET, NS_SCANNER_PATH) => match parse_query::<NsScannerCapabilityQuery>(&req) {
Ok(query) if query.ns_scanner_protocol == Some(NS_SCANNER_PROTOCOL_VERSION) => match query.ns_scanner_challenge {
Some(challenge) if !challenge.is_nil() => ns_scanner_capability_response(challenge),
Some(_) | None => response_with_status(StatusCode::BAD_REQUEST, "namespace scanner challenge is invalid"),
},
Ok(_) => response_with_status(StatusCode::UPGRADE_REQUIRED, "namespace scanner protocol is unsupported"),
Err(response) => *response,
},
(Method::POST, NS_SCANNER_PATH) => handle_ns_scanner(req).await,
(Method::PUT, PUT_FILE_STREAM_PATH) => handle_put_file(req).await,
_ => response_with_status(StatusCode::NOT_FOUND, "internode rpc route not found"),
};
@@ -346,6 +393,7 @@ fn internode_http_operation(path: &str) -> Option<&'static str> {
READ_FILE_STREAM_PATH => Some(INTERNODE_OPERATION_READ_FILE_STREAM),
PUT_FILE_STREAM_PATH => Some(INTERNODE_OPERATION_PUT_FILE_STREAM),
WALK_DIR_PATH => Some(INTERNODE_OPERATION_WALK_DIR),
NS_SCANNER_PATH => Some(INTERNODE_OPERATION_NS_SCANNER),
_ => None,
}
}
@@ -358,6 +406,64 @@ fn record_internode_rpc_error(operation: Option<&'static str>) {
}
}
fn ns_scanner_capability_response(challenge: uuid::Uuid) -> Response<Body> {
let server_epoch = *NS_SCANNER_SERVER_EPOCH;
let proof = match sign_ns_scanner_capability(challenge, server_epoch) {
Ok(proof) => proof,
Err(err) => {
error!(
event = EVENT_RPC_REQUEST_FAILED,
component = LOG_COMPONENT_INTERNODE_RPC,
subsystem = LOG_SUBSYSTEM_NAMESPACE_SCANNER,
operation = INTERNODE_OPERATION_NS_SCANNER,
result = "failed",
status_code = StatusCode::UPGRADE_REQUIRED.as_u16(),
rpc_path = NS_SCANNER_PATH,
method = %Method::GET,
reason = "capability_authentication_unavailable",
error = %err,
"internode rpc request failed"
);
return response_with_status(StatusCode::UPGRADE_REQUIRED, "namespace scanner RPC authentication is unavailable");
}
};
let body = match rmp_serde::to_vec_named(&NsScannerCapabilityResponse {
version: NS_SCANNER_PROTOCOL_VERSION,
server_epoch,
proof,
}) {
Ok(body) => body,
Err(err) => {
error!(
event = EVENT_RPC_REQUEST_FAILED,
component = LOG_COMPONENT_INTERNODE_RPC,
subsystem = LOG_SUBSYSTEM_NAMESPACE_SCANNER,
operation = INTERNODE_OPERATION_NS_SCANNER,
result = "failed",
status_code = StatusCode::INTERNAL_SERVER_ERROR.as_u16(),
rpc_path = NS_SCANNER_PATH,
method = %Method::GET,
reason = "capability_response_encode_failed",
error = %err,
"internode rpc request failed"
);
return response_with_status(
StatusCode::INTERNAL_SERVER_ERROR,
"namespace scanner capability response encoding failed",
);
}
};
let mut response = Response::new(Body::from(Bytes::from(body)));
response
.headers_mut()
.insert(http::header::CONTENT_TYPE, HeaderValue::from_static("application/msgpack"));
response
}
fn ns_scanner_server_epoch_matches(server_epoch: uuid::Uuid) -> bool {
server_epoch == *NS_SCANNER_SERVER_EPOCH
}
fn verify_internode_rpc_signature(uri: &Uri, method: &Method, headers: &HeaderMap) -> Result<(), RpcErrorResponse> {
if method == Method::HEAD {
return Ok(());
@@ -602,6 +708,304 @@ async fn handle_walk_dir(req: Request<Incoming>) -> Response<Body> {
.expect("failed to build walk dir response")
}
async fn handle_ns_scanner(req: Request<Incoming>) -> Response<Body> {
let query = match parse_query::<NsScannerQuery>(&req) {
Ok(query) => query,
Err(response) => return *response,
};
if !ns_scanner_server_epoch_matches(query.ns_scanner_server_epoch) {
warn!(
event = EVENT_RPC_REQUEST_REJECTED,
component = LOG_COMPONENT_INTERNODE_RPC,
subsystem = LOG_SUBSYSTEM_NAMESPACE_SCANNER,
operation = INTERNODE_OPERATION_NS_SCANNER,
result = "rejected",
status_code = StatusCode::CONFLICT.as_u16(),
rpc_path = NS_SCANNER_PATH,
method = %Method::POST,
reason = "server_epoch_mismatch",
"internode rpc request rejected"
);
return response_with_status(StatusCode::CONFLICT, "namespace scanner server epoch is stale");
}
let Some(disk) = find_local_disk_by_ref(&query.disk).await else {
warn!(
event = EVENT_RPC_REQUEST_REJECTED,
component = LOG_COMPONENT_INTERNODE_RPC,
subsystem = LOG_SUBSYSTEM_NAMESPACE_SCANNER,
operation = INTERNODE_OPERATION_NS_SCANNER,
result = "rejected",
status_code = StatusCode::BAD_REQUEST.as_u16(),
rpc_path = NS_SCANNER_PATH,
method = %Method::POST,
reason = "disk_not_found",
disk = %query.disk,
"internode rpc request rejected"
);
return response_with_status(StatusCode::BAD_REQUEST, "disk not found");
};
if let Err(err) = rustfs_scanner::preflight_remote_scanner_request(
disk.as_ref(),
query.ns_scanner_session_id,
query.ns_scanner_cycle,
query.ns_scanner_leader_epoch,
query.ns_scanner_session_sequence,
) {
let (status, reason, message) = remote_scanner_claim_rejection(&err);
warn!(
event = EVENT_RPC_REQUEST_REJECTED,
component = LOG_COMPONENT_INTERNODE_RPC,
subsystem = LOG_SUBSYSTEM_NAMESPACE_SCANNER,
operation = INTERNODE_OPERATION_NS_SCANNER,
result = "rejected",
status_code = status.as_u16(),
rpc_path = NS_SCANNER_PATH,
method = %Method::POST,
reason,
disk = %query.disk,
error = %err,
"internode rpc request rejected"
);
return response_with_status(status, message);
}
if let Err(err) =
rustfs_scanner::validate_remote_scanner_request_fence(query.ns_scanner_cycle, query.ns_scanner_leader_epoch).await
{
warn!(
event = EVENT_RPC_REQUEST_REJECTED,
component = LOG_COMPONENT_INTERNODE_RPC,
subsystem = LOG_SUBSYSTEM_NAMESPACE_SCANNER,
operation = INTERNODE_OPERATION_NS_SCANNER,
result = "rejected",
status_code = StatusCode::CONFLICT.as_u16(),
rpc_path = NS_SCANNER_PATH,
method = %Method::POST,
reason = "scanner_cycle_mismatch",
disk = %query.disk,
error = %err,
"internode rpc request rejected"
);
return response_with_status(StatusCode::CONFLICT, "namespace scanner cycle does not match persisted state");
}
// Acquire the per-disk permit before reading the request body. This bounds
// slow or duplicated authenticated uploads to one request per physical disk;
// dropping the request on timeout releases the permit without consuming its
// replay sequence.
let admission = match rustfs_scanner::admit_remote_scanner_request(disk.as_ref()) {
Ok(admission) => admission,
Err(rustfs_scanner::ScannerError::RemoteDiskBusy) => {
warn!(
event = EVENT_RPC_REQUEST_REJECTED,
component = LOG_COMPONENT_INTERNODE_RPC,
subsystem = LOG_SUBSYSTEM_NAMESPACE_SCANNER,
operation = INTERNODE_OPERATION_NS_SCANNER,
result = "rejected",
status_code = StatusCode::TOO_MANY_REQUESTS.as_u16(),
rpc_path = NS_SCANNER_PATH,
method = %Method::POST,
reason = "disk_scan_already_active",
disk = %query.disk,
"internode rpc request rejected"
);
return response_with_status(StatusCode::TOO_MANY_REQUESTS, "namespace scanner disk is already active");
}
Err(err) => {
let (status, reason, message) = remote_scanner_claim_rejection(&err);
warn!(
event = EVENT_RPC_REQUEST_REJECTED,
component = LOG_COMPONENT_INTERNODE_RPC,
subsystem = LOG_SUBSYSTEM_NAMESPACE_SCANNER,
operation = INTERNODE_OPERATION_NS_SCANNER,
result = "rejected",
status_code = status.as_u16(),
rpc_path = NS_SCANNER_PATH,
method = %Method::POST,
reason,
disk = %query.disk,
error = %err,
"internode rpc request rejected"
);
return response_with_status(status, message);
}
};
let body = match tokio::time::timeout(
NS_SCANNER_REQUEST_BODY_TIMEOUT,
Limited::new(req.into_body(), rustfs_scanner::NS_SCANNER_MAX_REQUEST_BODY_SIZE).collect(),
)
.await
{
Ok(Ok(body)) => body.to_bytes(),
Ok(Err(err)) => {
log_internode_rpc_response_failure!(
StatusCode::PAYLOAD_TOO_LARGE,
NS_SCANNER_PATH,
&Method::POST,
Some(INTERNODE_OPERATION_NS_SCANNER),
"request_body_read_failed",
"rejected",
Some(("disk", query.disk.as_str())),
Some(&err)
);
return response_with_status(StatusCode::PAYLOAD_TOO_LARGE, "namespace scanner request body is too large");
}
Err(_) => {
warn!(
event = EVENT_RPC_REQUEST_REJECTED,
component = LOG_COMPONENT_INTERNODE_RPC,
subsystem = LOG_SUBSYSTEM_NAMESPACE_SCANNER,
operation = INTERNODE_OPERATION_NS_SCANNER,
result = "rejected",
status_code = StatusCode::REQUEST_TIMEOUT.as_u16(),
rpc_path = NS_SCANNER_PATH,
method = %Method::POST,
reason = "request_body_timeout",
disk = %query.disk,
"internode rpc request rejected"
);
return response_with_status(StatusCode::REQUEST_TIMEOUT, "namespace scanner request body timed out");
}
};
if !verify_ns_scanner_body_digest(&query, &body) {
warn!(
event = EVENT_RPC_REQUEST_REJECTED,
component = LOG_COMPONENT_INTERNODE_RPC,
subsystem = LOG_SUBSYSTEM_NAMESPACE_SCANNER,
operation = INTERNODE_OPERATION_NS_SCANNER,
result = "rejected",
status_code = StatusCode::FORBIDDEN.as_u16(),
rpc_path = NS_SCANNER_PATH,
method = %Method::POST,
reason = "request_body_digest_mismatch",
disk = %query.disk,
"internode rpc request rejected"
);
return response_with_status(StatusCode::FORBIDDEN, "invalid request body digest");
}
let request = match rustfs_scanner::decode_remote_scanner_request(&body) {
Ok(request) => request,
Err(err) => {
warn!(
event = EVENT_RPC_REQUEST_REJECTED,
component = LOG_COMPONENT_INTERNODE_RPC,
subsystem = LOG_SUBSYSTEM_NAMESPACE_SCANNER,
operation = INTERNODE_OPERATION_NS_SCANNER,
result = "rejected",
status_code = StatusCode::BAD_REQUEST.as_u16(),
rpc_path = NS_SCANNER_PATH,
method = %Method::POST,
reason = "request_body_decode_failed",
disk = %query.disk,
error = %err,
"internode rpc request rejected"
);
return response_with_status(StatusCode::BAD_REQUEST, "invalid namespace scanner request");
}
};
if !rustfs_scanner::remote_scanner_request_matches_envelope(
&request,
query.ns_scanner_request_id,
query.ns_scanner_server_epoch,
query.ns_scanner_session_id,
query.ns_scanner_session_sequence,
query.ns_scanner_cycle,
query.ns_scanner_leader_epoch,
) {
warn!(
event = EVENT_RPC_REQUEST_REJECTED,
component = LOG_COMPONENT_INTERNODE_RPC,
subsystem = LOG_SUBSYSTEM_NAMESPACE_SCANNER,
operation = INTERNODE_OPERATION_NS_SCANNER,
result = "rejected",
status_code = StatusCode::FORBIDDEN.as_u16(),
rpc_path = NS_SCANNER_PATH,
method = %Method::POST,
reason = "request_envelope_mismatch",
disk = %query.disk,
"internode rpc request rejected"
);
return response_with_status(StatusCode::FORBIDDEN, "namespace scanner request envelope mismatch");
}
if let Err(err) = rustfs_scanner::claim_remote_scanner_request(
disk.as_ref(),
query.ns_scanner_session_id,
query.ns_scanner_cycle,
query.ns_scanner_leader_epoch,
query.ns_scanner_session_sequence,
) {
let (status, reason, message) = remote_scanner_claim_rejection(&err);
warn!(
event = EVENT_RPC_REQUEST_REJECTED,
component = LOG_COMPONENT_INTERNODE_RPC,
subsystem = LOG_SUBSYSTEM_NAMESPACE_SCANNER,
operation = INTERNODE_OPERATION_NS_SCANNER,
result = "rejected",
status_code = status.as_u16(),
rpc_path = NS_SCANNER_PATH,
method = %Method::POST,
reason,
disk = %query.disk,
error = %err,
"internode rpc request rejected"
);
return response_with_status(status, message);
}
let metrics = runtime_sources::current_internode_metrics();
metrics
.record_incoming_request_for_operation_and_backend(INTERNODE_OPERATION_NS_SCANNER, INTERNODE_TRANSPORT_BACKEND_TCP_HTTP);
metrics.record_recv_bytes_for_operation_and_backend(
INTERNODE_OPERATION_NS_SCANNER,
INTERNODE_TRANSPORT_BACKEND_TCP_HTTP,
body.len(),
);
let log_disk = query.disk;
let response_body = ns_scanner_response_body(move |writer, disconnect| async move {
let _admission = admission;
rustfs_scanner::serve_remote_scanner_request(disk, request, writer, disconnect)
.await
.map_err(|err| {
error!(
event = EVENT_RPC_BACKGROUND_TASK_FAILED,
component = LOG_COMPONENT_INTERNODE_RPC,
subsystem = LOG_SUBSYSTEM_NAMESPACE_SCANNER,
operation = INTERNODE_OPERATION_NS_SCANNER,
result = "failed",
disk = %log_disk,
error = %err,
"internode rpc background task failed"
);
io::Error::other("remote namespace scanner failed")
})
});
let mut response = Response::new(response_body);
response
.headers_mut()
.insert(http::header::CONTENT_TYPE, HeaderValue::from_static("application/msgpack"));
response
}
fn remote_scanner_claim_rejection(error: &rustfs_scanner::ScannerError) -> (StatusCode, &'static str, &'static str) {
match error {
rustfs_scanner::ScannerError::RemoteRequestReplay => {
(StatusCode::CONFLICT, "request_replay", "namespace scanner request was already accepted")
}
rustfs_scanner::ScannerError::RemoteReplayCapacity => (
StatusCode::TOO_MANY_REQUESTS,
"replay_capacity",
"namespace scanner request capacity is temporarily exhausted",
),
_ => (
StatusCode::SERVICE_UNAVAILABLE,
"replay_state_unavailable",
"namespace scanner replay state is unavailable",
),
}
}
fn walk_dir_response_body<F, Fut>(propagate_completion_errors: bool, producer: F) -> Body
where
F: FnOnce(tokio::io::DuplexStream) -> Fut + Send + 'static,
@@ -656,6 +1060,52 @@ where
)
}
fn ns_scanner_response_body<F, Fut>(producer: F) -> Body
where
F: FnOnce(tokio::io::DuplexStream, CancellationToken) -> Fut + Send + 'static,
Fut: Future<Output = io::Result<()>> + Send + 'static,
{
let (reader, writer) = tokio::io::duplex(NS_SCANNER_STREAM_BUFFER_SIZE);
let (mut completion_tx, completion_rx) = oneshot::channel();
let disconnect = CancellationToken::new();
spawn_traced(async move {
let producer = producer(writer, disconnect.clone());
tokio::pin!(producer);
tokio::select! {
biased;
result = &mut producer => {
let _ = completion_tx.send(result);
}
_ = completion_tx.closed() => {
disconnect.cancel();
let _ = producer.await;
}
}
});
let metrics = runtime_sources::current_internode_metrics();
let stream = ReaderStream::with_capacity(reader, NS_SCANNER_STREAM_BUFFER_SIZE).map_ok(move |bytes| {
metrics.record_sent_bytes_for_operation_and_backend(
INTERNODE_OPERATION_NS_SCANNER,
INTERNODE_TRANSPORT_BACKEND_TCP_HTTP,
bytes.len(),
);
bytes
});
let completion = stream::once(async move {
match completion_rx.await {
Ok(Ok(())) => None,
Ok(Err(err)) => Some(Err(err)),
Err(err) => Some(Err(io::Error::other(format!(
"remote namespace scanner task ended without a result: {err}"
)))),
}
})
.filter_map(std::future::ready);
Body::from(StreamingBlob::wrap(stream.chain(completion)))
}
async fn handle_put_file(req: Request<Incoming>) -> Response<Body> {
let method = req.method().clone();
let path = req.uri().path().to_string();
@@ -805,6 +1255,7 @@ fn response_with_status(status: StatusCode, message: impl Into<String>) -> Respo
fn internode_rpc_subsystem(operation: Option<&'static str>) -> &'static str {
match operation {
Some(INTERNODE_OPERATION_WALK_DIR) => LOG_SUBSYSTEM_DIRECTORY_WALK,
Some(INTERNODE_OPERATION_NS_SCANNER) => LOG_SUBSYSTEM_NAMESPACE_SCANNER,
Some(INTERNODE_OPERATION_READ_FILE_STREAM | INTERNODE_OPERATION_PUT_FILE_STREAM) => LOG_SUBSYSTEM_FILE_TRANSFER,
_ => LOG_SUBSYSTEM_ROUTING,
}
@@ -828,19 +1279,23 @@ fn put_file_stage_error_message(stage: &str, query: &PutFileQuery, err: &dyn std
#[cfg(test)]
mod tests {
use super::{
LOG_SUBSYSTEM_DIRECTORY_WALK, LOG_SUBSYSTEM_FILE_TRANSFER, LOG_SUBSYSTEM_ROUTING, PUT_FILE_STREAM_PATH, PutFileQuery,
LOG_SUBSYSTEM_DIRECTORY_WALK, LOG_SUBSYSTEM_FILE_TRANSFER, LOG_SUBSYSTEM_NAMESPACE_SCANNER, LOG_SUBSYSTEM_ROUTING,
NS_SCANNER_BODY_SHA256_QUERY, NS_SCANNER_CAPABILITY_CHALLENGE_QUERY, NS_SCANNER_CYCLE_QUERY,
NS_SCANNER_LEADER_EPOCH_QUERY, NS_SCANNER_PATH, NS_SCANNER_REQUEST_ID_QUERY, NS_SCANNER_SERVER_EPOCH_QUERY,
NS_SCANNER_SESSION_ID_QUERY, NS_SCANNER_SESSION_SEQUENCE_QUERY, NsScannerQuery, PUT_FILE_STREAM_PATH, PutFileQuery,
READ_FILE_STREAM_PATH, WALK_DIR_BODY_SHA256_QUERY, WALK_DIR_PATH, WalkDirQuery, append_walk_dir_completion,
internode_http_operation, internode_rpc_subsystem, is_internode_rpc_path, put_body_size_mismatch,
put_file_stage_error_message, read_file_body_stream, supports_walk_dir_stream_completion,
validate_walk_dir_completion_request, verify_internode_rpc_signature, verify_walk_dir_body_digest,
walk_dir_response_body, write_body_chunks_to_writer,
internode_http_operation, internode_rpc_subsystem, is_internode_rpc_path, ns_scanner_response_body,
ns_scanner_server_epoch_matches, put_body_size_mismatch, put_file_stage_error_message, read_file_body_stream,
remote_scanner_claim_rejection, supports_walk_dir_stream_completion, validate_walk_dir_completion_request,
verify_internode_rpc_signature, verify_ns_scanner_body_digest, verify_walk_dir_body_digest, walk_dir_response_body,
write_body_chunks_to_writer,
};
use bytes::Bytes;
use http::{HeaderMap, Method, StatusCode, Uri};
use http_body_util::BodyExt;
use rustfs_io_metrics::internode_metrics::{
INTERNODE_OPERATION_PUT_FILE_STREAM, INTERNODE_OPERATION_READ_FILE_STREAM, INTERNODE_OPERATION_WALK_DIR,
global_internode_metrics,
INTERNODE_OPERATION_NS_SCANNER, INTERNODE_OPERATION_PUT_FILE_STREAM, INTERNODE_OPERATION_READ_FILE_STREAM,
INTERNODE_OPERATION_WALK_DIR, global_internode_metrics,
};
use sha2::Digest as _;
use tokio::io;
@@ -862,6 +1317,7 @@ mod tests {
fn internode_rpc_path_matches_rpc_prefix() {
assert!(is_internode_rpc_path("/rustfs/rpc/read_file_stream"));
assert!(is_internode_rpc_path("/rustfs/rpc/walk_dir"));
assert!(is_internode_rpc_path("/rustfs/rpc/ns_scanner"));
assert!(!is_internode_rpc_path("/rustfs/admin/v3/info"));
}
@@ -873,16 +1329,37 @@ mod tests {
);
assert_eq!(internode_http_operation(PUT_FILE_STREAM_PATH), Some(INTERNODE_OPERATION_PUT_FILE_STREAM));
assert_eq!(internode_http_operation(WALK_DIR_PATH), Some(INTERNODE_OPERATION_WALK_DIR));
assert_eq!(internode_http_operation(NS_SCANNER_PATH), Some(INTERNODE_OPERATION_NS_SCANNER));
assert_eq!(internode_http_operation("/rustfs/rpc/unknown"), None);
}
#[test]
fn rpc_head_signature_verification_is_skipped() {
fn file_stream_head_signature_verification_is_skipped() {
let uri: Uri = READ_FILE_STREAM_PATH.parse().expect("uri");
let headers = HeaderMap::new();
assert!(verify_internode_rpc_signature(&uri, &Method::HEAD, &headers).is_ok());
}
#[test]
fn namespace_scanner_capability_get_requires_signature() {
let challenge = uuid::Uuid::new_v4();
let uri: Uri = format!(
"{NS_SCANNER_PATH}?ns_scanner_protocol={}&{NS_SCANNER_CAPABILITY_CHALLENGE_QUERY}={challenge}",
super::NS_SCANNER_PROTOCOL_VERSION
)
.parse()
.expect("uri");
let headers = HeaderMap::new();
let response = verify_internode_rpc_signature(&uri, &Method::GET, &headers).expect_err("response");
assert_eq!(response.status(), StatusCode::FORBIDDEN);
}
#[test]
fn namespace_scanner_rejects_requests_from_a_prior_server_epoch() {
assert!(ns_scanner_server_epoch_matches(*super::NS_SCANNER_SERVER_EPOCH));
assert!(!ns_scanner_server_epoch_matches(uuid::Uuid::nil()));
}
#[test]
fn rpc_get_request_requires_signature() {
let uri: Uri = READ_FILE_STREAM_PATH.parse().expect("uri");
@@ -941,9 +1418,92 @@ mod tests {
LOG_SUBSYSTEM_FILE_TRANSFER
);
assert_eq!(internode_rpc_subsystem(Some(INTERNODE_OPERATION_WALK_DIR)), LOG_SUBSYSTEM_DIRECTORY_WALK);
assert_eq!(
internode_rpc_subsystem(Some(INTERNODE_OPERATION_NS_SCANNER)),
LOG_SUBSYSTEM_NAMESPACE_SCANNER
);
assert_eq!(internode_rpc_subsystem(None), LOG_SUBSYSTEM_ROUTING);
}
#[test]
fn namespace_scanner_requires_matching_signed_body_digest() {
let body = b"scanner-request";
let digest = hex_simd::encode_to_string(sha2::Sha256::digest(body), hex_simd::AsciiCase::Lower);
let request_id = uuid::Uuid::new_v4();
let server_epoch = uuid::Uuid::new_v4();
let session_id = uuid::Uuid::new_v4();
let valid: NsScannerQuery = serde_urlencoded::from_str(&format!(
"disk=disk-a&{NS_SCANNER_REQUEST_ID_QUERY}={request_id}&{NS_SCANNER_SERVER_EPOCH_QUERY}={server_epoch}&{NS_SCANNER_SESSION_ID_QUERY}={session_id}&{NS_SCANNER_SESSION_SEQUENCE_QUERY}=0&{NS_SCANNER_CYCLE_QUERY}=7&{NS_SCANNER_LEADER_EPOCH_QUERY}=9&{NS_SCANNER_BODY_SHA256_QUERY}={digest}"
))
.expect("query should parse");
let missing: NsScannerQuery = serde_urlencoded::from_str(&format!(
"disk=disk-a&{NS_SCANNER_REQUEST_ID_QUERY}={request_id}&{NS_SCANNER_SERVER_EPOCH_QUERY}={server_epoch}&{NS_SCANNER_SESSION_ID_QUERY}={session_id}&{NS_SCANNER_SESSION_SEQUENCE_QUERY}=0&{NS_SCANNER_CYCLE_QUERY}=7&{NS_SCANNER_LEADER_EPOCH_QUERY}=9"
))
.expect("query should parse");
assert!(verify_ns_scanner_body_digest(&valid, body));
assert!(!verify_ns_scanner_body_digest(&valid, b"tampered"));
assert!(!verify_ns_scanner_body_digest(&missing, body));
}
#[test]
fn namespace_scanner_queries_reject_unknown_fields() {
let request_id = uuid::Uuid::new_v4();
let server_epoch = uuid::Uuid::new_v4();
let session_id = uuid::Uuid::new_v4();
let query = format!(
"disk=disk-a&{NS_SCANNER_REQUEST_ID_QUERY}={request_id}&{NS_SCANNER_SERVER_EPOCH_QUERY}={server_epoch}&{NS_SCANNER_SESSION_ID_QUERY}={session_id}&{NS_SCANNER_SESSION_SEQUENCE_QUERY}=0&{NS_SCANNER_CYCLE_QUERY}=7&{NS_SCANNER_LEADER_EPOCH_QUERY}=9&{NS_SCANNER_BODY_SHA256_QUERY}=digest&unexpected=true"
);
assert!(serde_urlencoded::from_str::<NsScannerQuery>(&query).is_err());
assert!(serde_urlencoded::from_str::<super::NsScannerCapabilityQuery>("ns_scanner_protocol=1&unexpected=true").is_err());
}
#[test]
fn namespace_scanner_replay_capacity_is_retryable_without_weakening_replay_rejection() {
let (replay_status, replay_reason, _) =
remote_scanner_claim_rejection(&rustfs_scanner::ScannerError::RemoteRequestReplay);
let (capacity_status, capacity_reason, _) =
remote_scanner_claim_rejection(&rustfs_scanner::ScannerError::RemoteReplayCapacity);
assert_eq!(replay_status, StatusCode::CONFLICT);
assert_eq!(replay_reason, "request_replay");
assert_eq!(capacity_status, StatusCode::TOO_MANY_REQUESTS);
assert_eq!(capacity_reason, "replay_capacity");
}
#[tokio::test]
async fn namespace_scanner_body_surfaces_background_failure() {
let body = ns_scanner_response_body(|mut writer, _disconnect| async move {
writer.write_all(b"partial scanner frame").await?;
Err(io::Error::other("remote namespace scanner failed"))
});
let err = BodyExt::collect(body)
.await
.expect_err("failed completion must fail body collection");
assert!(err.to_string().contains("remote namespace scanner failed"));
}
#[tokio::test]
async fn dropping_namespace_scanner_body_cancels_producer() {
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
let (dropped_tx, dropped_rx) = tokio::sync::oneshot::channel();
let body = ns_scanner_response_body(move |_writer, disconnect| async move {
let _drop_notifier = DropNotifier(Some(dropped_tx));
let _ = started_tx.send(());
disconnect.cancelled().await;
Ok(())
});
started_rx.await.expect("namespace scanner producer should start");
drop(body);
tokio::time::timeout(std::time::Duration::from_secs(1), dropped_rx)
.await
.expect("dropping the response body should cancel the namespace scanner producer")
.expect("drop notifier should send a cancellation signal");
}
#[tokio::test]
async fn write_body_chunks_to_writer_streams_all_chunks() {
let (mut reader, mut writer) = tokio::io::duplex(64);
+361 -55
View File
@@ -23,8 +23,9 @@ use crate::storage::storage_api::rpc_consumer::node_service::STORAGE_CLASS_SUB_S
use crate::storage::storage_api::rpc_consumer::node_service::{CollectMetricsOpts, MetricType};
use crate::storage::storage_api::rpc_consumer::node_service::{
DiskStore, ECStore, Error, LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS,
SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, StorageDiskRpcExt as _, StorageResult, all_local_disk_path,
find_local_disk_by_ref, reload_transition_tier_config,
SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION, SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SERVICE_SIGNAL_REFRESH_CONFIG,
SERVICE_SIGNAL_RELOAD_DYNAMIC, StorageDiskRpcExt as _, StorageResult, all_local_disk_path, find_local_disk_by_ref,
reload_transition_tier_config,
};
use crate::storage::storage_api::runtime_sources_consumer::{EndpointServerPools, runtime_sources};
use crate::storage::storage_api::{sign_tonic_rpc_response_proof, verify_tonic_canonical_body_digest};
@@ -78,6 +79,14 @@ const TIER_MUTATION_PEER_STATE_PREPARED_WIRE: i32 = 1;
const TIER_MUTATION_PEER_STATE_COMMITTED_WIRE: i32 = 2;
const TIER_MUTATION_PEER_STATE_ABORTED_WIRE: i32 = 3;
fn signal_service_response(success: bool, error_info: Option<String>) -> Response<SignalServiceResponse> {
Response::new(SignalServiceResponse {
success,
error_info,
protocol_version: rustfs_protos::DYNAMIC_CONFIG_PROTOCOL_VERSION,
})
}
fn supports_dynamic_config_rpc(sub_system: &str) -> bool {
NOTIFY_SUB_SYSTEMS.contains(&sub_system)
|| matches!(
@@ -174,11 +183,54 @@ fn validate_admin_heal_control_start(request: &rustfs_common::heal_channel::Heal
Ok(())
}
fn scanner_activity_response(namespace_generation: u64) -> ScannerActivityResponse {
fn scanner_activity_response(
namespace_generation: u64,
topology_digest: [u8; 32],
data_movement_active: bool,
dirty_usage: rustfs_scanner::ScannerDirtyUsageState,
) -> ScannerActivityResponse {
ScannerActivityResponse {
instance_id: rustfs_scanner::scanner_activity_epoch().to_string(),
namespace_generation,
maintenance_generation: rustfs_scanner::scanner_maintenance_generation(),
protocol_version: rustfs_scanner::SCANNER_ACTIVITY_PROTOCOL_VERSION,
topology_digest: topology_digest.to_vec().into(),
data_movement_active,
response_proof: Bytes::new(),
dirty_usage_generation: dirty_usage.generation,
dirty_usage_pending: dirty_usage.pending,
}
}
fn previous_scanner_activity_response(
namespace_generation: u64,
topology_digest: [u8; 32],
data_movement_active: bool,
) -> ScannerActivityResponse {
ScannerActivityResponse {
instance_id: rustfs_scanner::scanner_activity_epoch().to_string(),
namespace_generation,
maintenance_generation: rustfs_scanner::scanner_maintenance_generation(),
protocol_version: SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION,
topology_digest: topology_digest.to_vec().into(),
data_movement_active,
response_proof: Bytes::new(),
dirty_usage_generation: 0,
dirty_usage_pending: false,
}
}
fn legacy_scanner_activity_response(namespace_generation: u64) -> ScannerActivityResponse {
ScannerActivityResponse {
instance_id: rustfs_scanner::scanner_activity_epoch().to_string(),
namespace_generation,
maintenance_generation: rustfs_scanner::scanner_maintenance_generation(),
protocol_version: SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION,
topology_digest: Bytes::new(),
data_movement_active: false,
response_proof: Bytes::new(),
dirty_usage_generation: 0,
dirty_usage_pending: false,
}
}
@@ -1559,73 +1611,149 @@ impl Node for NodeService {
Some(value) => match value.parse::<bool>() {
Ok(value) => value,
Err(_) => {
return Ok(Response::new(SignalServiceResponse {
success: false,
error_info: Some(format!("invalid dry-run value: {value}")),
}));
return Ok(signal_service_response(false, Some(format!("invalid dry-run value: {value}"))));
}
},
};
match signal {
Some(SERVICE_SIGNAL_REFRESH_CONFIG) => match reload_runtime_config_snapshot().await {
Ok(()) => Ok(Response::new(SignalServiceResponse {
success: true,
error_info: None,
})),
Err(_) => Ok(Response::new(SignalServiceResponse {
success: false,
error_info: Some("runtime config snapshot reload failed".to_string()),
})),
Ok(()) => Ok(signal_service_response(true, None)),
Err(_) => Ok(signal_service_response(false, Some("runtime config snapshot reload failed".to_string()))),
},
Some(SERVICE_SIGNAL_RELOAD_DYNAMIC) => {
let supported = sub_system == MODULE_SWITCHES_SIGNAL_SUBSYSTEM || supports_dynamic_config_rpc(sub_system);
if !supported {
return Ok(Response::new(SignalServiceResponse {
success: false,
error_info: Some(format!("unsupported dynamic config subsystem: {sub_system}")),
}));
return Ok(signal_service_response(
false,
Some(format!("unsupported dynamic config subsystem: {sub_system}")),
));
}
if dry_run {
return Ok(Response::new(SignalServiceResponse {
success: true,
error_info: None,
}));
return Ok(signal_service_response(true, None));
}
match reload_dynamic_config_runtime_state(sub_system).await {
Ok(()) => Ok(Response::new(SignalServiceResponse {
success: true,
error_info: None,
})),
Err(_) => Ok(Response::new(SignalServiceResponse {
success: false,
error_info: Some(format!("dynamic config reload failed for {sub_system}")),
})),
Ok(()) => Ok(signal_service_response(true, None)),
Err(_) => Ok(signal_service_response(
false,
Some(format!("dynamic config reload failed for {sub_system}")),
)),
}
}
Some(other) => Ok(Response::new(SignalServiceResponse {
success: false,
error_info: Some(format!("unsupported service signal: {other}")),
})),
None if raw_signal.is_some() => Ok(Response::new(SignalServiceResponse {
success: false,
error_info: Some(format!("invalid service signal value: {}", raw_signal.unwrap_or_default())),
})),
None => Ok(Response::new(SignalServiceResponse {
success: false,
error_info: Some("missing service signal".to_string()),
})),
Some(other) => Ok(signal_service_response(false, Some(format!("unsupported service signal: {other}")))),
None if raw_signal.is_some() => Ok(signal_service_response(
false,
Some(format!("invalid service signal value: {}", raw_signal.unwrap_or_default())),
)),
None => Ok(signal_service_response(false, Some("missing service signal".to_string()))),
}
}
async fn scanner_activity(
&self,
_request: Request<ScannerActivityRequest>,
request: Request<ScannerActivityRequest>,
) -> Result<Response<ScannerActivityResponse>, Status> {
let request_protocol = request.get_ref().protocol_version;
match request_protocol {
// RUSTFS_COMPAT_TODO(ns-scanner-rpc-v3): legacy request body is unbound. Remove after protocol v0 peers are unsupported.
SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION => {
if !request.get_ref().acknowledge_instance_id.is_empty()
|| request.get_ref().acknowledge_dirty_usage_generation != 0
{
return Err(Status::invalid_argument("legacy scanner activity request cannot acknowledge dirty usage"));
}
}
SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION => {
verify_tonic_canonical_body_digest(&request, request.get_ref().challenge.as_ref())
.map_err(|err| Status::permission_denied(format!("scanner activity authentication failed: {err}")))?;
if !request.get_ref().acknowledge_instance_id.is_empty()
|| request.get_ref().acknowledge_dirty_usage_generation != 0
{
return Err(Status::invalid_argument("scanner activity protocol v4 cannot acknowledge dirty usage"));
}
}
rustfs_scanner::SCANNER_ACTIVITY_PROTOCOL_VERSION => {
let canonical = rustfs_protos::canonical_scanner_activity_request_body(request.get_ref())
.map_err(|_| Status::invalid_argument("scanner activity request is too large to authenticate"))?;
verify_tonic_canonical_body_digest(&request, &canonical)
.map_err(|err| Status::permission_denied(format!("scanner activity authentication failed: {err}")))?;
let has_acknowledgement = !request.get_ref().acknowledge_instance_id.is_empty();
if has_acknowledgement != (request.get_ref().acknowledge_dirty_usage_generation != 0) {
return Err(Status::invalid_argument(
"scanner dirty usage acknowledgement requires both instance ID and generation",
));
}
}
version => {
return Err(Status::failed_precondition(format!(
"unsupported scanner activity request protocol {version}"
)));
}
}
let challenge_len = request.get_ref().challenge.len();
if challenge_len != 16 && !(request_protocol == SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION && challenge_len == 0) {
return Err(Status::invalid_argument("scanner activity challenge must be 16 bytes"));
}
let store = self
.resolve_object_store()
.ok_or_else(|| Status::unavailable("storage layer is not initialized"))?;
Ok(Response::new(scanner_activity_response(store.scanner_namespace_mutation_generation())))
if request.get_ref().challenge.is_empty() {
// Older peers send an empty protocol-0 request and cannot establish
// the topology fence required for distributed usage publication.
return Ok(Response::new(legacy_scanner_activity_response(
store.scanner_namespace_mutation_generation(),
)));
}
let request = request.into_inner();
let challenge: [u8; 16] = request
.challenge
.as_ref()
.try_into()
.map_err(|_| Status::invalid_argument("scanner activity challenge must be 16 bytes"))?;
if !request.acknowledge_instance_id.is_empty() {
rustfs_scanner::acknowledge_dirty_usage_generation(
&request.acknowledge_instance_id,
request.acknowledge_dirty_usage_generation,
)
.map_err(|err| Status::failed_precondition(err.to_string()))?;
}
let namespace_generation = store.scanner_namespace_mutation_generation();
let topology_digest = rustfs_scanner::scanner_topology_digest(store.as_ref());
let data_movement_active = store.scanner_data_movement_active().await;
let mut response = match request_protocol {
SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION | SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION => {
previous_scanner_activity_response(namespace_generation, topology_digest, data_movement_active)
}
rustfs_scanner::SCANNER_ACTIVITY_PROTOCOL_VERSION => scanner_activity_response(
namespace_generation,
topology_digest,
data_movement_active,
rustfs_scanner::scanner_dirty_usage_state(),
),
version => {
return Err(Status::failed_precondition(format!(
"unsupported scanner activity request protocol {version}"
)));
}
};
let canonical = match response.protocol_version {
SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION => {
rustfs_protos::canonical_scanner_activity_v4_response_body(&challenge, &response)
}
rustfs_scanner::SCANNER_ACTIVITY_PROTOCOL_VERSION => {
rustfs_protos::canonical_scanner_activity_response_body(&challenge, &response)
}
version => {
return Err(Status::internal(format!(
"scanner activity response selected unsupported protocol {version}"
)));
}
}
.map_err(|_| Status::internal("scanner activity response is too large to authenticate"))?;
response.response_proof = sign_tonic_rpc_response_proof(&canonical)
.map_err(|_| Status::unavailable("scanner activity response authentication is unavailable"))?
.into();
Ok(Response::new(response))
}
async fn background_heal_status(
@@ -1882,11 +2010,13 @@ impl Node for NodeService {
mod tests {
use super::{
CollectMetricsOpts, DiskStore, Error, HEAL_CONTROL_PAYLOAD_MAX_SIZE, MetricType, Node as _, NodeService,
PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC,
PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION,
SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC,
STORAGE_CLASS_SUB_SYS, admit_heal_control_replay, background_rebalance_start_error_message,
execute_heal_control_envelope_with_manager, initialize_heal_topology_fingerprint, make_heal_control_server,
make_heal_control_server_with_cache, make_server, make_server_for_context, make_tier_mutation_control_server_for_context,
remove_heal_control_replay, scanner_activity_response, stop_rebalance_response,
execute_heal_control_envelope_with_manager, initialize_heal_topology_fingerprint, legacy_scanner_activity_response,
make_heal_control_server, make_heal_control_server_with_cache, make_server, make_server_for_context,
make_tier_mutation_control_server_for_context, previous_scanner_activity_response, remove_heal_control_replay,
scanner_activity_response, stop_rebalance_response,
};
use crate::storage::rpc::node_service::heal::heal_topology_fingerprint;
use crate::storage::storage_api::rpc_consumer::node_service::{HealBucketInfo, HealEndpoint};
@@ -4296,24 +4426,199 @@ mod tests {
}
#[tokio::test]
async fn test_scanner_activity_requires_storage_layer() {
async fn test_scanner_activity_requires_body_bound_auth_before_storage_lookup() {
let service = create_test_node_service();
let err = service
.scanner_activity(Request::new(ScannerActivityRequest {}))
let legacy = service
.scanner_activity(Request::new(ScannerActivityRequest {
challenge: vec![7; 16].into(),
protocol_version: SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION,
acknowledge_instance_id: String::new(),
acknowledge_dirty_usage_generation: 0,
}))
.await
.expect_err("activity queries must fail closed before storage is initialized");
.expect_err("a rolling-upgrade request should pass authentication before storage lookup");
assert_eq!(legacy.code(), tonic::Code::Unavailable);
assert_eq!(err.code(), tonic::Code::Unavailable);
let unsupported = service
.scanner_activity(Request::new(ScannerActivityRequest {
challenge: vec![7; 16].into(),
protocol_version: rustfs_scanner::SCANNER_ACTIVITY_PROTOCOL_VERSION + 1,
acknowledge_instance_id: String::new(),
acknowledge_dirty_usage_generation: 0,
}))
.await
.expect_err("an unknown request protocol must fail before storage lookup");
assert_eq!(unsupported.code(), tonic::Code::FailedPrecondition);
let malformed_legacy = service
.scanner_activity(Request::new(ScannerActivityRequest {
challenge: vec![7; 15].into(),
protocol_version: SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION,
acknowledge_instance_id: String::new(),
acknowledge_dirty_usage_generation: 0,
}))
.await
.expect_err("a malformed legacy challenge must fail before storage lookup");
assert_eq!(malformed_legacy.code(), tonic::Code::InvalidArgument);
let unsigned = service
.scanner_activity(Request::new(ScannerActivityRequest {
challenge: vec![7; 16].into(),
protocol_version: rustfs_scanner::SCANNER_ACTIVITY_PROTOCOL_VERSION,
acknowledge_instance_id: String::new(),
acknowledge_dirty_usage_generation: 0,
}))
.await
.expect_err("unsigned activity queries must fail before storage lookup");
assert_eq!(unsigned.code(), tonic::Code::PermissionDenied);
let mut malformed_current = Request::new(ScannerActivityRequest {
challenge: vec![7; 15].into(),
protocol_version: rustfs_scanner::SCANNER_ACTIVITY_PROTOCOL_VERSION,
acknowledge_instance_id: String::new(),
acknowledge_dirty_usage_generation: 0,
});
let malformed_canonical = rustfs_protos::canonical_scanner_activity_request_body(malformed_current.get_ref())
.expect("scanner activity request should encode");
set_tonic_canonical_body_digest(&mut malformed_current, &malformed_canonical).expect("digest metadata should encode");
mark_v2_authenticated(&mut malformed_current);
let malformed_current = service
.scanner_activity(malformed_current)
.await
.expect_err("a signed malformed challenge must fail before storage lookup");
assert_eq!(malformed_current.code(), tonic::Code::InvalidArgument);
let mut tampered = Request::new(ScannerActivityRequest {
challenge: vec![7; 16].into(),
protocol_version: rustfs_scanner::SCANNER_ACTIVITY_PROTOCOL_VERSION,
acknowledge_instance_id: String::new(),
acknowledge_dirty_usage_generation: 0,
});
let other = ScannerActivityRequest {
challenge: vec![8; 16].into(),
..tampered.get_ref().clone()
};
let other_canonical =
rustfs_protos::canonical_scanner_activity_request_body(&other).expect("scanner activity request should encode");
set_tonic_canonical_body_digest(&mut tampered, &other_canonical).expect("digest metadata should encode");
mark_v2_authenticated(&mut tampered);
let tampered = service
.scanner_activity(tampered)
.await
.expect_err("tampered activity challenge must fail before storage lookup");
assert_eq!(tampered.code(), tonic::Code::PermissionDenied);
let mut downgraded = Request::new(ScannerActivityRequest {
challenge: vec![7; 16].into(),
protocol_version: rustfs_scanner::SCANNER_ACTIVITY_PROTOCOL_VERSION,
acknowledge_instance_id: String::new(),
acknowledge_dirty_usage_generation: 0,
});
let current_canonical = rustfs_protos::canonical_scanner_activity_request_body(downgraded.get_ref())
.expect("scanner activity request should encode");
set_tonic_canonical_body_digest(&mut downgraded, &current_canonical).expect("digest metadata should encode");
downgraded.get_mut().protocol_version = SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION;
mark_v2_authenticated(&mut downgraded);
let downgraded = service
.scanner_activity(downgraded)
.await
.expect_err("a signed current request must not be downgraded to protocol v4");
assert_eq!(downgraded.code(), tonic::Code::PermissionDenied);
let mut incomplete_acknowledgement = Request::new(ScannerActivityRequest {
challenge: vec![7; 16].into(),
protocol_version: rustfs_scanner::SCANNER_ACTIVITY_PROTOCOL_VERSION,
acknowledge_instance_id: rustfs_scanner::scanner_activity_epoch().to_string(),
acknowledge_dirty_usage_generation: 0,
});
let acknowledgement_canonical =
rustfs_protos::canonical_scanner_activity_request_body(incomplete_acknowledgement.get_ref())
.expect("scanner activity request should encode");
set_tonic_canonical_body_digest(&mut incomplete_acknowledgement, &acknowledgement_canonical)
.expect("digest metadata should encode");
mark_v2_authenticated(&mut incomplete_acknowledgement);
let incomplete_acknowledgement = service
.scanner_activity(incomplete_acknowledgement)
.await
.expect_err("dirty usage acknowledgements require an instance ID and generation");
assert_eq!(incomplete_acknowledgement.code(), tonic::Code::InvalidArgument);
let mut previous = Request::new(ScannerActivityRequest {
challenge: vec![7; 16].into(),
protocol_version: SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION,
acknowledge_instance_id: String::new(),
acknowledge_dirty_usage_generation: 0,
});
set_tonic_canonical_body_digest(&mut previous, &[7; 16]).expect("protocol v4 digest metadata should encode");
mark_v2_authenticated(&mut previous);
let previous = service
.scanner_activity(previous)
.await
.expect_err("an authenticated protocol v4 request should reach storage lookup during rolling upgrades");
assert_eq!(previous.code(), tonic::Code::Unavailable);
let mut signed = Request::new(ScannerActivityRequest {
challenge: vec![7; 16].into(),
protocol_version: rustfs_scanner::SCANNER_ACTIVITY_PROTOCOL_VERSION,
acknowledge_instance_id: String::new(),
acknowledge_dirty_usage_generation: 0,
});
let signed_canonical = rustfs_protos::canonical_scanner_activity_request_body(signed.get_ref())
.expect("scanner activity request should encode");
set_tonic_canonical_body_digest(&mut signed, &signed_canonical).expect("digest metadata should encode");
mark_v2_authenticated(&mut signed);
let unavailable = service
.scanner_activity(signed)
.await
.expect_err("authenticated activity queries still require initialized storage");
assert_eq!(unavailable.code(), tonic::Code::Unavailable);
}
#[test]
fn test_scanner_activity_response_uses_process_epoch_and_generations() {
let response = scanner_activity_response(17);
let response = scanner_activity_response(
17,
[7; 32],
true,
rustfs_scanner::ScannerDirtyUsageState {
generation: 11,
pending: true,
},
);
assert_eq!(response.instance_id, rustfs_scanner::scanner_activity_epoch());
assert_eq!(response.namespace_generation, 17);
assert_eq!(response.maintenance_generation, rustfs_scanner::scanner_maintenance_generation());
assert_eq!(response.protocol_version, rustfs_scanner::SCANNER_ACTIVITY_PROTOCOL_VERSION);
assert_eq!(response.topology_digest.as_ref(), &[7; 32]);
assert!(response.data_movement_active);
assert_eq!(response.dirty_usage_generation, 11);
assert!(response.dirty_usage_pending);
}
#[test]
fn test_previous_scanner_activity_response_omits_dirty_usage_fields() {
let response = previous_scanner_activity_response(17, [7; 32], true);
assert_eq!(response.protocol_version, SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION);
assert_eq!(response.topology_digest.as_ref(), &[7; 32]);
assert!(response.data_movement_active);
assert_eq!(response.dirty_usage_generation, 0);
assert!(!response.dirty_usage_pending);
}
#[test]
fn test_legacy_scanner_activity_response_omits_extended_fields() {
let response = legacy_scanner_activity_response(17);
assert_eq!(response.namespace_generation, 17);
assert_eq!(response.protocol_version, SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION);
assert!(response.topology_digest.is_empty());
assert!(!response.data_movement_active);
assert!(response.response_proof.is_empty());
assert_eq!(response.dirty_usage_generation, 0);
assert!(!response.dirty_usage_pending);
}
#[tokio::test]
@@ -4373,6 +4678,7 @@ mod tests {
assert!(response.success, "new nodes must advertise notify lifecycle reload support");
assert!(response.error_info.is_none());
assert_eq!(response.protocol_version, rustfs_protos::DYNAMIC_CONFIG_PROTOCOL_VERSION);
}
#[tokio::test]
+18 -7
View File
@@ -698,13 +698,24 @@ impl NodeService {
Ok(volume_infos) => {
let volume_infos = volume_infos
.into_iter()
.filter_map(|volume_info| serde_json::to_string(&volume_info).ok())
.collect();
Ok(Response::new(ListVolumesResponse {
success: true,
volume_infos,
error: None,
}))
.enumerate()
.map(|(index, volume_info)| {
serde_json::to_string(&volume_info)
.map_err(|err| DiskError::other(format!("encode list volumes entry {index} failed: {err}")))
})
.collect::<std::result::Result<Vec<_>, DiskError>>();
match volume_infos {
Ok(volume_infos) => Ok(Response::new(ListVolumesResponse {
success: true,
volume_infos,
error: None,
})),
Err(err) => Ok(Response::new(ListVolumesResponse {
success: false,
volume_infos: Vec::new(),
error: Some(err.into()),
})),
}
}
Err(err) => Ok(Response::new(ListVolumesResponse {
success: false,
+21 -5
View File
@@ -212,12 +212,23 @@ pub(crate) mod rpc_consumer {
pub(crate) mod http_service {
pub(crate) const DEFAULT_READ_BUFFER_SIZE: usize = super::super::DEFAULT_READ_BUFFER_SIZE;
#[cfg(test)]
pub(crate) use super::super::storage_contracts::WALK_DIR_BODY_SHA256_QUERY;
pub(crate) use super::super::storage_contracts::WALK_DIR_STREAM_COMPLETION_V1;
pub(crate) use super::super::{StorageDiskRpcExt, WalkDirOptions, find_local_disk_by_ref, verify_rpc_signature};
pub(crate) use super::super::storage_contracts::{
NS_SCANNER_BODY_SHA256_QUERY, NS_SCANNER_CAPABILITY_CHALLENGE_QUERY, NS_SCANNER_CYCLE_QUERY,
NS_SCANNER_LEADER_EPOCH_QUERY, NS_SCANNER_REQUEST_ID_QUERY, NS_SCANNER_SERVER_EPOCH_QUERY,
NS_SCANNER_SESSION_ID_QUERY, NS_SCANNER_SESSION_SEQUENCE_QUERY, WALK_DIR_BODY_SHA256_QUERY,
};
pub(crate) use super::super::storage_contracts::{
NS_SCANNER_PROTOCOL_VERSION, NsScannerCapabilityResponse, WALK_DIR_STREAM_COMPLETION_V1,
};
pub(crate) use super::super::{
StorageDiskRpcExt, WalkDirOptions, find_local_disk_by_ref, sign_ns_scanner_capability, verify_rpc_signature,
};
}
pub(crate) mod node_service {
pub(crate) use super::super::storage_contracts::{
SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION, SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION,
};
pub(crate) use super::super::{
BatchReadVersionReq, BatchReadVersionResp, CollectMetricsOpts, DeleteOptions, DiskError, DiskInfoOptions, DiskStore,
ECStore, Error, FileInfoVersions, LocalPeerS3Client, MetricType, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS,
@@ -404,7 +415,7 @@ pub(crate) mod ecstore_data_usage {
apply_bucket_usage_memory_overlay, init_compression_total_memory_from_backend, load_data_usage_from_backend,
load_data_usage_from_backend_cached, record_bucket_delete_marker_memory, record_bucket_object_delete_memory,
record_bucket_object_version_write_memory, record_bucket_object_write_memory,
record_bucket_object_write_unknown_previous_memory, remove_bucket_usage_from_backend, store_compression_total_in_backend,
record_bucket_object_write_unknown_previous_memory, store_compression_total_in_backend,
};
// Test-only observables for the rustfs/backlog#1306 revert detector.
#[cfg(test)]
@@ -484,7 +495,8 @@ pub(crate) mod ecstore_rpc {
pub(crate) use rustfs_ecstore::api::rpc::{
LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, PeerS3Client,
SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, TONIC_RPC_PREFIX, normalize_tonic_rpc_audience,
sign_tonic_rpc_response_proof, verify_rpc_signature, verify_tonic_canonical_body_digest, verify_tonic_rpc_signature,
sign_ns_scanner_capability, sign_tonic_rpc_response_proof, verify_rpc_signature, verify_tonic_canonical_body_digest,
verify_tonic_rpc_signature,
};
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::rpc::{
@@ -1543,6 +1555,10 @@ pub(crate) fn verify_rpc_signature(url: &str, method: &http::Method, headers: &h
ecstore_rpc::verify_rpc_signature(url, method, headers)
}
pub(crate) fn sign_ns_scanner_capability(challenge: uuid::Uuid, server_epoch: uuid::Uuid) -> std::io::Result<Vec<u8>> {
ecstore_rpc::sign_ns_scanner_capability(challenge, server_epoch)
}
pub(crate) fn verify_tonic_rpc_signature(audience: &str, path: &str, headers: &http::HeaderMap) -> std::io::Result<()> {
ecstore_rpc::verify_tonic_rpc_signature(audience, path, headers)
}