mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-08 14:23:13 +00:00
fix(scanner): make distributed usage convergence authoritative (#5151)
* fix(scanner): make distributed usage cycles authoritative * fix(scanner): close distributed refresh races * fix(config): align scanner reload integration * fix(admin): scope config test helpers * fix(scanner): harden distributed usage convergence * fix(scanner): preserve rolling activity compatibility * fix(admin): expose non-secret optional config values * fix(scanner): acknowledge distributed dirty usage * fix(ecstore): make bucket mutations cancellation safe * fix(scanner): preserve pending dirty acknowledgements * test(obs): account for superseded scanner metric * fix(api): reject excess detached bucket mutations * test: close scanner convergence coverage gaps * fix(scanner): make path tracking cleanup one-shot --------- Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com> Co-authored-by: houseme <housemecn@gmail.com>
This commit is contained in:
@@ -259,8 +259,10 @@ impl Operation for AccountInfoHandler {
|
||||
.await
|
||||
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, e.to_string()))?;
|
||||
|
||||
// Serve the last persisted scanner snapshot plus the in-memory overlay.
|
||||
// This request path must never trigger a live full-version listing
|
||||
// Serve the last persisted scanner snapshot plus an authoritative
|
||||
// single-process overlay. Distributed nodes must not promote their
|
||||
// process-local absolute counters to cluster-wide bucket totals.
|
||||
// This path never triggers a live full-version listing
|
||||
// (rustfs/backlog#1306); freshness is owned by the scanner.
|
||||
let mut data_usage_info = map_data_usage_result(load_data_usage_from_backend_cached(store.clone()).await)?;
|
||||
apply_bucket_usage_memory_overlay(&mut data_usage_info).await;
|
||||
|
||||
@@ -12,11 +12,11 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::admin::handlers::supervise_admin_mutation;
|
||||
use crate::admin::handlers::target_descriptor::AdminTargetSpec;
|
||||
use crate::admin::runtime_sources::{AppContext, current_app_context, current_object_store_handle_for_context};
|
||||
use crate::admin::storage_api::config::{
|
||||
read_admin_config_without_migrate, read_admin_config_without_migrate_no_lock, save_admin_server_config_no_lock,
|
||||
with_admin_server_config_write_lock,
|
||||
read_admin_config_without_migrate, read_admin_server_config_snapshot, save_admin_server_config_snapshot,
|
||||
};
|
||||
use rustfs_audit::{audit_system, start_audit_system as start_global_audit_system, system::AuditSystemState};
|
||||
use rustfs_config::DEFAULT_DELIMITER;
|
||||
@@ -96,20 +96,20 @@ where
|
||||
let Some(store) = current_object_store_handle_for_context(context) else {
|
||||
return Err(s3_error!(InternalError, "server storage not initialized"));
|
||||
};
|
||||
|
||||
let specs = specs.to_vec();
|
||||
let lock_store = store.clone();
|
||||
with_admin_server_config_write_lock(lock_store, move || async move {
|
||||
let mut config = read_admin_config_without_migrate_no_lock(store.clone())
|
||||
supervise_admin_mutation("audit config update", async move {
|
||||
let snapshot = read_admin_server_config_snapshot(store.clone())
|
||||
.await
|
||||
.map_err(|e| s3_error!(InternalError, "failed to read server config: {}", e))?;
|
||||
let mut config = snapshot.config.clone();
|
||||
|
||||
if !modifier(&mut config) {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
save_admin_server_config_no_lock(store, &config)
|
||||
save_admin_server_config_snapshot(store, &config, &snapshot)
|
||||
.await
|
||||
.map(|_| ())
|
||||
.map_err(|e| s3_error!(InternalError, "failed to save audit config: {}", e))?;
|
||||
|
||||
// Keep persistence and runtime publication in one detached, serialized
|
||||
@@ -118,8 +118,6 @@ where
|
||||
apply_audit_runtime_config(&specs, config).await
|
||||
})
|
||||
.await
|
||||
.map_err(|err| s3_error!(InternalError, "failed to lock server config update: {}", err))??;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) async fn update_audit_config_and_reload<F>(specs: &[AdminTargetSpec], modifier: F) -> S3Result<()>
|
||||
|
||||
@@ -21,15 +21,14 @@ use crate::admin::runtime_sources::{
|
||||
};
|
||||
use crate::admin::service::config::{
|
||||
CONFIG_WORKER_RELOAD_FAILURE_STATE, EVENT_CONFIG_WORKER_RELOAD_FAILED, FULL_CONFIG_WORKER_SUBSYSTEMS, LOG_COMPONENT_ADMIN,
|
||||
LOG_SUBSYSTEM_CONFIG, PreparedRuntimeConfig, apply_dynamic_config_for_subsystem, is_dynamic_config_subsystem,
|
||||
preflight_dynamic_config_reload, prepare_server_config, signal_config_snapshot_reload_checked,
|
||||
signal_dynamic_config_reload_checked,
|
||||
LOG_SUBSYSTEM_CONFIG, PreparedRuntimeConfig, is_dynamic_config_subsystem, preflight_dynamic_config_reload,
|
||||
prepare_server_config, reload_dynamic_config_runtime_state, reload_runtime_config_snapshot, signal_config_snapshot_reload,
|
||||
signal_config_snapshot_reload_checked, signal_dynamic_config_reload_checked,
|
||||
};
|
||||
use crate::admin::storage_api::config::storageclass::{INLINE_BLOCK_ENV, OPTIMIZE_ENV, RRS_ENV, STANDARD_ENV};
|
||||
use crate::admin::storage_api::config::{
|
||||
RUSTFS_META_BUCKET, STORAGE_CLASS_SUB_SYS, delete_admin_config, read_admin_config, read_admin_config_without_migrate,
|
||||
read_admin_config_without_migrate_no_lock, save_admin_config, save_admin_server_config_no_lock,
|
||||
with_admin_server_config_write_lock,
|
||||
AdminServerConfigSnapshot, RUSTFS_META_BUCKET, STORAGE_CLASS_SUB_SYS, delete_admin_config, read_admin_config,
|
||||
read_admin_config_without_migrate, read_admin_server_config_snapshot, save_admin_config, save_admin_server_config_snapshot,
|
||||
};
|
||||
use crate::admin::storage_api::contract::list::ListOperations as _;
|
||||
use crate::admin::utils::{encode_compatible_admin_payload, is_compat_admin_request, read_compatible_admin_body};
|
||||
@@ -64,7 +63,7 @@ use rustfs_config::oidc::{
|
||||
};
|
||||
use rustfs_config::server_config::{Config as ServerConfig, DEFAULT_KVS, KV, KVS};
|
||||
use rustfs_config::{
|
||||
COMMENT_KEY, DEFAULT_DELIMITER, ENABLE_KEY, ENV_PREFIX, ENV_SCANNER_ALERT_EXCESS_FOLDERS,
|
||||
BASE_DSN_STRING, COMMENT_KEY, DEFAULT_DELIMITER, ENABLE_KEY, ENV_PREFIX, ENV_SCANNER_ALERT_EXCESS_FOLDERS,
|
||||
ENV_SCANNER_ALERT_EXCESS_VERSION_SIZE, ENV_SCANNER_ALERT_EXCESS_VERSIONS, ENV_SCANNER_BITROT_CYCLE_SECS,
|
||||
ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS, ENV_SCANNER_CYCLE, ENV_SCANNER_CYCLE_MAX_DIRECTORIES,
|
||||
ENV_SCANNER_CYCLE_MAX_DURATION_SECS, ENV_SCANNER_CYCLE_MAX_OBJECTS, ENV_SCANNER_DELAY, ENV_SCANNER_IDLE_MODE,
|
||||
@@ -86,7 +85,6 @@ use s3s::{Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error}
|
||||
use serde::Serialize;
|
||||
use std::collections::{BTreeSet, HashMap};
|
||||
use std::env;
|
||||
use std::future::Future;
|
||||
use std::mem::size_of;
|
||||
use time::OffsetDateTime;
|
||||
use tracing::warn;
|
||||
@@ -750,14 +748,6 @@ async fn load_server_config_from_store() -> S3Result<ServerConfig> {
|
||||
.map_err(Into::into)
|
||||
}
|
||||
|
||||
async fn load_server_config_from_store_locked() -> S3Result<ServerConfig> {
|
||||
let store = object_store()?;
|
||||
read_admin_config_without_migrate_no_lock(store)
|
||||
.await
|
||||
.map_err(ApiError::from)
|
||||
.map_err(Into::into)
|
||||
}
|
||||
|
||||
async fn load_active_server_config() -> S3Result<ServerConfig> {
|
||||
if let Ok(config) = load_server_config_from_store().await {
|
||||
return Ok(config);
|
||||
@@ -768,9 +758,17 @@ async fn load_active_server_config() -> S3Result<ServerConfig> {
|
||||
.ok_or_else(|| s3_error!(InternalError, "server config is not initialized"))
|
||||
}
|
||||
|
||||
async fn save_server_config_to_store_locked(config: &ServerConfig) -> S3Result<()> {
|
||||
async fn load_server_config_snapshot_from_store() -> S3Result<AdminServerConfigSnapshot> {
|
||||
let store = object_store()?;
|
||||
save_admin_server_config_no_lock(store, config)
|
||||
read_admin_server_config_snapshot(store)
|
||||
.await
|
||||
.map_err(ApiError::from)
|
||||
.map_err(Into::into)
|
||||
}
|
||||
|
||||
async fn save_server_config_to_store(config: &ServerConfig, snapshot: &AdminServerConfigSnapshot) -> S3Result<bool> {
|
||||
let store = object_store()?;
|
||||
save_admin_server_config_snapshot(store, config, snapshot)
|
||||
.await
|
||||
.map_err(ApiError::from)
|
||||
.map_err(Into::into)
|
||||
@@ -864,7 +862,17 @@ fn encode_config_history_recovery(config: &ServerConfig) -> Vec<u8> {
|
||||
|
||||
fn encode_config_history_snapshot(config: &ServerConfig) -> S3Result<Vec<u8>> {
|
||||
let recovery = encode_config_history_recovery(config);
|
||||
let display = render_full_config(config, true);
|
||||
let mut display_config = config.clone();
|
||||
for targets in display_config.0.values_mut() {
|
||||
for kvs in targets.values_mut() {
|
||||
for entry in &mut kvs.0 {
|
||||
if (entry.hidden_if_empty || is_sensitive_key_name(&entry.key)) && !entry.value.trim().is_empty() {
|
||||
entry.value = REDACTED_VALUE.to_string();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
let display = render_full_config(&display_config, false);
|
||||
let recovery_len =
|
||||
u64::try_from(recovery.len()).map_err(|_| s3_error!(InternalError, "config history recovery snapshot is too large"))?;
|
||||
let mut snapshot =
|
||||
@@ -1267,6 +1275,7 @@ fn is_sensitive_key_name(key: &str) -> bool {
|
||||
|| normalized.ends_with("_tls_client_key")
|
||||
|| normalized == "private_key"
|
||||
|| normalized.ends_with("_private_key")
|
||||
|| normalized == BASE_DSN_STRING
|
||||
}
|
||||
|
||||
fn is_sensitive_env_key_name(key: &str) -> bool {
|
||||
@@ -1415,7 +1424,7 @@ fn apply_delete_directives(config: &mut ServerConfig, directives: &[ConfigDirect
|
||||
}
|
||||
|
||||
fn render_entry_value(entry: &KV, redact_secrets: bool) -> String {
|
||||
if redact_secrets && (entry.hidden_if_empty || is_sensitive_key_name(&entry.key)) && !entry.value.trim().is_empty() {
|
||||
if redact_secrets && is_sensitive_key_name(&entry.key) && !entry.value.trim().is_empty() {
|
||||
REDACTED_VALUE.to_string()
|
||||
} else {
|
||||
entry.value.clone()
|
||||
@@ -1824,22 +1833,22 @@ fn build_help_response(sub_system: Option<&str>, key: Option<&str>, env_only: bo
|
||||
})
|
||||
}
|
||||
|
||||
async fn reject_lost_config_transaction<T>(snapshot: AdminServerConfigSnapshot, phase: &'static str) -> S3Result<T> {
|
||||
drop(snapshot);
|
||||
reload_runtime_config_snapshot().await?;
|
||||
signal_config_snapshot_reload().await;
|
||||
Err(s3_error!(
|
||||
InternalError,
|
||||
"server config transaction lock was lost {phase}; runtime state was reloaded from durable config"
|
||||
))
|
||||
}
|
||||
|
||||
fn publish_prepared_config_snapshots(config: ServerConfig, prepared: PreparedRuntimeConfig) -> S3Result<()> {
|
||||
prepared.publish_storage_class()?;
|
||||
publish_server_config(config);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn commit_prepared_config(
|
||||
config: ServerConfig,
|
||||
prepared: PreparedRuntimeConfig,
|
||||
persist: impl Future<Output = S3Result<()>>,
|
||||
publish: impl FnOnce(ServerConfig, PreparedRuntimeConfig) -> S3Result<()>,
|
||||
) -> S3Result<()> {
|
||||
persist.await?;
|
||||
publish(config, prepared)
|
||||
}
|
||||
|
||||
/// Re-apply local mutable worker families after a full-config replacement.
|
||||
/// Peers receive one full-snapshot signal after this returns; signaling each
|
||||
/// family here as well would recreate audit/scanner targets twice per peer.
|
||||
@@ -1851,9 +1860,26 @@ fn publish_notify_config_intent(
|
||||
.then(|| rustfs_notify::ensure_live_events().publish_config(config.clone()))
|
||||
}
|
||||
|
||||
async fn preflight_notify_config_intent(sub_system: Option<&str>) -> S3Result<()> {
|
||||
if sub_system.is_none() || sub_system.is_some_and(|sub_system| NOTIFY_SUB_SYSTEMS.contains(&sub_system)) {
|
||||
preflight_dynamic_config_reload(sub_system.unwrap_or(NOTIFY_WEBHOOK_SUB_SYS)).await?;
|
||||
fn config_preflight_subsystems(sub_system: Option<&str>) -> Vec<&str> {
|
||||
if let Some(sub_system) = sub_system {
|
||||
return is_dynamic_config_subsystem(sub_system)
|
||||
.then_some(sub_system)
|
||||
.into_iter()
|
||||
.collect();
|
||||
}
|
||||
|
||||
let mut sub_systems = vec![NOTIFY_WEBHOOK_SUB_SYS];
|
||||
for sub_system in FULL_CONFIG_WORKER_SUBSYSTEMS {
|
||||
if !NOTIFY_SUB_SYSTEMS.contains(&sub_system) {
|
||||
sub_systems.push(sub_system);
|
||||
}
|
||||
}
|
||||
sub_systems
|
||||
}
|
||||
|
||||
async fn preflight_config_intent(sub_system: Option<&str>) -> S3Result<()> {
|
||||
for sub_system in config_preflight_subsystems(sub_system) {
|
||||
preflight_dynamic_config_reload(sub_system).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -1869,13 +1895,13 @@ async fn wait_notify_config_intent(transition: Option<rustfs_notify::Notificatio
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
async fn apply_non_notify_dynamic_subsystems(config: &ServerConfig) -> Vec<String> {
|
||||
async fn reload_non_notify_dynamic_subsystems() -> Vec<String> {
|
||||
let mut failures = Vec::new();
|
||||
for sub_system in FULL_CONFIG_WORKER_SUBSYSTEMS {
|
||||
if NOTIFY_SUB_SYSTEMS.contains(&sub_system) {
|
||||
continue;
|
||||
}
|
||||
if apply_dynamic_config_for_subsystem(config, sub_system).await.is_err() {
|
||||
if reload_dynamic_config_runtime_state(sub_system).await.is_err() {
|
||||
failures.push(format!("local {sub_system}"));
|
||||
warn!(
|
||||
event = EVENT_CONFIG_WORKER_RELOAD_FAILED,
|
||||
@@ -1904,7 +1930,6 @@ fn finish_config_reconciliation(errors: Vec<String>) -> S3Result<()> {
|
||||
}
|
||||
|
||||
async fn reconcile_targeted_config(
|
||||
config: ServerConfig,
|
||||
sub_system: Option<String>,
|
||||
storage_class_applied: bool,
|
||||
notify_transition: Option<rustfs_notify::NotificationLifecycleTransition>,
|
||||
@@ -1933,8 +1958,8 @@ async fn reconcile_targeted_config(
|
||||
} else if let Some(sub_system) = sub_system.as_deref()
|
||||
&& is_dynamic_config_subsystem(sub_system)
|
||||
{
|
||||
let config_applied = match apply_dynamic_config_for_subsystem(&config, sub_system).await {
|
||||
Ok(applied) => applied,
|
||||
let config_applied = match reload_dynamic_config_runtime_state(sub_system).await {
|
||||
Ok(()) => true,
|
||||
Err(_) => {
|
||||
warn!(config_subsystem = sub_system, reason = "apply_failed", "Local config reload failed");
|
||||
errors.push(format!("local {sub_system}"));
|
||||
@@ -1958,10 +1983,7 @@ async fn reconcile_targeted_config(
|
||||
Ok(config_applied)
|
||||
}
|
||||
|
||||
async fn reconcile_full_config(
|
||||
config: ServerConfig,
|
||||
notify_transition: Option<rustfs_notify::NotificationLifecycleTransition>,
|
||||
) -> S3Result<()> {
|
||||
async fn reconcile_full_config(notify_transition: Option<rustfs_notify::NotificationLifecycleTransition>) -> S3Result<()> {
|
||||
let mut errors = Vec::new();
|
||||
if let Err(err) = wait_notify_config_intent(notify_transition).await {
|
||||
warn!(error = %err, "Local notification config failed to converge");
|
||||
@@ -1971,7 +1993,7 @@ async fn reconcile_full_config(
|
||||
warn!(error = %err, "Peer storage-class reload failed");
|
||||
errors.push(format!("peer {STORAGE_CLASS_SUB_SYS}"));
|
||||
}
|
||||
errors.extend(apply_non_notify_dynamic_subsystems(&config).await);
|
||||
errors.extend(reload_non_notify_dynamic_subsystems().await);
|
||||
if let Err(err) = signal_dynamic_config_reload_checked(NOTIFY_WEBHOOK_SUB_SYS).await {
|
||||
warn!(error = %err, "Peer notification config reload failed");
|
||||
errors.push("peer notify".to_string());
|
||||
@@ -1983,6 +2005,90 @@ async fn reconcile_full_config(
|
||||
finish_config_reconciliation(errors)
|
||||
}
|
||||
|
||||
struct PersistedConfigTransaction {
|
||||
previous_config: ServerConfig,
|
||||
history_restore_id: Option<String>,
|
||||
storage_class_applied: bool,
|
||||
notify_transition: Option<rustfs_notify::NotificationLifecycleTransition>,
|
||||
}
|
||||
|
||||
async fn persist_server_config_transaction(
|
||||
config: ServerConfig,
|
||||
prepared: PreparedRuntimeConfig,
|
||||
snapshot: AdminServerConfigSnapshot,
|
||||
sub_system: Option<&str>,
|
||||
) -> S3Result<PersistedConfigTransaction> {
|
||||
snapshot.ensure_lock_held().map_err(ApiError::from).map_err(S3Error::from)?;
|
||||
let previous_config = snapshot.config.clone();
|
||||
let history_restore_id = save_server_config_history_snapshot(&previous_config).await?;
|
||||
if snapshot.is_lock_lost() {
|
||||
cleanup_failed_config_history_snapshot(&history_restore_id).await;
|
||||
return reject_lost_config_transaction(snapshot, "before persistence").await;
|
||||
}
|
||||
|
||||
let persisted = match save_server_config_to_store(&config, &snapshot).await {
|
||||
Ok(persisted) => persisted,
|
||||
Err(err) => {
|
||||
cleanup_failed_config_history_snapshot(&history_restore_id).await;
|
||||
return Err(err);
|
||||
}
|
||||
};
|
||||
let history_restore_id = if persisted {
|
||||
Some(history_restore_id)
|
||||
} else {
|
||||
cleanup_failed_config_history_snapshot(&history_restore_id).await;
|
||||
None
|
||||
};
|
||||
if snapshot.is_lock_lost() {
|
||||
return reject_lost_config_transaction(snapshot, "after persistence").await;
|
||||
}
|
||||
|
||||
let storage_class_applied = sub_system.is_none_or(|value| value == STORAGE_CLASS_SUB_SYS);
|
||||
if storage_class_applied {
|
||||
if let Err(err) = publish_prepared_config_snapshots(config.clone(), prepared) {
|
||||
return Err(match history_restore_id.as_deref() {
|
||||
Some(restore_id) => s3_error!(
|
||||
InternalError,
|
||||
"config persisted but runtime publish failed; recovery snapshot restoreId={}: {}",
|
||||
restore_id,
|
||||
err
|
||||
),
|
||||
None => err,
|
||||
});
|
||||
}
|
||||
} else {
|
||||
publish_server_config(config.clone());
|
||||
}
|
||||
let notify_transition = publish_notify_config_intent(&config, sub_system);
|
||||
if snapshot.is_lock_lost() {
|
||||
return reject_lost_config_transaction(snapshot, "while publishing runtime snapshots").await;
|
||||
}
|
||||
|
||||
drop(snapshot);
|
||||
Ok(PersistedConfigTransaction {
|
||||
previous_config,
|
||||
history_restore_id,
|
||||
storage_class_applied,
|
||||
notify_transition,
|
||||
})
|
||||
}
|
||||
|
||||
async fn commit_server_config_transaction(
|
||||
config: ServerConfig,
|
||||
prepared: PreparedRuntimeConfig,
|
||||
snapshot: AdminServerConfigSnapshot,
|
||||
sub_system: Option<String>,
|
||||
) -> S3Result<bool> {
|
||||
let transaction = persist_server_config_transaction(config, prepared, snapshot, sub_system.as_deref()).await?;
|
||||
|
||||
if sub_system.is_none() {
|
||||
reconcile_full_config(transaction.notify_transition).await?;
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
reconcile_targeted_config(sub_system, transaction.storage_class_applied, transaction.notify_transition).await
|
||||
}
|
||||
|
||||
pub struct GetConfigKVHandler {}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
@@ -2016,41 +2122,13 @@ impl Operation for SetConfigKVHandler {
|
||||
validate_config_directives(&directives)?;
|
||||
|
||||
let sub_system = config_update_sub_system(&directives)?.map(str::to_owned);
|
||||
let transaction_sub_system = sub_system.clone();
|
||||
let config_store = object_store()?;
|
||||
let config_applied = supervise_admin_mutation("config mutation", async move {
|
||||
preflight_notify_config_intent(sub_system.as_deref()).await?;
|
||||
let (config, storage_class_applied, notify_transition) =
|
||||
with_admin_server_config_write_lock(config_store, move || async move {
|
||||
let sub_system = transaction_sub_system.as_deref();
|
||||
let previous_config = load_server_config_from_store_locked().await?;
|
||||
let mut config = previous_config.clone();
|
||||
apply_set_directives(&mut config, &directives)?;
|
||||
let prepared = prepare_server_config(&config, sub_system).await?;
|
||||
let history_restore_id = save_server_config_history_snapshot(&previous_config).await?;
|
||||
if let Err(err) = save_server_config_to_store_locked(&config).await {
|
||||
cleanup_failed_config_history_snapshot(&history_restore_id).await;
|
||||
return Err(err);
|
||||
}
|
||||
if sub_system == Some(STORAGE_CLASS_SUB_SYS) {
|
||||
publish_prepared_config_snapshots(config.clone(), prepared).map_err(|err| {
|
||||
s3_error!(
|
||||
InternalError,
|
||||
"config persisted but runtime publish failed; recovery snapshot restoreId={}: {}",
|
||||
history_restore_id,
|
||||
err
|
||||
)
|
||||
})?;
|
||||
} else {
|
||||
publish_server_config(config.clone());
|
||||
}
|
||||
let notify_transition = publish_notify_config_intent(&config, sub_system);
|
||||
Ok::<_, S3Error>((config, sub_system == Some(STORAGE_CLASS_SUB_SYS), notify_transition))
|
||||
})
|
||||
.await
|
||||
.map_err(|err| s3_error!(InternalError, "failed to lock server config update: {}", err))??;
|
||||
|
||||
reconcile_targeted_config(config, sub_system, storage_class_applied, notify_transition).await
|
||||
preflight_config_intent(sub_system.as_deref()).await?;
|
||||
let snapshot = load_server_config_snapshot_from_store().await?;
|
||||
let mut config = snapshot.config.clone();
|
||||
apply_set_directives(&mut config, &directives)?;
|
||||
let prepared = prepare_server_config(&config, sub_system.as_deref()).await?;
|
||||
commit_server_config_transaction(config, prepared, snapshot, sub_system).await
|
||||
})
|
||||
.await?;
|
||||
|
||||
@@ -2072,41 +2150,13 @@ impl Operation for DelConfigKVHandler {
|
||||
validate_config_directives(&directives)?;
|
||||
|
||||
let sub_system = config_update_sub_system(&directives)?.map(str::to_owned);
|
||||
let transaction_sub_system = sub_system.clone();
|
||||
let config_store = object_store()?;
|
||||
let config_applied = supervise_admin_mutation("config mutation", async move {
|
||||
preflight_notify_config_intent(sub_system.as_deref()).await?;
|
||||
let (config, storage_class_applied, notify_transition) =
|
||||
with_admin_server_config_write_lock(config_store, move || async move {
|
||||
let sub_system = transaction_sub_system.as_deref();
|
||||
let previous_config = load_server_config_from_store_locked().await?;
|
||||
let mut config = previous_config.clone();
|
||||
apply_delete_directives(&mut config, &directives);
|
||||
let prepared = prepare_server_config(&config, sub_system).await?;
|
||||
let history_restore_id = save_server_config_history_snapshot(&previous_config).await?;
|
||||
if let Err(err) = save_server_config_to_store_locked(&config).await {
|
||||
cleanup_failed_config_history_snapshot(&history_restore_id).await;
|
||||
return Err(err);
|
||||
}
|
||||
if sub_system == Some(STORAGE_CLASS_SUB_SYS) {
|
||||
publish_prepared_config_snapshots(config.clone(), prepared).map_err(|err| {
|
||||
s3_error!(
|
||||
InternalError,
|
||||
"config persisted but runtime publish failed; recovery snapshot restoreId={}: {}",
|
||||
history_restore_id,
|
||||
err
|
||||
)
|
||||
})?;
|
||||
} else {
|
||||
publish_server_config(config.clone());
|
||||
}
|
||||
let notify_transition = publish_notify_config_intent(&config, sub_system);
|
||||
Ok::<_, S3Error>((config, sub_system == Some(STORAGE_CLASS_SUB_SYS), notify_transition))
|
||||
})
|
||||
.await
|
||||
.map_err(|err| s3_error!(InternalError, "failed to lock server config update: {}", err))??;
|
||||
|
||||
reconcile_targeted_config(config, sub_system, storage_class_applied, notify_transition).await
|
||||
preflight_config_intent(sub_system.as_deref()).await?;
|
||||
let snapshot = load_server_config_snapshot_from_store().await?;
|
||||
let mut config = snapshot.config.clone();
|
||||
apply_delete_directives(&mut config, &directives);
|
||||
let prepared = prepare_server_config(&config, sub_system.as_deref()).await?;
|
||||
commit_server_config_transaction(config, prepared, snapshot, sub_system).await
|
||||
})
|
||||
.await?;
|
||||
|
||||
@@ -2186,79 +2236,108 @@ impl Operation for RestoreConfigHistoryKVHandler {
|
||||
.ok_or_else(|| s3_error!(InvalidRequest, "missing restoreId query parameter"))?;
|
||||
let history = read_server_config_history(restore_id).await?;
|
||||
let config = decode_config_history_snapshot(&history)?;
|
||||
let prepared = prepare_server_config(&config, None).await?;
|
||||
let config_store = object_store()?;
|
||||
supervise_admin_mutation("config mutation", async move {
|
||||
preflight_notify_config_intent(None).await?;
|
||||
let persisted_config = config.clone();
|
||||
preflight_config_intent(None).await?;
|
||||
let snapshot = load_server_config_snapshot_from_store().await?;
|
||||
let prepared = prepare_server_config(&config, None).await?;
|
||||
let restored_config = config.clone();
|
||||
let rollback_store = config_store.clone();
|
||||
let (previous_config, recovery_restore_id, notify_transition) =
|
||||
with_admin_server_config_write_lock(config_store, move || async move {
|
||||
let previous_config = load_server_config_from_store_locked().await?;
|
||||
let recovery_restore_id = save_server_config_history_snapshot(&previous_config).await?;
|
||||
if let Err(err) = save_server_config_to_store_locked(&persisted_config).await {
|
||||
cleanup_failed_config_history_snapshot(&recovery_restore_id).await;
|
||||
return Err(err);
|
||||
}
|
||||
publish_prepared_config_snapshots(persisted_config.clone(), prepared).map_err(|err| {
|
||||
s3_error!(
|
||||
InternalError,
|
||||
"config restore persisted but runtime publish failed; recovery snapshot restoreId={}: {}",
|
||||
recovery_restore_id,
|
||||
err
|
||||
)
|
||||
})?;
|
||||
let notify_transition = publish_notify_config_intent(&persisted_config, None);
|
||||
Ok::<_, S3Error>((previous_config, recovery_restore_id, notify_transition))
|
||||
})
|
||||
.await
|
||||
.map_err(|err| s3_error!(InternalError, "failed to lock server config restore: {}", err))??;
|
||||
|
||||
let Err(restore_error) = reconcile_full_config(config, notify_transition).await else {
|
||||
let transaction = persist_server_config_transaction(config, prepared, snapshot, None).await?;
|
||||
let Err(restore_error) = reconcile_full_config(transaction.notify_transition).await else {
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
let rollback_config = previous_config.clone();
|
||||
let rollback_expected_config = restored_config.clone();
|
||||
let rollback_transition = with_admin_server_config_write_lock(rollback_store, move || async move {
|
||||
let current_config = load_server_config_from_store_locked().await?;
|
||||
validate_restore_rollback_generation(¤t_config, &rollback_expected_config)?;
|
||||
let rollback_prepared = prepare_server_config(&rollback_config, None).await?;
|
||||
commit_prepared_config(
|
||||
rollback_config.clone(),
|
||||
rollback_prepared,
|
||||
save_server_config_to_store_locked(&rollback_config),
|
||||
publish_prepared_config_snapshots,
|
||||
)
|
||||
.await?;
|
||||
Ok::<_, S3Error>(publish_notify_config_intent(&rollback_config, None))
|
||||
})
|
||||
.await
|
||||
.map_err(|err| s3_error!(InternalError, "failed to lock server config rollback: {}", err))?;
|
||||
|
||||
let rollback_transition = match rollback_transition {
|
||||
Ok(transition) => transition,
|
||||
let previous_config = transaction.previous_config;
|
||||
let recovery_restore_id = transaction.history_restore_id;
|
||||
let recovery_reference = recovery_restore_id.as_deref().unwrap_or("not-created");
|
||||
let rollback_snapshot = match load_server_config_snapshot_from_store().await {
|
||||
Ok(snapshot) => snapshot,
|
||||
Err(rollback_error) => {
|
||||
return Err(s3_error!(
|
||||
InternalError,
|
||||
"config restore failed: {}; automatic rollback failed: {}; recovery snapshot restoreId={}",
|
||||
restore_error,
|
||||
rollback_error,
|
||||
recovery_restore_id
|
||||
recovery_reference
|
||||
));
|
||||
}
|
||||
};
|
||||
if let Err(rollback_error) = reconcile_full_config(previous_config, rollback_transition).await {
|
||||
if let Err(rollback_error) = validate_restore_rollback_generation(&rollback_snapshot.config, &restored_config) {
|
||||
return Err(s3_error!(
|
||||
InternalError,
|
||||
"config restore failed: {}; automatic rollback failed: {}; recovery snapshot restoreId={}",
|
||||
restore_error,
|
||||
rollback_error,
|
||||
recovery_reference
|
||||
));
|
||||
}
|
||||
|
||||
let rollback_prepared = prepare_server_config(&previous_config, None).await?;
|
||||
rollback_snapshot
|
||||
.ensure_lock_held()
|
||||
.map_err(ApiError::from)
|
||||
.map_err(S3Error::from)?;
|
||||
if let Err(rollback_error) = save_server_config_to_store(&previous_config, &rollback_snapshot).await {
|
||||
return Err(s3_error!(
|
||||
InternalError,
|
||||
"config restore failed: {}; automatic rollback failed: {}; recovery snapshot restoreId={}",
|
||||
restore_error,
|
||||
rollback_error,
|
||||
recovery_reference
|
||||
));
|
||||
}
|
||||
if rollback_snapshot.is_lock_lost() {
|
||||
if let Err(rollback_error) =
|
||||
reject_lost_config_transaction::<()>(rollback_snapshot, "after restore rollback persistence").await
|
||||
{
|
||||
return Err(s3_error!(
|
||||
InternalError,
|
||||
"config restore failed: {}; automatic rollback failed: {}; recovery snapshot restoreId={}",
|
||||
restore_error,
|
||||
rollback_error,
|
||||
recovery_reference
|
||||
));
|
||||
}
|
||||
return Err(s3_error!(InternalError, "restore rollback lock-loss handling returned unexpectedly"));
|
||||
}
|
||||
|
||||
if let Err(rollback_error) = publish_prepared_config_snapshots(previous_config.clone(), rollback_prepared) {
|
||||
return Err(s3_error!(
|
||||
InternalError,
|
||||
"config restore failed: {}; automatic rollback publish failed: {}; recovery snapshot restoreId={}",
|
||||
restore_error,
|
||||
rollback_error,
|
||||
recovery_reference
|
||||
));
|
||||
}
|
||||
let rollback_transition = publish_notify_config_intent(&previous_config, None);
|
||||
if rollback_snapshot.is_lock_lost() {
|
||||
if let Err(rollback_error) =
|
||||
reject_lost_config_transaction::<()>(rollback_snapshot, "while publishing restore rollback").await
|
||||
{
|
||||
return Err(s3_error!(
|
||||
InternalError,
|
||||
"config restore failed: {}; automatic rollback failed: {}; recovery snapshot restoreId={}",
|
||||
restore_error,
|
||||
rollback_error,
|
||||
recovery_reference
|
||||
));
|
||||
}
|
||||
return Err(s3_error!(InternalError, "restore rollback lock-loss handling returned unexpectedly"));
|
||||
}
|
||||
drop(rollback_snapshot);
|
||||
|
||||
if let Err(rollback_error) = reconcile_full_config(rollback_transition).await {
|
||||
return Err(s3_error!(
|
||||
InternalError,
|
||||
"config restore failed: {}; persisted rollback did not converge: {}; recovery snapshot restoreId={}",
|
||||
restore_error,
|
||||
rollback_error,
|
||||
recovery_restore_id
|
||||
recovery_reference
|
||||
));
|
||||
}
|
||||
cleanup_failed_config_history_snapshot(&recovery_restore_id).await;
|
||||
if let Some(recovery_restore_id) = recovery_restore_id.as_deref() {
|
||||
cleanup_failed_config_history_snapshot(recovery_restore_id).await;
|
||||
}
|
||||
Err(s3_error!(InternalError, "config restore failed and was rolled back: {}", restore_error))
|
||||
})
|
||||
.await?;
|
||||
@@ -2293,34 +2372,14 @@ impl Operation for SetConfigHandler {
|
||||
}
|
||||
validate_config_directives(&directives)?;
|
||||
|
||||
let mut config = ServerConfig::new();
|
||||
apply_set_directives(&mut config, &directives)?;
|
||||
let prepared = prepare_server_config(&config, None).await?;
|
||||
let config_store = object_store()?;
|
||||
supervise_admin_mutation("config mutation", async move {
|
||||
preflight_notify_config_intent(None).await?;
|
||||
let persisted_config = config.clone();
|
||||
let notify_transition = with_admin_server_config_write_lock(config_store, move || async move {
|
||||
let previous_config = load_server_config_from_store_locked().await?;
|
||||
let history_restore_id = save_server_config_history_snapshot(&previous_config).await?;
|
||||
if let Err(err) = save_server_config_to_store_locked(&persisted_config).await {
|
||||
cleanup_failed_config_history_snapshot(&history_restore_id).await;
|
||||
return Err(err);
|
||||
}
|
||||
publish_prepared_config_snapshots(persisted_config.clone(), prepared).map_err(|err| {
|
||||
s3_error!(
|
||||
InternalError,
|
||||
"full config persisted but runtime publish failed; recovery snapshot restoreId={}: {}",
|
||||
history_restore_id,
|
||||
err
|
||||
)
|
||||
})?;
|
||||
Ok::<_, S3Error>(publish_notify_config_intent(&persisted_config, None))
|
||||
})
|
||||
.await
|
||||
.map_err(|err| s3_error!(InternalError, "failed to lock full server config update: {}", err))??;
|
||||
|
||||
reconcile_full_config(config, notify_transition).await
|
||||
preflight_config_intent(None).await?;
|
||||
let snapshot = load_server_config_snapshot_from_store().await?;
|
||||
let mut config = ServerConfig::new();
|
||||
apply_set_directives(&mut config, &directives)?;
|
||||
let prepared = prepare_server_config(&config, None).await?;
|
||||
commit_server_config_transaction(config, prepared, snapshot, None).await?;
|
||||
Ok(())
|
||||
})
|
||||
.await?;
|
||||
|
||||
@@ -2332,56 +2391,21 @@ impl Operation for SetConfigHandler {
|
||||
mod tests {
|
||||
use super::*;
|
||||
use serial_test::serial;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use temp_env::with_vars;
|
||||
|
||||
#[tokio::test]
|
||||
async fn prepared_config_commit_persists_before_publish() {
|
||||
let events = Arc::new(Mutex::new(Vec::new()));
|
||||
let persist_events = events.clone();
|
||||
let publish_events = events.clone();
|
||||
|
||||
commit_prepared_config(
|
||||
ServerConfig::new(),
|
||||
PreparedRuntimeConfig::default(),
|
||||
async move {
|
||||
persist_events.lock().expect("persist events lock").push("persist");
|
||||
Ok(())
|
||||
},
|
||||
move |_, _| {
|
||||
publish_events.lock().expect("publish events lock").push("publish");
|
||||
Ok(())
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("prepared config commit");
|
||||
|
||||
assert_eq!(*events.lock().expect("result events lock"), ["persist", "publish"]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn prepared_config_commit_does_not_publish_after_persist_failure() {
|
||||
let events = Arc::new(Mutex::new(Vec::new()));
|
||||
let persist_events = events.clone();
|
||||
let publish_events = events.clone();
|
||||
|
||||
let err = commit_prepared_config(
|
||||
ServerConfig::new(),
|
||||
PreparedRuntimeConfig::default(),
|
||||
async move {
|
||||
persist_events.lock().expect("persist events lock").push("persist");
|
||||
Err(s3_error!(InternalError, "injected persistence failure"))
|
||||
},
|
||||
move |_, _| {
|
||||
publish_events.lock().expect("publish events lock").push("publish");
|
||||
Ok(())
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("persistence failure must propagate");
|
||||
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
assert_eq!(*events.lock().expect("result events lock"), ["persist"]);
|
||||
#[test]
|
||||
fn config_preflight_covers_each_runtime_worker_family() {
|
||||
assert_eq!(config_preflight_subsystems(Some(SCANNER_SUB_SYS)), [SCANNER_SUB_SYS]);
|
||||
assert_eq!(config_preflight_subsystems(Some(HEAL_SUB_SYS)), [HEAL_SUB_SYS]);
|
||||
assert!(config_preflight_subsystems(Some("identity_openid")).is_empty());
|
||||
assert_eq!(
|
||||
config_preflight_subsystems(None),
|
||||
[
|
||||
NOTIFY_WEBHOOK_SUB_SYS,
|
||||
rustfs_config::audit::AUDIT_WEBHOOK_SUB_SYS,
|
||||
SCANNER_SUB_SYS
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -2499,6 +2523,17 @@ mod tests {
|
||||
assert!(!rendered_after_delete.contains("client_secret="));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn config_rendering_redacts_database_dsn() {
|
||||
let entry = KV {
|
||||
key: BASE_DSN_STRING.to_string(),
|
||||
value: "postgres://user:password@db.example/database".to_string(),
|
||||
hidden_if_empty: true,
|
||||
};
|
||||
|
||||
assert_eq!(render_entry_value(&entry, true), REDACTED_VALUE);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn full_config_export_can_be_reapplied() {
|
||||
crate::admin::storage_api::config::init_admin_config_defaults();
|
||||
@@ -2621,6 +2656,36 @@ identity_openid config_url="https://issuer.example" client_id="console""#,
|
||||
validate_config_directives(&directives).expect("scanner pacing keys should be supported");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_cycle_config_set_and_get_round_trip() {
|
||||
let mut config = ServerConfig::new();
|
||||
let directives = parse_config_directives("scanner cycle=61", false).expect("parse scanner cycle directive");
|
||||
validate_config_directives(&directives).expect("scanner cycle should be supported");
|
||||
apply_set_directives(&mut config, &directives).expect("apply scanner cycle directive");
|
||||
|
||||
assert_eq!(
|
||||
config
|
||||
.get_value(SCANNER_SUB_SYS, DEFAULT_DELIMITER)
|
||||
.expect("scanner KVS should exist")
|
||||
.lookup(SCANNER_CYCLE)
|
||||
.as_deref(),
|
||||
Some("61")
|
||||
);
|
||||
let rendered = String::from_utf8(
|
||||
render_selected_config(
|
||||
&config,
|
||||
&ConfigSelector {
|
||||
sub_system: SCANNER_SUB_SYS.to_string(),
|
||||
target: Some(DEFAULT_DELIMITER.to_string()),
|
||||
},
|
||||
true,
|
||||
)
|
||||
.expect("render scanner cycle config"),
|
||||
)
|
||||
.expect("scanner config should be UTF-8");
|
||||
assert!(rendered.contains("cycle=\"61\""));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_help_response_reports_scanner_delay() {
|
||||
let response = build_help_response(Some(SCANNER_SUB_SYS), Some(SCANNER_DELAY), false).expect("scanner help response");
|
||||
|
||||
@@ -13,6 +13,7 @@
|
||||
// limitations under the License.
|
||||
|
||||
use crate::admin::auth::validate_admin_request;
|
||||
use crate::admin::handlers::supervise_admin_mutation;
|
||||
use crate::admin::router::{AdminOperation, Operation, S3Router};
|
||||
use crate::admin::runtime_sources::{
|
||||
current_app_context, current_federated_identity_service, current_object_store_handle_for_context,
|
||||
@@ -20,8 +21,7 @@ use crate::admin::runtime_sources::{
|
||||
};
|
||||
use crate::admin::service::federated_identity::DefaultFederatedSessionBinding;
|
||||
use crate::admin::storage_api::config::{
|
||||
read_admin_config_without_migrate, read_admin_config_without_migrate_no_lock, save_admin_server_config_no_lock,
|
||||
with_admin_server_config_write_lock,
|
||||
read_admin_config_without_migrate, read_admin_server_config_snapshot, save_admin_server_config_snapshot,
|
||||
};
|
||||
use crate::auth::{check_key_valid, get_session_token};
|
||||
use crate::server::{ADMIN_PREFIX, CONSOLE_PREFIX, MINIO_ADMIN_PREFIX, RemoteAddr};
|
||||
@@ -381,17 +381,13 @@ impl Operation for PutOidcConfigHandler {
|
||||
let provider_id = provider_id.to_owned();
|
||||
|
||||
let request: OidcConfigUpsertRequest = parse_json_body(&mut req).await?;
|
||||
let store = oidc_config_store()?;
|
||||
let lock_store = store.clone();
|
||||
with_admin_server_config_write_lock(lock_store, move || async move {
|
||||
let mut config = load_server_config_from_store_locked(store.clone()).await?;
|
||||
let existing_secret = persisted_provider_secret(&config, &provider_id);
|
||||
update_oidc_server_config(move |config| {
|
||||
let existing_secret = persisted_provider_secret(config, &provider_id);
|
||||
let provider_config = build_provider_config_from_upsert(&provider_id, request, existing_secret)?;
|
||||
upsert_persisted_provider_config(&mut config, &provider_config);
|
||||
save_server_config_to_store_locked(store, &config).await
|
||||
upsert_persisted_provider_config(config, &provider_config);
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
.map_err(|err| s3_error!(InternalError, "failed to lock server config update: {}", err))??;
|
||||
.await?;
|
||||
|
||||
json_response(
|
||||
StatusCode::OK,
|
||||
@@ -422,15 +418,11 @@ impl Operation for DeleteOidcConfigHandler {
|
||||
}
|
||||
let provider_id = provider_id.to_owned();
|
||||
|
||||
let store = oidc_config_store()?;
|
||||
let lock_store = store.clone();
|
||||
with_admin_server_config_write_lock(lock_store, move || async move {
|
||||
let mut config = load_server_config_from_store_locked(store.clone()).await?;
|
||||
delete_persisted_provider_config(&mut config, &provider_id)?;
|
||||
save_server_config_to_store_locked(store, &config).await
|
||||
update_oidc_server_config(move |config| {
|
||||
delete_persisted_provider_config(config, &provider_id)?;
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
.map_err(|err| s3_error!(InternalError, "failed to lock server config update: {}", err))??;
|
||||
.await?;
|
||||
|
||||
json_response(
|
||||
StatusCode::OK,
|
||||
@@ -916,21 +908,23 @@ fn oidc_config_store() -> S3Result<std::sync::Arc<crate::admin::storage_api::run
|
||||
.ok_or_else(|| s3_error!(InternalError, "storage layer not initialized"))
|
||||
}
|
||||
|
||||
async fn load_server_config_from_store_locked(
|
||||
store: std::sync::Arc<crate::admin::storage_api::runtime::ECStore>,
|
||||
) -> S3Result<ServerConfig> {
|
||||
read_admin_config_without_migrate_no_lock(store)
|
||||
.await
|
||||
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("failed to load server config: {e}")))
|
||||
}
|
||||
|
||||
async fn save_server_config_to_store_locked(
|
||||
store: std::sync::Arc<crate::admin::storage_api::runtime::ECStore>,
|
||||
config: &ServerConfig,
|
||||
) -> S3Result<()> {
|
||||
save_admin_server_config_no_lock(store, config)
|
||||
.await
|
||||
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("failed to save server config: {e}")))
|
||||
async fn update_oidc_server_config<F>(modifier: F) -> S3Result<()>
|
||||
where
|
||||
F: FnOnce(&mut ServerConfig) -> S3Result<()> + Send + 'static,
|
||||
{
|
||||
let store = oidc_config_store()?;
|
||||
supervise_admin_mutation("OIDC config update", async move {
|
||||
let snapshot = read_admin_server_config_snapshot(store.clone())
|
||||
.await
|
||||
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("failed to load server config: {e}")))?;
|
||||
let mut config = snapshot.config.clone();
|
||||
modifier(&mut config)?;
|
||||
save_admin_server_config_snapshot(store, &config, &snapshot)
|
||||
.await
|
||||
.map(|_| ())
|
||||
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("failed to save server config: {e}")))
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
fn is_env_managed_provider(provider_id: &str) -> bool {
|
||||
|
||||
@@ -30,7 +30,9 @@ pub(crate) use crate::runtime_sources::{
|
||||
current_replication_stats_handle_for_context, current_server_config_for_context, current_token_signing_key,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use crate::runtime_sources::{IamInterface, KmsInterface, ServerConfigInterface, StorageClassInterface};
|
||||
pub(crate) use crate::runtime_sources::{
|
||||
IamInterface, KmsInterface, NotificationSystemInterface, ServerConfigInterface, StorageClassInterface,
|
||||
};
|
||||
use rustfs_config::server_config::Config;
|
||||
use rustfs_kms::KmsServiceManager;
|
||||
use rustfs_tls_runtime::{GlobalPublishedOutboundTlsState, TlsGeneration};
|
||||
|
||||
@@ -37,9 +37,12 @@ use rustfs_targets::config::{
|
||||
};
|
||||
use s3s::{S3Error, S3ErrorCode, S3Result};
|
||||
use std::future::Future;
|
||||
use tokio::sync::Mutex as AsyncMutex;
|
||||
use tracing::warn;
|
||||
use url::Url;
|
||||
|
||||
static RUNTIME_CONFIG_RELOAD_MUTEX: AsyncMutex<()> = AsyncMutex::const_new(());
|
||||
|
||||
pub fn is_dynamic_config_subsystem(sub_system: &str) -> bool {
|
||||
NOTIFY_SUB_SYSTEMS.contains(&sub_system)
|
||||
|| matches!(
|
||||
@@ -393,6 +396,7 @@ pub async fn apply_dynamic_config_for_subsystem(config: &ServerConfig, sub_syste
|
||||
}
|
||||
|
||||
pub async fn reload_dynamic_config_runtime_state_for_context(context: Option<&AppContext>, sub_system: &str) -> S3Result<()> {
|
||||
let _reload_guard = RUNTIME_CONFIG_RELOAD_MUTEX.lock().await;
|
||||
if sub_system == MODULE_SWITCHES_SIGNAL_SUBSYSTEM {
|
||||
let store = resolve_runtime_config_store_for_context(context)?;
|
||||
let notify_result = reconcile_event_notifier_from_store(store).await;
|
||||
@@ -414,6 +418,14 @@ pub async fn reload_dynamic_config_runtime_state_for_context(context: Option<&Ap
|
||||
warn!("peer reload_dynamic_config: failed to load server config for {sub_system}: {err}");
|
||||
internal_error(format!("failed to load server config: {err}"))
|
||||
})?;
|
||||
|
||||
if matches!(sub_system, SCANNER_SUB_SYS | HEAL_SUB_SYS) {
|
||||
validate_server_config_for_context(context, &config, Some(sub_system)).await?;
|
||||
// Scanner cycles refresh from the process-wide server config before
|
||||
// each pass. Publish the same validated snapshot first so that refresh
|
||||
// cannot overwrite this peer's dynamic scanner update with stale data.
|
||||
publish_server_config_for_context(context, config.clone());
|
||||
}
|
||||
apply_dynamic_config_for_subsystem_for_context(context, &config, sub_system)
|
||||
.await
|
||||
.inspect_err(|_| {
|
||||
@@ -463,6 +475,7 @@ where
|
||||
}
|
||||
|
||||
pub async fn reload_runtime_config_snapshot_for_context(context: Option<&AppContext>) -> S3Result<()> {
|
||||
let _reload_guard = RUNTIME_CONFIG_RELOAD_MUTEX.lock().await;
|
||||
let store = resolve_runtime_config_store_for_context(context)?;
|
||||
|
||||
reload_runtime_config_snapshot_with(
|
||||
@@ -641,13 +654,16 @@ pub async fn signal_config_snapshot_reload_checked() -> S3Result<()> {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::admin::runtime_sources::{IamInterface, KmsInterface, ServerConfigInterface, StorageClassInterface};
|
||||
use crate::admin::runtime_sources::{
|
||||
IamInterface, KmsInterface, NotificationSystemInterface, ServerConfigInterface, StorageClassInterface,
|
||||
};
|
||||
use crate::admin::storage_api::bucket::metadata::{BUCKET_LIFECYCLE_CONFIG, BUCKET_REPLICATION_CONFIG};
|
||||
use crate::admin::storage_api::config::{
|
||||
read_admin_config_without_migrate, read_admin_config_without_migrate_no_lock, save_admin_server_config,
|
||||
save_admin_server_config_no_lock, with_admin_server_config_write_lock,
|
||||
};
|
||||
use crate::admin::storage_api::error::StorageError;
|
||||
use crate::admin::storage_api::runtime_sources::NotificationSys;
|
||||
use crate::server::{
|
||||
ModuleSwitchSource, PersistedModuleSwitches, current_module_switch_snapshot, is_event_notifier_reconciled,
|
||||
refresh_persisted_module_switches_from, save_persisted_module_switches_to,
|
||||
@@ -656,7 +672,7 @@ mod tests {
|
||||
use crate::storage_api::startup::storage::{init_local_disks_with_instance_ctx, new_instance_ctx};
|
||||
use rustfs_config::notify::{ENV_NOTIFY_WEBHOOK_ENABLE, ENV_NOTIFY_WEBHOOK_ENDPOINT, NOTIFY_WEBHOOK_SUB_SYS};
|
||||
use rustfs_config::oidc::{OIDC_CLIENT_ID, OIDC_CONFIG_URL, OIDC_SCOPES};
|
||||
use rustfs_config::{HEAL_SUB_SYS, SCANNER_SUB_SYS};
|
||||
use rustfs_config::{ENV_SCANNER_CYCLE, HEAL_SUB_SYS, SCANNER_CYCLE, SCANNER_SUB_SYS};
|
||||
use rustfs_config::{MQTT_BROKER, MQTT_QUEUE_DIR, MQTT_TOPIC, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR};
|
||||
use rustfs_iam::{store::object::ObjectStore, sys::IamSys};
|
||||
use rustfs_kms::KmsServiceManager;
|
||||
@@ -675,6 +691,36 @@ mod tests {
|
||||
const REAL_STORE_TEST_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30);
|
||||
const REPLICATION_RELOAD_LABEL: &str = "replication";
|
||||
|
||||
struct TestNotificationSystemInterface(Arc<NotificationSys>);
|
||||
|
||||
impl NotificationSystemInterface for TestNotificationSystemInterface {
|
||||
fn handle(&self) -> Option<Arc<NotificationSys>> {
|
||||
Some(self.0.clone())
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn checked_scanner_reload_reports_unreachable_peer() {
|
||||
let temp_dir = TempDir::new().expect("scanner reload temp dir");
|
||||
let store = build_isolated_heterogeneous_store(temp_dir.path()).await;
|
||||
let mut notification_system = NotificationSys::new(EndpointServerPools::default()).await;
|
||||
notification_system.peer_clients.push(None);
|
||||
let context = AppContext::new(
|
||||
store,
|
||||
Arc::new(TestIamInterface),
|
||||
Arc::new(TestKmsInterface {
|
||||
manager: Arc::new(KmsServiceManager::new()),
|
||||
}),
|
||||
)
|
||||
.with_test_notification_system_interface(Arc::new(TestNotificationSystemInterface(Arc::new(notification_system))));
|
||||
|
||||
let err = signal_dynamic_config_reload_checked_for_context(Some(&context), SCANNER_SUB_SYS)
|
||||
.await
|
||||
.expect_err("scanner config must not report success when a peer did not reload it");
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
assert_eq!(err.message(), Some("1 peer(s) failed dynamic config reload for scanner"));
|
||||
}
|
||||
|
||||
fn without_storage_class_env<R>(f: impl FnOnce() -> R) -> R {
|
||||
temp_env::with_vars_unset(
|
||||
[
|
||||
@@ -778,6 +824,16 @@ mod tests {
|
||||
config
|
||||
}
|
||||
|
||||
fn scanner_server_config(cycle: &str) -> ServerConfig {
|
||||
let mut config = ServerConfig::new();
|
||||
let mut kvs = KVS::new();
|
||||
kvs.insert(SCANNER_CYCLE.to_string(), cycle.to_string());
|
||||
config
|
||||
.0
|
||||
.insert(SCANNER_SUB_SYS.to_string(), HashMap::from([(DEFAULT_DELIMITER.to_string(), kvs)]));
|
||||
config
|
||||
}
|
||||
|
||||
async fn build_isolated_heterogeneous_store(temp_dir: &Path) -> Arc<ECStore> {
|
||||
let mut pools = Vec::new();
|
||||
for (pool_index, drives_per_set) in [4, 2].into_iter().enumerate() {
|
||||
@@ -1124,6 +1180,47 @@ mod tests {
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(scanner_env)]
|
||||
async fn peer_scanner_reload_publishes_server_snapshot_used_by_cycle_refresh() {
|
||||
temp_env::async_with_vars([(ENV_SCANNER_CYCLE, None::<&str>)], async {
|
||||
let fixture = runtime_config_reload_fixture().await;
|
||||
let candidate = scanner_server_config("61");
|
||||
save_admin_server_config(fixture.context.object_store(), &candidate)
|
||||
.await
|
||||
.expect("persist scanner config");
|
||||
|
||||
reload_dynamic_config_runtime_state_for_context(Some(&fixture.context), SCANNER_SUB_SYS)
|
||||
.await
|
||||
.expect("scanner peer reload should apply persisted config");
|
||||
|
||||
assert_eq!(fixture.server_set_calls.load(Ordering::SeqCst), 1);
|
||||
let snapshot = fixture
|
||||
.server_snapshot
|
||||
.lock()
|
||||
.expect("server config result lock")
|
||||
.clone()
|
||||
.expect("scanner peer reload should publish a server config snapshot");
|
||||
assert_eq!(
|
||||
snapshot
|
||||
.get_value(SCANNER_SUB_SYS, DEFAULT_DELIMITER)
|
||||
.expect("scanner defaults should be present")
|
||||
.get(SCANNER_CYCLE),
|
||||
"61",
|
||||
"scanner peer reload must keep the process-wide config source current"
|
||||
);
|
||||
let runtime = rustfs_scanner::scanner_runtime_config_status();
|
||||
assert_eq!(runtime.cycle_interval_seconds.value, 61);
|
||||
assert_eq!(
|
||||
runtime.cycle_interval_seconds.source,
|
||||
rustfs_scanner::runtime_config::ScannerRuntimeConfigSource::Config
|
||||
);
|
||||
|
||||
rustfs_scanner::apply_scanner_runtime_config(&ServerConfig::new()).expect("restore scanner runtime defaults");
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn peer_full_reload_rejects_later_pool_without_publishing() {
|
||||
|
||||
@@ -639,10 +639,13 @@ pub(crate) async fn read_admin_config_without_migrate(api: Arc<ECStore>) -> Resu
|
||||
ecstore_config::com::read_config_without_migrate(api).await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn read_admin_config_without_migrate_no_lock(api: Arc<ECStore>) -> Result<rustfs_config::server_config::Config> {
|
||||
ecstore_config::com::read_config_without_migrate_no_lock(api).await
|
||||
}
|
||||
|
||||
pub(crate) type AdminServerConfigSnapshot = ecstore_config::com::ServerConfigSnapshot;
|
||||
|
||||
pub(crate) async fn save_admin_config(api: Arc<ECStore>, file: &str, data: Vec<u8>) -> Result<()> {
|
||||
ecstore_config::com::save_config(api, file, data).await
|
||||
}
|
||||
@@ -656,6 +659,7 @@ pub(crate) async fn save_admin_server_config(api: Arc<ECStore>, cfg: &rustfs_con
|
||||
ecstore_config::com::save_server_config(api, cfg).await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn save_admin_server_config_no_lock(
|
||||
api: Arc<ECStore>,
|
||||
cfg: &rustfs_config::server_config::Config,
|
||||
@@ -663,6 +667,7 @@ pub(crate) async fn save_admin_server_config_no_lock(
|
||||
ecstore_config::com::save_server_config_no_lock(api, cfg).await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn with_admin_server_config_write_lock<F, Fut, T>(api: Arc<ECStore>, operation: F) -> Result<T>
|
||||
where
|
||||
F: FnOnce() -> Fut + Send + 'static,
|
||||
@@ -672,6 +677,18 @@ where
|
||||
ecstore_config::com::with_server_config_write_lock(api, operation).await
|
||||
}
|
||||
|
||||
pub(crate) async fn read_admin_server_config_snapshot(api: Arc<ECStore>) -> Result<AdminServerConfigSnapshot> {
|
||||
ecstore_config::com::read_server_config_snapshot(api).await
|
||||
}
|
||||
|
||||
pub(crate) async fn save_admin_server_config_snapshot(
|
||||
api: Arc<ECStore>,
|
||||
cfg: &rustfs_config::server_config::Config,
|
||||
snapshot: &AdminServerConfigSnapshot,
|
||||
) -> Result<bool> {
|
||||
ecstore_config::com::save_server_config_snapshot(api, cfg, snapshot).await
|
||||
}
|
||||
|
||||
pub(crate) fn init_admin_config_defaults() {
|
||||
ecstore_config::init();
|
||||
}
|
||||
@@ -769,13 +786,16 @@ pub(crate) mod cluster {
|
||||
}
|
||||
|
||||
pub(crate) mod config {
|
||||
#[cfg(test)]
|
||||
pub(crate) use super::save_admin_server_config;
|
||||
pub(crate) use super::storageclass;
|
||||
pub(crate) use super::{
|
||||
RUSTFS_META_BUCKET, STORAGE_CLASS_SUB_SYS, delete_admin_config, init_admin_config_defaults, read_admin_config,
|
||||
read_admin_config_without_migrate, read_admin_config_without_migrate_no_lock, save_admin_config,
|
||||
save_admin_server_config_no_lock, with_admin_server_config_write_lock,
|
||||
AdminServerConfigSnapshot, RUSTFS_META_BUCKET, STORAGE_CLASS_SUB_SYS, delete_admin_config, init_admin_config_defaults,
|
||||
read_admin_config, read_admin_config_without_migrate, read_admin_server_config_snapshot, save_admin_config,
|
||||
save_admin_server_config_snapshot,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use super::{
|
||||
read_admin_config_without_migrate_no_lock, save_admin_server_config, save_admin_server_config_no_lock,
|
||||
with_admin_server_config_write_lock,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -247,8 +247,10 @@ impl DefaultAdminUsecase {
|
||||
Self::query_data_usage_info_with_store(store).await
|
||||
}
|
||||
|
||||
/// Serve the last persisted scanner snapshot plus the in-memory overlay.
|
||||
/// This request path must never trigger a live full-version listing
|
||||
/// Serve the last persisted scanner snapshot plus an authoritative
|
||||
/// single-process overlay. Distributed nodes must not promote their
|
||||
/// process-local absolute counters to cluster-wide bucket totals.
|
||||
/// This path never triggers a live full-version listing
|
||||
/// (rustfs/backlog#1306); freshness is owned by the scanner.
|
||||
pub(crate) async fn query_data_usage_info_with_store(store: Arc<ECStore>) -> AdminUsecaseResult<DataUsageInfo> {
|
||||
let mut info = Self::map_data_usage_load_result(load_data_usage_from_backend_cached(store.clone()).await)?;
|
||||
|
||||
@@ -47,7 +47,6 @@ use super::storage_api::bucket_usecase::contract::bucket::{
|
||||
use super::storage_api::bucket_usecase::contract::list::{
|
||||
ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsV2Info as StorageListObjectsV2Info, ListOperations as _,
|
||||
};
|
||||
use super::storage_api::bucket_usecase::data_usage::remove_bucket_usage_from_backend;
|
||||
use super::storage_api::bucket_usecase::error::StorageError;
|
||||
use super::storage_api::bucket_usecase::helper::{OperationHelper, spawn_background_with_context};
|
||||
use super::storage_api::bucket_usecase::object_utils::to_s3s_etag;
|
||||
@@ -120,13 +119,18 @@ use std::{
|
||||
borrow::Cow,
|
||||
collections::{HashMap, HashSet},
|
||||
fmt::Display,
|
||||
future::Future,
|
||||
io::Write,
|
||||
sync::Arc,
|
||||
sync::{Arc, LazyLock},
|
||||
};
|
||||
use tracing::{debug, error, info, instrument, warn};
|
||||
use tokio::sync::{Semaphore, TryAcquireError};
|
||||
use tracing::{Instrument as _, debug, error, info, instrument, warn};
|
||||
|
||||
const LOG_COMPONENT_APP: &str = "app";
|
||||
const LOG_SUBSYSTEM_BUCKET: &str = "bucket";
|
||||
const BUCKET_OPERATION_CONCURRENCY: usize = 8;
|
||||
static BUCKET_OPERATION_ADMISSION: LazyLock<Arc<Semaphore>> =
|
||||
LazyLock::new(|| Arc::new(Semaphore::new(BUCKET_OPERATION_CONCURRENCY)));
|
||||
use urlencoding::encode;
|
||||
|
||||
type ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
|
||||
@@ -1087,6 +1091,40 @@ pub struct DefaultBucketUsecase {
|
||||
context: Option<Arc<AppContext>>,
|
||||
}
|
||||
|
||||
async fn await_bucket_usecase_on_fresh_task<T, F>(operation: &'static str, future: F) -> S3Result<T>
|
||||
where
|
||||
T: Send + 'static,
|
||||
F: Future<Output = S3Result<T>> + Send + 'static,
|
||||
{
|
||||
await_bucket_usecase_on_fresh_task_with_admission(operation, Arc::clone(&BUCKET_OPERATION_ADMISSION), future).await
|
||||
}
|
||||
|
||||
async fn await_bucket_usecase_on_fresh_task_with_admission<T, F>(
|
||||
operation: &'static str,
|
||||
admission: Arc<Semaphore>,
|
||||
future: F,
|
||||
) -> S3Result<T>
|
||||
where
|
||||
T: Send + 'static,
|
||||
F: Future<Output = S3Result<T>> + Send + 'static,
|
||||
{
|
||||
let permit = admission.try_acquire_owned().map_err(|err| match err {
|
||||
TryAcquireError::NoPermits => {
|
||||
S3Error::with_message(S3ErrorCode::SlowDown, format!("{operation} concurrency limit reached; retry later"))
|
||||
}
|
||||
TryAcquireError::Closed => S3Error::with_message(S3ErrorCode::InternalError, format!("{operation} admission closed")),
|
||||
})?;
|
||||
tokio::spawn(
|
||||
async move {
|
||||
let _permit = permit;
|
||||
future.await
|
||||
}
|
||||
.in_current_span(),
|
||||
)
|
||||
.await
|
||||
.map_err(|err| S3Error::with_message(S3ErrorCode::InternalError, format!("{operation} task failed: {err}")))?
|
||||
}
|
||||
|
||||
impl DefaultBucketUsecase {
|
||||
#[cfg(test)]
|
||||
pub fn without_context() -> Self {
|
||||
@@ -1121,6 +1159,11 @@ impl DefaultBucketUsecase {
|
||||
fields(start_time=?time::OffsetDateTime::now_utc())
|
||||
)]
|
||||
pub async fn execute_create_bucket(&self, req: S3Request<CreateBucketInput>) -> S3Result<S3Response<CreateBucketOutput>> {
|
||||
let usecase = self.clone();
|
||||
await_bucket_usecase_on_fresh_task("bucket creation", async move { usecase.execute_create_bucket_inner(req).await }).await
|
||||
}
|
||||
|
||||
async fn execute_create_bucket_inner(&self, req: S3Request<CreateBucketInput>) -> S3Result<S3Response<CreateBucketOutput>> {
|
||||
let helper = OperationHelper::new(&req, EventName::BucketCreated, S3Operation::CreateBucket);
|
||||
let requester_is_owner = match req_info_ref(&req) {
|
||||
Ok(r) => r.is_owner,
|
||||
@@ -1179,7 +1222,15 @@ impl DefaultBucketUsecase {
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
pub async fn execute_delete_bucket(&self, mut req: S3Request<DeleteBucketInput>) -> S3Result<S3Response<DeleteBucketOutput>> {
|
||||
pub async fn execute_delete_bucket(&self, req: S3Request<DeleteBucketInput>) -> S3Result<S3Response<DeleteBucketOutput>> {
|
||||
let usecase = self.clone();
|
||||
await_bucket_usecase_on_fresh_task("bucket deletion", async move { usecase.execute_delete_bucket_inner(req).await }).await
|
||||
}
|
||||
|
||||
async fn execute_delete_bucket_inner(
|
||||
&self,
|
||||
mut req: S3Request<DeleteBucketInput>,
|
||||
) -> S3Result<S3Response<DeleteBucketOutput>> {
|
||||
let helper = OperationHelper::new(&req, EventName::BucketRemoved, S3Operation::DeleteBucket);
|
||||
let input = req.input.clone();
|
||||
|
||||
@@ -1218,12 +1269,8 @@ impl DefaultBucketUsecase {
|
||||
// Invalidate bucket validation cache
|
||||
crate::storage::invalidate_bucket_validation_cache(&input.bucket);
|
||||
|
||||
// Re-evaluate lifecycle/replication after bucket removal and keep an
|
||||
// absent-bucket dirty marker until a complete usage snapshot is durable.
|
||||
// Re-evaluate lifecycle and replication after bucket removal.
|
||||
rustfs_scanner::record_scanner_maintenance_change(&input.bucket);
|
||||
if let Err(err) = remove_bucket_usage_from_backend(store.clone(), &input.bucket).await {
|
||||
warn!(bucket = %input.bucket, error = ?err, "failed to remove deleted bucket from data usage");
|
||||
}
|
||||
|
||||
if let Err(err) = site_replication_delete_bucket_hook(&input.bucket, force).await {
|
||||
warn!(bucket = %input.bucket, error = ?err, "site replication delete bucket hook failed");
|
||||
@@ -2633,6 +2680,155 @@ mod tests {
|
||||
ServerSideEncryptionConfiguration, Tag, Transition, TransitionStorageClass,
|
||||
};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
||||
use std::time::Duration;
|
||||
use tokio::sync::Notify;
|
||||
|
||||
#[tokio::test]
|
||||
async fn bucket_usecase_task_finishes_post_commit_hooks_after_parent_cancellation() {
|
||||
let admission = Arc::new(Semaphore::new(1));
|
||||
let committed = Arc::new(Notify::new());
|
||||
let committed_wait = committed.notified();
|
||||
let release_hook = Arc::new(Notify::new());
|
||||
let finished = Arc::new(Notify::new());
|
||||
let finished_wait = finished.notified();
|
||||
let hook_ran = Arc::new(AtomicBool::new(false));
|
||||
let committed_for_task = committed.clone();
|
||||
let release_hook_for_task = release_hook.clone();
|
||||
let finished_for_task = finished.clone();
|
||||
let hook_ran_for_task = hook_ran.clone();
|
||||
let parent = tokio::spawn(await_bucket_usecase_on_fresh_task_with_admission(
|
||||
"test bucket operation",
|
||||
admission.clone(),
|
||||
async move {
|
||||
committed_for_task.notify_one();
|
||||
release_hook_for_task.notified().await;
|
||||
hook_ran_for_task.store(true, Ordering::SeqCst);
|
||||
finished_for_task.notify_one();
|
||||
Ok(())
|
||||
},
|
||||
));
|
||||
committed_wait.await;
|
||||
|
||||
parent.abort();
|
||||
let _ = parent.await;
|
||||
release_hook.notify_waiters();
|
||||
tokio::time::timeout(Duration::from_secs(1), finished_wait)
|
||||
.await
|
||||
.expect("the post-commit hook should finish after request cancellation");
|
||||
|
||||
assert!(
|
||||
hook_ran.load(Ordering::SeqCst),
|
||||
"the fresh task must own both the storage mutation and its post-commit hooks"
|
||||
);
|
||||
assert_eq!(admission.available_permits(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn saturated_bucket_usecase_admission_does_not_start_work() {
|
||||
let admission = Arc::new(Semaphore::new(1));
|
||||
let held = admission
|
||||
.clone()
|
||||
.acquire_owned()
|
||||
.await
|
||||
.expect("test admission should remain open");
|
||||
let started = Arc::new(AtomicBool::new(false));
|
||||
let started_for_task = started.clone();
|
||||
let result = tokio::time::timeout(
|
||||
Duration::from_secs(1),
|
||||
await_bucket_usecase_on_fresh_task_with_admission("test bucket operation", admission, async move {
|
||||
started_for_task.store(true, Ordering::SeqCst);
|
||||
Ok(())
|
||||
}),
|
||||
)
|
||||
.await
|
||||
.expect("saturated admission must fail within the bounded timeout");
|
||||
drop(held);
|
||||
|
||||
assert!(
|
||||
!started.load(Ordering::SeqCst),
|
||||
"saturated admission must not start a detached bucket operation"
|
||||
);
|
||||
assert_eq!(result.expect_err("saturated admission must fail fast").code(), &S3ErrorCode::SlowDown);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bucket_usecase_admission_rejects_excess_detached_tasks() {
|
||||
let admission = Arc::new(Semaphore::new(BUCKET_OPERATION_CONCURRENCY));
|
||||
let release = Arc::new(Semaphore::new(0));
|
||||
let started = Arc::new(AtomicUsize::new(0));
|
||||
let mut tasks = Vec::with_capacity(BUCKET_OPERATION_CONCURRENCY);
|
||||
|
||||
for _ in 0..BUCKET_OPERATION_CONCURRENCY {
|
||||
let admission_for_task = admission.clone();
|
||||
let release_for_task = release.clone();
|
||||
let started_for_task = started.clone();
|
||||
tasks.push(tokio::spawn(await_bucket_usecase_on_fresh_task_with_admission(
|
||||
"test bucket operation",
|
||||
admission_for_task,
|
||||
async move {
|
||||
started_for_task.fetch_add(1, Ordering::SeqCst);
|
||||
let _release = release_for_task
|
||||
.acquire()
|
||||
.await
|
||||
.expect("test release gate should remain open");
|
||||
Ok(())
|
||||
},
|
||||
)));
|
||||
}
|
||||
|
||||
tokio::time::timeout(Duration::from_secs(1), async {
|
||||
while started.load(Ordering::SeqCst) != BUCKET_OPERATION_CONCURRENCY {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("all admitted operations should start");
|
||||
|
||||
let ninth_started = Arc::new(AtomicBool::new(false));
|
||||
let ninth_started_for_task = ninth_started.clone();
|
||||
let ninth_result = tokio::time::timeout(
|
||||
Duration::from_secs(1),
|
||||
await_bucket_usecase_on_fresh_task_with_admission("test bucket operation", admission.clone(), async move {
|
||||
ninth_started_for_task.store(true, Ordering::SeqCst);
|
||||
Ok(())
|
||||
}),
|
||||
)
|
||||
.await
|
||||
.expect("an excess bucket transaction must fail within the bounded timeout");
|
||||
assert!(
|
||||
!ninth_started.load(Ordering::SeqCst),
|
||||
"the ninth bucket transaction must not start when admission is saturated"
|
||||
);
|
||||
assert_eq!(
|
||||
ninth_result.expect_err("the ninth bucket transaction must fail fast").code(),
|
||||
&S3ErrorCode::SlowDown
|
||||
);
|
||||
|
||||
release.add_permits(BUCKET_OPERATION_CONCURRENCY);
|
||||
for task in tasks {
|
||||
task.await
|
||||
.expect("bucket transaction parent should join")
|
||||
.expect("bucket transaction should succeed");
|
||||
}
|
||||
|
||||
await_bucket_usecase_on_fresh_task_with_admission("test bucket operation", admission, async { Ok(()) })
|
||||
.await
|
||||
.expect("admission must recover after active transactions finish");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bucket_usecase_task_maps_panics_to_internal_errors() {
|
||||
async fn panicking_bucket_operation() -> S3Result<()> {
|
||||
panic!("injected bucket task panic");
|
||||
}
|
||||
|
||||
let result = await_bucket_usecase_on_fresh_task("test bucket operation", panicking_bucket_operation()).await;
|
||||
|
||||
let err = result.expect_err("a bucket task panic must be returned as an error");
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
assert!(err.message().is_some_and(|message| message.contains("task failed")));
|
||||
}
|
||||
|
||||
fn build_request<T>(input: T, method: Method) -> S3Request<T> {
|
||||
S3Request {
|
||||
@@ -2662,6 +2858,25 @@ mod tests {
|
||||
&rest[..end]
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bucket_create_and_delete_use_admitted_fresh_tasks() {
|
||||
let source = include_str!("bucket_usecase.rs");
|
||||
for (method, operation, inner_method) in [
|
||||
("execute_create_bucket", "bucket creation", "execute_create_bucket_inner"),
|
||||
("execute_delete_bucket", "bucket deletion", "execute_delete_bucket_inner"),
|
||||
] {
|
||||
let body = usecase_method_source(source, method);
|
||||
assert!(
|
||||
body.contains(&format!("await_bucket_usecase_on_fresh_task(\"{operation}\"")),
|
||||
"{method} must use bounded detached-task admission"
|
||||
);
|
||||
assert!(
|
||||
body.contains(&format!("{inner_method}(req).await")),
|
||||
"{method} must run its mutation inside the admitted task"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bucket_metadata_config_changes_notify_peer_metadata_reload() {
|
||||
let source = include_str!("bucket_usecase.rs");
|
||||
|
||||
@@ -354,6 +354,14 @@ impl AppContext {
|
||||
self.storage_class = storage_class;
|
||||
self
|
||||
}
|
||||
|
||||
pub(crate) fn with_test_notification_system_interface(
|
||||
mut self,
|
||||
notification_system: Arc<dyn NotificationSystemInterface>,
|
||||
) -> Self {
|
||||
self.notification_system = notification_system;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
static APP_CONTEXT_SINGLETON: OnceLock<Arc<AppContext>> = OnceLock::new();
|
||||
|
||||
@@ -110,13 +110,6 @@ pub(crate) mod data_usage {
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
pub(crate) async fn remove_bucket_usage_from_backend(
|
||||
store: Arc<crate::storage::storage_api::ECStore>,
|
||||
bucket: &str,
|
||||
) -> Result<(), crate::storage::storage_api::StorageError> {
|
||||
crate::storage::storage_api::ecstore_data_usage::remove_bucket_usage_from_backend(store, bucket).await
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) mod runtime {
|
||||
@@ -956,7 +949,7 @@ pub(crate) mod bucket_usecase {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) use super::{access, bucket, data_usage, error, helper, object_utils, request_context, s3_api};
|
||||
pub(crate) use super::{access, bucket, error, helper, object_utils, request_context, s3_api};
|
||||
pub(crate) use crate::storage::storage_api::{
|
||||
ECStore, StorageObjectInfo, get_validated_store, process_lambda_configurations, process_queue_configurations,
|
||||
process_topic_configurations, validate_list_object_unordered_with_delimiter,
|
||||
|
||||
@@ -61,7 +61,7 @@ pub(crate) fn set_test_outbound_tls_generation(generation: u64) {
|
||||
pub(crate) use context::install_test_app_context;
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) use context::{IamInterface, KmsInterface, ServerConfigInterface, StorageClassInterface};
|
||||
pub(crate) use context::{IamInterface, KmsInterface, NotificationSystemInterface, ServerConfigInterface, StorageClassInterface};
|
||||
|
||||
pub(crate) fn current_app_context() -> Option<Arc<AppContext>> {
|
||||
context::get_global_app_context()
|
||||
|
||||
@@ -14,22 +14,26 @@
|
||||
|
||||
use crate::server::RPC_PREFIX;
|
||||
use crate::storage::request_context::spawn_traced;
|
||||
#[cfg(test)]
|
||||
use crate::storage::storage_api::rpc_consumer::http_service::WALK_DIR_BODY_SHA256_QUERY;
|
||||
use crate::storage::storage_api::rpc_consumer::http_service::{
|
||||
DEFAULT_READ_BUFFER_SIZE, StorageDiskRpcExt as _, WALK_DIR_STREAM_COMPLETION_V1, WalkDirOptions, find_local_disk_by_ref,
|
||||
verify_rpc_signature,
|
||||
DEFAULT_READ_BUFFER_SIZE, NS_SCANNER_PROTOCOL_VERSION, NsScannerCapabilityResponse, StorageDiskRpcExt as _,
|
||||
WALK_DIR_STREAM_COMPLETION_V1, WalkDirOptions, find_local_disk_by_ref, sign_ns_scanner_capability, verify_rpc_signature,
|
||||
};
|
||||
#[cfg(test)]
|
||||
use crate::storage::storage_api::rpc_consumer::http_service::{
|
||||
NS_SCANNER_BODY_SHA256_QUERY, NS_SCANNER_CAPABILITY_CHALLENGE_QUERY, NS_SCANNER_CYCLE_QUERY, NS_SCANNER_LEADER_EPOCH_QUERY,
|
||||
NS_SCANNER_REQUEST_ID_QUERY, NS_SCANNER_SERVER_EPOCH_QUERY, NS_SCANNER_SESSION_ID_QUERY, NS_SCANNER_SESSION_SEQUENCE_QUERY,
|
||||
WALK_DIR_BODY_SHA256_QUERY,
|
||||
};
|
||||
use crate::storage::storage_api::runtime_sources_consumer::runtime_sources;
|
||||
use bytes::{Bytes, BytesMut};
|
||||
use futures_util::{Stream, StreamExt, TryStreamExt, stream};
|
||||
use http::{HeaderMap, Method, Request, Response, StatusCode, Uri};
|
||||
use http::{HeaderMap, HeaderValue, Method, Request, Response, StatusCode, Uri};
|
||||
use http_body_util::{BodyExt, Limited};
|
||||
use hyper::body::Incoming;
|
||||
use rustfs_config::MAX_ADMIN_REQUEST_BODY_SIZE;
|
||||
use rustfs_io_metrics::internode_metrics::{
|
||||
INTERNODE_OPERATION_PUT_FILE_STREAM, INTERNODE_OPERATION_READ_FILE_STREAM, INTERNODE_OPERATION_WALK_DIR,
|
||||
INTERNODE_TRANSPORT_BACKEND_TCP_HTTP,
|
||||
INTERNODE_OPERATION_NS_SCANNER, INTERNODE_OPERATION_PUT_FILE_STREAM, INTERNODE_OPERATION_READ_FILE_STREAM,
|
||||
INTERNODE_OPERATION_WALK_DIR, INTERNODE_TRANSPORT_BACKEND_TCP_HTTP,
|
||||
};
|
||||
use rustfs_utils::net::bytes_stream;
|
||||
use s3s::Body;
|
||||
@@ -39,11 +43,12 @@ use serde_urlencoded::from_bytes;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::future::Future;
|
||||
use std::pin::Pin;
|
||||
use std::sync::LazyLock;
|
||||
use std::task::{Context, Poll};
|
||||
use std::time::Instant;
|
||||
use std::time::{Duration, Instant};
|
||||
use tokio::io::{self, AsyncWriteExt};
|
||||
use tokio::sync::oneshot;
|
||||
use tokio_util::io::ReaderStream;
|
||||
use tokio_util::{io::ReaderStream, sync::CancellationToken};
|
||||
use tower::Service;
|
||||
use tracing::{error, warn};
|
||||
|
||||
@@ -52,6 +57,7 @@ type RpcErrorResponse = Box<Response<Body>>;
|
||||
const LOG_COMPONENT_INTERNODE_RPC: &str = "internode_rpc";
|
||||
const LOG_SUBSYSTEM_FILE_TRANSFER: &str = "file_transfer";
|
||||
const LOG_SUBSYSTEM_DIRECTORY_WALK: &str = "directory_walk";
|
||||
const LOG_SUBSYSTEM_NAMESPACE_SCANNER: &str = "namespace_scanner";
|
||||
const LOG_SUBSYSTEM_ROUTING: &str = "routing";
|
||||
const EVENT_RPC_REQUEST_REJECTED: &str = "rpc_request_rejected";
|
||||
const EVENT_RPC_REQUEST_FAILED: &str = "rpc_request_failed";
|
||||
@@ -60,6 +66,10 @@ const RPC_OPERATION_UNKNOWN: &str = "unknown";
|
||||
const READ_FILE_STREAM_PATH: &str = "/rustfs/rpc/read_file_stream";
|
||||
const PUT_FILE_STREAM_PATH: &str = "/rustfs/rpc/put_file_stream";
|
||||
const WALK_DIR_PATH: &str = "/rustfs/rpc/walk_dir";
|
||||
const NS_SCANNER_PATH: &str = "/rustfs/rpc/ns_scanner";
|
||||
const NS_SCANNER_REQUEST_BODY_TIMEOUT: Duration = Duration::from_secs(15);
|
||||
const NS_SCANNER_STREAM_BUFFER_SIZE: usize = 64 * 1024;
|
||||
static NS_SCANNER_SERVER_EPOCH: LazyLock<uuid::Uuid> = LazyLock::new(uuid::Uuid::new_v4);
|
||||
|
||||
macro_rules! log_internode_rpc_response_failure {
|
||||
($status:expr, $rpc_path:expr, $method:expr, $operation:expr, $reason:expr, $result:expr, Some(($context_key:expr, $context_value:expr)), Some($error_text:expr)) => {{
|
||||
@@ -247,6 +257,34 @@ struct WalkDirQuery {
|
||||
walk_dir_body_sha256: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, serde::Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct NsScannerQuery {
|
||||
disk: String,
|
||||
ns_scanner_request_id: uuid::Uuid,
|
||||
ns_scanner_server_epoch: uuid::Uuid,
|
||||
ns_scanner_session_id: uuid::Uuid,
|
||||
ns_scanner_session_sequence: u64,
|
||||
ns_scanner_cycle: u64,
|
||||
ns_scanner_leader_epoch: u64,
|
||||
ns_scanner_body_sha256: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, serde::Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct NsScannerCapabilityQuery {
|
||||
ns_scanner_protocol: Option<u16>,
|
||||
ns_scanner_challenge: Option<uuid::Uuid>,
|
||||
}
|
||||
|
||||
fn verify_ns_scanner_body_digest(query: &NsScannerQuery, body: &[u8]) -> bool {
|
||||
let Some(expected) = query.ns_scanner_body_sha256.as_deref() else {
|
||||
return false;
|
||||
};
|
||||
let actual = hex_simd::encode_to_string(Sha256::digest(body), hex_simd::AsciiCase::Lower);
|
||||
expected == actual
|
||||
}
|
||||
|
||||
fn supports_walk_dir_stream_completion(query: &WalkDirQuery) -> bool {
|
||||
query.walk_dir_stream_completion.as_deref() == Some(WALK_DIR_STREAM_COMPLETION_V1)
|
||||
}
|
||||
@@ -322,6 +360,15 @@ async fn handle_internode_rpc(req: Request<Incoming>) -> Response<Body> {
|
||||
let response = match (method, path) {
|
||||
(Method::GET, READ_FILE_STREAM_PATH) | (Method::HEAD, READ_FILE_STREAM_PATH) => handle_read_file(req).await,
|
||||
(Method::GET, WALK_DIR_PATH) | (Method::HEAD, WALK_DIR_PATH) => handle_walk_dir(req).await,
|
||||
(Method::GET, NS_SCANNER_PATH) => match parse_query::<NsScannerCapabilityQuery>(&req) {
|
||||
Ok(query) if query.ns_scanner_protocol == Some(NS_SCANNER_PROTOCOL_VERSION) => match query.ns_scanner_challenge {
|
||||
Some(challenge) if !challenge.is_nil() => ns_scanner_capability_response(challenge),
|
||||
Some(_) | None => response_with_status(StatusCode::BAD_REQUEST, "namespace scanner challenge is invalid"),
|
||||
},
|
||||
Ok(_) => response_with_status(StatusCode::UPGRADE_REQUIRED, "namespace scanner protocol is unsupported"),
|
||||
Err(response) => *response,
|
||||
},
|
||||
(Method::POST, NS_SCANNER_PATH) => handle_ns_scanner(req).await,
|
||||
(Method::PUT, PUT_FILE_STREAM_PATH) => handle_put_file(req).await,
|
||||
_ => response_with_status(StatusCode::NOT_FOUND, "internode rpc route not found"),
|
||||
};
|
||||
@@ -346,6 +393,7 @@ fn internode_http_operation(path: &str) -> Option<&'static str> {
|
||||
READ_FILE_STREAM_PATH => Some(INTERNODE_OPERATION_READ_FILE_STREAM),
|
||||
PUT_FILE_STREAM_PATH => Some(INTERNODE_OPERATION_PUT_FILE_STREAM),
|
||||
WALK_DIR_PATH => Some(INTERNODE_OPERATION_WALK_DIR),
|
||||
NS_SCANNER_PATH => Some(INTERNODE_OPERATION_NS_SCANNER),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
@@ -358,6 +406,64 @@ fn record_internode_rpc_error(operation: Option<&'static str>) {
|
||||
}
|
||||
}
|
||||
|
||||
fn ns_scanner_capability_response(challenge: uuid::Uuid) -> Response<Body> {
|
||||
let server_epoch = *NS_SCANNER_SERVER_EPOCH;
|
||||
let proof = match sign_ns_scanner_capability(challenge, server_epoch) {
|
||||
Ok(proof) => proof,
|
||||
Err(err) => {
|
||||
error!(
|
||||
event = EVENT_RPC_REQUEST_FAILED,
|
||||
component = LOG_COMPONENT_INTERNODE_RPC,
|
||||
subsystem = LOG_SUBSYSTEM_NAMESPACE_SCANNER,
|
||||
operation = INTERNODE_OPERATION_NS_SCANNER,
|
||||
result = "failed",
|
||||
status_code = StatusCode::UPGRADE_REQUIRED.as_u16(),
|
||||
rpc_path = NS_SCANNER_PATH,
|
||||
method = %Method::GET,
|
||||
reason = "capability_authentication_unavailable",
|
||||
error = %err,
|
||||
"internode rpc request failed"
|
||||
);
|
||||
return response_with_status(StatusCode::UPGRADE_REQUIRED, "namespace scanner RPC authentication is unavailable");
|
||||
}
|
||||
};
|
||||
let body = match rmp_serde::to_vec_named(&NsScannerCapabilityResponse {
|
||||
version: NS_SCANNER_PROTOCOL_VERSION,
|
||||
server_epoch,
|
||||
proof,
|
||||
}) {
|
||||
Ok(body) => body,
|
||||
Err(err) => {
|
||||
error!(
|
||||
event = EVENT_RPC_REQUEST_FAILED,
|
||||
component = LOG_COMPONENT_INTERNODE_RPC,
|
||||
subsystem = LOG_SUBSYSTEM_NAMESPACE_SCANNER,
|
||||
operation = INTERNODE_OPERATION_NS_SCANNER,
|
||||
result = "failed",
|
||||
status_code = StatusCode::INTERNAL_SERVER_ERROR.as_u16(),
|
||||
rpc_path = NS_SCANNER_PATH,
|
||||
method = %Method::GET,
|
||||
reason = "capability_response_encode_failed",
|
||||
error = %err,
|
||||
"internode rpc request failed"
|
||||
);
|
||||
return response_with_status(
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
"namespace scanner capability response encoding failed",
|
||||
);
|
||||
}
|
||||
};
|
||||
let mut response = Response::new(Body::from(Bytes::from(body)));
|
||||
response
|
||||
.headers_mut()
|
||||
.insert(http::header::CONTENT_TYPE, HeaderValue::from_static("application/msgpack"));
|
||||
response
|
||||
}
|
||||
|
||||
fn ns_scanner_server_epoch_matches(server_epoch: uuid::Uuid) -> bool {
|
||||
server_epoch == *NS_SCANNER_SERVER_EPOCH
|
||||
}
|
||||
|
||||
fn verify_internode_rpc_signature(uri: &Uri, method: &Method, headers: &HeaderMap) -> Result<(), RpcErrorResponse> {
|
||||
if method == Method::HEAD {
|
||||
return Ok(());
|
||||
@@ -602,6 +708,304 @@ async fn handle_walk_dir(req: Request<Incoming>) -> Response<Body> {
|
||||
.expect("failed to build walk dir response")
|
||||
}
|
||||
|
||||
async fn handle_ns_scanner(req: Request<Incoming>) -> Response<Body> {
|
||||
let query = match parse_query::<NsScannerQuery>(&req) {
|
||||
Ok(query) => query,
|
||||
Err(response) => return *response,
|
||||
};
|
||||
if !ns_scanner_server_epoch_matches(query.ns_scanner_server_epoch) {
|
||||
warn!(
|
||||
event = EVENT_RPC_REQUEST_REJECTED,
|
||||
component = LOG_COMPONENT_INTERNODE_RPC,
|
||||
subsystem = LOG_SUBSYSTEM_NAMESPACE_SCANNER,
|
||||
operation = INTERNODE_OPERATION_NS_SCANNER,
|
||||
result = "rejected",
|
||||
status_code = StatusCode::CONFLICT.as_u16(),
|
||||
rpc_path = NS_SCANNER_PATH,
|
||||
method = %Method::POST,
|
||||
reason = "server_epoch_mismatch",
|
||||
"internode rpc request rejected"
|
||||
);
|
||||
return response_with_status(StatusCode::CONFLICT, "namespace scanner server epoch is stale");
|
||||
}
|
||||
let Some(disk) = find_local_disk_by_ref(&query.disk).await else {
|
||||
warn!(
|
||||
event = EVENT_RPC_REQUEST_REJECTED,
|
||||
component = LOG_COMPONENT_INTERNODE_RPC,
|
||||
subsystem = LOG_SUBSYSTEM_NAMESPACE_SCANNER,
|
||||
operation = INTERNODE_OPERATION_NS_SCANNER,
|
||||
result = "rejected",
|
||||
status_code = StatusCode::BAD_REQUEST.as_u16(),
|
||||
rpc_path = NS_SCANNER_PATH,
|
||||
method = %Method::POST,
|
||||
reason = "disk_not_found",
|
||||
disk = %query.disk,
|
||||
"internode rpc request rejected"
|
||||
);
|
||||
return response_with_status(StatusCode::BAD_REQUEST, "disk not found");
|
||||
};
|
||||
if let Err(err) = rustfs_scanner::preflight_remote_scanner_request(
|
||||
disk.as_ref(),
|
||||
query.ns_scanner_session_id,
|
||||
query.ns_scanner_cycle,
|
||||
query.ns_scanner_leader_epoch,
|
||||
query.ns_scanner_session_sequence,
|
||||
) {
|
||||
let (status, reason, message) = remote_scanner_claim_rejection(&err);
|
||||
warn!(
|
||||
event = EVENT_RPC_REQUEST_REJECTED,
|
||||
component = LOG_COMPONENT_INTERNODE_RPC,
|
||||
subsystem = LOG_SUBSYSTEM_NAMESPACE_SCANNER,
|
||||
operation = INTERNODE_OPERATION_NS_SCANNER,
|
||||
result = "rejected",
|
||||
status_code = status.as_u16(),
|
||||
rpc_path = NS_SCANNER_PATH,
|
||||
method = %Method::POST,
|
||||
reason,
|
||||
disk = %query.disk,
|
||||
error = %err,
|
||||
"internode rpc request rejected"
|
||||
);
|
||||
return response_with_status(status, message);
|
||||
}
|
||||
if let Err(err) =
|
||||
rustfs_scanner::validate_remote_scanner_request_fence(query.ns_scanner_cycle, query.ns_scanner_leader_epoch).await
|
||||
{
|
||||
warn!(
|
||||
event = EVENT_RPC_REQUEST_REJECTED,
|
||||
component = LOG_COMPONENT_INTERNODE_RPC,
|
||||
subsystem = LOG_SUBSYSTEM_NAMESPACE_SCANNER,
|
||||
operation = INTERNODE_OPERATION_NS_SCANNER,
|
||||
result = "rejected",
|
||||
status_code = StatusCode::CONFLICT.as_u16(),
|
||||
rpc_path = NS_SCANNER_PATH,
|
||||
method = %Method::POST,
|
||||
reason = "scanner_cycle_mismatch",
|
||||
disk = %query.disk,
|
||||
error = %err,
|
||||
"internode rpc request rejected"
|
||||
);
|
||||
return response_with_status(StatusCode::CONFLICT, "namespace scanner cycle does not match persisted state");
|
||||
}
|
||||
// Acquire the per-disk permit before reading the request body. This bounds
|
||||
// slow or duplicated authenticated uploads to one request per physical disk;
|
||||
// dropping the request on timeout releases the permit without consuming its
|
||||
// replay sequence.
|
||||
let admission = match rustfs_scanner::admit_remote_scanner_request(disk.as_ref()) {
|
||||
Ok(admission) => admission,
|
||||
Err(rustfs_scanner::ScannerError::RemoteDiskBusy) => {
|
||||
warn!(
|
||||
event = EVENT_RPC_REQUEST_REJECTED,
|
||||
component = LOG_COMPONENT_INTERNODE_RPC,
|
||||
subsystem = LOG_SUBSYSTEM_NAMESPACE_SCANNER,
|
||||
operation = INTERNODE_OPERATION_NS_SCANNER,
|
||||
result = "rejected",
|
||||
status_code = StatusCode::TOO_MANY_REQUESTS.as_u16(),
|
||||
rpc_path = NS_SCANNER_PATH,
|
||||
method = %Method::POST,
|
||||
reason = "disk_scan_already_active",
|
||||
disk = %query.disk,
|
||||
"internode rpc request rejected"
|
||||
);
|
||||
return response_with_status(StatusCode::TOO_MANY_REQUESTS, "namespace scanner disk is already active");
|
||||
}
|
||||
Err(err) => {
|
||||
let (status, reason, message) = remote_scanner_claim_rejection(&err);
|
||||
warn!(
|
||||
event = EVENT_RPC_REQUEST_REJECTED,
|
||||
component = LOG_COMPONENT_INTERNODE_RPC,
|
||||
subsystem = LOG_SUBSYSTEM_NAMESPACE_SCANNER,
|
||||
operation = INTERNODE_OPERATION_NS_SCANNER,
|
||||
result = "rejected",
|
||||
status_code = status.as_u16(),
|
||||
rpc_path = NS_SCANNER_PATH,
|
||||
method = %Method::POST,
|
||||
reason,
|
||||
disk = %query.disk,
|
||||
error = %err,
|
||||
"internode rpc request rejected"
|
||||
);
|
||||
return response_with_status(status, message);
|
||||
}
|
||||
};
|
||||
let body = match tokio::time::timeout(
|
||||
NS_SCANNER_REQUEST_BODY_TIMEOUT,
|
||||
Limited::new(req.into_body(), rustfs_scanner::NS_SCANNER_MAX_REQUEST_BODY_SIZE).collect(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(Ok(body)) => body.to_bytes(),
|
||||
Ok(Err(err)) => {
|
||||
log_internode_rpc_response_failure!(
|
||||
StatusCode::PAYLOAD_TOO_LARGE,
|
||||
NS_SCANNER_PATH,
|
||||
&Method::POST,
|
||||
Some(INTERNODE_OPERATION_NS_SCANNER),
|
||||
"request_body_read_failed",
|
||||
"rejected",
|
||||
Some(("disk", query.disk.as_str())),
|
||||
Some(&err)
|
||||
);
|
||||
return response_with_status(StatusCode::PAYLOAD_TOO_LARGE, "namespace scanner request body is too large");
|
||||
}
|
||||
Err(_) => {
|
||||
warn!(
|
||||
event = EVENT_RPC_REQUEST_REJECTED,
|
||||
component = LOG_COMPONENT_INTERNODE_RPC,
|
||||
subsystem = LOG_SUBSYSTEM_NAMESPACE_SCANNER,
|
||||
operation = INTERNODE_OPERATION_NS_SCANNER,
|
||||
result = "rejected",
|
||||
status_code = StatusCode::REQUEST_TIMEOUT.as_u16(),
|
||||
rpc_path = NS_SCANNER_PATH,
|
||||
method = %Method::POST,
|
||||
reason = "request_body_timeout",
|
||||
disk = %query.disk,
|
||||
"internode rpc request rejected"
|
||||
);
|
||||
return response_with_status(StatusCode::REQUEST_TIMEOUT, "namespace scanner request body timed out");
|
||||
}
|
||||
};
|
||||
|
||||
if !verify_ns_scanner_body_digest(&query, &body) {
|
||||
warn!(
|
||||
event = EVENT_RPC_REQUEST_REJECTED,
|
||||
component = LOG_COMPONENT_INTERNODE_RPC,
|
||||
subsystem = LOG_SUBSYSTEM_NAMESPACE_SCANNER,
|
||||
operation = INTERNODE_OPERATION_NS_SCANNER,
|
||||
result = "rejected",
|
||||
status_code = StatusCode::FORBIDDEN.as_u16(),
|
||||
rpc_path = NS_SCANNER_PATH,
|
||||
method = %Method::POST,
|
||||
reason = "request_body_digest_mismatch",
|
||||
disk = %query.disk,
|
||||
"internode rpc request rejected"
|
||||
);
|
||||
return response_with_status(StatusCode::FORBIDDEN, "invalid request body digest");
|
||||
}
|
||||
|
||||
let request = match rustfs_scanner::decode_remote_scanner_request(&body) {
|
||||
Ok(request) => request,
|
||||
Err(err) => {
|
||||
warn!(
|
||||
event = EVENT_RPC_REQUEST_REJECTED,
|
||||
component = LOG_COMPONENT_INTERNODE_RPC,
|
||||
subsystem = LOG_SUBSYSTEM_NAMESPACE_SCANNER,
|
||||
operation = INTERNODE_OPERATION_NS_SCANNER,
|
||||
result = "rejected",
|
||||
status_code = StatusCode::BAD_REQUEST.as_u16(),
|
||||
rpc_path = NS_SCANNER_PATH,
|
||||
method = %Method::POST,
|
||||
reason = "request_body_decode_failed",
|
||||
disk = %query.disk,
|
||||
error = %err,
|
||||
"internode rpc request rejected"
|
||||
);
|
||||
return response_with_status(StatusCode::BAD_REQUEST, "invalid namespace scanner request");
|
||||
}
|
||||
};
|
||||
if !rustfs_scanner::remote_scanner_request_matches_envelope(
|
||||
&request,
|
||||
query.ns_scanner_request_id,
|
||||
query.ns_scanner_server_epoch,
|
||||
query.ns_scanner_session_id,
|
||||
query.ns_scanner_session_sequence,
|
||||
query.ns_scanner_cycle,
|
||||
query.ns_scanner_leader_epoch,
|
||||
) {
|
||||
warn!(
|
||||
event = EVENT_RPC_REQUEST_REJECTED,
|
||||
component = LOG_COMPONENT_INTERNODE_RPC,
|
||||
subsystem = LOG_SUBSYSTEM_NAMESPACE_SCANNER,
|
||||
operation = INTERNODE_OPERATION_NS_SCANNER,
|
||||
result = "rejected",
|
||||
status_code = StatusCode::FORBIDDEN.as_u16(),
|
||||
rpc_path = NS_SCANNER_PATH,
|
||||
method = %Method::POST,
|
||||
reason = "request_envelope_mismatch",
|
||||
disk = %query.disk,
|
||||
"internode rpc request rejected"
|
||||
);
|
||||
return response_with_status(StatusCode::FORBIDDEN, "namespace scanner request envelope mismatch");
|
||||
}
|
||||
if let Err(err) = rustfs_scanner::claim_remote_scanner_request(
|
||||
disk.as_ref(),
|
||||
query.ns_scanner_session_id,
|
||||
query.ns_scanner_cycle,
|
||||
query.ns_scanner_leader_epoch,
|
||||
query.ns_scanner_session_sequence,
|
||||
) {
|
||||
let (status, reason, message) = remote_scanner_claim_rejection(&err);
|
||||
warn!(
|
||||
event = EVENT_RPC_REQUEST_REJECTED,
|
||||
component = LOG_COMPONENT_INTERNODE_RPC,
|
||||
subsystem = LOG_SUBSYSTEM_NAMESPACE_SCANNER,
|
||||
operation = INTERNODE_OPERATION_NS_SCANNER,
|
||||
result = "rejected",
|
||||
status_code = status.as_u16(),
|
||||
rpc_path = NS_SCANNER_PATH,
|
||||
method = %Method::POST,
|
||||
reason,
|
||||
disk = %query.disk,
|
||||
error = %err,
|
||||
"internode rpc request rejected"
|
||||
);
|
||||
return response_with_status(status, message);
|
||||
}
|
||||
|
||||
let metrics = runtime_sources::current_internode_metrics();
|
||||
metrics
|
||||
.record_incoming_request_for_operation_and_backend(INTERNODE_OPERATION_NS_SCANNER, INTERNODE_TRANSPORT_BACKEND_TCP_HTTP);
|
||||
metrics.record_recv_bytes_for_operation_and_backend(
|
||||
INTERNODE_OPERATION_NS_SCANNER,
|
||||
INTERNODE_TRANSPORT_BACKEND_TCP_HTTP,
|
||||
body.len(),
|
||||
);
|
||||
|
||||
let log_disk = query.disk;
|
||||
let response_body = ns_scanner_response_body(move |writer, disconnect| async move {
|
||||
let _admission = admission;
|
||||
rustfs_scanner::serve_remote_scanner_request(disk, request, writer, disconnect)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
error!(
|
||||
event = EVENT_RPC_BACKGROUND_TASK_FAILED,
|
||||
component = LOG_COMPONENT_INTERNODE_RPC,
|
||||
subsystem = LOG_SUBSYSTEM_NAMESPACE_SCANNER,
|
||||
operation = INTERNODE_OPERATION_NS_SCANNER,
|
||||
result = "failed",
|
||||
disk = %log_disk,
|
||||
error = %err,
|
||||
"internode rpc background task failed"
|
||||
);
|
||||
io::Error::other("remote namespace scanner failed")
|
||||
})
|
||||
});
|
||||
|
||||
let mut response = Response::new(response_body);
|
||||
response
|
||||
.headers_mut()
|
||||
.insert(http::header::CONTENT_TYPE, HeaderValue::from_static("application/msgpack"));
|
||||
response
|
||||
}
|
||||
|
||||
fn remote_scanner_claim_rejection(error: &rustfs_scanner::ScannerError) -> (StatusCode, &'static str, &'static str) {
|
||||
match error {
|
||||
rustfs_scanner::ScannerError::RemoteRequestReplay => {
|
||||
(StatusCode::CONFLICT, "request_replay", "namespace scanner request was already accepted")
|
||||
}
|
||||
rustfs_scanner::ScannerError::RemoteReplayCapacity => (
|
||||
StatusCode::TOO_MANY_REQUESTS,
|
||||
"replay_capacity",
|
||||
"namespace scanner request capacity is temporarily exhausted",
|
||||
),
|
||||
_ => (
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"replay_state_unavailable",
|
||||
"namespace scanner replay state is unavailable",
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
fn walk_dir_response_body<F, Fut>(propagate_completion_errors: bool, producer: F) -> Body
|
||||
where
|
||||
F: FnOnce(tokio::io::DuplexStream) -> Fut + Send + 'static,
|
||||
@@ -656,6 +1060,52 @@ where
|
||||
)
|
||||
}
|
||||
|
||||
fn ns_scanner_response_body<F, Fut>(producer: F) -> Body
|
||||
where
|
||||
F: FnOnce(tokio::io::DuplexStream, CancellationToken) -> Fut + Send + 'static,
|
||||
Fut: Future<Output = io::Result<()>> + Send + 'static,
|
||||
{
|
||||
let (reader, writer) = tokio::io::duplex(NS_SCANNER_STREAM_BUFFER_SIZE);
|
||||
let (mut completion_tx, completion_rx) = oneshot::channel();
|
||||
let disconnect = CancellationToken::new();
|
||||
spawn_traced(async move {
|
||||
let producer = producer(writer, disconnect.clone());
|
||||
tokio::pin!(producer);
|
||||
tokio::select! {
|
||||
biased;
|
||||
result = &mut producer => {
|
||||
let _ = completion_tx.send(result);
|
||||
}
|
||||
_ = completion_tx.closed() => {
|
||||
disconnect.cancel();
|
||||
let _ = producer.await;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
let metrics = runtime_sources::current_internode_metrics();
|
||||
let stream = ReaderStream::with_capacity(reader, NS_SCANNER_STREAM_BUFFER_SIZE).map_ok(move |bytes| {
|
||||
metrics.record_sent_bytes_for_operation_and_backend(
|
||||
INTERNODE_OPERATION_NS_SCANNER,
|
||||
INTERNODE_TRANSPORT_BACKEND_TCP_HTTP,
|
||||
bytes.len(),
|
||||
);
|
||||
bytes
|
||||
});
|
||||
let completion = stream::once(async move {
|
||||
match completion_rx.await {
|
||||
Ok(Ok(())) => None,
|
||||
Ok(Err(err)) => Some(Err(err)),
|
||||
Err(err) => Some(Err(io::Error::other(format!(
|
||||
"remote namespace scanner task ended without a result: {err}"
|
||||
)))),
|
||||
}
|
||||
})
|
||||
.filter_map(std::future::ready);
|
||||
|
||||
Body::from(StreamingBlob::wrap(stream.chain(completion)))
|
||||
}
|
||||
|
||||
async fn handle_put_file(req: Request<Incoming>) -> Response<Body> {
|
||||
let method = req.method().clone();
|
||||
let path = req.uri().path().to_string();
|
||||
@@ -805,6 +1255,7 @@ fn response_with_status(status: StatusCode, message: impl Into<String>) -> Respo
|
||||
fn internode_rpc_subsystem(operation: Option<&'static str>) -> &'static str {
|
||||
match operation {
|
||||
Some(INTERNODE_OPERATION_WALK_DIR) => LOG_SUBSYSTEM_DIRECTORY_WALK,
|
||||
Some(INTERNODE_OPERATION_NS_SCANNER) => LOG_SUBSYSTEM_NAMESPACE_SCANNER,
|
||||
Some(INTERNODE_OPERATION_READ_FILE_STREAM | INTERNODE_OPERATION_PUT_FILE_STREAM) => LOG_SUBSYSTEM_FILE_TRANSFER,
|
||||
_ => LOG_SUBSYSTEM_ROUTING,
|
||||
}
|
||||
@@ -828,19 +1279,23 @@ fn put_file_stage_error_message(stage: &str, query: &PutFileQuery, err: &dyn std
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
LOG_SUBSYSTEM_DIRECTORY_WALK, LOG_SUBSYSTEM_FILE_TRANSFER, LOG_SUBSYSTEM_ROUTING, PUT_FILE_STREAM_PATH, PutFileQuery,
|
||||
LOG_SUBSYSTEM_DIRECTORY_WALK, LOG_SUBSYSTEM_FILE_TRANSFER, LOG_SUBSYSTEM_NAMESPACE_SCANNER, LOG_SUBSYSTEM_ROUTING,
|
||||
NS_SCANNER_BODY_SHA256_QUERY, NS_SCANNER_CAPABILITY_CHALLENGE_QUERY, NS_SCANNER_CYCLE_QUERY,
|
||||
NS_SCANNER_LEADER_EPOCH_QUERY, NS_SCANNER_PATH, NS_SCANNER_REQUEST_ID_QUERY, NS_SCANNER_SERVER_EPOCH_QUERY,
|
||||
NS_SCANNER_SESSION_ID_QUERY, NS_SCANNER_SESSION_SEQUENCE_QUERY, NsScannerQuery, PUT_FILE_STREAM_PATH, PutFileQuery,
|
||||
READ_FILE_STREAM_PATH, WALK_DIR_BODY_SHA256_QUERY, WALK_DIR_PATH, WalkDirQuery, append_walk_dir_completion,
|
||||
internode_http_operation, internode_rpc_subsystem, is_internode_rpc_path, put_body_size_mismatch,
|
||||
put_file_stage_error_message, read_file_body_stream, supports_walk_dir_stream_completion,
|
||||
validate_walk_dir_completion_request, verify_internode_rpc_signature, verify_walk_dir_body_digest,
|
||||
walk_dir_response_body, write_body_chunks_to_writer,
|
||||
internode_http_operation, internode_rpc_subsystem, is_internode_rpc_path, ns_scanner_response_body,
|
||||
ns_scanner_server_epoch_matches, put_body_size_mismatch, put_file_stage_error_message, read_file_body_stream,
|
||||
remote_scanner_claim_rejection, supports_walk_dir_stream_completion, validate_walk_dir_completion_request,
|
||||
verify_internode_rpc_signature, verify_ns_scanner_body_digest, verify_walk_dir_body_digest, walk_dir_response_body,
|
||||
write_body_chunks_to_writer,
|
||||
};
|
||||
use bytes::Bytes;
|
||||
use http::{HeaderMap, Method, StatusCode, Uri};
|
||||
use http_body_util::BodyExt;
|
||||
use rustfs_io_metrics::internode_metrics::{
|
||||
INTERNODE_OPERATION_PUT_FILE_STREAM, INTERNODE_OPERATION_READ_FILE_STREAM, INTERNODE_OPERATION_WALK_DIR,
|
||||
global_internode_metrics,
|
||||
INTERNODE_OPERATION_NS_SCANNER, INTERNODE_OPERATION_PUT_FILE_STREAM, INTERNODE_OPERATION_READ_FILE_STREAM,
|
||||
INTERNODE_OPERATION_WALK_DIR, global_internode_metrics,
|
||||
};
|
||||
use sha2::Digest as _;
|
||||
use tokio::io;
|
||||
@@ -862,6 +1317,7 @@ mod tests {
|
||||
fn internode_rpc_path_matches_rpc_prefix() {
|
||||
assert!(is_internode_rpc_path("/rustfs/rpc/read_file_stream"));
|
||||
assert!(is_internode_rpc_path("/rustfs/rpc/walk_dir"));
|
||||
assert!(is_internode_rpc_path("/rustfs/rpc/ns_scanner"));
|
||||
assert!(!is_internode_rpc_path("/rustfs/admin/v3/info"));
|
||||
}
|
||||
|
||||
@@ -873,16 +1329,37 @@ mod tests {
|
||||
);
|
||||
assert_eq!(internode_http_operation(PUT_FILE_STREAM_PATH), Some(INTERNODE_OPERATION_PUT_FILE_STREAM));
|
||||
assert_eq!(internode_http_operation(WALK_DIR_PATH), Some(INTERNODE_OPERATION_WALK_DIR));
|
||||
assert_eq!(internode_http_operation(NS_SCANNER_PATH), Some(INTERNODE_OPERATION_NS_SCANNER));
|
||||
assert_eq!(internode_http_operation("/rustfs/rpc/unknown"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rpc_head_signature_verification_is_skipped() {
|
||||
fn file_stream_head_signature_verification_is_skipped() {
|
||||
let uri: Uri = READ_FILE_STREAM_PATH.parse().expect("uri");
|
||||
let headers = HeaderMap::new();
|
||||
assert!(verify_internode_rpc_signature(&uri, &Method::HEAD, &headers).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn namespace_scanner_capability_get_requires_signature() {
|
||||
let challenge = uuid::Uuid::new_v4();
|
||||
let uri: Uri = format!(
|
||||
"{NS_SCANNER_PATH}?ns_scanner_protocol={}&{NS_SCANNER_CAPABILITY_CHALLENGE_QUERY}={challenge}",
|
||||
super::NS_SCANNER_PROTOCOL_VERSION
|
||||
)
|
||||
.parse()
|
||||
.expect("uri");
|
||||
let headers = HeaderMap::new();
|
||||
let response = verify_internode_rpc_signature(&uri, &Method::GET, &headers).expect_err("response");
|
||||
assert_eq!(response.status(), StatusCode::FORBIDDEN);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn namespace_scanner_rejects_requests_from_a_prior_server_epoch() {
|
||||
assert!(ns_scanner_server_epoch_matches(*super::NS_SCANNER_SERVER_EPOCH));
|
||||
assert!(!ns_scanner_server_epoch_matches(uuid::Uuid::nil()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rpc_get_request_requires_signature() {
|
||||
let uri: Uri = READ_FILE_STREAM_PATH.parse().expect("uri");
|
||||
@@ -941,9 +1418,92 @@ mod tests {
|
||||
LOG_SUBSYSTEM_FILE_TRANSFER
|
||||
);
|
||||
assert_eq!(internode_rpc_subsystem(Some(INTERNODE_OPERATION_WALK_DIR)), LOG_SUBSYSTEM_DIRECTORY_WALK);
|
||||
assert_eq!(
|
||||
internode_rpc_subsystem(Some(INTERNODE_OPERATION_NS_SCANNER)),
|
||||
LOG_SUBSYSTEM_NAMESPACE_SCANNER
|
||||
);
|
||||
assert_eq!(internode_rpc_subsystem(None), LOG_SUBSYSTEM_ROUTING);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn namespace_scanner_requires_matching_signed_body_digest() {
|
||||
let body = b"scanner-request";
|
||||
let digest = hex_simd::encode_to_string(sha2::Sha256::digest(body), hex_simd::AsciiCase::Lower);
|
||||
let request_id = uuid::Uuid::new_v4();
|
||||
let server_epoch = uuid::Uuid::new_v4();
|
||||
let session_id = uuid::Uuid::new_v4();
|
||||
let valid: NsScannerQuery = serde_urlencoded::from_str(&format!(
|
||||
"disk=disk-a&{NS_SCANNER_REQUEST_ID_QUERY}={request_id}&{NS_SCANNER_SERVER_EPOCH_QUERY}={server_epoch}&{NS_SCANNER_SESSION_ID_QUERY}={session_id}&{NS_SCANNER_SESSION_SEQUENCE_QUERY}=0&{NS_SCANNER_CYCLE_QUERY}=7&{NS_SCANNER_LEADER_EPOCH_QUERY}=9&{NS_SCANNER_BODY_SHA256_QUERY}={digest}"
|
||||
))
|
||||
.expect("query should parse");
|
||||
let missing: NsScannerQuery = serde_urlencoded::from_str(&format!(
|
||||
"disk=disk-a&{NS_SCANNER_REQUEST_ID_QUERY}={request_id}&{NS_SCANNER_SERVER_EPOCH_QUERY}={server_epoch}&{NS_SCANNER_SESSION_ID_QUERY}={session_id}&{NS_SCANNER_SESSION_SEQUENCE_QUERY}=0&{NS_SCANNER_CYCLE_QUERY}=7&{NS_SCANNER_LEADER_EPOCH_QUERY}=9"
|
||||
))
|
||||
.expect("query should parse");
|
||||
|
||||
assert!(verify_ns_scanner_body_digest(&valid, body));
|
||||
assert!(!verify_ns_scanner_body_digest(&valid, b"tampered"));
|
||||
assert!(!verify_ns_scanner_body_digest(&missing, body));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn namespace_scanner_queries_reject_unknown_fields() {
|
||||
let request_id = uuid::Uuid::new_v4();
|
||||
let server_epoch = uuid::Uuid::new_v4();
|
||||
let session_id = uuid::Uuid::new_v4();
|
||||
let query = format!(
|
||||
"disk=disk-a&{NS_SCANNER_REQUEST_ID_QUERY}={request_id}&{NS_SCANNER_SERVER_EPOCH_QUERY}={server_epoch}&{NS_SCANNER_SESSION_ID_QUERY}={session_id}&{NS_SCANNER_SESSION_SEQUENCE_QUERY}=0&{NS_SCANNER_CYCLE_QUERY}=7&{NS_SCANNER_LEADER_EPOCH_QUERY}=9&{NS_SCANNER_BODY_SHA256_QUERY}=digest&unexpected=true"
|
||||
);
|
||||
assert!(serde_urlencoded::from_str::<NsScannerQuery>(&query).is_err());
|
||||
assert!(serde_urlencoded::from_str::<super::NsScannerCapabilityQuery>("ns_scanner_protocol=1&unexpected=true").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn namespace_scanner_replay_capacity_is_retryable_without_weakening_replay_rejection() {
|
||||
let (replay_status, replay_reason, _) =
|
||||
remote_scanner_claim_rejection(&rustfs_scanner::ScannerError::RemoteRequestReplay);
|
||||
let (capacity_status, capacity_reason, _) =
|
||||
remote_scanner_claim_rejection(&rustfs_scanner::ScannerError::RemoteReplayCapacity);
|
||||
|
||||
assert_eq!(replay_status, StatusCode::CONFLICT);
|
||||
assert_eq!(replay_reason, "request_replay");
|
||||
assert_eq!(capacity_status, StatusCode::TOO_MANY_REQUESTS);
|
||||
assert_eq!(capacity_reason, "replay_capacity");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn namespace_scanner_body_surfaces_background_failure() {
|
||||
let body = ns_scanner_response_body(|mut writer, _disconnect| async move {
|
||||
writer.write_all(b"partial scanner frame").await?;
|
||||
Err(io::Error::other("remote namespace scanner failed"))
|
||||
});
|
||||
let err = BodyExt::collect(body)
|
||||
.await
|
||||
.expect_err("failed completion must fail body collection");
|
||||
|
||||
assert!(err.to_string().contains("remote namespace scanner failed"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dropping_namespace_scanner_body_cancels_producer() {
|
||||
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
|
||||
let (dropped_tx, dropped_rx) = tokio::sync::oneshot::channel();
|
||||
let body = ns_scanner_response_body(move |_writer, disconnect| async move {
|
||||
let _drop_notifier = DropNotifier(Some(dropped_tx));
|
||||
let _ = started_tx.send(());
|
||||
disconnect.cancelled().await;
|
||||
Ok(())
|
||||
});
|
||||
|
||||
started_rx.await.expect("namespace scanner producer should start");
|
||||
drop(body);
|
||||
|
||||
tokio::time::timeout(std::time::Duration::from_secs(1), dropped_rx)
|
||||
.await
|
||||
.expect("dropping the response body should cancel the namespace scanner producer")
|
||||
.expect("drop notifier should send a cancellation signal");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn write_body_chunks_to_writer_streams_all_chunks() {
|
||||
let (mut reader, mut writer) = tokio::io::duplex(64);
|
||||
|
||||
@@ -23,8 +23,9 @@ use crate::storage::storage_api::rpc_consumer::node_service::STORAGE_CLASS_SUB_S
|
||||
use crate::storage::storage_api::rpc_consumer::node_service::{CollectMetricsOpts, MetricType};
|
||||
use crate::storage::storage_api::rpc_consumer::node_service::{
|
||||
DiskStore, ECStore, Error, LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS,
|
||||
SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, StorageDiskRpcExt as _, StorageResult, all_local_disk_path,
|
||||
find_local_disk_by_ref, reload_transition_tier_config,
|
||||
SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION, SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SERVICE_SIGNAL_REFRESH_CONFIG,
|
||||
SERVICE_SIGNAL_RELOAD_DYNAMIC, StorageDiskRpcExt as _, StorageResult, all_local_disk_path, find_local_disk_by_ref,
|
||||
reload_transition_tier_config,
|
||||
};
|
||||
use crate::storage::storage_api::runtime_sources_consumer::{EndpointServerPools, runtime_sources};
|
||||
use crate::storage::storage_api::{sign_tonic_rpc_response_proof, verify_tonic_canonical_body_digest};
|
||||
@@ -78,6 +79,14 @@ const TIER_MUTATION_PEER_STATE_PREPARED_WIRE: i32 = 1;
|
||||
const TIER_MUTATION_PEER_STATE_COMMITTED_WIRE: i32 = 2;
|
||||
const TIER_MUTATION_PEER_STATE_ABORTED_WIRE: i32 = 3;
|
||||
|
||||
fn signal_service_response(success: bool, error_info: Option<String>) -> Response<SignalServiceResponse> {
|
||||
Response::new(SignalServiceResponse {
|
||||
success,
|
||||
error_info,
|
||||
protocol_version: rustfs_protos::DYNAMIC_CONFIG_PROTOCOL_VERSION,
|
||||
})
|
||||
}
|
||||
|
||||
fn supports_dynamic_config_rpc(sub_system: &str) -> bool {
|
||||
NOTIFY_SUB_SYSTEMS.contains(&sub_system)
|
||||
|| matches!(
|
||||
@@ -174,11 +183,54 @@ fn validate_admin_heal_control_start(request: &rustfs_common::heal_channel::Heal
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn scanner_activity_response(namespace_generation: u64) -> ScannerActivityResponse {
|
||||
fn scanner_activity_response(
|
||||
namespace_generation: u64,
|
||||
topology_digest: [u8; 32],
|
||||
data_movement_active: bool,
|
||||
dirty_usage: rustfs_scanner::ScannerDirtyUsageState,
|
||||
) -> ScannerActivityResponse {
|
||||
ScannerActivityResponse {
|
||||
instance_id: rustfs_scanner::scanner_activity_epoch().to_string(),
|
||||
namespace_generation,
|
||||
maintenance_generation: rustfs_scanner::scanner_maintenance_generation(),
|
||||
protocol_version: rustfs_scanner::SCANNER_ACTIVITY_PROTOCOL_VERSION,
|
||||
topology_digest: topology_digest.to_vec().into(),
|
||||
data_movement_active,
|
||||
response_proof: Bytes::new(),
|
||||
dirty_usage_generation: dirty_usage.generation,
|
||||
dirty_usage_pending: dirty_usage.pending,
|
||||
}
|
||||
}
|
||||
|
||||
fn previous_scanner_activity_response(
|
||||
namespace_generation: u64,
|
||||
topology_digest: [u8; 32],
|
||||
data_movement_active: bool,
|
||||
) -> ScannerActivityResponse {
|
||||
ScannerActivityResponse {
|
||||
instance_id: rustfs_scanner::scanner_activity_epoch().to_string(),
|
||||
namespace_generation,
|
||||
maintenance_generation: rustfs_scanner::scanner_maintenance_generation(),
|
||||
protocol_version: SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION,
|
||||
topology_digest: topology_digest.to_vec().into(),
|
||||
data_movement_active,
|
||||
response_proof: Bytes::new(),
|
||||
dirty_usage_generation: 0,
|
||||
dirty_usage_pending: false,
|
||||
}
|
||||
}
|
||||
|
||||
fn legacy_scanner_activity_response(namespace_generation: u64) -> ScannerActivityResponse {
|
||||
ScannerActivityResponse {
|
||||
instance_id: rustfs_scanner::scanner_activity_epoch().to_string(),
|
||||
namespace_generation,
|
||||
maintenance_generation: rustfs_scanner::scanner_maintenance_generation(),
|
||||
protocol_version: SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION,
|
||||
topology_digest: Bytes::new(),
|
||||
data_movement_active: false,
|
||||
response_proof: Bytes::new(),
|
||||
dirty_usage_generation: 0,
|
||||
dirty_usage_pending: false,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1559,73 +1611,149 @@ impl Node for NodeService {
|
||||
Some(value) => match value.parse::<bool>() {
|
||||
Ok(value) => value,
|
||||
Err(_) => {
|
||||
return Ok(Response::new(SignalServiceResponse {
|
||||
success: false,
|
||||
error_info: Some(format!("invalid dry-run value: {value}")),
|
||||
}));
|
||||
return Ok(signal_service_response(false, Some(format!("invalid dry-run value: {value}"))));
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
match signal {
|
||||
Some(SERVICE_SIGNAL_REFRESH_CONFIG) => match reload_runtime_config_snapshot().await {
|
||||
Ok(()) => Ok(Response::new(SignalServiceResponse {
|
||||
success: true,
|
||||
error_info: None,
|
||||
})),
|
||||
Err(_) => Ok(Response::new(SignalServiceResponse {
|
||||
success: false,
|
||||
error_info: Some("runtime config snapshot reload failed".to_string()),
|
||||
})),
|
||||
Ok(()) => Ok(signal_service_response(true, None)),
|
||||
Err(_) => Ok(signal_service_response(false, Some("runtime config snapshot reload failed".to_string()))),
|
||||
},
|
||||
Some(SERVICE_SIGNAL_RELOAD_DYNAMIC) => {
|
||||
let supported = sub_system == MODULE_SWITCHES_SIGNAL_SUBSYSTEM || supports_dynamic_config_rpc(sub_system);
|
||||
if !supported {
|
||||
return Ok(Response::new(SignalServiceResponse {
|
||||
success: false,
|
||||
error_info: Some(format!("unsupported dynamic config subsystem: {sub_system}")),
|
||||
}));
|
||||
return Ok(signal_service_response(
|
||||
false,
|
||||
Some(format!("unsupported dynamic config subsystem: {sub_system}")),
|
||||
));
|
||||
}
|
||||
if dry_run {
|
||||
return Ok(Response::new(SignalServiceResponse {
|
||||
success: true,
|
||||
error_info: None,
|
||||
}));
|
||||
return Ok(signal_service_response(true, None));
|
||||
}
|
||||
match reload_dynamic_config_runtime_state(sub_system).await {
|
||||
Ok(()) => Ok(Response::new(SignalServiceResponse {
|
||||
success: true,
|
||||
error_info: None,
|
||||
})),
|
||||
Err(_) => Ok(Response::new(SignalServiceResponse {
|
||||
success: false,
|
||||
error_info: Some(format!("dynamic config reload failed for {sub_system}")),
|
||||
})),
|
||||
Ok(()) => Ok(signal_service_response(true, None)),
|
||||
Err(_) => Ok(signal_service_response(
|
||||
false,
|
||||
Some(format!("dynamic config reload failed for {sub_system}")),
|
||||
)),
|
||||
}
|
||||
}
|
||||
Some(other) => Ok(Response::new(SignalServiceResponse {
|
||||
success: false,
|
||||
error_info: Some(format!("unsupported service signal: {other}")),
|
||||
})),
|
||||
None if raw_signal.is_some() => Ok(Response::new(SignalServiceResponse {
|
||||
success: false,
|
||||
error_info: Some(format!("invalid service signal value: {}", raw_signal.unwrap_or_default())),
|
||||
})),
|
||||
None => Ok(Response::new(SignalServiceResponse {
|
||||
success: false,
|
||||
error_info: Some("missing service signal".to_string()),
|
||||
})),
|
||||
Some(other) => Ok(signal_service_response(false, Some(format!("unsupported service signal: {other}")))),
|
||||
None if raw_signal.is_some() => Ok(signal_service_response(
|
||||
false,
|
||||
Some(format!("invalid service signal value: {}", raw_signal.unwrap_or_default())),
|
||||
)),
|
||||
None => Ok(signal_service_response(false, Some("missing service signal".to_string()))),
|
||||
}
|
||||
}
|
||||
|
||||
async fn scanner_activity(
|
||||
&self,
|
||||
_request: Request<ScannerActivityRequest>,
|
||||
request: Request<ScannerActivityRequest>,
|
||||
) -> Result<Response<ScannerActivityResponse>, Status> {
|
||||
let request_protocol = request.get_ref().protocol_version;
|
||||
match request_protocol {
|
||||
// RUSTFS_COMPAT_TODO(ns-scanner-rpc-v3): legacy request body is unbound. Remove after protocol v0 peers are unsupported.
|
||||
SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION => {
|
||||
if !request.get_ref().acknowledge_instance_id.is_empty()
|
||||
|| request.get_ref().acknowledge_dirty_usage_generation != 0
|
||||
{
|
||||
return Err(Status::invalid_argument("legacy scanner activity request cannot acknowledge dirty usage"));
|
||||
}
|
||||
}
|
||||
SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION => {
|
||||
verify_tonic_canonical_body_digest(&request, request.get_ref().challenge.as_ref())
|
||||
.map_err(|err| Status::permission_denied(format!("scanner activity authentication failed: {err}")))?;
|
||||
if !request.get_ref().acknowledge_instance_id.is_empty()
|
||||
|| request.get_ref().acknowledge_dirty_usage_generation != 0
|
||||
{
|
||||
return Err(Status::invalid_argument("scanner activity protocol v4 cannot acknowledge dirty usage"));
|
||||
}
|
||||
}
|
||||
rustfs_scanner::SCANNER_ACTIVITY_PROTOCOL_VERSION => {
|
||||
let canonical = rustfs_protos::canonical_scanner_activity_request_body(request.get_ref())
|
||||
.map_err(|_| Status::invalid_argument("scanner activity request is too large to authenticate"))?;
|
||||
verify_tonic_canonical_body_digest(&request, &canonical)
|
||||
.map_err(|err| Status::permission_denied(format!("scanner activity authentication failed: {err}")))?;
|
||||
let has_acknowledgement = !request.get_ref().acknowledge_instance_id.is_empty();
|
||||
if has_acknowledgement != (request.get_ref().acknowledge_dirty_usage_generation != 0) {
|
||||
return Err(Status::invalid_argument(
|
||||
"scanner dirty usage acknowledgement requires both instance ID and generation",
|
||||
));
|
||||
}
|
||||
}
|
||||
version => {
|
||||
return Err(Status::failed_precondition(format!(
|
||||
"unsupported scanner activity request protocol {version}"
|
||||
)));
|
||||
}
|
||||
}
|
||||
let challenge_len = request.get_ref().challenge.len();
|
||||
if challenge_len != 16 && !(request_protocol == SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION && challenge_len == 0) {
|
||||
return Err(Status::invalid_argument("scanner activity challenge must be 16 bytes"));
|
||||
}
|
||||
let store = self
|
||||
.resolve_object_store()
|
||||
.ok_or_else(|| Status::unavailable("storage layer is not initialized"))?;
|
||||
Ok(Response::new(scanner_activity_response(store.scanner_namespace_mutation_generation())))
|
||||
if request.get_ref().challenge.is_empty() {
|
||||
// Older peers send an empty protocol-0 request and cannot establish
|
||||
// the topology fence required for distributed usage publication.
|
||||
return Ok(Response::new(legacy_scanner_activity_response(
|
||||
store.scanner_namespace_mutation_generation(),
|
||||
)));
|
||||
}
|
||||
let request = request.into_inner();
|
||||
let challenge: [u8; 16] = request
|
||||
.challenge
|
||||
.as_ref()
|
||||
.try_into()
|
||||
.map_err(|_| Status::invalid_argument("scanner activity challenge must be 16 bytes"))?;
|
||||
if !request.acknowledge_instance_id.is_empty() {
|
||||
rustfs_scanner::acknowledge_dirty_usage_generation(
|
||||
&request.acknowledge_instance_id,
|
||||
request.acknowledge_dirty_usage_generation,
|
||||
)
|
||||
.map_err(|err| Status::failed_precondition(err.to_string()))?;
|
||||
}
|
||||
let namespace_generation = store.scanner_namespace_mutation_generation();
|
||||
let topology_digest = rustfs_scanner::scanner_topology_digest(store.as_ref());
|
||||
let data_movement_active = store.scanner_data_movement_active().await;
|
||||
let mut response = match request_protocol {
|
||||
SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION | SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION => {
|
||||
previous_scanner_activity_response(namespace_generation, topology_digest, data_movement_active)
|
||||
}
|
||||
rustfs_scanner::SCANNER_ACTIVITY_PROTOCOL_VERSION => scanner_activity_response(
|
||||
namespace_generation,
|
||||
topology_digest,
|
||||
data_movement_active,
|
||||
rustfs_scanner::scanner_dirty_usage_state(),
|
||||
),
|
||||
version => {
|
||||
return Err(Status::failed_precondition(format!(
|
||||
"unsupported scanner activity request protocol {version}"
|
||||
)));
|
||||
}
|
||||
};
|
||||
let canonical = match response.protocol_version {
|
||||
SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION => {
|
||||
rustfs_protos::canonical_scanner_activity_v4_response_body(&challenge, &response)
|
||||
}
|
||||
rustfs_scanner::SCANNER_ACTIVITY_PROTOCOL_VERSION => {
|
||||
rustfs_protos::canonical_scanner_activity_response_body(&challenge, &response)
|
||||
}
|
||||
version => {
|
||||
return Err(Status::internal(format!(
|
||||
"scanner activity response selected unsupported protocol {version}"
|
||||
)));
|
||||
}
|
||||
}
|
||||
.map_err(|_| Status::internal("scanner activity response is too large to authenticate"))?;
|
||||
response.response_proof = sign_tonic_rpc_response_proof(&canonical)
|
||||
.map_err(|_| Status::unavailable("scanner activity response authentication is unavailable"))?
|
||||
.into();
|
||||
Ok(Response::new(response))
|
||||
}
|
||||
|
||||
async fn background_heal_status(
|
||||
@@ -1882,11 +2010,13 @@ impl Node for NodeService {
|
||||
mod tests {
|
||||
use super::{
|
||||
CollectMetricsOpts, DiskStore, Error, HEAL_CONTROL_PAYLOAD_MAX_SIZE, MetricType, Node as _, NodeService,
|
||||
PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC,
|
||||
PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION,
|
||||
SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC,
|
||||
STORAGE_CLASS_SUB_SYS, admit_heal_control_replay, background_rebalance_start_error_message,
|
||||
execute_heal_control_envelope_with_manager, initialize_heal_topology_fingerprint, make_heal_control_server,
|
||||
make_heal_control_server_with_cache, make_server, make_server_for_context, make_tier_mutation_control_server_for_context,
|
||||
remove_heal_control_replay, scanner_activity_response, stop_rebalance_response,
|
||||
execute_heal_control_envelope_with_manager, initialize_heal_topology_fingerprint, legacy_scanner_activity_response,
|
||||
make_heal_control_server, make_heal_control_server_with_cache, make_server, make_server_for_context,
|
||||
make_tier_mutation_control_server_for_context, previous_scanner_activity_response, remove_heal_control_replay,
|
||||
scanner_activity_response, stop_rebalance_response,
|
||||
};
|
||||
use crate::storage::rpc::node_service::heal::heal_topology_fingerprint;
|
||||
use crate::storage::storage_api::rpc_consumer::node_service::{HealBucketInfo, HealEndpoint};
|
||||
@@ -4296,24 +4426,199 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_scanner_activity_requires_storage_layer() {
|
||||
async fn test_scanner_activity_requires_body_bound_auth_before_storage_lookup() {
|
||||
let service = create_test_node_service();
|
||||
|
||||
let err = service
|
||||
.scanner_activity(Request::new(ScannerActivityRequest {}))
|
||||
let legacy = service
|
||||
.scanner_activity(Request::new(ScannerActivityRequest {
|
||||
challenge: vec![7; 16].into(),
|
||||
protocol_version: SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION,
|
||||
acknowledge_instance_id: String::new(),
|
||||
acknowledge_dirty_usage_generation: 0,
|
||||
}))
|
||||
.await
|
||||
.expect_err("activity queries must fail closed before storage is initialized");
|
||||
.expect_err("a rolling-upgrade request should pass authentication before storage lookup");
|
||||
assert_eq!(legacy.code(), tonic::Code::Unavailable);
|
||||
|
||||
assert_eq!(err.code(), tonic::Code::Unavailable);
|
||||
let unsupported = service
|
||||
.scanner_activity(Request::new(ScannerActivityRequest {
|
||||
challenge: vec![7; 16].into(),
|
||||
protocol_version: rustfs_scanner::SCANNER_ACTIVITY_PROTOCOL_VERSION + 1,
|
||||
acknowledge_instance_id: String::new(),
|
||||
acknowledge_dirty_usage_generation: 0,
|
||||
}))
|
||||
.await
|
||||
.expect_err("an unknown request protocol must fail before storage lookup");
|
||||
assert_eq!(unsupported.code(), tonic::Code::FailedPrecondition);
|
||||
|
||||
let malformed_legacy = service
|
||||
.scanner_activity(Request::new(ScannerActivityRequest {
|
||||
challenge: vec![7; 15].into(),
|
||||
protocol_version: SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION,
|
||||
acknowledge_instance_id: String::new(),
|
||||
acknowledge_dirty_usage_generation: 0,
|
||||
}))
|
||||
.await
|
||||
.expect_err("a malformed legacy challenge must fail before storage lookup");
|
||||
assert_eq!(malformed_legacy.code(), tonic::Code::InvalidArgument);
|
||||
|
||||
let unsigned = service
|
||||
.scanner_activity(Request::new(ScannerActivityRequest {
|
||||
challenge: vec![7; 16].into(),
|
||||
protocol_version: rustfs_scanner::SCANNER_ACTIVITY_PROTOCOL_VERSION,
|
||||
acknowledge_instance_id: String::new(),
|
||||
acknowledge_dirty_usage_generation: 0,
|
||||
}))
|
||||
.await
|
||||
.expect_err("unsigned activity queries must fail before storage lookup");
|
||||
assert_eq!(unsigned.code(), tonic::Code::PermissionDenied);
|
||||
|
||||
let mut malformed_current = Request::new(ScannerActivityRequest {
|
||||
challenge: vec![7; 15].into(),
|
||||
protocol_version: rustfs_scanner::SCANNER_ACTIVITY_PROTOCOL_VERSION,
|
||||
acknowledge_instance_id: String::new(),
|
||||
acknowledge_dirty_usage_generation: 0,
|
||||
});
|
||||
let malformed_canonical = rustfs_protos::canonical_scanner_activity_request_body(malformed_current.get_ref())
|
||||
.expect("scanner activity request should encode");
|
||||
set_tonic_canonical_body_digest(&mut malformed_current, &malformed_canonical).expect("digest metadata should encode");
|
||||
mark_v2_authenticated(&mut malformed_current);
|
||||
let malformed_current = service
|
||||
.scanner_activity(malformed_current)
|
||||
.await
|
||||
.expect_err("a signed malformed challenge must fail before storage lookup");
|
||||
assert_eq!(malformed_current.code(), tonic::Code::InvalidArgument);
|
||||
|
||||
let mut tampered = Request::new(ScannerActivityRequest {
|
||||
challenge: vec![7; 16].into(),
|
||||
protocol_version: rustfs_scanner::SCANNER_ACTIVITY_PROTOCOL_VERSION,
|
||||
acknowledge_instance_id: String::new(),
|
||||
acknowledge_dirty_usage_generation: 0,
|
||||
});
|
||||
let other = ScannerActivityRequest {
|
||||
challenge: vec![8; 16].into(),
|
||||
..tampered.get_ref().clone()
|
||||
};
|
||||
let other_canonical =
|
||||
rustfs_protos::canonical_scanner_activity_request_body(&other).expect("scanner activity request should encode");
|
||||
set_tonic_canonical_body_digest(&mut tampered, &other_canonical).expect("digest metadata should encode");
|
||||
mark_v2_authenticated(&mut tampered);
|
||||
let tampered = service
|
||||
.scanner_activity(tampered)
|
||||
.await
|
||||
.expect_err("tampered activity challenge must fail before storage lookup");
|
||||
assert_eq!(tampered.code(), tonic::Code::PermissionDenied);
|
||||
|
||||
let mut downgraded = Request::new(ScannerActivityRequest {
|
||||
challenge: vec![7; 16].into(),
|
||||
protocol_version: rustfs_scanner::SCANNER_ACTIVITY_PROTOCOL_VERSION,
|
||||
acknowledge_instance_id: String::new(),
|
||||
acknowledge_dirty_usage_generation: 0,
|
||||
});
|
||||
let current_canonical = rustfs_protos::canonical_scanner_activity_request_body(downgraded.get_ref())
|
||||
.expect("scanner activity request should encode");
|
||||
set_tonic_canonical_body_digest(&mut downgraded, ¤t_canonical).expect("digest metadata should encode");
|
||||
downgraded.get_mut().protocol_version = SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION;
|
||||
mark_v2_authenticated(&mut downgraded);
|
||||
let downgraded = service
|
||||
.scanner_activity(downgraded)
|
||||
.await
|
||||
.expect_err("a signed current request must not be downgraded to protocol v4");
|
||||
assert_eq!(downgraded.code(), tonic::Code::PermissionDenied);
|
||||
|
||||
let mut incomplete_acknowledgement = Request::new(ScannerActivityRequest {
|
||||
challenge: vec![7; 16].into(),
|
||||
protocol_version: rustfs_scanner::SCANNER_ACTIVITY_PROTOCOL_VERSION,
|
||||
acknowledge_instance_id: rustfs_scanner::scanner_activity_epoch().to_string(),
|
||||
acknowledge_dirty_usage_generation: 0,
|
||||
});
|
||||
let acknowledgement_canonical =
|
||||
rustfs_protos::canonical_scanner_activity_request_body(incomplete_acknowledgement.get_ref())
|
||||
.expect("scanner activity request should encode");
|
||||
set_tonic_canonical_body_digest(&mut incomplete_acknowledgement, &acknowledgement_canonical)
|
||||
.expect("digest metadata should encode");
|
||||
mark_v2_authenticated(&mut incomplete_acknowledgement);
|
||||
let incomplete_acknowledgement = service
|
||||
.scanner_activity(incomplete_acknowledgement)
|
||||
.await
|
||||
.expect_err("dirty usage acknowledgements require an instance ID and generation");
|
||||
assert_eq!(incomplete_acknowledgement.code(), tonic::Code::InvalidArgument);
|
||||
|
||||
let mut previous = Request::new(ScannerActivityRequest {
|
||||
challenge: vec![7; 16].into(),
|
||||
protocol_version: SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION,
|
||||
acknowledge_instance_id: String::new(),
|
||||
acknowledge_dirty_usage_generation: 0,
|
||||
});
|
||||
set_tonic_canonical_body_digest(&mut previous, &[7; 16]).expect("protocol v4 digest metadata should encode");
|
||||
mark_v2_authenticated(&mut previous);
|
||||
let previous = service
|
||||
.scanner_activity(previous)
|
||||
.await
|
||||
.expect_err("an authenticated protocol v4 request should reach storage lookup during rolling upgrades");
|
||||
assert_eq!(previous.code(), tonic::Code::Unavailable);
|
||||
|
||||
let mut signed = Request::new(ScannerActivityRequest {
|
||||
challenge: vec![7; 16].into(),
|
||||
protocol_version: rustfs_scanner::SCANNER_ACTIVITY_PROTOCOL_VERSION,
|
||||
acknowledge_instance_id: String::new(),
|
||||
acknowledge_dirty_usage_generation: 0,
|
||||
});
|
||||
let signed_canonical = rustfs_protos::canonical_scanner_activity_request_body(signed.get_ref())
|
||||
.expect("scanner activity request should encode");
|
||||
set_tonic_canonical_body_digest(&mut signed, &signed_canonical).expect("digest metadata should encode");
|
||||
mark_v2_authenticated(&mut signed);
|
||||
let unavailable = service
|
||||
.scanner_activity(signed)
|
||||
.await
|
||||
.expect_err("authenticated activity queries still require initialized storage");
|
||||
assert_eq!(unavailable.code(), tonic::Code::Unavailable);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_scanner_activity_response_uses_process_epoch_and_generations() {
|
||||
let response = scanner_activity_response(17);
|
||||
let response = scanner_activity_response(
|
||||
17,
|
||||
[7; 32],
|
||||
true,
|
||||
rustfs_scanner::ScannerDirtyUsageState {
|
||||
generation: 11,
|
||||
pending: true,
|
||||
},
|
||||
);
|
||||
|
||||
assert_eq!(response.instance_id, rustfs_scanner::scanner_activity_epoch());
|
||||
assert_eq!(response.namespace_generation, 17);
|
||||
assert_eq!(response.maintenance_generation, rustfs_scanner::scanner_maintenance_generation());
|
||||
assert_eq!(response.protocol_version, rustfs_scanner::SCANNER_ACTIVITY_PROTOCOL_VERSION);
|
||||
assert_eq!(response.topology_digest.as_ref(), &[7; 32]);
|
||||
assert!(response.data_movement_active);
|
||||
assert_eq!(response.dirty_usage_generation, 11);
|
||||
assert!(response.dirty_usage_pending);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_previous_scanner_activity_response_omits_dirty_usage_fields() {
|
||||
let response = previous_scanner_activity_response(17, [7; 32], true);
|
||||
|
||||
assert_eq!(response.protocol_version, SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION);
|
||||
assert_eq!(response.topology_digest.as_ref(), &[7; 32]);
|
||||
assert!(response.data_movement_active);
|
||||
assert_eq!(response.dirty_usage_generation, 0);
|
||||
assert!(!response.dirty_usage_pending);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_legacy_scanner_activity_response_omits_extended_fields() {
|
||||
let response = legacy_scanner_activity_response(17);
|
||||
|
||||
assert_eq!(response.namespace_generation, 17);
|
||||
assert_eq!(response.protocol_version, SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION);
|
||||
assert!(response.topology_digest.is_empty());
|
||||
assert!(!response.data_movement_active);
|
||||
assert!(response.response_proof.is_empty());
|
||||
assert_eq!(response.dirty_usage_generation, 0);
|
||||
assert!(!response.dirty_usage_pending);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -4373,6 +4678,7 @@ mod tests {
|
||||
|
||||
assert!(response.success, "new nodes must advertise notify lifecycle reload support");
|
||||
assert!(response.error_info.is_none());
|
||||
assert_eq!(response.protocol_version, rustfs_protos::DYNAMIC_CONFIG_PROTOCOL_VERSION);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -698,13 +698,24 @@ impl NodeService {
|
||||
Ok(volume_infos) => {
|
||||
let volume_infos = volume_infos
|
||||
.into_iter()
|
||||
.filter_map(|volume_info| serde_json::to_string(&volume_info).ok())
|
||||
.collect();
|
||||
Ok(Response::new(ListVolumesResponse {
|
||||
success: true,
|
||||
volume_infos,
|
||||
error: None,
|
||||
}))
|
||||
.enumerate()
|
||||
.map(|(index, volume_info)| {
|
||||
serde_json::to_string(&volume_info)
|
||||
.map_err(|err| DiskError::other(format!("encode list volumes entry {index} failed: {err}")))
|
||||
})
|
||||
.collect::<std::result::Result<Vec<_>, DiskError>>();
|
||||
match volume_infos {
|
||||
Ok(volume_infos) => Ok(Response::new(ListVolumesResponse {
|
||||
success: true,
|
||||
volume_infos,
|
||||
error: None,
|
||||
})),
|
||||
Err(err) => Ok(Response::new(ListVolumesResponse {
|
||||
success: false,
|
||||
volume_infos: Vec::new(),
|
||||
error: Some(err.into()),
|
||||
})),
|
||||
}
|
||||
}
|
||||
Err(err) => Ok(Response::new(ListVolumesResponse {
|
||||
success: false,
|
||||
|
||||
@@ -212,12 +212,23 @@ pub(crate) mod rpc_consumer {
|
||||
pub(crate) mod http_service {
|
||||
pub(crate) const DEFAULT_READ_BUFFER_SIZE: usize = super::super::DEFAULT_READ_BUFFER_SIZE;
|
||||
#[cfg(test)]
|
||||
pub(crate) use super::super::storage_contracts::WALK_DIR_BODY_SHA256_QUERY;
|
||||
pub(crate) use super::super::storage_contracts::WALK_DIR_STREAM_COMPLETION_V1;
|
||||
pub(crate) use super::super::{StorageDiskRpcExt, WalkDirOptions, find_local_disk_by_ref, verify_rpc_signature};
|
||||
pub(crate) use super::super::storage_contracts::{
|
||||
NS_SCANNER_BODY_SHA256_QUERY, NS_SCANNER_CAPABILITY_CHALLENGE_QUERY, NS_SCANNER_CYCLE_QUERY,
|
||||
NS_SCANNER_LEADER_EPOCH_QUERY, NS_SCANNER_REQUEST_ID_QUERY, NS_SCANNER_SERVER_EPOCH_QUERY,
|
||||
NS_SCANNER_SESSION_ID_QUERY, NS_SCANNER_SESSION_SEQUENCE_QUERY, WALK_DIR_BODY_SHA256_QUERY,
|
||||
};
|
||||
pub(crate) use super::super::storage_contracts::{
|
||||
NS_SCANNER_PROTOCOL_VERSION, NsScannerCapabilityResponse, WALK_DIR_STREAM_COMPLETION_V1,
|
||||
};
|
||||
pub(crate) use super::super::{
|
||||
StorageDiskRpcExt, WalkDirOptions, find_local_disk_by_ref, sign_ns_scanner_capability, verify_rpc_signature,
|
||||
};
|
||||
}
|
||||
|
||||
pub(crate) mod node_service {
|
||||
pub(crate) use super::super::storage_contracts::{
|
||||
SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION, SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION,
|
||||
};
|
||||
pub(crate) use super::super::{
|
||||
BatchReadVersionReq, BatchReadVersionResp, CollectMetricsOpts, DeleteOptions, DiskError, DiskInfoOptions, DiskStore,
|
||||
ECStore, Error, FileInfoVersions, LocalPeerS3Client, MetricType, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS,
|
||||
@@ -404,7 +415,7 @@ pub(crate) mod ecstore_data_usage {
|
||||
apply_bucket_usage_memory_overlay, init_compression_total_memory_from_backend, load_data_usage_from_backend,
|
||||
load_data_usage_from_backend_cached, record_bucket_delete_marker_memory, record_bucket_object_delete_memory,
|
||||
record_bucket_object_version_write_memory, record_bucket_object_write_memory,
|
||||
record_bucket_object_write_unknown_previous_memory, remove_bucket_usage_from_backend, store_compression_total_in_backend,
|
||||
record_bucket_object_write_unknown_previous_memory, store_compression_total_in_backend,
|
||||
};
|
||||
// Test-only observables for the rustfs/backlog#1306 revert detector.
|
||||
#[cfg(test)]
|
||||
@@ -484,7 +495,8 @@ pub(crate) mod ecstore_rpc {
|
||||
pub(crate) use rustfs_ecstore::api::rpc::{
|
||||
LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, PeerS3Client,
|
||||
SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, TONIC_RPC_PREFIX, normalize_tonic_rpc_audience,
|
||||
sign_tonic_rpc_response_proof, verify_rpc_signature, verify_tonic_canonical_body_digest, verify_tonic_rpc_signature,
|
||||
sign_ns_scanner_capability, sign_tonic_rpc_response_proof, verify_rpc_signature, verify_tonic_canonical_body_digest,
|
||||
verify_tonic_rpc_signature,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::rpc::{
|
||||
@@ -1543,6 +1555,10 @@ pub(crate) fn verify_rpc_signature(url: &str, method: &http::Method, headers: &h
|
||||
ecstore_rpc::verify_rpc_signature(url, method, headers)
|
||||
}
|
||||
|
||||
pub(crate) fn sign_ns_scanner_capability(challenge: uuid::Uuid, server_epoch: uuid::Uuid) -> std::io::Result<Vec<u8>> {
|
||||
ecstore_rpc::sign_ns_scanner_capability(challenge, server_epoch)
|
||||
}
|
||||
|
||||
pub(crate) fn verify_tonic_rpc_signature(audience: &str, path: &str, headers: &http::HeaderMap) -> std::io::Result<()> {
|
||||
ecstore_rpc::verify_tonic_rpc_signature(audience, path, headers)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user