mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-22 12:26:37 +00:00
Compare commits
8 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| ce9b69d811 | |||
| adb90fc6e1 | |||
| cdfac5d7e3 | |||
| ca4adea0c9 | |||
| 23a0f6324c | |||
| cdd9ab1124 | |||
| 122a69df65 | |||
| dfeb732ac8 |
@@ -189,6 +189,7 @@ jobs:
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
fail-fast: false
|
||||
max-parallel: 1
|
||||
matrix:
|
||||
include:
|
||||
- arch: x86_64
|
||||
@@ -510,15 +511,13 @@ jobs:
|
||||
|
||||
CHECKSUM_DIR="$(mktemp -d)"
|
||||
gh release download "$TAG" -p 'SHA256SUMS' -p 'SHA512SUMS' \
|
||||
-D "$CHECKSUM_DIR" --clobber 2>/dev/null || true
|
||||
-D "$CHECKSUM_DIR" --clobber
|
||||
|
||||
for spec in "SHA256SUMS:sha256sum" "SHA512SUMS:sha512sum"; do
|
||||
asset="${spec%%:*}"
|
||||
checksum_cmd="${spec##*:}"
|
||||
checksum_file="${CHECKSUM_DIR}/${asset}"
|
||||
|
||||
touch "$checksum_file"
|
||||
|
||||
for f in "$DEB_FILE" "$RPM_FILE"; do
|
||||
if [[ -n "$f" && -f "$f" ]]; then
|
||||
base="$(basename "$f")"
|
||||
@@ -531,7 +530,8 @@ jobs:
|
||||
grep -Fv -- "$base" "$checksum_file" > "${checksum_file}.tmp" || true
|
||||
grep -Fv -- "$github_base" "${checksum_file}.tmp" > "${checksum_file}.tmp2" || true
|
||||
mv "${checksum_file}.tmp2" "$checksum_file"
|
||||
(cd "$(dirname "$f")" && "$checksum_cmd" -- "$github_base") >> "$checksum_file"
|
||||
digest=$("$checksum_cmd" -- "$f" | awk '{print $1}')
|
||||
printf '%s %s\n' "$digest" "$github_base" >> "$checksum_file"
|
||||
fi
|
||||
done
|
||||
|
||||
|
||||
@@ -199,12 +199,12 @@ pub mod bucket {
|
||||
VersionPurgeStatusType, XferStats, commit_force_delete_intent, complete_force_delete_intent,
|
||||
delete_replication_state_from_config, delete_replication_version_id, get_global_replication_pool,
|
||||
get_global_replication_stats, get_proxy_targets, init_background_replication,
|
||||
invalid_replication_config_status_field, persist_force_delete_intent, read_durable_mrf_backlog,
|
||||
replication_state_to_filemeta, replication_status_to_filemeta, replication_statuses_map, replication_target_arns,
|
||||
resync_start_conflict_id, should_remove_replication_target, should_schedule_delete_replication,
|
||||
should_use_existing_delete_replication_info, should_use_existing_delete_replication_source,
|
||||
unsupported_replication_config_field, validate_replication_config_structure, validate_replication_config_target_arns,
|
||||
version_purge_status_to_filemeta,
|
||||
invalid_replication_config_status_field, is_site_replication_rule, merge_incoming_replication_config,
|
||||
persist_force_delete_intent, read_durable_mrf_backlog, replication_state_to_filemeta, replication_status_to_filemeta,
|
||||
replication_statuses_map, replication_target_arn_deployment_id, replication_target_arns, resync_start_conflict_id,
|
||||
should_remove_replication_target, should_schedule_delete_replication, should_use_existing_delete_replication_info,
|
||||
should_use_existing_delete_replication_source, unsupported_replication_config_field,
|
||||
validate_replication_config_structure, validate_replication_config_target_arns, version_purge_status_to_filemeta,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -41,6 +41,7 @@ const IAM_FORMAT_FILE_PATH: &str = "config/iam/format.json";
|
||||
const IAM_USERS_PREFIX: &str = "config/iam/users/";
|
||||
const IAM_SERVICE_ACCOUNTS_PREFIX: &str = "config/iam/service-accounts/";
|
||||
const IAM_STS_PREFIX: &str = "config/iam/sts/";
|
||||
const MINIO_GO_ZERO_TIME: OffsetDateTime = time::macros::datetime!(0001-01-01 00:00 UTC);
|
||||
const IAM_GROUPS_PREFIX: &str = "config/iam/groups/";
|
||||
const IAM_POLICIES_PREFIX: &str = "config/iam/policies/";
|
||||
const IAM_POLICY_DB_PREFIX: &str = "config/iam/policydb/";
|
||||
@@ -120,6 +121,15 @@ fn normalize_iam_config_blob(path: &str, data: &[u8]) -> std::result::Result<Opt
|
||||
if is_identity_path(path) {
|
||||
let mut identity: UserIdentity =
|
||||
serde_json::from_slice(data).map_err(|err| format!("parse IAM identity failed: {err}"))?;
|
||||
if (path.starts_with(IAM_USERS_PREFIX) || path.starts_with(IAM_SERVICE_ACCOUNTS_PREFIX))
|
||||
&& identity
|
||||
.credentials
|
||||
.expiration
|
||||
.as_ref()
|
||||
.is_some_and(|expiration| *expiration == MINIO_GO_ZERO_TIME || *expiration == OffsetDateTime::UNIX_EPOCH)
|
||||
{
|
||||
identity.credentials.expiration = None;
|
||||
}
|
||||
if identity.update_at.is_none() {
|
||||
identity.update_at = Some(OffsetDateTime::now_utc());
|
||||
}
|
||||
@@ -441,7 +451,10 @@ mod tests {
|
||||
use crate::bucket::replication::{
|
||||
BucketReplicationResyncStatus, ReplicationMigrationBridge, ResyncStatusType, TargetReplicationResyncStatus,
|
||||
};
|
||||
use rustfs_policy::auth::UserIdentity;
|
||||
use std::collections::HashMap;
|
||||
use time::OffsetDateTime;
|
||||
use time::format_description::well_known::Rfc3339;
|
||||
|
||||
#[test]
|
||||
fn test_normalize_policy_mapping_legacy_timestamp_and_fields() {
|
||||
@@ -493,6 +506,54 @@ mod tests {
|
||||
assert!(v.get("updatedAt").is_some(), "normalize should backfill updatedAt");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_normalize_minio_permanent_credential_expiration() {
|
||||
let cases = [
|
||||
("config/iam/users/alice/identity.json", "0001-01-01T00:00:00Z", true),
|
||||
("config/iam/users/alice/identity.json", "1970-01-01T00:00:00Z", true),
|
||||
("config/iam/service-accounts/svc/identity.json", "0001-01-01T00:00:00Z", true),
|
||||
("config/iam/service-accounts/svc/identity.json", "1970-01-01T00:00:00Z", true),
|
||||
("config/iam/service-accounts/svc/identity.json", "1970-01-01T00:00:00.000000001Z", false),
|
||||
("config/iam/sts/temp/identity.json", "0001-01-01T00:00:00Z", false),
|
||||
("config/iam/sts/temp/identity.json", "1970-01-01T00:00:00Z", false),
|
||||
("config/iam/users/alice/identity.json", "1969-12-31T23:59:59Z", false),
|
||||
("config/iam/users/alice/identity.json", "1970-01-01T00:00:00.000000001Z", false),
|
||||
("config/iam/users/alice/identity.json", "0001-01-01T00:00:00.000000001Z", false),
|
||||
("config/iam/users/alice/identity.json", "2030-01-01T00:00:00Z", false),
|
||||
];
|
||||
|
||||
for (path, expiration, should_clear) in cases {
|
||||
let input = serde_json::json!({
|
||||
"version": 1,
|
||||
"credentials": {
|
||||
"accessKey": "test-access",
|
||||
"secretKey": "test-secret",
|
||||
"sessionToken": "test-session-token",
|
||||
"parentUser": "test-parent",
|
||||
"expiration": expiration,
|
||||
}
|
||||
});
|
||||
let output = normalize_iam_config_blob(path, &serde_json::to_vec(&input).expect("serialize identity fixture"))
|
||||
.expect("normalize should succeed")
|
||||
.expect("identity path should be supported");
|
||||
let identity: UserIdentity = serde_json::from_slice(&output).expect("deserialize normalized identity");
|
||||
|
||||
assert_eq!(identity.credentials.access_key, "test-access");
|
||||
assert_eq!(identity.credentials.secret_key, "test-secret");
|
||||
assert_eq!(identity.credentials.session_token, "test-session-token");
|
||||
assert_eq!(identity.credentials.parent_user, "test-parent");
|
||||
if should_clear {
|
||||
assert_eq!(identity.credentials.expiration, None, "path: {path}, expiration: {expiration}");
|
||||
} else {
|
||||
assert_eq!(
|
||||
identity.credentials.expiration,
|
||||
Some(OffsetDateTime::parse(expiration, &Rfc3339).expect("parse expected expiration")),
|
||||
"path: {path}, expiration: {expiration}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_normalize_bucket_meta_blob_resync_reencode() {
|
||||
let path = ".buckets/test/.replication/resync.bin";
|
||||
|
||||
@@ -47,7 +47,8 @@ pub use replication_config_boundary::{
|
||||
ObjectOpts, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS,
|
||||
REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS,
|
||||
ReplicationConfigStructureError, ReplicationConfigurationExt, ReplicationTargetValidationError,
|
||||
invalid_replication_config_status_field, replication_target_arns, should_remove_replication_target,
|
||||
invalid_replication_config_status_field, is_site_replication_rule, merge_incoming_replication_config,
|
||||
replication_target_arn_deployment_id, replication_target_arns, should_remove_replication_target,
|
||||
unsupported_replication_config_field, validate_replication_config_structure, validate_replication_config_target_arns,
|
||||
};
|
||||
pub(crate) use replication_filemeta_boundary::version_purge_statuses_map;
|
||||
|
||||
@@ -16,6 +16,7 @@ pub use rustfs_replication::{
|
||||
ObjectOpts, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS,
|
||||
REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS,
|
||||
ReplicationConfigStructureError, ReplicationConfigurationExt, ReplicationRuleExt, ReplicationTargetValidationError,
|
||||
invalid_replication_config_status_field, replication_target_arns, should_remove_replication_target,
|
||||
invalid_replication_config_status_field, is_site_replication_rule, merge_incoming_replication_config,
|
||||
replication_target_arn_deployment_id, replication_target_arns, should_remove_replication_target,
|
||||
unsupported_replication_config_field, validate_replication_config_structure, validate_replication_config_target_arns,
|
||||
};
|
||||
|
||||
@@ -94,6 +94,9 @@ const DECOMMISSION_BUCKET_CONCURRENCY_DEFAULT_CAP: usize = 4;
|
||||
const DECOMMISSION_TARGET_CAPACITY_OVERHEAD_PERCENT: usize = 30;
|
||||
const DECOMMISSION_LISTING_MAX_ATTEMPTS: usize = 3;
|
||||
const DECOMMISSION_LISTING_RETRY_DELAY: std::time::Duration = std::time::Duration::from_secs(5);
|
||||
/// Background decommission walks must tolerate slow object migrations; the
|
||||
/// stall timeout is the drive-health bound, not the total listing duration.
|
||||
const DECOMMISSION_BACKGROUND_WALKDIR_STALL_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60);
|
||||
|
||||
pub const POOL_META_NAME: &str = "pool.bin";
|
||||
pub const POOL_META_FORMAT: u16 = 1;
|
||||
@@ -5047,6 +5050,8 @@ impl SetDisks {
|
||||
path: bucket_info.prefix.clone(),
|
||||
recursive: true,
|
||||
min_disks: listing_quorum,
|
||||
skip_walkdir_total_timeout: true,
|
||||
walkdir_stall_timeout: Some(DECOMMISSION_BACKGROUND_WALKDIR_STALL_TIMEOUT),
|
||||
agreed: Some(Box::new(move |entry: MetaCacheEntry| Box::pin(cb1(entry)))),
|
||||
partial: Some(Box::new(move |entries: MetaCacheEntries, _: &[Option<DiskError>]| {
|
||||
let resolver = resolver.clone();
|
||||
|
||||
@@ -23,7 +23,7 @@ use std::{
|
||||
io,
|
||||
path::{Component, Path, PathBuf},
|
||||
sync::{Arc, LazyLock, Weak},
|
||||
time::Instant,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
use tokio::fs;
|
||||
use tokio::sync::{
|
||||
@@ -328,6 +328,9 @@ const ENV_DST_DIR_FSYNC_GROUP_COMMIT_ENABLE: &str = "RUSTFS_EXPERIMENTAL_DST_DIR
|
||||
const DEFAULT_DST_DIR_FSYNC_GROUP_COMMIT_ENABLE: bool = false;
|
||||
const ENV_FILE_FDATASYNC_GROUP_COMMIT_ENABLE: &str = "RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_ENABLE";
|
||||
const DEFAULT_FILE_FDATASYNC_GROUP_COMMIT_ENABLE: bool = false;
|
||||
const ENV_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS: &str = "RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS";
|
||||
const DEFAULT_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS: u64 = 0;
|
||||
const MAX_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS: u64 = 1_000;
|
||||
#[cfg(not(test))]
|
||||
const MAX_DST_DIR_FSYNC_GROUPS: usize = 1024;
|
||||
#[cfg(test)]
|
||||
@@ -354,6 +357,16 @@ static DST_DIR_FSYNC_GROUP_COMMIT_ENABLED: LazyLock<bool> = LazyLock::new(|| {
|
||||
static FILE_FDATASYNC_GROUP_COMMIT_ENABLED: LazyLock<bool> = LazyLock::new(|| {
|
||||
rustfs_utils::get_env_bool(ENV_FILE_FDATASYNC_GROUP_COMMIT_ENABLE, DEFAULT_FILE_FDATASYNC_GROUP_COMMIT_ENABLE)
|
||||
});
|
||||
fn file_fdatasync_group_commit_wait_duration(wait_micros: u64) -> Duration {
|
||||
Duration::from_micros(wait_micros.min(MAX_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS))
|
||||
}
|
||||
|
||||
static FILE_FDATASYNC_GROUP_COMMIT_WAIT: LazyLock<Duration> = LazyLock::new(|| {
|
||||
file_fdatasync_group_commit_wait_duration(rustfs_utils::get_env_u64(
|
||||
ENV_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS,
|
||||
DEFAULT_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS,
|
||||
))
|
||||
});
|
||||
|
||||
#[cfg(test)]
|
||||
mod dst_dir_fsync_group_commit_override {
|
||||
@@ -402,6 +415,7 @@ mod file_fdatasync_group_commit_override {
|
||||
use std::sync::{Mutex, MutexGuard, PoisonError, RwLock};
|
||||
|
||||
static OVERRIDE: RwLock<Option<bool>> = RwLock::new(None);
|
||||
static WAIT_OVERRIDE_MICROS: RwLock<Option<u64>> = RwLock::new(None);
|
||||
static SERIAL: Mutex<()> = Mutex::new(());
|
||||
|
||||
pub(crate) fn get() -> Option<bool> {
|
||||
@@ -415,6 +429,7 @@ mod file_fdatasync_group_commit_override {
|
||||
impl Drop for OverrideGuard {
|
||||
fn drop(&mut self) {
|
||||
*OVERRIDE.write().unwrap_or_else(PoisonError::into_inner) = None;
|
||||
*WAIT_OVERRIDE_MICROS.write().unwrap_or_else(PoisonError::into_inner) = None;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -423,6 +438,14 @@ mod file_fdatasync_group_commit_override {
|
||||
*OVERRIDE.write().unwrap_or_else(PoisonError::into_inner) = Some(enabled);
|
||||
OverrideGuard { _serial: serial }
|
||||
}
|
||||
|
||||
pub(crate) fn set_wait_micros(wait_micros: u64) {
|
||||
*WAIT_OVERRIDE_MICROS.write().unwrap_or_else(PoisonError::into_inner) = Some(wait_micros);
|
||||
}
|
||||
|
||||
pub(crate) fn wait_micros() -> Option<u64> {
|
||||
*WAIT_OVERRIDE_MICROS.read().unwrap_or_else(PoisonError::into_inner)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -430,6 +453,11 @@ pub(crate) fn set_file_fdatasync_group_commit_for_test(enabled: bool) -> file_fd
|
||||
file_fdatasync_group_commit_override::set(enabled)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn set_file_fdatasync_group_commit_wait_for_test(wait_micros: u64) {
|
||||
file_fdatasync_group_commit_override::set_wait_micros(wait_micros);
|
||||
}
|
||||
|
||||
fn file_fdatasync_group_commit_enabled() -> bool {
|
||||
#[cfg(test)]
|
||||
if let Some(enabled) = file_fdatasync_group_commit_override::get() {
|
||||
@@ -439,6 +467,15 @@ fn file_fdatasync_group_commit_enabled() -> bool {
|
||||
*FILE_FDATASYNC_GROUP_COMMIT_ENABLED
|
||||
}
|
||||
|
||||
fn file_fdatasync_group_commit_wait() -> Duration {
|
||||
#[cfg(test)]
|
||||
if let Some(wait_micros) = file_fdatasync_group_commit_override::wait_micros() {
|
||||
return file_fdatasync_group_commit_wait_duration(wait_micros);
|
||||
}
|
||||
|
||||
*FILE_FDATASYNC_GROUP_COMMIT_WAIT
|
||||
}
|
||||
|
||||
#[derive(Clone, Eq, Hash, PartialEq)]
|
||||
struct DstDirFsyncGroupKey {
|
||||
canonical_path: PathBuf,
|
||||
@@ -934,6 +971,10 @@ async fn run_file_fdatasync_group_worker(group: Arc<FileFdatasyncGroup>) {
|
||||
#[cfg(test)]
|
||||
file_sync_probe::run_before_group_batch();
|
||||
tokio::task::yield_now().await;
|
||||
let wait = file_fdatasync_group_commit_wait();
|
||||
if !wait.is_zero() {
|
||||
tokio::time::sleep(wait).await;
|
||||
}
|
||||
let (batch, batch_file_count): (Vec<FileFdatasyncWaiter>, usize) = {
|
||||
let mut group_state = group.inner.lock();
|
||||
let batch_file_count = group_state.pending_files;
|
||||
@@ -6075,6 +6116,7 @@ mod tests {
|
||||
use std::sync::mpsc;
|
||||
|
||||
let _group_commit = set_file_fdatasync_group_commit_for_test(true);
|
||||
set_file_fdatasync_group_commit_wait_for_test(0);
|
||||
clear_file_fdatasync_group_commit_for_test();
|
||||
let temp_dir = tempdir().expect("create temp dir");
|
||||
let first_dir = temp_dir.path().join("first");
|
||||
@@ -6141,12 +6183,105 @@ mod tests {
|
||||
assert_eq!(file_fdatasync_group_commit_counts_for_test(), (0, 0, 0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn file_fdatasync_group_commit_wait_duration_uses_default_and_cap() {
|
||||
assert_eq!(DEFAULT_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS, 0);
|
||||
assert_eq!(
|
||||
file_fdatasync_group_commit_wait_duration(DEFAULT_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS),
|
||||
Duration::ZERO
|
||||
);
|
||||
assert_eq!(file_fdatasync_group_commit_wait_duration(250), Duration::from_micros(250));
|
||||
assert_eq!(
|
||||
file_fdatasync_group_commit_wait_duration(MAX_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS),
|
||||
Duration::from_micros(MAX_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS)
|
||||
);
|
||||
assert_eq!(
|
||||
file_fdatasync_group_commit_wait_duration(u64::MAX),
|
||||
Duration::from_micros(MAX_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread", start_paused = true)]
|
||||
#[serial_test::serial(file_sync_probe)]
|
||||
async fn file_fdatasync_group_commit_wait_budget_batches_late_follower() {
|
||||
use std::sync::mpsc;
|
||||
|
||||
let _group_commit = set_file_fdatasync_group_commit_for_test(true);
|
||||
let wait_budget_micros = 1_000;
|
||||
let wait_budget = file_fdatasync_group_commit_wait_duration(wait_budget_micros);
|
||||
set_file_fdatasync_group_commit_wait_for_test(wait_budget_micros);
|
||||
clear_file_fdatasync_group_commit_for_test();
|
||||
let temp_dir = tempdir().expect("create temp dir");
|
||||
let first_dir = temp_dir.path().join("first");
|
||||
let second_dir = temp_dir.path().join("second");
|
||||
std::fs::create_dir(&first_dir).expect("create first dir");
|
||||
std::fs::create_dir(&second_dir).expect("create second dir");
|
||||
std::fs::write(first_dir.join("part.1"), b"first").expect("write first part");
|
||||
std::fs::write(second_dir.join("part.1"), b"second").expect("write second part");
|
||||
let _probe = file_sync_probe::set_blocking(temp_dir.path());
|
||||
let (entered_tx, entered_rx) = mpsc::channel();
|
||||
file_sync_probe::set_before_group_batch(move || {
|
||||
entered_tx.send(()).expect("signal first file fdatasync group worker");
|
||||
});
|
||||
|
||||
let limiter = file_sync_limiter();
|
||||
let first_limiter = limiter.clone();
|
||||
let first_path = first_dir.clone();
|
||||
let first = tokio::spawn(async move { sync_dir_files_with_limiter(first_path, first_limiter).await });
|
||||
tokio::task::spawn_blocking(move || entered_rx.recv_timeout(Duration::from_secs(30)))
|
||||
.await
|
||||
.expect("group worker hook waiter should run")
|
||||
.expect("first file fdatasync group worker should start");
|
||||
|
||||
let second_limiter = limiter.clone();
|
||||
let second_path = second_dir.clone();
|
||||
let second = tokio::spawn(async move { sync_dir_files_with_limiter(second_path, second_limiter).await });
|
||||
tokio::time::timeout(Duration::from_secs(30), async {
|
||||
loop {
|
||||
if file_fdatasync_group_commit_counts_for_test().1 == 2 {
|
||||
return;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("second waiter should enqueue during the configured wait budget");
|
||||
tokio::time::advance(wait_budget).await;
|
||||
tokio::task::yield_now().await;
|
||||
file_sync_probe::wait_for_active(1).await;
|
||||
|
||||
assert_eq!(
|
||||
file_sync_probe::group_batches(),
|
||||
vec![2],
|
||||
"configured wait budget should let a follower join the leader's batch"
|
||||
);
|
||||
file_sync_probe::release();
|
||||
first
|
||||
.await
|
||||
.expect("join first wait-budget file sync")
|
||||
.expect("first wait-budget file sync must succeed");
|
||||
second
|
||||
.await
|
||||
.expect("join second wait-budget file sync")
|
||||
.expect("second wait-budget file sync must succeed");
|
||||
assert!(
|
||||
fsync_dir_recorder::was_fsynced(&first_dir),
|
||||
"first source directory must still be fsynced"
|
||||
);
|
||||
assert!(
|
||||
fsync_dir_recorder::was_fsynced(&second_dir),
|
||||
"second source directory must still be fsynced"
|
||||
);
|
||||
assert_eq!(file_fdatasync_group_commit_counts_for_test(), (0, 0, 0));
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(file_sync_probe)]
|
||||
async fn file_fdatasync_group_commit_failure_fails_all_waiters_before_dir_fsync() {
|
||||
use std::sync::mpsc;
|
||||
|
||||
let _group_commit = set_file_fdatasync_group_commit_for_test(true);
|
||||
set_file_fdatasync_group_commit_wait_for_test(0);
|
||||
clear_file_fdatasync_group_commit_for_test();
|
||||
let temp_dir = tempdir().expect("create temp dir");
|
||||
let first_dir = temp_dir.path().join("first");
|
||||
|
||||
@@ -343,6 +343,23 @@ impl ECStore {
|
||||
let (decommission, rebalance) = tokio::join!(self.is_decommission_running(), self.is_rebalance_started());
|
||||
decommission || rebalance
|
||||
}
|
||||
|
||||
/// Returns whether scanner metadata may still be hidden by a local
|
||||
/// data-movement state. Terminal failed/canceled decommission entries
|
||||
/// remain suspended until an operator clears or retries them, so they are
|
||||
/// a publication barrier even after the worker has stopped.
|
||||
pub async fn scanner_data_usage_publication_blocked(&self) -> bool {
|
||||
if self.scanner_data_movement_active().await {
|
||||
return true;
|
||||
}
|
||||
|
||||
let pool_meta = self.pool_meta.read().await;
|
||||
pool_meta.pools.iter().any(|pool| {
|
||||
pool.decommission
|
||||
.as_ref()
|
||||
.is_some_and(|info| !info.queued && (info.failed || info.canceled))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// impl Clone for ECStore {
|
||||
@@ -875,6 +892,7 @@ impl crate::storage_api_contracts::admin::StorageAdminApi for ECStore {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::core::pools::{PoolDecommissionInfo, PoolStatus};
|
||||
use crate::layout::endpoints::{Endpoints, PoolEndpoints, SetupType};
|
||||
use crate::runtime::global::reset_local_disk_test_state;
|
||||
use crate::runtime::sources::{clear_local_disk_id_map_for_test, local_disk_path_by_id};
|
||||
@@ -911,6 +929,72 @@ mod tests {
|
||||
})
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_data_usage_publication_blocks_active_and_unqueued_terminal_decommission() {
|
||||
let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
|
||||
let cases = [
|
||||
(
|
||||
"active",
|
||||
PoolDecommissionInfo {
|
||||
start_time: Some(OffsetDateTime::now_utc()),
|
||||
..Default::default()
|
||||
},
|
||||
true,
|
||||
),
|
||||
(
|
||||
"failed",
|
||||
PoolDecommissionInfo {
|
||||
failed: true,
|
||||
..Default::default()
|
||||
},
|
||||
true,
|
||||
),
|
||||
(
|
||||
"canceled",
|
||||
PoolDecommissionInfo {
|
||||
canceled: true,
|
||||
..Default::default()
|
||||
},
|
||||
true,
|
||||
),
|
||||
(
|
||||
"queued_failed",
|
||||
PoolDecommissionInfo {
|
||||
failed: true,
|
||||
queued: true,
|
||||
..Default::default()
|
||||
},
|
||||
false,
|
||||
),
|
||||
(
|
||||
"complete",
|
||||
PoolDecommissionInfo {
|
||||
complete: true,
|
||||
..Default::default()
|
||||
},
|
||||
false,
|
||||
),
|
||||
("idle", PoolDecommissionInfo::default(), false),
|
||||
];
|
||||
|
||||
for (name, decommission, expected) in cases {
|
||||
*store.pool_meta.write().await = PoolMeta {
|
||||
pools: vec![PoolStatus {
|
||||
id: 0,
|
||||
cmd_line: format!("scanner-publication-{name}"),
|
||||
last_update: OffsetDateTime::now_utc(),
|
||||
decommission: Some(decommission),
|
||||
}],
|
||||
..Default::default()
|
||||
};
|
||||
assert_eq!(
|
||||
store.scanner_data_usage_publication_blocked().await,
|
||||
expected,
|
||||
"unexpected scanner publication barrier state for {name}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// The object graph is the isolation carrier: two ECStore instances holding
|
||||
// distinct contexts report independent erasure state through their real
|
||||
// `&self` accessors — no cross-contamination.
|
||||
|
||||
@@ -17,11 +17,11 @@
|
||||
//! All direct `rustfs_ecstore` facade imports used by tests in this crate
|
||||
//! must go through this module (architecture migration rule:
|
||||
//! `check_architecture_migration_rules.sh`). Keep the surface minimal —
|
||||
//! only what the tests actually need to build a temp-disk ECStore fixture
|
||||
//! and to flip the erasure setup type for lock-quorum fault injection.
|
||||
//! only what the tests actually need to run storage-backed IAM scenarios.
|
||||
|
||||
#[allow(unused_imports)]
|
||||
pub(crate) mod fixture {
|
||||
pub(crate) use rustfs_ecstore::api::bucket::migration::try_migrate_iam_config;
|
||||
pub(crate) use rustfs_ecstore::api::layout::SetupType;
|
||||
|
||||
// `update_erasure_type` is a write-side global facade entry. Its use is
|
||||
|
||||
@@ -0,0 +1,182 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
mod ecstore_test_compat;
|
||||
|
||||
use ecstore_test_compat::fixture::try_migrate_iam_config;
|
||||
use rustfs_credentials::{get_global_action_cred, init_global_action_credentials};
|
||||
use rustfs_iam::store::object::{
|
||||
IAM_CONFIG_POLICY_DB_SERVICE_ACCOUNTS_PREFIX, IAM_CONFIG_POLICY_DB_USERS_PREFIX, IAM_CONFIG_SERVICE_ACCOUNTS_PREFIX,
|
||||
IAM_CONFIG_USERS_PREFIX, ObjectStore,
|
||||
};
|
||||
use rustfs_iam::store::{Store, UserType};
|
||||
use rustfs_iam::utils::generate_jwt;
|
||||
use rustfs_policy::auth::UserIdentity;
|
||||
use serde_json::{Value, json};
|
||||
use std::collections::HashMap;
|
||||
|
||||
const LEGACY_META_BUCKET: &str = ".minio.sys";
|
||||
const REGULAR_USER: &str = "minio-user";
|
||||
const SERVICE_ACCOUNT: &str = "minio-service-account";
|
||||
|
||||
async fn seed_legacy_iam_object(env: &rustfs_test_utils::TestECStoreEnv, path: &str, value: &Value) {
|
||||
env.put_object_bytes(
|
||||
LEGACY_META_BUCKET,
|
||||
path,
|
||||
serde_json::to_vec(value).expect("legacy IAM object must serialize"),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
fn assert_identity_fields(actual: &UserIdentity, expected: &Value) {
|
||||
assert_eq!(
|
||||
serde_json::to_value(actual).expect("loaded identity must serialize"),
|
||||
*expected,
|
||||
"migration must preserve every credential field except expiration",
|
||||
);
|
||||
}
|
||||
|
||||
async fn assert_identity_survives(
|
||||
store: &ObjectStore,
|
||||
identity_path: &str,
|
||||
name: &str,
|
||||
user_type: UserType,
|
||||
source: &Value,
|
||||
expected_policy: &Value,
|
||||
) {
|
||||
let mut expected = source.clone();
|
||||
expected["credentials"]["expiration"] = Value::Null;
|
||||
|
||||
let persisted: UserIdentity = store
|
||||
.load_iam_config(identity_path)
|
||||
.await
|
||||
.expect("migrated identity must be persisted");
|
||||
assert_identity_fields(&persisted, &expected);
|
||||
|
||||
for _ in 0..2 {
|
||||
let actual = store
|
||||
.load_user_identity(name, user_type)
|
||||
.await
|
||||
.expect("migrated permanent identity must remain loadable");
|
||||
assert_identity_fields(&actual, &expected);
|
||||
}
|
||||
|
||||
let mut mappings = HashMap::new();
|
||||
store
|
||||
.load_mapped_policy(name, user_type, false, &mut mappings)
|
||||
.await
|
||||
.expect("loading the identity must not delete its policy mapping");
|
||||
let actual_policy = mappings.get(name).expect("migrated policy mapping must exist");
|
||||
assert_eq!(
|
||||
serde_json::to_value(actual_policy).expect("loaded policy mapping must serialize"),
|
||||
*expected_policy,
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn minio_permanent_identities_survive_migration_and_repeated_iam_loads() {
|
||||
if get_global_action_cred().is_none() {
|
||||
init_global_action_credentials(Some("MINIOMIGRATIONROOT".to_string()), Some("minio-migration-root-secret".to_string()))
|
||||
.expect("root credentials must initialize for JWT validation");
|
||||
}
|
||||
|
||||
let temp_dir = tempfile::TempDir::with_prefix("rustfs_minio_iam_migration_").expect("temp directory must be created");
|
||||
let env = rustfs_test_utils::TestECStoreEnv::builder()
|
||||
.base_dir(temp_dir.path())
|
||||
.init_bucket_metadata(false)
|
||||
.build()
|
||||
.await;
|
||||
for disk_path in &env.disk_paths {
|
||||
tokio::fs::create_dir_all(disk_path.join(LEGACY_META_BUCKET))
|
||||
.await
|
||||
.expect("legacy metadata volume must be created");
|
||||
}
|
||||
|
||||
let regular_source = json!({
|
||||
"version": 1,
|
||||
"credentials": {
|
||||
"accessKey": REGULAR_USER,
|
||||
"secretKey": "regular-user-secret",
|
||||
"sessionToken": "",
|
||||
"expiration": "0001-01-01T00:00:00Z",
|
||||
"status": "on",
|
||||
"parentUser": "regular-parent",
|
||||
"groups": ["engineering", "operations"],
|
||||
"claims": {"tenant": "alpha"},
|
||||
"name": "MinIO regular user",
|
||||
"description": "migrated regular identity"
|
||||
},
|
||||
"updatedAt": "2025-03-07T12:00:00Z"
|
||||
});
|
||||
let service_claims = json!({"sa-policy": "inherited-policy", "tenant": "alpha"});
|
||||
let service_secret = "service-account-secret";
|
||||
let service_source = json!({
|
||||
"version": 1,
|
||||
"credentials": {
|
||||
"accessKey": SERVICE_ACCOUNT,
|
||||
"secretKey": service_secret,
|
||||
"sessionToken": generate_jwt(&service_claims, service_secret).expect("service-account JWT must be generated"),
|
||||
"expiration": "1970-01-01T00:00:00Z",
|
||||
"status": "on",
|
||||
"parentUser": REGULAR_USER,
|
||||
"groups": ["service-accounts"],
|
||||
"claims": service_claims,
|
||||
"name": "MinIO service account",
|
||||
"description": "migrated service identity"
|
||||
},
|
||||
"updatedAt": "2025-03-07T12:00:00Z"
|
||||
});
|
||||
let regular_policy_source = json!({"version": 1, "policy": "readwrite", "updatedAt": "2025-03-07T12:00:00Z"});
|
||||
let service_policy_source = json!({"version": 1, "policy": "readonly", "updatedAt": "2025-03-07T12:00:00Z"});
|
||||
|
||||
let regular_identity_path = format!("{}{REGULAR_USER}/identity.json", IAM_CONFIG_USERS_PREFIX.as_str());
|
||||
let service_identity_path = format!("{}{SERVICE_ACCOUNT}/identity.json", IAM_CONFIG_SERVICE_ACCOUNTS_PREFIX.as_str());
|
||||
|
||||
seed_legacy_iam_object(&env, ®ular_identity_path, ®ular_source).await;
|
||||
seed_legacy_iam_object(&env, &service_identity_path, &service_source).await;
|
||||
seed_legacy_iam_object(
|
||||
&env,
|
||||
&format!("{}{REGULAR_USER}.json", IAM_CONFIG_POLICY_DB_USERS_PREFIX.as_str()),
|
||||
®ular_policy_source,
|
||||
)
|
||||
.await;
|
||||
seed_legacy_iam_object(
|
||||
&env,
|
||||
&format!("{}{SERVICE_ACCOUNT}.json", IAM_CONFIG_POLICY_DB_SERVICE_ACCOUNTS_PREFIX.as_str()),
|
||||
&service_policy_source,
|
||||
)
|
||||
.await;
|
||||
|
||||
try_migrate_iam_config(env.ecstore.clone(), None).await;
|
||||
|
||||
let store = ObjectStore::new(env.ecstore);
|
||||
assert_identity_survives(
|
||||
&store,
|
||||
®ular_identity_path,
|
||||
REGULAR_USER,
|
||||
UserType::Reg,
|
||||
®ular_source,
|
||||
®ular_policy_source,
|
||||
)
|
||||
.await;
|
||||
assert_identity_survives(
|
||||
&store,
|
||||
&service_identity_path,
|
||||
SERVICE_ACCOUNT,
|
||||
UserType::Svc,
|
||||
&service_source,
|
||||
&service_policy_source,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
@@ -265,6 +265,78 @@ pub fn active_replication_rule_destination_arns(config: &ReplicationConfiguratio
|
||||
arns
|
||||
}
|
||||
|
||||
/// Deployment id extracted from a site-replication target ARN
|
||||
/// (`arn:{rustfs|minio}:replication::<deployment-id>:<bucket>`), or `None`
|
||||
/// for an operator-authored ARN.
|
||||
pub fn replication_target_arn_deployment_id(arn: &str) -> Option<String> {
|
||||
let parts: Vec<_> = arn.split(':').collect();
|
||||
if parts.len() == 6
|
||||
&& parts[0] == "arn"
|
||||
&& matches!(parts[1], "rustfs" | "minio")
|
||||
&& parts[2] == "replication"
|
||||
&& !parts[4].is_empty()
|
||||
{
|
||||
return Some(parts[4].to_string());
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
/// Whether `rule` is a site-replication rule (`site-repl-*` id) owned by the
|
||||
/// local site's reconciler rather than authored by an operator.
|
||||
pub fn is_site_replication_rule(rule: &ReplicationRule) -> bool {
|
||||
rule.id.as_deref().is_some_and(|id| id.starts_with("site-repl-"))
|
||||
}
|
||||
|
||||
/// Merge an incoming replication config into the local one.
|
||||
///
|
||||
/// `site-repl-*` rules encode the *holder's* outbound direction — their
|
||||
/// destination ARN names another site — so applying an external rule set
|
||||
/// verbatim replaces the local reverse rule with one this site can never
|
||||
/// satisfy (no bucket target backs it) and replication silently stops. Only
|
||||
/// operator-authored rules travel: the site-replication peer ingestion path
|
||||
/// and the S3 put/delete-bucket-replication path both keep the local site's
|
||||
/// `site-repl-*` rules through this merge. `incoming == None` models a
|
||||
/// delete of the operator-authored rules.
|
||||
pub fn merge_incoming_replication_config(
|
||||
incoming: Option<ReplicationConfiguration>,
|
||||
local: Option<ReplicationConfiguration>,
|
||||
) -> Option<ReplicationConfiguration> {
|
||||
let incoming_role = incoming.as_ref().map(|config| config.role.clone()).unwrap_or_default();
|
||||
// Operator rules first, then the local site rules — the same order the
|
||||
// site-replication reconciler produces, so its no-op check matches and
|
||||
// the bucket metadata is written once per broadcast, not twice.
|
||||
let mut rules: Vec<ReplicationRule> = incoming
|
||||
.into_iter()
|
||||
.flat_map(|config| config.rules)
|
||||
.filter(|rule| !is_site_replication_rule(rule))
|
||||
.collect();
|
||||
rules.extend(
|
||||
local
|
||||
.into_iter()
|
||||
.flat_map(|config| config.rules)
|
||||
.filter(is_site_replication_rule),
|
||||
);
|
||||
|
||||
if rules.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
for (index, rule) in rules.iter_mut().enumerate() {
|
||||
rule.priority = Some(i32::try_from(index + 1).unwrap_or(i32::MAX));
|
||||
}
|
||||
|
||||
// A site-replication ARN in `role` is the sender's, and the reconciler's
|
||||
// per-peer target lookup reads it — carrying it over would pin the
|
||||
// receiver's targets to the sender's identity.
|
||||
let role = match replication_target_arn_deployment_id(&incoming_role) {
|
||||
Some(_) => String::new(),
|
||||
None => incoming_role,
|
||||
};
|
||||
|
||||
Some(ReplicationConfiguration { role, rules })
|
||||
}
|
||||
|
||||
pub fn replication_target_arns(config: &ReplicationConfiguration) -> HashSet<String> {
|
||||
let role = config.role.trim();
|
||||
if !role.is_empty() {
|
||||
|
||||
@@ -32,7 +32,8 @@ pub use config::{
|
||||
ObjectOpts, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS,
|
||||
REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS,
|
||||
ReplicationConfigStructureError, ReplicationConfigurationExt, ReplicationTargetValidationError,
|
||||
active_replication_rule_destination_arns, invalid_replication_config_status_field, replication_target_arns,
|
||||
active_replication_rule_destination_arns, invalid_replication_config_status_field, is_site_replication_rule,
|
||||
merge_incoming_replication_config, replication_target_arn_deployment_id, replication_target_arns,
|
||||
should_remove_replication_target, unsupported_replication_config_field, validate_replication_config_structure,
|
||||
validate_replication_config_target_arns,
|
||||
};
|
||||
|
||||
+166
-78
@@ -1081,18 +1081,6 @@ async fn run_data_scanner_cycle(
|
||||
}
|
||||
};
|
||||
let (sender, receiver) = mpsc::channel::<DataUsageInfo>(1);
|
||||
let storeapi_clone = storeapi.clone();
|
||||
let ctx_clone = ctx.clone();
|
||||
let mut usage_persist_task = AbortOnDropHandle::new(tokio::spawn(async move {
|
||||
store_data_usage_in_backend_with_outcome_for_epoch_and_baseline(
|
||||
ctx_clone,
|
||||
storeapi_clone,
|
||||
receiver,
|
||||
Some(leader_epoch),
|
||||
Some(usage_persist_baseline),
|
||||
)
|
||||
.await
|
||||
}));
|
||||
|
||||
let done_cycle = Metrics::time(Metric::ScanCycle);
|
||||
let cycle_budget = ScannerCycleBudget::new(ctx, cycle_budget_config);
|
||||
@@ -1107,47 +1095,78 @@ async fn run_data_scanner_cycle(
|
||||
scan_mode,
|
||||
)
|
||||
.await;
|
||||
let publication_defer_reason = match &scan_result {
|
||||
Ok(result) => final_data_usage_publication_defer_reason(storeapi.as_ref(), result.status).await,
|
||||
Err(_) => Some(ScannerCycleDeferReason::ActivityBaselineUnavailable),
|
||||
};
|
||||
let budget_elapsed = cycle_budget.budget_elapsed() && !ctx.is_cancelled();
|
||||
let usage_persist_outcome = match wait_for_data_usage_persist_task(ctx, &mut usage_persist_task, usage_persist_timeout).await
|
||||
{
|
||||
DataUsagePersistTaskResult::Completed(outcome) => outcome,
|
||||
DataUsagePersistTaskResult::JoinFailed(err) => {
|
||||
error!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_PERSIST_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
cycle = cycle_info.current,
|
||||
state = "usage_persist_task_failed",
|
||||
error = %err,
|
||||
"Scanner data usage persistence task failed"
|
||||
);
|
||||
DataUsagePersistOutcome::Failed
|
||||
let usage_persist_outcome = match publication_defer_reason {
|
||||
Some(reason) => {
|
||||
drop(receiver);
|
||||
DataUsagePersistOutcome::Deferred(reason)
|
||||
}
|
||||
DataUsagePersistTaskResult::Cancelled => {
|
||||
debug!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_PERSIST_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
cycle = cycle_info.current,
|
||||
state = "usage_persist_task_cancelled",
|
||||
"Scanner data usage persistence task cancelled"
|
||||
);
|
||||
DataUsagePersistOutcome::Failed
|
||||
}
|
||||
DataUsagePersistTaskResult::TimedOut => {
|
||||
error!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_PERSIST_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
cycle = cycle_info.current,
|
||||
timeout = ?usage_persist_timeout,
|
||||
state = "usage_persist_task_timed_out",
|
||||
"Scanner data usage persistence task timed out"
|
||||
);
|
||||
DataUsagePersistOutcome::Failed
|
||||
None => {
|
||||
// ScannerIO emits its complete or observational update only after
|
||||
// all set workers finish. Persist after the final activity fence;
|
||||
// this also avoids blocking the scanner on a denied publication.
|
||||
let storeapi_clone = storeapi.clone();
|
||||
let ctx_clone = ctx.clone();
|
||||
let route_probe_store = storeapi.clone();
|
||||
let mut usage_persist_task = AbortOnDropHandle::new(tokio::spawn(async move {
|
||||
store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe(
|
||||
ctx_clone,
|
||||
storeapi_clone,
|
||||
receiver,
|
||||
Some(leader_epoch),
|
||||
Some(usage_persist_baseline),
|
||||
move || {
|
||||
let storeapi = route_probe_store.clone();
|
||||
async move { storeapi.scanner_data_usage_publication_blocked().await }
|
||||
},
|
||||
)
|
||||
.await
|
||||
}));
|
||||
match wait_for_data_usage_persist_task(ctx, &mut usage_persist_task, usage_persist_timeout).await {
|
||||
DataUsagePersistTaskResult::Completed(outcome) => outcome,
|
||||
DataUsagePersistTaskResult::JoinFailed(err) => {
|
||||
error!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_PERSIST_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
cycle = cycle_info.current,
|
||||
state = "usage_persist_task_failed",
|
||||
error = %err,
|
||||
"Scanner data usage persistence task failed"
|
||||
);
|
||||
DataUsagePersistOutcome::Failed
|
||||
}
|
||||
DataUsagePersistTaskResult::Cancelled => {
|
||||
debug!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_PERSIST_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
cycle = cycle_info.current,
|
||||
state = "usage_persist_task_cancelled",
|
||||
"Scanner data usage persistence task cancelled"
|
||||
);
|
||||
DataUsagePersistOutcome::Failed
|
||||
}
|
||||
DataUsagePersistTaskResult::TimedOut => {
|
||||
error!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_PERSIST_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
cycle = cycle_info.current,
|
||||
timeout = ?usage_persist_timeout,
|
||||
state = "usage_persist_task_timed_out",
|
||||
"Scanner data usage persistence task timed out"
|
||||
);
|
||||
DataUsagePersistOutcome::Failed
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
let unresolved_heal_work = global_metrics().current_scan_cycle_has_unresolved_heal_work();
|
||||
@@ -1191,33 +1210,51 @@ async fn run_data_scanner_cycle(
|
||||
mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await;
|
||||
return ScannerCycleOutcome::Failed;
|
||||
}
|
||||
if let Some(required_cycle) = scan_cycle_result.required_cycle_floor() {
|
||||
warn!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_CYCLE_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
cycle = cycle_info.current,
|
||||
required_cycle,
|
||||
state = "cache_cycle_ahead",
|
||||
"Scanner cycle is recovering to a newer durable cache generation"
|
||||
);
|
||||
emit_scan_cycle_partial_with_source(cycle_start.elapsed(), ScanCyclePartialReason::Unknown, None);
|
||||
return if persist_required_scanner_cycle_floor(
|
||||
ctx,
|
||||
storeapi.clone(),
|
||||
cycle_info,
|
||||
cycle_revision,
|
||||
leader_epoch,
|
||||
required_cycle,
|
||||
&mut cycle_metrics_guard,
|
||||
)
|
||||
.await
|
||||
{
|
||||
ScannerCycleOutcome::Partial
|
||||
} else {
|
||||
ScannerCycleOutcome::Failed
|
||||
};
|
||||
match scanner_cycle_pre_commit_outcome(scan_cycle_result.required_cycle_floor(), &usage_persist_outcome) {
|
||||
Some(ScannerCyclePreCommitOutcome::RecoverCacheCycle(required_cycle)) => {
|
||||
warn!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_CYCLE_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
cycle = cycle_info.current,
|
||||
required_cycle,
|
||||
state = "cache_cycle_ahead",
|
||||
"Scanner cycle is recovering to a newer durable cache generation"
|
||||
);
|
||||
emit_scan_cycle_partial_with_source(cycle_start.elapsed(), ScanCyclePartialReason::Unknown, None);
|
||||
return if persist_required_scanner_cycle_floor(
|
||||
ctx,
|
||||
storeapi.clone(),
|
||||
cycle_info,
|
||||
cycle_revision,
|
||||
leader_epoch,
|
||||
required_cycle,
|
||||
&mut cycle_metrics_guard,
|
||||
)
|
||||
.await
|
||||
{
|
||||
ScannerCycleOutcome::Partial
|
||||
} else {
|
||||
ScannerCycleOutcome::Failed
|
||||
};
|
||||
}
|
||||
Some(ScannerCyclePreCommitOutcome::Deferred(reason)) => {
|
||||
info!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_CYCLE_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
cycle = cycle_info.current,
|
||||
reason = reason.as_str(),
|
||||
state = "deferred",
|
||||
"Scanner cycle deferred before data usage publication"
|
||||
);
|
||||
emit_scan_cycle_deferred(cycle_start.elapsed());
|
||||
mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await;
|
||||
return ScannerCycleOutcome::Deferred(reason);
|
||||
}
|
||||
None => {}
|
||||
}
|
||||
if usage_persist_outcome == DataUsagePersistOutcome::Failed {
|
||||
error!(
|
||||
@@ -2000,6 +2037,56 @@ impl Drop for ScannerScanModeGuard {
|
||||
}
|
||||
}
|
||||
|
||||
async fn final_data_usage_publication_defer_reason(
|
||||
storeapi: &ECStore,
|
||||
status: ScannerCycleStatus,
|
||||
) -> Option<ScannerCycleDeferReason> {
|
||||
match status {
|
||||
ScannerCycleStatus::Complete | ScannerCycleStatus::Superseded => {
|
||||
if storeapi.scanner_data_usage_publication_blocked().await {
|
||||
return Some(ScannerCycleDeferReason::DataMovement);
|
||||
}
|
||||
if status == ScannerCycleStatus::Complete {
|
||||
let distributed = storeapi.setup_is_dist_erasure().await;
|
||||
match probe_scanner_activity(storeapi, distributed).await {
|
||||
Ok(snapshot) if scanner_activity_allows_usage_publication(&snapshot) => None,
|
||||
Ok(_) => Some(ScannerCycleDeferReason::DataMovement),
|
||||
Err(_) => Some(ScannerCycleDeferReason::ActivityBaselineUnavailable),
|
||||
}
|
||||
} else {
|
||||
// A superseded cycle is explicitly observational and cannot
|
||||
// replace the authoritative snapshot. It may still be
|
||||
// persisted as a convergence baseline for the next cycle.
|
||||
None
|
||||
}
|
||||
}
|
||||
ScannerCycleStatus::Deferred(reason) => Some(reason),
|
||||
// Incomplete cycles do not publish a usage snapshot. Keep the
|
||||
// decision permissive so existing partial-cycle handling remains
|
||||
// unchanged if a future scanner path emits a bookkeeping update.
|
||||
ScannerCycleStatus::Incomplete => None,
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
enum ScannerCyclePreCommitOutcome {
|
||||
RecoverCacheCycle(u64),
|
||||
Deferred(ScannerCycleDeferReason),
|
||||
}
|
||||
|
||||
fn scanner_cycle_pre_commit_outcome(
|
||||
required_cycle_floor: Option<u64>,
|
||||
usage_persist_outcome: &DataUsagePersistOutcome,
|
||||
) -> Option<ScannerCyclePreCommitOutcome> {
|
||||
// Keep the publication barrier fail-closed: `.bloomcycle.bin` uses the
|
||||
// same routed writer and its floor must remain pending while data movement
|
||||
// hides the source pool.
|
||||
match usage_persist_outcome {
|
||||
DataUsagePersistOutcome::Deferred(reason) => Some(ScannerCyclePreCommitOutcome::Deferred(*reason)),
|
||||
_ => required_cycle_floor.map(ScannerCyclePreCommitOutcome::RecoverCacheCycle),
|
||||
}
|
||||
}
|
||||
|
||||
fn scanner_cycle_completion_outcome(
|
||||
scan_status: ScannerCycleStatus,
|
||||
usage_persist_outcome: DataUsagePersistOutcome,
|
||||
@@ -2007,6 +2094,7 @@ fn scanner_cycle_completion_outcome(
|
||||
has_failed_dirty_usage: bool,
|
||||
) -> ScannerCycleOutcome {
|
||||
match (scan_status, usage_persist_outcome) {
|
||||
(_, DataUsagePersistOutcome::Deferred(reason)) => ScannerCycleOutcome::Deferred(reason),
|
||||
(_, DataUsagePersistOutcome::Failed) => ScannerCycleOutcome::Failed,
|
||||
(ScannerCycleStatus::Deferred(reason), DataUsagePersistOutcome::NoUpdate)
|
||||
if !has_dirty_usage && !has_failed_dirty_usage =>
|
||||
|
||||
@@ -153,6 +153,7 @@ struct MemoryConfigStore {
|
||||
objects: Mutex<HashMap<String, Vec<u8>>>,
|
||||
revisions: Mutex<HashMap<String, u64>>,
|
||||
fail_put_number: Mutex<HashMap<String, usize>>,
|
||||
object_not_found_put_number: Mutex<HashMap<String, usize>>,
|
||||
error_after_commit_put_number: Mutex<HashMap<String, usize>>,
|
||||
interleaving_puts: Mutex<HashMap<String, (usize, Vec<u8>)>>,
|
||||
cancel_after_interleaving_puts: Mutex<HashMap<String, CancellationToken>>,
|
||||
@@ -224,6 +225,9 @@ impl crate::storage_api::scanner_io::ObjectIO for MemoryConfigStore {
|
||||
if self.fail_put_number.lock().await.get(&key) == Some(&put_count) {
|
||||
return Err(EcstoreError::other("injected put failure"));
|
||||
}
|
||||
if self.object_not_found_put_number.lock().await.get(&key) == Some(&put_count) {
|
||||
return Err(EcstoreError::ObjectNotFound(bucket.to_string(), object.to_string()));
|
||||
}
|
||||
|
||||
let interleaving_data = {
|
||||
let mut interleaving_puts = self.interleaving_puts.lock().await;
|
||||
@@ -1431,6 +1435,170 @@ async fn test_store_data_usage_in_backend_preserves_newer_snapshot() {
|
||||
assert_eq!(outcome, DataUsagePersistOutcome::Current);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_usage_save_object_not_found_defers_only_with_a_fresh_route_barrier() {
|
||||
for (route_blocked, expected) in [
|
||||
(true, DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement)),
|
||||
(false, DataUsagePersistOutcome::Failed),
|
||||
] {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str());
|
||||
let baseline = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(10)), 1);
|
||||
let baseline_data = serde_json::to_vec(&baseline).expect("baseline usage snapshot should encode");
|
||||
store.objects.lock().await.insert(key.clone(), baseline_data.clone());
|
||||
store.revisions.lock().await.insert(key.clone(), 1);
|
||||
store.object_not_found_put_number.lock().await.insert(key.clone(), 1);
|
||||
|
||||
let (sender, receiver) = mpsc::channel(1);
|
||||
sender
|
||||
.send(complete_usage_with_bucket_count(
|
||||
Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)),
|
||||
2,
|
||||
))
|
||||
.await
|
||||
.expect("new usage snapshot should enqueue");
|
||||
drop(sender);
|
||||
let probe_calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let route_probe_calls = probe_calls.clone();
|
||||
|
||||
let outcome = store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe(
|
||||
CancellationToken::new(),
|
||||
store.clone(),
|
||||
receiver,
|
||||
None,
|
||||
Some(DataUsagePersistBaseline {
|
||||
data: Some(Bytes::from(baseline_data.clone())),
|
||||
revision: DataUsageCacheRevision::Etag("memory-1".to_string()),
|
||||
}),
|
||||
move || {
|
||||
let probe_calls = route_probe_calls.clone();
|
||||
async move {
|
||||
let call = probe_calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||
route_blocked && call > 1
|
||||
}
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(outcome, expected);
|
||||
assert_eq!(
|
||||
probe_calls.load(std::sync::atomic::Ordering::SeqCst),
|
||||
3,
|
||||
"ObjectNotFound must be followed by a fresh route-barrier probe"
|
||||
);
|
||||
assert_eq!(
|
||||
store.objects.lock().await.get(&key),
|
||||
Some(&baseline_data),
|
||||
"a route failure must not replace the authoritative baseline"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_usage_save_route_barrier_prevents_missing_snapshot_creation() {
|
||||
for observational in [false, true] {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
let target_path = if observational {
|
||||
DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str()
|
||||
} else {
|
||||
DATA_USAGE_OBJ_NAME_PATH.as_str()
|
||||
};
|
||||
let target_key = memory_config_key(RUSTFS_META_BUCKET, target_path);
|
||||
let mut incoming = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)), 1);
|
||||
incoming.usage_snapshot_converged = Some(!observational);
|
||||
let (sender, receiver) = mpsc::channel(1);
|
||||
sender.send(incoming).await.expect("usage snapshot should enqueue");
|
||||
drop(sender);
|
||||
|
||||
let outcome = store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe(
|
||||
CancellationToken::new(),
|
||||
store.clone(),
|
||||
receiver,
|
||||
None,
|
||||
Some(DataUsagePersistBaseline {
|
||||
data: None,
|
||||
revision: DataUsageCacheRevision::Missing,
|
||||
}),
|
||||
|| async { true },
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(outcome, DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement));
|
||||
assert!(!store.objects.lock().await.contains_key(&target_key));
|
||||
assert_eq!(
|
||||
store.put_counts.lock().await.get(&target_key),
|
||||
None,
|
||||
"the final pool-state fence must run before the first PUT"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_usage_route_barrier_precedes_durable_reconciliation() {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str());
|
||||
let snapshot = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)), 1);
|
||||
let snapshot_data = serde_json::to_vec(&snapshot).expect("usage snapshot should encode");
|
||||
let (sender, receiver) = mpsc::channel(1);
|
||||
sender.send(snapshot).await.expect("usage snapshot should enqueue");
|
||||
drop(sender);
|
||||
|
||||
let outcome = store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe(
|
||||
CancellationToken::new(),
|
||||
store.clone(),
|
||||
receiver,
|
||||
None,
|
||||
Some(DataUsagePersistBaseline {
|
||||
data: Some(Bytes::from(snapshot_data)),
|
||||
revision: DataUsageCacheRevision::Etag("memory-1".to_string()),
|
||||
}),
|
||||
|| async { true },
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(outcome, DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement));
|
||||
assert_eq!(store.put_counts.lock().await.get(&key), None);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_deferred_usage_save_keeps_last_real_save_metric() {
|
||||
let metrics = global_metrics();
|
||||
metrics.record_scanner_usage_save_result(ScannerUsageSaveResult::Success);
|
||||
let before = metrics.report().await.usage_freshness;
|
||||
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
let (sender, receiver) = mpsc::channel(1);
|
||||
sender
|
||||
.send(complete_usage_with_bucket_count(
|
||||
Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)),
|
||||
1,
|
||||
))
|
||||
.await
|
||||
.expect("usage snapshot should enqueue");
|
||||
drop(sender);
|
||||
|
||||
let outcome = store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe(
|
||||
CancellationToken::new(),
|
||||
store,
|
||||
receiver,
|
||||
None,
|
||||
Some(DataUsagePersistBaseline {
|
||||
data: None,
|
||||
revision: DataUsageCacheRevision::Missing,
|
||||
}),
|
||||
|| async { true },
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(outcome, DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement));
|
||||
let after = metrics.report().await.usage_freshness;
|
||||
assert_eq!(after.last_usage_save_result, before.last_usage_save_result);
|
||||
assert_eq!(after.last_usage_save_result_code, before.last_usage_save_result_code);
|
||||
assert_eq!(after.last_usage_save_unix_secs, before.last_usage_save_unix_secs);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_store_data_usage_in_backend_fences_interleaving_newer_writer() {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
@@ -2325,6 +2493,15 @@ async fn test_store_data_usage_in_backend_reports_missing_snapshot() {
|
||||
|
||||
#[test]
|
||||
fn test_scanner_cycle_completion_prioritizes_persist_failure() {
|
||||
assert_eq!(
|
||||
scanner_cycle_completion_outcome(
|
||||
ScannerCycleStatus::Complete,
|
||||
DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement),
|
||||
true,
|
||||
false,
|
||||
),
|
||||
ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement)
|
||||
);
|
||||
assert_eq!(
|
||||
scanner_cycle_completion_outcome(
|
||||
ScannerCycleStatus::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable),
|
||||
@@ -2421,6 +2598,33 @@ fn test_scanner_cycle_completion_prioritizes_persist_failure() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_cycle_cache_floor_stays_pending_during_deferred_usage_publication() {
|
||||
for reason in [
|
||||
ScannerCycleDeferReason::DataMovement,
|
||||
ScannerCycleDeferReason::ActivityBaselineUnavailable,
|
||||
] {
|
||||
let deferred = DataUsagePersistOutcome::Deferred(reason);
|
||||
assert_eq!(
|
||||
scanner_cycle_pre_commit_outcome(Some(19), &deferred),
|
||||
Some(ScannerCyclePreCommitOutcome::Deferred(reason)),
|
||||
"a blocked publication must not persist the routed scanner cycle floor"
|
||||
);
|
||||
assert_eq!(
|
||||
scanner_cycle_pre_commit_outcome(None, &deferred),
|
||||
Some(ScannerCyclePreCommitOutcome::Deferred(reason))
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
scanner_cycle_pre_commit_outcome(Some(19), &DataUsagePersistOutcome::Saved),
|
||||
Some(ScannerCyclePreCommitOutcome::RecoverCacheCycle(19))
|
||||
);
|
||||
assert_eq!(
|
||||
scanner_cycle_pre_commit_outcome(Some(19), &DataUsagePersistOutcome::Failed),
|
||||
Some(ScannerCyclePreCommitOutcome::RecoverCacheCycle(19))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn finalizing_a_saved_cycle_acknowledges_its_exact_dirty_snapshot() {
|
||||
@@ -2448,6 +2652,23 @@ fn finalizing_a_saved_cycle_acknowledges_its_exact_dirty_snapshot() {
|
||||
assert!(!crate::scanner_io::dirty_usage_buckets_pending());
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn finalizing_a_deferred_usage_save_keeps_dirty_work_pending() {
|
||||
crate::scanner_io::clear_dirty_usage_bucket("photos");
|
||||
crate::scanner_io::record_dirty_usage_bucket("photos");
|
||||
let dirty_snapshot = crate::scanner_io::dirty_usage_buckets_for_tests();
|
||||
let deferred = crate::scanner_io::ScannerCycleResult::new(ScannerCycleStatus::Complete, Some(dirty_snapshot));
|
||||
|
||||
let (outcome, _, acknowledgements) =
|
||||
finalize_scanner_cycle_result(deferred, DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement));
|
||||
|
||||
assert_eq!(outcome, ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement));
|
||||
assert!(acknowledgements.is_empty());
|
||||
assert!(crate::scanner_io::dirty_usage_buckets_pending());
|
||||
crate::scanner_io::clear_dirty_usage_bucket("photos");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_cycle_keeps_remote_pending_acknowledgement() {
|
||||
let pending = remote_dirty_usage_acknowledgement_pending(7, 1, std::future::ready(Ok::<bool, std::io::Error>(true))).await;
|
||||
|
||||
@@ -22,6 +22,10 @@ pub(super) enum DataUsagePersistOutcome {
|
||||
AlreadyDurable,
|
||||
PriorCycleDurable,
|
||||
Saved,
|
||||
/// The metadata route is temporarily unavailable (for example while a
|
||||
/// terminal decommission state keeps the source pool suspended). The
|
||||
/// caller must retry without acknowledging dirty usage.
|
||||
Deferred(ScannerCycleDeferReason),
|
||||
Failed,
|
||||
}
|
||||
|
||||
@@ -92,10 +96,33 @@ pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch(
|
||||
pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_baseline(
|
||||
ctx: CancellationToken,
|
||||
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
|
||||
mut receiver: mpsc::Receiver<DataUsageInfo>,
|
||||
receiver: mpsc::Receiver<DataUsageInfo>,
|
||||
leader_epoch: Option<u64>,
|
||||
initial_baseline: Option<DataUsagePersistBaseline>,
|
||||
) -> DataUsagePersistOutcome {
|
||||
store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe(
|
||||
ctx,
|
||||
storeapi,
|
||||
receiver,
|
||||
leader_epoch,
|
||||
initial_baseline,
|
||||
|| async { false },
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe<F, Fut>(
|
||||
ctx: CancellationToken,
|
||||
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
|
||||
mut receiver: mpsc::Receiver<DataUsageInfo>,
|
||||
leader_epoch: Option<u64>,
|
||||
initial_baseline: Option<DataUsagePersistBaseline>,
|
||||
route_probe: F,
|
||||
) -> DataUsagePersistOutcome
|
||||
where
|
||||
F: Fn() -> Fut + Send + Sync,
|
||||
Fut: Future<Output = bool> + Send,
|
||||
{
|
||||
let mut outcome = DataUsagePersistOutcome::NoUpdate;
|
||||
let mut next_baseline = initial_baseline;
|
||||
|
||||
@@ -113,6 +140,19 @@ pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_basel
|
||||
} else {
|
||||
DATA_USAGE_OBJ_NAME_PATH.as_str()
|
||||
};
|
||||
if route_probe().await {
|
||||
debug!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_PERSIST_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
path = %target_path,
|
||||
state = "publication_blocked_before_reconcile",
|
||||
"Scanner data usage publication deferred by the pool-state fence"
|
||||
);
|
||||
outcome = DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
|
||||
break;
|
||||
}
|
||||
|
||||
if observational && data_usage_info.usage_snapshot_authoritative_baseline.is_none() {
|
||||
let authoritative_data = match next_baseline.as_ref() {
|
||||
@@ -275,6 +315,18 @@ pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_basel
|
||||
if ctx.is_cancelled() {
|
||||
break 'updates;
|
||||
}
|
||||
if route_probe().await {
|
||||
debug!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_PERSIST_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
path = %target_path,
|
||||
state = "publication_blocked_before_save",
|
||||
"Scanner data usage publication deferred by the final pool-state fence"
|
||||
);
|
||||
break DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
|
||||
}
|
||||
|
||||
let done_save = Metrics::time(Metric::SaveUsage);
|
||||
let save_result = save_config_shared_with_preconditions(
|
||||
@@ -313,6 +365,33 @@ pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_basel
|
||||
"Scanner data usage CAS conflict will be reconciled"
|
||||
);
|
||||
}
|
||||
Err(e @ EcstoreError::ObjectNotFound(_, _)) => {
|
||||
let route_blocked = route_probe().await;
|
||||
if route_blocked {
|
||||
warn!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_PERSIST_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
path = %target_path,
|
||||
state = "publication_deferred",
|
||||
error = %e,
|
||||
"Scanner data usage route is blocked by data movement; retrying later"
|
||||
);
|
||||
break DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
|
||||
}
|
||||
error!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_PERSIST_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
path = %target_path,
|
||||
state = "save_failed",
|
||||
error = %e,
|
||||
"Scanner data usage save failed"
|
||||
);
|
||||
break DataUsagePersistOutcome::Failed;
|
||||
}
|
||||
Err(e) => {
|
||||
error!(
|
||||
target: "rustfs::scanner",
|
||||
@@ -370,6 +449,13 @@ pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_basel
|
||||
outcome = DataUsagePersistOutcome::Failed;
|
||||
continue;
|
||||
}
|
||||
DataUsagePersistOutcome::Deferred(reason) => {
|
||||
// A deferred publication is an intentional retryable state, not a
|
||||
// failed save. Keep the last real save result so admin freshness
|
||||
// reporting does not turn a pool-recovery fence into a false error.
|
||||
outcome = DataUsagePersistOutcome::Deferred(reason);
|
||||
break 'updates;
|
||||
}
|
||||
DataUsagePersistOutcome::Saved => {
|
||||
if observational {
|
||||
invalidate_admin_data_usage_snapshot_cache().await;
|
||||
|
||||
@@ -49,6 +49,25 @@ impl ScannerIOCycle for ECStore {
|
||||
) -> Result<ScannerCycleResult> {
|
||||
let child_token = ctx.child_token();
|
||||
|
||||
// Check the local pool metadata before listing buckets. A failed or
|
||||
// canceled decommission remains suspended after its worker exits, so
|
||||
// starting a scan in that state could build a snapshot that cannot be
|
||||
// routed to the authoritative metadata object.
|
||||
if self.scanner_data_usage_publication_blocked().await {
|
||||
debug!(
|
||||
target: "rustfs::scanner::io",
|
||||
event = EVENT_SCANNER_SET_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_IO,
|
||||
state = "cycle_data_usage_route_blocked",
|
||||
"Scanner cycle deferred while data usage metadata remains hidden by data movement"
|
||||
);
|
||||
return Ok(ScannerCycleResult::new(
|
||||
ScannerCycleStatus::Deferred(ScannerCycleDeferReason::DataMovement),
|
||||
None,
|
||||
));
|
||||
}
|
||||
|
||||
let distributed = self.setup_is_dist_erasure().await;
|
||||
let activity_before = match scanner_activity_preflight(crate::scanner::probe_scanner_activity(self, distributed).await) {
|
||||
ScannerActivityPreflight::Ready(snapshot) => snapshot,
|
||||
|
||||
@@ -17,7 +17,9 @@ use super::io_disk::tier_stats_template;
|
||||
use super::*;
|
||||
use crate::scanner_budget::ScannerCycleBudgetConfig;
|
||||
use crate::scanner_folder::ScannerItem;
|
||||
use crate::storage_api::owner::{EcstoreRebalStatus, EcstoreRebalanceInfo, EcstoreRebalanceMeta, EcstoreRebalanceStats};
|
||||
use crate::storage_api::owner::{
|
||||
EcstorePoolDecommissionInfo, EcstoreRebalStatus, EcstoreRebalanceInfo, EcstoreRebalanceMeta, EcstoreRebalanceStats,
|
||||
};
|
||||
use crate::storage_api::scan::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions, ObjectIO as _};
|
||||
use crate::{
|
||||
DiskOption, ECStore, Endpoint, EndpointServerPools, Endpoints, InstanceContext, PoolEndpoints, ScannerObjectOptions,
|
||||
@@ -182,6 +184,39 @@ async fn scanner_cycle_is_deferred_while_rebalance_is_active() {
|
||||
assert!(receiver.recv().await.is_none(), "rebalance-deferred cycle must not publish usage");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn scanner_cycle_is_deferred_while_terminal_decommission_is_blocked() {
|
||||
let (_temp_dir, store) = setup_two_pool_scanner_store().await;
|
||||
for decommission in [
|
||||
EcstorePoolDecommissionInfo {
|
||||
failed: true,
|
||||
..Default::default()
|
||||
},
|
||||
EcstorePoolDecommissionInfo {
|
||||
canceled: true,
|
||||
..Default::default()
|
||||
},
|
||||
] {
|
||||
store.pool_meta.write().await.pools[0].decommission = Some(decommission);
|
||||
assert!(store.scanner_data_usage_publication_blocked().await);
|
||||
|
||||
let ctx = CancellationToken::new();
|
||||
let budget = ScannerCycleBudget::new(&ctx, ScannerCycleBudgetConfig::default());
|
||||
let (updates, mut receiver) = mpsc::channel(1);
|
||||
let result = tokio::time::timeout(
|
||||
Duration::from_secs(30),
|
||||
ScannerIOCycle::nsscanner_with_status(store.as_ref(), ctx, budget, updates, 1, 1, HealScanMode::Normal),
|
||||
)
|
||||
.await
|
||||
.expect("terminal-decommission-deferred scanner cycle should finish")
|
||||
.expect("terminal-decommission-deferred scanner cycle should succeed");
|
||||
|
||||
assert_eq!(result.status, ScannerCycleStatus::Deferred(ScannerCycleDeferReason::DataMovement));
|
||||
assert!(receiver.recv().await.is_none(), "blocked cycle must not publish usage");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn data_usage_publish_fails_when_receiver_is_closed() {
|
||||
let (updates, receiver) = mpsc::channel(1);
|
||||
@@ -236,6 +271,10 @@ async fn multi_pool_scanner_cycle_publishes_combined_usage() {
|
||||
assert_eq!(bucket_usage.size, 11);
|
||||
assert_eq!(usage.objects_total_count, 2);
|
||||
assert_eq!(usage.objects_total_size, 11);
|
||||
assert!(
|
||||
receiver.recv().await.is_none(),
|
||||
"a scanner cycle must publish at most one terminal usage snapshot"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -47,6 +47,8 @@ pub(crate) use rustfs_ecstore::api::bucket::versioning_sys::BucketVersioningSys
|
||||
pub(crate) use rustfs_ecstore::api::cache::{
|
||||
ListPathRawOptions as EcstoreListPathRawOptions, list_path_raw as ecstore_list_path_raw,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::capacity::PoolDecommissionInfo as EcstorePoolDecommissionInfo;
|
||||
pub(crate) use rustfs_ecstore::api::capacity::{
|
||||
is_reserved_or_invalid_bucket as ecstore_is_reserved_or_invalid_bucket, path2_bucket_object as ecstore_path2_bucket_object,
|
||||
path2_bucket_object_with_base_path as ecstore_path2_bucket_object_with_base_path,
|
||||
@@ -127,9 +129,9 @@ pub(crate) mod owner {
|
||||
#[cfg(test)]
|
||||
pub(crate) use super::{
|
||||
EcstoreDiskOption, EcstoreDiskStore, EcstoreEndpoint, EcstoreEndpointServerPools, EcstoreEndpoints,
|
||||
EcstoreInstanceContext, EcstorePoolEndpoints, EcstoreRebalStatus, EcstoreRebalanceInfo, EcstoreRebalanceMeta,
|
||||
EcstoreRebalanceStats, ecstore_config_init, ecstore_init_bucket_metadata_sys, ecstore_init_local_disks_with_instance_ctx,
|
||||
ecstore_new_disk,
|
||||
EcstoreInstanceContext, EcstorePoolDecommissionInfo, EcstorePoolEndpoints, EcstoreRebalStatus, EcstoreRebalanceInfo,
|
||||
EcstoreRebalanceMeta, EcstoreRebalanceStats, ecstore_config_init, ecstore_init_bucket_metadata_sys,
|
||||
ecstore_init_local_disks_with_instance_ctx, ecstore_new_disk,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -31,6 +31,9 @@ use crate::admin::storage_api::bucket::metadata::{
|
||||
use crate::admin::storage_api::bucket::metadata_sys;
|
||||
use crate::admin::storage_api::bucket::quota::BucketQuota;
|
||||
use crate::admin::storage_api::bucket::replication;
|
||||
use crate::admin::storage_api::bucket::replication::{
|
||||
is_site_replication_rule, merge_incoming_replication_config, replication_target_arn_deployment_id,
|
||||
};
|
||||
use crate::admin::storage_api::bucket::target::{ARN, BucketTarget, BucketTargetType, BucketTargets, Credentials};
|
||||
use crate::admin::storage_api::bucket::target_sys::BucketTargetSys;
|
||||
use crate::admin::storage_api::bucket::utils::{deserialize, serialize};
|
||||
@@ -1117,6 +1120,14 @@ async fn load_site_replication_state() -> S3Result<SiteReplicationState> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether this deployment participates in site replication (two or more
|
||||
/// peers in the persisted state). Read by the S3 interface layer to gate
|
||||
/// replication-config edits (MinIO `ErrReplicationDenyEditError` semantics,
|
||||
/// issue #1948); a state-read failure propagates so the gate fails closed.
|
||||
pub(crate) async fn site_replication_enabled() -> S3Result<bool> {
|
||||
Ok(load_site_replication_state().await?.enabled())
|
||||
}
|
||||
|
||||
async fn load_site_replication_state_no_lock(store: Arc<ECStore>) -> S3Result<SiteReplicationState> {
|
||||
match read_config_no_lock(store, SITE_REPLICATION_STATE_PATH).await {
|
||||
Ok(data) => parse_site_replication_state(&data),
|
||||
@@ -7748,20 +7759,6 @@ fn bucket_target_deployment_id(target: &BucketTarget) -> Option<String> {
|
||||
replication_target_arn_deployment_id(&target.arn)
|
||||
}
|
||||
|
||||
fn replication_target_arn_deployment_id(arn: &str) -> Option<String> {
|
||||
let parts: Vec<_> = arn.split(':').collect();
|
||||
if parts.len() == 6
|
||||
&& parts[0] == "arn"
|
||||
&& matches!(parts[1], "rustfs" | "minio")
|
||||
&& parts[2] == "replication"
|
||||
&& !parts[4].is_empty()
|
||||
{
|
||||
return Some(parts[4].to_string());
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
fn prune_removed_site_replication_bucket_targets(
|
||||
existing: BucketTargets,
|
||||
removed_deployment_ids: &HashSet<String>,
|
||||
@@ -7786,10 +7783,6 @@ fn prune_removed_site_replication_bucket_targets(
|
||||
(BucketTargets { targets }, removed)
|
||||
}
|
||||
|
||||
fn is_site_replication_rule(rule: &ReplicationRule) -> bool {
|
||||
rule.id.as_deref().is_some_and(|id| id.starts_with("site-repl-"))
|
||||
}
|
||||
|
||||
/// Whether every `site-repl-*` rule on this bucket resolves to a live remote target.
|
||||
///
|
||||
/// The rule set alone cannot answer this: a rule can be perfectly formed while the endpoint
|
||||
@@ -7815,52 +7808,6 @@ async fn site_replication_targets_online(bucket: &str, replication_config_xml: &
|
||||
true
|
||||
}
|
||||
|
||||
/// Merge a peer's replication config into the local one.
|
||||
///
|
||||
/// `site-repl-*` rules encode the *sender's* outbound direction — their destination ARN
|
||||
/// names the receiver — so applying a peer's rule set verbatim replaces the receiver's
|
||||
/// reverse rule with one pointing at itself. No bucket target can satisfy that ARN
|
||||
/// (`reconcile_site_replication_bucket_targets` skips the local peer), so the receiver
|
||||
/// silently stops replicating back: the one-directional symptom. Only operator-authored
|
||||
/// rules travel between sites; each site owns its own `site-repl-*` rules.
|
||||
fn merge_incoming_replication_config(
|
||||
incoming: Option<ReplicationConfiguration>,
|
||||
local: Option<ReplicationConfiguration>,
|
||||
) -> Option<ReplicationConfiguration> {
|
||||
let incoming_role = incoming.as_ref().map(|config| config.role.clone()).unwrap_or_default();
|
||||
// Operator rules first, then the local site rules — the same order
|
||||
// `ensure_site_replication_bucket_replication_config_with_runtime` produces, so its
|
||||
// no-op check matches and the bucket metadata is written once per broadcast, not twice.
|
||||
let mut rules: Vec<ReplicationRule> = incoming
|
||||
.into_iter()
|
||||
.flat_map(|config| config.rules)
|
||||
.filter(|rule| !is_site_replication_rule(rule))
|
||||
.collect();
|
||||
rules.extend(
|
||||
local
|
||||
.into_iter()
|
||||
.flat_map(|config| config.rules)
|
||||
.filter(is_site_replication_rule),
|
||||
);
|
||||
|
||||
if rules.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
for (index, rule) in rules.iter_mut().enumerate() {
|
||||
rule.priority = Some(i32::try_from(index + 1).unwrap_or(i32::MAX));
|
||||
}
|
||||
|
||||
// A site-replication ARN in `role` is the sender's, and `site_replication_target_arns_by_peer`
|
||||
// reads it — carrying it over would pin the receiver's targets to the sender's identity.
|
||||
let role = match replication_target_arn_deployment_id(&incoming_role) {
|
||||
Some(_) => String::new(),
|
||||
None => incoming_role,
|
||||
};
|
||||
|
||||
Some(ReplicationConfiguration { role, rules })
|
||||
}
|
||||
|
||||
/// Merge a peer's ILM expiry document into the local lifecycle config.
|
||||
///
|
||||
/// Mirrors MinIO's `mergeWithCurrentLCConfig` with one hardening: incoming
|
||||
|
||||
@@ -443,6 +443,7 @@ pub(crate) mod replication {
|
||||
pub(crate) use super::ecstore_bucket::replication::{
|
||||
REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS,
|
||||
REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS,
|
||||
is_site_replication_rule, merge_incoming_replication_config, replication_target_arn_deployment_id,
|
||||
};
|
||||
pub(crate) type BucketReplicationResyncStatus = super::ecstore_bucket::replication::BucketReplicationResyncStatus;
|
||||
pub(crate) type BucketStats = super::ecstore_bucket::replication::BucketStats;
|
||||
|
||||
@@ -38,9 +38,9 @@ use super::storage_api::bucket_usecase::bucket::{
|
||||
metadata_sys,
|
||||
policy_sys::PolicySys,
|
||||
replication::{
|
||||
ReplicationTargetValidationError, invalid_replication_config_status_field, replication_target_arns,
|
||||
should_remove_replication_target, unsupported_replication_config_field, validate_replication_config_structure,
|
||||
validate_replication_config_target_arns,
|
||||
ReplicationTargetValidationError, invalid_replication_config_status_field, is_site_replication_rule,
|
||||
merge_incoming_replication_config, replication_target_arns, should_remove_replication_target,
|
||||
unsupported_replication_config_field, validate_replication_config_structure, validate_replication_config_target_arns,
|
||||
},
|
||||
target::{BucketTargetType, BucketTargets},
|
||||
utils::serialize,
|
||||
@@ -623,11 +623,50 @@ async fn validate_bucket_replication_update(bucket: &str, config: &ReplicationCo
|
||||
validate_replication_config_targets(&targets, config)
|
||||
}
|
||||
|
||||
async fn replication_targets_without_config_targets(
|
||||
/// Defense in depth for site-replication-managed buckets (issue #1948): an S3
|
||||
/// PutBucketReplication replaces the operator-authored rules but must not wipe
|
||||
/// the local `site-repl-*` rules the reconciler owns — until its next pass
|
||||
/// (600s period) every peer link on this bucket would be silently dead. The
|
||||
/// same merge also drops incoming `site-repl-*` impostor rules, matching the
|
||||
/// peer bucket-meta ingestion path. Buckets without site-replication rules
|
||||
/// keep the verbatim overwrite semantics.
|
||||
fn merge_user_replication_config_update(
|
||||
incoming: ReplicationConfiguration,
|
||||
existing: Option<ReplicationConfiguration>,
|
||||
) -> ReplicationConfiguration {
|
||||
let has_site_rules = existing
|
||||
.as_ref()
|
||||
.is_some_and(|config| config.rules.iter().any(is_site_replication_rule));
|
||||
if !has_site_rules {
|
||||
return incoming;
|
||||
}
|
||||
// `existing` holds at least one site-replication rule the merge keeps, so
|
||||
// the merged rule set is non-empty; the fallback only guards the type.
|
||||
merge_incoming_replication_config(Some(incoming.clone()), existing).unwrap_or(incoming)
|
||||
}
|
||||
|
||||
/// Split of an S3 DeleteBucketReplication on the stored config (issue #1948):
|
||||
/// the operator-authored rules are removed, the local `site-repl-*` rules
|
||||
/// survive (`None` means nothing survives and the config is deleted), and the
|
||||
/// returned ARNs are the ones whose bucket targets may be garbage-collected —
|
||||
/// never an ARN a surviving site-replication rule still points at.
|
||||
fn split_replication_config_for_user_delete(
|
||||
config: ReplicationConfiguration,
|
||||
) -> (Option<ReplicationConfiguration>, HashSet<String>) {
|
||||
let mut removable_arns = replication_target_arns(&config);
|
||||
let remaining = merge_incoming_replication_config(None, Some(config));
|
||||
if let Some(remaining) = remaining.as_ref() {
|
||||
for rule in &remaining.rules {
|
||||
removable_arns.remove(rule.destination.bucket.trim());
|
||||
}
|
||||
}
|
||||
(remaining, removable_arns)
|
||||
}
|
||||
|
||||
async fn replication_targets_without_arns(
|
||||
bucket: &str,
|
||||
config: &ReplicationConfiguration,
|
||||
target_arns: &HashSet<String>,
|
||||
) -> S3Result<Option<(BucketTargets, usize)>> {
|
||||
let target_arns = replication_target_arns(config);
|
||||
if target_arns.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
@@ -638,7 +677,7 @@ async fn replication_targets_without_config_targets(
|
||||
Err(err) => return Err(ApiError::from(err).into()),
|
||||
};
|
||||
|
||||
let removed = remove_replication_targets_from_config_targets(&mut targets, &target_arns);
|
||||
let removed = remove_replication_targets_from_config_targets(&mut targets, target_arns);
|
||||
if removed == 0 {
|
||||
return Ok(None);
|
||||
}
|
||||
@@ -1604,15 +1643,29 @@ impl DefaultBucketUsecase {
|
||||
Err(StorageError::ConfigNotFound) => None,
|
||||
Err(err) => return Err(ApiError::from(err).into()),
|
||||
};
|
||||
let updated_targets = if let Some(config) = replication_config.as_ref() {
|
||||
replication_targets_without_config_targets(&bucket, config).await?
|
||||
let (remaining_config, updated_targets) = if let Some(config) = replication_config.as_ref() {
|
||||
let (remaining, removable_arns) = split_replication_config_for_user_delete(config.clone());
|
||||
let targets = replication_targets_without_arns(&bucket, &removable_arns).await?;
|
||||
(remaining, targets)
|
||||
} else {
|
||||
None
|
||||
(None, None)
|
||||
};
|
||||
|
||||
delete_bucket_config_for_incarnation(&bucket, BUCKET_REPLICATION_CONFIG, expected_incarnation_id)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
match remaining_config {
|
||||
// Site-replication rules and the targets backing them survive the
|
||||
// S3 delete (issue #1948); only the operator-authored rules go.
|
||||
Some(remaining) => {
|
||||
let data = serialize_config(&remaining)?;
|
||||
update_bucket_config_for_incarnation(&bucket, BUCKET_REPLICATION_CONFIG, data, expected_incarnation_id)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
}
|
||||
None => {
|
||||
delete_bucket_config_for_incarnation(&bucket, BUCKET_REPLICATION_CONFIG, expected_incarnation_id)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
}
|
||||
}
|
||||
if let Some((targets, removed)) = updated_targets
|
||||
&& let Err(err) =
|
||||
write_replication_targets_after_config_delete(&bucket, &targets, removed, expected_incarnation_id).await
|
||||
@@ -2485,6 +2538,12 @@ impl DefaultBucketUsecase {
|
||||
|
||||
let targets_guard = lock_bucket_targets_metadata(&bucket).await;
|
||||
validate_bucket_replication_update(&bucket, &replication_configuration).await?;
|
||||
let existing_config = match metadata_sys::get_replication_config(&bucket).await {
|
||||
Ok((config, _)) => Some(config),
|
||||
Err(StorageError::ConfigNotFound) => None,
|
||||
Err(err) => return Err(ApiError::from(err).into()),
|
||||
};
|
||||
let replication_configuration = merge_user_replication_config_update(replication_configuration, existing_config);
|
||||
let data = serialize_config(&replication_configuration)?;
|
||||
update_bucket_config_for_incarnation(&bucket, BUCKET_REPLICATION_CONFIG, data, expected_incarnation_id)
|
||||
.await
|
||||
@@ -3114,6 +3173,127 @@ mod tests {
|
||||
assert!(arns.contains(destination));
|
||||
}
|
||||
|
||||
fn replication_rule_with_id(arn: &str, id: &str, priority: i32) -> ReplicationRule {
|
||||
let mut rule = replication_rule_for_target(arn);
|
||||
rule.id = Some(id.to_string());
|
||||
rule.priority = Some(priority);
|
||||
rule
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn put_replication_merge_preserves_site_replication_rules() {
|
||||
let existing = ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![
|
||||
replication_rule_with_id("arn:rustfs:replication::peer-dep:bucket", "site-repl-peer-dep", 1),
|
||||
replication_rule_with_id("arn:rustfs:replication:us-east-1:old:bucket", "old-user-rule", 2),
|
||||
],
|
||||
};
|
||||
let incoming = ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![
|
||||
replication_rule_with_id("arn:rustfs:replication:us-east-1:new:bucket", "new-user-rule", 1),
|
||||
replication_rule_with_id("arn:rustfs:replication::forged-dep:bucket", "site-repl-forged", 2),
|
||||
],
|
||||
};
|
||||
|
||||
let merged = merge_user_replication_config_update(incoming, Some(existing));
|
||||
|
||||
let ids: Vec<_> = merged
|
||||
.rules
|
||||
.iter()
|
||||
.map(|rule| rule.id.as_deref().unwrap_or_default())
|
||||
.collect();
|
||||
assert_eq!(
|
||||
ids,
|
||||
vec!["new-user-rule", "site-repl-peer-dep"],
|
||||
"user rules replaced, local site-replication rule preserved, forged incoming site-repl rule dropped"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn put_replication_merge_returns_incoming_verbatim_without_site_rules() {
|
||||
let existing = ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![replication_rule_with_id(
|
||||
"arn:rustfs:replication:us-east-1:old:bucket",
|
||||
"old-user-rule",
|
||||
7,
|
||||
)],
|
||||
};
|
||||
let incoming = ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![replication_rule_with_id(
|
||||
"arn:rustfs:replication:us-east-1:new:bucket",
|
||||
"new-user-rule",
|
||||
5,
|
||||
)],
|
||||
};
|
||||
|
||||
let merged = merge_user_replication_config_update(incoming.clone(), Some(existing));
|
||||
|
||||
assert_eq!(merged.role, incoming.role);
|
||||
assert_eq!(merged.rules, incoming.rules, "non-SR buckets keep the verbatim overwrite semantics");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_replication_split_keeps_site_rules_and_their_targets() {
|
||||
let sr_arn = "arn:rustfs:replication::peer-dep:bucket";
|
||||
let user_arn = "arn:rustfs:replication:us-east-1:user:bucket";
|
||||
let config = ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![
|
||||
replication_rule_with_id(user_arn, "user-rule", 1),
|
||||
replication_rule_with_id(sr_arn, "site-repl-peer-dep", 2),
|
||||
],
|
||||
};
|
||||
|
||||
let (remaining, removable) = split_replication_config_for_user_delete(config);
|
||||
|
||||
let remaining = remaining.expect("site-replication rules must survive a user delete");
|
||||
let ids: Vec<_> = remaining
|
||||
.rules
|
||||
.iter()
|
||||
.map(|rule| rule.id.as_deref().unwrap_or_default())
|
||||
.collect();
|
||||
assert_eq!(ids, vec!["site-repl-peer-dep"]);
|
||||
assert_eq!(removable, HashSet::from([user_arn.to_string()]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_replication_split_protects_targets_shared_with_site_rules() {
|
||||
let sr_arn = "arn:rustfs:replication::peer-dep:bucket";
|
||||
let config = ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![
|
||||
replication_rule_with_id(sr_arn, "user-rule-on-sr-target", 1),
|
||||
replication_rule_with_id(sr_arn, "site-repl-peer-dep", 2),
|
||||
],
|
||||
};
|
||||
|
||||
let (remaining, removable) = split_replication_config_for_user_delete(config);
|
||||
|
||||
assert!(remaining.is_some());
|
||||
assert!(
|
||||
removable.is_empty(),
|
||||
"a target still referenced by a surviving site-replication rule must not be removed"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_replication_split_removes_everything_without_site_rules() {
|
||||
let user_arn = "arn:rustfs:replication:us-east-1:user:bucket";
|
||||
let config = ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![replication_rule_with_id(user_arn, "user-rule", 1)],
|
||||
};
|
||||
|
||||
let (remaining, removable) = split_replication_config_for_user_delete(config);
|
||||
|
||||
assert!(remaining.is_none(), "without site-replication rules the whole config is deleted");
|
||||
assert_eq!(removable, HashSet::from([user_arn.to_string()]));
|
||||
}
|
||||
|
||||
fn replication_targets_with_arn(arns: &[&str]) -> BucketTargets {
|
||||
BucketTargets {
|
||||
targets: arns
|
||||
|
||||
@@ -3969,6 +3969,14 @@ fn delete_creates_delete_marker(opts: &ObjectOptions) -> bool {
|
||||
opts.version_id.is_none() && opts.versioned && !opts.version_suspended
|
||||
}
|
||||
|
||||
/// `DeleteObjects` is idempotent. A raw filesystem `NotFound` can cross the
|
||||
/// distributed delete path instead of its usual typed missing-object error.
|
||||
fn is_delete_objects_not_found(error: &EcstoreError) -> bool {
|
||||
is_err_object_not_found(error)
|
||||
|| is_err_version_not_found(error)
|
||||
|| matches!(error, StorageError::Io(source) if source.kind() == std::io::ErrorKind::NotFound)
|
||||
}
|
||||
|
||||
/// Bounded concurrency for the per-object pre-delete stat fanout in
|
||||
/// `execute_delete_objects` (backlog#929 / HP-8). Keeps the metadata reads for
|
||||
/// a 1000-key batch from serializing while capping the disk fanout pressure.
|
||||
@@ -4030,6 +4038,27 @@ fn delete_response_version_id(version_id: Option<Uuid>, synthetic_version_id: bo
|
||||
}
|
||||
}
|
||||
|
||||
fn reduce_delete_objects_result<'a>(
|
||||
object: &ObjectToDelete,
|
||||
deleted: &'a StorageDeletedObject,
|
||||
error: Option<&EcstoreError>,
|
||||
synthetic_version_id: bool,
|
||||
) -> Result<&'a StorageDeletedObject, s3s::dto::Error> {
|
||||
match error {
|
||||
None => Ok(deleted),
|
||||
Some(error) if is_delete_objects_not_found(error) => Ok(deleted),
|
||||
Some(error) => {
|
||||
let api_error = ApiError::from(error.clone());
|
||||
Err(s3s::dto::Error {
|
||||
code: Some(api_error.code.as_str().to_string()),
|
||||
key: Some(object.object_name.clone()),
|
||||
message: Some(api_error.message),
|
||||
version_id: delete_response_version_id(object.version_id, synthetic_version_id),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn resolve_put_object_extract_options(headers: &HeaderMap) -> S3Result<PutObjectExtractOptions> {
|
||||
let prefix = snowball_meta_value(headers, SNOWBALL_PREFIX_HEADER_KEYS, SNOWBALL_PREFIX_SUFFIX_LOWER)
|
||||
.map(|value| normalize_snowball_prefix(&value))
|
||||
@@ -8476,39 +8505,31 @@ impl DefaultObjectUsecase {
|
||||
for (i, err) in errs.iter().enumerate() {
|
||||
let didx = object_to_delete_idx[i];
|
||||
|
||||
if err.is_none()
|
||||
|| err
|
||||
.clone()
|
||||
.is_some_and(|v| is_err_object_not_found(&v) || is_err_version_not_found(&v))
|
||||
{
|
||||
delete_results[didx].delete_object = Some(dobjs[i].clone());
|
||||
let (versioned, version_suspended) = object_versioning[i];
|
||||
let creates_delete_marker = object_to_delete[i].version_id.is_none() && versioned && !version_suspended;
|
||||
if creates_delete_marker {
|
||||
record_bucket_delete_marker_memory(&bucket).await;
|
||||
} else {
|
||||
let size = object_sizes[i].max(0) as u64;
|
||||
record_bucket_object_delete_memory(
|
||||
&bucket,
|
||||
size,
|
||||
existing_object_infos[i].is_some() && object_to_delete[i].version_id.is_none(),
|
||||
)
|
||||
.await;
|
||||
match reduce_delete_objects_result(
|
||||
&object_to_delete[i],
|
||||
&dobjs[i],
|
||||
err.as_ref(),
|
||||
delete_results[didx].synthetic_version_id,
|
||||
) {
|
||||
Ok(deleted_object) => {
|
||||
delete_results[didx].delete_object = Some(deleted_object.clone());
|
||||
let (versioned, version_suspended) = object_versioning[i];
|
||||
let creates_delete_marker = object_to_delete[i].version_id.is_none() && versioned && !version_suspended;
|
||||
if creates_delete_marker {
|
||||
record_bucket_delete_marker_memory(&bucket).await;
|
||||
} else {
|
||||
let size = object_sizes[i].max(0) as u64;
|
||||
record_bucket_object_delete_memory(
|
||||
&bucket,
|
||||
size,
|
||||
existing_object_infos[i].is_some() && object_to_delete[i].version_id.is_none(),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
Err(error) => {
|
||||
delete_results[didx].error = Some(error);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if let Some(err) = err.clone() {
|
||||
let api_error = ApiError::from(err);
|
||||
delete_results[didx].error = Some(s3s::dto::Error {
|
||||
code: Some(api_error.code.as_str().to_string()),
|
||||
key: Some(object_to_delete[i].object_name.clone()),
|
||||
message: Some(api_error.message),
|
||||
version_id: delete_response_version_id(
|
||||
object_to_delete[i].version_id,
|
||||
delete_results[didx].synthetic_version_id,
|
||||
),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17692,6 +17713,35 @@ mod tests {
|
||||
assert_eq!(internal_version_id, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_objects_treats_raw_io_not_found_as_idempotent() {
|
||||
assert!(is_delete_objects_not_found(&StorageError::FileNotFound));
|
||||
assert!(is_delete_objects_not_found(&StorageError::Io(std::io::Error::from(
|
||||
std::io::ErrorKind::NotFound,
|
||||
))));
|
||||
assert!(!is_delete_objects_not_found(&StorageError::Io(std::io::Error::from(
|
||||
std::io::ErrorKind::PermissionDenied,
|
||||
))));
|
||||
assert!(!is_delete_objects_not_found(&StorageError::DiskNotFound));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_objects_result_reducer_reports_raw_not_found_as_deleted() {
|
||||
let object = ObjectToDelete {
|
||||
object_name: "missing-key".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
let deleted = StorageDeletedObject {
|
||||
object_name: object.object_name.clone(),
|
||||
..Default::default()
|
||||
};
|
||||
let error = StorageError::Io(std::io::Error::from(std::io::ErrorKind::NotFound));
|
||||
|
||||
let deleted = reduce_delete_objects_result(&object, &deleted, Some(&error), false)
|
||||
.expect("raw not-found must produce a deleted result");
|
||||
assert_eq!(deleted.object_name, "missing-key");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recursive_force_delete_requires_administrative_or_replica_context() {
|
||||
let mut headers = HeaderMap::new();
|
||||
|
||||
@@ -614,6 +614,8 @@ pub(crate) mod bucket {
|
||||
|
||||
use crate::storage::storage_api::ecstore_bucket::replication as replication_contracts;
|
||||
|
||||
pub(crate) use replication_contracts::{is_site_replication_rule, merge_incoming_replication_config};
|
||||
|
||||
type ReplicationObjectBridge = crate::storage::storage_api::ecstore_bucket::replication::ReplicationObjectBridge;
|
||||
pub(crate) type DeleteReplicationConfigSnapshot =
|
||||
crate::storage::storage_api::ecstore_bucket::replication::DeleteReplicationConfigSnapshot;
|
||||
|
||||
@@ -1537,7 +1537,7 @@ fn process_connection(
|
||||
None
|
||||
}
|
||||
};
|
||||
// ── Canonical Middleware Stack Order (outermost → innermost) ──
|
||||
// ── Canonical External Middleware Stack Order (outermost → innermost) ──
|
||||
// This order MUST be preserved across refactorings.
|
||||
// Only AddExtensionLayer (layers 1-2) are per-connection; most remaining layers are stateless.
|
||||
//
|
||||
@@ -1565,6 +1565,8 @@ fn process_connection(
|
||||
// 22. PublicHealthEndpointLayer — handles public health before s3s host parsing
|
||||
// 23. VirtualHostStyleHintLayer — actionable error for unroutable virtual-hosted-style (conditional)
|
||||
// 24. DoubleSlashListBucketsCompatLayer — rewrites `GET //` to `GET /` for ListBuckets (MinIO browser compat)
|
||||
// The internode lane below intentionally keeps only the shared
|
||||
// transport/auth/observability subset needed by `/rustfs/rpc/...`.
|
||||
// ─────────────────────────────────────────────────────────────
|
||||
let build_external_stack = |service| {
|
||||
ServiceBuilder::new()
|
||||
@@ -1747,16 +1749,9 @@ fn process_connection(
|
||||
.layer(PropagateRequestIdLayer::x_request_id())
|
||||
.layer(CompressionLayer::new().compress_when(PathAwareHttpCompressionPredicate::new(compression_config.clone())))
|
||||
.option_layer(compression_config.enabled.then_some(PathCategoryInjectionLayer))
|
||||
.layer(S3ErrorMessageCompatLayer)
|
||||
.layer(IcebergRestErrorCompatLayer)
|
||||
.layer(ObjectAttributesEtagFixLayer)
|
||||
.layer(ConditionalCorsLayer::new())
|
||||
.option_layer(if is_console { Some(RedirectLayer) } else { None })
|
||||
.layer(BodylessStatusFixLayer)
|
||||
.layer(HeadRequestBodyFixLayer)
|
||||
.layer(PublicHealthEndpointLayer::new(Arc::clone(&server_ctx)))
|
||||
.option_layer((!server_domains_configured && !is_console).then_some(VirtualHostStyleHintLayer))
|
||||
.layer(DoubleSlashListBucketsCompatLayer)
|
||||
// The internode lane only serves `/rustfs/rpc/...` gRPC requests.
|
||||
// Keep safety/observability layers above, but leave S3/REST
|
||||
// compatibility rewrites on the external lane.
|
||||
.service(service)
|
||||
};
|
||||
let external_stack_service = build_external_stack(external_service);
|
||||
|
||||
@@ -63,6 +63,54 @@ use crate::app::storage_api::object_usecase::bucket::replication::{
|
||||
};
|
||||
use crate::storage::storage_api::ecfs_consumer::StorageObjectOptions as ObjectOptions;
|
||||
|
||||
#[cfg(test)]
|
||||
static SITE_REPLICATION_GATE_TEST_OVERRIDE: std::sync::atomic::AtomicU8 = std::sync::atomic::AtomicU8::new(0);
|
||||
#[cfg(test)]
|
||||
const SITE_REPLICATION_GATE_FORCE_DISABLED: u8 = 1;
|
||||
#[cfg(test)]
|
||||
const SITE_REPLICATION_GATE_FORCE_ENABLED: u8 = 2;
|
||||
|
||||
async fn site_replication_gate_enabled() -> S3Result<bool> {
|
||||
#[cfg(test)]
|
||||
match SITE_REPLICATION_GATE_TEST_OVERRIDE.load(std::sync::atomic::Ordering::SeqCst) {
|
||||
SITE_REPLICATION_GATE_FORCE_DISABLED => return Ok(false),
|
||||
SITE_REPLICATION_GATE_FORCE_ENABLED => return Ok(true),
|
||||
_ => {}
|
||||
}
|
||||
crate::admin::handlers::site_replication::site_replication_enabled().await
|
||||
}
|
||||
|
||||
/// MinIO `ErrReplicationDenyEditError`.
|
||||
fn replication_deny_edit_error() -> S3Error {
|
||||
let mut err = S3Error::with_message(
|
||||
S3ErrorCode::Custom("XMinioReplicationDenyEdit".into()),
|
||||
"Sub-User is not allowed to edit Replication configuration",
|
||||
);
|
||||
err.set_status_code(StatusCode::BAD_REQUEST);
|
||||
err
|
||||
}
|
||||
|
||||
/// Site-replication gate for S3 replication-config edits (issue #1948).
|
||||
///
|
||||
/// On a site-replication deployment the bucket's replication config carries
|
||||
/// the operator-managed `site-repl-*` rules that keep every peer in sync, and
|
||||
/// a successful edit is broadcast to all peers — so a user holding only
|
||||
/// bucket-scoped `s3:PutReplicationConfiguration` could rewrite or erase
|
||||
/// replication net-wide. MinIO parity (`ErrReplicationDenyEditError`): only
|
||||
/// owner credentials (root or root-parented) may edit. Runs after the policy
|
||||
/// authorization in the access layer and only on the external S3 path — the
|
||||
/// reconciler and peer bucket-meta ingestion never route through these
|
||||
/// handlers.
|
||||
async fn deny_replication_config_edit_for_non_owner<T>(req: &S3Request<T>) -> S3Result<()> {
|
||||
if crate::storage::access::req_info_ref(req)?.is_owner {
|
||||
return Ok(());
|
||||
}
|
||||
if site_replication_gate_enabled().await? {
|
||||
return Err(replication_deny_edit_error());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct FS {
|
||||
/// This server's late-bound application-context slot (backlog#1052 S2).
|
||||
@@ -500,6 +548,7 @@ impl S3 for FS {
|
||||
&self,
|
||||
req: S3Request<DeleteBucketReplicationInput>,
|
||||
) -> S3Result<S3Response<DeleteBucketReplicationOutput>> {
|
||||
deny_replication_config_edit_for_non_owner(&req).await?;
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
usecase.execute_delete_bucket_replication(req).await
|
||||
}
|
||||
@@ -1353,6 +1402,7 @@ impl S3 for FS {
|
||||
&self,
|
||||
req: S3Request<PutBucketReplicationInput>,
|
||||
) -> S3Result<S3Response<PutBucketReplicationOutput>> {
|
||||
deny_replication_config_edit_for_non_owner(&req).await?;
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
usecase.execute_put_bucket_replication(req).await
|
||||
}
|
||||
@@ -1919,3 +1969,103 @@ impl S3 for FS {
|
||||
Box::pin(usecase.execute_upload_part_copy(req)).await
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
FS, SITE_REPLICATION_GATE_FORCE_DISABLED, SITE_REPLICATION_GATE_FORCE_ENABLED, SITE_REPLICATION_GATE_TEST_OVERRIDE,
|
||||
};
|
||||
use crate::storage::access::ReqInfo;
|
||||
use http::Method;
|
||||
use http::StatusCode;
|
||||
use s3s::dto::{DeleteBucketReplicationInput, PutBucketReplicationInput, ReplicationConfiguration};
|
||||
use s3s::{S3, S3Error, S3ErrorCode, S3Request};
|
||||
use std::sync::atomic::Ordering;
|
||||
|
||||
fn replication_config_edit_request<T>(input: T, is_owner: bool) -> S3Request<T> {
|
||||
let mut req = S3Request {
|
||||
input,
|
||||
method: Method::PUT,
|
||||
uri: http::Uri::from_static("/"),
|
||||
headers: http::HeaderMap::new(),
|
||||
extensions: http::Extensions::new(),
|
||||
credentials: None,
|
||||
region: None,
|
||||
service: None,
|
||||
trailing_headers: None,
|
||||
};
|
||||
req.extensions.insert(ReqInfo {
|
||||
is_owner,
|
||||
..Default::default()
|
||||
});
|
||||
req
|
||||
}
|
||||
|
||||
fn put_bucket_replication_input() -> PutBucketReplicationInput {
|
||||
PutBucketReplicationInput {
|
||||
bucket: "test-bucket".to_string(),
|
||||
checksum_algorithm: None,
|
||||
content_md5: None,
|
||||
expected_bucket_owner: None,
|
||||
replication_configuration: ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: Vec::new(),
|
||||
},
|
||||
token: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn delete_bucket_replication_input() -> DeleteBucketReplicationInput {
|
||||
DeleteBucketReplicationInput {
|
||||
bucket: "test-bucket".to_string(),
|
||||
expected_bucket_owner: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn assert_replication_deny_edit(err: &S3Error) {
|
||||
match err.code() {
|
||||
S3ErrorCode::Custom(code) => assert_eq!(code, "XMinioReplicationDenyEdit"),
|
||||
other => panic!("expected XMinioReplicationDenyEdit, got {other:?}"),
|
||||
}
|
||||
assert_eq!(err.status_code(), Some(StatusCode::BAD_REQUEST));
|
||||
}
|
||||
|
||||
/// Single test on purpose: the branches share the process-wide gate
|
||||
/// override, and parallel tests would race it.
|
||||
#[tokio::test]
|
||||
async fn replication_config_edit_gate_denies_only_non_owner_under_site_replication() {
|
||||
let fs = FS::new();
|
||||
SITE_REPLICATION_GATE_TEST_OVERRIDE.store(SITE_REPLICATION_GATE_FORCE_ENABLED, Ordering::SeqCst);
|
||||
|
||||
// Non-owner PUT/DELETE through the real S3 handlers: denied by the
|
||||
// gate before the usecase (and thus the store) is ever touched.
|
||||
let err = fs
|
||||
.put_bucket_replication(replication_config_edit_request(put_bucket_replication_input(), false))
|
||||
.await
|
||||
.expect_err("non-owner PutBucketReplication must be denied while site replication is enabled");
|
||||
assert_replication_deny_edit(&err);
|
||||
let err = fs
|
||||
.delete_bucket_replication(replication_config_edit_request(delete_bucket_replication_input(), false))
|
||||
.await
|
||||
.expect_err("non-owner DeleteBucketReplication must be denied while site replication is enabled");
|
||||
assert_replication_deny_edit(&err);
|
||||
|
||||
// Owner passes the gate (the usecase's empty-rules structure error
|
||||
// proves the request reached the usecase instead of the deny path).
|
||||
let err = fs
|
||||
.put_bucket_replication(replication_config_edit_request(put_bucket_replication_input(), true))
|
||||
.await
|
||||
.expect_err("owner request should pass the gate and fail later on config validation");
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
|
||||
|
||||
// Without site replication the policy check alone still governs the edit.
|
||||
SITE_REPLICATION_GATE_TEST_OVERRIDE.store(SITE_REPLICATION_GATE_FORCE_DISABLED, Ordering::SeqCst);
|
||||
let err = fs
|
||||
.put_bucket_replication(replication_config_edit_request(put_bucket_replication_input(), false))
|
||||
.await
|
||||
.expect_err("non-owner request should pass the gate and fail later on config validation");
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
|
||||
|
||||
SITE_REPLICATION_GATE_TEST_OVERRIDE.store(0, Ordering::SeqCst);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user