mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-23 04:39:04 +00:00
Compare commits
8 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 4b7a1ac050 | |||
| b6ba89d9e4 | |||
| 648d5166e2 | |||
| 84eb5aebef | |||
| 74171bd673 | |||
| a42d81b26a | |||
| a3733c1a1c | |||
| ce9b69d811 |
@@ -30,7 +30,8 @@ make build-docker BUILD_OS=ubuntu22.04
|
||||
- Crate membership: `Cargo.toml` `[workspace].members`
|
||||
- Architecture, layering, crate map: [ARCHITECTURE.md](ARCHITECTURE.md)
|
||||
- Migration guardrails & readiness contracts: [docs/architecture/](docs/architecture/README.md)
|
||||
- CI gates: `.github/workflows/ci.yml` (source of truth; never copy its steps into docs)
|
||||
- CI workflow steps: `.github/workflows/`; event, timeout, and required-status
|
||||
matrix: [docs/testing/ci-gates.md](docs/testing/ci-gates.md)
|
||||
- Test-layer taxonomy, per-layer entry commands, serial/nextest rules, flake
|
||||
policy: [docs/testing/README.md](docs/testing/README.md)
|
||||
- Tier/ILM transition debugging (xl.meta inspection, versionId tracing):
|
||||
|
||||
@@ -70,6 +70,8 @@ make pre-pr
|
||||
|
||||
> For the full test-layer taxonomy (unit / ecstore black-box / e2e / s3s-e2e / S3 compatibility / chaos / fuzz / bench), each layer's entry command, the naming conventions the migration gate depends on, and the serial/nextest rules, see [docs/testing/README.md](docs/testing/README.md).
|
||||
|
||||
> For the event, timeout, required-status, and local reproduction matrix, see [docs/testing/ci-gates.md](docs/testing/ci-gates.md).
|
||||
|
||||
### 🔒 Automated Pre-commit Hooks
|
||||
#### What `make pre-commit` and `make pre-pr` actually run
|
||||
|
||||
|
||||
Generated
+22
-27
@@ -1858,9 +1858,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.4.3"
|
||||
version = "1.4.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "509591b7bcd67f4ef775afad7662703b4935daaa6ec0e5605cfb1090b32a2b6d"
|
||||
checksum = "0ad534f4357a5264cce5019c989cf66a4f0dc4e0d1b1d15f8aacec0ff7360273"
|
||||
dependencies = [
|
||||
"find-msvc-tools",
|
||||
"jobserver",
|
||||
@@ -2522,12 +2522,6 @@ dependencies = [
|
||||
"subtle",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cty"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b365fabc795046672053e29c954733ec3b05e4be654ab130fe8f1f94d7051f35"
|
||||
|
||||
[[package]]
|
||||
name = "curve25519-dalek"
|
||||
version = "4.1.3"
|
||||
@@ -5988,15 +5982,6 @@ version = "0.2.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981"
|
||||
|
||||
[[package]]
|
||||
name = "libmimalloc-sys"
|
||||
version = "0.1.49"
|
||||
source = "git+https://github.com/xonatius/mimalloc_rust.git?rev=6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11#6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"cty",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "libredox"
|
||||
version = "0.1.20"
|
||||
@@ -6397,14 +6382,6 @@ dependencies = [
|
||||
"synstructure 0.13.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "mimalloc"
|
||||
version = "0.1.52"
|
||||
source = "git+https://github.com/xonatius/mimalloc_rust.git?rev=6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11#6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11"
|
||||
dependencies = [
|
||||
"libmimalloc-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "mime"
|
||||
version = "0.3.17"
|
||||
@@ -9162,13 +9139,11 @@ dependencies = [
|
||||
"insta",
|
||||
"jiff",
|
||||
"libc",
|
||||
"libmimalloc-sys",
|
||||
"libsystemd",
|
||||
"matchit 0.9.2",
|
||||
"md-5 0.11.0",
|
||||
"metrics",
|
||||
"metrics-util",
|
||||
"mimalloc",
|
||||
"mime_guess",
|
||||
"opentelemetry",
|
||||
"opentelemetry_sdk",
|
||||
@@ -9204,6 +9179,8 @@ dependencies = [
|
||||
"rustfs-lock",
|
||||
"rustfs-log-analyzer",
|
||||
"rustfs-madmin",
|
||||
"rustfs-mimalloc",
|
||||
"rustfs-mimalloc-sys",
|
||||
"rustfs-notify",
|
||||
"rustfs-object-capacity",
|
||||
"rustfs-object-data-cache",
|
||||
@@ -9875,6 +9852,24 @@ dependencies = [
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-mimalloc"
|
||||
version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a406f4aa07084301d485beec873af6dccc8e3f8762da244743df92038b1db1a6"
|
||||
dependencies = [
|
||||
"rustfs-mimalloc-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-mimalloc-sys"
|
||||
version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c3051b819175f58445d4c369a72f0ab88149f3885ba8bea2aff3be01f53fe7cd"
|
||||
dependencies = [
|
||||
"cc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-notify"
|
||||
version = "1.0.0-rc.3"
|
||||
|
||||
+2
-2
@@ -350,8 +350,8 @@ russh-sftp = "2.4.0"
|
||||
dav-server = "0.11.0"
|
||||
|
||||
# Performance Analysis and Memory Profiling
|
||||
mimalloc = { version = "0.1.52", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11" }
|
||||
libmimalloc-sys = { version = "0.1.49", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11", features = ["extended"] }
|
||||
rustfs-mimalloc = { version = "0.5.0" }
|
||||
rustfs-mimalloc-sys = { version = "0.5.0" }
|
||||
hotpath = { version = "0.23.3", default-features = false }
|
||||
# Snapshot testing for output format regression detection
|
||||
insta = { version = "1.48" }
|
||||
|
||||
@@ -236,15 +236,6 @@ pub struct DataUsageInfo {
|
||||
/// without relying on synchronized clocks.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub usage_snapshot_authoritative_baseline: Option<DataUsageSnapshotIdentity>,
|
||||
/// Per-set freshness for an observational aggregate. A set entry is
|
||||
/// never sufficient to make the aggregate authoritative; it only records
|
||||
/// which last-known-good generation contributed to the view.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub usage_snapshot_set_states: Vec<DataUsageSnapshotSetState>,
|
||||
/// An observational view may contain only the sets that completed this
|
||||
/// cycle (or retained a compatible last-known-good cache).
|
||||
#[serde(default)]
|
||||
pub usage_snapshot_partial: bool,
|
||||
/// Deprecated kept here for backward compatibility reasons
|
||||
pub bucket_sizes: HashMap<String, u64>,
|
||||
/// Per-disk snapshot information when available
|
||||
@@ -261,22 +252,6 @@ pub struct DataUsageSnapshotIdentity {
|
||||
pub scanner_epoch: Option<u64>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct DataUsageSnapshotSetState {
|
||||
pub pool_index: u64,
|
||||
pub set_index: u64,
|
||||
#[serde(default)]
|
||||
pub scanner_cycle: Option<u64>,
|
||||
#[serde(default)]
|
||||
pub scanner_epoch: Option<u64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub scan_plan_digest: Option<[u8; 32]>,
|
||||
#[serde(default)]
|
||||
pub complete: bool,
|
||||
#[serde(default)]
|
||||
pub tombstone: bool,
|
||||
}
|
||||
|
||||
impl DataUsageInfo {
|
||||
pub fn snapshot_identity(&self) -> DataUsageSnapshotIdentity {
|
||||
DataUsageSnapshotIdentity {
|
||||
@@ -316,7 +291,7 @@ pub fn data_usage_snapshot_is_newer(candidate: &DataUsageInfo, baseline: &DataUs
|
||||
/// rollback delete/recreate fences the previous bucket incarnation too.
|
||||
pub fn observed_data_usage_is_newer(observed: &DataUsageInfo, authoritative: &DataUsageInfo) -> bool {
|
||||
observed.usage_snapshot_converged == Some(false)
|
||||
&& (observed.is_complete_bucket_usage_snapshot() || observed.is_valid_partial_snapshot())
|
||||
&& observed.is_complete_bucket_usage_snapshot()
|
||||
&& observed.usage_snapshot_authoritative_baseline.as_ref() == Some(&authoritative.snapshot_identity())
|
||||
&& data_usage_snapshot_is_newer(observed, authoritative)
|
||||
}
|
||||
@@ -1461,39 +1436,6 @@ impl DataUsageInfo {
|
||||
&& u64::try_from(self.buckets_usage.len()).ok() == Some(self.buckets_count)
|
||||
}
|
||||
|
||||
/// Validate provenance before an observational view can be selected for
|
||||
/// admin display. Partial data is accepted only with unique set states,
|
||||
/// a plan digest for every state, and at least one usable generation.
|
||||
pub fn is_valid_partial_snapshot(&self) -> bool {
|
||||
if !self.usage_snapshot_partial
|
||||
|| self.usage_snapshot_converged != Some(false)
|
||||
|| self.last_update.is_none()
|
||||
|| self.scanner_cycle.is_none()
|
||||
|| self.scanner_epoch.is_none()
|
||||
|| self.usage_snapshot_set_states.is_empty()
|
||||
|| u64::try_from(self.buckets_usage.len()).ok() != Some(self.buckets_count)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
let mut previous = None;
|
||||
let mut plan_digest = None;
|
||||
let mut has_source = false;
|
||||
for state in &self.usage_snapshot_set_states {
|
||||
if state.scan_plan_digest.is_none()
|
||||
|| plan_digest.is_some_and(|digest| Some(digest) != state.scan_plan_digest)
|
||||
|| state.scanner_cycle.is_some() != state.scanner_epoch.is_some()
|
||||
|| previous.is_some_and(|(pool, set)| (pool, set) >= (state.pool_index, state.set_index))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
previous = Some((state.pool_index, state.set_index));
|
||||
plan_digest = state.scan_plan_digest;
|
||||
has_source |= state.scanner_cycle.is_some() && !state.tombstone;
|
||||
}
|
||||
has_source
|
||||
}
|
||||
|
||||
/// Add object metadata to data usage statistics
|
||||
pub fn add_object(&mut self, object_path: &str, meta_object: &rustfs_filemeta::MetaObject) {
|
||||
// This method is kept for backward compatibility
|
||||
@@ -2321,55 +2263,6 @@ mod tests {
|
||||
assert!(!observed_data_usage_is_newer(&candidate(2, 9, Some(false), true), &authoritative));
|
||||
assert!(!observed_data_usage_is_newer(&candidate(2, 11, Some(true), true), &authoritative));
|
||||
assert!(!observed_data_usage_is_newer(&candidate(2, 11, Some(false), false), &authoritative));
|
||||
|
||||
let mut partial = candidate(2, 11, Some(false), false);
|
||||
partial.usage_snapshot_partial = true;
|
||||
partial.usage_snapshot_set_states = vec![DataUsageSnapshotSetState {
|
||||
pool_index: 0,
|
||||
set_index: 0,
|
||||
scanner_cycle: Some(10),
|
||||
scanner_epoch: Some(2),
|
||||
scan_plan_digest: Some([1; 32]),
|
||||
complete: false,
|
||||
tombstone: false,
|
||||
}];
|
||||
assert!(observed_data_usage_is_newer(&partial, &authoritative));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mixed_topology_snapshot_is_rejected() {
|
||||
let mut partial = DataUsageInfo {
|
||||
last_update: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(2)),
|
||||
scanner_cycle: Some(11),
|
||||
scanner_epoch: Some(2),
|
||||
buckets_count: 0,
|
||||
usage_snapshot_converged: Some(false),
|
||||
usage_snapshot_partial: true,
|
||||
usage_snapshot_set_states: vec![
|
||||
DataUsageSnapshotSetState {
|
||||
pool_index: 0,
|
||||
set_index: 0,
|
||||
scanner_cycle: Some(11),
|
||||
scanner_epoch: Some(2),
|
||||
scan_plan_digest: Some([1; 32]),
|
||||
complete: true,
|
||||
tombstone: false,
|
||||
},
|
||||
DataUsageSnapshotSetState {
|
||||
pool_index: 1,
|
||||
set_index: 0,
|
||||
scanner_cycle: Some(10),
|
||||
scanner_epoch: Some(2),
|
||||
scan_plan_digest: Some([2; 32]),
|
||||
complete: false,
|
||||
tombstone: false,
|
||||
},
|
||||
],
|
||||
..Default::default()
|
||||
};
|
||||
assert!(!partial.is_valid_partial_snapshot());
|
||||
partial.usage_snapshot_set_states[1].scan_plan_digest = Some([1; 32]);
|
||||
assert!(partial.is_valid_partial_snapshot());
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -196,15 +196,16 @@ pub mod bucket {
|
||||
ReplicationOperation, ReplicationPoolTrait, ReplicationPriority, ReplicationQueueAdmission, ReplicationScannerBridge,
|
||||
ReplicationState, ReplicationStats, ReplicationStatusType, ReplicationStorage, ReplicationTargetValidationError,
|
||||
ReplicationType, ResyncOpts, ResyncStatusType, RuntimeReplicationTargetBacklog, TargetReplicationResyncStatus,
|
||||
VersionPurgeStatusType, XferStats, commit_force_delete_intent, complete_force_delete_intent,
|
||||
delete_replication_state_from_config, delete_replication_version_id, get_global_replication_pool,
|
||||
get_global_replication_stats, get_proxy_targets, init_background_replication,
|
||||
invalid_replication_config_status_field, persist_force_delete_intent, read_durable_mrf_backlog,
|
||||
replication_state_to_filemeta, replication_status_to_filemeta, replication_statuses_map, replication_target_arns,
|
||||
resync_start_conflict_id, should_remove_replication_target, should_schedule_delete_replication,
|
||||
should_use_existing_delete_replication_info, should_use_existing_delete_replication_source,
|
||||
unsupported_replication_config_field, validate_replication_config_structure, validate_replication_config_target_arns,
|
||||
version_purge_status_to_filemeta,
|
||||
VersionPurgeStatusType, XferStats, assign_site_replication_rule_priorities, commit_force_delete_intent,
|
||||
complete_force_delete_intent, delete_replication_state_from_config, delete_replication_version_id,
|
||||
get_global_replication_pool, get_global_replication_stats, get_proxy_targets, init_background_replication,
|
||||
invalid_replication_config_status_field, is_site_replication_rule, merge_incoming_replication_config,
|
||||
merge_user_replication_config, persist_force_delete_intent, read_durable_mrf_backlog, replication_state_to_filemeta,
|
||||
replication_status_to_filemeta, replication_statuses_map, replication_target_arn_deployment_id,
|
||||
replication_target_arns, resync_start_conflict_id, should_remove_replication_target,
|
||||
should_schedule_delete_replication, should_use_existing_delete_replication_info,
|
||||
should_use_existing_delete_replication_source, unsupported_replication_config_field,
|
||||
validate_replication_config_structure, validate_replication_config_target_arns, version_purge_status_to_filemeta,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -47,8 +47,10 @@ pub use replication_config_boundary::{
|
||||
ObjectOpts, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS,
|
||||
REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS,
|
||||
ReplicationConfigStructureError, ReplicationConfigurationExt, ReplicationTargetValidationError,
|
||||
invalid_replication_config_status_field, replication_target_arns, should_remove_replication_target,
|
||||
unsupported_replication_config_field, validate_replication_config_structure, validate_replication_config_target_arns,
|
||||
assign_site_replication_rule_priorities, invalid_replication_config_status_field, is_site_replication_rule,
|
||||
merge_incoming_replication_config, merge_user_replication_config, replication_target_arn_deployment_id,
|
||||
replication_target_arns, should_remove_replication_target, unsupported_replication_config_field,
|
||||
validate_replication_config_structure, validate_replication_config_target_arns,
|
||||
};
|
||||
pub(crate) use replication_filemeta_boundary::version_purge_statuses_map;
|
||||
pub use replication_filemeta_boundary::{
|
||||
|
||||
@@ -16,6 +16,8 @@ pub use rustfs_replication::{
|
||||
ObjectOpts, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS,
|
||||
REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS,
|
||||
ReplicationConfigStructureError, ReplicationConfigurationExt, ReplicationRuleExt, ReplicationTargetValidationError,
|
||||
invalid_replication_config_status_field, replication_target_arns, should_remove_replication_target,
|
||||
unsupported_replication_config_field, validate_replication_config_structure, validate_replication_config_target_arns,
|
||||
assign_site_replication_rule_priorities, invalid_replication_config_status_field, is_site_replication_rule,
|
||||
merge_incoming_replication_config, merge_user_replication_config, replication_target_arn_deployment_id,
|
||||
replication_target_arns, should_remove_replication_target, unsupported_replication_config_field,
|
||||
validate_replication_config_structure, validate_replication_config_target_arns,
|
||||
};
|
||||
|
||||
@@ -73,16 +73,6 @@ struct CachedBucketUsage {
|
||||
// mutation. A strictly later generation is required before the mutation
|
||||
// evidence can be discarded.
|
||||
pending_scanner_position: Option<(u64, u64)>,
|
||||
// Deletes are visible to admin immediately, but quota admission keeps
|
||||
// them pending until a complete scanner generation reconciles the set.
|
||||
// This marker intentionally remains process-local: the delete request
|
||||
// updates this overlay before the scanner writes a durable snapshot. If
|
||||
// the process restarts first, loading the persisted complete snapshot
|
||||
// restores the pre-reconciliation (larger) baseline, which is
|
||||
// conservative for quota admission. A persisted post-delete snapshot is
|
||||
// necessarily a complete scanner reconciliation and therefore creates a
|
||||
// fresh cache entry with no pending hold.
|
||||
pending_negative_delta: u64,
|
||||
}
|
||||
|
||||
type UsageMemoryCache = Arc<RwLock<HashMap<String, CachedBucketUsage>>>;
|
||||
@@ -958,12 +948,7 @@ async fn load_observed_data_usage_snapshot(store: Arc<ECStore>) -> Option<DataUs
|
||||
};
|
||||
|
||||
match parse_usage_snapshot(&data) {
|
||||
Ok(info)
|
||||
if info.usage_snapshot_converged == Some(false)
|
||||
&& (info.is_complete_bucket_usage_snapshot() || info.is_valid_partial_snapshot()) =>
|
||||
{
|
||||
Some(info)
|
||||
}
|
||||
Ok(info) if info.usage_snapshot_converged == Some(false) && info.is_complete_bucket_usage_snapshot() => Some(info),
|
||||
Ok(_) => {
|
||||
error!(
|
||||
event = "data_usage_snapshot_load_failed",
|
||||
@@ -1008,7 +993,7 @@ async fn load_admin_data_usage_from_backend(store: Arc<ECStore>) -> Result<DataU
|
||||
}
|
||||
|
||||
fn discard_incomplete_bucket_usage(data_usage_info: &mut DataUsageInfo) {
|
||||
if !data_usage_info.is_complete_bucket_usage_snapshot() && !data_usage_info.usage_snapshot_partial {
|
||||
if !data_usage_info.is_complete_bucket_usage_snapshot() {
|
||||
data_usage_info.usage_snapshot_complete = false;
|
||||
data_usage_info.buckets_usage.clear();
|
||||
data_usage_info.bucket_sizes.clear();
|
||||
@@ -1658,7 +1643,6 @@ fn cached_bucket_usage_from_backend(usage: BucketUsageInfo, updated_at: SystemTi
|
||||
dirty: false,
|
||||
stale_snapshot_pending: false,
|
||||
pending_scanner_position: None,
|
||||
pending_negative_delta: 0,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1672,7 +1656,6 @@ fn cached_bucket_usage_now(usage: BucketUsageInfo) -> CachedBucketUsage {
|
||||
dirty: false,
|
||||
stale_snapshot_pending: false,
|
||||
pending_scanner_position: None,
|
||||
pending_negative_delta: 0,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1825,7 +1808,6 @@ pub async fn record_bucket_object_delete_memory(bucket: &str, deleted_size: u64,
|
||||
.or_insert_with(|| cached_bucket_usage_now(BucketUsageInfo::default()));
|
||||
|
||||
entry.usage.size = entry.usage.size.saturating_sub(deleted_size);
|
||||
entry.pending_negative_delta = entry.pending_negative_delta.saturating_add(deleted_size);
|
||||
if removed_current_object {
|
||||
entry.usage.objects_count = entry.usage.objects_count.saturating_sub(1);
|
||||
entry.usage.versions_count = entry.usage.versions_count.saturating_sub(1);
|
||||
@@ -1881,7 +1863,7 @@ pub async fn get_bucket_usage_memory(bucket: &str) -> Option<u64> {
|
||||
cache
|
||||
.get(bucket)
|
||||
.filter(|cached| cached.authoritative)
|
||||
.map(|cached| cached.usage.size.saturating_add(cached.pending_negative_delta))
|
||||
.map(|cached| cached.usage.size)
|
||||
}
|
||||
|
||||
async fn update_usage_cache_if_needed() {
|
||||
@@ -2961,45 +2943,6 @@ mod tests {
|
||||
assert_eq!(selected.usage_snapshot_converged, Some(true));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn persisted_authoritative_stalls_but_memory_overlay_remains_visible() {
|
||||
let authoritative = DataUsageInfo {
|
||||
last_update: Some(SystemTime::UNIX_EPOCH),
|
||||
scanner_epoch: Some(4),
|
||||
scanner_cycle: Some(10),
|
||||
usage_snapshot_complete: true,
|
||||
..Default::default()
|
||||
};
|
||||
let mut partial = authoritative.clone();
|
||||
partial.last_update = Some(SystemTime::UNIX_EPOCH + Duration::from_secs(1));
|
||||
partial.scanner_cycle = Some(11);
|
||||
partial.usage_snapshot_complete = false;
|
||||
partial.usage_snapshot_partial = true;
|
||||
partial.usage_snapshot_converged = Some(false);
|
||||
partial.usage_snapshot_authoritative_baseline = Some(authoritative.snapshot_identity());
|
||||
partial.usage_snapshot_set_states = vec![rustfs_data_usage::DataUsageSnapshotSetState {
|
||||
pool_index: 0,
|
||||
set_index: 0,
|
||||
scanner_cycle: Some(10),
|
||||
scanner_epoch: Some(4),
|
||||
scan_plan_digest: Some([1; 32]),
|
||||
complete: false,
|
||||
tombstone: false,
|
||||
}];
|
||||
partial.buckets_usage.insert(
|
||||
"bucket".to_string(),
|
||||
BucketUsageInfo {
|
||||
size: 100,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
partial.buckets_count = 1;
|
||||
|
||||
let (selected, _) = select_admin_data_usage_snapshot(authoritative, true, Some(partial));
|
||||
assert!(selected.usage_snapshot_partial);
|
||||
assert_eq!(selected.buckets_usage.get("bucket").map(|usage| usage.size), Some(100));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn authoritative_save_cleanup_removes_observed_snapshot_best_effort() {
|
||||
let store = UsageCasStore::default();
|
||||
@@ -4722,55 +4665,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn partial_usage_is_observational_not_authoritative_for_quota() {
|
||||
clear_usage_memory_cache_for_test().await;
|
||||
|
||||
let mut partial = data_usage_info_for_test("bucket-a", 10, 100, SystemTime::now());
|
||||
partial.usage_snapshot_complete = false;
|
||||
partial.usage_snapshot_partial = true;
|
||||
replace_bucket_usage_memory_from_info(&partial).await;
|
||||
|
||||
assert_eq!(get_bucket_usage_memory("bucket-a").await, None);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn stale_quota_uses_complete_baseline_plus_positive_deltas() {
|
||||
clear_usage_memory_cache_for_test().await;
|
||||
|
||||
let baseline = data_usage_info_for_test("bucket-a", 1, 100, SystemTime::now());
|
||||
replace_bucket_usage_memory_from_info(&baseline).await;
|
||||
record_bucket_object_write_memory("bucket-a", None, 25).await;
|
||||
|
||||
assert_eq!(get_bucket_usage_memory("bucket-a").await, Some(125));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn negative_delta_waits_for_set_reconciliation() {
|
||||
clear_usage_memory_cache_for_test().await;
|
||||
|
||||
let baseline = data_usage_info_for_test("bucket-a", 1, 100, SystemTime::UNIX_EPOCH + Duration::from_secs(100));
|
||||
replace_bucket_usage_memory_from_info(&baseline).await;
|
||||
record_bucket_object_delete_memory("bucket-a", 25, true).await;
|
||||
|
||||
assert_eq!(get_bucket_usage_memory("bucket-a").await, Some(100));
|
||||
|
||||
// Simulate a process restart: the request-path overlay is gone, but
|
||||
// the persisted authoritative snapshot is still the pre-reconciliation
|
||||
// baseline. Quota must remain conservative until a complete scanner
|
||||
// result proves the delete.
|
||||
clear_usage_memory_cache_for_test().await;
|
||||
replace_bucket_usage_memory_from_info(&baseline).await;
|
||||
assert_eq!(get_bucket_usage_memory("bucket-a").await, Some(100));
|
||||
|
||||
let reconciled = data_usage_info_for_test("bucket-a", 0, 75, SystemTime::UNIX_EPOCH + Duration::from_secs(101));
|
||||
replace_bucket_usage_memory_from_info(&reconciled).await;
|
||||
assert_eq!(get_bucket_usage_memory("bucket-a").await, Some(75));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn memory_overlay_counts_versioned_overwrite_as_new_version() {
|
||||
|
||||
@@ -2124,26 +2124,13 @@ impl SetDisks {
|
||||
|
||||
let put_object_size = known_put_object_storage_size(data.size());
|
||||
let shard_file_size_raw = erasure.shard_file_size(put_object_size);
|
||||
let is_inline_buffer =
|
||||
storage_class_config.should_inline(shard_file_size_raw, erasure.data_shards, opts.versioned);
|
||||
let is_inline_buffer = storage_class_config.should_inline(shard_file_size_raw, erasure.data_shards, opts.versioned);
|
||||
|
||||
let collect_stage_timing = rustfs_io_metrics::put_stage_metrics_enabled() || issue3031_diag_enabled();
|
||||
let shard_file_size = shard_file_size_raw;
|
||||
let shard_size = erasure.shard_size();
|
||||
let write_path = classify_put_write_path(is_inline_buffer, put_object_size, fi.erasure.block_size);
|
||||
let direct_inline_commit = matches!(write_path, SmallWritePath::Inline);
|
||||
{
|
||||
use std::io::Write;
|
||||
let msg = format!(
|
||||
"INLINE_DEBUG: bucket={} obj={} size={} shard_fs={} ds={} bs={} inline={} direct={} path={} iblock={} ver={}\n",
|
||||
bucket, object, put_object_size, shard_file_size_raw, erasure.data_shards, fi.erasure.block_size,
|
||||
is_inline_buffer, direct_inline_commit, write_path.metric_label(), storage_class_config.inline_block(), opts.versioned
|
||||
);
|
||||
if let Ok(mut f) = std::fs::OpenOptions::new().create(true).append(true).open("/tmp/rustfs_inline_debug.log") {
|
||||
let _ = f.write_all(msg.as_bytes());
|
||||
}
|
||||
let _ = std::io::stderr().write_all(msg.as_bytes());
|
||||
}
|
||||
rustfs_io_metrics::record_put_object_path(write_path.metric_label());
|
||||
let writer_setup_stage_start = collect_stage_timing.then(Instant::now);
|
||||
let (mut writers, errors) = if direct_inline_commit {
|
||||
|
||||
@@ -3194,7 +3194,7 @@ impl ECStore {
|
||||
|
||||
// Default return value
|
||||
let mut del_objects = vec![DeletedObject::default(); objects.len()];
|
||||
let mut accounting = vec![None; objects.len()];
|
||||
let accounting = vec![None; objects.len()];
|
||||
|
||||
let mut del_errs = Vec::with_capacity(objects.len());
|
||||
for _ in 0..objects.len() {
|
||||
|
||||
@@ -271,7 +271,7 @@ pub(super) fn resolve_latest_object_info_candidates(
|
||||
.filter(|candidate| latest_candidate_mod_time(candidate) == Some(latest_mod_time))
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
latest_candidates.sort_by(|left, right| right.idx.cmp(&left.idx));
|
||||
latest_candidates.sort_by_key(|candidate| std::cmp::Reverse(candidate.idx));
|
||||
|
||||
let Some(winner) = latest_candidates.first() else {
|
||||
return Err(Error::ErasureReadQuorum);
|
||||
|
||||
@@ -270,6 +270,160 @@ pub fn active_replication_rule_destination_arns(config: &ReplicationConfiguratio
|
||||
arns
|
||||
}
|
||||
|
||||
/// Deployment id extracted from a site-replication target ARN
|
||||
/// (`arn:{rustfs|minio}:replication::<deployment-id>:<bucket>`), or `None`
|
||||
/// for an operator-authored ARN.
|
||||
pub fn replication_target_arn_deployment_id(arn: &str) -> Option<String> {
|
||||
let parts: Vec<_> = arn.split(':').collect();
|
||||
if parts.len() == 6
|
||||
&& parts[0] == "arn"
|
||||
&& matches!(parts[1], "rustfs" | "minio")
|
||||
&& parts[2] == "replication"
|
||||
&& !parts[4].is_empty()
|
||||
{
|
||||
return Some(parts[4].to_string());
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
/// Rule id prefix the site-replication reconciler stamps on the rules it
|
||||
/// derives (`site-repl-<peer deployment id>`).
|
||||
pub const SITE_REPLICATION_RULE_ID_PREFIX: &str = "site-repl-";
|
||||
|
||||
/// Whether `rule` carries a site-replication rule id (`site-repl-*`). The
|
||||
/// reconciler and the peer ingestion path treat the whole namespace as theirs
|
||||
/// on a site-replication bucket; the S3 edit path must not — rule ids are not
|
||||
/// reserved, so see [`site_replication_rule_deployment_id`].
|
||||
pub fn is_site_replication_rule(rule: &ReplicationRule) -> bool {
|
||||
rule.id
|
||||
.as_deref()
|
||||
.is_some_and(|id| id.starts_with(SITE_REPLICATION_RULE_ID_PREFIX))
|
||||
}
|
||||
|
||||
/// Deployment id of the peer a reconciler-derived rule replicates to, or
|
||||
/// `None` for any other rule. The reconciler builds each rule from one peer:
|
||||
/// the id is `site-repl-<deployment id>` and the destination ARN names that
|
||||
/// same deployment id — an operator-authored `site-repl-user` rule, or a
|
||||
/// `site-repl-<peer>` id pasted onto a foreign ARN, fails the agreement check.
|
||||
/// Callers that know the current peer set must also confirm the id is one of
|
||||
/// those peers before treating the rule as reconciler-owned.
|
||||
pub fn site_replication_rule_deployment_id(rule: &ReplicationRule) -> Option<&str> {
|
||||
let deployment_id = rule.id.as_deref()?.strip_prefix(SITE_REPLICATION_RULE_ID_PREFIX)?;
|
||||
(!deployment_id.is_empty()
|
||||
&& replication_target_arn_deployment_id(&rule.destination.bucket).as_deref() == Some(deployment_id))
|
||||
.then_some(deployment_id)
|
||||
}
|
||||
|
||||
/// Whether `rule` is one the local reconciler derived for a current remote
|
||||
/// site-replication peer in `peer_deployment_ids`. With an empty peer set
|
||||
/// (site replication disabled) nothing qualifies, so a bucket outside site
|
||||
/// replication keeps the verbatim S3 put/delete semantics.
|
||||
pub fn is_reconciler_owned_site_replication_rule(rule: &ReplicationRule, peer_deployment_ids: &HashSet<String>) -> bool {
|
||||
site_replication_rule_deployment_id(rule).is_some_and(|deployment_id| peer_deployment_ids.contains(deployment_id))
|
||||
}
|
||||
|
||||
/// Merge an incoming replication config into the local one.
|
||||
///
|
||||
/// `site-repl-*` rules encode the *holder's* outbound direction — their
|
||||
/// destination ARN names another site — so applying an external rule set
|
||||
/// verbatim replaces the local reverse rule with one this site can never
|
||||
/// satisfy (no bucket target backs it) and replication silently stops. Only
|
||||
/// operator-authored rules travel: the site-replication peer ingestion path
|
||||
/// and the S3 put/delete-bucket-replication path both keep the local site's
|
||||
/// `site-repl-*` rules through this merge. `incoming == None` models a
|
||||
/// delete of the operator-authored rules.
|
||||
pub fn merge_incoming_replication_config(
|
||||
incoming: Option<ReplicationConfiguration>,
|
||||
local: Option<ReplicationConfiguration>,
|
||||
) -> Option<ReplicationConfiguration> {
|
||||
merge_replication_config_keeping_site_rules(incoming, local, is_site_replication_rule)
|
||||
}
|
||||
|
||||
/// [`merge_incoming_replication_config`] for the S3 put/delete-bucket-replication
|
||||
/// path (issue #1948): only rules the local reconciler derived for a current
|
||||
/// peer in `peer_deployment_ids` survive as site rules; every other stored
|
||||
/// rule — including an operator-authored `site-repl-*` id — is operator state
|
||||
/// that the request replaces or deletes. An incoming rule whose id is a
|
||||
/// current peer's `site-repl-<id>` is dropped whatever its ARN: accepting it
|
||||
/// would duplicate the reconciler rule's id.
|
||||
pub fn merge_user_replication_config(
|
||||
incoming: Option<ReplicationConfiguration>,
|
||||
local: Option<ReplicationConfiguration>,
|
||||
peer_deployment_ids: &HashSet<String>,
|
||||
) -> Option<ReplicationConfiguration> {
|
||||
let incoming = incoming.map(|mut config| {
|
||||
config.rules.retain(|rule| {
|
||||
!rule
|
||||
.id
|
||||
.as_deref()
|
||||
.and_then(|id| id.strip_prefix(SITE_REPLICATION_RULE_ID_PREFIX))
|
||||
.is_some_and(|deployment_id| peer_deployment_ids.contains(deployment_id))
|
||||
});
|
||||
config
|
||||
});
|
||||
merge_replication_config_keeping_site_rules(incoming, local, |rule| {
|
||||
is_reconciler_owned_site_replication_rule(rule, peer_deployment_ids)
|
||||
})
|
||||
}
|
||||
|
||||
fn merge_replication_config_keeping_site_rules(
|
||||
incoming: Option<ReplicationConfiguration>,
|
||||
local: Option<ReplicationConfiguration>,
|
||||
is_site_rule: impl Fn(&ReplicationRule) -> bool,
|
||||
) -> Option<ReplicationConfiguration> {
|
||||
let incoming_role = incoming.as_ref().map(|config| config.role.clone()).unwrap_or_default();
|
||||
// Operator rules first, then the local site rules — the same order the
|
||||
// site-replication reconciler produces, so its no-op check matches and
|
||||
// the bucket metadata is written once per broadcast, not twice.
|
||||
let mut rules: Vec<ReplicationRule> = incoming
|
||||
.into_iter()
|
||||
.flat_map(|config| config.rules)
|
||||
.filter(|rule| !is_site_rule(rule))
|
||||
.collect();
|
||||
rules.extend(local.into_iter().flat_map(|config| config.rules).filter(&is_site_rule));
|
||||
|
||||
if rules.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
assign_site_replication_rule_priorities(&mut rules, &is_site_rule);
|
||||
|
||||
// A site-replication ARN in `role` is the sender's, and the reconciler's
|
||||
// per-peer target lookup reads it — carrying it over would pin the
|
||||
// receiver's targets to the sender's identity.
|
||||
let role = match replication_target_arn_deployment_id(&incoming_role) {
|
||||
Some(_) => String::new(),
|
||||
None => incoming_role,
|
||||
};
|
||||
|
||||
Some(ReplicationConfiguration { role, rules })
|
||||
}
|
||||
|
||||
/// Give the site rules in `rules` the lowest priorities no operator rule uses,
|
||||
/// in rule order, leaving every operator rule's priority untouched. Operator
|
||||
/// priorities decide which rule wins per target, so they are part of the
|
||||
/// submitted policy; site rules are derived state and only need to be unique
|
||||
/// (`validate_replication_config_structure` rejects duplicates). The result
|
||||
/// is a pure function of the rule list, so the site-replication reconciler,
|
||||
/// the peer ingestion merge and the S3 edit merge all converge on the same
|
||||
/// bytes and the reconciler's no-op check holds.
|
||||
pub fn assign_site_replication_rule_priorities(rules: &mut [ReplicationRule], is_site_rule: impl Fn(&ReplicationRule) -> bool) {
|
||||
let taken: HashSet<i32> = rules
|
||||
.iter()
|
||||
.filter(|rule| !is_site_rule(rule))
|
||||
.map(|rule| rule.priority.unwrap_or(0))
|
||||
.collect();
|
||||
let mut next = 1;
|
||||
for rule in rules.iter_mut().filter(|rule| is_site_rule(rule)) {
|
||||
while taken.contains(&next) {
|
||||
next += 1;
|
||||
}
|
||||
rule.priority = Some(next);
|
||||
next = next.saturating_add(1);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn replication_target_arns(config: &ReplicationConfiguration) -> HashSet<String> {
|
||||
let role = config.role.trim();
|
||||
if !role.is_empty() {
|
||||
@@ -1544,4 +1698,103 @@ mod tests {
|
||||
"the child rule must win for target A while the overlapping child target B remains eligible"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn site_replication_rule_deployment_id_requires_id_and_arn_agreement() {
|
||||
let reconciler_rule = replication_rule("site-repl-peer-dep", "arn:rustfs:replication::peer-dep:bucket");
|
||||
assert_eq!(site_replication_rule_deployment_id(&reconciler_rule), Some("peer-dep"));
|
||||
|
||||
// A remote-target ARN carries the remote's deployment id (or a random
|
||||
// uuid), never the operator's rule id.
|
||||
let operator_named_rule = replication_rule("site-repl-user", "arn:minio:replication:us-east-1:2f1c-remote:bucket");
|
||||
assert_eq!(site_replication_rule_deployment_id(&operator_named_rule), None);
|
||||
|
||||
let foreign_arn = replication_rule("site-repl-peer-dep", "arn:rustfs:replication::other-dep:bucket");
|
||||
assert_eq!(site_replication_rule_deployment_id(&foreign_arn), None);
|
||||
|
||||
let empty_id = replication_rule("site-repl-", "arn:rustfs:replication::peer-dep:bucket");
|
||||
assert_eq!(site_replication_rule_deployment_id(&empty_id), None);
|
||||
|
||||
let peers = HashSet::from(["peer-dep".to_string()]);
|
||||
assert!(is_reconciler_owned_site_replication_rule(&reconciler_rule, &peers));
|
||||
assert!(!is_reconciler_owned_site_replication_rule(&reconciler_rule, &HashSet::new()));
|
||||
let removed_peer = replication_rule("site-repl-gone-dep", "arn:rustfs:replication::gone-dep:bucket");
|
||||
assert!(!is_reconciler_owned_site_replication_rule(&removed_peer, &peers));
|
||||
}
|
||||
|
||||
// The merge must not rewrite the operator's priorities: with the
|
||||
// priority-5 rule listed first and renumbered 1 then 2, the priority-1
|
||||
// delete-marker-disabled rule would win the replication decision.
|
||||
#[test]
|
||||
fn merge_keeps_operator_priorities_and_replication_decision() {
|
||||
let user_arn = "arn:minio:replication:us-east-1:2f1c-remote:bucket";
|
||||
let peer_arn = "arn:rustfs:replication::peer-dep:bucket";
|
||||
let incoming = ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![
|
||||
delete_marker_rule("dm-enabled", user_arn, "logs/", 5, true),
|
||||
delete_marker_rule("dm-disabled", user_arn, "logs/2026/", 1, false),
|
||||
],
|
||||
};
|
||||
let mut site_rule = delete_marker_rule("site-repl-peer-dep", peer_arn, "", 7, true);
|
||||
site_rule.prefix = None;
|
||||
let local = structure_config(vec![site_rule]);
|
||||
let opts = ObjectOpts {
|
||||
name: "logs/2026/app.log".to_string(),
|
||||
op_type: ReplicationType::Delete,
|
||||
delete_marker: true,
|
||||
version_id: None,
|
||||
..Default::default()
|
||||
};
|
||||
let submitted: Vec<_> = incoming.filter_target_replication_decisions(&opts);
|
||||
|
||||
let peers = HashSet::from(["peer-dep".to_string()]);
|
||||
let merged = merge_user_replication_config(Some(incoming.clone()), Some(local.clone()), &peers).expect("rules");
|
||||
|
||||
let priorities: Vec<_> = merged
|
||||
.rules
|
||||
.iter()
|
||||
.map(|rule| (rule.id.as_deref().unwrap(), rule.priority))
|
||||
.collect();
|
||||
assert_eq!(
|
||||
priorities,
|
||||
vec![
|
||||
("dm-enabled", Some(5)),
|
||||
("dm-disabled", Some(1)),
|
||||
("site-repl-peer-dep", Some(2))
|
||||
],
|
||||
"operator priorities are kept verbatim; the site rule takes the lowest free slot"
|
||||
);
|
||||
assert!(validate_replication_config_structure(&merged).is_ok());
|
||||
let mut decisions = merged.filter_target_replication_decisions(&opts);
|
||||
decisions.retain(|(arn, _)| arn == user_arn);
|
||||
assert_eq!(decisions, submitted, "the merged config must replicate exactly as the operator submitted");
|
||||
assert_eq!(decisions, vec![(user_arn.to_string(), true)]);
|
||||
|
||||
// The peer ingestion merge follows the same rule.
|
||||
let merged = merge_incoming_replication_config(Some(incoming), Some(local)).expect("rules");
|
||||
let priorities: Vec<_> = merged.rules.iter().map(|rule| rule.priority).collect();
|
||||
assert_eq!(priorities, vec![Some(5), Some(1), Some(2)]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn site_rule_priorities_skip_every_operator_priority() {
|
||||
let mut rules = vec![
|
||||
delete_marker_rule("a", "arn:a", "", 2, true),
|
||||
delete_marker_rule("site-repl-x", "arn:rustfs:replication::x:b", "", 9, true),
|
||||
delete_marker_rule("b", "arn:a", "", 1, true),
|
||||
delete_marker_rule("site-repl-y", "arn:rustfs:replication::y:b", "", 9, true),
|
||||
delete_marker_rule("c", "arn:a", "", 4, true),
|
||||
];
|
||||
assign_site_replication_rule_priorities(&mut rules, is_site_replication_rule);
|
||||
let priorities: Vec<_> = rules.iter().map(|rule| rule.priority).collect();
|
||||
assert_eq!(priorities, vec![Some(2), Some(3), Some(1), Some(5), Some(4)]);
|
||||
assert!(validate_replication_config_structure(&structure_config(rules.clone())).is_ok());
|
||||
|
||||
// Idempotent, so the reconciler's pass over an already-merged config
|
||||
// is a byte-stable no-op rather than a rewrite every period.
|
||||
let settled = rules.clone();
|
||||
assign_site_replication_rule_priorities(&mut rules, is_site_replication_rule);
|
||||
assert_eq!(rules, settled);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -32,9 +32,11 @@ pub use config::{
|
||||
ObjectOpts, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS,
|
||||
REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS,
|
||||
ReplicationConfigStructureError, ReplicationConfigurationExt, ReplicationTargetValidationError,
|
||||
active_replication_rule_destination_arns, invalid_replication_config_status_field, replication_target_arns,
|
||||
should_remove_replication_target, unsupported_replication_config_field, validate_replication_config_structure,
|
||||
validate_replication_config_target_arns,
|
||||
active_replication_rule_destination_arns, assign_site_replication_rule_priorities, invalid_replication_config_status_field,
|
||||
is_reconciler_owned_site_replication_rule, is_site_replication_rule, merge_incoming_replication_config,
|
||||
merge_user_replication_config, replication_target_arn_deployment_id, replication_target_arns,
|
||||
should_remove_replication_target, site_replication_rule_deployment_id, unsupported_replication_config_field,
|
||||
validate_replication_config_structure, validate_replication_config_target_arns,
|
||||
};
|
||||
pub use delete::{
|
||||
DeletedObjectReplicationInfo, delete_marker_purge_mrf_entry, delete_marker_purge_version_id,
|
||||
|
||||
@@ -28,9 +28,8 @@ use rustfs_common::heal_channel::HealScanMode;
|
||||
use rustfs_config::ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS;
|
||||
pub use rustfs_data_usage::{
|
||||
AllTierStats, BucketTargetUsageInfo, BucketUsageInfo, DATA_USAGE_OBJECT_NAME, DATA_USAGE_OBSERVED_OBJECT_NAME,
|
||||
DataUsageEntry, DataUsageHash, DataUsageHashMap, DataUsageInfo, DataUsageSnapshotSetState, LEGACY_DATA_USAGE_OBJECT_NAME,
|
||||
PrefixUsageEntry, PrefixUsageQuery, PrefixUsageSummary, ReplTargetSizeSummary, SizeSummary, TierStats, hash_path,
|
||||
prefix_usage_in_cache,
|
||||
DataUsageEntry, DataUsageHash, DataUsageHashMap, DataUsageInfo, LEGACY_DATA_USAGE_OBJECT_NAME, PrefixUsageEntry,
|
||||
PrefixUsageQuery, PrefixUsageSummary, ReplTargetSizeSummary, SizeSummary, TierStats, hash_path, prefix_usage_in_cache,
|
||||
};
|
||||
use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf};
|
||||
use tokio::time::{Duration, Instant, sleep, timeout};
|
||||
@@ -345,18 +344,6 @@ pub struct DataUsageCacheInfo {
|
||||
pub scan_plan_digest: Option<DataUsageScanPlanDigest>,
|
||||
#[serde(default)]
|
||||
pub cache_key_format: u16,
|
||||
/// Whether the entries retained while a set scan was incomplete come
|
||||
/// from a prior complete set snapshot. This is observational input only.
|
||||
#[serde(default)]
|
||||
pub lkg_snapshot_complete: bool,
|
||||
#[serde(default)]
|
||||
pub lkg_next_cycle: Option<u64>,
|
||||
#[serde(default)]
|
||||
pub lkg_last_update: Option<SystemTime>,
|
||||
#[serde(default)]
|
||||
pub lkg_leader_epoch: Option<u64>,
|
||||
#[serde(default)]
|
||||
pub lkg_scan_plan_digest: Option<DataUsageScanPlanDigest>,
|
||||
}
|
||||
|
||||
impl Serialize for DataUsageCacheInfo {
|
||||
@@ -366,7 +353,7 @@ impl Serialize for DataUsageCacheInfo {
|
||||
{
|
||||
// Keep this metadata map-encoded so older readers can ignore fields
|
||||
// appended by newer scanner versions during rolling upgrades.
|
||||
let mut state = serializer.serialize_map(Some(21))?;
|
||||
let mut state = serializer.serialize_map(Some(16))?;
|
||||
state.serialize_entry("name", &self.name)?;
|
||||
state.serialize_entry("next_cycle", &self.next_cycle)?;
|
||||
state.serialize_entry("leader_epoch", &self.leader_epoch)?;
|
||||
@@ -383,11 +370,6 @@ impl Serialize for DataUsageCacheInfo {
|
||||
state.serialize_entry("snapshot_complete", &self.snapshot_complete)?;
|
||||
state.serialize_entry("scan_plan_digest", &self.scan_plan_digest)?;
|
||||
state.serialize_entry("cache_key_format", &self.cache_key_format)?;
|
||||
state.serialize_entry("lkg_snapshot_complete", &self.lkg_snapshot_complete)?;
|
||||
state.serialize_entry("lkg_next_cycle", &self.lkg_next_cycle)?;
|
||||
state.serialize_entry("lkg_last_update", &self.lkg_last_update)?;
|
||||
state.serialize_entry("lkg_leader_epoch", &self.lkg_leader_epoch)?;
|
||||
state.serialize_entry("lkg_scan_plan_digest", &self.lkg_scan_plan_digest)?;
|
||||
state.end()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2274,8 +2274,9 @@ async fn final_data_usage_publication_defer_reason(
|
||||
}
|
||||
}
|
||||
ScannerCycleStatus::Deferred(reason) => Some(reason),
|
||||
// Incomplete cycles may publish a non-authoritative observational
|
||||
// snapshot when at least one set has a usable current/LKG view.
|
||||
// Incomplete cycles do not publish a usage snapshot. Keep the
|
||||
// decision permissive so existing partial-cycle handling remains
|
||||
// unchanged if a future scanner path emits a bookkeeping update.
|
||||
ScannerCycleStatus::Incomplete => None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -198,7 +198,7 @@ where
|
||||
data_usage_info.usage_snapshot_authoritative_baseline = Some(authoritative.snapshot_identity());
|
||||
}
|
||||
|
||||
if !data_usage_info.is_complete_bucket_usage_snapshot() && !data_usage_info.usage_snapshot_partial {
|
||||
if !data_usage_info.is_complete_bucket_usage_snapshot() {
|
||||
error!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_PERSIST_STATE,
|
||||
|
||||
@@ -18,8 +18,8 @@ use crate::scanner_folder::{ScannerItem, scan_data_folder};
|
||||
use crate::sleeper::SCANNER_SLEEPER;
|
||||
use crate::{
|
||||
DATA_USAGE_CACHE_NAME, DATA_USAGE_ROOT, DataUsageCache, DataUsageCacheInfo, DataUsageCachePrepareOutcome,
|
||||
DataUsageCacheSource, DataUsageEntry, DataUsageEntryInfo, DataUsageInfo, DataUsageScanPlanDigest, DataUsageSnapshotSetState,
|
||||
ScannerError, SizeSummary, TierStats,
|
||||
DataUsageCacheSource, DataUsageEntry, DataUsageEntryInfo, DataUsageInfo, DataUsageScanPlanDigest, ScannerError, SizeSummary,
|
||||
TierStats,
|
||||
};
|
||||
use futures::future::join_all;
|
||||
use metrics::counter;
|
||||
@@ -278,17 +278,6 @@ async fn publish_usage_snapshot(
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
async fn publish_observational_snapshot(
|
||||
updates: &mpsc::Sender<DataUsageInfo>,
|
||||
mut data_usage_info: DataUsageInfo,
|
||||
) -> Result<bool> {
|
||||
data_usage_info.usage_snapshot_complete = false;
|
||||
data_usage_info.usage_snapshot_partial = true;
|
||||
data_usage_info.usage_snapshot_converged = Some(false);
|
||||
send_data_usage_update(updates, data_usage_info).await?;
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
enum ScannerCycleActivityStatus {
|
||||
Unchanged,
|
||||
|
||||
@@ -188,7 +188,7 @@ pub(super) fn completed_data_usage_info(
|
||||
}
|
||||
|
||||
let mut total = DataUsageEntry::default();
|
||||
let mut bucket_entries = HashMap::with_capacity(all_buckets.len());
|
||||
let mut buckets_usage = HashMap::with_capacity(all_buckets.len());
|
||||
for bucket in all_buckets {
|
||||
let mut merged = DataUsageEntry::default();
|
||||
for result in results {
|
||||
@@ -200,14 +200,10 @@ pub(super) fn completed_data_usage_info(
|
||||
if !total.checked_merge(&merged) {
|
||||
return None;
|
||||
}
|
||||
bucket_entries.insert(bucket.clone(), merged);
|
||||
buckets_usage.insert(bucket.clone(), checked_bucket_usage_info(&merged)?);
|
||||
}
|
||||
|
||||
let merged_last_update = results.iter().filter_map(|result| result.info.last_update).max()?;
|
||||
let buckets_usage = bucket_entries
|
||||
.iter()
|
||||
.map(|(bucket, entry)| Some((bucket.clone(), checked_bucket_usage_info(entry)?)))
|
||||
.collect::<Option<HashMap<_, _>>>()?;
|
||||
let bucket_sizes = buckets_usage
|
||||
.iter()
|
||||
.map(|(bucket, usage)| (bucket.clone(), usage.size))
|
||||
@@ -229,145 +225,6 @@ pub(super) fn completed_data_usage_info(
|
||||
Some((data_usage_info, merged_last_update))
|
||||
}
|
||||
|
||||
/// Build a non-authoritative view from the set snapshots that completed this
|
||||
/// cycle plus compatible per-set last-known-good caches. The caller must
|
||||
/// persist this result only on the observational object; a missing set is
|
||||
/// intentionally represented by an incomplete state and is never treated as
|
||||
/// an empty set.
|
||||
pub(super) fn observational_data_usage_info(
|
||||
results: &[DataUsageCache],
|
||||
expected_sources: &HashSet<DataUsageCacheSource>,
|
||||
all_buckets: &[String],
|
||||
expected_plan_digest: DataUsageScanPlanDigest,
|
||||
scanner_cycle: u64,
|
||||
leader_epoch: u64,
|
||||
) -> Option<(DataUsageInfo, SystemTime)> {
|
||||
let mut by_source = HashMap::with_capacity(results.len());
|
||||
for result in results {
|
||||
let source = result.info.source?;
|
||||
if !expected_sources.contains(&source) || by_source.insert(source, result).is_some() {
|
||||
return None;
|
||||
}
|
||||
}
|
||||
|
||||
let mut usable = Vec::new();
|
||||
let mut set_states = Vec::with_capacity(expected_sources.len());
|
||||
let mut sources = expected_sources.iter().copied().collect::<Vec<_>>();
|
||||
sources.sort_by_key(|source| (source.pool_index, source.set_index));
|
||||
for source in sources {
|
||||
let result = by_source.get(&source).copied();
|
||||
let current = result.filter(|result| {
|
||||
result.info.snapshot_complete
|
||||
&& result.info.next_cycle == scanner_cycle
|
||||
&& result.info.leader_epoch == leader_epoch
|
||||
&& result.info.scan_plan_digest == Some(expected_plan_digest)
|
||||
});
|
||||
let lkg = result.filter(|result| {
|
||||
!result.info.snapshot_complete
|
||||
&& result.info.lkg_snapshot_complete
|
||||
&& result.info.lkg_scan_plan_digest == Some(expected_plan_digest)
|
||||
&& result.info.lkg_leader_epoch.is_some_and(|epoch| {
|
||||
epoch < leader_epoch
|
||||
|| (epoch == leader_epoch && result.info.lkg_next_cycle.is_some_and(|cycle| cycle <= scanner_cycle))
|
||||
})
|
||||
});
|
||||
let current_snapshot = current.is_some();
|
||||
let selected = current.or(lkg);
|
||||
if let Some(selected) = selected {
|
||||
let (cycle, epoch, digest, last_update, complete) = if current_snapshot {
|
||||
(
|
||||
Some(selected.info.next_cycle),
|
||||
Some(selected.info.leader_epoch),
|
||||
selected.info.scan_plan_digest.map(|digest| digest.0),
|
||||
selected.info.last_update,
|
||||
true,
|
||||
)
|
||||
} else {
|
||||
(
|
||||
selected.info.lkg_next_cycle,
|
||||
selected.info.lkg_leader_epoch,
|
||||
selected.info.lkg_scan_plan_digest.map(|digest| digest.0),
|
||||
selected.info.lkg_last_update,
|
||||
false,
|
||||
)
|
||||
};
|
||||
set_states.push(DataUsageSnapshotSetState {
|
||||
pool_index: u64::try_from(source.pool_index).ok()?,
|
||||
set_index: u64::try_from(source.set_index).ok()?,
|
||||
scanner_cycle: cycle,
|
||||
scanner_epoch: epoch,
|
||||
scan_plan_digest: digest,
|
||||
complete,
|
||||
tombstone: false,
|
||||
});
|
||||
usable.push((selected, last_update));
|
||||
} else {
|
||||
set_states.push(DataUsageSnapshotSetState {
|
||||
pool_index: u64::try_from(source.pool_index).ok()?,
|
||||
set_index: u64::try_from(source.set_index).ok()?,
|
||||
scanner_cycle: None,
|
||||
scanner_epoch: None,
|
||||
scan_plan_digest: Some(expected_plan_digest.0),
|
||||
complete: false,
|
||||
tombstone: false,
|
||||
});
|
||||
}
|
||||
}
|
||||
if usable.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut total = DataUsageEntry::default();
|
||||
let mut bucket_entries = HashMap::with_capacity(all_buckets.len());
|
||||
let mut merged_last_update = None;
|
||||
for (result, last_update) in usable {
|
||||
if let Some(update) = last_update {
|
||||
merged_last_update = Some(merged_last_update.map_or(update, |current: SystemTime| current.max(update)));
|
||||
}
|
||||
for bucket in all_buckets {
|
||||
let Some(entry) = result.checked_flatten(bucket) else {
|
||||
continue;
|
||||
};
|
||||
let bucket_entry = bucket_entries.entry(bucket.clone()).or_insert_with(DataUsageEntry::default);
|
||||
if !bucket_entry.checked_merge(&entry) {
|
||||
return None;
|
||||
}
|
||||
if !total.checked_merge(&entry) {
|
||||
return None;
|
||||
}
|
||||
}
|
||||
}
|
||||
let merged_last_update = merged_last_update?;
|
||||
let buckets_usage = bucket_entries
|
||||
.iter()
|
||||
.map(|(bucket, entry)| Some((bucket.clone(), checked_bucket_usage_info(entry)?)))
|
||||
.collect::<Option<HashMap<_, _>>>()?;
|
||||
Some((
|
||||
DataUsageInfo {
|
||||
last_update: Some(merged_last_update),
|
||||
scanner_cycle: Some(scanner_cycle),
|
||||
scanner_epoch: Some(leader_epoch),
|
||||
objects_total_count: u64::try_from(total.objects).ok()?,
|
||||
versions_total_count: u64::try_from(total.versions).ok()?,
|
||||
delete_markers_total_count: u64::try_from(total.delete_markers).ok()?,
|
||||
objects_total_size: u64::try_from(total.size).ok()?,
|
||||
tier_stats: total.all_tier_stats.filter(|tiers| !tiers.is_empty()),
|
||||
buckets_count: u64::try_from(buckets_usage.len()).ok()?,
|
||||
bucket_sizes: buckets_usage
|
||||
.iter()
|
||||
.map(|(bucket, usage)| (bucket.clone(), usage.size))
|
||||
.collect(),
|
||||
buckets_usage,
|
||||
usage_snapshot_complete: false,
|
||||
usage_snapshot_partial: true,
|
||||
usage_snapshot_converged: Some(false),
|
||||
usage_snapshot_set_states: set_states,
|
||||
..Default::default()
|
||||
},
|
||||
merged_last_update,
|
||||
))
|
||||
}
|
||||
|
||||
pub(super) async fn send_cache_root_entry_info(
|
||||
bucket_result_tx: &mpsc::Sender<DataUsageEntryInfo>,
|
||||
cache: &DataUsageCache,
|
||||
|
||||
@@ -40,21 +40,6 @@ impl ScannerIOCache for SetDisks {
|
||||
let set_label = self.set_index.to_string();
|
||||
|
||||
let source = DataUsageCacheSource::new(self.pool_index, self.set_index);
|
||||
let mut old_cache = DataUsageCache::default();
|
||||
if let Err(e) = old_cache.load(self.clone(), DATA_USAGE_CACHE_NAME).await {
|
||||
warn!(
|
||||
target: "rustfs::scanner::io",
|
||||
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_IO,
|
||||
pool = self.pool_index,
|
||||
set = self.set_index,
|
||||
cache_name = DATA_USAGE_CACHE_NAME,
|
||||
state = "old_cache_load_failed",
|
||||
error = %e,
|
||||
"Scanner old data usage cache load failed; rebuilding from bucket caches"
|
||||
);
|
||||
}
|
||||
if buckets.is_empty() {
|
||||
let now = SystemTime::now();
|
||||
let mut cache = DataUsageCache {
|
||||
@@ -95,24 +80,6 @@ impl ScannerIOCache for SetDisks {
|
||||
"Scanner set state found no online disks"
|
||||
);
|
||||
reset_disk_bucket_scan_gauges(&pool_label, &set_label);
|
||||
let lkg = old_cache.info.snapshot_complete.then(|| old_cache.clone());
|
||||
let mut incomplete_scope = lkg.clone().unwrap_or_default();
|
||||
incomplete_scope.info.name = DATA_USAGE_ROOT.to_string();
|
||||
incomplete_scope.info.next_cycle = want_cycle;
|
||||
incomplete_scope.info.last_update = None;
|
||||
incomplete_scope.info.leader_epoch = leader_epoch;
|
||||
incomplete_scope.info.source = Some(source);
|
||||
incomplete_scope.info.snapshot_complete = false;
|
||||
incomplete_scope.info.scan_plan_digest = Some(scan_plan_digest);
|
||||
incomplete_scope.info.cache_key_format = DATA_USAGE_CACHE_KEY_FORMAT;
|
||||
if let Some(lkg) = lkg {
|
||||
incomplete_scope.info.lkg_snapshot_complete = true;
|
||||
incomplete_scope.info.lkg_next_cycle = Some(lkg.info.next_cycle);
|
||||
incomplete_scope.info.lkg_last_update = lkg.info.last_update;
|
||||
incomplete_scope.info.lkg_leader_epoch = Some(lkg.info.leader_epoch);
|
||||
incomplete_scope.info.lkg_scan_plan_digest = lkg.info.scan_plan_digest;
|
||||
}
|
||||
let _ = updates.send(incomplete_scope).await;
|
||||
return Ok(());
|
||||
}
|
||||
// Preserve the original set topology across capability filtering. During
|
||||
@@ -195,24 +162,6 @@ impl ScannerIOCache for SetDisks {
|
||||
"Scanner set state found no usable namespace scanner disks"
|
||||
);
|
||||
reset_disk_bucket_scan_gauges(&pool_label, &set_label);
|
||||
let lkg = old_cache.info.snapshot_complete.then(|| old_cache.clone());
|
||||
let mut incomplete_scope = lkg.clone().unwrap_or_default();
|
||||
incomplete_scope.info.name = DATA_USAGE_ROOT.to_string();
|
||||
incomplete_scope.info.next_cycle = want_cycle;
|
||||
incomplete_scope.info.last_update = None;
|
||||
incomplete_scope.info.leader_epoch = leader_epoch;
|
||||
incomplete_scope.info.source = Some(source);
|
||||
incomplete_scope.info.snapshot_complete = false;
|
||||
incomplete_scope.info.scan_plan_digest = Some(scan_plan_digest);
|
||||
incomplete_scope.info.cache_key_format = DATA_USAGE_CACHE_KEY_FORMAT;
|
||||
if let Some(lkg) = lkg {
|
||||
incomplete_scope.info.lkg_snapshot_complete = true;
|
||||
incomplete_scope.info.lkg_next_cycle = Some(lkg.info.next_cycle);
|
||||
incomplete_scope.info.lkg_last_update = lkg.info.last_update;
|
||||
incomplete_scope.info.lkg_leader_epoch = Some(lkg.info.leader_epoch);
|
||||
incomplete_scope.info.lkg_scan_plan_digest = lkg.info.scan_plan_digest;
|
||||
}
|
||||
let _ = updates.send(incomplete_scope).await;
|
||||
return Ok(());
|
||||
}
|
||||
let set_disk_inventory = Arc::new(scanner_set_disk_inventory(self.as_ref()).await);
|
||||
@@ -254,15 +203,22 @@ impl ScannerIOCache for SetDisks {
|
||||
record_disk_bucket_scans_active(0, &pool_label, &set_label);
|
||||
let _reset_disk_bucket_scan_gauges = DiskBucketScanGaugeReset::new(pool_label.clone(), set_label.clone());
|
||||
|
||||
let old_lkg = old_cache.info.snapshot_complete.then(|| {
|
||||
(
|
||||
old_cache.info.next_cycle,
|
||||
old_cache.info.last_update,
|
||||
old_cache.info.leader_epoch,
|
||||
old_cache.info.scan_plan_digest,
|
||||
)
|
||||
});
|
||||
let prepare_outcome = match old_cache.prepare_for_scan(
|
||||
let mut old_cache = DataUsageCache::default();
|
||||
if let Err(e) = old_cache.load(self.clone(), DATA_USAGE_CACHE_NAME).await {
|
||||
warn!(
|
||||
target: "rustfs::scanner::io",
|
||||
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_IO,
|
||||
pool = self.pool_index,
|
||||
set = self.set_index,
|
||||
cache_name = DATA_USAGE_CACHE_NAME,
|
||||
state = "old_cache_load_failed",
|
||||
error = %e,
|
||||
"Scanner old data usage cache load failed; rebuilding from bucket caches"
|
||||
);
|
||||
}
|
||||
match old_cache.prepare_for_scan(
|
||||
DATA_USAGE_ROOT,
|
||||
want_cycle,
|
||||
leader_epoch,
|
||||
@@ -303,16 +259,7 @@ impl ScannerIOCache for SetDisks {
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
outcome => outcome,
|
||||
};
|
||||
if matches!(prepare_outcome, DataUsageCachePrepareOutcome::Reused)
|
||||
&& let Some((cycle, last_update, epoch, digest)) = old_lkg
|
||||
{
|
||||
old_cache.info.lkg_snapshot_complete = true;
|
||||
old_cache.info.lkg_next_cycle = Some(cycle);
|
||||
old_cache.info.lkg_last_update = last_update;
|
||||
old_cache.info.lkg_leader_epoch = Some(epoch);
|
||||
old_cache.info.lkg_scan_plan_digest = digest;
|
||||
DataUsageCachePrepareOutcome::Reused | DataUsageCachePrepareOutcome::Reset => {}
|
||||
}
|
||||
|
||||
let mut cache = DataUsageCache {
|
||||
@@ -1152,29 +1099,23 @@ impl ScannerIOCache for SetDisks {
|
||||
cache.info.next_cycle = want_cycle;
|
||||
cache.info.last_update.get_or_insert_with(SystemTime::now);
|
||||
cache.info.snapshot_complete = true;
|
||||
cache.info.lkg_snapshot_complete = false;
|
||||
cache.info.lkg_next_cycle = None;
|
||||
cache.info.lkg_last_update = None;
|
||||
cache.info.lkg_leader_epoch = None;
|
||||
cache.info.lkg_scan_plan_digest = None;
|
||||
cache.clone()
|
||||
};
|
||||
let _ = persist_and_publish_cache_snapshot(self.clone(), &updates, cache_snapshot, cache_cycle_floor.as_ref()).await;
|
||||
} else {
|
||||
let mut incomplete_scope = cache_mutex.lock().await.clone();
|
||||
incomplete_scope.info.name = DATA_USAGE_ROOT.to_string();
|
||||
incomplete_scope.info.next_cycle = want_cycle;
|
||||
incomplete_scope.info.last_update = None;
|
||||
incomplete_scope.info.leader_epoch = leader_epoch;
|
||||
incomplete_scope.info.source = Some(source);
|
||||
incomplete_scope.info.snapshot_complete = false;
|
||||
incomplete_scope.info.scan_plan_digest = Some(scan_plan_digest);
|
||||
incomplete_scope.info.cache_key_format = DATA_USAGE_CACHE_KEY_FORMAT;
|
||||
incomplete_scope.info.lkg_snapshot_complete = old_cache.info.lkg_snapshot_complete;
|
||||
incomplete_scope.info.lkg_next_cycle = old_cache.info.lkg_next_cycle;
|
||||
incomplete_scope.info.lkg_last_update = old_cache.info.lkg_last_update;
|
||||
incomplete_scope.info.lkg_leader_epoch = old_cache.info.lkg_leader_epoch;
|
||||
incomplete_scope.info.lkg_scan_plan_digest = old_cache.info.lkg_scan_plan_digest;
|
||||
let incomplete_scope = DataUsageCache {
|
||||
info: DataUsageCacheInfo {
|
||||
name: DATA_USAGE_ROOT.to_string(),
|
||||
next_cycle: want_cycle,
|
||||
leader_epoch,
|
||||
source: Some(source),
|
||||
snapshot_complete: false,
|
||||
scan_plan_digest: Some(scan_plan_digest),
|
||||
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
|
||||
..Default::default()
|
||||
},
|
||||
cache: HashMap::new(),
|
||||
};
|
||||
if let Err(e) = updates.send(incomplete_scope).await {
|
||||
error!(
|
||||
target: "rustfs::scanner::io",
|
||||
|
||||
@@ -234,7 +234,6 @@ impl ScannerIOCycle for ECStore {
|
||||
let active_set_scans_clone = active_set_scans.clone();
|
||||
|
||||
let (tx, mut rx) = mpsc::channel::<DataUsageCache>(1);
|
||||
let failed_scope_tx = tx.clone();
|
||||
|
||||
// Spawn task to receive and store results
|
||||
let receiver_fut = tokio::spawn(async move {
|
||||
@@ -315,21 +314,6 @@ impl ScannerIOCycle for ECStore {
|
||||
state = "set_scan_failed",
|
||||
"Scanner set scan failed; continuing cycle"
|
||||
);
|
||||
let _ = failed_scope_tx
|
||||
.send(DataUsageCache {
|
||||
info: DataUsageCacheInfo {
|
||||
name: DATA_USAGE_ROOT.to_string(),
|
||||
next_cycle: want_cycle_clone,
|
||||
leader_epoch,
|
||||
source: Some(source),
|
||||
snapshot_complete: false,
|
||||
scan_plan_digest: Some(scan_plan_digest),
|
||||
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
|
||||
..Default::default()
|
||||
},
|
||||
cache: HashMap::new(),
|
||||
})
|
||||
.await;
|
||||
let mut first_err = first_err_mutex_clone.lock().await;
|
||||
record_set_scan_failure(&mut first_err, e);
|
||||
}
|
||||
@@ -386,19 +370,6 @@ impl ScannerIOCycle for ECStore {
|
||||
budget_elapsed,
|
||||
ctx.is_cancelled(),
|
||||
);
|
||||
let observational_usage = completed_usage
|
||||
.is_none()
|
||||
.then(|| {
|
||||
observational_data_usage_info(
|
||||
&results,
|
||||
&expected_sources,
|
||||
&all_bucket_names,
|
||||
scan_plan_digest,
|
||||
want_cycle,
|
||||
leader_epoch,
|
||||
)
|
||||
})
|
||||
.flatten();
|
||||
let structurally_complete_snapshot = result.is_ok() && completed_all_sets && completed_usage.is_some();
|
||||
let cycle_status = classify_nsscanner_cycle(
|
||||
structurally_complete_snapshot,
|
||||
@@ -410,10 +381,6 @@ impl ScannerIOCycle for ECStore {
|
||||
);
|
||||
if let Some((data_usage_info, _)) = completed_usage {
|
||||
publish_usage_snapshot(&updates, cycle_status, data_usage_info).await?;
|
||||
} else if !ctx.is_cancelled()
|
||||
&& let Some((data_usage_info, _)) = observational_usage
|
||||
{
|
||||
publish_observational_snapshot(&updates, data_usage_info).await?;
|
||||
}
|
||||
let dirty_usage_clear = should_clear_dirty_usage_snapshot(
|
||||
result.is_ok(),
|
||||
|
||||
@@ -105,160 +105,6 @@ fn completed_data_usage_info_for_test(
|
||||
completed_data_usage_info(results, &expected_sources, all_buckets, true, budget_elapsed, cancelled)
|
||||
}
|
||||
|
||||
fn lkg_root_cache(bucket: &str, objects: usize, source: DataUsageCacheSource) -> DataUsageCache {
|
||||
let mut cache = completed_root_cache(bucket, objects, 10, source);
|
||||
cache.info.snapshot_complete = false;
|
||||
cache.info.next_cycle = 8;
|
||||
cache.info.leader_epoch = 3;
|
||||
cache.info.lkg_snapshot_complete = true;
|
||||
cache.info.lkg_next_cycle = Some(7);
|
||||
cache.info.lkg_last_update = cache.info.last_update;
|
||||
cache.info.lkg_leader_epoch = Some(3);
|
||||
cache.info.lkg_scan_plan_digest = Some(TEST_PLAN_DIGEST);
|
||||
cache
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn partial_usage_is_observational_not_authoritative_for_quota() {
|
||||
let all_buckets = vec!["bucket".to_string()];
|
||||
let current_source = DataUsageCacheSource::new(0, 0);
|
||||
let stalled_source = DataUsageCacheSource::new(1, 0);
|
||||
let mut current = completed_root_cache("bucket", 2, 20, current_source);
|
||||
current.info.next_cycle = 8;
|
||||
current.info.leader_epoch = 3;
|
||||
let stalled = lkg_root_cache("bucket", 1, stalled_source);
|
||||
let expected = HashSet::from([current_source, stalled_source]);
|
||||
|
||||
assert!(
|
||||
completed_data_usage_info(&[current.clone(), stalled.clone()], &expected, &all_buckets, true, false, false).is_none()
|
||||
);
|
||||
let (observed, _) = observational_data_usage_info(&[current, stalled], &expected, &all_buckets, TEST_PLAN_DIGEST, 8, 3)
|
||||
.expect("a completed set should produce an observational view");
|
||||
assert!(observed.usage_snapshot_partial);
|
||||
assert!(!observed.usage_snapshot_complete);
|
||||
assert_eq!(observed.usage_snapshot_converged, Some(false));
|
||||
assert_eq!(observed.usage_snapshot_set_states.len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lkg_scope_does_not_count_as_current_cycle_completion() {
|
||||
let source = DataUsageCacheSource::new(0, 0);
|
||||
let mut lkg = lkg_root_cache("bucket", 1, source);
|
||||
lkg.info.last_update = None;
|
||||
let expected = HashSet::from([source]);
|
||||
assert!(!scanner_results_form_complete_snapshot(&[lkg], &expected));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stale_quota_uses_complete_baseline_plus_positive_deltas() {
|
||||
let all_buckets = vec!["bucket".to_string()];
|
||||
let source = DataUsageCacheSource::new(0, 0);
|
||||
let mut current = completed_root_cache("bucket", 3, 20, source);
|
||||
current.info.next_cycle = 8;
|
||||
current.info.leader_epoch = 3;
|
||||
let expected = HashSet::from([source]);
|
||||
let (observed, _) = observational_data_usage_info(&[current], &expected, &all_buckets, TEST_PLAN_DIGEST, 8, 3)
|
||||
.expect("complete set data is a valid observational baseline");
|
||||
assert_eq!(observed.objects_total_size, 30);
|
||||
assert_eq!(observed.usage_snapshot_set_states[0].complete, true);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn negative_delta_waits_for_set_reconciliation() {
|
||||
let all_buckets = vec!["bucket".to_string()];
|
||||
let source = DataUsageCacheSource::new(0, 0);
|
||||
let mut stalled = lkg_root_cache("bucket", 4, source);
|
||||
stalled.info.lkg_scan_plan_digest = Some(DataUsageScanPlanDigest([9; 32]));
|
||||
let expected = HashSet::from([source]);
|
||||
assert!(observational_data_usage_info(&[stalled], &expected, &all_buckets, TEST_PLAN_DIGEST, 8, 3).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn set_membership_add_remove_uses_generation_and_tombstone() {
|
||||
let state = DataUsageSnapshotSetState {
|
||||
pool_index: 1,
|
||||
set_index: 2,
|
||||
scanner_cycle: Some(9),
|
||||
scanner_epoch: Some(4),
|
||||
scan_plan_digest: Some(TEST_PLAN_DIGEST.0),
|
||||
complete: false,
|
||||
tombstone: true,
|
||||
};
|
||||
let encoded = serde_json::to_vec(&state).expect("set state should serialize");
|
||||
let decoded: DataUsageSnapshotSetState = serde_json::from_slice(&encoded).expect("set state should deserialize");
|
||||
assert_eq!(decoded, state);
|
||||
|
||||
let snapshot = DataUsageInfo {
|
||||
last_update: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(10)),
|
||||
scanner_cycle: Some(9),
|
||||
scanner_epoch: Some(4),
|
||||
buckets_count: 0,
|
||||
usage_snapshot_converged: Some(false),
|
||||
usage_snapshot_partial: true,
|
||||
usage_snapshot_set_states: vec![
|
||||
DataUsageSnapshotSetState {
|
||||
pool_index: 0,
|
||||
set_index: 0,
|
||||
scanner_cycle: Some(9),
|
||||
scanner_epoch: Some(4),
|
||||
scan_plan_digest: Some(TEST_PLAN_DIGEST.0),
|
||||
complete: true,
|
||||
tombstone: false,
|
||||
},
|
||||
state,
|
||||
],
|
||||
..Default::default()
|
||||
};
|
||||
assert!(snapshot.is_valid_partial_snapshot());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn old_set_completion_cannot_overwrite_new_aggregate() {
|
||||
let all_buckets = vec!["bucket".to_string()];
|
||||
let source = DataUsageCacheSource::new(0, 0);
|
||||
let mut old = completed_root_cache("bucket", 1, 20, source);
|
||||
old.info.next_cycle = 7;
|
||||
old.info.leader_epoch = 2;
|
||||
let expected = HashSet::from([source]);
|
||||
assert!(observational_data_usage_info(&[old], &expected, &all_buckets, TEST_PLAN_DIGEST, 8, 3).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn usage_aggregate_survives_restart_and_leader_failover() {
|
||||
let all_buckets = vec!["bucket".to_string()];
|
||||
let source = DataUsageCacheSource::new(0, 0);
|
||||
let mut lkg = lkg_root_cache("bucket", 5, source);
|
||||
lkg.info.lkg_leader_epoch = Some(4);
|
||||
lkg.info.lkg_next_cycle = Some(9);
|
||||
let expected = HashSet::from([source]);
|
||||
let (observed, _) = observational_data_usage_info(&[lkg], &expected, &all_buckets, TEST_PLAN_DIGEST, 10, 5)
|
||||
.expect("compatible LKG should survive a leader change");
|
||||
assert_eq!(observed.usage_snapshot_set_states[0].scanner_epoch, Some(4));
|
||||
assert_eq!(observed.objects_total_size, 50);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn usage_aggregate_cost_is_linear_in_set_count() {
|
||||
let all_buckets = vec!["bucket".to_string()];
|
||||
let mut results = Vec::new();
|
||||
let mut expected = HashSet::new();
|
||||
for index in 0..32 {
|
||||
let source = DataUsageCacheSource::new(index, 0);
|
||||
expected.insert(source);
|
||||
let mut cache = completed_root_cache("bucket", 1, 20, source);
|
||||
cache.info.next_cycle = 8;
|
||||
cache.info.leader_epoch = 3;
|
||||
results.push(cache);
|
||||
}
|
||||
let (observed, _) = observational_data_usage_info(&results, &expected, &all_buckets, TEST_PLAN_DIGEST, 8, 3)
|
||||
.expect("all set snapshots should aggregate");
|
||||
assert_eq!(observed.objects_total_count, 32);
|
||||
let reversed = results.iter().rev().cloned().collect::<Vec<_>>();
|
||||
let (reversed_observed, _) = observational_data_usage_info(&reversed, &expected, &all_buckets, TEST_PLAN_DIGEST, 8, 3)
|
||||
.expect("reordered set snapshots should aggregate");
|
||||
assert_eq!(observed.usage_snapshot_set_states, reversed_observed.usage_snapshot_set_states);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn completed_data_usage_info_publishes_tier_stats_across_sets() {
|
||||
let all_buckets = vec!["bucket-a".to_string(), "bucket-b".to_string()];
|
||||
|
||||
@@ -43,9 +43,6 @@ allow-git = [
|
||||
# RustFS fork carrying presigned expiry and constant-time authentication fixes.
|
||||
# owner: rustfs-maintainers review: 2026-10
|
||||
"https://github.com/rustfs/s3s.git",
|
||||
# MiMalloc fork pinned for hotpath allocation counting support.
|
||||
# owner: houseme review: 2026-10
|
||||
"https://github.com/xonatius/mimalloc_rust.git",
|
||||
]
|
||||
|
||||
[bans]
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
# CI gate matrix
|
||||
|
||||
This file is the source of truth for which validation runs on each event, its
|
||||
configured wall-clock budget, and whether it can block a merge. Test taxonomy,
|
||||
naming, and nextest serialization rules remain in [README.md](README.md); e2e
|
||||
membership and counts remain in
|
||||
[e2e-suite-inventory.md](e2e-suite-inventory.md).
|
||||
|
||||
The distinction between **required** and **report-only** is load-bearing:
|
||||
a failing job blocks a merge only when its exact check name is present in the
|
||||
live `main` ruleset. A workflow name, a `merge_group` trigger, or a red PR check
|
||||
does not make a job required by itself.
|
||||
|
||||
## Required merge checks
|
||||
|
||||
The live `main` ruleset (`6436880`) currently requires exactly these contexts:
|
||||
|
||||
| Required context | Producer | Validation |
|
||||
|---|---|---|
|
||||
| `CLA Check` | `.github/workflows/cla.yml` | Contributor agreement |
|
||||
| `Quick Checks` | `.github/workflows/ci.yml` | Formatting and repository guard scripts |
|
||||
| `Test and Lint` | `.github/workflows/ci.yml` | Clippy, workspace nextest excluding `e2e_test`, doctests, and migration proofs |
|
||||
|
||||
For pull requests limited to the paths excluded by the main CI workflow,
|
||||
`.github/workflows/ci-docs-only.yml` reports `Quick Checks` and
|
||||
`Test and Lint` under the same names. It runs the real quick checks and the
|
||||
planning-document guard; it does not claim that Rust compilation or runtime
|
||||
tests ran. Despite the workflow name, these paths also include selected deploy,
|
||||
workflow, and lock files.
|
||||
|
||||
Verify the live rule rather than trusting this snapshot before changing merge
|
||||
policy:
|
||||
|
||||
```bash
|
||||
gh api repos/rustfs/rustfs/rulesets/6436880 \
|
||||
--jq '.rules[] | select(.type == "required_status_checks") | .parameters'
|
||||
```
|
||||
|
||||
The ruleset currently has `strict_required_status_checks_policy=false`.
|
||||
`Continuous Integration` accepts `merge_group` events and runs `e2e-full` for
|
||||
them, but `End-to-End Tests (full merge gate)` is not currently a required
|
||||
context. Therefore the repository is prepared to test a merge-queue SHA, but
|
||||
the workflow alone does not prove that every merge passed that lane.
|
||||
|
||||
## Pull request and merge matrix
|
||||
|
||||
Budgets below are job `timeout-minutes`, not typical runtimes. “Report-only”
|
||||
means the result is visible and actionable but is not in the live required
|
||||
context list.
|
||||
|
||||
| Event | Validation | Budget | Merge status | Reproduction |
|
||||
|---|---|---:|---|---|
|
||||
| PR, non-doc change | `Quick Checks` | 10 min | Required | `make pre-commit` (broader local umbrella) |
|
||||
| PR, non-doc change | `Test and Lint` | 90 min | Required | `cargo nextest run --profile ci --all --exclude e2e_test` |
|
||||
| PR, non-doc change | `Typos` | 10 min | Report-only | `typos` |
|
||||
| PR, non-doc change | `ILM Integration (serial)` | 90 min | Report-only | Use the exact command in `.github/workflows/ci.yml` |
|
||||
| PR, non-doc change | rio-v2 / swift / sftp test-and-lint variants | 90 min each | Report-only | `cargo nextest run` with the workflow's feature set |
|
||||
| PR, non-doc change | `Build RustFS Debug Binary` | 30 min | Report-only; prerequisite for black-box lanes | `cargo build -p rustfs --bins` |
|
||||
| PR, non-doc change | `io_uring Integration (real)` | 30 min | Report-only | `cargo test -p rustfs-ecstore --lib uring_ -- --test-threads=1 --nocapture` |
|
||||
| PR, non-doc change | `End-to-End Tests` (`e2e-smoke` plus `s3s-e2e`) | 30 min | Report-only | `cargo nextest run --profile e2e-smoke -p e2e_test`; then `./scripts/e2e-run.sh ./target/debug/rustfs <data-dir>` |
|
||||
| PR, non-doc change | `S3 Implemented Tests` | 60 min | Report-only | Build `rustfs`, then run `scripts/s3-tests/run.sh` with `DEPLOY_MODE=binary`, `TEST_MODE=single`, and `MAXFAIL=0` |
|
||||
| PR, non-doc change | `S3 Lifecycle Behavior Tests` | 30 min | Report-only | Use the accelerated scanner environment in `.github/workflows/ci.yml` with `scripts/s3-tests/run.sh` |
|
||||
| PR touching dependency or workflow inputs | Cargo Deny / Workflow Pin Report / Dependency Review | 20 / 5 / 30 min | Report-only | `cargo deny check`; `scripts/security/check_workflow_pins.sh` |
|
||||
| PR touching architecture rules or architecture docs | `Architecture Migration Rules` | 10 min | Report-only | `scripts/check_architecture_migration_rules.sh` |
|
||||
| PR touching Nix or workspace manifests | `Nix Build & Check` | 60 min | Report-only | `nix flake check` |
|
||||
| PR limited to main-CI-excluded paths | companion `Quick Checks` and `Test and Lint` | 10 min each | Required | `git diff --check`; `make doc-paths-check` when documentation paths changed |
|
||||
| `merge_group` | Standard CI plus `e2e-full` | 55 min for `e2e-full` | Standard required contexts only; `e2e-full` report-only | `cargo nextest run --profile e2e-full -p e2e_test` |
|
||||
| Push to `main` | Standard CI plus `e2e-full` | 55 min for `e2e-full` | Post-merge detection | Same as `merge_group` |
|
||||
| PR touching fuzz inputs or harness paths | Build plus five 60-second fuzz smoke targets | 60 min build; 30 min per target | Report-only | `MAX_TOTAL_TIME=60 ./scripts/fuzz/run.sh` |
|
||||
| PR touching selected ecstore disk/format paths | `Rename Safety` on Windows | 60 min | Report-only | Run the four `cargo test -p rustfs-ecstore --lib <filter>` commands in `windows-filesystem.yml` on Windows |
|
||||
|
||||
The authoritative e2e filters live in `.config/nextest.toml`; extend a profile
|
||||
instead of adding a second ad-hoc selector. Before a profile runs,
|
||||
`scripts/check_test_wiring.py` compares its exact membership to the committed
|
||||
digest so a silent test drop fails closed.
|
||||
|
||||
## Scheduled and manual validation
|
||||
|
||||
Scheduled lanes are independent fault domains. They do not block a pull
|
||||
request, but their workflow-local gate can fail the run and scheduled failures
|
||||
are routed to the shared failure-issue action. The scheduled-validation
|
||||
watchdog and freshness workflow separately detect incomplete runs and missing
|
||||
schedules.
|
||||
|
||||
| Cadence (UTC unless noted) | Workflow / validation | Budget | Verdict and artifacts | Reproduction |
|
||||
|---|---|---:|---|---|
|
||||
| Daily 02:17 | Fuzz: five nightly corpus targets | 60 min build; 60 min per target | Gate; corpus/crash artifacts, scheduled failure alert | `MAX_TOTAL_TIME=<seconds> ./scripts/fuzz/run.sh` |
|
||||
| Daily 03:17 | MinIO interop (EC + SSE read parity) | 40 min | Gate; scheduled failure alert | Dispatch `minio-interop.yml` or follow its pinned Docker fixture steps |
|
||||
| Daily 04:29 | Replication / cluster-fault / protocol e2e | 45 / 90 / 90 min | Three independent gates; JUnit, membership, and server logs | `cargo nextest run --profile e2e-repl-nightly -p e2e_test`; `--profile e2e-nightly`; `-j 1 --profile e2e-protocols` |
|
||||
| Daily 06:31 | Warp performance A/B | 180 min | Regression budget gate; A/B summaries and server logs | `bash scripts/run_hotpath_warp_abba.sh --help` |
|
||||
| Daily 00:07 Asia/Shanghai (16:07 UTC previous day) | Nightly GNU build and Vault lanes | 150 / 90 / 60 min | Build, live Vault, and HA failover gates | Use the commands and pinned Vault images in `nightly-gnu.yml` |
|
||||
| Daily 03:23 | Security Audit | 20 / 5 min, plus 30 min on PR dependency review | Cargo Deny and workflow-pin gates; scheduled failure alert | `cargo deny check`; `scripts/security/check_workflow_pins.sh` |
|
||||
| Daily 23:47 | Scheduled Validation Freshness | 10 min | Fails when a critical schedule was never created or is stale | Dispatch `scheduled-validation-freshness.yml` |
|
||||
| Sunday 00:11 | Full `Continuous Integration` matrix | Per-job budgets above | Weekly variant coverage, including dormant rio-v2 binary/e2e lanes | Dispatch `ci.yml` |
|
||||
| Sunday 01:13 | Seven-platform build matrix | 150 min per platform | Build/package integrity; scheduled failure alert | Dispatch `build.yml` with an exact platform set |
|
||||
| Sunday 02:19 | Ceph s3-tests full sweep: single and real four-node, four shards each | 180 min per shard | Compatibility gate; report, JUnit, exact node IDs, and server logs | `scripts/s3-tests/run.sh` against an existing single or distributed target |
|
||||
| Sunday 06:41 | Mint | 120 min | **Report-only by design**; per-suite PASS/FAIL/NA and raw `log.json` | Reproduce the pinned Docker sequence in `mint.yml` or dispatch it |
|
||||
| Sunday 07:43 | Workspace line coverage | 120 min | Report-only trend; lcov and JSON retained 90 days | `make coverage` |
|
||||
| Monthly, day 1 06:37 | Runner Hygiene | 15 min | Validates runner ephemerality; scheduled failure alert | Dispatch `runner-hygiene.yml` |
|
||||
|
||||
Manual `workflow_dispatch` exists for the scheduled workflows above. Manual
|
||||
runs are debugging evidence and intentionally do not open scheduled-failure
|
||||
issues. A manual performance run may explicitly allow a known regression; that
|
||||
override must not be treated as an ordinary passing baseline.
|
||||
|
||||
## Release validation
|
||||
|
||||
Release validation is post-merge and tag-driven; it does not substitute for a
|
||||
pull-request gate.
|
||||
|
||||
| Event | Validation | Budget | Result |
|
||||
|---|---|---:|---|
|
||||
| Push to `main` or weekly schedule | `Build and Release` platform matrix | 150 min per platform | Build artifacts for all selected targets; no release publication on a main push |
|
||||
| Valid release or preview tag | `Build and Release` plus asset checks | 150 min per platform | Draft release, checksummed assets, and publish step |
|
||||
| Successful non-preview release-tag build | Docker image build and image scan | 60 min build; 30 min scan | Multi-architecture images plus vulnerability report |
|
||||
| Successful release-tag build | DEB/RPM packaging | 30 min per architecture | Packages and checksum files uploaded to the release |
|
||||
| Successful non-preview release-tag build | Helm template test and package | 30 min build; 30 min publish | Versioned chart and repository index |
|
||||
|
||||
Use an exact preview tag for end-to-end release rehearsal. Manual dispatches
|
||||
are backfill/debug paths and do not prove the automatic `workflow_run` chain.
|
||||
|
||||
## Evidence requirements
|
||||
|
||||
A green check is useful only when it proves the intended behavior ran:
|
||||
|
||||
- Record the exact commit SHA and run URL.
|
||||
- Separate product failure from runner prerequisites, service readiness, and
|
||||
cancellation. Repair the precondition, then rerun the exact workload.
|
||||
- Preserve membership manifests, JUnit, raw compatibility logs, seeds, and
|
||||
server logs where the workflow provides them.
|
||||
- For a bug fix or a new fault checker, provide sensitivity evidence: the old
|
||||
behavior or an intentional mutation must fail the new oracle, and the fixed
|
||||
behavior must pass it.
|
||||
- Never promote a report-only lane to required from one green run. Require at
|
||||
least 14 days and 30 representative pull requests with at least 99% complete
|
||||
execution, then update the ruleset and this table together.
|
||||
|
||||
## Change checklist
|
||||
|
||||
Update this file in the same pull request when any of these change:
|
||||
|
||||
- workflow triggers, job names, timeouts, or nextest profile ownership;
|
||||
- required status contexts or strict/merge-queue policy;
|
||||
- scheduled cadence, alert routing, artifact contract, or local reproduction;
|
||||
- report-only versus gating semantics.
|
||||
|
||||
Do not copy per-module test counts here. Update
|
||||
[e2e-suite-inventory.md](e2e-suite-inventory.md) and its enforced membership
|
||||
digest instead.
|
||||
+2
-2
@@ -336,13 +336,13 @@ opentelemetry = { workspace = true }
|
||||
tracing-opentelemetry = { workspace = true }
|
||||
# Data structures
|
||||
hashbrown = { workspace = true, features = ["serde", "rayon"] }
|
||||
mimalloc = { workspace = true }
|
||||
rustfs-mimalloc = { workspace = true }
|
||||
|
||||
[target.'cfg(target_os = "linux")'.dependencies]
|
||||
libsystemd.workspace = true
|
||||
|
||||
[target.'cfg(not(target_os = "windows"))'.dependencies]
|
||||
libmimalloc-sys.workspace = true
|
||||
rustfs-mimalloc-sys.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
uuid = { workspace = true, features = ["v4", "v5", "fast-rng", "macro-diagnostics"] }
|
||||
|
||||
@@ -31,6 +31,10 @@ use crate::admin::storage_api::bucket::metadata::{
|
||||
use crate::admin::storage_api::bucket::metadata_sys;
|
||||
use crate::admin::storage_api::bucket::quota::BucketQuota;
|
||||
use crate::admin::storage_api::bucket::replication;
|
||||
use crate::admin::storage_api::bucket::replication::{
|
||||
assign_site_replication_rule_priorities, is_site_replication_rule, merge_incoming_replication_config,
|
||||
replication_target_arn_deployment_id,
|
||||
};
|
||||
use crate::admin::storage_api::bucket::target::{ARN, BucketTarget, BucketTargetType, BucketTargets, Credentials};
|
||||
use crate::admin::storage_api::bucket::target_sys::BucketTargetSys;
|
||||
use crate::admin::storage_api::bucket::utils::{deserialize, serialize};
|
||||
@@ -1118,6 +1122,36 @@ async fn load_site_replication_state() -> S3Result<SiteReplicationState> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether this deployment participates in site replication (two or more
|
||||
/// peers in the persisted state). Read by the S3 interface layer to gate
|
||||
/// replication-config edits (MinIO `ErrReplicationDenyEditError` semantics,
|
||||
/// issue #1948); a state-read failure propagates so the gate fails closed.
|
||||
pub(crate) async fn site_replication_enabled() -> S3Result<bool> {
|
||||
Ok(load_site_replication_state().await?.enabled())
|
||||
}
|
||||
|
||||
/// Deployment ids of the remote peers the reconciler derives a
|
||||
/// `site-repl-<id>` rule for on every bucket (the same peer filter as
|
||||
/// `build_site_replication_config`); empty when site replication is not
|
||||
/// enabled. Read by the bucket usecase so an S3 replication-config edit keeps
|
||||
/// exactly the reconciler-owned rules (issue #1948); a state-read failure
|
||||
/// propagates so the edit fails closed.
|
||||
pub(crate) async fn site_replication_remote_peer_deployment_ids() -> S3Result<HashSet<String>> {
|
||||
let state = load_site_replication_state().await?;
|
||||
if !state.enabled() {
|
||||
return Ok(HashSet::new());
|
||||
}
|
||||
let local_peer = current_local_runtime_peer(&state);
|
||||
Ok(state
|
||||
.peers
|
||||
.values()
|
||||
.filter(|peer| {
|
||||
peer.deployment_id != local_peer.deployment_id && !same_identity_endpoint(&peer.endpoint, &local_peer.endpoint)
|
||||
})
|
||||
.map(|peer| peer.deployment_id.clone())
|
||||
.collect())
|
||||
}
|
||||
|
||||
async fn load_site_replication_state_no_lock(store: Arc<ECStore>) -> S3Result<SiteReplicationState> {
|
||||
match read_config_no_lock(store, SITE_REPLICATION_STATE_PATH).await {
|
||||
Ok(data) => parse_site_replication_state(&data),
|
||||
@@ -7762,20 +7796,6 @@ fn bucket_target_deployment_id(target: &BucketTarget) -> Option<String> {
|
||||
replication_target_arn_deployment_id(&target.arn)
|
||||
}
|
||||
|
||||
fn replication_target_arn_deployment_id(arn: &str) -> Option<String> {
|
||||
let parts: Vec<_> = arn.split(':').collect();
|
||||
if parts.len() == 6
|
||||
&& parts[0] == "arn"
|
||||
&& matches!(parts[1], "rustfs" | "minio")
|
||||
&& parts[2] == "replication"
|
||||
&& !parts[4].is_empty()
|
||||
{
|
||||
return Some(parts[4].to_string());
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
fn prune_removed_site_replication_bucket_targets(
|
||||
existing: BucketTargets,
|
||||
removed_deployment_ids: &HashSet<String>,
|
||||
@@ -7800,10 +7820,6 @@ fn prune_removed_site_replication_bucket_targets(
|
||||
(BucketTargets { targets }, removed)
|
||||
}
|
||||
|
||||
fn is_site_replication_rule(rule: &ReplicationRule) -> bool {
|
||||
rule.id.as_deref().is_some_and(|id| id.starts_with("site-repl-"))
|
||||
}
|
||||
|
||||
/// Whether every `site-repl-*` rule on this bucket resolves to a live remote target.
|
||||
///
|
||||
/// The rule set alone cannot answer this: a rule can be perfectly formed while the endpoint
|
||||
@@ -7829,52 +7845,6 @@ async fn site_replication_targets_online(bucket: &str, replication_config_xml: &
|
||||
true
|
||||
}
|
||||
|
||||
/// Merge a peer's replication config into the local one.
|
||||
///
|
||||
/// `site-repl-*` rules encode the *sender's* outbound direction — their destination ARN
|
||||
/// names the receiver — so applying a peer's rule set verbatim replaces the receiver's
|
||||
/// reverse rule with one pointing at itself. No bucket target can satisfy that ARN
|
||||
/// (`reconcile_site_replication_bucket_targets` skips the local peer), so the receiver
|
||||
/// silently stops replicating back: the one-directional symptom. Only operator-authored
|
||||
/// rules travel between sites; each site owns its own `site-repl-*` rules.
|
||||
fn merge_incoming_replication_config(
|
||||
incoming: Option<ReplicationConfiguration>,
|
||||
local: Option<ReplicationConfiguration>,
|
||||
) -> Option<ReplicationConfiguration> {
|
||||
let incoming_role = incoming.as_ref().map(|config| config.role.clone()).unwrap_or_default();
|
||||
// Operator rules first, then the local site rules — the same order
|
||||
// `ensure_site_replication_bucket_replication_config_with_runtime` produces, so its
|
||||
// no-op check matches and the bucket metadata is written once per broadcast, not twice.
|
||||
let mut rules: Vec<ReplicationRule> = incoming
|
||||
.into_iter()
|
||||
.flat_map(|config| config.rules)
|
||||
.filter(|rule| !is_site_replication_rule(rule))
|
||||
.collect();
|
||||
rules.extend(
|
||||
local
|
||||
.into_iter()
|
||||
.flat_map(|config| config.rules)
|
||||
.filter(is_site_replication_rule),
|
||||
);
|
||||
|
||||
if rules.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
for (index, rule) in rules.iter_mut().enumerate() {
|
||||
rule.priority = Some(i32::try_from(index + 1).unwrap_or(i32::MAX));
|
||||
}
|
||||
|
||||
// A site-replication ARN in `role` is the sender's, and `site_replication_target_arns_by_peer`
|
||||
// reads it — carrying it over would pin the receiver's targets to the sender's identity.
|
||||
let role = match replication_target_arn_deployment_id(&incoming_role) {
|
||||
Some(_) => String::new(),
|
||||
None => incoming_role,
|
||||
};
|
||||
|
||||
Some(ReplicationConfiguration { role, rules })
|
||||
}
|
||||
|
||||
/// Merge a peer's ILM expiry document into the local lifecycle config.
|
||||
///
|
||||
/// Mirrors MinIO's `mergeWithCurrentLCConfig` with one hardening: incoming
|
||||
@@ -8213,9 +8183,7 @@ fn prune_removed_site_replication_rules(
|
||||
return (None, removed);
|
||||
}
|
||||
|
||||
for (index, rule) in config.rules.iter_mut().enumerate() {
|
||||
rule.priority = Some(i32::try_from(index + 1).unwrap_or(i32::MAX));
|
||||
}
|
||||
assign_site_replication_rule_priorities(&mut config.rules, is_site_replication_rule);
|
||||
|
||||
(Some(config), removed)
|
||||
}
|
||||
@@ -8389,9 +8357,10 @@ async fn ensure_site_replication_bucket_replication_config_with_runtime(
|
||||
.cloned()
|
||||
.collect();
|
||||
rules.extend(desired.rules);
|
||||
for (index, rule) in rules.iter_mut().enumerate() {
|
||||
rule.priority = Some(i32::try_from(index + 1).unwrap_or(i32::MAX));
|
||||
}
|
||||
// Operator priorities are the operator's policy; only the derived rules
|
||||
// take free slots, by the same function as the config merges so a merged
|
||||
// write and this pass agree byte for byte.
|
||||
assign_site_replication_rule_priorities(&mut rules, is_site_replication_rule);
|
||||
|
||||
// Only a site-replication ARN in `role` is ours to drop — an operator-authored role is
|
||||
// part of the bucket's S3-visible configuration, and repairing a reverse rule must not
|
||||
@@ -17089,7 +17058,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_prune_removed_site_replication_rules_removes_site_rule_and_reorders_priorities() {
|
||||
fn test_prune_removed_site_replication_rules_removes_site_rule_and_keeps_operator_priority() {
|
||||
let removed_deployment_ids = HashSet::from(["removed-dep".to_string()]);
|
||||
let kept_rule = build_site_replication_rule("arn:rustfs:replication::kept-dep:photos", 3, "site-repl-kept-dep");
|
||||
let removed_rule = build_site_replication_rule("arn:rustfs:replication::removed-dep:photos", 1, "site-repl-removed-dep");
|
||||
@@ -17106,9 +17075,9 @@ mod tests {
|
||||
assert!(updated.role.is_empty());
|
||||
assert_eq!(updated.rules.len(), 2);
|
||||
assert_eq!(updated.rules[0].id.as_deref(), Some("user-managed-rule"));
|
||||
assert_eq!(updated.rules[0].priority, Some(1));
|
||||
assert_eq!(updated.rules[0].priority, Some(9), "the operator's priority is policy and stays");
|
||||
assert_eq!(updated.rules[1].id.as_deref(), Some("site-repl-kept-dep"));
|
||||
assert_eq!(updated.rules[1].priority, Some(2));
|
||||
assert_eq!(updated.rules[1].priority, Some(1), "the derived rule moves to the lowest free slot");
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -443,6 +443,8 @@ pub(crate) mod replication {
|
||||
pub(crate) use super::ecstore_bucket::replication::{
|
||||
REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS,
|
||||
REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS,
|
||||
assign_site_replication_rule_priorities, is_site_replication_rule, merge_incoming_replication_config,
|
||||
replication_target_arn_deployment_id,
|
||||
};
|
||||
pub(crate) type BucketReplicationResyncStatus = super::ecstore_bucket::replication::BucketReplicationResyncStatus;
|
||||
pub(crate) type BucketStats = super::ecstore_bucket::replication::BucketStats;
|
||||
|
||||
@@ -369,14 +369,8 @@ pub fn allocator_reclaim_controller_snapshot(ctx: &CancellationToken) -> Allocat
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "windows"))]
|
||||
#[allow(unsafe_code)]
|
||||
fn collect_allocator_memory(force: bool) -> Result<(), String> {
|
||||
// SAFETY: `mi_collect` is provided by the active global allocator backend
|
||||
// on this target family. It is explicitly intended to reclaim retained
|
||||
// pages/segments and does not require additional invariants from the caller.
|
||||
unsafe {
|
||||
libmimalloc_sys::mi_collect(force);
|
||||
}
|
||||
rustfs_mimalloc::MiMalloc::collect(force);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -38,9 +38,9 @@ use super::storage_api::bucket_usecase::bucket::{
|
||||
metadata_sys,
|
||||
policy_sys::PolicySys,
|
||||
replication::{
|
||||
ReplicationTargetValidationError, invalid_replication_config_status_field, replication_target_arns,
|
||||
should_remove_replication_target, unsupported_replication_config_field, validate_replication_config_structure,
|
||||
validate_replication_config_target_arns,
|
||||
ReplicationTargetValidationError, invalid_replication_config_status_field, merge_user_replication_config,
|
||||
replication_target_arns, should_remove_replication_target, unsupported_replication_config_field,
|
||||
validate_replication_config_structure, validate_replication_config_target_arns,
|
||||
},
|
||||
target::{BucketTargetType, BucketTargets},
|
||||
utils::serialize,
|
||||
@@ -623,11 +623,52 @@ async fn validate_bucket_replication_update(bucket: &str, config: &ReplicationCo
|
||||
validate_replication_config_targets(&targets, config)
|
||||
}
|
||||
|
||||
async fn replication_targets_without_config_targets(
|
||||
/// Defense in depth for site-replication-managed buckets (issue #1948): an S3
|
||||
/// PutBucketReplication replaces the operator-authored rules but must not wipe
|
||||
/// the rules the reconciler derived for the current remote peers
|
||||
/// (`site_peer_deployment_ids`) — until its next pass (600s period) every
|
||||
/// peer link on this bucket would be silently dead. The same merge also drops
|
||||
/// incoming impostors of those rules. An empty peer set (site replication
|
||||
/// disabled) keeps the verbatim overwrite semantics: rule ids are not
|
||||
/// reserved, so an operator's own `site-repl-*` rule is ordinary state there.
|
||||
fn merge_user_replication_config_update(
|
||||
incoming: ReplicationConfiguration,
|
||||
existing: Option<ReplicationConfiguration>,
|
||||
site_peer_deployment_ids: &HashSet<String>,
|
||||
) -> ReplicationConfiguration {
|
||||
if site_peer_deployment_ids.is_empty() {
|
||||
return incoming;
|
||||
}
|
||||
// `incoming` passed structure validation, so it holds at least one rule;
|
||||
// `None` is only reachable when every incoming rule impersonates a
|
||||
// reconciler rule, and then the stored reconciler rules are what remains.
|
||||
merge_user_replication_config(Some(incoming.clone()), existing, site_peer_deployment_ids).unwrap_or(incoming)
|
||||
}
|
||||
|
||||
/// Split of an S3 DeleteBucketReplication on the stored config (issue #1948):
|
||||
/// the operator-authored rules are removed, the rules the reconciler derived
|
||||
/// for the current remote peers survive (`None` means nothing survives and
|
||||
/// the config is deleted), and the returned ARNs are the ones whose bucket
|
||||
/// targets may be garbage-collected — never an ARN a surviving reconciler
|
||||
/// rule still points at.
|
||||
fn split_replication_config_for_user_delete(
|
||||
config: ReplicationConfiguration,
|
||||
site_peer_deployment_ids: &HashSet<String>,
|
||||
) -> (Option<ReplicationConfiguration>, HashSet<String>) {
|
||||
let mut removable_arns = replication_target_arns(&config);
|
||||
let remaining = merge_user_replication_config(None, Some(config), site_peer_deployment_ids);
|
||||
if let Some(remaining) = remaining.as_ref() {
|
||||
for rule in &remaining.rules {
|
||||
removable_arns.remove(rule.destination.bucket.trim());
|
||||
}
|
||||
}
|
||||
(remaining, removable_arns)
|
||||
}
|
||||
|
||||
async fn replication_targets_without_arns(
|
||||
bucket: &str,
|
||||
config: &ReplicationConfiguration,
|
||||
target_arns: &HashSet<String>,
|
||||
) -> S3Result<Option<(BucketTargets, usize)>> {
|
||||
let target_arns = replication_target_arns(config);
|
||||
if target_arns.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
@@ -638,7 +679,7 @@ async fn replication_targets_without_config_targets(
|
||||
Err(err) => return Err(ApiError::from(err).into()),
|
||||
};
|
||||
|
||||
let removed = remove_replication_targets_from_config_targets(&mut targets, &target_arns);
|
||||
let removed = remove_replication_targets_from_config_targets(&mut targets, target_arns);
|
||||
if removed == 0 {
|
||||
return Ok(None);
|
||||
}
|
||||
@@ -1582,9 +1623,15 @@ impl DefaultBucketUsecase {
|
||||
Ok(S3Response::new(DeleteBucketPolicyOutput {}))
|
||||
}
|
||||
|
||||
/// `site_peers` is the set of remote site-replication peer deployment ids
|
||||
/// (empty when site replication is disabled). The interface layer reads it
|
||||
/// from the persisted state and fails closed on a read error, so this
|
||||
/// usecase stays a pure function of its inputs (layer rule: app never
|
||||
/// imports interface).
|
||||
pub async fn execute_delete_bucket_replication(
|
||||
&self,
|
||||
req: S3Request<DeleteBucketReplicationInput>,
|
||||
site_peers: HashSet<String>,
|
||||
) -> S3Result<S3Response<DeleteBucketReplicationOutput>> {
|
||||
let expected_incarnation_id = bucket_config_mutation_incarnation(&req, &req.input.bucket)?;
|
||||
let request_context = req.extensions.get::<request_context::RequestContext>().cloned();
|
||||
@@ -1604,15 +1651,29 @@ impl DefaultBucketUsecase {
|
||||
Err(StorageError::ConfigNotFound) => None,
|
||||
Err(err) => return Err(ApiError::from(err).into()),
|
||||
};
|
||||
let updated_targets = if let Some(config) = replication_config.as_ref() {
|
||||
replication_targets_without_config_targets(&bucket, config).await?
|
||||
let (remaining_config, updated_targets) = if let Some(config) = replication_config.as_ref() {
|
||||
let (remaining, removable_arns) = split_replication_config_for_user_delete(config.clone(), &site_peers);
|
||||
let targets = replication_targets_without_arns(&bucket, &removable_arns).await?;
|
||||
(remaining, targets)
|
||||
} else {
|
||||
None
|
||||
(None, None)
|
||||
};
|
||||
|
||||
delete_bucket_config_for_incarnation(&bucket, BUCKET_REPLICATION_CONFIG, expected_incarnation_id)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
match remaining_config {
|
||||
// Site-replication rules and the targets backing them survive the
|
||||
// S3 delete (issue #1948); only the operator-authored rules go.
|
||||
Some(remaining) => {
|
||||
let data = serialize_config(&remaining)?;
|
||||
update_bucket_config_for_incarnation(&bucket, BUCKET_REPLICATION_CONFIG, data, expected_incarnation_id)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
}
|
||||
None => {
|
||||
delete_bucket_config_for_incarnation(&bucket, BUCKET_REPLICATION_CONFIG, expected_incarnation_id)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
}
|
||||
}
|
||||
if let Some((targets, removed)) = updated_targets
|
||||
&& let Err(err) =
|
||||
write_replication_targets_after_config_delete(&bucket, &targets, removed, expected_incarnation_id).await
|
||||
@@ -2459,9 +2520,11 @@ impl DefaultBucketUsecase {
|
||||
Ok(S3Response::new(PutBucketCorsOutput::default()))
|
||||
}
|
||||
|
||||
/// See [`Self::execute_delete_bucket_replication`] for `site_peers`.
|
||||
pub async fn execute_put_bucket_replication(
|
||||
&self,
|
||||
req: S3Request<PutBucketReplicationInput>,
|
||||
site_peers: HashSet<String>,
|
||||
) -> S3Result<S3Response<PutBucketReplicationOutput>> {
|
||||
let expected_incarnation_id = bucket_config_mutation_incarnation(&req, &req.input.bucket)?;
|
||||
let request_context = req.extensions.get::<request_context::RequestContext>().cloned();
|
||||
@@ -2485,6 +2548,13 @@ impl DefaultBucketUsecase {
|
||||
|
||||
let targets_guard = lock_bucket_targets_metadata(&bucket).await;
|
||||
validate_bucket_replication_update(&bucket, &replication_configuration).await?;
|
||||
let existing_config = match metadata_sys::get_replication_config(&bucket).await {
|
||||
Ok((config, _)) => Some(config),
|
||||
Err(StorageError::ConfigNotFound) => None,
|
||||
Err(err) => return Err(ApiError::from(err).into()),
|
||||
};
|
||||
let replication_configuration =
|
||||
merge_user_replication_config_update(replication_configuration, existing_config, &site_peers);
|
||||
let data = serialize_config(&replication_configuration)?;
|
||||
update_bucket_config_for_incarnation(&bucket, BUCKET_REPLICATION_CONFIG, data, expected_incarnation_id)
|
||||
.await
|
||||
@@ -3114,6 +3184,185 @@ mod tests {
|
||||
assert!(arns.contains(destination));
|
||||
}
|
||||
|
||||
fn replication_rule_with_id(arn: &str, id: &str, priority: i32) -> ReplicationRule {
|
||||
let mut rule = replication_rule_for_target(arn);
|
||||
rule.id = Some(id.to_string());
|
||||
rule.priority = Some(priority);
|
||||
rule
|
||||
}
|
||||
|
||||
fn site_peers(deployment_ids: &[&str]) -> HashSet<String> {
|
||||
deployment_ids.iter().map(|id| id.to_string()).collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn put_replication_merge_preserves_site_replication_rules() {
|
||||
let existing = ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![
|
||||
replication_rule_with_id("arn:rustfs:replication::peer-dep:bucket", "site-repl-peer-dep", 1),
|
||||
replication_rule_with_id("arn:rustfs:replication:us-east-1:old:bucket", "old-user-rule", 2),
|
||||
],
|
||||
};
|
||||
let incoming = ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![
|
||||
replication_rule_with_id("arn:rustfs:replication:us-east-1:new:bucket", "new-user-rule", 1),
|
||||
replication_rule_with_id("arn:rustfs:replication::forged-dep:bucket", "site-repl-peer-dep", 2),
|
||||
replication_rule_with_id("arn:rustfs:replication::other-dep:bucket", "site-repl-other", 3),
|
||||
],
|
||||
};
|
||||
|
||||
let merged = merge_user_replication_config_update(incoming, Some(existing), &site_peers(&["peer-dep"]));
|
||||
|
||||
let rules: Vec<_> = merged
|
||||
.rules
|
||||
.iter()
|
||||
.map(|rule| (rule.id.as_deref().unwrap_or_default(), rule.destination.bucket.as_str()))
|
||||
.collect();
|
||||
assert_eq!(
|
||||
rules,
|
||||
vec![
|
||||
("new-user-rule", "arn:rustfs:replication:us-east-1:new:bucket"),
|
||||
("site-repl-other", "arn:rustfs:replication::other-dep:bucket"),
|
||||
("site-repl-peer-dep", "arn:rustfs:replication::peer-dep:bucket"),
|
||||
],
|
||||
"user rules replaced, the reconciler rule for the current peer kept over the incoming impostor, \
|
||||
a site-repl-* id that names no current peer is ordinary operator state"
|
||||
);
|
||||
}
|
||||
|
||||
// Rule ids do not reserve `site-repl-*`: outside site replication an
|
||||
// owner's `site-repl-user` rule is ordinary state, so PUT stores it
|
||||
// verbatim and DELETE removes it and garbage-collects its target.
|
||||
#[test]
|
||||
fn put_then_delete_replication_without_site_replication_treats_site_repl_id_as_user_rule() {
|
||||
let user_arn = "arn:minio:replication:us-east-1:2f1c-remote:bucket";
|
||||
let incoming = ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![replication_rule_with_id(user_arn, "site-repl-user", 1)],
|
||||
};
|
||||
|
||||
let stored = merge_user_replication_config_update(incoming.clone(), None, &HashSet::new());
|
||||
assert_eq!(stored, incoming, "PUT on a non-site-replication bucket is verbatim");
|
||||
|
||||
let (remaining, removable) = split_replication_config_for_user_delete(stored, &HashSet::new());
|
||||
assert!(remaining.is_none(), "DELETE must remove the operator's site-repl-* rule");
|
||||
assert_eq!(removable, HashSet::from([user_arn.to_string()]));
|
||||
}
|
||||
|
||||
// Under site replication only a rule the reconciler would derive — id
|
||||
// `site-repl-<peer>` for a current peer, destination ARN naming the same
|
||||
// peer — is reconciler-owned. Everything else is operator state.
|
||||
#[test]
|
||||
fn delete_replication_split_keeps_only_reconciler_derived_rules() {
|
||||
let peer_arn = "arn:rustfs:replication::peer-dep:bucket";
|
||||
let user_arn = "arn:minio:replication:us-east-1:2f1c-remote:bucket";
|
||||
let config = ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![
|
||||
replication_rule_with_id(user_arn, "site-repl-user", 1),
|
||||
replication_rule_with_id(user_arn, "site-repl-peer-dep", 2),
|
||||
replication_rule_with_id("arn:rustfs:replication::gone-dep:bucket", "site-repl-gone-dep", 3),
|
||||
replication_rule_with_id(peer_arn, "site-repl-peer-dep", 4),
|
||||
],
|
||||
};
|
||||
|
||||
let (remaining, removable) = split_replication_config_for_user_delete(config, &site_peers(&["peer-dep"]));
|
||||
|
||||
let remaining = remaining.expect("the reconciler-derived rule must survive");
|
||||
assert_eq!(remaining.rules.len(), 1);
|
||||
assert_eq!(remaining.rules[0].destination.bucket, peer_arn);
|
||||
assert_eq!(
|
||||
removable,
|
||||
HashSet::from([user_arn.to_string(), "arn:rustfs:replication::gone-dep:bucket".to_string()]),
|
||||
"targets of operator rules and of a removed peer are garbage-collected"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn put_replication_merge_returns_incoming_verbatim_without_site_rules() {
|
||||
let existing = ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![replication_rule_with_id(
|
||||
"arn:rustfs:replication:us-east-1:old:bucket",
|
||||
"old-user-rule",
|
||||
7,
|
||||
)],
|
||||
};
|
||||
let incoming = ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![replication_rule_with_id(
|
||||
"arn:rustfs:replication:us-east-1:new:bucket",
|
||||
"new-user-rule",
|
||||
5,
|
||||
)],
|
||||
};
|
||||
|
||||
let merged = merge_user_replication_config_update(incoming.clone(), Some(existing), &HashSet::new());
|
||||
|
||||
assert_eq!(merged.role, incoming.role);
|
||||
assert_eq!(merged.rules, incoming.rules, "non-SR buckets keep the verbatim overwrite semantics");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_replication_split_keeps_site_rules_and_their_targets() {
|
||||
let sr_arn = "arn:rustfs:replication::peer-dep:bucket";
|
||||
let user_arn = "arn:rustfs:replication:us-east-1:user:bucket";
|
||||
let config = ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![
|
||||
replication_rule_with_id(user_arn, "user-rule", 1),
|
||||
replication_rule_with_id(sr_arn, "site-repl-peer-dep", 2),
|
||||
],
|
||||
};
|
||||
|
||||
let (remaining, removable) = split_replication_config_for_user_delete(config, &site_peers(&["peer-dep"]));
|
||||
|
||||
let remaining = remaining.expect("site-replication rules must survive a user delete");
|
||||
let ids: Vec<_> = remaining
|
||||
.rules
|
||||
.iter()
|
||||
.map(|rule| rule.id.as_deref().unwrap_or_default())
|
||||
.collect();
|
||||
assert_eq!(ids, vec!["site-repl-peer-dep"]);
|
||||
assert_eq!(removable, HashSet::from([user_arn.to_string()]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_replication_split_protects_targets_shared_with_site_rules() {
|
||||
let sr_arn = "arn:rustfs:replication::peer-dep:bucket";
|
||||
let config = ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![
|
||||
replication_rule_with_id(sr_arn, "user-rule-on-sr-target", 1),
|
||||
replication_rule_with_id(sr_arn, "site-repl-peer-dep", 2),
|
||||
],
|
||||
};
|
||||
|
||||
let (remaining, removable) = split_replication_config_for_user_delete(config, &site_peers(&["peer-dep"]));
|
||||
|
||||
assert!(remaining.is_some());
|
||||
assert!(
|
||||
removable.is_empty(),
|
||||
"a target still referenced by a surviving site-replication rule must not be removed"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_replication_split_removes_everything_without_site_rules() {
|
||||
let user_arn = "arn:rustfs:replication:us-east-1:user:bucket";
|
||||
let config = ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![replication_rule_with_id(user_arn, "user-rule", 1)],
|
||||
};
|
||||
|
||||
let (remaining, removable) = split_replication_config_for_user_delete(config, &site_peers(&["peer-dep"]));
|
||||
|
||||
assert!(remaining.is_none(), "without site-replication rules the whole config is deleted");
|
||||
assert_eq!(removable, HashSet::from([user_arn.to_string()]));
|
||||
}
|
||||
|
||||
fn replication_targets_with_arn(arns: &[&str]) -> BucketTargets {
|
||||
BucketTargets {
|
||||
targets: arns
|
||||
@@ -3451,7 +3700,10 @@ mod tests {
|
||||
let req = build_request(input, Method::DELETE);
|
||||
let usecase = DefaultBucketUsecase::without_context();
|
||||
|
||||
let err = usecase.execute_delete_bucket_replication(req).await.unwrap_err();
|
||||
let err = usecase
|
||||
.execute_delete_bucket_replication(req, HashSet::new())
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
@@ -4537,7 +4789,7 @@ mod tests {
|
||||
let req = build_request(input, Method::PUT);
|
||||
let usecase = DefaultBucketUsecase::without_context();
|
||||
|
||||
let err = usecase.execute_put_bucket_replication(req).await.unwrap_err();
|
||||
let err = usecase.execute_put_bucket_replication(req, HashSet::new()).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
@@ -4555,7 +4807,7 @@ mod tests {
|
||||
.unwrap();
|
||||
|
||||
let err = DefaultBucketUsecase::without_context()
|
||||
.execute_put_bucket_replication(build_request(input, Method::PUT))
|
||||
.execute_put_bucket_replication(build_request(input, Method::PUT), HashSet::new())
|
||||
.await
|
||||
.expect_err("unsupported fields must be rejected before store access");
|
||||
|
||||
|
||||
@@ -619,6 +619,8 @@ pub(crate) mod bucket {
|
||||
|
||||
use crate::storage::storage_api::ecstore_bucket::replication as replication_contracts;
|
||||
|
||||
pub(crate) use replication_contracts::merge_user_replication_config;
|
||||
|
||||
type ReplicationObjectBridge = crate::storage::storage_api::ecstore_bucket::replication::ReplicationObjectBridge;
|
||||
pub(crate) type DeleteReplicationConfigSnapshot =
|
||||
crate::storage::storage_api::ecstore_bucket::replication::DeleteReplicationConfigSnapshot;
|
||||
|
||||
+10
-8
@@ -26,22 +26,22 @@ struct MiMallocAllocator;
|
||||
unsafe impl GlobalAlloc for MiMallocAllocator {
|
||||
unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
|
||||
// SAFETY: the caller upholds GlobalAlloc's contract for layout.
|
||||
unsafe { mimalloc::MiMalloc.alloc(layout) }
|
||||
unsafe { rustfs_mimalloc::MiMalloc.alloc(layout) }
|
||||
}
|
||||
|
||||
unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 {
|
||||
// SAFETY: the caller upholds GlobalAlloc's contract for layout.
|
||||
unsafe { mimalloc::MiMalloc.alloc_zeroed(layout) }
|
||||
unsafe { rustfs_mimalloc::MiMalloc.alloc_zeroed(layout) }
|
||||
}
|
||||
|
||||
unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) {
|
||||
// SAFETY: ptr and layout came from this allocator and are forwarded unchanged.
|
||||
unsafe { mimalloc::MiMalloc.dealloc(ptr, layout) }
|
||||
unsafe { rustfs_mimalloc::MiMalloc.dealloc(ptr, layout) }
|
||||
}
|
||||
|
||||
unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 {
|
||||
// SAFETY: ptr and layout came from this allocator and are forwarded unchanged.
|
||||
unsafe { mimalloc::MiMalloc.realloc(ptr, layout, new_size) }
|
||||
unsafe { rustfs_mimalloc::MiMalloc.realloc(ptr, layout, new_size) }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -51,7 +51,7 @@ static GLOBAL: hotpath::CountingAllocator<MiMallocAllocator> = hotpath::Counting
|
||||
|
||||
#[cfg(not(all(feature = "hotpath", feature = "hotpath-alloc")))]
|
||||
#[global_allocator]
|
||||
static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;
|
||||
static GLOBAL: rustfs_mimalloc::MiMalloc = rustfs_mimalloc::MiMalloc;
|
||||
|
||||
fn main() {
|
||||
let _hotpath_guard = hotpath::HotpathGuardBuilder::new("main").build();
|
||||
@@ -71,8 +71,9 @@ mod tests {
|
||||
allocation.extend_from_slice(&[7_u8; 64]);
|
||||
|
||||
assert_eq!(allocation.len(), 64);
|
||||
let heap = rustfs_mimalloc::heap::Heap::main();
|
||||
// SAFETY: the live Vec pointer is valid to inspect for heap ownership.
|
||||
assert!(unsafe { libmimalloc_sys::mi_is_in_heap_region(allocation.as_ptr().cast()) });
|
||||
assert!(unsafe { heap.contains(allocation.as_ptr()) });
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -85,12 +86,13 @@ mod tests {
|
||||
let layout = Layout::from_size_align(32, 8).expect("valid test allocation layout");
|
||||
let grown_layout = Layout::from_size_align(64, 8).expect("valid grown test allocation layout");
|
||||
let allocator = super::MiMallocAllocator;
|
||||
let heap = rustfs_mimalloc::heap::Heap::main();
|
||||
|
||||
// SAFETY: The pointer is checked for null before use and later released
|
||||
// through the same allocator with the corresponding layout.
|
||||
let ptr = unsafe { allocator.alloc_zeroed(layout) };
|
||||
assert!(!ptr.is_null());
|
||||
assert!(unsafe { libmimalloc_sys::mi_is_in_heap_region(ptr.cast()) });
|
||||
assert!(unsafe { heap.contains(ptr) });
|
||||
assert!(unsafe { std::slice::from_raw_parts(ptr, 32).iter().all(|byte| *byte == 0) });
|
||||
|
||||
// SAFETY: `ptr` was allocated by `allocator` with `layout`; on failure
|
||||
@@ -102,7 +104,7 @@ mod tests {
|
||||
panic!("mimalloc realloc failed in allocator smoke test");
|
||||
}
|
||||
|
||||
assert!(unsafe { libmimalloc_sys::mi_is_in_heap_region(grown_ptr.cast()) });
|
||||
assert!(unsafe { heap.contains(grown_ptr) });
|
||||
// SAFETY: `grown_ptr` was reallocated by `allocator` and is released
|
||||
// with the matching grown layout.
|
||||
unsafe { allocator.dealloc(grown_ptr, grown_layout) };
|
||||
|
||||
@@ -17,10 +17,7 @@ use rustfs_io_metrics::{
|
||||
record_cpu_usage, record_memory_usage, record_process_memory_split,
|
||||
};
|
||||
use serde::Serialize;
|
||||
#[cfg(any(test, not(target_os = "windows")))]
|
||||
use serde_json::Value;
|
||||
#[cfg(not(target_os = "windows"))]
|
||||
use std::ffi::CStr;
|
||||
use std::path::Path;
|
||||
use std::sync::{Arc, Mutex, OnceLock};
|
||||
use std::time::Duration;
|
||||
@@ -231,7 +228,18 @@ fn read_cgroup_memory_snapshot() -> Option<CgroupMemorySnapshot> {
|
||||
read_cgroup_v2().or_else(read_cgroup_v1)
|
||||
}
|
||||
|
||||
#[cfg(any(test, not(target_os = "windows")))]
|
||||
fn read_allocator_memory_snapshot() -> Option<AllocatorMemorySnapshot> {
|
||||
let json = rustfs_mimalloc::MiMalloc::stats_json();
|
||||
if json.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let observation = parse_mimalloc_stats_json(&json)?;
|
||||
Some(AllocatorMemorySnapshot {
|
||||
backend: crate::allocator_reclaim::allocator_backend(),
|
||||
observation,
|
||||
})
|
||||
}
|
||||
|
||||
fn numeric_json_value(value: &Value) -> Option<u64> {
|
||||
match value {
|
||||
Value::Number(number) => number
|
||||
@@ -242,7 +250,6 @@ fn numeric_json_value(value: &Value) -> Option<u64> {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(test, not(target_os = "windows")))]
|
||||
fn numeric_json_field(value: &Value, field: &str) -> Option<u64> {
|
||||
match value {
|
||||
Value::Object(fields) => fields
|
||||
@@ -254,7 +261,6 @@ fn numeric_json_field(value: &Value, field: &str) -> Option<u64> {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(test, not(target_os = "windows")))]
|
||||
fn mimalloc_stat_field(value: &Value, metric: &str, field: &str) -> Option<u64> {
|
||||
match value {
|
||||
Value::Object(fields) => {
|
||||
@@ -271,12 +277,10 @@ fn mimalloc_stat_field(value: &Value, metric: &str, field: &str) -> Option<u64>
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(test, not(target_os = "windows")))]
|
||||
fn mimalloc_stat_current(value: &Value, metric: &str) -> Option<u64> {
|
||||
mimalloc_stat_field(value, metric, "current")
|
||||
}
|
||||
|
||||
#[cfg(any(test, not(target_os = "windows")))]
|
||||
fn mimalloc_stat_sum(value: &Value, metrics: &[&str], field: &str) -> Option<u64> {
|
||||
metrics
|
||||
.iter()
|
||||
@@ -285,7 +289,6 @@ fn mimalloc_stat_sum(value: &Value, metrics: &[&str], field: &str) -> Option<u64
|
||||
.filter(|value| *value > 0)
|
||||
}
|
||||
|
||||
#[cfg(any(test, not(target_os = "windows")))]
|
||||
fn parse_mimalloc_stats_json(stats_json: &str) -> Option<AllocatorMemoryObservation> {
|
||||
let value = serde_json::from_str::<Value>(stats_json).ok()?;
|
||||
let malloc_metrics = ["malloc_normal", "malloc_huge"];
|
||||
@@ -312,33 +315,6 @@ fn parse_mimalloc_stats_json(stats_json: &str) -> Option<AllocatorMemoryObservat
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "windows"))]
|
||||
#[allow(unsafe_code)]
|
||||
fn read_allocator_memory_snapshot() -> Option<AllocatorMemorySnapshot> {
|
||||
// SAFETY: `mi_stats_get_json` returns a null-terminated JSON buffer owned by
|
||||
// mimalloc when called with a null input buffer. The mimalloc API requires
|
||||
// freeing that buffer with `mi_free`; parsing finishes before the buffer is freed.
|
||||
let observation = unsafe {
|
||||
let stats_ptr = libmimalloc_sys::mi_stats_get_json(0, std::ptr::null_mut());
|
||||
if stats_ptr.is_null() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let observation = CStr::from_ptr(stats_ptr).to_str().ok().and_then(parse_mimalloc_stats_json);
|
||||
libmimalloc_sys::mi_free(stats_ptr.cast());
|
||||
observation?
|
||||
};
|
||||
Some(AllocatorMemorySnapshot {
|
||||
backend: crate::allocator_reclaim::allocator_backend(),
|
||||
observation,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
fn read_allocator_memory_snapshot() -> Option<AllocatorMemorySnapshot> {
|
||||
None
|
||||
}
|
||||
|
||||
fn configured_memory_observability_interval_secs() -> u64 {
|
||||
rustfs_utils::get_env_u64(ENV_MEMORY_OBSERVABILITY_INTERVAL_SECS, DEFAULT_MEMORY_OBSERVABILITY_INTERVAL_SECS).max(1)
|
||||
}
|
||||
@@ -566,6 +542,13 @@ mod tests {
|
||||
assert_eq!(parse_mimalloc_stats_json(r#"{ "allocator": "unknown" }"#), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn read_allocator_memory_snapshot_uses_mimalloc_stats_json() {
|
||||
let snapshot = super::read_allocator_memory_snapshot();
|
||||
#[cfg(not(target_os = "windows"))]
|
||||
assert!(snapshot.is_some(), "allocator snapshot should be available on non-Windows");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn memory_observability_snapshot_reports_disabled_when_metrics_are_disabled() {
|
||||
let snapshot = build_memory_observability_status_snapshot(false, 15, false);
|
||||
|
||||
+169
-2
@@ -63,6 +63,69 @@ use crate::app::storage_api::object_usecase::bucket::replication::{
|
||||
};
|
||||
use crate::storage::storage_api::ecfs_consumer::StorageObjectOptions as ObjectOptions;
|
||||
|
||||
#[cfg(test)]
|
||||
static SITE_REPLICATION_GATE_TEST_OVERRIDE: std::sync::atomic::AtomicU8 = std::sync::atomic::AtomicU8::new(0);
|
||||
#[cfg(test)]
|
||||
const SITE_REPLICATION_GATE_FORCE_DISABLED: u8 = 1;
|
||||
#[cfg(test)]
|
||||
const SITE_REPLICATION_GATE_FORCE_ENABLED: u8 = 2;
|
||||
|
||||
async fn site_replication_gate_enabled() -> S3Result<bool> {
|
||||
#[cfg(test)]
|
||||
match SITE_REPLICATION_GATE_TEST_OVERRIDE.load(std::sync::atomic::Ordering::SeqCst) {
|
||||
SITE_REPLICATION_GATE_FORCE_DISABLED => return Ok(false),
|
||||
SITE_REPLICATION_GATE_FORCE_ENABLED => return Ok(true),
|
||||
_ => {}
|
||||
}
|
||||
crate::admin::handlers::site_replication::site_replication_enabled().await
|
||||
}
|
||||
|
||||
/// Remote site-replication peer deployment ids handed to the bucket usecase
|
||||
/// so an S3 replication-config edit keeps exactly the reconciler-owned rules
|
||||
/// (issue #1948). Read here, in the interface layer, because the usecase must
|
||||
/// not import the admin handlers (layer guard); a state-read failure
|
||||
/// propagates so the edit fails closed.
|
||||
async fn site_replication_peer_deployment_ids_for_edit() -> S3Result<std::collections::HashSet<String>> {
|
||||
// While the gate override is in effect the test exercises the deny/allow
|
||||
// branch, not the peer set; there is no persisted state to read.
|
||||
#[cfg(test)]
|
||||
if SITE_REPLICATION_GATE_TEST_OVERRIDE.load(std::sync::atomic::Ordering::SeqCst) != 0 {
|
||||
return Ok(std::collections::HashSet::new());
|
||||
}
|
||||
crate::admin::handlers::site_replication::site_replication_remote_peer_deployment_ids().await
|
||||
}
|
||||
|
||||
/// MinIO `ErrReplicationDenyEditError`.
|
||||
fn replication_deny_edit_error() -> S3Error {
|
||||
let mut err = S3Error::with_message(
|
||||
S3ErrorCode::Custom("XMinioReplicationDenyEdit".into()),
|
||||
"Sub-User is not allowed to edit Replication configuration",
|
||||
);
|
||||
err.set_status_code(StatusCode::BAD_REQUEST);
|
||||
err
|
||||
}
|
||||
|
||||
/// Site-replication gate for S3 replication-config edits (issue #1948).
|
||||
///
|
||||
/// On a site-replication deployment the bucket's replication config carries
|
||||
/// the operator-managed `site-repl-*` rules that keep every peer in sync, and
|
||||
/// a successful edit is broadcast to all peers — so a user holding only
|
||||
/// bucket-scoped `s3:PutReplicationConfiguration` could rewrite or erase
|
||||
/// replication net-wide. MinIO parity (`ErrReplicationDenyEditError`): only
|
||||
/// owner credentials (root or root-parented) may edit. Runs after the policy
|
||||
/// authorization in the access layer and only on the external S3 path — the
|
||||
/// reconciler and peer bucket-meta ingestion never route through these
|
||||
/// handlers.
|
||||
async fn deny_replication_config_edit_for_non_owner<T>(req: &S3Request<T>) -> S3Result<()> {
|
||||
if crate::storage::access::req_info_ref(req)?.is_owner {
|
||||
return Ok(());
|
||||
}
|
||||
if site_replication_gate_enabled().await? {
|
||||
return Err(replication_deny_edit_error());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct FS {
|
||||
/// This server's late-bound application-context slot (backlog#1052 S2).
|
||||
@@ -500,8 +563,10 @@ impl S3 for FS {
|
||||
&self,
|
||||
req: S3Request<DeleteBucketReplicationInput>,
|
||||
) -> S3Result<S3Response<DeleteBucketReplicationOutput>> {
|
||||
deny_replication_config_edit_for_non_owner(&req).await?;
|
||||
let site_peers = site_replication_peer_deployment_ids_for_edit().await?;
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
usecase.execute_delete_bucket_replication(req).await
|
||||
usecase.execute_delete_bucket_replication(req, site_peers).await
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
@@ -1353,8 +1418,10 @@ impl S3 for FS {
|
||||
&self,
|
||||
req: S3Request<PutBucketReplicationInput>,
|
||||
) -> S3Result<S3Response<PutBucketReplicationOutput>> {
|
||||
deny_replication_config_edit_for_non_owner(&req).await?;
|
||||
let site_peers = site_replication_peer_deployment_ids_for_edit().await?;
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
usecase.execute_put_bucket_replication(req).await
|
||||
usecase.execute_put_bucket_replication(req, site_peers).await
|
||||
}
|
||||
|
||||
async fn put_bucket_request_payment(
|
||||
@@ -1919,3 +1986,103 @@ impl S3 for FS {
|
||||
Box::pin(usecase.execute_upload_part_copy(req)).await
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
FS, SITE_REPLICATION_GATE_FORCE_DISABLED, SITE_REPLICATION_GATE_FORCE_ENABLED, SITE_REPLICATION_GATE_TEST_OVERRIDE,
|
||||
};
|
||||
use crate::storage::access::ReqInfo;
|
||||
use http::Method;
|
||||
use http::StatusCode;
|
||||
use s3s::dto::{DeleteBucketReplicationInput, PutBucketReplicationInput, ReplicationConfiguration};
|
||||
use s3s::{S3, S3Error, S3ErrorCode, S3Request};
|
||||
use std::sync::atomic::Ordering;
|
||||
|
||||
fn replication_config_edit_request<T>(input: T, is_owner: bool) -> S3Request<T> {
|
||||
let mut req = S3Request {
|
||||
input,
|
||||
method: Method::PUT,
|
||||
uri: http::Uri::from_static("/"),
|
||||
headers: http::HeaderMap::new(),
|
||||
extensions: http::Extensions::new(),
|
||||
credentials: None,
|
||||
region: None,
|
||||
service: None,
|
||||
trailing_headers: None,
|
||||
};
|
||||
req.extensions.insert(ReqInfo {
|
||||
is_owner,
|
||||
..Default::default()
|
||||
});
|
||||
req
|
||||
}
|
||||
|
||||
fn put_bucket_replication_input() -> PutBucketReplicationInput {
|
||||
PutBucketReplicationInput {
|
||||
bucket: "test-bucket".to_string(),
|
||||
checksum_algorithm: None,
|
||||
content_md5: None,
|
||||
expected_bucket_owner: None,
|
||||
replication_configuration: ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: Vec::new(),
|
||||
},
|
||||
token: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn delete_bucket_replication_input() -> DeleteBucketReplicationInput {
|
||||
DeleteBucketReplicationInput {
|
||||
bucket: "test-bucket".to_string(),
|
||||
expected_bucket_owner: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn assert_replication_deny_edit(err: &S3Error) {
|
||||
match err.code() {
|
||||
S3ErrorCode::Custom(code) => assert_eq!(code, "XMinioReplicationDenyEdit"),
|
||||
other => panic!("expected XMinioReplicationDenyEdit, got {other:?}"),
|
||||
}
|
||||
assert_eq!(err.status_code(), Some(StatusCode::BAD_REQUEST));
|
||||
}
|
||||
|
||||
/// Single test on purpose: the branches share the process-wide gate
|
||||
/// override, and parallel tests would race it.
|
||||
#[tokio::test]
|
||||
async fn replication_config_edit_gate_denies_only_non_owner_under_site_replication() {
|
||||
let fs = FS::new();
|
||||
SITE_REPLICATION_GATE_TEST_OVERRIDE.store(SITE_REPLICATION_GATE_FORCE_ENABLED, Ordering::SeqCst);
|
||||
|
||||
// Non-owner PUT/DELETE through the real S3 handlers: denied by the
|
||||
// gate before the usecase (and thus the store) is ever touched.
|
||||
let err = fs
|
||||
.put_bucket_replication(replication_config_edit_request(put_bucket_replication_input(), false))
|
||||
.await
|
||||
.expect_err("non-owner PutBucketReplication must be denied while site replication is enabled");
|
||||
assert_replication_deny_edit(&err);
|
||||
let err = fs
|
||||
.delete_bucket_replication(replication_config_edit_request(delete_bucket_replication_input(), false))
|
||||
.await
|
||||
.expect_err("non-owner DeleteBucketReplication must be denied while site replication is enabled");
|
||||
assert_replication_deny_edit(&err);
|
||||
|
||||
// Owner passes the gate (the usecase's empty-rules structure error
|
||||
// proves the request reached the usecase instead of the deny path).
|
||||
let err = fs
|
||||
.put_bucket_replication(replication_config_edit_request(put_bucket_replication_input(), true))
|
||||
.await
|
||||
.expect_err("owner request should pass the gate and fail later on config validation");
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
|
||||
|
||||
// Without site replication the policy check alone still governs the edit.
|
||||
SITE_REPLICATION_GATE_TEST_OVERRIDE.store(SITE_REPLICATION_GATE_FORCE_DISABLED, Ordering::SeqCst);
|
||||
let err = fs
|
||||
.put_bucket_replication(replication_config_edit_request(put_bucket_replication_input(), false))
|
||||
.await
|
||||
.expect_err("non-owner request should pass the gate and fail later on config validation");
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
|
||||
|
||||
SITE_REPLICATION_GATE_TEST_OVERRIDE.store(0, Ordering::SeqCst);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user