mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-22 12:26:37 +00:00
Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| ce9b69d811 |
Generated
+4
@@ -3843,6 +3843,7 @@ dependencies = [
|
||||
"s3s",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"serial_test",
|
||||
"sha2 0.11.0",
|
||||
"suppaftp",
|
||||
"time",
|
||||
@@ -9803,6 +9804,7 @@ dependencies = [
|
||||
"rustfs-replication",
|
||||
"rustfs-storage-api",
|
||||
"s3s",
|
||||
"serial_test",
|
||||
"temp-env",
|
||||
"time",
|
||||
"tokio",
|
||||
@@ -9918,6 +9920,7 @@ dependencies = [
|
||||
"rustfs-config",
|
||||
"rustfs-io-metrics",
|
||||
"rustfs-utils",
|
||||
"serial_test",
|
||||
"temp-env",
|
||||
"tempfile",
|
||||
"tokio",
|
||||
@@ -10290,6 +10293,7 @@ dependencies = [
|
||||
"s3s",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"serial_test",
|
||||
"sha2 0.11.0",
|
||||
"temp-env",
|
||||
"tempfile",
|
||||
|
||||
@@ -96,6 +96,7 @@ tokio-stream = { workspace = true }
|
||||
rustfs-madmin.workspace = true
|
||||
rustfs-filemeta.workspace = true
|
||||
bytes = { workspace = true, features = ["serde"] }
|
||||
serial_test = { workspace = true }
|
||||
aws-sdk-s3 = { workspace = true, default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
|
||||
aws-sdk-sts = { workspace = true, default-features = false, features = ["default-https-client", "rt-tokio"] }
|
||||
aws-config = { workspace = true }
|
||||
|
||||
@@ -55,6 +55,7 @@ mod tests {
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
|
||||
use serial_test::serial;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::error::Error;
|
||||
use tokio::time::{Duration, timeout};
|
||||
@@ -268,6 +269,7 @@ mod tests {
|
||||
/// stripes) and a multipart object (3 parts × 5 MiB) must GET back as a
|
||||
/// full, byte-identical body with the correct Content-Length. No early EOF.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn degraded_read_large_objects_with_one_disk_offline_return_full_body() -> TestResult {
|
||||
init_logging();
|
||||
info!("dist-13 (a): large-object degraded read with one of four disks offline");
|
||||
@@ -333,6 +335,7 @@ mod tests {
|
||||
/// mid-stream — the exact window the fixes had to reconstruct through rather
|
||||
/// than truncate.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn degraded_read_reconstructs_through_midstream_bitrot_within_quorum() -> TestResult {
|
||||
init_logging();
|
||||
info!("dist-13 (b): mid-stream bitrot within quorum must reconstruct a full body");
|
||||
@@ -390,6 +393,7 @@ mod tests {
|
||||
/// Content-Length. `get_checked` panics on that forbidden outcome, so this
|
||||
/// test fails loudly if the truncation bug ever returns.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn beyond_quorum_degraded_read_never_silently_truncates() -> TestResult {
|
||||
init_logging();
|
||||
info!("dist-13 (c): beyond-quorum degraded read must fail, never 200+truncated");
|
||||
|
||||
@@ -51,6 +51,7 @@ mod tests {
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
|
||||
use serial_test::serial;
|
||||
use std::error::Error;
|
||||
use tokio::time::{Duration, timeout};
|
||||
use tracing::info;
|
||||
@@ -128,6 +129,7 @@ mod tests {
|
||||
/// the body — and assert the server log names the object, at the log level a
|
||||
/// default deployment actually runs with.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn midstream_get_failure_is_logged_with_the_object_at_default_log_level() -> TestResult {
|
||||
init_logging();
|
||||
info!("rustfs#4784: a mid-stream GET failure must name its object in the source log");
|
||||
|
||||
@@ -46,6 +46,7 @@ use prost::Message;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
use serial_test::serial;
|
||||
use std::collections::BTreeMap;
|
||||
use std::convert::Infallible;
|
||||
use std::error::Error;
|
||||
@@ -1694,6 +1695,7 @@ fn assert_storage_layout(
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn four_node_inline_storage_and_get_boundaries() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
@@ -1765,6 +1767,7 @@ async fn four_node_inline_storage_and_get_boundaries() -> TestResult {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn four_node_empty_legacy_volumes_start_as_fresh() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
@@ -1802,6 +1805,7 @@ async fn four_node_empty_legacy_volumes_start_as_fresh() -> TestResult {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn four_node_inline_fallback_controls() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
@@ -1866,6 +1870,7 @@ async fn four_node_inline_fallback_controls() -> TestResult {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn four_node_compressed_inline_fallback() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
@@ -1900,6 +1905,7 @@ async fn four_node_compressed_inline_fallback() -> TestResult {
|
||||
/// Multipart disk compression is live again, so a compression-enabled cluster classifies multipart objects as compressed and the roundtrip (full GET plus partNumber GET) must still return the original bytes.
|
||||
/// Reverting the multipart compression fix must fail this test.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn four_node_multipart_disk_compression_roundtrip() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
@@ -1946,6 +1952,7 @@ async fn four_node_multipart_disk_compression_roundtrip() -> TestResult {
|
||||
/// read costs on the order of the covering part's block size against a ~5 MiB
|
||||
/// object.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn four_node_compressed_multipart_tail_range_reads_are_bounded() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
@@ -2012,6 +2019,7 @@ async fn four_node_compressed_multipart_tail_range_reads_are_bounded() -> TestRe
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn four_node_mixed_msgpack_compat_mode_preserves_fallback_controls() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
@@ -2115,6 +2123,7 @@ async fn four_node_mixed_msgpack_compat_mode_preserves_fallback_controls() -> Te
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn four_node_add_tier_converges() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
@@ -2133,6 +2142,7 @@ async fn four_node_add_tier_converges() -> TestResult {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn four_node_add_tier_converges_after_offline_node_restart_without_second_mutation() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
@@ -2154,6 +2164,7 @@ async fn four_node_add_tier_converges_after_offline_node_restart_without_second_
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn four_node_manual_transition_job_status_survives_node_restart() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
@@ -2228,6 +2239,7 @@ async fn four_node_manual_transition_job_status_survives_node_restart() -> TestR
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn four_node_manual_transition_distributed_admission_conflict_reports_status_and_backpressure() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
@@ -2369,6 +2381,7 @@ async fn four_node_manual_transition_distributed_admission_conflict_reports_stat
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
#[ignore = "manual #1508 evidence harness: starts a 4-node cluster, a remote tier, and an in-flight transition job"]
|
||||
async fn four_node_manual_transition_rollout_non_empty_restart_readback() -> TestResult {
|
||||
init_logging();
|
||||
@@ -2473,6 +2486,7 @@ async fn four_node_manual_transition_rollout_non_empty_restart_readback() -> Tes
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn four_node_mixed_msgpack_compat_mode_preserves_fallback_controls_during_transition() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
@@ -2584,6 +2598,7 @@ async fn four_node_mixed_msgpack_compat_mode_preserves_fallback_controls_during_
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn four_node_transitioned_inline_fallback() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
|
||||
@@ -19,6 +19,7 @@
|
||||
//! multipart upload behaviour.
|
||||
|
||||
use crate::common::{TEST_BUCKET, init_logging};
|
||||
use serial_test::serial;
|
||||
use tokio::time::{Duration, sleep};
|
||||
use tracing::{error, info};
|
||||
|
||||
@@ -61,6 +62,7 @@ impl VaultKmsTestContext {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_vault_kms_end_to_end() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
if skip_if_kms_admin_tool_unavailable("test_vault_kms_end_to_end") {
|
||||
@@ -116,6 +118,7 @@ async fn test_vault_kms_end_to_end() -> Result<(), Box<dyn std::error::Error + S
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_vault_kms_key_isolation() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
if skip_if_kms_admin_tool_unavailable("test_vault_kms_key_isolation") {
|
||||
@@ -202,6 +205,7 @@ async fn test_vault_kms_key_isolation() -> Result<(), Box<dyn std::error::Error
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_vault_kms_large_file() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
if skip_if_kms_admin_tool_unavailable("test_vault_kms_large_file") {
|
||||
@@ -266,6 +270,7 @@ async fn test_vault_kms_large_file() -> Result<(), Box<dyn std::error::Error + S
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_vault_kms_multipart_upload() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
if skip_if_kms_admin_tool_unavailable("test_vault_kms_multipart_upload") {
|
||||
@@ -296,6 +301,7 @@ async fn test_vault_kms_multipart_upload() -> Result<(), Box<dyn std::error::Err
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_vault_kms_key_operations() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
if skip_if_kms_admin_tool_unavailable("test_vault_kms_key_operations") {
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
use crate::common::{awscurl_delete, awscurl_put, init_logging};
|
||||
use crate::policy::test_env::PolicyTestEnvironment;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use serial_test::serial;
|
||||
use tracing::info;
|
||||
|
||||
/// Helper function to create a regular user with given credentials
|
||||
@@ -121,6 +122,7 @@ async fn cleanup_user_and_policy(env: &PolicyTestEnvironment, username: &str, po
|
||||
|
||||
/// Test AWS policy variables with single-value scenarios
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
#[ignore = "Starts a rustfs server; enable when running full E2E"]
|
||||
pub async fn test_aws_policy_variables_single_value() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
test_aws_policy_variables_single_value_impl().await
|
||||
@@ -273,6 +275,7 @@ pub async fn test_aws_policy_variables_single_value_impl_with_env(
|
||||
|
||||
/// Test AWS policy variables with multi-value scenarios
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
#[ignore = "Starts a rustfs server; enable when running full E2E"]
|
||||
pub async fn test_aws_policy_variables_multi_value() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
test_aws_policy_variables_multi_value_impl().await
|
||||
@@ -398,6 +401,7 @@ pub async fn test_aws_policy_variables_multi_value_impl_with_env(
|
||||
|
||||
/// Test AWS policy variables with variable concatenation
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
#[ignore = "Starts a rustfs server; enable when running full E2E"]
|
||||
pub async fn test_aws_policy_variables_concatenation() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
test_aws_policy_variables_concatenation_impl().await
|
||||
@@ -487,6 +491,7 @@ pub async fn test_aws_policy_variables_concatenation_impl_with_env(
|
||||
|
||||
/// Test AWS policy variables with nested scenarios
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
#[ignore = "Starts a rustfs server; enable when running full E2E"]
|
||||
pub async fn test_aws_policy_variables_nested() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
test_aws_policy_variables_nested_impl().await
|
||||
@@ -504,6 +509,7 @@ pub async fn test_aws_policy_variables_nested_impl() -> Result<(), Box<dyn std::
|
||||
|
||||
/// Test AWS policy variables with STS temporary credentials
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
#[ignore = "Starts a rustfs server; enable when running full E2E"]
|
||||
pub async fn test_aws_policy_variables_sts() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
test_aws_policy_variables_sts_impl().await
|
||||
@@ -699,6 +705,7 @@ pub async fn test_aws_policy_variables_sts_impl_with_env(
|
||||
|
||||
/// Test AWS policy variables with deny scenarios
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
#[ignore = "Starts a rustfs server; enable when running full E2E"]
|
||||
pub async fn test_aws_policy_variables_deny() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
test_aws_policy_variables_deny_impl().await
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
|
||||
use crate::common::init_logging;
|
||||
use crate::policy::test_env::PolicyTestEnvironment;
|
||||
use serial_test::serial;
|
||||
use std::time::Instant;
|
||||
use tokio::time::{Duration, sleep};
|
||||
use tracing::{error, info};
|
||||
@@ -212,6 +213,7 @@ impl PolicyTestSuite {
|
||||
|
||||
/// Test suite
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
#[ignore = "Connects to existing rustfs server"]
|
||||
async fn test_policy_critical_suite() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let config = TestSuiteConfig {
|
||||
|
||||
@@ -41,6 +41,7 @@ use reqwest::Client;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
use serial_test::serial;
|
||||
use tokio::process::Command;
|
||||
use tracing::info;
|
||||
|
||||
@@ -820,6 +821,7 @@ pub async fn test_webdav_core_operations() -> Result<()> {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_webdav_core_operations_direct() -> Result<()> {
|
||||
test_webdav_core_operations().await
|
||||
}
|
||||
|
||||
@@ -27,6 +27,7 @@ mod tests {
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, VersioningConfiguration};
|
||||
use serial_test::serial;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::HashSet;
|
||||
use std::error::Error;
|
||||
@@ -156,6 +157,7 @@ mod tests {
|
||||
/// content, degraded writes must succeed, and everything must still
|
||||
/// verify after the disk returns.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_degraded_read_write_with_one_disk_offline() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
info!("Reliability: degraded read/write with one of four disks offline");
|
||||
@@ -208,6 +210,7 @@ mod tests {
|
||||
/// bytes to a reader: per-shard bitrot checksums reject the bad shard and
|
||||
/// the object is reconstructed from the remaining shards.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_bitrot_corrupted_shard_read_returns_correct_data() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
info!("Reliability: GET must read through a bitrot-corrupted shard");
|
||||
@@ -250,6 +253,7 @@ mod tests {
|
||||
/// heal, and require the replaced disk to be rebuilt and all content to
|
||||
/// verify against the sha256 manifest.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_fresh_disk_replacement_heals_after_sigkill_restart() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
info!("Reliability: fresh-disk replacement heals after SIGKILL restart");
|
||||
@@ -323,6 +327,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_versioned_shard_census_selects_each_version_data_dir() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
info!("Reliability: physical shard census selects the requested object version");
|
||||
|
||||
@@ -29,6 +29,7 @@ mod tests {
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, VersioningConfiguration};
|
||||
use http::Method;
|
||||
use serial_test::serial;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::BTreeSet;
|
||||
use std::error::Error;
|
||||
@@ -1060,6 +1061,7 @@ mod tests {
|
||||
/// Linux mount namespaces are per-thread; keep mount setup and process
|
||||
/// spawning on one OS thread so child RustFS nodes inherit the test mounts.
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
#[serial]
|
||||
#[ignore = "requires Linux root/CAP_SYS_ADMIN and RUSTFS_PRIVILEGED_REPLACEMENT_E2E=1"]
|
||||
async fn test_privileged_3x4_auto_replacement_rebuilds_ec8_plus_4_without_admin_heal()
|
||||
-> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
@@ -1073,6 +1075,7 @@ mod tests {
|
||||
/// Linux mount namespaces are per-thread; keep mount setup and process
|
||||
/// spawning on one OS thread so child RustFS nodes inherit the test mounts.
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
#[serial]
|
||||
#[ignore = "requires Linux root/CAP_SYS_ADMIN and RUSTFS_PRIVILEGED_REPLACEMENT_E2E=1"]
|
||||
async fn test_privileged_3x4_auto_replacement_rebuilds_ec6_plus_6_without_admin_heal()
|
||||
-> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
|
||||
@@ -199,12 +199,12 @@ pub mod bucket {
|
||||
VersionPurgeStatusType, XferStats, commit_force_delete_intent, complete_force_delete_intent,
|
||||
delete_replication_state_from_config, delete_replication_version_id, get_global_replication_pool,
|
||||
get_global_replication_stats, get_proxy_targets, init_background_replication,
|
||||
invalid_replication_config_status_field, persist_force_delete_intent, read_durable_mrf_backlog,
|
||||
replication_state_to_filemeta, replication_status_to_filemeta, replication_statuses_map, replication_target_arns,
|
||||
resync_start_conflict_id, should_remove_replication_target, should_schedule_delete_replication,
|
||||
should_use_existing_delete_replication_info, should_use_existing_delete_replication_source,
|
||||
unsupported_replication_config_field, validate_replication_config_structure, validate_replication_config_target_arns,
|
||||
version_purge_status_to_filemeta,
|
||||
invalid_replication_config_status_field, is_site_replication_rule, merge_incoming_replication_config,
|
||||
persist_force_delete_intent, read_durable_mrf_backlog, replication_state_to_filemeta, replication_status_to_filemeta,
|
||||
replication_statuses_map, replication_target_arn_deployment_id, replication_target_arns, resync_start_conflict_id,
|
||||
should_remove_replication_target, should_schedule_delete_replication, should_use_existing_delete_replication_info,
|
||||
should_use_existing_delete_replication_source, unsupported_replication_config_field,
|
||||
validate_replication_config_structure, validate_replication_config_target_arns, version_purge_status_to_filemeta,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -47,7 +47,8 @@ pub use replication_config_boundary::{
|
||||
ObjectOpts, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS,
|
||||
REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS,
|
||||
ReplicationConfigStructureError, ReplicationConfigurationExt, ReplicationTargetValidationError,
|
||||
invalid_replication_config_status_field, replication_target_arns, should_remove_replication_target,
|
||||
invalid_replication_config_status_field, is_site_replication_rule, merge_incoming_replication_config,
|
||||
replication_target_arn_deployment_id, replication_target_arns, should_remove_replication_target,
|
||||
unsupported_replication_config_field, validate_replication_config_structure, validate_replication_config_target_arns,
|
||||
};
|
||||
pub(crate) use replication_filemeta_boundary::version_purge_statuses_map;
|
||||
|
||||
@@ -16,6 +16,7 @@ pub use rustfs_replication::{
|
||||
ObjectOpts, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS,
|
||||
REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS,
|
||||
ReplicationConfigStructureError, ReplicationConfigurationExt, ReplicationRuleExt, ReplicationTargetValidationError,
|
||||
invalid_replication_config_status_field, replication_target_arns, should_remove_replication_target,
|
||||
invalid_replication_config_status_field, is_site_replication_rule, merge_incoming_replication_config,
|
||||
replication_target_arn_deployment_id, replication_target_arns, should_remove_replication_target,
|
||||
unsupported_replication_config_field, validate_replication_config_structure, validate_replication_config_target_arns,
|
||||
};
|
||||
|
||||
@@ -69,7 +69,7 @@ uuid = { workspace = true, features = ["v4", "serde", "fast-rng", "macro-diagnos
|
||||
[dev-dependencies]
|
||||
metrics-util = { workspace = true, features = ["debugging"] }
|
||||
proptest = "1"
|
||||
serial_test = { workspace = true }
|
||||
serial_test.workspace = true
|
||||
temp-env.workspace = true
|
||||
tokio = { workspace = true, features = ["macros", "fs", "rt-multi-thread"] }
|
||||
|
||||
|
||||
@@ -1564,6 +1564,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn abort_incomplete_multipart_upload_due_accepts_zero_days() {
|
||||
let initiated = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
|
||||
let lc = BucketLifecycleConfiguration {
|
||||
@@ -1624,6 +1625,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn predict_expiration_selects_closest_expiry_for_put_object() {
|
||||
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
|
||||
let lc = BucketLifecycleConfiguration {
|
||||
@@ -1870,6 +1872,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn empty_transition_vectors_are_not_active_or_due() {
|
||||
let lc = BucketLifecycleConfiguration {
|
||||
expiry_updated_at: None,
|
||||
@@ -1935,6 +1938,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn eval_inner_keeps_latest_object_before_days_due() {
|
||||
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
|
||||
let lc = BucketLifecycleConfiguration {
|
||||
@@ -1968,6 +1972,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn eval_inner_transitions_latest_object_after_days_due() {
|
||||
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
|
||||
let lc = BucketLifecycleConfiguration {
|
||||
@@ -2005,6 +2010,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn eval_inner_transitions_latest_object_after_date_due() {
|
||||
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
|
||||
let transition_date = base_time - Duration::days(1);
|
||||
@@ -2044,6 +2050,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn eval_inner_selects_earliest_due_among_multiple_past_due_events() {
|
||||
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
|
||||
// Two enabled rules both yield a past-due DeleteAction and a third yields a
|
||||
@@ -2157,6 +2164,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn eval_inner_expires_noncurrent_version_after_due() {
|
||||
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
|
||||
let lc = BucketLifecycleConfiguration {
|
||||
@@ -2194,6 +2202,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn eval_inner_skips_noncurrent_expiration_without_successor() {
|
||||
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).expect("valid fixed test timestamp");
|
||||
let lc = BucketLifecycleConfiguration {
|
||||
@@ -2229,6 +2238,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn eval_inner_missing_successor_does_not_skip_noncurrent_transition() {
|
||||
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).expect("valid fixed test timestamp");
|
||||
let lc = BucketLifecycleConfiguration {
|
||||
@@ -2271,6 +2281,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn eval_inner_noncurrent_expiration_one_day_respects_due_boundary() {
|
||||
let successor_time = datetime!(2025-06-15 12:00:00 UTC);
|
||||
let due = expected_expiry_time(successor_time, 1);
|
||||
@@ -2312,6 +2323,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn eval_inner_expires_noncurrent_version_immediately_when_zero_days() {
|
||||
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
|
||||
let lc = BucketLifecycleConfiguration {
|
||||
@@ -2349,6 +2361,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn eval_inner_transitions_noncurrent_version_after_due() {
|
||||
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
|
||||
let lc = BucketLifecycleConfiguration {
|
||||
@@ -2424,6 +2437,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn evaluator_honors_newer_noncurrent_versions_retention_count() {
|
||||
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
|
||||
let lc = Arc::new(BucketLifecycleConfiguration {
|
||||
@@ -2712,6 +2726,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn expired_object_delete_marker_ignores_marker_with_noncurrent_versions_present() {
|
||||
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
|
||||
let lc = BucketLifecycleConfiguration {
|
||||
@@ -2788,6 +2803,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn expired_object_delete_marker_deletes_only_delete_marker_immediately() {
|
||||
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
|
||||
let lc = BucketLifecycleConfiguration {
|
||||
@@ -2865,6 +2881,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn expiration_days_deletes_only_expired_delete_marker_when_due() {
|
||||
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
|
||||
let lc = BucketLifecycleConfiguration {
|
||||
@@ -2915,6 +2932,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn expiration_days_uses_earliest_due_rule_for_expired_delete_marker() {
|
||||
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
|
||||
let make_rule = |id: &str, days| LifecycleRule {
|
||||
@@ -3245,6 +3263,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn del_marker_expiration_deletes_marker_and_older_versions_when_due() {
|
||||
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).expect("fixed timestamp should be valid");
|
||||
let lc = BucketLifecycleConfiguration {
|
||||
@@ -3284,6 +3303,7 @@ mod tests {
|
||||
// --- TASK-003 tests: Round up to next UTC processing boundary ---
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn expected_expiry_time_rounds_up_to_next_midnight_utc() {
|
||||
with_default_ilm_process_time(|| {
|
||||
// Object created at 2025-01-15T10:30:45Z, expire in 30 days
|
||||
@@ -3299,6 +3319,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn expected_expiry_time_immediate_expiry_returns_epoch() {
|
||||
with_default_ilm_process_time(|| {
|
||||
let mod_time = datetime!(2025-06-01 12:00:00 UTC);
|
||||
@@ -3308,6 +3329,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn expected_expiry_time_preserves_exact_midnight_boundary() {
|
||||
with_default_ilm_process_time(|| {
|
||||
let mod_time = datetime!(2025-03-01 00:00:00 UTC);
|
||||
@@ -3317,6 +3339,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn expected_expiry_time_rounds_end_of_day_to_following_midnight() {
|
||||
with_default_ilm_process_time(|| {
|
||||
let mod_time = datetime!(2025-06-15 23:59:59 UTC);
|
||||
@@ -3326,6 +3349,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn expected_expiry_time_uses_canonical_process_time_boundary() {
|
||||
let mod_time = datetime!(2025-01-15 10:30:45 UTC);
|
||||
|
||||
@@ -3338,6 +3362,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn expected_expiry_time_uses_deprecated_process_time_alias() {
|
||||
let mod_time = datetime!(2025-01-15 10:30:45 UTC);
|
||||
|
||||
@@ -3350,6 +3375,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn expected_expiry_time_uses_default_boundary_when_process_time_is_zero_or_invalid() {
|
||||
let mod_time = datetime!(2025-01-15 10:30:45 UTC);
|
||||
|
||||
@@ -3372,6 +3398,7 @@ mod tests {
|
||||
|
||||
// (a) Default path (env unset) is byte-identical: one day == 86400s.
|
||||
#[test]
|
||||
#[serial]
|
||||
fn ilm_day_secs_defaults_to_86400_when_unset() {
|
||||
temp_env::with_var_unset(ENV_ILM_DEBUG_DAY_SECS, || {
|
||||
assert_eq!(ilm_day_secs(), DEFAULT_ILM_DAY_SECS);
|
||||
@@ -3400,6 +3427,7 @@ mod tests {
|
||||
|
||||
// (b) End-to-end env read scales the day length.
|
||||
#[test]
|
||||
#[serial]
|
||||
fn ilm_day_secs_scales_when_env_set() {
|
||||
temp_env::with_var(ENV_ILM_DEBUG_DAY_SECS, Some("2"), || {
|
||||
assert_eq!(ilm_day_secs(), 2);
|
||||
@@ -3408,6 +3436,7 @@ mod tests {
|
||||
|
||||
// (c) Invalid env value falls back to 86400.
|
||||
#[test]
|
||||
#[serial]
|
||||
fn ilm_day_secs_falls_back_on_invalid_env() {
|
||||
temp_env::with_var(ENV_ILM_DEBUG_DAY_SECS, Some("bogus"), || {
|
||||
assert_eq!(ilm_day_secs(), DEFAULT_ILM_DAY_SECS);
|
||||
@@ -3420,6 +3449,7 @@ mod tests {
|
||||
// Deadline math scales: with a 1s day and PROCESS_TIME unset, a Days=1 rule is
|
||||
// due 1s after mod_time (rounded up to the next 1s boundary => same instant).
|
||||
#[test]
|
||||
#[serial]
|
||||
fn expected_expiry_time_scales_with_debug_day_secs() {
|
||||
let mod_time = datetime!(2025-01-15 10:30:45 UTC);
|
||||
temp_env::with_var(ENV_ILM_DEBUG_DAY_SECS, Some("1"), || {
|
||||
@@ -3435,6 +3465,7 @@ mod tests {
|
||||
|
||||
// days == 0 still yields the immediate-expiry sentinel regardless of the switch.
|
||||
#[test]
|
||||
#[serial]
|
||||
fn expected_expiry_time_zero_days_ignores_debug_day_secs() {
|
||||
let mod_time = datetime!(2025-06-01 12:00:00 UTC);
|
||||
temp_env::with_var(ENV_ILM_DEBUG_DAY_SECS, Some("2"), || {
|
||||
@@ -3445,6 +3476,7 @@ mod tests {
|
||||
// (③) Interaction with an explicit RUSTFS_ILM_PROCESS_TIME: the deadline offset
|
||||
// uses the accelerated day length, but the rounding boundary honors PROCESS_TIME.
|
||||
#[test]
|
||||
#[serial]
|
||||
fn expected_expiry_time_debug_day_secs_respects_explicit_process_time() {
|
||||
let mod_time = datetime!(2025-01-15 10:30:00 UTC);
|
||||
// day == 10s, but round up to the next 60s (PROCESS_TIME) boundary.
|
||||
@@ -3461,6 +3493,7 @@ mod tests {
|
||||
|
||||
// (③) With the switch unset, an explicit PROCESS_TIME behaves exactly as before.
|
||||
#[test]
|
||||
#[serial]
|
||||
fn expected_expiry_time_unset_debug_day_secs_matches_legacy_process_time() {
|
||||
let mod_time = datetime!(2025-01-15 10:30:45 UTC);
|
||||
temp_env::with_var_unset(ENV_ILM_DEBUG_DAY_SECS, || {
|
||||
@@ -3488,6 +3521,7 @@ mod tests {
|
||||
|
||||
// The abort-incomplete-multipart deadline path also scales through the switch.
|
||||
#[test]
|
||||
#[serial]
|
||||
fn abort_incomplete_multipart_due_scales_with_debug_day_secs() {
|
||||
use s3s::dto::AbortIncompleteMultipartUpload;
|
||||
let initiated = datetime!(2025-01-15 10:30:45 UTC);
|
||||
@@ -3532,6 +3566,7 @@ mod tests {
|
||||
// (⑤ evaluator seam) A Days=1 rule fires under RUSTFS_ILM_DEBUG_DAY_SECS=1 once
|
||||
// `now` advances a few seconds past a mod_time only ~seconds in the past.
|
||||
#[test]
|
||||
#[serial]
|
||||
fn eval_inner_expires_days_one_rule_under_debug_day_secs() {
|
||||
let lc = BucketLifecycleConfiguration {
|
||||
expiry_updated_at: None,
|
||||
@@ -3580,6 +3615,7 @@ mod tests {
|
||||
|
||||
// Absolute Date-based rules must NOT scale with the switch (regression guard).
|
||||
#[test]
|
||||
#[serial]
|
||||
fn eval_inner_date_rule_ignores_debug_day_secs() {
|
||||
let expiry_date = datetime!(2025-06-01 00:00:00 UTC);
|
||||
let lc = BucketLifecycleConfiguration {
|
||||
@@ -3837,6 +3873,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn eval_inner_triggers_delete_all_versions_when_expired_object_all_versions_set() {
|
||||
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
|
||||
let lc = BucketLifecycleConfiguration {
|
||||
@@ -3875,6 +3912,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn expired_object_all_versions_does_not_apply_to_current_delete_marker() {
|
||||
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).expect("fixed timestamp should be valid");
|
||||
let lc = BucketLifecycleConfiguration {
|
||||
@@ -3904,6 +3942,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn eval_inner_uses_delete_action_when_all_versions_not_set() {
|
||||
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
|
||||
let lc = BucketLifecycleConfiguration {
|
||||
@@ -4022,6 +4061,7 @@ mod tests {
|
||||
use super::*;
|
||||
use proptest::prelude::*;
|
||||
use s3s::dto::{NoncurrentVersionExpiration, Tag};
|
||||
use serial_test::serial;
|
||||
|
||||
const DAY_SECS: i64 = 86400;
|
||||
|
||||
@@ -4252,6 +4292,7 @@ mod tests {
|
||||
/// combination, and must be deterministic: the same input
|
||||
/// evaluated twice yields an identical event.
|
||||
#[test]
|
||||
#[serial]
|
||||
fn eval_inner_never_panics_and_is_deterministic(
|
||||
rules in prop::collection::vec(arb_rule(), 0..4),
|
||||
obj in arb_object_opts(),
|
||||
@@ -4391,6 +4432,7 @@ mod tests {
|
||||
/// candidate set — earliest due wins, ties prefer delete-class —
|
||||
/// and must be `NoneAction` exactly when that set is empty.
|
||||
#[test]
|
||||
#[serial]
|
||||
fn eval_inner_winner_matches_selection_oracle(
|
||||
rules in prop::collection::vec(arb_selection_rule(), 0..5),
|
||||
mod_off in 0i64..(2 * DAY_SECS),
|
||||
@@ -4444,6 +4486,7 @@ mod tests {
|
||||
/// non-decreasing in `days` (days == 0 maps to UNIX_EPOCH, below
|
||||
/// any post-1970 deadline).
|
||||
#[test]
|
||||
#[serial]
|
||||
fn expected_expiry_time_is_monotonic_in_days(
|
||||
mod_off in 0i64..(3650 * DAY_SECS),
|
||||
d1 in 0i32..2000,
|
||||
@@ -4465,6 +4508,7 @@ mod tests {
|
||||
/// to the next whole-day boundary: the result is day-aligned, not
|
||||
/// before `mod_time + days`, and less than one boundary beyond it.
|
||||
#[test]
|
||||
#[serial]
|
||||
fn expected_expiry_time_lands_on_default_day_boundary(
|
||||
mod_off in 0i64..(3650 * DAY_SECS),
|
||||
days in 1i32..2000,
|
||||
@@ -4482,6 +4526,7 @@ mod tests {
|
||||
/// to that boundary instead: aligned to it, never early, and less
|
||||
/// than one boundary late.
|
||||
#[test]
|
||||
#[serial]
|
||||
fn expected_expiry_time_lands_on_explicit_process_boundary(
|
||||
mod_off in 0i64..(365 * DAY_SECS),
|
||||
days in 1i32..400,
|
||||
|
||||
@@ -73,6 +73,7 @@ walkdir = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
criterion = { workspace = true, features = ["html_reports"] }
|
||||
serial_test = { workspace = true }
|
||||
temp-env = { workspace = true, features = ["async_closure"] }
|
||||
tempfile = { workspace = true }
|
||||
tokio = { workspace = true, features = ["test-util", "macros", "fs", "rt-multi-thread"] }
|
||||
|
||||
@@ -1484,6 +1484,7 @@ mod tests {
|
||||
ENV_CAPACITY_SAMPLE_RATE, ENV_CAPACITY_STAT_TIMEOUT, ENV_CAPACITY_WRITE_FREQUENCY_THRESHOLD,
|
||||
ENV_CAPACITY_WRITE_TRIGGER_DELAY,
|
||||
};
|
||||
use serial_test::serial;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
@@ -1668,6 +1669,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn test_config_getter_defaults() {
|
||||
for (env_var, getter, default, _, _) in config_getter_cases() {
|
||||
temp_env::with_var(env_var, None::<&str>, || {
|
||||
@@ -1677,6 +1679,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn test_config_getter_env_overrides() {
|
||||
for (env_var, getter, _, override_value, expected) in config_getter_cases() {
|
||||
temp_env::with_var(env_var, Some(override_value), || {
|
||||
@@ -1686,6 +1689,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn test_zero_env_values_clamp_to_defaults() {
|
||||
// A zero threshold makes small disks report 0 bytes; a zero timeout
|
||||
// (with dynamic timeout off) makes every scan fail. Both must fall
|
||||
@@ -1705,6 +1709,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_update_capacity_preserves_retrieval_metadata() {
|
||||
let manager = HybridCapacityManager::from_env();
|
||||
|
||||
@@ -1720,6 +1725,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_record_write_operation() {
|
||||
let manager = HybridCapacityManager::from_env();
|
||||
|
||||
@@ -1730,6 +1736,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_write_frequency_window() {
|
||||
let manager = HybridCapacityManager::from_env();
|
||||
|
||||
@@ -1817,6 +1824,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn test_recent_write_count_ignores_future_buckets() {
|
||||
let record = WriteRecord::new();
|
||||
record.write_buckets[0].store(120, 3);
|
||||
@@ -1830,6 +1838,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_needs_fast_update() {
|
||||
let manager = HybridCapacityManager::from_env();
|
||||
|
||||
@@ -1846,6 +1855,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_cache_age_tracking() {
|
||||
let manager = HybridCapacityManager::from_env();
|
||||
|
||||
@@ -1865,6 +1875,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_data_source_tracking() {
|
||||
let manager = HybridCapacityManager::from_env();
|
||||
|
||||
@@ -1880,6 +1891,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_needs_fast_update_waits_for_write_trigger_delay() {
|
||||
let manager = create_isolated_manager(HybridStrategyConfig {
|
||||
scheduled_update_interval: Duration::from_secs(60),
|
||||
@@ -1910,6 +1922,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_needs_fast_update_respects_enable_write_trigger() {
|
||||
let manager = create_isolated_manager(HybridStrategyConfig {
|
||||
scheduled_update_interval: Duration::from_secs(60),
|
||||
@@ -1936,6 +1949,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_concurrent_access() {
|
||||
let manager = Arc::new(HybridCapacityManager::from_env());
|
||||
let mut handles = Vec::new();
|
||||
@@ -1962,6 +1976,7 @@ mod tests {
|
||||
// exact under heavy same-second contention or the frequency window (and the
|
||||
// write-trigger decision) would undercount.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 8)]
|
||||
#[serial]
|
||||
async fn test_record_write_operation_lock_free_is_exact_under_contention() {
|
||||
let manager = Arc::new(HybridCapacityManager::from_env());
|
||||
let mut handles = Vec::new();
|
||||
@@ -1986,6 +2001,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_performance_overhead() {
|
||||
let manager = Arc::new(HybridCapacityManager::from_env());
|
||||
let start = Instant::now();
|
||||
@@ -2002,6 +2018,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_refresh_or_join_singleflight() {
|
||||
let manager = Arc::new(HybridCapacityManager::from_env());
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
@@ -2041,6 +2058,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_refresh_or_join_recovers_after_leader_cancellation() {
|
||||
let manager = Arc::new(HybridCapacityManager::from_env());
|
||||
|
||||
@@ -2069,6 +2087,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_refresh_or_join_cancelled_leader_unblocks_joiner() {
|
||||
let manager = Arc::new(HybridCapacityManager::from_env());
|
||||
|
||||
@@ -2096,6 +2115,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_spawn_refresh_if_needed_deduplicates_background_refresh() {
|
||||
let manager = Arc::new(HybridCapacityManager::from_env());
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
@@ -2133,6 +2153,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_record_write_operation_with_scope_token_marks_dirty_disks() {
|
||||
let manager = create_isolated_manager(HybridStrategyConfig::default());
|
||||
let token = uuid::Uuid::new_v4();
|
||||
@@ -2156,6 +2177,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_get_dirty_disks_drains_global_dirty_scope_registry() {
|
||||
let manager = create_isolated_manager(HybridStrategyConfig::default());
|
||||
record_global_dirty_scope(CapacityScope {
|
||||
@@ -2175,6 +2197,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_update_capacity_recomputes_total_from_disk_cache_for_subset_refresh() {
|
||||
let manager = create_isolated_manager(HybridStrategyConfig::default());
|
||||
|
||||
@@ -2285,6 +2308,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_update_capacity_degraded_full_refresh_merges_cache_and_does_not_oscillate() {
|
||||
let manager = create_isolated_manager(HybridStrategyConfig::default());
|
||||
|
||||
@@ -2330,6 +2354,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_update_capacity_degraded_with_empty_per_disk_serves_merged_cache() {
|
||||
let manager = create_isolated_manager(HybridStrategyConfig::default());
|
||||
manager.update_capacity(full_two_disk_update(), DataSource::RealTime).await;
|
||||
@@ -2359,6 +2384,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_update_capacity_degraded_without_complete_cache_keeps_partial_sum() {
|
||||
let manager = create_isolated_manager(HybridStrategyConfig::default());
|
||||
|
||||
@@ -2399,6 +2425,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_commit_keeps_dirty_marks_recorded_after_scan_start() {
|
||||
let manager = create_isolated_manager(HybridStrategyConfig::default());
|
||||
let disk = scope_disk("node-a", "/tmp/disk-a");
|
||||
@@ -2429,6 +2456,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_commit_clears_dirty_marks_recorded_before_scan_start() {
|
||||
let manager = create_isolated_manager(HybridStrategyConfig::default());
|
||||
let disk = scope_disk("node-a", "/tmp/disk-a");
|
||||
@@ -2449,6 +2477,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_retain_dirty_disks_within_drops_ghost_entries() {
|
||||
let manager = create_isolated_manager(HybridStrategyConfig::default());
|
||||
let local = scope_disk("node-a", "/tmp/disk-a");
|
||||
@@ -2467,6 +2496,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_spawn_refresh_recovers_from_construction_panic() {
|
||||
let manager = create_isolated_manager(HybridStrategyConfig::default());
|
||||
|
||||
@@ -2533,6 +2563,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
#[serial]
|
||||
async fn test_refresh_or_join_joiner_times_out_when_leader_wedges() {
|
||||
let manager = create_isolated_manager(HybridStrategyConfig::default());
|
||||
|
||||
@@ -2560,6 +2591,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_refresh_or_join_returns_cluster_total_for_dirty_subset() {
|
||||
let manager = create_isolated_manager(HybridStrategyConfig::default());
|
||||
|
||||
@@ -2641,6 +2673,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_config_from_env() {
|
||||
let config = HybridStrategyConfig::from_env();
|
||||
|
||||
@@ -2654,6 +2687,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_config_from_env_with_override() {
|
||||
temp_env::with_var(ENV_CAPACITY_SCHEDULED_INTERVAL, Some("600"), || {
|
||||
let config = HybridStrategyConfig::from_env();
|
||||
|
||||
@@ -1069,6 +1069,7 @@ mod tests {
|
||||
#[cfg(unix)]
|
||||
use rustfs_config::ENV_CAPACITY_FOLLOW_SYMLINKS;
|
||||
use rustfs_config::{ENV_CAPACITY_MAX_FILES_THRESHOLD, ENV_CAPACITY_SAMPLE_RATE};
|
||||
use serial_test::serial;
|
||||
|
||||
/// Reference implementation using unbounded `u128` arithmetic, clamped to
|
||||
/// `u64::MAX`, used as the source of truth for the sampling extrapolation.
|
||||
@@ -1273,6 +1274,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_get_dir_size_async_nonexistent_directory() {
|
||||
let result = get_dir_size_async(Path::new("/nonexistent/path")).await;
|
||||
assert!(result.is_err());
|
||||
@@ -1646,6 +1648,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_metadata_incomplete_aggregate_does_not_replace_disk_cache() {
|
||||
use std::fs::File;
|
||||
use std::io::Write;
|
||||
@@ -1780,6 +1783,7 @@ mod tests {
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_get_dir_size_async_ignores_symlink_targets_when_follow_disabled() {
|
||||
use std::fs::File;
|
||||
use std::io::Write;
|
||||
@@ -1805,6 +1809,7 @@ mod tests {
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_get_dir_size_async_counts_symlink_targets_when_follow_enabled() {
|
||||
use std::fs::File;
|
||||
use std::io::Write;
|
||||
|
||||
@@ -75,7 +75,7 @@ pub struct BucketReplicationBandwidthStats {
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct BucketReplicationMetricsSnapshot {
|
||||
pub struct BucketReplicationStats {
|
||||
pub bucket: String,
|
||||
pub total_failed_bytes: u64,
|
||||
pub total_failed_count: u64,
|
||||
@@ -107,7 +107,7 @@ pub struct BucketReplicationMetricsSnapshot {
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub(crate) struct BucketReplicationRuntimeStats {
|
||||
pub(crate) stats: BucketReplicationMetricsSnapshot,
|
||||
pub(crate) stats: BucketReplicationStats,
|
||||
pub(crate) target_flows: Vec<BucketReplicationTargetFlowStats>,
|
||||
}
|
||||
|
||||
@@ -182,7 +182,7 @@ fn push_proxy_request_result_metrics(
|
||||
}
|
||||
}
|
||||
|
||||
pub fn collect_bucket_replication_metrics(stats: &[BucketReplicationMetricsSnapshot]) -> Vec<PrometheusMetric> {
|
||||
pub fn collect_bucket_replication_metrics(stats: &[BucketReplicationStats]) -> Vec<PrometheusMetric> {
|
||||
if stats.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
@@ -572,7 +572,7 @@ mod tests {
|
||||
#[test]
|
||||
fn test_collect_bucket_replication_metrics() {
|
||||
let stats = vec![BucketReplicationRuntimeStats {
|
||||
stats: BucketReplicationMetricsSnapshot {
|
||||
stats: BucketReplicationStats {
|
||||
bucket: "b1".to_string(),
|
||||
total_failed_bytes: 64,
|
||||
total_failed_count: 2,
|
||||
@@ -876,7 +876,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_collect_bucket_replication_metrics_empty() {
|
||||
let stats: Vec<BucketReplicationMetricsSnapshot> = Vec::new();
|
||||
let stats: Vec<BucketReplicationStats> = Vec::new();
|
||||
let metrics = collect_bucket_replication_metrics(&stats);
|
||||
assert!(metrics.is_empty());
|
||||
}
|
||||
|
||||
@@ -48,7 +48,7 @@ pub(crate) use bucket_replication::{
|
||||
BucketReplicationTargetFlowStats, collect_bucket_replication_backlog_metrics, collect_bucket_replication_runtime_metrics,
|
||||
};
|
||||
pub use bucket_replication::{
|
||||
BucketReplicationBandwidthStats, BucketReplicationMetricsSnapshot, BucketReplicationTargetStats,
|
||||
BucketReplicationBandwidthStats, BucketReplicationStats, BucketReplicationTargetStats,
|
||||
collect_bucket_replication_bandwidth_metrics, collect_bucket_replication_metrics,
|
||||
};
|
||||
pub use cluster::{ClusterStats, collect_cluster_metrics};
|
||||
@@ -68,8 +68,8 @@ pub(crate) use notification::collect_notification_runtime_metrics;
|
||||
pub use notification::{NotificationStats, collect_notification_metrics};
|
||||
pub(crate) use notification_target::{NotificationTargetRuntimeStats, collect_notification_target_runtime_metrics};
|
||||
pub use notification_target::{NotificationTargetStats, collect_notification_target_metrics};
|
||||
pub use replication::{ReplicationMetricsSnapshot, collect_replication_metrics};
|
||||
pub(crate) use replication::{ReplicationRuntimeStats, collect_replication_runtime_metrics};
|
||||
pub use replication::{ReplicationStats, collect_replication_metrics};
|
||||
pub(crate) use request::{ApiRequestMetricSupport, ApiRequestStats, collect_request_metrics};
|
||||
pub use resource::{ResourceStats, collect_resource_metrics};
|
||||
pub(crate) use scanner::{ScannerRuntimeStats, collect_scanner_runtime_metrics};
|
||||
|
||||
@@ -22,7 +22,7 @@ use crate::metrics::schema::replication::*;
|
||||
|
||||
/// Replication statistics.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct ReplicationMetricsSnapshot {
|
||||
pub struct ReplicationStats {
|
||||
/// Average number of active replication workers
|
||||
pub average_active_workers: f64,
|
||||
/// Average queued bytes since server start
|
||||
@@ -54,13 +54,13 @@ pub struct ReplicationMetricsSnapshot {
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub(crate) struct ReplicationRuntimeStats {
|
||||
pub(crate) server: String,
|
||||
pub(crate) stats: ReplicationMetricsSnapshot,
|
||||
pub(crate) stats: ReplicationStats,
|
||||
}
|
||||
|
||||
/// Collects replication metrics from the given stats.
|
||||
///
|
||||
/// Returns a vector of Prometheus metrics for replication statistics.
|
||||
pub fn collect_replication_metrics(stats: &ReplicationMetricsSnapshot) -> Vec<PrometheusMetric> {
|
||||
pub fn collect_replication_metrics(stats: &ReplicationStats) -> Vec<PrometheusMetric> {
|
||||
vec![
|
||||
PrometheusMetric::from_descriptor(&REPLICATION_AVERAGE_ACTIVE_WORKERS_MD, stats.average_active_workers),
|
||||
PrometheusMetric::from_descriptor(&REPLICATION_AVERAGE_QUEUED_BYTES_MD, stats.average_queued_bytes as f64),
|
||||
@@ -120,7 +120,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_collect_replication_metrics() {
|
||||
let stats = ReplicationMetricsSnapshot {
|
||||
let stats = ReplicationStats {
|
||||
average_active_workers: 8.5,
|
||||
average_queued_bytes: 1024 * 1024 * 40,
|
||||
average_queued_count: 240,
|
||||
@@ -182,7 +182,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_collect_replication_metrics_default() {
|
||||
let stats = ReplicationMetricsSnapshot::default();
|
||||
let stats = ReplicationStats::default();
|
||||
let metrics = collect_replication_metrics(&stats);
|
||||
|
||||
assert_eq!(metrics.len(), 13);
|
||||
@@ -194,7 +194,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn replication_stats_struct_literal_keeps_legacy_fields() {
|
||||
let stats = ReplicationMetricsSnapshot {
|
||||
let stats = ReplicationStats {
|
||||
average_active_workers: 1.0,
|
||||
average_queued_bytes: 2,
|
||||
average_queued_count: 3,
|
||||
|
||||
@@ -2811,14 +2811,14 @@ mod tests {
|
||||
#[test]
|
||||
fn replication_proxy_bucket_keys_detect_removed_buckets() {
|
||||
let previous = repl_proxy_bucket_live_keys(&[BucketReplicationRuntimeStats {
|
||||
stats: crate::metrics::BucketReplicationMetricsSnapshot {
|
||||
stats: crate::metrics::BucketReplicationStats {
|
||||
bucket: "photos".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
}]);
|
||||
let current = repl_proxy_bucket_live_keys(&[BucketReplicationRuntimeStats {
|
||||
stats: crate::metrics::BucketReplicationMetricsSnapshot {
|
||||
stats: crate::metrics::BucketReplicationStats {
|
||||
bucket: "logs".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
|
||||
@@ -21,12 +21,12 @@
|
||||
use crate::metrics::collectors::scanner::{ScannerActiveBucketDriveStats, ScannerBucketDriveResultStats, ScannerSourceWorkStats};
|
||||
use crate::metrics::collectors::{
|
||||
ApiRequestMetricSupport, ApiRequestStats, BucketReplicationBacklogStats, BucketReplicationBandwidthStats,
|
||||
BucketReplicationMetricsSnapshot, BucketReplicationRuntimeStats, BucketReplicationTargetBacklogStats,
|
||||
BucketReplicationTargetFlowStats, BucketReplicationTargetStats, BucketStats, BucketUsageStats, ClusterConfigStats,
|
||||
ClusterHealthStats, ClusterStats, ClusterUsageStats, CompressionClusterStats, CpuStats, DiskStats, DriveCountStats,
|
||||
DriveDetailedStats, DriveRuntimeDetailedStats, ErasureSetStats, HostNetworkStats, IamStats, IlmActionTaskStats,
|
||||
IlmBackpressureStats, IlmQueueTaskStats, IlmRuntimeStats, IlmStats, IlmTaskEventStats, MemoryStats, NetworkStats,
|
||||
ProcessStats, ProcessStatusType, ReplicationMetricsSnapshot, ResourceStats, ScannerRuntimeStats, ScannerStats,
|
||||
BucketReplicationRuntimeStats, BucketReplicationStats, BucketReplicationTargetBacklogStats, BucketReplicationTargetFlowStats,
|
||||
BucketReplicationTargetStats, BucketStats, BucketUsageStats, ClusterConfigStats, ClusterHealthStats, ClusterStats,
|
||||
ClusterUsageStats, CompressionClusterStats, CpuStats, DiskStats, DriveCountStats, DriveDetailedStats,
|
||||
DriveRuntimeDetailedStats, ErasureSetStats, HostNetworkStats, IamStats, IlmActionTaskStats, IlmBackpressureStats,
|
||||
IlmQueueTaskStats, IlmRuntimeStats, IlmStats, IlmTaskEventStats, MemoryStats, NetworkStats, ProcessStats, ProcessStatusType,
|
||||
ReplicationStats, ResourceStats, ScannerRuntimeStats, ScannerStats,
|
||||
};
|
||||
use crate::metrics::runtime_sources::{ObsIlmRuntimeSnapshot, bucket_monitor_handle, iam_metrics_snapshot, ilm_runtime_snapshot};
|
||||
use crate::metrics::{
|
||||
@@ -266,7 +266,7 @@ fn bucket_replication_detail_from_snapshot(stats: ObsBucketReplicationStatsSnaps
|
||||
|
||||
BucketReplicationRuntimeStats {
|
||||
target_flows,
|
||||
stats: BucketReplicationMetricsSnapshot {
|
||||
stats: BucketReplicationStats {
|
||||
bucket,
|
||||
total_failed_bytes: stats.total_failed_bytes,
|
||||
total_failed_count: stats.total_failed_count,
|
||||
@@ -298,7 +298,7 @@ fn bucket_replication_detail_from_snapshot(stats: ObsBucketReplicationStatsSnaps
|
||||
}
|
||||
}
|
||||
|
||||
async fn obs_site_replication_stats() -> ReplicationMetricsSnapshot {
|
||||
async fn obs_site_replication_stats() -> ReplicationStats {
|
||||
let current_data_transfer_rate = obs_bucket_replication_bandwidth_stats()
|
||||
.into_iter()
|
||||
.flatten()
|
||||
@@ -306,7 +306,7 @@ async fn obs_site_replication_stats() -> ReplicationMetricsSnapshot {
|
||||
.sum::<f64>();
|
||||
let stats = obs_replication_site_stats_snapshot(current_data_transfer_rate).await;
|
||||
|
||||
ReplicationMetricsSnapshot {
|
||||
ReplicationStats {
|
||||
average_active_workers: stats.average_active_workers,
|
||||
average_queued_bytes: stats.average_queued_bytes,
|
||||
average_queued_count: stats.average_queued_count,
|
||||
@@ -648,7 +648,7 @@ pub fn collect_bucket_replication_bandwidth_stats() -> Vec<BucketReplicationBand
|
||||
}
|
||||
|
||||
/// Collect bucket and target level replication stats from the global replication runtime.
|
||||
pub async fn collect_bucket_replication_detail_stats() -> Vec<BucketReplicationMetricsSnapshot> {
|
||||
pub async fn collect_bucket_replication_detail_stats() -> Vec<BucketReplicationStats> {
|
||||
obs_bucket_replication_stats_snapshot()
|
||||
.await
|
||||
.into_iter()
|
||||
@@ -662,7 +662,7 @@ pub(crate) async fn collect_bucket_replication_stats_bundle()
|
||||
}
|
||||
|
||||
/// Collect site-level replication stats from the global replication runtime.
|
||||
pub async fn collect_replication_stats() -> ReplicationMetricsSnapshot {
|
||||
pub async fn collect_replication_stats() -> ReplicationStats {
|
||||
obs_site_replication_stats().await
|
||||
}
|
||||
|
||||
|
||||
@@ -265,6 +265,78 @@ pub fn active_replication_rule_destination_arns(config: &ReplicationConfiguratio
|
||||
arns
|
||||
}
|
||||
|
||||
/// Deployment id extracted from a site-replication target ARN
|
||||
/// (`arn:{rustfs|minio}:replication::<deployment-id>:<bucket>`), or `None`
|
||||
/// for an operator-authored ARN.
|
||||
pub fn replication_target_arn_deployment_id(arn: &str) -> Option<String> {
|
||||
let parts: Vec<_> = arn.split(':').collect();
|
||||
if parts.len() == 6
|
||||
&& parts[0] == "arn"
|
||||
&& matches!(parts[1], "rustfs" | "minio")
|
||||
&& parts[2] == "replication"
|
||||
&& !parts[4].is_empty()
|
||||
{
|
||||
return Some(parts[4].to_string());
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
/// Whether `rule` is a site-replication rule (`site-repl-*` id) owned by the
|
||||
/// local site's reconciler rather than authored by an operator.
|
||||
pub fn is_site_replication_rule(rule: &ReplicationRule) -> bool {
|
||||
rule.id.as_deref().is_some_and(|id| id.starts_with("site-repl-"))
|
||||
}
|
||||
|
||||
/// Merge an incoming replication config into the local one.
|
||||
///
|
||||
/// `site-repl-*` rules encode the *holder's* outbound direction — their
|
||||
/// destination ARN names another site — so applying an external rule set
|
||||
/// verbatim replaces the local reverse rule with one this site can never
|
||||
/// satisfy (no bucket target backs it) and replication silently stops. Only
|
||||
/// operator-authored rules travel: the site-replication peer ingestion path
|
||||
/// and the S3 put/delete-bucket-replication path both keep the local site's
|
||||
/// `site-repl-*` rules through this merge. `incoming == None` models a
|
||||
/// delete of the operator-authored rules.
|
||||
pub fn merge_incoming_replication_config(
|
||||
incoming: Option<ReplicationConfiguration>,
|
||||
local: Option<ReplicationConfiguration>,
|
||||
) -> Option<ReplicationConfiguration> {
|
||||
let incoming_role = incoming.as_ref().map(|config| config.role.clone()).unwrap_or_default();
|
||||
// Operator rules first, then the local site rules — the same order the
|
||||
// site-replication reconciler produces, so its no-op check matches and
|
||||
// the bucket metadata is written once per broadcast, not twice.
|
||||
let mut rules: Vec<ReplicationRule> = incoming
|
||||
.into_iter()
|
||||
.flat_map(|config| config.rules)
|
||||
.filter(|rule| !is_site_replication_rule(rule))
|
||||
.collect();
|
||||
rules.extend(
|
||||
local
|
||||
.into_iter()
|
||||
.flat_map(|config| config.rules)
|
||||
.filter(is_site_replication_rule),
|
||||
);
|
||||
|
||||
if rules.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
for (index, rule) in rules.iter_mut().enumerate() {
|
||||
rule.priority = Some(i32::try_from(index + 1).unwrap_or(i32::MAX));
|
||||
}
|
||||
|
||||
// A site-replication ARN in `role` is the sender's, and the reconciler's
|
||||
// per-peer target lookup reads it — carrying it over would pin the
|
||||
// receiver's targets to the sender's identity.
|
||||
let role = match replication_target_arn_deployment_id(&incoming_role) {
|
||||
Some(_) => String::new(),
|
||||
None => incoming_role,
|
||||
};
|
||||
|
||||
Some(ReplicationConfiguration { role, rules })
|
||||
}
|
||||
|
||||
pub fn replication_target_arns(config: &ReplicationConfiguration) -> HashSet<String> {
|
||||
let role = config.role.trim();
|
||||
if !role.is_empty() {
|
||||
|
||||
@@ -32,7 +32,8 @@ pub use config::{
|
||||
ObjectOpts, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS,
|
||||
REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS,
|
||||
ReplicationConfigStructureError, ReplicationConfigurationExt, ReplicationTargetValidationError,
|
||||
active_replication_rule_destination_arns, invalid_replication_config_status_field, replication_target_arns,
|
||||
active_replication_rule_destination_arns, invalid_replication_config_status_field, is_site_replication_rule,
|
||||
merge_incoming_replication_config, replication_target_arn_deployment_id, replication_target_arns,
|
||||
should_remove_replication_target, unsupported_replication_config_field, validate_replication_config_structure,
|
||||
validate_replication_config_target_arns,
|
||||
};
|
||||
|
||||
@@ -103,7 +103,8 @@ hex-simd.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
tracing-subscriber = { workspace = true, features = ["json", "env-filter", "time"] }
|
||||
temp-env = { workspace = true, features = ["async_closure"] }
|
||||
serial_test = { workspace = true }
|
||||
temp-env = { workspace = true }
|
||||
tempfile = { workspace = true }
|
||||
uuid = { workspace = true, features = ["v4", "serde", "fast-rng", "macro-diagnostics"] }
|
||||
tokio = { workspace = true, features = ["test-util", "fs", "rt-multi-thread"] }
|
||||
|
||||
@@ -599,8 +599,10 @@ impl ScannerConfigObjectDelete for ECStore {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use serial_test::serial;
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn runtime_tier_names_serves_cached_arc_within_ttl() {
|
||||
reset_tier_name_cache_for_test();
|
||||
// The tier config manager is unconfigured in unit tests, so the
|
||||
@@ -614,6 +616,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn foreground_read_guard_tracks_stream_lifetime() {
|
||||
reset_foreground_read_activity_for_test();
|
||||
assert_eq!(current_foreground_read_activity(), 0);
|
||||
@@ -627,6 +630,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn foreground_read_activity_keeps_larger_signal() {
|
||||
reset_foreground_read_activity_for_test();
|
||||
let _guard = ForegroundReadGuard::new();
|
||||
@@ -639,6 +643,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn scanner_runtime_guard_tracks_runtime_lifetime() {
|
||||
reset_scanner_runtime_instances_for_test();
|
||||
assert!(!scanner_runtime_initialized());
|
||||
|
||||
@@ -868,6 +868,7 @@ mod tests {
|
||||
SCANNER_CYCLE_MAX_DIRECTORIES, SCANNER_CYCLE_MAX_DURATION, SCANNER_CYCLE_MAX_OBJECTS, SCANNER_DELAY, SCANNER_IDLE_MODE,
|
||||
SCANNER_SPEED, SCANNER_SUB_SYS, ScannerSpeed,
|
||||
};
|
||||
use serial_test::serial;
|
||||
use std::collections::HashMap;
|
||||
use std::time::Duration;
|
||||
use temp_env::{with_var, with_var_unset};
|
||||
@@ -915,6 +916,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn scanner_runtime_config_uses_persisted_values_when_env_is_unset() {
|
||||
let config = server_config_with_scanner(&[
|
||||
(SCANNER_SPEED, "slow"),
|
||||
@@ -942,6 +944,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn scanner_runtime_config_normalizes_persisted_default_speed() {
|
||||
let config = server_config_with_scanner(&[(SCANNER_SPEED, "default")]);
|
||||
|
||||
@@ -957,6 +960,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn scanner_runtime_config_prefers_env_over_persisted_config() {
|
||||
let config = server_config_with_scanner(&[(SCANNER_SPEED, "slowest"), (SCANNER_CYCLE, "600")]);
|
||||
|
||||
@@ -973,6 +977,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn scanner_runtime_config_prefers_heal_bitrot_cycle_over_scanner_compat_config() {
|
||||
let config = server_config_with_scanner_and_heal(&[(SCANNER_BITROT_CYCLE, "3600")], &[(HEAL_BITROT_CYCLE, "off")]);
|
||||
|
||||
@@ -985,6 +990,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn scanner_runtime_config_marks_scanner_bitrot_cycle_as_compat_source() {
|
||||
let config = server_config_with_scanner(&[(SCANNER_BITROT_CYCLE, "3600")]);
|
||||
|
||||
@@ -1001,6 +1007,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn scanner_runtime_config_normalizes_persisted_default_bitrot_cycles() {
|
||||
let default_cycle = DEFAULT_HEAL_BITROT_CYCLE_SECS.to_string();
|
||||
for config in [
|
||||
@@ -1025,6 +1032,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn scanner_runtime_config_validation_rejects_invalid_persisted_speed_with_env_override() {
|
||||
let config = server_config_with_scanner(&[(SCANNER_SPEED, "warp")]);
|
||||
|
||||
@@ -1058,6 +1066,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn scanner_runtime_config_uses_derived_delay_for_excessive_env_override() {
|
||||
let config = server_config_with_scanner(&[(SCANNER_SPEED, "slow")]);
|
||||
|
||||
@@ -1078,6 +1087,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn scanner_runtime_config_status_reports_value_sources() {
|
||||
let config = server_config_with_scanner(&[(SCANNER_CYCLE_MAX_OBJECTS, "100"), (SCANNER_CACHE_SAVE_TIMEOUT, "5")]);
|
||||
|
||||
@@ -1098,6 +1108,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn applied_runtime_config_is_the_authoritative_scheduler_state() {
|
||||
let config = server_config_with_scanner(&[(SCANNER_CYCLE, "321")]);
|
||||
|
||||
@@ -1114,6 +1125,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn scanner_runtime_config_status_reports_persisted_pacing_overrides() {
|
||||
let config = server_config_with_scanner(&[("delay", "3.5"), ("max_wait", "7")]);
|
||||
|
||||
@@ -1135,6 +1147,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn scanner_runtime_config_status_prefers_env_pacing_overrides() {
|
||||
let config = server_config_with_scanner(&[("delay", "3.5"), ("max_wait", "7")]);
|
||||
|
||||
@@ -1156,6 +1169,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn scanner_runtime_config_status_preserves_subsecond_max_wait() {
|
||||
let config = server_config_with_scanner(&[(SCANNER_SPEED, "fast")]);
|
||||
|
||||
|
||||
@@ -19,6 +19,7 @@ use crate::{
|
||||
ScannerObjectInfo as ObjectInfo, ScannerObjectOptions as ObjectOptions, ScannerPutObjReader as PutObjReader,
|
||||
init_bucket_metadata_sys_for_scanner_tests, init_ecstore_config_for_scanner_tests, init_local_disks_with_instance_ctx,
|
||||
};
|
||||
use serial_test::serial;
|
||||
use std::collections::HashMap;
|
||||
use std::io::Cursor;
|
||||
use std::task::Poll;
|
||||
@@ -361,6 +362,7 @@ fn test_initial_scanner_delay_uses_configured_start_delay() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn test_initial_scanner_delay_uses_cycle_without_explicit_start_delay() {
|
||||
with_var(ENV_SCANNER_CYCLE, Some("120"), || {
|
||||
crate::runtime_config::refresh_scanner_runtime_config_for_tests();
|
||||
@@ -407,6 +409,7 @@ fn test_initial_scanner_delay_keeps_delay_for_replication_without_buckets() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn test_scanner_cycle_max_duration_uses_env() {
|
||||
with_var(ENV_SCANNER_CYCLE_MAX_DURATION_SECS, Some("42"), || {
|
||||
assert_eq!(scanner_cycle_max_duration(), Some(Duration::from_secs(42)));
|
||||
@@ -414,6 +417,7 @@ fn test_scanner_cycle_max_duration_uses_env() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn test_scanner_cycle_max_duration_default_is_disabled() {
|
||||
with_var_unset(ENV_SCANNER_CYCLE_MAX_DURATION_SECS, || {
|
||||
assert_eq!(scanner_cycle_max_duration(), None);
|
||||
@@ -457,6 +461,7 @@ async fn test_scanner_cycle_budget_drop_cancels_child_without_elapsed() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn test_scanner_cycle_budget_config_uses_work_budget_env() {
|
||||
with_var(ENV_SCANNER_CYCLE_MAX_OBJECTS, Some("100"), || {
|
||||
with_var(ENV_SCANNER_CYCLE_MAX_DIRECTORIES, Some("25"), || {
|
||||
@@ -468,6 +473,7 @@ fn test_scanner_cycle_budget_config_uses_work_budget_env() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn test_scanner_cycle_budget_config_disables_zero_work_budgets() {
|
||||
with_var(ENV_SCANNER_CYCLE_MAX_OBJECTS, Some("0"), || {
|
||||
with_var(ENV_SCANNER_CYCLE_MAX_DIRECTORIES, Some("0"), || {
|
||||
@@ -510,6 +516,7 @@ fn test_scan_cycle_partial_source_maps_budget_reason() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_mark_scan_cycle_idle_clears_published_cycle_state() {
|
||||
let mut cycle_info = CurrentCycle {
|
||||
current: 12,
|
||||
@@ -538,6 +545,7 @@ async fn test_mark_scan_cycle_idle_clears_published_cycle_state() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn scanner_cycle_metrics_guard_covers_published_first_cycle_lifetime() {
|
||||
let cycle_started = Utc::now() - chrono::Duration::seconds(5);
|
||||
let mut cycle_info = CurrentCycle {
|
||||
@@ -564,6 +572,7 @@ async fn scanner_cycle_metrics_guard_covers_published_first_cycle_lifetime() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn scanner_cycle_metrics_guard_keeps_active_cycle_published_during_finalization() {
|
||||
let mut cycle_info = CurrentCycle {
|
||||
current: 12,
|
||||
@@ -588,6 +597,7 @@ async fn scanner_cycle_metrics_guard_keeps_active_cycle_published_during_finaliz
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn scanner_cycle_metrics_guard_drop_clears_activity() {
|
||||
let guard = ScannerCycleMetricsGuard::new(CurrentCycle {
|
||||
current: 12,
|
||||
@@ -605,6 +615,7 @@ async fn scanner_cycle_metrics_guard_drop_clears_activity() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn run_data_scanner_cycle_publishes_activity_for_owner_lifetime() {
|
||||
let (_temp_dir, store) = setup_scanner_cycle_store().await;
|
||||
let ctx = CancellationToken::new();
|
||||
@@ -655,6 +666,7 @@ async fn run_data_scanner_cycle_publishes_activity_for_owner_lifetime() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_finalize_partial_scan_cycle_advances_and_persists_counter() {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
let ctx = CancellationToken::new();
|
||||
@@ -690,6 +702,7 @@ async fn test_finalize_partial_scan_cycle_advances_and_persists_counter() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn scanner_cycle_recovers_to_newer_durable_cache_floor() {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
let ctx = CancellationToken::new();
|
||||
@@ -729,6 +742,7 @@ async fn scanner_cycle_recovers_to_newer_durable_cache_floor() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn scanner_cycle_rejects_invalid_cache_floor() {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
let ctx = CancellationToken::new();
|
||||
@@ -999,6 +1013,7 @@ async fn scanner_usage_floor_fails_closed_on_corrupt_or_exhausted_usage_state()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn scanner_usage_backup_uses_durable_cycle_cadence_across_tasks() {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
let ctx = CancellationToken::new();
|
||||
@@ -1072,6 +1087,7 @@ fn scanner_cycle_advance_fails_before_reserved_exhausted_value() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_finalize_partial_scan_cycle_reports_persist_failure() {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
let ctx = CancellationToken::new();
|
||||
@@ -1095,6 +1111,7 @@ async fn test_finalize_partial_scan_cycle_reports_persist_failure() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_persist_scanner_cycle_state_reconciles_newer_winner() {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
let ctx = CancellationToken::new();
|
||||
@@ -1517,6 +1534,7 @@ async fn test_usage_save_route_barrier_prevents_missing_snapshot_creation() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_usage_route_barrier_precedes_durable_reconciliation() {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str());
|
||||
@@ -1544,6 +1562,7 @@ async fn test_usage_route_barrier_precedes_durable_reconciliation() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_deferred_usage_save_keeps_last_real_save_metric() {
|
||||
let metrics = global_metrics();
|
||||
metrics.record_scanner_usage_save_result(ScannerUsageSaveResult::Success);
|
||||
@@ -2607,6 +2626,7 @@ fn scanner_cycle_cache_floor_stays_pending_during_deferred_usage_publication() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn finalizing_a_saved_cycle_acknowledges_its_exact_dirty_snapshot() {
|
||||
crate::scanner_io::clear_dirty_usage_bucket("photos");
|
||||
crate::scanner_io::record_dirty_usage_bucket("photos");
|
||||
@@ -2633,6 +2653,7 @@ fn finalizing_a_saved_cycle_acknowledges_its_exact_dirty_snapshot() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn finalizing_a_deferred_usage_save_keeps_dirty_work_pending() {
|
||||
crate::scanner_io::clear_dirty_usage_bucket("photos");
|
||||
crate::scanner_io::record_dirty_usage_bucket("photos");
|
||||
@@ -2675,6 +2696,7 @@ async fn scanner_cycle_keeps_remote_pending_acknowledgement() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn finalizing_an_already_durable_cycle_acknowledges_its_exact_dirty_snapshot() {
|
||||
crate::scanner_io::clear_dirty_usage_bucket("photos");
|
||||
crate::scanner_io::record_dirty_usage_bucket("photos");
|
||||
@@ -2689,6 +2711,7 @@ fn finalizing_an_already_durable_cycle_acknowledges_its_exact_dirty_snapshot() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn finalizing_a_prior_same_cycle_snapshot_keeps_new_dirty_work_pending() {
|
||||
crate::scanner_io::clear_dirty_usage_bucket("photos");
|
||||
crate::scanner_io::record_dirty_usage_bucket("photos");
|
||||
@@ -2704,6 +2727,7 @@ fn finalizing_a_prior_same_cycle_snapshot_keeps_new_dirty_work_pending() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn finalizing_a_durable_superseded_snapshot_keeps_dirty_work_pending() {
|
||||
crate::scanner_io::clear_dirty_usage_bucket("photos");
|
||||
crate::scanner_io::record_dirty_usage_bucket("photos");
|
||||
@@ -2719,6 +2743,7 @@ fn finalizing_a_durable_superseded_snapshot_keeps_dirty_work_pending() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn data_usage_persist_wait_covers_cache_retries_and_backup() {
|
||||
with_var(rustfs_config::ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS, Some("7"), || {
|
||||
crate::runtime_config::refresh_scanner_runtime_config_for_tests();
|
||||
@@ -2771,6 +2796,7 @@ async fn maintenance_feature_inspection_preserves_base_cycle_after_timeout() {
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
#[serial]
|
||||
async fn stable_maintenance_detection_preserves_base_cycle_after_timeout() {
|
||||
let ctx = CancellationToken::new();
|
||||
|
||||
@@ -2836,6 +2862,7 @@ async fn maintenance_feature_inspection_stops_on_cancellation() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn test_cycle_interval_prefers_explicit_cycle_override() {
|
||||
with_var(ENV_SCANNER_SPEED, Some("slowest"), || {
|
||||
with_var(ENV_SCANNER_CYCLE, Some("42"), || {
|
||||
@@ -2845,6 +2872,7 @@ fn test_cycle_interval_prefers_explicit_cycle_override() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn test_cycle_interval_prefers_explicit_cycle_over_default_cycle() {
|
||||
let _guard = ScannerDefaultCycleGuard::set(TEST_DEFAULT_SCANNER_CYCLE_SECS);
|
||||
|
||||
@@ -2854,6 +2882,7 @@ fn test_cycle_interval_prefers_explicit_cycle_over_default_cycle() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn test_cycle_interval_uses_scanner_default_speed_override_when_unconfigured() {
|
||||
let _guard = ScannerDefaultSpeedGuard::set(ScannerSpeed::Slowest);
|
||||
|
||||
@@ -2863,6 +2892,7 @@ fn test_cycle_interval_uses_scanner_default_speed_override_when_unconfigured() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn test_cycle_interval_prefers_explicit_speed_over_default_speed_override() {
|
||||
let _guard = ScannerDefaultSpeedGuard::set(ScannerSpeed::Slowest);
|
||||
|
||||
@@ -2880,6 +2910,7 @@ fn test_cycle_interval_prefers_explicit_speed_over_default_speed_override() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn test_cycle_interval_uses_default_cycle_override_when_unconfigured() {
|
||||
let _guard = ScannerDefaultCycleGuard::set(TEST_DEFAULT_SCANNER_CYCLE_SECS);
|
||||
|
||||
@@ -3043,6 +3074,7 @@ fn scanner_cycle_wait_plan_drives_growth_resets_and_bitrot_cap() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn scanner_cycle_schedule_status_reports_effective_backoff() {
|
||||
record_scanner_cycle_schedule(Duration::from_millis(86_400_001), true, 2_048, true, 7);
|
||||
|
||||
@@ -3322,6 +3354,7 @@ fn dirty_usage_wakes_are_disabled_for_explicit_cycle_policy() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn clean_idle_cap_preserves_default_bitrot_coverage_window() {
|
||||
let config = ScannerRuntimeConfig {
|
||||
bitrot_cycle: Some(Duration::from_secs(30 * 24 * 60 * 60)),
|
||||
@@ -3351,6 +3384,7 @@ fn clean_idle_cap_allows_policy_max_when_bitrot_is_disabled() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn clean_idle_cap_never_shortens_the_base_cycle() {
|
||||
let config = ScannerRuntimeConfig {
|
||||
bitrot_cycle: Some(Duration::from_secs(60)),
|
||||
@@ -3364,6 +3398,7 @@ fn clean_idle_cap_never_shortens_the_base_cycle() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn test_cycle_interval_keeps_default_cycle_with_explicit_speed() {
|
||||
let _guard = ScannerDefaultCycleGuard::set(TEST_DEFAULT_SCANNER_CYCLE_SECS);
|
||||
|
||||
@@ -3381,6 +3416,7 @@ fn test_cycle_interval_keeps_default_cycle_with_explicit_speed() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn test_cycle_interval_prefers_explicit_start_delay_over_default_cycle() {
|
||||
let _guard = ScannerDefaultCycleGuard::set(TEST_DEFAULT_SCANNER_CYCLE_SECS);
|
||||
|
||||
@@ -3394,6 +3430,7 @@ fn test_cycle_interval_prefers_explicit_start_delay_over_default_cycle() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn test_cycle_interval_supports_minio_speed_alias() {
|
||||
with_var_unset(ENV_SCANNER_SPEED, || {
|
||||
with_var_unset(ENV_SCANNER_CYCLE, || {
|
||||
@@ -3407,6 +3444,7 @@ fn test_cycle_interval_supports_minio_speed_alias() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn test_cycle_interval_supports_minio_cycle_alias() {
|
||||
with_var_unset(ENV_SCANNER_CYCLE, || {
|
||||
with_var_unset(ENV_SCANNER_START_DELAY_SECS, || {
|
||||
@@ -3426,6 +3464,7 @@ fn test_randomized_cycle_delay_handles_small_start_delay() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_wait_for_next_scanner_cycle_wakes_for_dirty_usage() {
|
||||
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
||||
|
||||
@@ -3451,6 +3490,7 @@ async fn test_wait_for_next_scanner_cycle_wakes_for_dirty_usage() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_wait_for_next_scanner_cycle_sees_unattempted_dirty_usage() {
|
||||
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
||||
let dirty_generation = crate::scanner_io::dirty_usage_generation();
|
||||
@@ -3472,6 +3512,7 @@ async fn test_wait_for_next_scanner_cycle_sees_unattempted_dirty_usage() {
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
#[serial]
|
||||
async fn test_wait_for_next_scanner_cycle_retries_stable_dirty_usage_on_timer() {
|
||||
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
||||
crate::scanner_io::record_dirty_usage_bucket("photos");
|
||||
@@ -3493,6 +3534,7 @@ async fn test_wait_for_next_scanner_cycle_retries_stable_dirty_usage_on_timer()
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
#[serial]
|
||||
async fn test_wait_for_next_scanner_cycle_can_defer_dirty_wakes_until_timer() {
|
||||
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
||||
let ctx = CancellationToken::new();
|
||||
@@ -3511,6 +3553,7 @@ async fn test_wait_for_next_scanner_cycle_can_defer_dirty_wakes_until_timer() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_wait_for_next_scanner_cycle_wakes_for_repeated_dirty_bucket() {
|
||||
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
||||
crate::scanner_io::record_dirty_usage_bucket("photos");
|
||||
@@ -3536,6 +3579,7 @@ async fn test_wait_for_next_scanner_cycle_wakes_for_repeated_dirty_bucket() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_wait_for_next_scanner_cycle_reschedules_for_runtime_config() {
|
||||
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
||||
let observed_generation = crate::runtime_config::scanner_runtime_config_generation();
|
||||
@@ -3563,6 +3607,7 @@ async fn test_wait_for_next_scanner_cycle_reschedules_for_runtime_config() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_wait_for_next_scanner_cycle_reschedules_for_maintenance_change() {
|
||||
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
||||
let observed_generation = crate::scanner_io::scanner_maintenance_generation();
|
||||
@@ -3806,6 +3851,7 @@ fn scanner_activity_after_a_cycle_restores_the_base_interval() {
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
#[serial]
|
||||
async fn distributed_clean_idle_wait_wakes_at_base_interval_for_remote_activity() {
|
||||
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
||||
let ctx = CancellationToken::new();
|
||||
@@ -3833,6 +3879,7 @@ async fn distributed_clean_idle_wait_wakes_at_base_interval_for_remote_activity(
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
#[serial]
|
||||
async fn superseded_retry_wait_defers_dirty_cluster_activity_until_timer() {
|
||||
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
||||
let ctx = CancellationToken::new();
|
||||
@@ -3860,6 +3907,7 @@ async fn superseded_retry_wait_defers_dirty_cluster_activity_until_timer() {
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
#[serial]
|
||||
async fn distributed_clean_idle_wait_blocks_backoff_for_unpropagated_maintenance() {
|
||||
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
||||
let ctx = CancellationToken::new();
|
||||
@@ -3886,6 +3934,7 @@ async fn distributed_clean_idle_wait_blocks_backoff_for_unpropagated_maintenance
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
#[serial]
|
||||
async fn distributed_clean_idle_wait_fails_closed_when_a_peer_is_unverifiable() {
|
||||
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
||||
let ctx = CancellationToken::new();
|
||||
@@ -3912,6 +3961,7 @@ async fn distributed_clean_idle_wait_fails_closed_when_a_peer_is_unverifiable()
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
#[serial]
|
||||
async fn distributed_clean_idle_wait_keeps_the_extended_deadline_when_peers_are_clean() {
|
||||
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
||||
let ctx = CancellationToken::new();
|
||||
@@ -3939,6 +3989,7 @@ async fn distributed_clean_idle_wait_keeps_the_extended_deadline_when_peers_are_
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
#[serial]
|
||||
async fn scanner_activity_probe_wait_is_cancellation_aware() {
|
||||
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
||||
let ctx = CancellationToken::new();
|
||||
@@ -3969,6 +4020,7 @@ async fn scanner_activity_probe_wait_is_cancellation_aware() {
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
#[serial]
|
||||
async fn scanner_activity_probe_wait_stops_after_leader_lock_loss() {
|
||||
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
||||
let ctx = CancellationToken::new();
|
||||
@@ -4000,6 +4052,7 @@ async fn scanner_activity_probe_wait_stops_after_leader_lock_loss() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn test_get_cycle_scan_mode_runs_deep_until_selection_window_completes() {
|
||||
with_var(ENV_SCANNER_BITROT_CYCLE_SECS, Some("3600"), || {
|
||||
let mode = get_cycle_scan_mode(10, 0, Some(Utc::now()), bitrot_scan_cycle());
|
||||
@@ -4008,6 +4061,7 @@ fn test_get_cycle_scan_mode_runs_deep_until_selection_window_completes() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn test_get_cycle_scan_mode_respects_elapsed_bitrot_cycle() {
|
||||
with_var(ENV_SCANNER_BITROT_CYCLE_SECS, Some("3600"), || {
|
||||
let recent = Utc::now() - chrono::Duration::minutes(30);
|
||||
@@ -4019,6 +4073,7 @@ fn test_get_cycle_scan_mode_respects_elapsed_bitrot_cycle() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn test_get_cycle_scan_mode_can_disable_periodic_deep_scan() {
|
||||
with_var(ENV_SCANNER_BITROT_CYCLE_SECS, Some("off"), || {
|
||||
assert_eq!(get_cycle_scan_mode(1, 0, None, bitrot_scan_cycle()), HealScanMode::Normal);
|
||||
@@ -4026,6 +4081,7 @@ fn test_get_cycle_scan_mode_can_disable_periodic_deep_scan() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn test_background_heal_info_for_scan_start_marks_deep_active() {
|
||||
let now = Utc::now();
|
||||
let info =
|
||||
@@ -4038,6 +4094,7 @@ fn test_background_heal_info_for_scan_start_marks_deep_active() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn test_background_heal_info_for_scan_start_keeps_deep_window_start() {
|
||||
with_var_unset(ENV_SCANNER_BITROT_CYCLE_SECS, || {
|
||||
let started_at = Utc::now();
|
||||
|
||||
@@ -18,6 +18,7 @@ use super::*;
|
||||
use crate::storage_api::VersionPurgeStatusType;
|
||||
use crate::{DiskOption, Endpoint, STORAGE_FORMAT_FILE, TierStats, new_disk, storageclass};
|
||||
use rustfs_filemeta::{FileInfo, FileMeta};
|
||||
use serial_test::serial;
|
||||
use std::io::Write;
|
||||
#[cfg(unix)]
|
||||
use std::os::unix::fs::{PermissionsExt, symlink};
|
||||
@@ -355,6 +356,7 @@ impl Drop for TestGuard {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_should_skip_failed_respects_ttl() {
|
||||
let (mut scanner, temp_dir) = build_test_scanner().await;
|
||||
let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir);
|
||||
@@ -376,6 +378,7 @@ async fn test_should_skip_failed_respects_ttl() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_record_failed_ttl_zero_noop() {
|
||||
let (mut scanner, temp_dir) = build_test_scanner().await;
|
||||
let _guard = TestGuard::new(0, 100, &mut scanner, temp_dir);
|
||||
@@ -464,6 +467,7 @@ fn test_should_account_replication_stats_only_for_live_object_versions() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_heal_replication_only_queues_pending_null_deletes() {
|
||||
async fn replication_skipped_count() -> u64 {
|
||||
global_metrics()
|
||||
@@ -712,6 +716,7 @@ async fn test_scanner_heal_admission_accounting_maps_deep_scan_to_bitrot() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn test_excessive_version_alert_thresholds_use_env() {
|
||||
with_var(rustfs_config::ENV_SCANNER_ALERT_EXCESS_VERSIONS, Some("3"), || {
|
||||
with_var(rustfs_config::ENV_SCANNER_ALERT_EXCESS_VERSION_SIZE, Some("100"), || {
|
||||
@@ -726,6 +731,7 @@ fn test_excessive_version_alert_thresholds_use_env() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn test_excessive_folders_threshold_uses_env() {
|
||||
with_var(rustfs_config::ENV_SCANNER_ALERT_EXCESS_FOLDERS, Some("3"), || {
|
||||
crate::runtime_config::refresh_scanner_runtime_config_for_tests();
|
||||
@@ -735,6 +741,7 @@ fn test_excessive_folders_threshold_uses_env() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn test_excessive_folders_threshold_default_supports_pbs_layout() {
|
||||
with_var_unset(rustfs_config::ENV_SCANNER_ALERT_EXCESS_FOLDERS, || {
|
||||
crate::runtime_config::refresh_scanner_runtime_config_for_tests();
|
||||
@@ -744,6 +751,7 @@ fn test_excessive_folders_threshold_default_supports_pbs_layout() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn test_scanner_yield_every_n_objects_uses_env() {
|
||||
with_var(rustfs_config::ENV_SCANNER_YIELD_EVERY_N_OBJECTS, Some("32"), || {
|
||||
crate::runtime_config::refresh_scanner_runtime_config_for_tests();
|
||||
@@ -753,6 +761,7 @@ fn test_scanner_yield_every_n_objects_uses_env() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn test_scanner_yield_every_n_objects_uses_default() {
|
||||
with_var_unset(rustfs_config::ENV_SCANNER_YIELD_EVERY_N_OBJECTS, || {
|
||||
crate::runtime_config::refresh_scanner_runtime_config_for_tests();
|
||||
@@ -879,6 +888,7 @@ fn test_order_folders_for_resume_reports_stale_hint() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_record_failed_prunes_to_max_entries() {
|
||||
let (mut scanner, temp_dir) = build_test_scanner().await;
|
||||
let _guard = TestGuard::new(1000, 2, &mut scanner, temp_dir);
|
||||
@@ -910,6 +920,7 @@ async fn test_record_failed_prunes_to_max_entries() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_prune_failed_objects_cache_drops_expired() {
|
||||
let (mut scanner, temp_dir) = build_test_scanner().await;
|
||||
let _guard = TestGuard::new(5, 10, &mut scanner, temp_dir);
|
||||
@@ -933,6 +944,7 @@ async fn test_prune_failed_objects_cache_drops_expired() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_prune_failed_objects_max_zero_keeps_fresh() {
|
||||
let (mut scanner, temp_dir) = build_test_scanner().await;
|
||||
let _guard = TestGuard::new(60, 0, &mut scanner, temp_dir);
|
||||
@@ -1689,6 +1701,7 @@ async fn test_heal_actions_returns_actual_size_without_inline_heal() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
#[cfg(unix)]
|
||||
async fn test_scan_folder_skips_unreadable_child_directory() {
|
||||
let (mut scanner, temp_dir) = build_test_scanner().await;
|
||||
@@ -1721,6 +1734,7 @@ async fn test_scan_folder_skips_unreadable_child_directory() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_scan_folder_exits_when_abandoned_child_listing_finishes() {
|
||||
let (mut scanner, temp_dir) = build_test_scanner().await;
|
||||
let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir.clone());
|
||||
@@ -1799,6 +1813,7 @@ async fn test_scan_folder_exits_when_abandoned_child_listing_finishes() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_scan_folder_xl_meta_named_directory_uses_namespace_descent() {
|
||||
let (mut scanner, temp_dir) = build_test_scanner().await;
|
||||
let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir.clone());
|
||||
@@ -1844,6 +1859,7 @@ async fn test_scan_folder_xl_meta_named_directory_uses_namespace_descent() {
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
#[serial]
|
||||
async fn test_scan_folder_corrupt_xl_meta_stops_erasure_data_dir_descent() {
|
||||
let logs = CapturedLogs::default();
|
||||
let subscriber = tracing_subscriber::fmt()
|
||||
@@ -2005,6 +2021,7 @@ async fn test_scan_folder_corrupt_xl_meta_stops_erasure_data_dir_descent() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_scan_folder_missing_xl_meta_stops_erasure_data_dir_descent() {
|
||||
let (mut scanner, temp_dir) = build_test_scanner().await;
|
||||
let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir.clone());
|
||||
@@ -2082,6 +2099,7 @@ async fn test_scan_folder_missing_xl_meta_stops_erasure_data_dir_descent() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_scan_folder_uuid_namespace_part_name_directory_is_not_data_dir() {
|
||||
let (mut scanner, temp_dir) = build_test_scanner().await;
|
||||
let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir.clone());
|
||||
@@ -2143,6 +2161,7 @@ async fn test_scan_folder_uuid_namespace_part_name_directory_is_not_data_dir() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_scan_folder_non_erasure_metadata_keeps_namespace_descent() {
|
||||
let (mut scanner, temp_dir) = build_test_scanner().await;
|
||||
let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir.clone());
|
||||
@@ -2184,6 +2203,7 @@ async fn test_scan_folder_non_erasure_metadata_keeps_namespace_descent() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_scan_folder_compacted_parent_sends_partial_update() {
|
||||
let (mut scanner, temp_dir) = build_test_scanner().await;
|
||||
let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir.clone());
|
||||
@@ -2225,6 +2245,7 @@ async fn test_scan_folder_compacted_parent_sends_partial_update() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_scan_data_folder_cancelled_before_scan_clears_current_path() {
|
||||
let (scanner, temp_dir) = build_test_scanner().await;
|
||||
let _guard = TestGuard {
|
||||
@@ -2269,6 +2290,7 @@ async fn test_scan_data_folder_cancelled_before_scan_clears_current_path() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_scan_data_folder_returns_partial_cache_on_budget_cancel() {
|
||||
let (mut scanner, temp_dir) = build_test_scanner().await;
|
||||
let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir.clone());
|
||||
@@ -2324,6 +2346,7 @@ async fn test_scan_data_folder_returns_partial_cache_on_budget_cancel() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_scan_data_folder_reports_invalid_checkpoint_ignored_once() {
|
||||
let (scanner, temp_dir) = build_test_scanner().await;
|
||||
let _guard = TestGuard {
|
||||
@@ -2368,6 +2391,7 @@ async fn test_scan_data_folder_reports_invalid_checkpoint_ignored_once() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_scan_data_folder_resume_hint_prioritizes_next_existing_folder() {
|
||||
let (scanner, temp_dir) = build_test_scanner().await;
|
||||
let _guard = TestGuard {
|
||||
@@ -2441,6 +2465,7 @@ async fn test_scan_data_folder_resume_hint_prioritizes_next_existing_folder() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn scan_data_folder_missing_bucket_returns_partial() {
|
||||
let (scanner, temp_dir) = build_test_scanner().await;
|
||||
let _guard = TestGuard {
|
||||
@@ -2492,6 +2517,7 @@ async fn scan_data_folder_missing_bucket_returns_partial() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn scan_data_folder_missing_scan_root_returns_partial() {
|
||||
let (scanner, temp_dir) = build_test_scanner().await;
|
||||
tokio::fs::remove_dir_all(&temp_dir)
|
||||
@@ -2537,6 +2563,7 @@ async fn scan_data_folder_missing_scan_root_returns_partial() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_scan_data_folder_resume_hint_orders_across_new_and_existing_folders() {
|
||||
let (scanner, temp_dir) = build_test_scanner().await;
|
||||
let _guard = TestGuard {
|
||||
@@ -2605,6 +2632,7 @@ async fn test_scan_data_folder_resume_hint_orders_across_new_and_existing_folder
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_scan_data_folder_partial_object_budget_accumulates_progress() {
|
||||
let (scanner, temp_dir) = build_test_scanner().await;
|
||||
let _guard = TestGuard {
|
||||
@@ -2687,6 +2715,7 @@ async fn test_scan_data_folder_partial_object_budget_accumulates_progress() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_partial_compacted_entry_does_not_carry_children() {
|
||||
let (mut scanner, temp_dir) = build_test_scanner().await;
|
||||
let _guard = TestGuard {
|
||||
@@ -2732,6 +2761,7 @@ async fn test_partial_compacted_entry_does_not_carry_children() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_partial_entry_does_not_carry_missing_old_child() {
|
||||
let (mut scanner, temp_dir) = build_test_scanner().await;
|
||||
let _guard = TestGuard {
|
||||
@@ -2764,6 +2794,7 @@ async fn test_partial_entry_does_not_carry_missing_old_child() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_legacy_windows_cache_rebuilds_and_round_trips_portable_keys() {
|
||||
let (scanner, temp_dir) = build_test_scanner().await;
|
||||
let _guard = TestGuard {
|
||||
@@ -2830,6 +2861,7 @@ async fn test_legacy_windows_cache_rebuilds_and_round_trips_portable_keys() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_scan_data_folder_success_clears_resume_hint() {
|
||||
let (scanner, temp_dir) = build_test_scanner().await;
|
||||
let _guard = TestGuard {
|
||||
@@ -2872,6 +2904,7 @@ async fn test_scan_data_folder_success_clears_resume_hint() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_scan_data_folder_keeps_unresolved_objects_partial() {
|
||||
let (scanner, temp_dir) = build_test_scanner().await;
|
||||
let _guard = TestGuard {
|
||||
@@ -2918,6 +2951,7 @@ async fn test_scan_data_folder_keeps_unresolved_objects_partial() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
#[cfg(unix)]
|
||||
async fn test_scan_folder_ignores_symlinked_child_directory() {
|
||||
let (mut scanner, temp_dir) = build_test_scanner().await;
|
||||
|
||||
@@ -27,6 +27,7 @@ use crate::{
|
||||
init_local_disks_with_instance_ctx, new_disk, path2_bucket_object_with_base_path,
|
||||
};
|
||||
use rustfs_filemeta::FileInfo;
|
||||
use serial_test::serial;
|
||||
use temp_env::with_var;
|
||||
use time::OffsetDateTime;
|
||||
use uuid::Uuid;
|
||||
@@ -102,6 +103,7 @@ async fn setup_two_pool_scanner_store() -> (tempfile::TempDir, Arc<ECStore>) {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn scanner_cache_locks_block_same_source_workers() {
|
||||
let (_temp_dir, store) = setup_two_pool_scanner_store().await;
|
||||
let set = &store.pools[0].disk_set[0];
|
||||
@@ -128,6 +130,7 @@ async fn scanner_cache_locks_block_same_source_workers() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn scanner_cache_locks_allow_cross_source_workers() {
|
||||
let (_temp_dir, store) = setup_two_pool_scanner_store().await;
|
||||
let first_set = &store.pools[0].disk_set[0];
|
||||
@@ -146,6 +149,7 @@ async fn scanner_cache_locks_allow_cross_source_workers() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn scanner_cycle_is_deferred_while_rebalance_is_active() {
|
||||
let (_temp_dir, store) = setup_two_pool_scanner_store().await;
|
||||
let mut pool_stats = vec![EcstoreRebalanceStats::default(); store.pools.len()];
|
||||
@@ -181,6 +185,7 @@ async fn scanner_cycle_is_deferred_while_rebalance_is_active() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn scanner_cycle_is_deferred_while_terminal_decommission_is_blocked() {
|
||||
let (_temp_dir, store) = setup_two_pool_scanner_store().await;
|
||||
for decommission in [
|
||||
@@ -225,6 +230,7 @@ async fn data_usage_publish_fails_when_receiver_is_closed() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn multi_pool_scanner_cycle_publishes_combined_usage() {
|
||||
let (_temp_dir, store) = setup_two_pool_scanner_store().await;
|
||||
let bucket = format!("scanner-union-{}", Uuid::new_v4().simple());
|
||||
@@ -272,6 +278,7 @@ async fn multi_pool_scanner_cycle_publishes_combined_usage() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn multi_pool_scanner_cycle_zero_fills_bucket_absent_from_first_pool() {
|
||||
let (_temp_dir, store) = setup_two_pool_scanner_store().await;
|
||||
let bucket = format!("scanner-second-pool-{}", Uuid::new_v4().simple());
|
||||
@@ -359,6 +366,7 @@ fn object_lock_config_enabled_accepts_enabled_only() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn dirty_usage_snapshot_clear_preserves_newer_generation() {
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
record_dirty_usage_bucket("photos");
|
||||
@@ -373,6 +381,7 @@ fn dirty_usage_snapshot_clear_preserves_newer_generation() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn dirty_usage_generation_acknowledgement_preserves_newer_mutations() {
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
record_dirty_usage_bucket("photos");
|
||||
@@ -398,6 +407,7 @@ fn dirty_usage_generation_acknowledgement_preserves_newer_mutations() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn dirty_usage_generation_acknowledgement_rejects_stale_process_and_future_generation() {
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
record_dirty_usage_bucket("photos");
|
||||
@@ -427,6 +437,7 @@ fn dirty_usage_generation_acknowledgement_rejects_stale_process_and_future_gener
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn dirty_usage_snapshot_detects_uncovered_generation() {
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
record_dirty_usage_bucket("photos");
|
||||
@@ -451,6 +462,7 @@ fn generation_saturates_instead_of_wrapping() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn dirty_usage_snapshot_clears_a_stably_absent_bucket_after_durable_save() {
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
record_dirty_usage_bucket("photos");
|
||||
@@ -472,6 +484,7 @@ fn dirty_usage_snapshot_clears_a_stably_absent_bucket_after_durable_save() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn dirty_usage_snapshot_preserves_an_absent_bucket_recorded_after_listing_started() {
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
let generation_before_bucket_list = dirty_usage_generation();
|
||||
@@ -486,6 +499,7 @@ fn dirty_usage_snapshot_preserves_an_absent_bucket_recorded_after_listing_starte
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn deleting_a_clean_bucket_invalidates_an_inflight_usage_snapshot() {
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
let snapshot = snapshot_dirty_usage_buckets(&[bucket_info("photos")], dirty_usage_generation());
|
||||
@@ -499,6 +513,7 @@ fn deleting_a_clean_bucket_invalidates_an_inflight_usage_snapshot() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn deleting_a_bucket_during_listing_invalidates_the_resulting_usage_snapshot() {
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
let generation_before_bucket_list = dirty_usage_generation();
|
||||
@@ -512,6 +527,7 @@ fn deleting_a_bucket_during_listing_invalidates_the_resulting_usage_snapshot() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn scanner_maintenance_change_advances_generation_and_marks_usage_dirty() {
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
let generation = scanner_maintenance_generation();
|
||||
@@ -524,6 +540,7 @@ fn scanner_maintenance_change_advances_generation_and_marks_usage_dirty() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn dirty_usage_clear_excludes_failed_buckets() {
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
record_dirty_usage_bucket("photos");
|
||||
@@ -555,6 +572,7 @@ fn dirty_usage_clear_plan_excludes_cache_save_failures() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn dirty_usage_is_acknowledged_only_after_durable_usage_confirmation() {
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
record_dirty_usage_bucket("photos");
|
||||
@@ -572,6 +590,7 @@ fn dirty_usage_is_acknowledged_only_after_durable_usage_confirmation() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn clear_dirty_usage_bucket_removes_deleted_bucket_marker() {
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
record_dirty_usage_bucket("photos");
|
||||
@@ -898,30 +917,35 @@ async fn bucket_cache_pending_heal_reaches_cycle_maintenance_state() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn scanner_concurrency_limit_preserves_available_when_unconfigured() {
|
||||
crate::reset_foreground_read_activity_for_test();
|
||||
assert_eq!(scanner_concurrency_limit(0, 4), 4);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn scanner_concurrency_limit_caps_to_configured_value() {
|
||||
crate::reset_foreground_read_activity_for_test();
|
||||
assert_eq!(scanner_concurrency_limit(2, 4), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn scanner_concurrency_limit_never_exceeds_available_work() {
|
||||
crate::reset_foreground_read_activity_for_test();
|
||||
assert_eq!(scanner_concurrency_limit(8, 4), 4);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn scanner_concurrency_limit_handles_no_available_work() {
|
||||
crate::reset_foreground_read_activity_for_test();
|
||||
assert_eq!(scanner_concurrency_limit(2, 0), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn scanner_concurrency_limit_yields_to_foreground_reads() {
|
||||
crate::reset_foreground_read_activity_for_test();
|
||||
crate::set_foreground_read_activity(8);
|
||||
@@ -931,6 +955,7 @@ fn scanner_concurrency_limit_yields_to_foreground_reads() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn scanner_concurrency_limit_yields_to_streaming_reads() {
|
||||
crate::reset_foreground_read_activity_for_test();
|
||||
let _guard = crate::ForegroundReadGuard::new();
|
||||
@@ -954,6 +979,7 @@ fn increment_atomic_usize_saturates_at_max() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn scanner_max_concurrent_set_scans_uses_env_cap() {
|
||||
with_var(ENV_SCANNER_MAX_CONCURRENT_SET_SCANS, Some("2"), || {
|
||||
crate::runtime_config::refresh_scanner_runtime_config_for_tests();
|
||||
@@ -963,6 +989,7 @@ fn scanner_max_concurrent_set_scans_uses_env_cap() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn scanner_max_concurrent_disk_scans_uses_env_cap() {
|
||||
with_var(ENV_SCANNER_MAX_CONCURRENT_DISK_SCANS, Some("1"), || {
|
||||
crate::runtime_config::refresh_scanner_runtime_config_for_tests();
|
||||
|
||||
@@ -258,6 +258,7 @@ impl SleepTimer {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use serial_test::serial;
|
||||
use temp_env::{with_var, with_var_unset};
|
||||
|
||||
struct ScannerDefaultSpeedGuard;
|
||||
@@ -325,6 +326,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn test_refresh_from_env_applies_speed_and_idle_mode_for_next_cycle() {
|
||||
let prev_mode = SCANNER_IDLE_MODE.load(Ordering::Relaxed);
|
||||
SCANNER_IDLE_MODE.store(true, Ordering::Relaxed);
|
||||
@@ -344,6 +346,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn test_refresh_from_env_uses_default_speed_override_when_speed_unset() {
|
||||
let _guard = ScannerDefaultSpeedGuard::set(ScannerSpeed::Slowest);
|
||||
let s = DynamicSleeper::new(ScannerSpeed::Default);
|
||||
@@ -359,6 +362,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
#[serial]
|
||||
async fn test_fastest_never_sleeps() {
|
||||
let prev_mode = SCANNER_IDLE_MODE.load(Ordering::Relaxed);
|
||||
SCANNER_IDLE_MODE.store(true, Ordering::Relaxed);
|
||||
@@ -372,6 +376,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
#[serial]
|
||||
async fn test_idle_mode_off_skips_sleep() {
|
||||
let prev_mode = SCANNER_IDLE_MODE.load(Ordering::Relaxed);
|
||||
SCANNER_IDLE_MODE.store(false, Ordering::Relaxed);
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
|
||||
#![recursion_limit = "256"]
|
||||
|
||||
use futures::FutureExt;
|
||||
use rustfs_config::ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT;
|
||||
use rustfs_scanner::scanner_folder::ScannerItem;
|
||||
use rustfs_scanner::scanner_io::ScannerIODisk;
|
||||
@@ -22,8 +23,10 @@ use rustfs_scanner::{
|
||||
scanner::init_data_scanner,
|
||||
};
|
||||
use s3s::dto::RestoreRequest;
|
||||
use serial_test::serial;
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
env,
|
||||
path::{Path, PathBuf},
|
||||
sync::{Arc, Once, OnceLock},
|
||||
time::Duration,
|
||||
@@ -532,15 +535,31 @@ async fn wait_for_transition(ecstore: &Arc<ECStore>, bucket: &str, object: &str,
|
||||
}
|
||||
}
|
||||
|
||||
// Run `test_fn` with `ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT`
|
||||
// set to `"1"` for its duration. `temp_env` serializes environment mutations
|
||||
// globally, preventing data races when multiple tests run in parallel.
|
||||
// SAFETY: this helper is used only by `#[serial]` tests and runs under the single-threaded Tokio
|
||||
// runtime (`worker_threads = 1`), so no concurrent test can mutate process environment during the
|
||||
// `env::set_var` / `env::remove_var` window.
|
||||
#[allow(unsafe_code)]
|
||||
async fn with_forced_immediate_enqueue_timeout<F, Fut>(test_fn: F)
|
||||
where
|
||||
F: FnOnce() -> Fut,
|
||||
Fut: std::future::Future<Output = ()>,
|
||||
{
|
||||
temp_env::async_with_vars([(ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT, Some("1"))], test_fn()).await;
|
||||
let original = env::var_os(ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT);
|
||||
unsafe {
|
||||
env::set_var(ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT, "1");
|
||||
}
|
||||
let result = std::panic::AssertUnwindSafe(test_fn()).catch_unwind().await;
|
||||
match original {
|
||||
Some(value) => unsafe {
|
||||
env::set_var(ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT, value);
|
||||
},
|
||||
None => unsafe {
|
||||
env::remove_var(ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT);
|
||||
},
|
||||
}
|
||||
if let Err(err) = result {
|
||||
std::panic::resume_unwind(err);
|
||||
}
|
||||
}
|
||||
|
||||
mod serial_tests {
|
||||
@@ -573,6 +592,7 @@ mod serial_tests {
|
||||
/// body (GET won) or a clean object/version-not-found (expiry won). A
|
||||
/// tier-fetch failure -- the #3491 symptom -- is never tolerated.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[serial]
|
||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-2)"]
|
||||
async fn test_expire_transitioned_object_never_races_concurrent_get() {
|
||||
let (_disk_paths, ecstore) = setup_isolated_test_env(false).await;
|
||||
@@ -718,6 +738,7 @@ mod serial_tests {
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[serial]
|
||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial"]
|
||||
async fn rejected_transition_candidate_is_recovered_from_persisted_delete_journal() {
|
||||
let (_disk_paths, ecstore) = setup_isolated_test_env(false).await;
|
||||
@@ -804,6 +825,7 @@ mod serial_tests {
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[serial]
|
||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial"]
|
||||
async fn cancelled_before_cleanup_store_resolution_persists_journal() {
|
||||
let (_disk_paths, ecstore) = setup_isolated_test_env(false).await;
|
||||
@@ -897,6 +919,7 @@ mod serial_tests {
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[serial]
|
||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial"]
|
||||
async fn rejected_transition_cleanup_durability_matrix() {
|
||||
#[derive(Clone, Copy)]
|
||||
@@ -1036,6 +1059,7 @@ mod serial_tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
|
||||
fn test_transition_and_restore_flows() {
|
||||
std::thread::Builder::new()
|
||||
@@ -1361,6 +1385,7 @@ mod serial_tests {
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[serial]
|
||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
|
||||
async fn test_scanner_enqueues_free_version_cleanup_for_stale_transitioned_object() {
|
||||
let (disk_paths, ecstore) = setup_isolated_test_env(false).await;
|
||||
@@ -1421,6 +1446,7 @@ mod serial_tests {
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[serial]
|
||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
|
||||
async fn test_scanner_cleanup_still_works_after_immediate_compensation_transition() {
|
||||
let (disk_paths, ecstore) = setup_isolated_test_env(false).await;
|
||||
@@ -1478,6 +1504,7 @@ mod serial_tests {
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[serial]
|
||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
|
||||
async fn test_existing_object_backfill_is_idempotent_after_immediate_compensation_transition() {
|
||||
let (_disk_paths, ecstore) = setup_isolated_test_env(false).await;
|
||||
@@ -1520,6 +1547,7 @@ mod serial_tests {
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[serial]
|
||||
#[ignore = "FAILING on main: excluded from the serial ILM lane pending a fix, see rustfs/backlog#1148 (ilm-1 partial)"]
|
||||
async fn test_noncurrent_expiry_still_works_after_immediate_compensation_transition() {
|
||||
let (disk_paths, ecstore) = setup_isolated_test_env(true).await;
|
||||
@@ -1603,6 +1631,7 @@ mod serial_tests {
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[serial]
|
||||
#[ignore = "FAILING on main: excluded from the serial ILM lane pending a fix, see rustfs/backlog#1148 (ilm-1 partial)"]
|
||||
async fn test_noncurrent_transition_still_works_after_immediate_compensation_transition() {
|
||||
let (disk_paths, ecstore) = setup_isolated_test_env(true).await;
|
||||
@@ -1685,6 +1714,7 @@ mod serial_tests {
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[serial]
|
||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
|
||||
async fn test_modeled_versioned_delete_creates_delete_marker_after_immediate_compensation_transition() {
|
||||
let (_disk_paths, ecstore) = setup_isolated_test_env(true).await;
|
||||
@@ -1732,6 +1762,7 @@ mod serial_tests {
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[serial]
|
||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
|
||||
async fn test_modeled_delete_marker_cleanup_after_immediate_compensation_transition() {
|
||||
let (disk_paths, ecstore) = setup_isolated_test_env(true).await;
|
||||
@@ -1808,6 +1839,7 @@ mod serial_tests {
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[serial]
|
||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
|
||||
async fn test_scanner_expires_zero_day_current_version() {
|
||||
let (disk_paths, ecstore) = setup_isolated_test_env(false).await;
|
||||
@@ -1834,6 +1866,7 @@ mod serial_tests {
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[serial]
|
||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
|
||||
async fn test_put_object_immediately_enqueues_zero_day_current_expiry() {
|
||||
let (_disk_paths, ecstore) = setup_isolated_test_env(true).await;
|
||||
@@ -1871,6 +1904,7 @@ mod serial_tests {
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[serial]
|
||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
|
||||
async fn test_scanner_expires_zero_day_noncurrent_version() {
|
||||
let (disk_paths, ecstore) = setup_isolated_test_env(false).await;
|
||||
@@ -1937,6 +1971,7 @@ mod serial_tests {
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[serial]
|
||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
|
||||
async fn test_put_object_immediately_enqueues_zero_day_noncurrent_expiry() {
|
||||
let (_disk_paths, ecstore) = setup_isolated_test_env(true).await;
|
||||
@@ -1997,6 +2032,7 @@ mod serial_tests {
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[serial]
|
||||
async fn test_background_scanner_expires_zero_day_current_version() {
|
||||
let (_disk_paths, ecstore) = setup_isolated_test_env(true).await;
|
||||
|
||||
@@ -2020,6 +2056,7 @@ mod serial_tests {
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[serial]
|
||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
|
||||
async fn test_background_scanner_expires_zero_day_current_version_for_exact_key_prefix() {
|
||||
let (_disk_paths, ecstore) = setup_isolated_test_env(true).await;
|
||||
@@ -2085,6 +2122,7 @@ mod serial_tests {
|
||||
/// tier object is untouched (zero `remove` calls) -> GET streams from the
|
||||
/// tier again -> a second restore succeeds.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[serial]
|
||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-8)"]
|
||||
async fn test_restore_chain_local_read_expiry_keeps_remote_and_allows_re_restore() {
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
@@ -2216,6 +2254,7 @@ mod serial_tests {
|
||||
/// parts) must reassemble the exact part layout: part count and sizes,
|
||||
/// the multipart ETag, and byte-identical content across part boundaries.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[serial]
|
||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-8)"]
|
||||
async fn test_multipart_restore_preserves_parts_and_etag() {
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
|
||||
@@ -31,6 +31,9 @@ use crate::admin::storage_api::bucket::metadata::{
|
||||
use crate::admin::storage_api::bucket::metadata_sys;
|
||||
use crate::admin::storage_api::bucket::quota::BucketQuota;
|
||||
use crate::admin::storage_api::bucket::replication;
|
||||
use crate::admin::storage_api::bucket::replication::{
|
||||
is_site_replication_rule, merge_incoming_replication_config, replication_target_arn_deployment_id,
|
||||
};
|
||||
use crate::admin::storage_api::bucket::target::{ARN, BucketTarget, BucketTargetType, BucketTargets, Credentials};
|
||||
use crate::admin::storage_api::bucket::target_sys::BucketTargetSys;
|
||||
use crate::admin::storage_api::bucket::utils::{deserialize, serialize};
|
||||
@@ -1117,6 +1120,14 @@ async fn load_site_replication_state() -> S3Result<SiteReplicationState> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether this deployment participates in site replication (two or more
|
||||
/// peers in the persisted state). Read by the S3 interface layer to gate
|
||||
/// replication-config edits (MinIO `ErrReplicationDenyEditError` semantics,
|
||||
/// issue #1948); a state-read failure propagates so the gate fails closed.
|
||||
pub(crate) async fn site_replication_enabled() -> S3Result<bool> {
|
||||
Ok(load_site_replication_state().await?.enabled())
|
||||
}
|
||||
|
||||
async fn load_site_replication_state_no_lock(store: Arc<ECStore>) -> S3Result<SiteReplicationState> {
|
||||
match read_config_no_lock(store, SITE_REPLICATION_STATE_PATH).await {
|
||||
Ok(data) => parse_site_replication_state(&data),
|
||||
@@ -7748,20 +7759,6 @@ fn bucket_target_deployment_id(target: &BucketTarget) -> Option<String> {
|
||||
replication_target_arn_deployment_id(&target.arn)
|
||||
}
|
||||
|
||||
fn replication_target_arn_deployment_id(arn: &str) -> Option<String> {
|
||||
let parts: Vec<_> = arn.split(':').collect();
|
||||
if parts.len() == 6
|
||||
&& parts[0] == "arn"
|
||||
&& matches!(parts[1], "rustfs" | "minio")
|
||||
&& parts[2] == "replication"
|
||||
&& !parts[4].is_empty()
|
||||
{
|
||||
return Some(parts[4].to_string());
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
fn prune_removed_site_replication_bucket_targets(
|
||||
existing: BucketTargets,
|
||||
removed_deployment_ids: &HashSet<String>,
|
||||
@@ -7786,10 +7783,6 @@ fn prune_removed_site_replication_bucket_targets(
|
||||
(BucketTargets { targets }, removed)
|
||||
}
|
||||
|
||||
fn is_site_replication_rule(rule: &ReplicationRule) -> bool {
|
||||
rule.id.as_deref().is_some_and(|id| id.starts_with("site-repl-"))
|
||||
}
|
||||
|
||||
/// Whether every `site-repl-*` rule on this bucket resolves to a live remote target.
|
||||
///
|
||||
/// The rule set alone cannot answer this: a rule can be perfectly formed while the endpoint
|
||||
@@ -7815,52 +7808,6 @@ async fn site_replication_targets_online(bucket: &str, replication_config_xml: &
|
||||
true
|
||||
}
|
||||
|
||||
/// Merge a peer's replication config into the local one.
|
||||
///
|
||||
/// `site-repl-*` rules encode the *sender's* outbound direction — their destination ARN
|
||||
/// names the receiver — so applying a peer's rule set verbatim replaces the receiver's
|
||||
/// reverse rule with one pointing at itself. No bucket target can satisfy that ARN
|
||||
/// (`reconcile_site_replication_bucket_targets` skips the local peer), so the receiver
|
||||
/// silently stops replicating back: the one-directional symptom. Only operator-authored
|
||||
/// rules travel between sites; each site owns its own `site-repl-*` rules.
|
||||
fn merge_incoming_replication_config(
|
||||
incoming: Option<ReplicationConfiguration>,
|
||||
local: Option<ReplicationConfiguration>,
|
||||
) -> Option<ReplicationConfiguration> {
|
||||
let incoming_role = incoming.as_ref().map(|config| config.role.clone()).unwrap_or_default();
|
||||
// Operator rules first, then the local site rules — the same order
|
||||
// `ensure_site_replication_bucket_replication_config_with_runtime` produces, so its
|
||||
// no-op check matches and the bucket metadata is written once per broadcast, not twice.
|
||||
let mut rules: Vec<ReplicationRule> = incoming
|
||||
.into_iter()
|
||||
.flat_map(|config| config.rules)
|
||||
.filter(|rule| !is_site_replication_rule(rule))
|
||||
.collect();
|
||||
rules.extend(
|
||||
local
|
||||
.into_iter()
|
||||
.flat_map(|config| config.rules)
|
||||
.filter(is_site_replication_rule),
|
||||
);
|
||||
|
||||
if rules.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
for (index, rule) in rules.iter_mut().enumerate() {
|
||||
rule.priority = Some(i32::try_from(index + 1).unwrap_or(i32::MAX));
|
||||
}
|
||||
|
||||
// A site-replication ARN in `role` is the sender's, and `site_replication_target_arns_by_peer`
|
||||
// reads it — carrying it over would pin the receiver's targets to the sender's identity.
|
||||
let role = match replication_target_arn_deployment_id(&incoming_role) {
|
||||
Some(_) => String::new(),
|
||||
None => incoming_role,
|
||||
};
|
||||
|
||||
Some(ReplicationConfiguration { role, rules })
|
||||
}
|
||||
|
||||
/// Merge a peer's ILM expiry document into the local lifecycle config.
|
||||
///
|
||||
/// Mirrors MinIO's `mergeWithCurrentLCConfig` with one hardening: incoming
|
||||
|
||||
@@ -443,6 +443,7 @@ pub(crate) mod replication {
|
||||
pub(crate) use super::ecstore_bucket::replication::{
|
||||
REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS,
|
||||
REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS,
|
||||
is_site_replication_rule, merge_incoming_replication_config, replication_target_arn_deployment_id,
|
||||
};
|
||||
pub(crate) type BucketReplicationResyncStatus = super::ecstore_bucket::replication::BucketReplicationResyncStatus;
|
||||
pub(crate) type BucketStats = super::ecstore_bucket::replication::BucketStats;
|
||||
|
||||
@@ -38,9 +38,9 @@ use super::storage_api::bucket_usecase::bucket::{
|
||||
metadata_sys,
|
||||
policy_sys::PolicySys,
|
||||
replication::{
|
||||
ReplicationTargetValidationError, invalid_replication_config_status_field, replication_target_arns,
|
||||
should_remove_replication_target, unsupported_replication_config_field, validate_replication_config_structure,
|
||||
validate_replication_config_target_arns,
|
||||
ReplicationTargetValidationError, invalid_replication_config_status_field, is_site_replication_rule,
|
||||
merge_incoming_replication_config, replication_target_arns, should_remove_replication_target,
|
||||
unsupported_replication_config_field, validate_replication_config_structure, validate_replication_config_target_arns,
|
||||
},
|
||||
target::{BucketTargetType, BucketTargets},
|
||||
utils::serialize,
|
||||
@@ -623,11 +623,50 @@ async fn validate_bucket_replication_update(bucket: &str, config: &ReplicationCo
|
||||
validate_replication_config_targets(&targets, config)
|
||||
}
|
||||
|
||||
async fn replication_targets_without_config_targets(
|
||||
/// Defense in depth for site-replication-managed buckets (issue #1948): an S3
|
||||
/// PutBucketReplication replaces the operator-authored rules but must not wipe
|
||||
/// the local `site-repl-*` rules the reconciler owns — until its next pass
|
||||
/// (600s period) every peer link on this bucket would be silently dead. The
|
||||
/// same merge also drops incoming `site-repl-*` impostor rules, matching the
|
||||
/// peer bucket-meta ingestion path. Buckets without site-replication rules
|
||||
/// keep the verbatim overwrite semantics.
|
||||
fn merge_user_replication_config_update(
|
||||
incoming: ReplicationConfiguration,
|
||||
existing: Option<ReplicationConfiguration>,
|
||||
) -> ReplicationConfiguration {
|
||||
let has_site_rules = existing
|
||||
.as_ref()
|
||||
.is_some_and(|config| config.rules.iter().any(is_site_replication_rule));
|
||||
if !has_site_rules {
|
||||
return incoming;
|
||||
}
|
||||
// `existing` holds at least one site-replication rule the merge keeps, so
|
||||
// the merged rule set is non-empty; the fallback only guards the type.
|
||||
merge_incoming_replication_config(Some(incoming.clone()), existing).unwrap_or(incoming)
|
||||
}
|
||||
|
||||
/// Split of an S3 DeleteBucketReplication on the stored config (issue #1948):
|
||||
/// the operator-authored rules are removed, the local `site-repl-*` rules
|
||||
/// survive (`None` means nothing survives and the config is deleted), and the
|
||||
/// returned ARNs are the ones whose bucket targets may be garbage-collected —
|
||||
/// never an ARN a surviving site-replication rule still points at.
|
||||
fn split_replication_config_for_user_delete(
|
||||
config: ReplicationConfiguration,
|
||||
) -> (Option<ReplicationConfiguration>, HashSet<String>) {
|
||||
let mut removable_arns = replication_target_arns(&config);
|
||||
let remaining = merge_incoming_replication_config(None, Some(config));
|
||||
if let Some(remaining) = remaining.as_ref() {
|
||||
for rule in &remaining.rules {
|
||||
removable_arns.remove(rule.destination.bucket.trim());
|
||||
}
|
||||
}
|
||||
(remaining, removable_arns)
|
||||
}
|
||||
|
||||
async fn replication_targets_without_arns(
|
||||
bucket: &str,
|
||||
config: &ReplicationConfiguration,
|
||||
target_arns: &HashSet<String>,
|
||||
) -> S3Result<Option<(BucketTargets, usize)>> {
|
||||
let target_arns = replication_target_arns(config);
|
||||
if target_arns.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
@@ -638,7 +677,7 @@ async fn replication_targets_without_config_targets(
|
||||
Err(err) => return Err(ApiError::from(err).into()),
|
||||
};
|
||||
|
||||
let removed = remove_replication_targets_from_config_targets(&mut targets, &target_arns);
|
||||
let removed = remove_replication_targets_from_config_targets(&mut targets, target_arns);
|
||||
if removed == 0 {
|
||||
return Ok(None);
|
||||
}
|
||||
@@ -1604,15 +1643,29 @@ impl DefaultBucketUsecase {
|
||||
Err(StorageError::ConfigNotFound) => None,
|
||||
Err(err) => return Err(ApiError::from(err).into()),
|
||||
};
|
||||
let updated_targets = if let Some(config) = replication_config.as_ref() {
|
||||
replication_targets_without_config_targets(&bucket, config).await?
|
||||
let (remaining_config, updated_targets) = if let Some(config) = replication_config.as_ref() {
|
||||
let (remaining, removable_arns) = split_replication_config_for_user_delete(config.clone());
|
||||
let targets = replication_targets_without_arns(&bucket, &removable_arns).await?;
|
||||
(remaining, targets)
|
||||
} else {
|
||||
None
|
||||
(None, None)
|
||||
};
|
||||
|
||||
delete_bucket_config_for_incarnation(&bucket, BUCKET_REPLICATION_CONFIG, expected_incarnation_id)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
match remaining_config {
|
||||
// Site-replication rules and the targets backing them survive the
|
||||
// S3 delete (issue #1948); only the operator-authored rules go.
|
||||
Some(remaining) => {
|
||||
let data = serialize_config(&remaining)?;
|
||||
update_bucket_config_for_incarnation(&bucket, BUCKET_REPLICATION_CONFIG, data, expected_incarnation_id)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
}
|
||||
None => {
|
||||
delete_bucket_config_for_incarnation(&bucket, BUCKET_REPLICATION_CONFIG, expected_incarnation_id)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
}
|
||||
}
|
||||
if let Some((targets, removed)) = updated_targets
|
||||
&& let Err(err) =
|
||||
write_replication_targets_after_config_delete(&bucket, &targets, removed, expected_incarnation_id).await
|
||||
@@ -2485,6 +2538,12 @@ impl DefaultBucketUsecase {
|
||||
|
||||
let targets_guard = lock_bucket_targets_metadata(&bucket).await;
|
||||
validate_bucket_replication_update(&bucket, &replication_configuration).await?;
|
||||
let existing_config = match metadata_sys::get_replication_config(&bucket).await {
|
||||
Ok((config, _)) => Some(config),
|
||||
Err(StorageError::ConfigNotFound) => None,
|
||||
Err(err) => return Err(ApiError::from(err).into()),
|
||||
};
|
||||
let replication_configuration = merge_user_replication_config_update(replication_configuration, existing_config);
|
||||
let data = serialize_config(&replication_configuration)?;
|
||||
update_bucket_config_for_incarnation(&bucket, BUCKET_REPLICATION_CONFIG, data, expected_incarnation_id)
|
||||
.await
|
||||
@@ -3114,6 +3173,127 @@ mod tests {
|
||||
assert!(arns.contains(destination));
|
||||
}
|
||||
|
||||
fn replication_rule_with_id(arn: &str, id: &str, priority: i32) -> ReplicationRule {
|
||||
let mut rule = replication_rule_for_target(arn);
|
||||
rule.id = Some(id.to_string());
|
||||
rule.priority = Some(priority);
|
||||
rule
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn put_replication_merge_preserves_site_replication_rules() {
|
||||
let existing = ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![
|
||||
replication_rule_with_id("arn:rustfs:replication::peer-dep:bucket", "site-repl-peer-dep", 1),
|
||||
replication_rule_with_id("arn:rustfs:replication:us-east-1:old:bucket", "old-user-rule", 2),
|
||||
],
|
||||
};
|
||||
let incoming = ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![
|
||||
replication_rule_with_id("arn:rustfs:replication:us-east-1:new:bucket", "new-user-rule", 1),
|
||||
replication_rule_with_id("arn:rustfs:replication::forged-dep:bucket", "site-repl-forged", 2),
|
||||
],
|
||||
};
|
||||
|
||||
let merged = merge_user_replication_config_update(incoming, Some(existing));
|
||||
|
||||
let ids: Vec<_> = merged
|
||||
.rules
|
||||
.iter()
|
||||
.map(|rule| rule.id.as_deref().unwrap_or_default())
|
||||
.collect();
|
||||
assert_eq!(
|
||||
ids,
|
||||
vec!["new-user-rule", "site-repl-peer-dep"],
|
||||
"user rules replaced, local site-replication rule preserved, forged incoming site-repl rule dropped"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn put_replication_merge_returns_incoming_verbatim_without_site_rules() {
|
||||
let existing = ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![replication_rule_with_id(
|
||||
"arn:rustfs:replication:us-east-1:old:bucket",
|
||||
"old-user-rule",
|
||||
7,
|
||||
)],
|
||||
};
|
||||
let incoming = ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![replication_rule_with_id(
|
||||
"arn:rustfs:replication:us-east-1:new:bucket",
|
||||
"new-user-rule",
|
||||
5,
|
||||
)],
|
||||
};
|
||||
|
||||
let merged = merge_user_replication_config_update(incoming.clone(), Some(existing));
|
||||
|
||||
assert_eq!(merged.role, incoming.role);
|
||||
assert_eq!(merged.rules, incoming.rules, "non-SR buckets keep the verbatim overwrite semantics");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_replication_split_keeps_site_rules_and_their_targets() {
|
||||
let sr_arn = "arn:rustfs:replication::peer-dep:bucket";
|
||||
let user_arn = "arn:rustfs:replication:us-east-1:user:bucket";
|
||||
let config = ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![
|
||||
replication_rule_with_id(user_arn, "user-rule", 1),
|
||||
replication_rule_with_id(sr_arn, "site-repl-peer-dep", 2),
|
||||
],
|
||||
};
|
||||
|
||||
let (remaining, removable) = split_replication_config_for_user_delete(config);
|
||||
|
||||
let remaining = remaining.expect("site-replication rules must survive a user delete");
|
||||
let ids: Vec<_> = remaining
|
||||
.rules
|
||||
.iter()
|
||||
.map(|rule| rule.id.as_deref().unwrap_or_default())
|
||||
.collect();
|
||||
assert_eq!(ids, vec!["site-repl-peer-dep"]);
|
||||
assert_eq!(removable, HashSet::from([user_arn.to_string()]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_replication_split_protects_targets_shared_with_site_rules() {
|
||||
let sr_arn = "arn:rustfs:replication::peer-dep:bucket";
|
||||
let config = ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![
|
||||
replication_rule_with_id(sr_arn, "user-rule-on-sr-target", 1),
|
||||
replication_rule_with_id(sr_arn, "site-repl-peer-dep", 2),
|
||||
],
|
||||
};
|
||||
|
||||
let (remaining, removable) = split_replication_config_for_user_delete(config);
|
||||
|
||||
assert!(remaining.is_some());
|
||||
assert!(
|
||||
removable.is_empty(),
|
||||
"a target still referenced by a surviving site-replication rule must not be removed"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_replication_split_removes_everything_without_site_rules() {
|
||||
let user_arn = "arn:rustfs:replication:us-east-1:user:bucket";
|
||||
let config = ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![replication_rule_with_id(user_arn, "user-rule", 1)],
|
||||
};
|
||||
|
||||
let (remaining, removable) = split_replication_config_for_user_delete(config);
|
||||
|
||||
assert!(remaining.is_none(), "without site-replication rules the whole config is deleted");
|
||||
assert_eq!(removable, HashSet::from([user_arn.to_string()]));
|
||||
}
|
||||
|
||||
fn replication_targets_with_arn(arns: &[&str]) -> BucketTargets {
|
||||
BucketTargets {
|
||||
targets: arns
|
||||
|
||||
@@ -614,6 +614,8 @@ pub(crate) mod bucket {
|
||||
|
||||
use crate::storage::storage_api::ecstore_bucket::replication as replication_contracts;
|
||||
|
||||
pub(crate) use replication_contracts::{is_site_replication_rule, merge_incoming_replication_config};
|
||||
|
||||
type ReplicationObjectBridge = crate::storage::storage_api::ecstore_bucket::replication::ReplicationObjectBridge;
|
||||
pub(crate) type DeleteReplicationConfigSnapshot =
|
||||
crate::storage::storage_api::ecstore_bucket::replication::DeleteReplicationConfigSnapshot;
|
||||
|
||||
@@ -63,6 +63,54 @@ use crate::app::storage_api::object_usecase::bucket::replication::{
|
||||
};
|
||||
use crate::storage::storage_api::ecfs_consumer::StorageObjectOptions as ObjectOptions;
|
||||
|
||||
#[cfg(test)]
|
||||
static SITE_REPLICATION_GATE_TEST_OVERRIDE: std::sync::atomic::AtomicU8 = std::sync::atomic::AtomicU8::new(0);
|
||||
#[cfg(test)]
|
||||
const SITE_REPLICATION_GATE_FORCE_DISABLED: u8 = 1;
|
||||
#[cfg(test)]
|
||||
const SITE_REPLICATION_GATE_FORCE_ENABLED: u8 = 2;
|
||||
|
||||
async fn site_replication_gate_enabled() -> S3Result<bool> {
|
||||
#[cfg(test)]
|
||||
match SITE_REPLICATION_GATE_TEST_OVERRIDE.load(std::sync::atomic::Ordering::SeqCst) {
|
||||
SITE_REPLICATION_GATE_FORCE_DISABLED => return Ok(false),
|
||||
SITE_REPLICATION_GATE_FORCE_ENABLED => return Ok(true),
|
||||
_ => {}
|
||||
}
|
||||
crate::admin::handlers::site_replication::site_replication_enabled().await
|
||||
}
|
||||
|
||||
/// MinIO `ErrReplicationDenyEditError`.
|
||||
fn replication_deny_edit_error() -> S3Error {
|
||||
let mut err = S3Error::with_message(
|
||||
S3ErrorCode::Custom("XMinioReplicationDenyEdit".into()),
|
||||
"Sub-User is not allowed to edit Replication configuration",
|
||||
);
|
||||
err.set_status_code(StatusCode::BAD_REQUEST);
|
||||
err
|
||||
}
|
||||
|
||||
/// Site-replication gate for S3 replication-config edits (issue #1948).
|
||||
///
|
||||
/// On a site-replication deployment the bucket's replication config carries
|
||||
/// the operator-managed `site-repl-*` rules that keep every peer in sync, and
|
||||
/// a successful edit is broadcast to all peers — so a user holding only
|
||||
/// bucket-scoped `s3:PutReplicationConfiguration` could rewrite or erase
|
||||
/// replication net-wide. MinIO parity (`ErrReplicationDenyEditError`): only
|
||||
/// owner credentials (root or root-parented) may edit. Runs after the policy
|
||||
/// authorization in the access layer and only on the external S3 path — the
|
||||
/// reconciler and peer bucket-meta ingestion never route through these
|
||||
/// handlers.
|
||||
async fn deny_replication_config_edit_for_non_owner<T>(req: &S3Request<T>) -> S3Result<()> {
|
||||
if crate::storage::access::req_info_ref(req)?.is_owner {
|
||||
return Ok(());
|
||||
}
|
||||
if site_replication_gate_enabled().await? {
|
||||
return Err(replication_deny_edit_error());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct FS {
|
||||
/// This server's late-bound application-context slot (backlog#1052 S2).
|
||||
@@ -500,6 +548,7 @@ impl S3 for FS {
|
||||
&self,
|
||||
req: S3Request<DeleteBucketReplicationInput>,
|
||||
) -> S3Result<S3Response<DeleteBucketReplicationOutput>> {
|
||||
deny_replication_config_edit_for_non_owner(&req).await?;
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
usecase.execute_delete_bucket_replication(req).await
|
||||
}
|
||||
@@ -1353,6 +1402,7 @@ impl S3 for FS {
|
||||
&self,
|
||||
req: S3Request<PutBucketReplicationInput>,
|
||||
) -> S3Result<S3Response<PutBucketReplicationOutput>> {
|
||||
deny_replication_config_edit_for_non_owner(&req).await?;
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
usecase.execute_put_bucket_replication(req).await
|
||||
}
|
||||
@@ -1919,3 +1969,103 @@ impl S3 for FS {
|
||||
Box::pin(usecase.execute_upload_part_copy(req)).await
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
FS, SITE_REPLICATION_GATE_FORCE_DISABLED, SITE_REPLICATION_GATE_FORCE_ENABLED, SITE_REPLICATION_GATE_TEST_OVERRIDE,
|
||||
};
|
||||
use crate::storage::access::ReqInfo;
|
||||
use http::Method;
|
||||
use http::StatusCode;
|
||||
use s3s::dto::{DeleteBucketReplicationInput, PutBucketReplicationInput, ReplicationConfiguration};
|
||||
use s3s::{S3, S3Error, S3ErrorCode, S3Request};
|
||||
use std::sync::atomic::Ordering;
|
||||
|
||||
fn replication_config_edit_request<T>(input: T, is_owner: bool) -> S3Request<T> {
|
||||
let mut req = S3Request {
|
||||
input,
|
||||
method: Method::PUT,
|
||||
uri: http::Uri::from_static("/"),
|
||||
headers: http::HeaderMap::new(),
|
||||
extensions: http::Extensions::new(),
|
||||
credentials: None,
|
||||
region: None,
|
||||
service: None,
|
||||
trailing_headers: None,
|
||||
};
|
||||
req.extensions.insert(ReqInfo {
|
||||
is_owner,
|
||||
..Default::default()
|
||||
});
|
||||
req
|
||||
}
|
||||
|
||||
fn put_bucket_replication_input() -> PutBucketReplicationInput {
|
||||
PutBucketReplicationInput {
|
||||
bucket: "test-bucket".to_string(),
|
||||
checksum_algorithm: None,
|
||||
content_md5: None,
|
||||
expected_bucket_owner: None,
|
||||
replication_configuration: ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: Vec::new(),
|
||||
},
|
||||
token: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn delete_bucket_replication_input() -> DeleteBucketReplicationInput {
|
||||
DeleteBucketReplicationInput {
|
||||
bucket: "test-bucket".to_string(),
|
||||
expected_bucket_owner: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn assert_replication_deny_edit(err: &S3Error) {
|
||||
match err.code() {
|
||||
S3ErrorCode::Custom(code) => assert_eq!(code, "XMinioReplicationDenyEdit"),
|
||||
other => panic!("expected XMinioReplicationDenyEdit, got {other:?}"),
|
||||
}
|
||||
assert_eq!(err.status_code(), Some(StatusCode::BAD_REQUEST));
|
||||
}
|
||||
|
||||
/// Single test on purpose: the branches share the process-wide gate
|
||||
/// override, and parallel tests would race it.
|
||||
#[tokio::test]
|
||||
async fn replication_config_edit_gate_denies_only_non_owner_under_site_replication() {
|
||||
let fs = FS::new();
|
||||
SITE_REPLICATION_GATE_TEST_OVERRIDE.store(SITE_REPLICATION_GATE_FORCE_ENABLED, Ordering::SeqCst);
|
||||
|
||||
// Non-owner PUT/DELETE through the real S3 handlers: denied by the
|
||||
// gate before the usecase (and thus the store) is ever touched.
|
||||
let err = fs
|
||||
.put_bucket_replication(replication_config_edit_request(put_bucket_replication_input(), false))
|
||||
.await
|
||||
.expect_err("non-owner PutBucketReplication must be denied while site replication is enabled");
|
||||
assert_replication_deny_edit(&err);
|
||||
let err = fs
|
||||
.delete_bucket_replication(replication_config_edit_request(delete_bucket_replication_input(), false))
|
||||
.await
|
||||
.expect_err("non-owner DeleteBucketReplication must be denied while site replication is enabled");
|
||||
assert_replication_deny_edit(&err);
|
||||
|
||||
// Owner passes the gate (the usecase's empty-rules structure error
|
||||
// proves the request reached the usecase instead of the deny path).
|
||||
let err = fs
|
||||
.put_bucket_replication(replication_config_edit_request(put_bucket_replication_input(), true))
|
||||
.await
|
||||
.expect_err("owner request should pass the gate and fail later on config validation");
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
|
||||
|
||||
// Without site replication the policy check alone still governs the edit.
|
||||
SITE_REPLICATION_GATE_TEST_OVERRIDE.store(SITE_REPLICATION_GATE_FORCE_DISABLED, Ordering::SeqCst);
|
||||
let err = fs
|
||||
.put_bucket_replication(replication_config_edit_request(put_bucket_replication_input(), false))
|
||||
.await
|
||||
.expect_err("non-owner request should pass the gate and fail later on config validation");
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
|
||||
|
||||
SITE_REPLICATION_GATE_TEST_OVERRIDE.store(0, Ordering::SeqCst);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user