Compare commits

..

4 Commits

Author SHA1 Message Date
overtrue d33e489d84 style: cargo fmt 2026-08-22 07:53:36 +08:00
overtrue 378d46c54a fix(lifecycle): restore #[serial] on 2 tests that read env vars without temp_env
eval_inner_expires_latest_object_after_days_due and
eval_inner_does_not_panic_on_many_equal_due_events call eval_inner()
which reads ENV_ILM_PROCESS_TIME via std::env::var(). Without #[serial]
they race with other tests that set these vars via temp_env.
2026-08-22 01:22:19 +08:00
overtrue 3f6cd10946 cleanup: remove #[serial] annotations from e2e_test, scanner, lifecycle, and object-capacity crates
Remove no-op #[serial] attributes (nextest ignores serial_test) and the
serial_test dependency from four crates. All tests already use temp_env
for env-var isolation, making #[serial] purely redundant.

Crates cleaned:
- e2e_test (37 annotations, 9 imports, removed serial_test dep)
- rustfs-scanner (115 annotations across 7 files, removed serial_test dep)
- rustfs-lifecycle (46 annotations, removed serial_test dep)
- rustfs-object-capacity (38 annotations, removed serial_test dep)

Also converted scanner/tests/lifecycle_integration_test.rs
with_forced_immediate_enqueue_timeout helper from unsafe raw
env::set_var/remove_var to temp_env::async_with_vars for proper
isolation, and added async_closure feature to scanner's temp-env dep.
2026-08-22 00:28:25 +08:00
overtrue 5eaa6c1745 chore(obs): ReplicationStats -> ReplicationMetricsSnapshot, BucketReplicationStats -> BucketReplicationMetricsSnapshot
Rename in-obs-crate ReplicationStats and BucketReplicationStats to
ReplicationMetricsSnapshot and BucketReplicationMetricsSnapshot respectively.
No serde impact (these types are Prometheus metric collectors, not serialized).
No external consumers found outside the obs crate.
2026-08-22 00:04:12 +08:00
36 changed files with 38 additions and 1154 deletions
Generated
-4
View File
@@ -3843,7 +3843,6 @@ dependencies = [
"s3s",
"serde",
"serde_json",
"serial_test",
"sha2 0.11.0",
"suppaftp",
"time",
@@ -9804,7 +9803,6 @@ dependencies = [
"rustfs-replication",
"rustfs-storage-api",
"s3s",
"serial_test",
"temp-env",
"time",
"tokio",
@@ -9920,7 +9918,6 @@ dependencies = [
"rustfs-config",
"rustfs-io-metrics",
"rustfs-utils",
"serial_test",
"temp-env",
"tempfile",
"tokio",
@@ -10293,7 +10290,6 @@ dependencies = [
"s3s",
"serde",
"serde_json",
"serial_test",
"sha2 0.11.0",
"temp-env",
"tempfile",
-1
View File
@@ -96,7 +96,6 @@ tokio-stream = { workspace = true }
rustfs-madmin.workspace = true
rustfs-filemeta.workspace = true
bytes = { workspace = true, features = ["serde"] }
serial_test = { workspace = true }
aws-sdk-s3 = { workspace = true, default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
aws-sdk-sts = { workspace = true, default-features = false, features = ["default-https-client", "rt-tokio"] }
aws-config = { workspace = true }
@@ -55,7 +55,6 @@ mod tests {
use aws_sdk_s3::Client;
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
use serial_test::serial;
use sha2::{Digest, Sha256};
use std::error::Error;
use tokio::time::{Duration, timeout};
@@ -269,7 +268,6 @@ mod tests {
/// stripes) and a multipart object (3 parts × 5 MiB) must GET back as a
/// full, byte-identical body with the correct Content-Length. No early EOF.
#[tokio::test]
#[serial]
async fn degraded_read_large_objects_with_one_disk_offline_return_full_body() -> TestResult {
init_logging();
info!("dist-13 (a): large-object degraded read with one of four disks offline");
@@ -335,7 +333,6 @@ mod tests {
/// mid-stream — the exact window the fixes had to reconstruct through rather
/// than truncate.
#[tokio::test]
#[serial]
async fn degraded_read_reconstructs_through_midstream_bitrot_within_quorum() -> TestResult {
init_logging();
info!("dist-13 (b): mid-stream bitrot within quorum must reconstruct a full body");
@@ -393,7 +390,6 @@ mod tests {
/// Content-Length. `get_checked` panics on that forbidden outcome, so this
/// test fails loudly if the truncation bug ever returns.
#[tokio::test]
#[serial]
async fn beyond_quorum_degraded_read_never_silently_truncates() -> TestResult {
init_logging();
info!("dist-13 (c): beyond-quorum degraded read must fail, never 200+truncated");
@@ -51,7 +51,6 @@ mod tests {
use aws_sdk_s3::Client;
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
use serial_test::serial;
use std::error::Error;
use tokio::time::{Duration, timeout};
use tracing::info;
@@ -129,7 +128,6 @@ mod tests {
/// the body — and assert the server log names the object, at the log level a
/// default deployment actually runs with.
#[tokio::test]
#[serial]
async fn midstream_get_failure_is_logged_with_the_object_at_default_log_level() -> TestResult {
init_logging();
info!("rustfs#4784: a mid-stream GET failure must name its object in the source log");
@@ -46,7 +46,6 @@ use prost::Message;
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
use rustfs_signer::sign_v4;
use s3s::Body;
use serial_test::serial;
use std::collections::BTreeMap;
use std::convert::Infallible;
use std::error::Error;
@@ -1695,7 +1694,6 @@ fn assert_storage_layout(
}
#[tokio::test]
#[serial]
async fn four_node_inline_storage_and_get_boundaries() -> TestResult {
init_logging();
@@ -1767,7 +1765,6 @@ async fn four_node_inline_storage_and_get_boundaries() -> TestResult {
}
#[tokio::test]
#[serial]
async fn four_node_empty_legacy_volumes_start_as_fresh() -> TestResult {
init_logging();
@@ -1805,7 +1802,6 @@ async fn four_node_empty_legacy_volumes_start_as_fresh() -> TestResult {
}
#[tokio::test]
#[serial]
async fn four_node_inline_fallback_controls() -> TestResult {
init_logging();
@@ -1870,7 +1866,6 @@ async fn four_node_inline_fallback_controls() -> TestResult {
}
#[tokio::test]
#[serial]
async fn four_node_compressed_inline_fallback() -> TestResult {
init_logging();
@@ -1905,7 +1900,6 @@ async fn four_node_compressed_inline_fallback() -> TestResult {
/// Multipart disk compression is live again, so a compression-enabled cluster classifies multipart objects as compressed and the roundtrip (full GET plus partNumber GET) must still return the original bytes.
/// Reverting the multipart compression fix must fail this test.
#[tokio::test]
#[serial]
async fn four_node_multipart_disk_compression_roundtrip() -> TestResult {
init_logging();
@@ -1952,7 +1946,6 @@ async fn four_node_multipart_disk_compression_roundtrip() -> TestResult {
/// read costs on the order of the covering part's block size against a ~5 MiB
/// object.
#[tokio::test]
#[serial]
async fn four_node_compressed_multipart_tail_range_reads_are_bounded() -> TestResult {
init_logging();
@@ -2019,7 +2012,6 @@ async fn four_node_compressed_multipart_tail_range_reads_are_bounded() -> TestRe
}
#[tokio::test]
#[serial]
async fn four_node_mixed_msgpack_compat_mode_preserves_fallback_controls() -> TestResult {
init_logging();
@@ -2123,7 +2115,6 @@ async fn four_node_mixed_msgpack_compat_mode_preserves_fallback_controls() -> Te
}
#[tokio::test]
#[serial]
async fn four_node_add_tier_converges() -> TestResult {
init_logging();
@@ -2142,7 +2133,6 @@ async fn four_node_add_tier_converges() -> TestResult {
}
#[tokio::test]
#[serial]
async fn four_node_add_tier_converges_after_offline_node_restart_without_second_mutation() -> TestResult {
init_logging();
@@ -2164,7 +2154,6 @@ async fn four_node_add_tier_converges_after_offline_node_restart_without_second_
}
#[tokio::test]
#[serial]
async fn four_node_manual_transition_job_status_survives_node_restart() -> TestResult {
init_logging();
@@ -2239,7 +2228,6 @@ async fn four_node_manual_transition_job_status_survives_node_restart() -> TestR
}
#[tokio::test]
#[serial]
async fn four_node_manual_transition_distributed_admission_conflict_reports_status_and_backpressure() -> TestResult {
init_logging();
@@ -2381,7 +2369,6 @@ async fn four_node_manual_transition_distributed_admission_conflict_reports_stat
}
#[tokio::test]
#[serial]
#[ignore = "manual #1508 evidence harness: starts a 4-node cluster, a remote tier, and an in-flight transition job"]
async fn four_node_manual_transition_rollout_non_empty_restart_readback() -> TestResult {
init_logging();
@@ -2486,7 +2473,6 @@ async fn four_node_manual_transition_rollout_non_empty_restart_readback() -> Tes
}
#[tokio::test]
#[serial]
async fn four_node_mixed_msgpack_compat_mode_preserves_fallback_controls_during_transition() -> TestResult {
init_logging();
@@ -2598,7 +2584,6 @@ async fn four_node_mixed_msgpack_compat_mode_preserves_fallback_controls_during_
}
#[tokio::test]
#[serial]
async fn four_node_transitioned_inline_fallback() -> TestResult {
init_logging();
@@ -19,7 +19,6 @@
//! multipart upload behaviour.
use crate::common::{TEST_BUCKET, init_logging};
use serial_test::serial;
use tokio::time::{Duration, sleep};
use tracing::{error, info};
@@ -62,7 +61,6 @@ impl VaultKmsTestContext {
}
#[tokio::test]
#[serial]
async fn test_vault_kms_end_to_end() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if skip_if_kms_admin_tool_unavailable("test_vault_kms_end_to_end") {
@@ -118,7 +116,6 @@ async fn test_vault_kms_end_to_end() -> Result<(), Box<dyn std::error::Error + S
}
#[tokio::test]
#[serial]
async fn test_vault_kms_key_isolation() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if skip_if_kms_admin_tool_unavailable("test_vault_kms_key_isolation") {
@@ -205,7 +202,6 @@ async fn test_vault_kms_key_isolation() -> Result<(), Box<dyn std::error::Error
}
#[tokio::test]
#[serial]
async fn test_vault_kms_large_file() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if skip_if_kms_admin_tool_unavailable("test_vault_kms_large_file") {
@@ -270,7 +266,6 @@ async fn test_vault_kms_large_file() -> Result<(), Box<dyn std::error::Error + S
}
#[tokio::test]
#[serial]
async fn test_vault_kms_multipart_upload() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if skip_if_kms_admin_tool_unavailable("test_vault_kms_multipart_upload") {
@@ -301,7 +296,6 @@ async fn test_vault_kms_multipart_upload() -> Result<(), Box<dyn std::error::Err
}
#[tokio::test]
#[serial]
async fn test_vault_kms_key_operations() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if skip_if_kms_admin_tool_unavailable("test_vault_kms_key_operations") {
-5
View File
@@ -61,11 +61,6 @@ mod get_codec_streaming_compat_test;
#[cfg(test)]
mod version_id_regression_test;
// Receiver-side replication LWW (rustfs/backlog#1953): stale inbound
// replication metadata must not overwrite a newer local category state.
#[cfg(test)]
mod replication_lww_receiver_test;
// Data usage regression tests
#[cfg(test)]
mod data_usage_test;
@@ -17,7 +17,6 @@
use crate::common::{awscurl_delete, awscurl_put, init_logging};
use crate::policy::test_env::PolicyTestEnvironment;
use aws_sdk_s3::primitives::ByteStream;
use serial_test::serial;
use tracing::info;
/// Helper function to create a regular user with given credentials
@@ -122,7 +121,6 @@ async fn cleanup_user_and_policy(env: &PolicyTestEnvironment, username: &str, po
/// Test AWS policy variables with single-value scenarios
#[tokio::test(flavor = "multi_thread")]
#[serial]
#[ignore = "Starts a rustfs server; enable when running full E2E"]
pub async fn test_aws_policy_variables_single_value() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
test_aws_policy_variables_single_value_impl().await
@@ -275,7 +273,6 @@ pub async fn test_aws_policy_variables_single_value_impl_with_env(
/// Test AWS policy variables with multi-value scenarios
#[tokio::test(flavor = "multi_thread")]
#[serial]
#[ignore = "Starts a rustfs server; enable when running full E2E"]
pub async fn test_aws_policy_variables_multi_value() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
test_aws_policy_variables_multi_value_impl().await
@@ -401,7 +398,6 @@ pub async fn test_aws_policy_variables_multi_value_impl_with_env(
/// Test AWS policy variables with variable concatenation
#[tokio::test(flavor = "multi_thread")]
#[serial]
#[ignore = "Starts a rustfs server; enable when running full E2E"]
pub async fn test_aws_policy_variables_concatenation() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
test_aws_policy_variables_concatenation_impl().await
@@ -491,7 +487,6 @@ pub async fn test_aws_policy_variables_concatenation_impl_with_env(
/// Test AWS policy variables with nested scenarios
#[tokio::test(flavor = "multi_thread")]
#[serial]
#[ignore = "Starts a rustfs server; enable when running full E2E"]
pub async fn test_aws_policy_variables_nested() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
test_aws_policy_variables_nested_impl().await
@@ -509,7 +504,6 @@ pub async fn test_aws_policy_variables_nested_impl() -> Result<(), Box<dyn std::
/// Test AWS policy variables with STS temporary credentials
#[tokio::test(flavor = "multi_thread")]
#[serial]
#[ignore = "Starts a rustfs server; enable when running full E2E"]
pub async fn test_aws_policy_variables_sts() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
test_aws_policy_variables_sts_impl().await
@@ -705,7 +699,6 @@ pub async fn test_aws_policy_variables_sts_impl_with_env(
/// Test AWS policy variables with deny scenarios
#[tokio::test(flavor = "multi_thread")]
#[serial]
#[ignore = "Starts a rustfs server; enable when running full E2E"]
pub async fn test_aws_policy_variables_deny() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
test_aws_policy_variables_deny_impl().await
@@ -14,7 +14,6 @@
use crate::common::init_logging;
use crate::policy::test_env::PolicyTestEnvironment;
use serial_test::serial;
use std::time::Instant;
use tokio::time::{Duration, sleep};
use tracing::{error, info};
@@ -213,7 +212,6 @@ impl PolicyTestSuite {
/// Test suite
#[tokio::test]
#[serial]
#[ignore = "Connects to existing rustfs server"]
async fn test_policy_critical_suite() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let config = TestSuiteConfig {
@@ -41,7 +41,6 @@ use reqwest::Client;
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
use rustfs_signer::sign_v4;
use s3s::Body;
use serial_test::serial;
use tokio::process::Command;
use tracing::info;
@@ -821,7 +820,6 @@ pub async fn test_webdav_core_operations() -> Result<()> {
}
#[tokio::test]
#[serial]
async fn test_webdav_core_operations_direct() -> Result<()> {
test_webdav_core_operations().await
}
@@ -27,7 +27,6 @@ mod tests {
use aws_sdk_s3::Client;
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, VersioningConfiguration};
use serial_test::serial;
use sha2::{Digest, Sha256};
use std::collections::HashSet;
use std::error::Error;
@@ -157,7 +156,6 @@ mod tests {
/// content, degraded writes must succeed, and everything must still
/// verify after the disk returns.
#[tokio::test]
#[serial]
async fn test_degraded_read_write_with_one_disk_offline() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
info!("Reliability: degraded read/write with one of four disks offline");
@@ -210,7 +208,6 @@ mod tests {
/// bytes to a reader: per-shard bitrot checksums reject the bad shard and
/// the object is reconstructed from the remaining shards.
#[tokio::test]
#[serial]
async fn test_bitrot_corrupted_shard_read_returns_correct_data() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
info!("Reliability: GET must read through a bitrot-corrupted shard");
@@ -253,7 +250,6 @@ mod tests {
/// heal, and require the replaced disk to be rebuilt and all content to
/// verify against the sha256 manifest.
#[tokio::test]
#[serial]
async fn test_fresh_disk_replacement_heals_after_sigkill_restart() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
info!("Reliability: fresh-disk replacement heals after SIGKILL restart");
@@ -327,7 +323,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn test_versioned_shard_census_selects_each_version_data_dir() -> Result<(), Box<dyn Error + Send + Sync>> {
init_logging();
info!("Reliability: physical shard census selects the requested object version");
@@ -29,7 +29,6 @@ mod tests {
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, VersioningConfiguration};
use http::Method;
use serial_test::serial;
use sha2::{Digest, Sha256};
use std::collections::BTreeSet;
use std::error::Error;
@@ -1061,7 +1060,6 @@ mod tests {
/// Linux mount namespaces are per-thread; keep mount setup and process
/// spawning on one OS thread so child RustFS nodes inherit the test mounts.
#[tokio::test(flavor = "current_thread")]
#[serial]
#[ignore = "requires Linux root/CAP_SYS_ADMIN and RUSTFS_PRIVILEGED_REPLACEMENT_E2E=1"]
async fn test_privileged_3x4_auto_replacement_rebuilds_ec8_plus_4_without_admin_heal()
-> Result<(), Box<dyn Error + Send + Sync>> {
@@ -1075,7 +1073,6 @@ mod tests {
/// Linux mount namespaces are per-thread; keep mount setup and process
/// spawning on one OS thread so child RustFS nodes inherit the test mounts.
#[tokio::test(flavor = "current_thread")]
#[serial]
#[ignore = "requires Linux root/CAP_SYS_ADMIN and RUSTFS_PRIVILEGED_REPLACEMENT_E2E=1"]
async fn test_privileged_3x4_auto_replacement_rebuilds_ec6_plus_6_without_admin_heal()
-> Result<(), Box<dyn Error + Send + Sync>> {
@@ -1,150 +0,0 @@
#![cfg(test)]
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Receiver-side replication LWW over the wire (rustfs/backlog#1953, audit
//! A4/P1-6).
//!
//! In an active-active topology both sites' metadata states arrive at the
//! peer as authorized replication PUTs carrying per-category source
//! timestamps (`x-rustfs-source-replication-tagging-timestamp` header
//! family). Before the fix the receiver applied them unconditionally, so a
//! stale delivery overwrote a newer local state and the two sites diverged
//! permanently while both reported COMPLETED. This test drives one live
//! `rustfs` server with simulated inbound replication PUTs for the same
//! object version and asserts the newer tagging state wins regardless of
//! delivery order, while a stale delivery still succeeds at the object level
//! (a failure would loop through MRF re-delivering the stale value).
use crate::common::{RustFSTestEnvironment, init_logging};
use aws_sdk_s3::Client;
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
const HDR_SOURCE_REPLICATION_REQUEST: &str = "x-rustfs-source-replication-request";
const HDR_SOURCE_VERSION_ID: &str = "x-rustfs-source-version-id";
const HDR_SOURCE_MTIME: &str = "x-rustfs-source-mtime";
const HDR_SOURCE_TAGGING_TIMESTAMP: &str = "x-rustfs-source-replication-tagging-timestamp";
const SOURCE_MTIME: &str = "2026-01-01T00:00:00Z";
const T_STALE: &str = "2026-01-01T00:00:01Z";
const T_LOCAL: &str = "2026-02-01T00:00:00Z";
const T_NEWER: &str = "2026-03-01T00:00:00Z";
/// Simulated inbound authorized replication PUT: same object version, tags and
/// the source-authored tagging timestamp carried in transport headers.
async fn inbound_replication_put(
client: &Client,
bucket: &str,
key: &str,
version_id: &str,
tags: &str,
tagging_timestamp: &str,
) -> TestResult {
let version_id = version_id.to_string();
let tagging_timestamp = tagging_timestamp.to_string();
client
.put_object()
.bucket(bucket)
.key(key)
.body(ByteStream::from_static(b"lww-e2e-body"))
.tagging(tags)
.customize()
.mutate_request(move |req| {
req.headers_mut().insert(HDR_SOURCE_REPLICATION_REQUEST, "true");
req.headers_mut().insert(HDR_SOURCE_VERSION_ID, version_id.clone());
req.headers_mut().insert(HDR_SOURCE_MTIME, SOURCE_MTIME);
req.headers_mut()
.insert(HDR_SOURCE_TAGGING_TIMESTAMP, tagging_timestamp.clone());
})
.send()
.await?;
Ok(())
}
async fn tag_value(client: &Client, bucket: &str, key: &str, version_id: &str, tag_key: &str) -> Option<String> {
let tagging = client
.get_object_tagging()
.bucket(bucket)
.key(key)
.version_id(version_id)
.send()
.await
.expect("object tagging should be readable");
tagging
.tag_set()
.iter()
.find(|tag| tag.key() == tag_key)
.map(|tag| tag.value().to_string())
}
#[tokio::test(flavor = "multi_thread")]
async fn receiver_lww_keeps_newer_tags_across_delivery_orders() -> TestResult {
init_logging();
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server(vec![]).await?;
let client = env.create_s3_client();
let bucket = "replication-lww-receiver";
let key = "object";
client.create_bucket().bucket(bucket).send().await?;
client
.put_bucket_versioning()
.bucket(bucket)
.versioning_configuration(
VersioningConfiguration::builder()
.status(BucketVersioningStatus::Enabled)
.build(),
)
.send()
.await?;
// First delivery establishes version V with tags stamped T_LOCAL.
let version_id = uuid::Uuid::new_v4().to_string();
inbound_replication_put(&client, bucket, key, &version_id, "site=local", T_LOCAL).await?;
assert_eq!(
tag_value(&client, bucket, key, &version_id, "site").await.as_deref(),
Some("local"),
"the first delivery must establish the tagged version"
);
// A stale delivery (older source timestamp) must succeed at the object
// level but must NOT overwrite the newer tags.
inbound_replication_put(&client, bucket, key, &version_id, "site=stale", T_STALE).await?;
assert_eq!(
tag_value(&client, bucket, key, &version_id, "site").await.as_deref(),
Some("local"),
"a stale inbound delivery must not overwrite newer tags (rustfs/backlog#1953)"
);
// A newer delivery still converges the version onto the newest state.
inbound_replication_put(&client, bucket, key, &version_id, "site=newer", T_NEWER).await?;
assert_eq!(
tag_value(&client, bucket, key, &version_id, "site").await.as_deref(),
Some("newer"),
"a newer inbound delivery must overwrite older tags"
);
client
.delete_object()
.bucket(bucket)
.key(key)
.version_id(&version_id)
.send()
.await?;
env.delete_test_bucket(bucket).await.ok();
Ok(())
}
@@ -3868,7 +3868,6 @@ async fn replicate_object_with_multipart<S: ReplicationObjectIO>(ctx: MultipartR
actual_size,
object_info.etag.clone().unwrap_or_default(),
object_info.mod_time,
&put_opts.internal,
),
)
.await
@@ -472,7 +472,6 @@ pub(crate) fn replication_complete_multipart_options(
actual_size: String,
source_etag: String,
source_mtime: Option<OffsetDateTime>,
source_internal: &AdvancedPutOptions,
) -> PutObjectOptions {
let mut user_metadata = HashMap::new();
insert_header_map(&mut user_metadata, SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE, actual_size);
@@ -485,14 +484,6 @@ pub(crate) fn replication_complete_multipart_options(
// mtime must degrade to epoch so header() suppresses the header
// instead of asserting the replication time as the object's mtime.
source_mtime: source_mtime.unwrap_or(OffsetDateTime::UNIX_EPOCH),
// Carry the per-category LWW timestamps on the complete request as
// well: the receiver's CompleteMultipartUpload options builder
// parses the same headers, so the multipart transport gets the
// same receiver-side LWW as the single-PUT transport
// (rustfs/backlog#1953). Epoch values keep the headers suppressed.
tagging_timestamp: source_internal.tagging_timestamp,
retention_timestamp: source_internal.retention_timestamp,
legalhold_timestamp: source_internal.legalhold_timestamp,
replication_status: ReplicationStatusType::Replica,
replication_request: true,
..Default::default()
@@ -658,39 +649,20 @@ mod tests {
#[test]
fn replication_complete_multipart_options_sets_actual_size() {
let source_mtime = OffsetDateTime::from_unix_timestamp(1_716_170_000).expect("valid test timestamp");
let source_internal = AdvancedPutOptions {
tagging_timestamp: OffsetDateTime::from_unix_timestamp(1_716_170_100).expect("valid test timestamp"),
retention_timestamp: OffsetDateTime::from_unix_timestamp(1_716_170_200).expect("valid test timestamp"),
legalhold_timestamp: OffsetDateTime::from_unix_timestamp(1_716_170_300).expect("valid test timestamp"),
..Default::default()
};
let options = replication_complete_multipart_options(
"1024".to_string(),
"0123456789abcdef0123456789abcdef-3".to_string(),
Some(source_mtime),
&source_internal,
);
assert_eq!(options.internal.source_etag, "0123456789abcdef0123456789abcdef-3");
assert_eq!(options.internal.source_mtime, source_mtime);
// The complete request must carry the same per-category LWW timestamps
// as the initiate request; the receiver reads them from the complete
// headers (rustfs/backlog#1953).
assert_eq!(options.internal.tagging_timestamp, source_internal.tagging_timestamp);
assert_eq!(options.internal.retention_timestamp, source_internal.retention_timestamp);
assert_eq!(options.internal.legalhold_timestamp, source_internal.legalhold_timestamp);
// Absent source mtime must degrade to epoch (header suppressed), not
// the AdvancedPutOptions default of now_utc() — that default would
// stamp the replication time as the replica's mtime and break the
// multipart HEAD convergence. Unset category timestamps stay epoch so
// header() keeps suppressing them.
let options_no_mtime =
replication_complete_multipart_options("1024".to_string(), String::new(), None, &AdvancedPutOptions::default());
// multipart HEAD convergence.
let options_no_mtime = replication_complete_multipart_options("1024".to_string(), String::new(), None);
assert_eq!(options_no_mtime.internal.source_mtime.unix_timestamp(), 0);
assert_eq!(options_no_mtime.internal.tagging_timestamp.unix_timestamp(), 0);
assert_eq!(options_no_mtime.internal.retention_timestamp.unix_timestamp(), 0);
assert_eq!(options_no_mtime.internal.legalhold_timestamp.unix_timestamp(), 0);
assert_eq!(
get_header_map(&options.user_metadata, SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE).as_deref(),
@@ -2223,57 +2223,6 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
fi.set_data_moved();
}
// Receiver-side LWW (rustfs/backlog#1953): the multipart replication
// transport carries the category values at CreateMultipartUpload (in
// the staged upload metadata) and the source category timestamps on
// the complete request. Read the destination version under the held
// object write lock and keep any category this site modified more
// recently. Read failures (version absent on first replication, quorum
// errors) keep today's overwrite semantics: failing the complete would
// loop through MRF, re-delivering the stale value forever.
if crate::set_disk::ops::object::replication_lww_applicable(opts)
&& let Some(version_id) = fi.version_id
{
match self
.get_object_info(
bucket,
object,
&ObjectOptions {
version_id: Some(version_id.to_string()),
no_lock: true,
metadata_cache_safe: false,
versioned: opts.versioned,
version_suspended: opts.version_suspended,
..Default::default()
},
)
.await
{
Ok(existing) => {
let stored = crate::set_disk::ops::object::stored_replication_category_metadata(&existing);
crate::set_disk::ops::object::merge_replication_metadata_lww(&mut fi.metadata, &stored, opts);
}
// Version absent: first replication of this version, nothing
// local to compare — the normal path, not a degraded one.
Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {}
Err(err) => {
// Degraded path: without the stored state the inbound
// metadata is applied unchanged — exactly the overwrite
// LWW exists to prevent — so this must be operator-visible.
warn!(
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_SET_DISK,
bucket,
object,
version_id = %version_id,
error = %err,
state = "replication_lww_read_unavailable",
"SetDisk multipart replication LWW read skipped; inbound metadata applied without comparison"
);
}
}
}
for meta in parts_metadatas.iter_mut() {
if meta.has_valid_erasure_geometry() {
meta.size = fi.size;
@@ -7011,97 +6960,6 @@ mod tests {
.await
}
/// Receiver-side LWW on the multipart replication transport
/// (rustfs/backlog#1953): a metadata-only replication of a multipart
/// source object rides CreateMultipartUpload (category values in the
/// upload metadata) + CompleteMultipartUpload (category timestamps in
/// the complete options). A stale inbound tagging timestamp must not
/// overwrite a newer locally-tagged destination version.
#[tokio::test]
#[serial]
async fn complete_multipart_upload_stale_replication_tags_keep_local() {
use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING;
use rustfs_utils::http::{SUFFIX_TAGGING_TIMESTAMP, get_str};
use time::format_description::well_known::Rfc3339;
const T_OLD: &str = "2026-01-01T00:00:00Z";
const T_LOCAL: &str = "2026-02-01T00:00:00Z";
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "multipart-replication-lww-bucket";
let object = "object";
make_bucket_on_all(&disk_stores, bucket).await;
// Local destination version with newer tags.
let version_id = Uuid::new_v4();
let mut local_metadata = HashMap::new();
local_metadata.insert(AMZ_OBJECT_TAGGING.to_string(), "site=local".to_string());
rustfs_utils::http::insert_str(&mut local_metadata, SUFFIX_TAGGING_TIMESTAMP, T_LOCAL.to_string());
let mut local_reader = PutObjReader::from_vec(b"local body".to_vec());
set_disks
.put_object(
bucket,
object,
&mut local_reader,
&ObjectOptions {
versioned: true,
version_id: Some(version_id.to_string()),
user_defined: local_metadata,
// Explicit-version PUTs require the bucket Object Lock snapshot.
object_lock_config_snapshot: Some(Arc::new(crate::set_disk::ObjectLockConfigSnapshot::new(
crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
))),
..Default::default()
},
)
.await
.expect("local versioned put should commit");
// Inbound replication upload carrying older tags for the same version.
let mut inbound_metadata = HashMap::new();
inbound_metadata.insert(AMZ_OBJECT_TAGGING.to_string(), "site=remote".to_string());
rustfs_utils::http::insert_str(&mut inbound_metadata, SUFFIX_TAGGING_TIMESTAMP, T_OLD.to_string());
let create_opts = ObjectOptions {
versioned: true,
user_defined: inbound_metadata,
..Default::default()
};
let (upload_id, parts) =
stage_upload_with_create_opts(&set_disks, bucket, object, &payload(0x5a), &create_opts).await;
rewrite_staged_upload_version_id(&set_disks, bucket, object, &upload_id, Some(version_id)).await;
let complete_opts = ObjectOptions {
versioned: true,
replication_request: true,
replication_tagging_timestamp: Some(OffsetDateTime::parse(T_OLD, &Rfc3339).expect("test timestamp should parse")),
..Default::default()
};
set_disks
.clone()
.complete_multipart_upload(bucket, object, &upload_id, parts, &complete_opts)
.await
.expect("replication multipart completion should succeed even when a category keeps local values");
let info = set_disks
.get_object_info(
bucket,
object,
&ObjectOptions {
versioned: true,
version_id: Some(version_id.to_string()),
..Default::default()
},
)
.await
.expect("completed version should be readable");
assert_eq!(
info.user_tags.as_str(),
"site=local",
"older inbound multipart tags must not overwrite newer local tags"
);
assert_eq!(get_str(&info.user_defined, SUFFIX_TAGGING_TIMESTAMP).as_deref(), Some(T_LOCAL));
}
#[tokio::test]
#[serial]
async fn complete_multipart_upload_assigns_completion_version_id() {
-471
View File
@@ -1880,110 +1880,6 @@ fn delete_file_info_with_replication_transport_metadata(fi: &FileInfo) -> FileIn
transported
}
/// True when an authorized replication write carries at least one per-category
/// source timestamp, i.e. receiver-side LWW has something to judge.
pub(in crate::set_disk) fn replication_lww_applicable(opts: &ObjectOptions) -> bool {
opts.replication_request
&& (opts.replication_tagging_timestamp.is_some()
|| opts.replication_retention_timestamp.is_some()
|| opts.replication_legalhold_timestamp.is_some())
}
/// The stored per-category state of a destination version, as compared by
/// [`merge_replication_metadata_lww`]. `ObjectInfo::from_file_info`
/// externalizes tags into `user_tags` (stripping the metadata key), so the
/// tag value is folded back into map form here.
pub(in crate::set_disk) fn stored_replication_category_metadata(existing: &ObjectInfo) -> HashMap<String, String> {
let mut stored = (*existing.user_defined).clone();
if !existing.user_tags.is_empty() {
stored.insert(rustfs_utils::http::headers::AMZ_OBJECT_TAGGING.to_string(), (*existing.user_tags).clone());
}
stored
}
/// Receiver-side last-writer-wins for authorized replication writes
/// (rustfs/backlog#1953, audit A4/P1-6). Metadata-only replication reuses the
/// whole-object transports, so in active-active topologies an inbound write
/// carries the source's tags / retention / legal hold verbatim and would
/// otherwise overwrite a category the destination modified more recently —
/// both sites end up permanently diverged while reporting COMPLETED.
///
/// Judged per category, only when the inbound request carries that category's
/// source timestamp (`ObjectOptions::replication_*_timestamp`):
/// - stored timestamp newer than inbound: the local category values and
/// timestamp are kept; the rest of the write proceeds per the inbound
/// metadata and the object-level result stays successful (failing the write
/// instead would loop through MRF, re-delivering the stale value forever);
/// - otherwise the inbound category wins and its internal timestamp key is
/// pinned to the source-authored time — the PUT path re-stamps the
/// object-lock timestamps with the receiver's clock
/// (`parse_object_lock_retention` / `parse_object_lock_legal_hold` insert
/// `now()` via `eval_metadata`), which would make the replica's clock the
/// LWW authority and wedge later convergence;
/// - no stored timestamp (pre-P1-6 data) or no inbound timestamp: the current
/// overwrite behavior is preserved.
///
/// Returns whether `inbound` was modified. Callers must hold the object write
/// lock so the stored values compared here are the ones being replaced.
pub(in crate::set_disk) fn merge_replication_metadata_lww(
inbound: &mut HashMap<String, String>,
existing: &HashMap<String, String>,
opts: &ObjectOptions,
) -> bool {
use rustfs_utils::http::headers::{
AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER, AMZ_OBJECT_TAGGING,
};
use rustfs_utils::http::metadata_compat::{
SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, SUFFIX_TAGGING_TIMESTAMP, get_str,
remove_str,
};
use time::format_description::well_known::Rfc3339;
let categories: [(Option<OffsetDateTime>, &str, &[&str]); 3] = [
(opts.replication_tagging_timestamp, SUFFIX_TAGGING_TIMESTAMP, &[AMZ_OBJECT_TAGGING]),
(
opts.replication_retention_timestamp,
SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP,
&[AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER],
),
(
opts.replication_legalhold_timestamp,
SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP,
&[AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER],
),
];
let mut changed = false;
for (inbound_timestamp, timestamp_suffix, value_keys) in categories {
let Some(inbound_timestamp) = inbound_timestamp else { continue };
let is_category_value_key = |key: &str| value_keys.iter().any(|value_key| key.eq_ignore_ascii_case(value_key));
let stored_timestamp = get_str(existing, timestamp_suffix).and_then(|value| OffsetDateTime::parse(&value, &Rfc3339).ok());
if stored_timestamp.is_some_and(|stored| stored > inbound_timestamp) {
inbound.retain(|key, _| !is_category_value_key(key));
remove_str(inbound, timestamp_suffix);
for (key, value) in existing {
if is_category_value_key(key) {
inbound.insert(key.clone(), value.clone());
}
}
// Restore the winning timestamp via insert_str, not a verbatim key
// copy: a MinIO-written version may carry only the
// x-minio-internal- key, and the dual-key invariant requires every
// write to produce both keys.
if let Some(stored_value) = get_str(existing, timestamp_suffix) {
rustfs_utils::http::insert_str(inbound, timestamp_suffix, stored_value);
}
changed = true;
} else if let Ok(source_authored) = inbound_timestamp.format(&Rfc3339)
&& get_str(inbound, timestamp_suffix).as_deref() != Some(source_authored.as_str())
{
rustfs_utils::http::insert_str(inbound, timestamp_suffix, source_authored);
changed = true;
}
}
changed
}
impl SetDisks {
pub(in crate::set_disk) async fn persist_old_data_cleanup_receipts(
&self,
@@ -2664,22 +2560,6 @@ impl SetDisks {
if check_object_lock_for_deletion_with_state(object_lock_config.state(), &existing, false)?.is_some() {
return Err(StorageError::PrefixAccessDenied(bucket.to_string(), object.to_string()));
}
// Receiver-side LWW (rustfs/backlog#1953): reuse this
// commit-lock read of the destination version so a
// category (tags / retention / legal hold) modified
// more recently on this site is kept instead of being
// overwritten by the inbound replication metadata.
if replication_lww_applicable(opts) {
let stored = stored_replication_category_metadata(&existing);
let mut merged = parts_metadatas[response_metadata_slot].metadata.clone();
if merge_replication_metadata_lww(&mut merged, &stored, opts) {
for (pfi, disk) in parts_metadatas.iter_mut().zip(shuffle_disks.iter()) {
if disk.is_some() {
pfi.metadata = merged.clone();
}
}
}
}
}
Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {}
Err(err) => return Err(err),
@@ -8007,357 +7887,6 @@ mod replication_quota_safety_tests {
}
}
#[cfg(test)]
mod replication_lww_tests {
//! Receiver-side LWW for authorized replication writes (rustfs/backlog#1953,
//! audit A4/P1-6): an inbound replication PUT whose per-category timestamp
//! (tags / retention / legal hold) is older than the destination version's
//! stored timestamp must keep the local category values instead of
//! overwriting them; categories are judged independently and the write
//! itself still succeeds.
use super::hermetic_set_disks_support::hermetic_set_disks_isolated as hermetic_set_disks;
use super::*;
use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
use rustfs_utils::http::headers::{
AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER, AMZ_OBJECT_TAGGING,
};
use rustfs_utils::http::{
SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, SUFFIX_TAGGING_TIMESTAMP, get_str,
insert_str,
};
use time::format_description::well_known::Rfc3339;
const T_OLD: &str = "2026-01-01T00:00:00Z";
const T_LOCAL: &str = "2026-02-01T00:00:00Z";
const T_NEW: &str = "2026-03-01T00:00:00Z";
fn parse_ts(value: &str) -> OffsetDateTime {
OffsetDateTime::parse(value, &Rfc3339).expect("test timestamp should parse")
}
async fn make_bucket(disks: &[DiskStore], bucket: &str) {
for disk in disks {
disk.make_volume(bucket).await.expect("bucket volume should be created");
}
}
async fn put_version(set_disks: &Arc<SetDisks>, bucket: &str, object: &str, version_id: &str, opts: &ObjectOptions) {
let mut reader = PutObjReader::from_vec(b"lww-body".to_vec());
set_disks
.put_object(bucket, object, &mut reader, opts)
.await
.expect("versioned put should commit");
assert_eq!(opts.version_id.as_deref(), Some(version_id));
}
fn versioned_opts(version_id: &str, user_defined: HashMap<String, String>) -> ObjectOptions {
ObjectOptions {
versioned: true,
version_id: Some(version_id.to_string()),
user_defined,
// Explicit-version PUTs require the bucket Object Lock snapshot.
object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(
crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
))),
..Default::default()
}
}
/// Local state: version `version_id` with tags "site=local" stamped `T_LOCAL`.
async fn seed_local_tagged_version(set_disks: &Arc<SetDisks>, bucket: &str, object: &str, version_id: &str) {
let mut user_defined = HashMap::new();
user_defined.insert(AMZ_OBJECT_TAGGING.to_string(), "site=local".to_string());
insert_str(&mut user_defined, SUFFIX_TAGGING_TIMESTAMP, T_LOCAL.to_string());
put_version(set_disks, bucket, object, version_id, &versioned_opts(version_id, user_defined)).await;
}
fn inbound_tagging_opts(version_id: &str, tags: &str, timestamp: &str) -> ObjectOptions {
let mut user_defined = HashMap::new();
user_defined.insert(AMZ_OBJECT_TAGGING.to_string(), tags.to_string());
insert_str(&mut user_defined, SUFFIX_TAGGING_TIMESTAMP, timestamp.to_string());
ObjectOptions {
replication_request: true,
replication_tagging_timestamp: Some(parse_ts(timestamp)),
..versioned_opts(version_id, user_defined)
}
}
async fn version_info(set_disks: &Arc<SetDisks>, bucket: &str, object: &str, version_id: &str) -> ObjectInfo {
set_disks
.get_object_info(bucket, object, &versioned_opts(version_id, HashMap::new()))
.await
.expect("version should be readable")
}
#[tokio::test]
async fn inbound_stale_tagging_keeps_newer_local_tags() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "lww-tagging-stale";
let object = "object";
let version_id = Uuid::new_v4().to_string();
make_bucket(&disk_stores, bucket).await;
seed_local_tagged_version(&set_disks, bucket, object, &version_id).await;
put_version(
&set_disks,
bucket,
object,
&version_id,
&inbound_tagging_opts(&version_id, "site=remote", T_OLD),
)
.await;
let info = version_info(&set_disks, bucket, object, &version_id).await;
assert_eq!(
info.user_tags.as_str(),
"site=local",
"older inbound tags must not overwrite newer local tags"
);
assert_eq!(
get_str(&info.user_defined, SUFFIX_TAGGING_TIMESTAMP).as_deref(),
Some(T_LOCAL),
"the winning local tagging timestamp must be preserved"
);
}
#[tokio::test]
async fn inbound_newer_tagging_overwrites_local_tags() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "lww-tagging-newer";
let object = "object";
let version_id = Uuid::new_v4().to_string();
make_bucket(&disk_stores, bucket).await;
seed_local_tagged_version(&set_disks, bucket, object, &version_id).await;
put_version(
&set_disks,
bucket,
object,
&version_id,
&inbound_tagging_opts(&version_id, "site=remote", T_NEW),
)
.await;
let info = version_info(&set_disks, bucket, object, &version_id).await;
assert_eq!(
info.user_tags.as_str(),
"site=remote",
"newer inbound tags must overwrite older local tags"
);
assert_eq!(get_str(&info.user_defined, SUFFIX_TAGGING_TIMESTAMP).as_deref(), Some(T_NEW));
}
#[tokio::test]
async fn inbound_wins_when_local_has_no_tagging_timestamp() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "lww-tagging-no-local-ts";
let object = "object";
let version_id = Uuid::new_v4().to_string();
make_bucket(&disk_stores, bucket).await;
// Pre-P1-6 data: local tags without a stored tagging timestamp.
let mut user_defined = HashMap::new();
user_defined.insert(AMZ_OBJECT_TAGGING.to_string(), "site=local".to_string());
put_version(&set_disks, bucket, object, &version_id, &versioned_opts(&version_id, user_defined)).await;
put_version(
&set_disks,
bucket,
object,
&version_id,
&inbound_tagging_opts(&version_id, "site=remote", T_OLD),
)
.await;
let info = version_info(&set_disks, bucket, object, &version_id).await;
assert_eq!(
info.user_tags.as_str(),
"site=remote",
"without a local timestamp the inbound category must win (pre-LWW data compatibility)"
);
}
#[tokio::test]
async fn categories_are_judged_independently() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "lww-category-independent";
let object = "object";
let version_id = Uuid::new_v4().to_string();
make_bucket(&disk_stores, bucket).await;
// Local: newer tags (T_LOCAL), older *cleared* retention (T_OLD) —
// timestamp key only, the shape a replicated retention clear stores.
// (An active local retention would already block the overwrite at the
// WORM gate; the LWW-reachable retention states are cleared/expired.)
let mut local = HashMap::new();
local.insert(AMZ_OBJECT_TAGGING.to_string(), "site=local".to_string());
insert_str(&mut local, SUFFIX_TAGGING_TIMESTAMP, T_LOCAL.to_string());
insert_str(&mut local, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, T_OLD.to_string());
put_version(&set_disks, bucket, object, &version_id, &versioned_opts(&version_id, local)).await;
// Inbound: older tags (T_OLD), newer retention (T_NEW).
let mut inbound = HashMap::new();
inbound.insert(AMZ_OBJECT_TAGGING.to_string(), "site=remote".to_string());
insert_str(&mut inbound, SUFFIX_TAGGING_TIMESTAMP, T_OLD.to_string());
inbound.insert(AMZ_OBJECT_LOCK_MODE_LOWER.to_string(), "COMPLIANCE".to_string());
inbound.insert(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER.to_string(), "2028-01-01T00:00:00Z".to_string());
insert_str(&mut inbound, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, T_NEW.to_string());
let opts = ObjectOptions {
replication_request: true,
replication_tagging_timestamp: Some(parse_ts(T_OLD)),
replication_retention_timestamp: Some(parse_ts(T_NEW)),
..versioned_opts(&version_id, inbound)
};
put_version(&set_disks, bucket, object, &version_id, &opts).await;
let info = version_info(&set_disks, bucket, object, &version_id).await;
assert_eq!(info.user_tags.as_str(), "site=local", "the stale tagging category must keep local values");
assert_eq!(
info.user_defined.get(AMZ_OBJECT_LOCK_MODE_LOWER).map(String::as_str),
Some("COMPLIANCE"),
"the newer retention category must be applied in the same write"
);
assert_eq!(get_str(&info.user_defined, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP).as_deref(), Some(T_NEW));
}
#[tokio::test]
async fn inbound_stale_legal_hold_keeps_local_value() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "lww-legalhold-stale";
let object = "object";
let version_id = Uuid::new_v4().to_string();
make_bucket(&disk_stores, bucket).await;
// Local: legal hold released (OFF) at T_LOCAL. (A local hold that is
// still ON already blocks the overwrite at the WORM gate; the
// LWW-reachable divergence is a stale inbound ON resurrecting a hold
// that was released more recently on this site.)
let mut local = HashMap::new();
local.insert(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER.to_string(), "OFF".to_string());
insert_str(&mut local, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, T_LOCAL.to_string());
put_version(&set_disks, bucket, object, &version_id, &versioned_opts(&version_id, local)).await;
let mut inbound = HashMap::new();
inbound.insert(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER.to_string(), "ON".to_string());
insert_str(&mut inbound, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, T_OLD.to_string());
let opts = ObjectOptions {
replication_request: true,
replication_legalhold_timestamp: Some(parse_ts(T_OLD)),
..versioned_opts(&version_id, inbound)
};
put_version(&set_disks, bucket, object, &version_id, &opts).await;
let info = version_info(&set_disks, bucket, object, &version_id).await;
assert_eq!(
info.user_defined.get(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER).map(String::as_str),
Some("OFF"),
"a stale inbound legal hold must not resurrect a hold released more recently"
);
assert_eq!(
get_str(&info.user_defined, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP).as_deref(),
Some(T_LOCAL)
);
}
/// Dual-key invariant under LWW: a MinIO-written destination version may
/// carry only the x-minio-internal timestamp key; when the local category
/// wins, the restored map must still hold BOTH compatibility keys.
#[test]
fn local_win_restores_both_internal_timestamp_keys_for_minio_only_metadata() {
let mut inbound = HashMap::new();
inbound.insert(AMZ_OBJECT_TAGGING.to_string(), "site=remote".to_string());
insert_str(&mut inbound, SUFFIX_TAGGING_TIMESTAMP, T_OLD.to_string());
let existing = HashMap::from([
(AMZ_OBJECT_TAGGING.to_string(), "site=local".to_string()),
("X-Minio-Internal-Tagging-Timestamp".to_string(), T_LOCAL.to_string()),
]);
let opts = ObjectOptions {
replication_request: true,
replication_tagging_timestamp: Some(parse_ts(T_OLD)),
..Default::default()
};
assert!(merge_replication_metadata_lww(&mut inbound, &existing, &opts));
assert_eq!(inbound.get(AMZ_OBJECT_TAGGING).map(String::as_str), Some("site=local"));
assert_eq!(
inbound.get("x-rustfs-internal-tagging-timestamp").map(String::as_str),
Some(T_LOCAL),
"the RustFS twin key must be materialized even when the source version only had the MinIO key"
);
assert_eq!(inbound.get("x-minio-internal-tagging-timestamp").map(String::as_str), Some(T_LOCAL));
}
/// When the inbound category wins, the stored timestamp must be the
/// source-authored one: the PUT path's eval_metadata stamps the
/// object-lock timestamps with the receiver's clock
/// (`parse_object_lock_retention`), which would otherwise make this
/// replica's clock the LWW authority and wedge later convergence.
#[tokio::test]
async fn inbound_win_pins_stored_timestamp_to_source_authored_value() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "lww-retention-ts-pinned";
let object = "object";
let version_id = Uuid::new_v4().to_string();
make_bucket(&disk_stores, bucket).await;
// Local cleared retention at T_OLD.
let mut local = HashMap::new();
insert_str(&mut local, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, T_OLD.to_string());
put_version(&set_disks, bucket, object, &version_id, &versioned_opts(&version_id, local)).await;
// Inbound newer retention: the source authored T_LOCAL, but the PUT
// path's eval_metadata stomped the metadata key with receiver-now
// (simulated by T_NEW here).
let mut inbound = HashMap::new();
inbound.insert(AMZ_OBJECT_LOCK_MODE_LOWER.to_string(), "GOVERNANCE".to_string());
inbound.insert(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER.to_string(), "2028-01-01T00:00:00Z".to_string());
insert_str(&mut inbound, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, T_NEW.to_string());
let opts = ObjectOptions {
replication_request: true,
replication_retention_timestamp: Some(parse_ts(T_LOCAL)),
..versioned_opts(&version_id, inbound)
};
put_version(&set_disks, bucket, object, &version_id, &opts).await;
let info = version_info(&set_disks, bucket, object, &version_id).await;
assert_eq!(
get_str(&info.user_defined, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP).as_deref(),
Some(T_LOCAL),
"the stored category timestamp must be the source-authored time, not the receiver's clock"
);
assert_eq!(info.user_defined.get(AMZ_OBJECT_LOCK_MODE_LOWER).map(String::as_str), Some("GOVERNANCE"));
}
#[tokio::test]
async fn newer_local_tag_deletion_survives_stale_inbound_tags() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "lww-tagging-deleted";
let object = "object";
let version_id = Uuid::new_v4().to_string();
make_bucket(&disk_stores, bucket).await;
// Local DeleteObjectTagging state: no tags, but a newer tagging timestamp.
let mut local = HashMap::new();
insert_str(&mut local, SUFFIX_TAGGING_TIMESTAMP, T_LOCAL.to_string());
put_version(&set_disks, bucket, object, &version_id, &versioned_opts(&version_id, local)).await;
put_version(
&set_disks,
bucket,
object,
&version_id,
&inbound_tagging_opts(&version_id, "site=remote", T_OLD),
)
.await;
let info = version_info(&set_disks, bucket, object, &version_id).await;
assert!(
info.user_tags.is_empty(),
"a newer local tag deletion must not be resurrected by older inbound tags"
);
assert_eq!(get_str(&info.user_defined, SUFFIX_TAGGING_TIMESTAMP).as_deref(), Some(T_LOCAL));
}
}
#[cfg(test)]
mod inline_put_commit_path_tests {
use super::hermetic_set_disks_support::hermetic_set_disks_isolated as hermetic_set_disks;
+1 -1
View File
@@ -69,7 +69,7 @@ uuid = { workspace = true, features = ["v4", "serde", "fast-rng", "macro-diagnos
[dev-dependencies]
metrics-util = { workspace = true, features = ["debugging"] }
proptest = "1"
serial_test.workspace = true
serial_test = { workspace = true }
temp-env.workspace = true
tokio = { workspace = true, features = ["macros", "fs", "rt-multi-thread"] }
-45
View File
@@ -1564,7 +1564,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn abort_incomplete_multipart_upload_due_accepts_zero_days() {
let initiated = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
let lc = BucketLifecycleConfiguration {
@@ -1625,7 +1624,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn predict_expiration_selects_closest_expiry_for_put_object() {
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
let lc = BucketLifecycleConfiguration {
@@ -1872,7 +1870,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn empty_transition_vectors_are_not_active_or_due() {
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
@@ -1938,7 +1935,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn eval_inner_keeps_latest_object_before_days_due() {
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
let lc = BucketLifecycleConfiguration {
@@ -1972,7 +1968,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn eval_inner_transitions_latest_object_after_days_due() {
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
let lc = BucketLifecycleConfiguration {
@@ -2010,7 +2005,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn eval_inner_transitions_latest_object_after_date_due() {
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
let transition_date = base_time - Duration::days(1);
@@ -2050,7 +2044,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn eval_inner_selects_earliest_due_among_multiple_past_due_events() {
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
// Two enabled rules both yield a past-due DeleteAction and a third yields a
@@ -2164,7 +2157,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn eval_inner_expires_noncurrent_version_after_due() {
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
let lc = BucketLifecycleConfiguration {
@@ -2202,7 +2194,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn eval_inner_skips_noncurrent_expiration_without_successor() {
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).expect("valid fixed test timestamp");
let lc = BucketLifecycleConfiguration {
@@ -2238,7 +2229,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn eval_inner_missing_successor_does_not_skip_noncurrent_transition() {
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).expect("valid fixed test timestamp");
let lc = BucketLifecycleConfiguration {
@@ -2281,7 +2271,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn eval_inner_noncurrent_expiration_one_day_respects_due_boundary() {
let successor_time = datetime!(2025-06-15 12:00:00 UTC);
let due = expected_expiry_time(successor_time, 1);
@@ -2323,7 +2312,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn eval_inner_expires_noncurrent_version_immediately_when_zero_days() {
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
let lc = BucketLifecycleConfiguration {
@@ -2361,7 +2349,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn eval_inner_transitions_noncurrent_version_after_due() {
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
let lc = BucketLifecycleConfiguration {
@@ -2437,7 +2424,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn evaluator_honors_newer_noncurrent_versions_retention_count() {
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
let lc = Arc::new(BucketLifecycleConfiguration {
@@ -2726,7 +2712,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn expired_object_delete_marker_ignores_marker_with_noncurrent_versions_present() {
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
let lc = BucketLifecycleConfiguration {
@@ -2803,7 +2788,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn expired_object_delete_marker_deletes_only_delete_marker_immediately() {
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
let lc = BucketLifecycleConfiguration {
@@ -2881,7 +2865,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn expiration_days_deletes_only_expired_delete_marker_when_due() {
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
let lc = BucketLifecycleConfiguration {
@@ -2932,7 +2915,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn expiration_days_uses_earliest_due_rule_for_expired_delete_marker() {
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
let make_rule = |id: &str, days| LifecycleRule {
@@ -3263,7 +3245,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn del_marker_expiration_deletes_marker_and_older_versions_when_due() {
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).expect("fixed timestamp should be valid");
let lc = BucketLifecycleConfiguration {
@@ -3303,7 +3284,6 @@ mod tests {
// --- TASK-003 tests: Round up to next UTC processing boundary ---
#[test]
#[serial]
fn expected_expiry_time_rounds_up_to_next_midnight_utc() {
with_default_ilm_process_time(|| {
// Object created at 2025-01-15T10:30:45Z, expire in 30 days
@@ -3319,7 +3299,6 @@ mod tests {
}
#[test]
#[serial]
fn expected_expiry_time_immediate_expiry_returns_epoch() {
with_default_ilm_process_time(|| {
let mod_time = datetime!(2025-06-01 12:00:00 UTC);
@@ -3329,7 +3308,6 @@ mod tests {
}
#[test]
#[serial]
fn expected_expiry_time_preserves_exact_midnight_boundary() {
with_default_ilm_process_time(|| {
let mod_time = datetime!(2025-03-01 00:00:00 UTC);
@@ -3339,7 +3317,6 @@ mod tests {
}
#[test]
#[serial]
fn expected_expiry_time_rounds_end_of_day_to_following_midnight() {
with_default_ilm_process_time(|| {
let mod_time = datetime!(2025-06-15 23:59:59 UTC);
@@ -3349,7 +3326,6 @@ mod tests {
}
#[test]
#[serial]
fn expected_expiry_time_uses_canonical_process_time_boundary() {
let mod_time = datetime!(2025-01-15 10:30:45 UTC);
@@ -3362,7 +3338,6 @@ mod tests {
}
#[test]
#[serial]
fn expected_expiry_time_uses_deprecated_process_time_alias() {
let mod_time = datetime!(2025-01-15 10:30:45 UTC);
@@ -3375,7 +3350,6 @@ mod tests {
}
#[test]
#[serial]
fn expected_expiry_time_uses_default_boundary_when_process_time_is_zero_or_invalid() {
let mod_time = datetime!(2025-01-15 10:30:45 UTC);
@@ -3398,7 +3372,6 @@ mod tests {
// (a) Default path (env unset) is byte-identical: one day == 86400s.
#[test]
#[serial]
fn ilm_day_secs_defaults_to_86400_when_unset() {
temp_env::with_var_unset(ENV_ILM_DEBUG_DAY_SECS, || {
assert_eq!(ilm_day_secs(), DEFAULT_ILM_DAY_SECS);
@@ -3427,7 +3400,6 @@ mod tests {
// (b) End-to-end env read scales the day length.
#[test]
#[serial]
fn ilm_day_secs_scales_when_env_set() {
temp_env::with_var(ENV_ILM_DEBUG_DAY_SECS, Some("2"), || {
assert_eq!(ilm_day_secs(), 2);
@@ -3436,7 +3408,6 @@ mod tests {
// (c) Invalid env value falls back to 86400.
#[test]
#[serial]
fn ilm_day_secs_falls_back_on_invalid_env() {
temp_env::with_var(ENV_ILM_DEBUG_DAY_SECS, Some("bogus"), || {
assert_eq!(ilm_day_secs(), DEFAULT_ILM_DAY_SECS);
@@ -3449,7 +3420,6 @@ mod tests {
// Deadline math scales: with a 1s day and PROCESS_TIME unset, a Days=1 rule is
// due 1s after mod_time (rounded up to the next 1s boundary => same instant).
#[test]
#[serial]
fn expected_expiry_time_scales_with_debug_day_secs() {
let mod_time = datetime!(2025-01-15 10:30:45 UTC);
temp_env::with_var(ENV_ILM_DEBUG_DAY_SECS, Some("1"), || {
@@ -3465,7 +3435,6 @@ mod tests {
// days == 0 still yields the immediate-expiry sentinel regardless of the switch.
#[test]
#[serial]
fn expected_expiry_time_zero_days_ignores_debug_day_secs() {
let mod_time = datetime!(2025-06-01 12:00:00 UTC);
temp_env::with_var(ENV_ILM_DEBUG_DAY_SECS, Some("2"), || {
@@ -3476,7 +3445,6 @@ mod tests {
// (③) Interaction with an explicit RUSTFS_ILM_PROCESS_TIME: the deadline offset
// uses the accelerated day length, but the rounding boundary honors PROCESS_TIME.
#[test]
#[serial]
fn expected_expiry_time_debug_day_secs_respects_explicit_process_time() {
let mod_time = datetime!(2025-01-15 10:30:00 UTC);
// day == 10s, but round up to the next 60s (PROCESS_TIME) boundary.
@@ -3493,7 +3461,6 @@ mod tests {
// (③) With the switch unset, an explicit PROCESS_TIME behaves exactly as before.
#[test]
#[serial]
fn expected_expiry_time_unset_debug_day_secs_matches_legacy_process_time() {
let mod_time = datetime!(2025-01-15 10:30:45 UTC);
temp_env::with_var_unset(ENV_ILM_DEBUG_DAY_SECS, || {
@@ -3521,7 +3488,6 @@ mod tests {
// The abort-incomplete-multipart deadline path also scales through the switch.
#[test]
#[serial]
fn abort_incomplete_multipart_due_scales_with_debug_day_secs() {
use s3s::dto::AbortIncompleteMultipartUpload;
let initiated = datetime!(2025-01-15 10:30:45 UTC);
@@ -3566,7 +3532,6 @@ mod tests {
// (⑤ evaluator seam) A Days=1 rule fires under RUSTFS_ILM_DEBUG_DAY_SECS=1 once
// `now` advances a few seconds past a mod_time only ~seconds in the past.
#[test]
#[serial]
fn eval_inner_expires_days_one_rule_under_debug_day_secs() {
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
@@ -3615,7 +3580,6 @@ mod tests {
// Absolute Date-based rules must NOT scale with the switch (regression guard).
#[test]
#[serial]
fn eval_inner_date_rule_ignores_debug_day_secs() {
let expiry_date = datetime!(2025-06-01 00:00:00 UTC);
let lc = BucketLifecycleConfiguration {
@@ -3873,7 +3837,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn eval_inner_triggers_delete_all_versions_when_expired_object_all_versions_set() {
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
let lc = BucketLifecycleConfiguration {
@@ -3912,7 +3875,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn expired_object_all_versions_does_not_apply_to_current_delete_marker() {
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).expect("fixed timestamp should be valid");
let lc = BucketLifecycleConfiguration {
@@ -3942,7 +3904,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn eval_inner_uses_delete_action_when_all_versions_not_set() {
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
let lc = BucketLifecycleConfiguration {
@@ -4061,7 +4022,6 @@ mod tests {
use super::*;
use proptest::prelude::*;
use s3s::dto::{NoncurrentVersionExpiration, Tag};
use serial_test::serial;
const DAY_SECS: i64 = 86400;
@@ -4292,7 +4252,6 @@ mod tests {
/// combination, and must be deterministic: the same input
/// evaluated twice yields an identical event.
#[test]
#[serial]
fn eval_inner_never_panics_and_is_deterministic(
rules in prop::collection::vec(arb_rule(), 0..4),
obj in arb_object_opts(),
@@ -4432,7 +4391,6 @@ mod tests {
/// candidate set — earliest due wins, ties prefer delete-class —
/// and must be `NoneAction` exactly when that set is empty.
#[test]
#[serial]
fn eval_inner_winner_matches_selection_oracle(
rules in prop::collection::vec(arb_selection_rule(), 0..5),
mod_off in 0i64..(2 * DAY_SECS),
@@ -4486,7 +4444,6 @@ mod tests {
/// non-decreasing in `days` (days == 0 maps to UNIX_EPOCH, below
/// any post-1970 deadline).
#[test]
#[serial]
fn expected_expiry_time_is_monotonic_in_days(
mod_off in 0i64..(3650 * DAY_SECS),
d1 in 0i32..2000,
@@ -4508,7 +4465,6 @@ mod tests {
/// to the next whole-day boundary: the result is day-aligned, not
/// before `mod_time + days`, and less than one boundary beyond it.
#[test]
#[serial]
fn expected_expiry_time_lands_on_default_day_boundary(
mod_off in 0i64..(3650 * DAY_SECS),
days in 1i32..2000,
@@ -4526,7 +4482,6 @@ mod tests {
/// to that boundary instead: aligned to it, never early, and less
/// than one boundary late.
#[test]
#[serial]
fn expected_expiry_time_lands_on_explicit_process_boundary(
mod_off in 0i64..(365 * DAY_SECS),
days in 1i32..400,
-1
View File
@@ -73,7 +73,6 @@ walkdir = { workspace = true }
[dev-dependencies]
criterion = { workspace = true, features = ["html_reports"] }
serial_test = { workspace = true }
temp-env = { workspace = true, features = ["async_closure"] }
tempfile = { workspace = true }
tokio = { workspace = true, features = ["test-util", "macros", "fs", "rt-multi-thread"] }
@@ -1484,7 +1484,6 @@ mod tests {
ENV_CAPACITY_SAMPLE_RATE, ENV_CAPACITY_STAT_TIMEOUT, ENV_CAPACITY_WRITE_FREQUENCY_THRESHOLD,
ENV_CAPACITY_WRITE_TRIGGER_DELAY,
};
use serial_test::serial;
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
@@ -1669,7 +1668,6 @@ mod tests {
}
#[test]
#[serial]
fn test_config_getter_defaults() {
for (env_var, getter, default, _, _) in config_getter_cases() {
temp_env::with_var(env_var, None::<&str>, || {
@@ -1679,7 +1677,6 @@ mod tests {
}
#[test]
#[serial]
fn test_config_getter_env_overrides() {
for (env_var, getter, _, override_value, expected) in config_getter_cases() {
temp_env::with_var(env_var, Some(override_value), || {
@@ -1689,7 +1686,6 @@ mod tests {
}
#[test]
#[serial]
fn test_zero_env_values_clamp_to_defaults() {
// A zero threshold makes small disks report 0 bytes; a zero timeout
// (with dynamic timeout off) makes every scan fail. Both must fall
@@ -1709,7 +1705,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn test_update_capacity_preserves_retrieval_metadata() {
let manager = HybridCapacityManager::from_env();
@@ -1725,7 +1720,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn test_record_write_operation() {
let manager = HybridCapacityManager::from_env();
@@ -1736,7 +1730,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn test_write_frequency_window() {
let manager = HybridCapacityManager::from_env();
@@ -1824,7 +1817,6 @@ mod tests {
}
#[test]
#[serial]
fn test_recent_write_count_ignores_future_buckets() {
let record = WriteRecord::new();
record.write_buckets[0].store(120, 3);
@@ -1838,7 +1830,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn test_needs_fast_update() {
let manager = HybridCapacityManager::from_env();
@@ -1855,7 +1846,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn test_cache_age_tracking() {
let manager = HybridCapacityManager::from_env();
@@ -1875,7 +1865,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn test_data_source_tracking() {
let manager = HybridCapacityManager::from_env();
@@ -1891,7 +1880,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn test_needs_fast_update_waits_for_write_trigger_delay() {
let manager = create_isolated_manager(HybridStrategyConfig {
scheduled_update_interval: Duration::from_secs(60),
@@ -1922,7 +1910,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn test_needs_fast_update_respects_enable_write_trigger() {
let manager = create_isolated_manager(HybridStrategyConfig {
scheduled_update_interval: Duration::from_secs(60),
@@ -1949,7 +1936,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn test_concurrent_access() {
let manager = Arc::new(HybridCapacityManager::from_env());
let mut handles = Vec::new();
@@ -1976,7 +1962,6 @@ mod tests {
// exact under heavy same-second contention or the frequency window (and the
// write-trigger decision) would undercount.
#[tokio::test(flavor = "multi_thread", worker_threads = 8)]
#[serial]
async fn test_record_write_operation_lock_free_is_exact_under_contention() {
let manager = Arc::new(HybridCapacityManager::from_env());
let mut handles = Vec::new();
@@ -2001,7 +1986,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn test_performance_overhead() {
let manager = Arc::new(HybridCapacityManager::from_env());
let start = Instant::now();
@@ -2018,7 +2002,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn test_refresh_or_join_singleflight() {
let manager = Arc::new(HybridCapacityManager::from_env());
let calls = Arc::new(AtomicUsize::new(0));
@@ -2058,7 +2041,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn test_refresh_or_join_recovers_after_leader_cancellation() {
let manager = Arc::new(HybridCapacityManager::from_env());
@@ -2087,7 +2069,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn test_refresh_or_join_cancelled_leader_unblocks_joiner() {
let manager = Arc::new(HybridCapacityManager::from_env());
@@ -2115,7 +2096,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn test_spawn_refresh_if_needed_deduplicates_background_refresh() {
let manager = Arc::new(HybridCapacityManager::from_env());
let calls = Arc::new(AtomicUsize::new(0));
@@ -2153,7 +2133,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn test_record_write_operation_with_scope_token_marks_dirty_disks() {
let manager = create_isolated_manager(HybridStrategyConfig::default());
let token = uuid::Uuid::new_v4();
@@ -2177,7 +2156,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn test_get_dirty_disks_drains_global_dirty_scope_registry() {
let manager = create_isolated_manager(HybridStrategyConfig::default());
record_global_dirty_scope(CapacityScope {
@@ -2197,7 +2175,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn test_update_capacity_recomputes_total_from_disk_cache_for_subset_refresh() {
let manager = create_isolated_manager(HybridStrategyConfig::default());
@@ -2308,7 +2285,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn test_update_capacity_degraded_full_refresh_merges_cache_and_does_not_oscillate() {
let manager = create_isolated_manager(HybridStrategyConfig::default());
@@ -2354,7 +2330,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn test_update_capacity_degraded_with_empty_per_disk_serves_merged_cache() {
let manager = create_isolated_manager(HybridStrategyConfig::default());
manager.update_capacity(full_two_disk_update(), DataSource::RealTime).await;
@@ -2384,7 +2359,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn test_update_capacity_degraded_without_complete_cache_keeps_partial_sum() {
let manager = create_isolated_manager(HybridStrategyConfig::default());
@@ -2425,7 +2399,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn test_commit_keeps_dirty_marks_recorded_after_scan_start() {
let manager = create_isolated_manager(HybridStrategyConfig::default());
let disk = scope_disk("node-a", "/tmp/disk-a");
@@ -2456,7 +2429,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn test_commit_clears_dirty_marks_recorded_before_scan_start() {
let manager = create_isolated_manager(HybridStrategyConfig::default());
let disk = scope_disk("node-a", "/tmp/disk-a");
@@ -2477,7 +2449,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn test_retain_dirty_disks_within_drops_ghost_entries() {
let manager = create_isolated_manager(HybridStrategyConfig::default());
let local = scope_disk("node-a", "/tmp/disk-a");
@@ -2496,7 +2467,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn test_spawn_refresh_recovers_from_construction_panic() {
let manager = create_isolated_manager(HybridStrategyConfig::default());
@@ -2563,7 +2533,6 @@ mod tests {
}
#[tokio::test(start_paused = true)]
#[serial]
async fn test_refresh_or_join_joiner_times_out_when_leader_wedges() {
let manager = create_isolated_manager(HybridStrategyConfig::default());
@@ -2591,7 +2560,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn test_refresh_or_join_returns_cluster_total_for_dirty_subset() {
let manager = create_isolated_manager(HybridStrategyConfig::default());
@@ -2673,7 +2641,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn test_config_from_env() {
let config = HybridStrategyConfig::from_env();
@@ -2687,7 +2654,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn test_config_from_env_with_override() {
temp_env::with_var(ENV_CAPACITY_SCHEDULED_INTERVAL, Some("600"), || {
let config = HybridStrategyConfig::from_env();
-5
View File
@@ -1069,7 +1069,6 @@ mod tests {
#[cfg(unix)]
use rustfs_config::ENV_CAPACITY_FOLLOW_SYMLINKS;
use rustfs_config::{ENV_CAPACITY_MAX_FILES_THRESHOLD, ENV_CAPACITY_SAMPLE_RATE};
use serial_test::serial;
/// Reference implementation using unbounded `u128` arithmetic, clamped to
/// `u64::MAX`, used as the source of truth for the sampling extrapolation.
@@ -1274,7 +1273,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn test_get_dir_size_async_nonexistent_directory() {
let result = get_dir_size_async(Path::new("/nonexistent/path")).await;
assert!(result.is_err());
@@ -1648,7 +1646,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn test_metadata_incomplete_aggregate_does_not_replace_disk_cache() {
use std::fs::File;
use std::io::Write;
@@ -1783,7 +1780,6 @@ mod tests {
#[cfg(unix)]
#[tokio::test]
#[serial]
async fn test_get_dir_size_async_ignores_symlink_targets_when_follow_disabled() {
use std::fs::File;
use std::io::Write;
@@ -1809,7 +1805,6 @@ mod tests {
#[cfg(unix)]
#[tokio::test]
#[serial]
async fn test_get_dir_size_async_counts_symlink_targets_when_follow_enabled() {
use std::fs::File;
use std::io::Write;
@@ -75,7 +75,7 @@ pub struct BucketReplicationBandwidthStats {
}
#[derive(Debug, Clone, Default)]
pub struct BucketReplicationStats {
pub struct BucketReplicationMetricsSnapshot {
pub bucket: String,
pub total_failed_bytes: u64,
pub total_failed_count: u64,
@@ -107,7 +107,7 @@ pub struct BucketReplicationStats {
#[derive(Debug, Clone, Default)]
pub(crate) struct BucketReplicationRuntimeStats {
pub(crate) stats: BucketReplicationStats,
pub(crate) stats: BucketReplicationMetricsSnapshot,
pub(crate) target_flows: Vec<BucketReplicationTargetFlowStats>,
}
@@ -182,7 +182,7 @@ fn push_proxy_request_result_metrics(
}
}
pub fn collect_bucket_replication_metrics(stats: &[BucketReplicationStats]) -> Vec<PrometheusMetric> {
pub fn collect_bucket_replication_metrics(stats: &[BucketReplicationMetricsSnapshot]) -> Vec<PrometheusMetric> {
if stats.is_empty() {
return Vec::new();
}
@@ -572,7 +572,7 @@ mod tests {
#[test]
fn test_collect_bucket_replication_metrics() {
let stats = vec![BucketReplicationRuntimeStats {
stats: BucketReplicationStats {
stats: BucketReplicationMetricsSnapshot {
bucket: "b1".to_string(),
total_failed_bytes: 64,
total_failed_count: 2,
@@ -876,7 +876,7 @@ mod tests {
#[test]
fn test_collect_bucket_replication_metrics_empty() {
let stats: Vec<BucketReplicationStats> = Vec::new();
let stats: Vec<BucketReplicationMetricsSnapshot> = Vec::new();
let metrics = collect_bucket_replication_metrics(&stats);
assert!(metrics.is_empty());
}
+2 -2
View File
@@ -48,7 +48,7 @@ pub(crate) use bucket_replication::{
BucketReplicationTargetFlowStats, collect_bucket_replication_backlog_metrics, collect_bucket_replication_runtime_metrics,
};
pub use bucket_replication::{
BucketReplicationBandwidthStats, BucketReplicationStats, BucketReplicationTargetStats,
BucketReplicationBandwidthStats, BucketReplicationMetricsSnapshot, BucketReplicationTargetStats,
collect_bucket_replication_bandwidth_metrics, collect_bucket_replication_metrics,
};
pub use cluster::{ClusterStats, collect_cluster_metrics};
@@ -68,8 +68,8 @@ pub(crate) use notification::collect_notification_runtime_metrics;
pub use notification::{NotificationStats, collect_notification_metrics};
pub(crate) use notification_target::{NotificationTargetRuntimeStats, collect_notification_target_runtime_metrics};
pub use notification_target::{NotificationTargetStats, collect_notification_target_metrics};
pub use replication::{ReplicationMetricsSnapshot, collect_replication_metrics};
pub(crate) use replication::{ReplicationRuntimeStats, collect_replication_runtime_metrics};
pub use replication::{ReplicationStats, collect_replication_metrics};
pub(crate) use request::{ApiRequestMetricSupport, ApiRequestStats, collect_request_metrics};
pub use resource::{ResourceStats, collect_resource_metrics};
pub(crate) use scanner::{ScannerRuntimeStats, collect_scanner_runtime_metrics};
@@ -22,7 +22,7 @@ use crate::metrics::schema::replication::*;
/// Replication statistics.
#[derive(Debug, Clone, Default)]
pub struct ReplicationStats {
pub struct ReplicationMetricsSnapshot {
/// Average number of active replication workers
pub average_active_workers: f64,
/// Average queued bytes since server start
@@ -54,13 +54,13 @@ pub struct ReplicationStats {
#[derive(Debug, Clone, Default)]
pub(crate) struct ReplicationRuntimeStats {
pub(crate) server: String,
pub(crate) stats: ReplicationStats,
pub(crate) stats: ReplicationMetricsSnapshot,
}
/// Collects replication metrics from the given stats.
///
/// Returns a vector of Prometheus metrics for replication statistics.
pub fn collect_replication_metrics(stats: &ReplicationStats) -> Vec<PrometheusMetric> {
pub fn collect_replication_metrics(stats: &ReplicationMetricsSnapshot) -> Vec<PrometheusMetric> {
vec![
PrometheusMetric::from_descriptor(&REPLICATION_AVERAGE_ACTIVE_WORKERS_MD, stats.average_active_workers),
PrometheusMetric::from_descriptor(&REPLICATION_AVERAGE_QUEUED_BYTES_MD, stats.average_queued_bytes as f64),
@@ -120,7 +120,7 @@ mod tests {
#[test]
fn test_collect_replication_metrics() {
let stats = ReplicationStats {
let stats = ReplicationMetricsSnapshot {
average_active_workers: 8.5,
average_queued_bytes: 1024 * 1024 * 40,
average_queued_count: 240,
@@ -182,7 +182,7 @@ mod tests {
#[test]
fn test_collect_replication_metrics_default() {
let stats = ReplicationStats::default();
let stats = ReplicationMetricsSnapshot::default();
let metrics = collect_replication_metrics(&stats);
assert_eq!(metrics.len(), 13);
@@ -194,7 +194,7 @@ mod tests {
#[test]
fn replication_stats_struct_literal_keeps_legacy_fields() {
let stats = ReplicationStats {
let stats = ReplicationMetricsSnapshot {
average_active_workers: 1.0,
average_queued_bytes: 2,
average_queued_count: 3,
+2 -2
View File
@@ -2811,14 +2811,14 @@ mod tests {
#[test]
fn replication_proxy_bucket_keys_detect_removed_buckets() {
let previous = repl_proxy_bucket_live_keys(&[BucketReplicationRuntimeStats {
stats: crate::metrics::BucketReplicationStats {
stats: crate::metrics::BucketReplicationMetricsSnapshot {
bucket: "photos".to_string(),
..Default::default()
},
..Default::default()
}]);
let current = repl_proxy_bucket_live_keys(&[BucketReplicationRuntimeStats {
stats: crate::metrics::BucketReplicationStats {
stats: crate::metrics::BucketReplicationMetricsSnapshot {
bucket: "logs".to_string(),
..Default::default()
},
+11 -11
View File
@@ -21,12 +21,12 @@
use crate::metrics::collectors::scanner::{ScannerActiveBucketDriveStats, ScannerBucketDriveResultStats, ScannerSourceWorkStats};
use crate::metrics::collectors::{
ApiRequestMetricSupport, ApiRequestStats, BucketReplicationBacklogStats, BucketReplicationBandwidthStats,
BucketReplicationRuntimeStats, BucketReplicationStats, BucketReplicationTargetBacklogStats, BucketReplicationTargetFlowStats,
BucketReplicationTargetStats, BucketStats, BucketUsageStats, ClusterConfigStats, ClusterHealthStats, ClusterStats,
ClusterUsageStats, CompressionClusterStats, CpuStats, DiskStats, DriveCountStats, DriveDetailedStats,
DriveRuntimeDetailedStats, ErasureSetStats, HostNetworkStats, IamStats, IlmActionTaskStats, IlmBackpressureStats,
IlmQueueTaskStats, IlmRuntimeStats, IlmStats, IlmTaskEventStats, MemoryStats, NetworkStats, ProcessStats, ProcessStatusType,
ReplicationStats, ResourceStats, ScannerRuntimeStats, ScannerStats,
BucketReplicationMetricsSnapshot, BucketReplicationRuntimeStats, BucketReplicationTargetBacklogStats,
BucketReplicationTargetFlowStats, BucketReplicationTargetStats, BucketStats, BucketUsageStats, ClusterConfigStats,
ClusterHealthStats, ClusterStats, ClusterUsageStats, CompressionClusterStats, CpuStats, DiskStats, DriveCountStats,
DriveDetailedStats, DriveRuntimeDetailedStats, ErasureSetStats, HostNetworkStats, IamStats, IlmActionTaskStats,
IlmBackpressureStats, IlmQueueTaskStats, IlmRuntimeStats, IlmStats, IlmTaskEventStats, MemoryStats, NetworkStats,
ProcessStats, ProcessStatusType, ReplicationMetricsSnapshot, ResourceStats, ScannerRuntimeStats, ScannerStats,
};
use crate::metrics::runtime_sources::{ObsIlmRuntimeSnapshot, bucket_monitor_handle, iam_metrics_snapshot, ilm_runtime_snapshot};
use crate::metrics::{
@@ -266,7 +266,7 @@ fn bucket_replication_detail_from_snapshot(stats: ObsBucketReplicationStatsSnaps
BucketReplicationRuntimeStats {
target_flows,
stats: BucketReplicationStats {
stats: BucketReplicationMetricsSnapshot {
bucket,
total_failed_bytes: stats.total_failed_bytes,
total_failed_count: stats.total_failed_count,
@@ -298,7 +298,7 @@ fn bucket_replication_detail_from_snapshot(stats: ObsBucketReplicationStatsSnaps
}
}
async fn obs_site_replication_stats() -> ReplicationStats {
async fn obs_site_replication_stats() -> ReplicationMetricsSnapshot {
let current_data_transfer_rate = obs_bucket_replication_bandwidth_stats()
.into_iter()
.flatten()
@@ -306,7 +306,7 @@ async fn obs_site_replication_stats() -> ReplicationStats {
.sum::<f64>();
let stats = obs_replication_site_stats_snapshot(current_data_transfer_rate).await;
ReplicationStats {
ReplicationMetricsSnapshot {
average_active_workers: stats.average_active_workers,
average_queued_bytes: stats.average_queued_bytes,
average_queued_count: stats.average_queued_count,
@@ -648,7 +648,7 @@ pub fn collect_bucket_replication_bandwidth_stats() -> Vec<BucketReplicationBand
}
/// Collect bucket and target level replication stats from the global replication runtime.
pub async fn collect_bucket_replication_detail_stats() -> Vec<BucketReplicationStats> {
pub async fn collect_bucket_replication_detail_stats() -> Vec<BucketReplicationMetricsSnapshot> {
obs_bucket_replication_stats_snapshot()
.await
.into_iter()
@@ -662,7 +662,7 @@ pub(crate) async fn collect_bucket_replication_stats_bundle()
}
/// Collect site-level replication stats from the global replication runtime.
pub async fn collect_replication_stats() -> ReplicationStats {
pub async fn collect_replication_stats() -> ReplicationMetricsSnapshot {
obs_site_replication_stats().await
}
+1 -2
View File
@@ -103,8 +103,7 @@ hex-simd.workspace = true
[dev-dependencies]
tracing-subscriber = { workspace = true, features = ["json", "env-filter", "time"] }
serial_test = { workspace = true }
temp-env = { workspace = true }
temp-env = { workspace = true, features = ["async_closure"] }
tempfile = { workspace = true }
uuid = { workspace = true, features = ["v4", "serde", "fast-rng", "macro-diagnostics"] }
tokio = { workspace = true, features = ["test-util", "fs", "rt-multi-thread"] }
-5
View File
@@ -599,10 +599,8 @@ impl ScannerConfigObjectDelete for ECStore {
#[cfg(test)]
mod tests {
use super::*;
use serial_test::serial;
#[tokio::test]
#[serial]
async fn runtime_tier_names_serves_cached_arc_within_ttl() {
reset_tier_name_cache_for_test();
// The tier config manager is unconfigured in unit tests, so the
@@ -616,7 +614,6 @@ mod tests {
}
#[test]
#[serial]
fn foreground_read_guard_tracks_stream_lifetime() {
reset_foreground_read_activity_for_test();
assert_eq!(current_foreground_read_activity(), 0);
@@ -630,7 +627,6 @@ mod tests {
}
#[test]
#[serial]
fn foreground_read_activity_keeps_larger_signal() {
reset_foreground_read_activity_for_test();
let _guard = ForegroundReadGuard::new();
@@ -643,7 +639,6 @@ mod tests {
}
#[test]
#[serial]
fn scanner_runtime_guard_tracks_runtime_lifetime() {
reset_scanner_runtime_instances_for_test();
assert!(!scanner_runtime_initialized());
-14
View File
@@ -868,7 +868,6 @@ mod tests {
SCANNER_CYCLE_MAX_DIRECTORIES, SCANNER_CYCLE_MAX_DURATION, SCANNER_CYCLE_MAX_OBJECTS, SCANNER_DELAY, SCANNER_IDLE_MODE,
SCANNER_SPEED, SCANNER_SUB_SYS, ScannerSpeed,
};
use serial_test::serial;
use std::collections::HashMap;
use std::time::Duration;
use temp_env::{with_var, with_var_unset};
@@ -916,7 +915,6 @@ mod tests {
}
#[test]
#[serial]
fn scanner_runtime_config_uses_persisted_values_when_env_is_unset() {
let config = server_config_with_scanner(&[
(SCANNER_SPEED, "slow"),
@@ -944,7 +942,6 @@ mod tests {
}
#[test]
#[serial]
fn scanner_runtime_config_normalizes_persisted_default_speed() {
let config = server_config_with_scanner(&[(SCANNER_SPEED, "default")]);
@@ -960,7 +957,6 @@ mod tests {
}
#[test]
#[serial]
fn scanner_runtime_config_prefers_env_over_persisted_config() {
let config = server_config_with_scanner(&[(SCANNER_SPEED, "slowest"), (SCANNER_CYCLE, "600")]);
@@ -977,7 +973,6 @@ mod tests {
}
#[test]
#[serial]
fn scanner_runtime_config_prefers_heal_bitrot_cycle_over_scanner_compat_config() {
let config = server_config_with_scanner_and_heal(&[(SCANNER_BITROT_CYCLE, "3600")], &[(HEAL_BITROT_CYCLE, "off")]);
@@ -990,7 +985,6 @@ mod tests {
}
#[test]
#[serial]
fn scanner_runtime_config_marks_scanner_bitrot_cycle_as_compat_source() {
let config = server_config_with_scanner(&[(SCANNER_BITROT_CYCLE, "3600")]);
@@ -1007,7 +1001,6 @@ mod tests {
}
#[test]
#[serial]
fn scanner_runtime_config_normalizes_persisted_default_bitrot_cycles() {
let default_cycle = DEFAULT_HEAL_BITROT_CYCLE_SECS.to_string();
for config in [
@@ -1032,7 +1025,6 @@ mod tests {
}
#[test]
#[serial]
fn scanner_runtime_config_validation_rejects_invalid_persisted_speed_with_env_override() {
let config = server_config_with_scanner(&[(SCANNER_SPEED, "warp")]);
@@ -1066,7 +1058,6 @@ mod tests {
}
#[test]
#[serial]
fn scanner_runtime_config_uses_derived_delay_for_excessive_env_override() {
let config = server_config_with_scanner(&[(SCANNER_SPEED, "slow")]);
@@ -1087,7 +1078,6 @@ mod tests {
}
#[test]
#[serial]
fn scanner_runtime_config_status_reports_value_sources() {
let config = server_config_with_scanner(&[(SCANNER_CYCLE_MAX_OBJECTS, "100"), (SCANNER_CACHE_SAVE_TIMEOUT, "5")]);
@@ -1108,7 +1098,6 @@ mod tests {
}
#[test]
#[serial]
fn applied_runtime_config_is_the_authoritative_scheduler_state() {
let config = server_config_with_scanner(&[(SCANNER_CYCLE, "321")]);
@@ -1125,7 +1114,6 @@ mod tests {
}
#[test]
#[serial]
fn scanner_runtime_config_status_reports_persisted_pacing_overrides() {
let config = server_config_with_scanner(&[("delay", "3.5"), ("max_wait", "7")]);
@@ -1147,7 +1135,6 @@ mod tests {
}
#[test]
#[serial]
fn scanner_runtime_config_status_prefers_env_pacing_overrides() {
let config = server_config_with_scanner(&[("delay", "3.5"), ("max_wait", "7")]);
@@ -1169,7 +1156,6 @@ mod tests {
}
#[test]
#[serial]
fn scanner_runtime_config_status_preserves_subsecond_max_wait() {
let config = server_config_with_scanner(&[(SCANNER_SPEED, "fast")]);
-57
View File
@@ -19,7 +19,6 @@ use crate::{
ScannerObjectInfo as ObjectInfo, ScannerObjectOptions as ObjectOptions, ScannerPutObjReader as PutObjReader,
init_bucket_metadata_sys_for_scanner_tests, init_ecstore_config_for_scanner_tests, init_local_disks_with_instance_ctx,
};
use serial_test::serial;
use std::collections::HashMap;
use std::io::Cursor;
use std::task::Poll;
@@ -362,7 +361,6 @@ fn test_initial_scanner_delay_uses_configured_start_delay() {
}
#[test]
#[serial]
fn test_initial_scanner_delay_uses_cycle_without_explicit_start_delay() {
with_var(ENV_SCANNER_CYCLE, Some("120"), || {
crate::runtime_config::refresh_scanner_runtime_config_for_tests();
@@ -409,7 +407,6 @@ fn test_initial_scanner_delay_keeps_delay_for_replication_without_buckets() {
}
#[test]
#[serial]
fn test_scanner_cycle_max_duration_uses_env() {
with_var(ENV_SCANNER_CYCLE_MAX_DURATION_SECS, Some("42"), || {
assert_eq!(scanner_cycle_max_duration(), Some(Duration::from_secs(42)));
@@ -417,7 +414,6 @@ fn test_scanner_cycle_max_duration_uses_env() {
}
#[test]
#[serial]
fn test_scanner_cycle_max_duration_default_is_disabled() {
with_var_unset(ENV_SCANNER_CYCLE_MAX_DURATION_SECS, || {
assert_eq!(scanner_cycle_max_duration(), None);
@@ -461,7 +457,6 @@ async fn test_scanner_cycle_budget_drop_cancels_child_without_elapsed() {
}
#[test]
#[serial]
fn test_scanner_cycle_budget_config_uses_work_budget_env() {
with_var(ENV_SCANNER_CYCLE_MAX_OBJECTS, Some("100"), || {
with_var(ENV_SCANNER_CYCLE_MAX_DIRECTORIES, Some("25"), || {
@@ -473,7 +468,6 @@ fn test_scanner_cycle_budget_config_uses_work_budget_env() {
}
#[test]
#[serial]
fn test_scanner_cycle_budget_config_disables_zero_work_budgets() {
with_var(ENV_SCANNER_CYCLE_MAX_OBJECTS, Some("0"), || {
with_var(ENV_SCANNER_CYCLE_MAX_DIRECTORIES, Some("0"), || {
@@ -516,7 +510,6 @@ fn test_scan_cycle_partial_source_maps_budget_reason() {
}
#[tokio::test]
#[serial]
async fn test_mark_scan_cycle_idle_clears_published_cycle_state() {
let mut cycle_info = CurrentCycle {
current: 12,
@@ -545,7 +538,6 @@ async fn test_mark_scan_cycle_idle_clears_published_cycle_state() {
}
#[tokio::test]
#[serial]
async fn scanner_cycle_metrics_guard_covers_published_first_cycle_lifetime() {
let cycle_started = Utc::now() - chrono::Duration::seconds(5);
let mut cycle_info = CurrentCycle {
@@ -572,7 +564,6 @@ async fn scanner_cycle_metrics_guard_covers_published_first_cycle_lifetime() {
}
#[tokio::test]
#[serial]
async fn scanner_cycle_metrics_guard_keeps_active_cycle_published_during_finalization() {
let mut cycle_info = CurrentCycle {
current: 12,
@@ -597,7 +588,6 @@ async fn scanner_cycle_metrics_guard_keeps_active_cycle_published_during_finaliz
}
#[tokio::test]
#[serial]
async fn scanner_cycle_metrics_guard_drop_clears_activity() {
let guard = ScannerCycleMetricsGuard::new(CurrentCycle {
current: 12,
@@ -615,7 +605,6 @@ async fn scanner_cycle_metrics_guard_drop_clears_activity() {
}
#[tokio::test]
#[serial]
async fn run_data_scanner_cycle_publishes_activity_for_owner_lifetime() {
let (_temp_dir, store) = setup_scanner_cycle_store().await;
let ctx = CancellationToken::new();
@@ -666,7 +655,6 @@ async fn run_data_scanner_cycle_publishes_activity_for_owner_lifetime() {
}
#[tokio::test]
#[serial]
async fn test_finalize_partial_scan_cycle_advances_and_persists_counter() {
let store = Arc::new(MemoryConfigStore::default());
let ctx = CancellationToken::new();
@@ -702,7 +690,6 @@ async fn test_finalize_partial_scan_cycle_advances_and_persists_counter() {
}
#[tokio::test]
#[serial]
async fn scanner_cycle_recovers_to_newer_durable_cache_floor() {
let store = Arc::new(MemoryConfigStore::default());
let ctx = CancellationToken::new();
@@ -742,7 +729,6 @@ async fn scanner_cycle_recovers_to_newer_durable_cache_floor() {
}
#[tokio::test]
#[serial]
async fn scanner_cycle_rejects_invalid_cache_floor() {
let store = Arc::new(MemoryConfigStore::default());
let ctx = CancellationToken::new();
@@ -1013,7 +999,6 @@ async fn scanner_usage_floor_fails_closed_on_corrupt_or_exhausted_usage_state()
}
#[tokio::test]
#[serial]
async fn scanner_usage_backup_uses_durable_cycle_cadence_across_tasks() {
let store = Arc::new(MemoryConfigStore::default());
let ctx = CancellationToken::new();
@@ -1087,7 +1072,6 @@ fn scanner_cycle_advance_fails_before_reserved_exhausted_value() {
}
#[tokio::test]
#[serial]
async fn test_finalize_partial_scan_cycle_reports_persist_failure() {
let store = Arc::new(MemoryConfigStore::default());
let ctx = CancellationToken::new();
@@ -1111,7 +1095,6 @@ async fn test_finalize_partial_scan_cycle_reports_persist_failure() {
}
#[tokio::test]
#[serial]
async fn test_persist_scanner_cycle_state_reconciles_newer_winner() {
let store = Arc::new(MemoryConfigStore::default());
let ctx = CancellationToken::new();
@@ -1534,7 +1517,6 @@ async fn test_usage_save_route_barrier_prevents_missing_snapshot_creation() {
}
#[tokio::test]
#[serial]
async fn test_usage_route_barrier_precedes_durable_reconciliation() {
let store = Arc::new(MemoryConfigStore::default());
let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str());
@@ -1562,7 +1544,6 @@ async fn test_usage_route_barrier_precedes_durable_reconciliation() {
}
#[tokio::test]
#[serial]
async fn test_deferred_usage_save_keeps_last_real_save_metric() {
let metrics = global_metrics();
metrics.record_scanner_usage_save_result(ScannerUsageSaveResult::Success);
@@ -2626,7 +2607,6 @@ fn scanner_cycle_cache_floor_stays_pending_during_deferred_usage_publication() {
}
#[test]
#[serial]
fn finalizing_a_saved_cycle_acknowledges_its_exact_dirty_snapshot() {
crate::scanner_io::clear_dirty_usage_bucket("photos");
crate::scanner_io::record_dirty_usage_bucket("photos");
@@ -2653,7 +2633,6 @@ fn finalizing_a_saved_cycle_acknowledges_its_exact_dirty_snapshot() {
}
#[test]
#[serial]
fn finalizing_a_deferred_usage_save_keeps_dirty_work_pending() {
crate::scanner_io::clear_dirty_usage_bucket("photos");
crate::scanner_io::record_dirty_usage_bucket("photos");
@@ -2696,7 +2675,6 @@ async fn scanner_cycle_keeps_remote_pending_acknowledgement() {
}
#[test]
#[serial]
fn finalizing_an_already_durable_cycle_acknowledges_its_exact_dirty_snapshot() {
crate::scanner_io::clear_dirty_usage_bucket("photos");
crate::scanner_io::record_dirty_usage_bucket("photos");
@@ -2711,7 +2689,6 @@ fn finalizing_an_already_durable_cycle_acknowledges_its_exact_dirty_snapshot() {
}
#[test]
#[serial]
fn finalizing_a_prior_same_cycle_snapshot_keeps_new_dirty_work_pending() {
crate::scanner_io::clear_dirty_usage_bucket("photos");
crate::scanner_io::record_dirty_usage_bucket("photos");
@@ -2727,7 +2704,6 @@ fn finalizing_a_prior_same_cycle_snapshot_keeps_new_dirty_work_pending() {
}
#[test]
#[serial]
fn finalizing_a_durable_superseded_snapshot_keeps_dirty_work_pending() {
crate::scanner_io::clear_dirty_usage_bucket("photos");
crate::scanner_io::record_dirty_usage_bucket("photos");
@@ -2743,7 +2719,6 @@ fn finalizing_a_durable_superseded_snapshot_keeps_dirty_work_pending() {
}
#[test]
#[serial]
fn data_usage_persist_wait_covers_cache_retries_and_backup() {
with_var(rustfs_config::ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS, Some("7"), || {
crate::runtime_config::refresh_scanner_runtime_config_for_tests();
@@ -2796,7 +2771,6 @@ async fn maintenance_feature_inspection_preserves_base_cycle_after_timeout() {
}
#[tokio::test(start_paused = true)]
#[serial]
async fn stable_maintenance_detection_preserves_base_cycle_after_timeout() {
let ctx = CancellationToken::new();
@@ -2862,7 +2836,6 @@ async fn maintenance_feature_inspection_stops_on_cancellation() {
}
#[test]
#[serial]
fn test_cycle_interval_prefers_explicit_cycle_override() {
with_var(ENV_SCANNER_SPEED, Some("slowest"), || {
with_var(ENV_SCANNER_CYCLE, Some("42"), || {
@@ -2872,7 +2845,6 @@ fn test_cycle_interval_prefers_explicit_cycle_override() {
}
#[test]
#[serial]
fn test_cycle_interval_prefers_explicit_cycle_over_default_cycle() {
let _guard = ScannerDefaultCycleGuard::set(TEST_DEFAULT_SCANNER_CYCLE_SECS);
@@ -2882,7 +2854,6 @@ fn test_cycle_interval_prefers_explicit_cycle_over_default_cycle() {
}
#[test]
#[serial]
fn test_cycle_interval_uses_scanner_default_speed_override_when_unconfigured() {
let _guard = ScannerDefaultSpeedGuard::set(ScannerSpeed::Slowest);
@@ -2892,7 +2863,6 @@ fn test_cycle_interval_uses_scanner_default_speed_override_when_unconfigured() {
}
#[test]
#[serial]
fn test_cycle_interval_prefers_explicit_speed_over_default_speed_override() {
let _guard = ScannerDefaultSpeedGuard::set(ScannerSpeed::Slowest);
@@ -2910,7 +2880,6 @@ fn test_cycle_interval_prefers_explicit_speed_over_default_speed_override() {
}
#[test]
#[serial]
fn test_cycle_interval_uses_default_cycle_override_when_unconfigured() {
let _guard = ScannerDefaultCycleGuard::set(TEST_DEFAULT_SCANNER_CYCLE_SECS);
@@ -3074,7 +3043,6 @@ fn scanner_cycle_wait_plan_drives_growth_resets_and_bitrot_cap() {
}
#[test]
#[serial]
fn scanner_cycle_schedule_status_reports_effective_backoff() {
record_scanner_cycle_schedule(Duration::from_millis(86_400_001), true, 2_048, true, 7);
@@ -3354,7 +3322,6 @@ fn dirty_usage_wakes_are_disabled_for_explicit_cycle_policy() {
}
#[test]
#[serial]
fn clean_idle_cap_preserves_default_bitrot_coverage_window() {
let config = ScannerRuntimeConfig {
bitrot_cycle: Some(Duration::from_secs(30 * 24 * 60 * 60)),
@@ -3384,7 +3351,6 @@ fn clean_idle_cap_allows_policy_max_when_bitrot_is_disabled() {
}
#[test]
#[serial]
fn clean_idle_cap_never_shortens_the_base_cycle() {
let config = ScannerRuntimeConfig {
bitrot_cycle: Some(Duration::from_secs(60)),
@@ -3398,7 +3364,6 @@ fn clean_idle_cap_never_shortens_the_base_cycle() {
}
#[test]
#[serial]
fn test_cycle_interval_keeps_default_cycle_with_explicit_speed() {
let _guard = ScannerDefaultCycleGuard::set(TEST_DEFAULT_SCANNER_CYCLE_SECS);
@@ -3416,7 +3381,6 @@ fn test_cycle_interval_keeps_default_cycle_with_explicit_speed() {
}
#[test]
#[serial]
fn test_cycle_interval_prefers_explicit_start_delay_over_default_cycle() {
let _guard = ScannerDefaultCycleGuard::set(TEST_DEFAULT_SCANNER_CYCLE_SECS);
@@ -3430,7 +3394,6 @@ fn test_cycle_interval_prefers_explicit_start_delay_over_default_cycle() {
}
#[test]
#[serial]
fn test_cycle_interval_supports_minio_speed_alias() {
with_var_unset(ENV_SCANNER_SPEED, || {
with_var_unset(ENV_SCANNER_CYCLE, || {
@@ -3444,7 +3407,6 @@ fn test_cycle_interval_supports_minio_speed_alias() {
}
#[test]
#[serial]
fn test_cycle_interval_supports_minio_cycle_alias() {
with_var_unset(ENV_SCANNER_CYCLE, || {
with_var_unset(ENV_SCANNER_START_DELAY_SECS, || {
@@ -3464,7 +3426,6 @@ fn test_randomized_cycle_delay_handles_small_start_delay() {
}
#[tokio::test]
#[serial]
async fn test_wait_for_next_scanner_cycle_wakes_for_dirty_usage() {
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
@@ -3490,7 +3451,6 @@ async fn test_wait_for_next_scanner_cycle_wakes_for_dirty_usage() {
}
#[tokio::test]
#[serial]
async fn test_wait_for_next_scanner_cycle_sees_unattempted_dirty_usage() {
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
let dirty_generation = crate::scanner_io::dirty_usage_generation();
@@ -3512,7 +3472,6 @@ async fn test_wait_for_next_scanner_cycle_sees_unattempted_dirty_usage() {
}
#[tokio::test(start_paused = true)]
#[serial]
async fn test_wait_for_next_scanner_cycle_retries_stable_dirty_usage_on_timer() {
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
crate::scanner_io::record_dirty_usage_bucket("photos");
@@ -3534,7 +3493,6 @@ async fn test_wait_for_next_scanner_cycle_retries_stable_dirty_usage_on_timer()
}
#[tokio::test(start_paused = true)]
#[serial]
async fn test_wait_for_next_scanner_cycle_can_defer_dirty_wakes_until_timer() {
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
let ctx = CancellationToken::new();
@@ -3553,7 +3511,6 @@ async fn test_wait_for_next_scanner_cycle_can_defer_dirty_wakes_until_timer() {
}
#[tokio::test]
#[serial]
async fn test_wait_for_next_scanner_cycle_wakes_for_repeated_dirty_bucket() {
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
crate::scanner_io::record_dirty_usage_bucket("photos");
@@ -3579,7 +3536,6 @@ async fn test_wait_for_next_scanner_cycle_wakes_for_repeated_dirty_bucket() {
}
#[tokio::test]
#[serial]
async fn test_wait_for_next_scanner_cycle_reschedules_for_runtime_config() {
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
let observed_generation = crate::runtime_config::scanner_runtime_config_generation();
@@ -3607,7 +3563,6 @@ async fn test_wait_for_next_scanner_cycle_reschedules_for_runtime_config() {
}
#[tokio::test]
#[serial]
async fn test_wait_for_next_scanner_cycle_reschedules_for_maintenance_change() {
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
let observed_generation = crate::scanner_io::scanner_maintenance_generation();
@@ -3851,7 +3806,6 @@ fn scanner_activity_after_a_cycle_restores_the_base_interval() {
}
#[tokio::test(start_paused = true)]
#[serial]
async fn distributed_clean_idle_wait_wakes_at_base_interval_for_remote_activity() {
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
let ctx = CancellationToken::new();
@@ -3879,7 +3833,6 @@ async fn distributed_clean_idle_wait_wakes_at_base_interval_for_remote_activity(
}
#[tokio::test(start_paused = true)]
#[serial]
async fn superseded_retry_wait_defers_dirty_cluster_activity_until_timer() {
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
let ctx = CancellationToken::new();
@@ -3907,7 +3860,6 @@ async fn superseded_retry_wait_defers_dirty_cluster_activity_until_timer() {
}
#[tokio::test(start_paused = true)]
#[serial]
async fn distributed_clean_idle_wait_blocks_backoff_for_unpropagated_maintenance() {
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
let ctx = CancellationToken::new();
@@ -3934,7 +3886,6 @@ async fn distributed_clean_idle_wait_blocks_backoff_for_unpropagated_maintenance
}
#[tokio::test(start_paused = true)]
#[serial]
async fn distributed_clean_idle_wait_fails_closed_when_a_peer_is_unverifiable() {
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
let ctx = CancellationToken::new();
@@ -3961,7 +3912,6 @@ async fn distributed_clean_idle_wait_fails_closed_when_a_peer_is_unverifiable()
}
#[tokio::test(start_paused = true)]
#[serial]
async fn distributed_clean_idle_wait_keeps_the_extended_deadline_when_peers_are_clean() {
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
let ctx = CancellationToken::new();
@@ -3989,7 +3939,6 @@ async fn distributed_clean_idle_wait_keeps_the_extended_deadline_when_peers_are_
}
#[tokio::test(start_paused = true)]
#[serial]
async fn scanner_activity_probe_wait_is_cancellation_aware() {
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
let ctx = CancellationToken::new();
@@ -4020,7 +3969,6 @@ async fn scanner_activity_probe_wait_is_cancellation_aware() {
}
#[tokio::test(start_paused = true)]
#[serial]
async fn scanner_activity_probe_wait_stops_after_leader_lock_loss() {
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
let ctx = CancellationToken::new();
@@ -4052,7 +4000,6 @@ async fn scanner_activity_probe_wait_stops_after_leader_lock_loss() {
}
#[test]
#[serial]
fn test_get_cycle_scan_mode_runs_deep_until_selection_window_completes() {
with_var(ENV_SCANNER_BITROT_CYCLE_SECS, Some("3600"), || {
let mode = get_cycle_scan_mode(10, 0, Some(Utc::now()), bitrot_scan_cycle());
@@ -4061,7 +4008,6 @@ fn test_get_cycle_scan_mode_runs_deep_until_selection_window_completes() {
}
#[test]
#[serial]
fn test_get_cycle_scan_mode_respects_elapsed_bitrot_cycle() {
with_var(ENV_SCANNER_BITROT_CYCLE_SECS, Some("3600"), || {
let recent = Utc::now() - chrono::Duration::minutes(30);
@@ -4073,7 +4019,6 @@ fn test_get_cycle_scan_mode_respects_elapsed_bitrot_cycle() {
}
#[test]
#[serial]
fn test_get_cycle_scan_mode_can_disable_periodic_deep_scan() {
with_var(ENV_SCANNER_BITROT_CYCLE_SECS, Some("off"), || {
assert_eq!(get_cycle_scan_mode(1, 0, None, bitrot_scan_cycle()), HealScanMode::Normal);
@@ -4081,7 +4026,6 @@ fn test_get_cycle_scan_mode_can_disable_periodic_deep_scan() {
}
#[test]
#[serial]
fn test_background_heal_info_for_scan_start_marks_deep_active() {
let now = Utc::now();
let info =
@@ -4094,7 +4038,6 @@ fn test_background_heal_info_for_scan_start_marks_deep_active() {
}
#[test]
#[serial]
fn test_background_heal_info_for_scan_start_keeps_deep_window_start() {
with_var_unset(ENV_SCANNER_BITROT_CYCLE_SECS, || {
let started_at = Utc::now();
@@ -18,7 +18,6 @@ use super::*;
use crate::storage_api::VersionPurgeStatusType;
use crate::{DiskOption, Endpoint, STORAGE_FORMAT_FILE, TierStats, new_disk, storageclass};
use rustfs_filemeta::{FileInfo, FileMeta};
use serial_test::serial;
use std::io::Write;
#[cfg(unix)]
use std::os::unix::fs::{PermissionsExt, symlink};
@@ -356,7 +355,6 @@ impl Drop for TestGuard {
}
#[tokio::test]
#[serial]
async fn test_should_skip_failed_respects_ttl() {
let (mut scanner, temp_dir) = build_test_scanner().await;
let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir);
@@ -378,7 +376,6 @@ async fn test_should_skip_failed_respects_ttl() {
}
#[tokio::test]
#[serial]
async fn test_record_failed_ttl_zero_noop() {
let (mut scanner, temp_dir) = build_test_scanner().await;
let _guard = TestGuard::new(0, 100, &mut scanner, temp_dir);
@@ -467,7 +464,6 @@ fn test_should_account_replication_stats_only_for_live_object_versions() {
}
#[tokio::test]
#[serial]
async fn test_heal_replication_only_queues_pending_null_deletes() {
async fn replication_skipped_count() -> u64 {
global_metrics()
@@ -716,7 +712,6 @@ async fn test_scanner_heal_admission_accounting_maps_deep_scan_to_bitrot() {
}
#[test]
#[serial]
fn test_excessive_version_alert_thresholds_use_env() {
with_var(rustfs_config::ENV_SCANNER_ALERT_EXCESS_VERSIONS, Some("3"), || {
with_var(rustfs_config::ENV_SCANNER_ALERT_EXCESS_VERSION_SIZE, Some("100"), || {
@@ -731,7 +726,6 @@ fn test_excessive_version_alert_thresholds_use_env() {
}
#[test]
#[serial]
fn test_excessive_folders_threshold_uses_env() {
with_var(rustfs_config::ENV_SCANNER_ALERT_EXCESS_FOLDERS, Some("3"), || {
crate::runtime_config::refresh_scanner_runtime_config_for_tests();
@@ -741,7 +735,6 @@ fn test_excessive_folders_threshold_uses_env() {
}
#[test]
#[serial]
fn test_excessive_folders_threshold_default_supports_pbs_layout() {
with_var_unset(rustfs_config::ENV_SCANNER_ALERT_EXCESS_FOLDERS, || {
crate::runtime_config::refresh_scanner_runtime_config_for_tests();
@@ -751,7 +744,6 @@ fn test_excessive_folders_threshold_default_supports_pbs_layout() {
}
#[test]
#[serial]
fn test_scanner_yield_every_n_objects_uses_env() {
with_var(rustfs_config::ENV_SCANNER_YIELD_EVERY_N_OBJECTS, Some("32"), || {
crate::runtime_config::refresh_scanner_runtime_config_for_tests();
@@ -761,7 +753,6 @@ fn test_scanner_yield_every_n_objects_uses_env() {
}
#[test]
#[serial]
fn test_scanner_yield_every_n_objects_uses_default() {
with_var_unset(rustfs_config::ENV_SCANNER_YIELD_EVERY_N_OBJECTS, || {
crate::runtime_config::refresh_scanner_runtime_config_for_tests();
@@ -888,7 +879,6 @@ fn test_order_folders_for_resume_reports_stale_hint() {
}
#[tokio::test]
#[serial]
async fn test_record_failed_prunes_to_max_entries() {
let (mut scanner, temp_dir) = build_test_scanner().await;
let _guard = TestGuard::new(1000, 2, &mut scanner, temp_dir);
@@ -920,7 +910,6 @@ async fn test_record_failed_prunes_to_max_entries() {
}
#[tokio::test]
#[serial]
async fn test_prune_failed_objects_cache_drops_expired() {
let (mut scanner, temp_dir) = build_test_scanner().await;
let _guard = TestGuard::new(5, 10, &mut scanner, temp_dir);
@@ -944,7 +933,6 @@ async fn test_prune_failed_objects_cache_drops_expired() {
}
#[tokio::test]
#[serial]
async fn test_prune_failed_objects_max_zero_keeps_fresh() {
let (mut scanner, temp_dir) = build_test_scanner().await;
let _guard = TestGuard::new(60, 0, &mut scanner, temp_dir);
@@ -1701,7 +1689,6 @@ async fn test_heal_actions_returns_actual_size_without_inline_heal() {
}
#[tokio::test]
#[serial]
#[cfg(unix)]
async fn test_scan_folder_skips_unreadable_child_directory() {
let (mut scanner, temp_dir) = build_test_scanner().await;
@@ -1734,7 +1721,6 @@ async fn test_scan_folder_skips_unreadable_child_directory() {
}
#[tokio::test]
#[serial]
async fn test_scan_folder_exits_when_abandoned_child_listing_finishes() {
let (mut scanner, temp_dir) = build_test_scanner().await;
let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir.clone());
@@ -1813,7 +1799,6 @@ async fn test_scan_folder_exits_when_abandoned_child_listing_finishes() {
}
#[tokio::test]
#[serial]
async fn test_scan_folder_xl_meta_named_directory_uses_namespace_descent() {
let (mut scanner, temp_dir) = build_test_scanner().await;
let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir.clone());
@@ -1859,7 +1844,6 @@ async fn test_scan_folder_xl_meta_named_directory_uses_namespace_descent() {
}
#[tokio::test(flavor = "current_thread")]
#[serial]
async fn test_scan_folder_corrupt_xl_meta_stops_erasure_data_dir_descent() {
let logs = CapturedLogs::default();
let subscriber = tracing_subscriber::fmt()
@@ -2021,7 +2005,6 @@ async fn test_scan_folder_corrupt_xl_meta_stops_erasure_data_dir_descent() {
}
#[tokio::test]
#[serial]
async fn test_scan_folder_missing_xl_meta_stops_erasure_data_dir_descent() {
let (mut scanner, temp_dir) = build_test_scanner().await;
let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir.clone());
@@ -2099,7 +2082,6 @@ async fn test_scan_folder_missing_xl_meta_stops_erasure_data_dir_descent() {
}
#[tokio::test]
#[serial]
async fn test_scan_folder_uuid_namespace_part_name_directory_is_not_data_dir() {
let (mut scanner, temp_dir) = build_test_scanner().await;
let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir.clone());
@@ -2161,7 +2143,6 @@ async fn test_scan_folder_uuid_namespace_part_name_directory_is_not_data_dir() {
}
#[tokio::test]
#[serial]
async fn test_scan_folder_non_erasure_metadata_keeps_namespace_descent() {
let (mut scanner, temp_dir) = build_test_scanner().await;
let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir.clone());
@@ -2203,7 +2184,6 @@ async fn test_scan_folder_non_erasure_metadata_keeps_namespace_descent() {
}
#[tokio::test]
#[serial]
async fn test_scan_folder_compacted_parent_sends_partial_update() {
let (mut scanner, temp_dir) = build_test_scanner().await;
let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir.clone());
@@ -2245,7 +2225,6 @@ async fn test_scan_folder_compacted_parent_sends_partial_update() {
}
#[tokio::test]
#[serial]
async fn test_scan_data_folder_cancelled_before_scan_clears_current_path() {
let (scanner, temp_dir) = build_test_scanner().await;
let _guard = TestGuard {
@@ -2290,7 +2269,6 @@ async fn test_scan_data_folder_cancelled_before_scan_clears_current_path() {
}
#[tokio::test]
#[serial]
async fn test_scan_data_folder_returns_partial_cache_on_budget_cancel() {
let (mut scanner, temp_dir) = build_test_scanner().await;
let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir.clone());
@@ -2346,7 +2324,6 @@ async fn test_scan_data_folder_returns_partial_cache_on_budget_cancel() {
}
#[tokio::test]
#[serial]
async fn test_scan_data_folder_reports_invalid_checkpoint_ignored_once() {
let (scanner, temp_dir) = build_test_scanner().await;
let _guard = TestGuard {
@@ -2391,7 +2368,6 @@ async fn test_scan_data_folder_reports_invalid_checkpoint_ignored_once() {
}
#[tokio::test]
#[serial]
async fn test_scan_data_folder_resume_hint_prioritizes_next_existing_folder() {
let (scanner, temp_dir) = build_test_scanner().await;
let _guard = TestGuard {
@@ -2465,7 +2441,6 @@ async fn test_scan_data_folder_resume_hint_prioritizes_next_existing_folder() {
}
#[tokio::test]
#[serial]
async fn scan_data_folder_missing_bucket_returns_partial() {
let (scanner, temp_dir) = build_test_scanner().await;
let _guard = TestGuard {
@@ -2517,7 +2492,6 @@ async fn scan_data_folder_missing_bucket_returns_partial() {
}
#[tokio::test]
#[serial]
async fn scan_data_folder_missing_scan_root_returns_partial() {
let (scanner, temp_dir) = build_test_scanner().await;
tokio::fs::remove_dir_all(&temp_dir)
@@ -2563,7 +2537,6 @@ async fn scan_data_folder_missing_scan_root_returns_partial() {
}
#[tokio::test]
#[serial]
async fn test_scan_data_folder_resume_hint_orders_across_new_and_existing_folders() {
let (scanner, temp_dir) = build_test_scanner().await;
let _guard = TestGuard {
@@ -2632,7 +2605,6 @@ async fn test_scan_data_folder_resume_hint_orders_across_new_and_existing_folder
}
#[tokio::test]
#[serial]
async fn test_scan_data_folder_partial_object_budget_accumulates_progress() {
let (scanner, temp_dir) = build_test_scanner().await;
let _guard = TestGuard {
@@ -2715,7 +2687,6 @@ async fn test_scan_data_folder_partial_object_budget_accumulates_progress() {
}
#[tokio::test]
#[serial]
async fn test_partial_compacted_entry_does_not_carry_children() {
let (mut scanner, temp_dir) = build_test_scanner().await;
let _guard = TestGuard {
@@ -2761,7 +2732,6 @@ async fn test_partial_compacted_entry_does_not_carry_children() {
}
#[tokio::test]
#[serial]
async fn test_partial_entry_does_not_carry_missing_old_child() {
let (mut scanner, temp_dir) = build_test_scanner().await;
let _guard = TestGuard {
@@ -2794,7 +2764,6 @@ async fn test_partial_entry_does_not_carry_missing_old_child() {
}
#[tokio::test]
#[serial]
async fn test_legacy_windows_cache_rebuilds_and_round_trips_portable_keys() {
let (scanner, temp_dir) = build_test_scanner().await;
let _guard = TestGuard {
@@ -2861,7 +2830,6 @@ async fn test_legacy_windows_cache_rebuilds_and_round_trips_portable_keys() {
}
#[tokio::test]
#[serial]
async fn test_scan_data_folder_success_clears_resume_hint() {
let (scanner, temp_dir) = build_test_scanner().await;
let _guard = TestGuard {
@@ -2904,7 +2872,6 @@ async fn test_scan_data_folder_success_clears_resume_hint() {
}
#[tokio::test]
#[serial]
async fn test_scan_data_folder_keeps_unresolved_objects_partial() {
let (scanner, temp_dir) = build_test_scanner().await;
let _guard = TestGuard {
@@ -2951,7 +2918,6 @@ async fn test_scan_data_folder_keeps_unresolved_objects_partial() {
}
#[tokio::test]
#[serial]
#[cfg(unix)]
async fn test_scan_folder_ignores_symlinked_child_directory() {
let (mut scanner, temp_dir) = build_test_scanner().await;
-27
View File
@@ -27,7 +27,6 @@ use crate::{
init_local_disks_with_instance_ctx, new_disk, path2_bucket_object_with_base_path,
};
use rustfs_filemeta::FileInfo;
use serial_test::serial;
use temp_env::with_var;
use time::OffsetDateTime;
use uuid::Uuid;
@@ -103,7 +102,6 @@ async fn setup_two_pool_scanner_store() -> (tempfile::TempDir, Arc<ECStore>) {
}
#[tokio::test]
#[serial]
async fn scanner_cache_locks_block_same_source_workers() {
let (_temp_dir, store) = setup_two_pool_scanner_store().await;
let set = &store.pools[0].disk_set[0];
@@ -130,7 +128,6 @@ async fn scanner_cache_locks_block_same_source_workers() {
}
#[tokio::test]
#[serial]
async fn scanner_cache_locks_allow_cross_source_workers() {
let (_temp_dir, store) = setup_two_pool_scanner_store().await;
let first_set = &store.pools[0].disk_set[0];
@@ -149,7 +146,6 @@ async fn scanner_cache_locks_allow_cross_source_workers() {
}
#[tokio::test]
#[serial]
async fn scanner_cycle_is_deferred_while_rebalance_is_active() {
let (_temp_dir, store) = setup_two_pool_scanner_store().await;
let mut pool_stats = vec![EcstoreRebalanceStats::default(); store.pools.len()];
@@ -185,7 +181,6 @@ async fn scanner_cycle_is_deferred_while_rebalance_is_active() {
}
#[tokio::test]
#[serial]
async fn scanner_cycle_is_deferred_while_terminal_decommission_is_blocked() {
let (_temp_dir, store) = setup_two_pool_scanner_store().await;
for decommission in [
@@ -230,7 +225,6 @@ async fn data_usage_publish_fails_when_receiver_is_closed() {
}
#[tokio::test]
#[serial]
async fn multi_pool_scanner_cycle_publishes_combined_usage() {
let (_temp_dir, store) = setup_two_pool_scanner_store().await;
let bucket = format!("scanner-union-{}", Uuid::new_v4().simple());
@@ -278,7 +272,6 @@ async fn multi_pool_scanner_cycle_publishes_combined_usage() {
}
#[tokio::test]
#[serial]
async fn multi_pool_scanner_cycle_zero_fills_bucket_absent_from_first_pool() {
let (_temp_dir, store) = setup_two_pool_scanner_store().await;
let bucket = format!("scanner-second-pool-{}", Uuid::new_v4().simple());
@@ -366,7 +359,6 @@ fn object_lock_config_enabled_accepts_enabled_only() {
}
#[test]
#[serial]
fn dirty_usage_snapshot_clear_preserves_newer_generation() {
clear_dirty_usage_buckets_for_tests();
record_dirty_usage_bucket("photos");
@@ -381,7 +373,6 @@ fn dirty_usage_snapshot_clear_preserves_newer_generation() {
}
#[test]
#[serial]
fn dirty_usage_generation_acknowledgement_preserves_newer_mutations() {
clear_dirty_usage_buckets_for_tests();
record_dirty_usage_bucket("photos");
@@ -407,7 +398,6 @@ fn dirty_usage_generation_acknowledgement_preserves_newer_mutations() {
}
#[test]
#[serial]
fn dirty_usage_generation_acknowledgement_rejects_stale_process_and_future_generation() {
clear_dirty_usage_buckets_for_tests();
record_dirty_usage_bucket("photos");
@@ -437,7 +427,6 @@ fn dirty_usage_generation_acknowledgement_rejects_stale_process_and_future_gener
}
#[test]
#[serial]
fn dirty_usage_snapshot_detects_uncovered_generation() {
clear_dirty_usage_buckets_for_tests();
record_dirty_usage_bucket("photos");
@@ -462,7 +451,6 @@ fn generation_saturates_instead_of_wrapping() {
}
#[test]
#[serial]
fn dirty_usage_snapshot_clears_a_stably_absent_bucket_after_durable_save() {
clear_dirty_usage_buckets_for_tests();
record_dirty_usage_bucket("photos");
@@ -484,7 +472,6 @@ fn dirty_usage_snapshot_clears_a_stably_absent_bucket_after_durable_save() {
}
#[test]
#[serial]
fn dirty_usage_snapshot_preserves_an_absent_bucket_recorded_after_listing_started() {
clear_dirty_usage_buckets_for_tests();
let generation_before_bucket_list = dirty_usage_generation();
@@ -499,7 +486,6 @@ fn dirty_usage_snapshot_preserves_an_absent_bucket_recorded_after_listing_starte
}
#[test]
#[serial]
fn deleting_a_clean_bucket_invalidates_an_inflight_usage_snapshot() {
clear_dirty_usage_buckets_for_tests();
let snapshot = snapshot_dirty_usage_buckets(&[bucket_info("photos")], dirty_usage_generation());
@@ -513,7 +499,6 @@ fn deleting_a_clean_bucket_invalidates_an_inflight_usage_snapshot() {
}
#[test]
#[serial]
fn deleting_a_bucket_during_listing_invalidates_the_resulting_usage_snapshot() {
clear_dirty_usage_buckets_for_tests();
let generation_before_bucket_list = dirty_usage_generation();
@@ -527,7 +512,6 @@ fn deleting_a_bucket_during_listing_invalidates_the_resulting_usage_snapshot() {
}
#[test]
#[serial]
fn scanner_maintenance_change_advances_generation_and_marks_usage_dirty() {
clear_dirty_usage_buckets_for_tests();
let generation = scanner_maintenance_generation();
@@ -540,7 +524,6 @@ fn scanner_maintenance_change_advances_generation_and_marks_usage_dirty() {
}
#[test]
#[serial]
fn dirty_usage_clear_excludes_failed_buckets() {
clear_dirty_usage_buckets_for_tests();
record_dirty_usage_bucket("photos");
@@ -572,7 +555,6 @@ fn dirty_usage_clear_plan_excludes_cache_save_failures() {
}
#[test]
#[serial]
fn dirty_usage_is_acknowledged_only_after_durable_usage_confirmation() {
clear_dirty_usage_buckets_for_tests();
record_dirty_usage_bucket("photos");
@@ -590,7 +572,6 @@ fn dirty_usage_is_acknowledged_only_after_durable_usage_confirmation() {
}
#[test]
#[serial]
fn clear_dirty_usage_bucket_removes_deleted_bucket_marker() {
clear_dirty_usage_buckets_for_tests();
record_dirty_usage_bucket("photos");
@@ -917,35 +898,30 @@ async fn bucket_cache_pending_heal_reaches_cycle_maintenance_state() {
}
#[test]
#[serial]
fn scanner_concurrency_limit_preserves_available_when_unconfigured() {
crate::reset_foreground_read_activity_for_test();
assert_eq!(scanner_concurrency_limit(0, 4), 4);
}
#[test]
#[serial]
fn scanner_concurrency_limit_caps_to_configured_value() {
crate::reset_foreground_read_activity_for_test();
assert_eq!(scanner_concurrency_limit(2, 4), 2);
}
#[test]
#[serial]
fn scanner_concurrency_limit_never_exceeds_available_work() {
crate::reset_foreground_read_activity_for_test();
assert_eq!(scanner_concurrency_limit(8, 4), 4);
}
#[test]
#[serial]
fn scanner_concurrency_limit_handles_no_available_work() {
crate::reset_foreground_read_activity_for_test();
assert_eq!(scanner_concurrency_limit(2, 0), 0);
}
#[test]
#[serial]
fn scanner_concurrency_limit_yields_to_foreground_reads() {
crate::reset_foreground_read_activity_for_test();
crate::set_foreground_read_activity(8);
@@ -955,7 +931,6 @@ fn scanner_concurrency_limit_yields_to_foreground_reads() {
}
#[test]
#[serial]
fn scanner_concurrency_limit_yields_to_streaming_reads() {
crate::reset_foreground_read_activity_for_test();
let _guard = crate::ForegroundReadGuard::new();
@@ -979,7 +954,6 @@ fn increment_atomic_usize_saturates_at_max() {
}
#[test]
#[serial]
fn scanner_max_concurrent_set_scans_uses_env_cap() {
with_var(ENV_SCANNER_MAX_CONCURRENT_SET_SCANS, Some("2"), || {
crate::runtime_config::refresh_scanner_runtime_config_for_tests();
@@ -989,7 +963,6 @@ fn scanner_max_concurrent_set_scans_uses_env_cap() {
}
#[test]
#[serial]
fn scanner_max_concurrent_disk_scans_uses_env_cap() {
with_var(ENV_SCANNER_MAX_CONCURRENT_DISK_SCANS, Some("1"), || {
crate::runtime_config::refresh_scanner_runtime_config_for_tests();
-5
View File
@@ -258,7 +258,6 @@ impl SleepTimer {
#[cfg(test)]
mod tests {
use super::*;
use serial_test::serial;
use temp_env::{with_var, with_var_unset};
struct ScannerDefaultSpeedGuard;
@@ -326,7 +325,6 @@ mod tests {
}
#[test]
#[serial]
fn test_refresh_from_env_applies_speed_and_idle_mode_for_next_cycle() {
let prev_mode = SCANNER_IDLE_MODE.load(Ordering::Relaxed);
SCANNER_IDLE_MODE.store(true, Ordering::Relaxed);
@@ -346,7 +344,6 @@ mod tests {
}
#[test]
#[serial]
fn test_refresh_from_env_uses_default_speed_override_when_speed_unset() {
let _guard = ScannerDefaultSpeedGuard::set(ScannerSpeed::Slowest);
let s = DynamicSleeper::new(ScannerSpeed::Default);
@@ -362,7 +359,6 @@ mod tests {
}
#[tokio::test(start_paused = true)]
#[serial]
async fn test_fastest_never_sleeps() {
let prev_mode = SCANNER_IDLE_MODE.load(Ordering::Relaxed);
SCANNER_IDLE_MODE.store(true, Ordering::Relaxed);
@@ -376,7 +372,6 @@ mod tests {
}
#[tokio::test(start_paused = true)]
#[serial]
async fn test_idle_mode_off_skips_sleep() {
let prev_mode = SCANNER_IDLE_MODE.load(Ordering::Relaxed);
SCANNER_IDLE_MODE.store(false, Ordering::Relaxed);
@@ -14,7 +14,6 @@
#![recursion_limit = "256"]
use futures::FutureExt;
use rustfs_config::ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT;
use rustfs_scanner::scanner_folder::ScannerItem;
use rustfs_scanner::scanner_io::ScannerIODisk;
@@ -23,10 +22,8 @@ use rustfs_scanner::{
scanner::init_data_scanner,
};
use s3s::dto::RestoreRequest;
use serial_test::serial;
use std::{
collections::HashMap,
env,
path::{Path, PathBuf},
sync::{Arc, Once, OnceLock},
time::Duration,
@@ -535,31 +532,15 @@ async fn wait_for_transition(ecstore: &Arc<ECStore>, bucket: &str, object: &str,
}
}
// SAFETY: this helper is used only by `#[serial]` tests and runs under the single-threaded Tokio
// runtime (`worker_threads = 1`), so no concurrent test can mutate process environment during the
// `env::set_var` / `env::remove_var` window.
#[allow(unsafe_code)]
// Run `test_fn` with `ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT`
// set to `"1"` for its duration. `temp_env` serializes environment mutations
// globally, preventing data races when multiple tests run in parallel.
async fn with_forced_immediate_enqueue_timeout<F, Fut>(test_fn: F)
where
F: FnOnce() -> Fut,
Fut: std::future::Future<Output = ()>,
{
let original = env::var_os(ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT);
unsafe {
env::set_var(ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT, "1");
}
let result = std::panic::AssertUnwindSafe(test_fn()).catch_unwind().await;
match original {
Some(value) => unsafe {
env::set_var(ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT, value);
},
None => unsafe {
env::remove_var(ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT);
},
}
if let Err(err) = result {
std::panic::resume_unwind(err);
}
temp_env::async_with_vars([(ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT, Some("1"))], test_fn()).await;
}
mod serial_tests {
@@ -592,7 +573,6 @@ mod serial_tests {
/// body (GET won) or a clean object/version-not-found (expiry won). A
/// tier-fetch failure -- the #3491 symptom -- is never tolerated.
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
#[serial]
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-2)"]
async fn test_expire_transitioned_object_never_races_concurrent_get() {
let (_disk_paths, ecstore) = setup_isolated_test_env(false).await;
@@ -738,7 +718,6 @@ mod serial_tests {
}
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
#[serial]
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial"]
async fn rejected_transition_candidate_is_recovered_from_persisted_delete_journal() {
let (_disk_paths, ecstore) = setup_isolated_test_env(false).await;
@@ -825,7 +804,6 @@ mod serial_tests {
}
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
#[serial]
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial"]
async fn cancelled_before_cleanup_store_resolution_persists_journal() {
let (_disk_paths, ecstore) = setup_isolated_test_env(false).await;
@@ -919,7 +897,6 @@ mod serial_tests {
}
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
#[serial]
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial"]
async fn rejected_transition_cleanup_durability_matrix() {
#[derive(Clone, Copy)]
@@ -1059,7 +1036,6 @@ mod serial_tests {
}
#[test]
#[serial]
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
fn test_transition_and_restore_flows() {
std::thread::Builder::new()
@@ -1385,7 +1361,6 @@ mod serial_tests {
}
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
#[serial]
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
async fn test_scanner_enqueues_free_version_cleanup_for_stale_transitioned_object() {
let (disk_paths, ecstore) = setup_isolated_test_env(false).await;
@@ -1446,7 +1421,6 @@ mod serial_tests {
}
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
#[serial]
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
async fn test_scanner_cleanup_still_works_after_immediate_compensation_transition() {
let (disk_paths, ecstore) = setup_isolated_test_env(false).await;
@@ -1504,7 +1478,6 @@ mod serial_tests {
}
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
#[serial]
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
async fn test_existing_object_backfill_is_idempotent_after_immediate_compensation_transition() {
let (_disk_paths, ecstore) = setup_isolated_test_env(false).await;
@@ -1547,7 +1520,6 @@ mod serial_tests {
}
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
#[serial]
#[ignore = "FAILING on main: excluded from the serial ILM lane pending a fix, see rustfs/backlog#1148 (ilm-1 partial)"]
async fn test_noncurrent_expiry_still_works_after_immediate_compensation_transition() {
let (disk_paths, ecstore) = setup_isolated_test_env(true).await;
@@ -1631,7 +1603,6 @@ mod serial_tests {
}
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
#[serial]
#[ignore = "FAILING on main: excluded from the serial ILM lane pending a fix, see rustfs/backlog#1148 (ilm-1 partial)"]
async fn test_noncurrent_transition_still_works_after_immediate_compensation_transition() {
let (disk_paths, ecstore) = setup_isolated_test_env(true).await;
@@ -1714,7 +1685,6 @@ mod serial_tests {
}
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
#[serial]
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
async fn test_modeled_versioned_delete_creates_delete_marker_after_immediate_compensation_transition() {
let (_disk_paths, ecstore) = setup_isolated_test_env(true).await;
@@ -1762,7 +1732,6 @@ mod serial_tests {
}
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
#[serial]
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
async fn test_modeled_delete_marker_cleanup_after_immediate_compensation_transition() {
let (disk_paths, ecstore) = setup_isolated_test_env(true).await;
@@ -1839,7 +1808,6 @@ mod serial_tests {
}
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
#[serial]
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
async fn test_scanner_expires_zero_day_current_version() {
let (disk_paths, ecstore) = setup_isolated_test_env(false).await;
@@ -1866,7 +1834,6 @@ mod serial_tests {
}
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
#[serial]
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
async fn test_put_object_immediately_enqueues_zero_day_current_expiry() {
let (_disk_paths, ecstore) = setup_isolated_test_env(true).await;
@@ -1904,7 +1871,6 @@ mod serial_tests {
}
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
#[serial]
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
async fn test_scanner_expires_zero_day_noncurrent_version() {
let (disk_paths, ecstore) = setup_isolated_test_env(false).await;
@@ -1971,7 +1937,6 @@ mod serial_tests {
}
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
#[serial]
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
async fn test_put_object_immediately_enqueues_zero_day_noncurrent_expiry() {
let (_disk_paths, ecstore) = setup_isolated_test_env(true).await;
@@ -2032,7 +1997,6 @@ mod serial_tests {
}
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
#[serial]
async fn test_background_scanner_expires_zero_day_current_version() {
let (_disk_paths, ecstore) = setup_isolated_test_env(true).await;
@@ -2056,7 +2020,6 @@ mod serial_tests {
}
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
#[serial]
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
async fn test_background_scanner_expires_zero_day_current_version_for_exact_key_prefix() {
let (_disk_paths, ecstore) = setup_isolated_test_env(true).await;
@@ -2122,7 +2085,6 @@ mod serial_tests {
/// tier object is untouched (zero `remove` calls) -> GET streams from the
/// tier again -> a second restore succeeds.
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
#[serial]
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-8)"]
async fn test_restore_chain_local_read_expiry_keeps_remote_and_allows_re_restore() {
let (_disk_paths, ecstore) = setup_test_env().await;
@@ -2254,7 +2216,6 @@ mod serial_tests {
/// parts) must reassemble the exact part layout: part count and sizes,
/// the multipart ETag, and byte-identical content across part boundaries.
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
#[serial]
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-8)"]
async fn test_multipart_restore_preserves_parts_and_etag() {
let (_disk_paths, ecstore) = setup_test_env().await;
+4 -5
View File
@@ -554,11 +554,10 @@ fn apply_replication_timestamps_from_headers(headers: &HeaderMap<HeaderValue>, o
// Persist into the internal metadata keys so a later outbound replication
// pass (replication_target_boundary) reads the source's modification
// times instead of falling back to mod_time. Receiver-side LWW happens at
// the set layer under the object write lock
// (ecstore set_disk::ops::object::merge_replication_metadata_lww,
// rustfs/backlog#1953): a category whose stored timestamp is newer than
// the inbound one keeps the local values.
// times instead of falling back to mod_time.
// TODO(P1-6): receiver-side LWW is still missing — when the stored
// per-category timestamp is newer than the inbound one, the existing
// tags/retention/legal-hold should win instead of being overwritten.
for (timestamp, suffix) in [
(opts.replication_tagging_timestamp, SUFFIX_TAGGING_TIMESTAMP),
(opts.replication_retention_timestamp, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP),