mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-22 04:16:38 +00:00
Compare commits
11 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| d33e489d84 | |||
| 378d46c54a | |||
| 3f6cd10946 | |||
| 5eaa6c1745 | |||
| adb90fc6e1 | |||
| cdfac5d7e3 | |||
| ca4adea0c9 | |||
| 23a0f6324c | |||
| cdd9ab1124 | |||
| 122a69df65 | |||
| dfeb732ac8 |
@@ -189,6 +189,7 @@ jobs:
|
|||||||
timeout-minutes: 30
|
timeout-minutes: 30
|
||||||
strategy:
|
strategy:
|
||||||
fail-fast: false
|
fail-fast: false
|
||||||
|
max-parallel: 1
|
||||||
matrix:
|
matrix:
|
||||||
include:
|
include:
|
||||||
- arch: x86_64
|
- arch: x86_64
|
||||||
@@ -510,15 +511,13 @@ jobs:
|
|||||||
|
|
||||||
CHECKSUM_DIR="$(mktemp -d)"
|
CHECKSUM_DIR="$(mktemp -d)"
|
||||||
gh release download "$TAG" -p 'SHA256SUMS' -p 'SHA512SUMS' \
|
gh release download "$TAG" -p 'SHA256SUMS' -p 'SHA512SUMS' \
|
||||||
-D "$CHECKSUM_DIR" --clobber 2>/dev/null || true
|
-D "$CHECKSUM_DIR" --clobber
|
||||||
|
|
||||||
for spec in "SHA256SUMS:sha256sum" "SHA512SUMS:sha512sum"; do
|
for spec in "SHA256SUMS:sha256sum" "SHA512SUMS:sha512sum"; do
|
||||||
asset="${spec%%:*}"
|
asset="${spec%%:*}"
|
||||||
checksum_cmd="${spec##*:}"
|
checksum_cmd="${spec##*:}"
|
||||||
checksum_file="${CHECKSUM_DIR}/${asset}"
|
checksum_file="${CHECKSUM_DIR}/${asset}"
|
||||||
|
|
||||||
touch "$checksum_file"
|
|
||||||
|
|
||||||
for f in "$DEB_FILE" "$RPM_FILE"; do
|
for f in "$DEB_FILE" "$RPM_FILE"; do
|
||||||
if [[ -n "$f" && -f "$f" ]]; then
|
if [[ -n "$f" && -f "$f" ]]; then
|
||||||
base="$(basename "$f")"
|
base="$(basename "$f")"
|
||||||
@@ -531,7 +530,8 @@ jobs:
|
|||||||
grep -Fv -- "$base" "$checksum_file" > "${checksum_file}.tmp" || true
|
grep -Fv -- "$base" "$checksum_file" > "${checksum_file}.tmp" || true
|
||||||
grep -Fv -- "$github_base" "${checksum_file}.tmp" > "${checksum_file}.tmp2" || true
|
grep -Fv -- "$github_base" "${checksum_file}.tmp" > "${checksum_file}.tmp2" || true
|
||||||
mv "${checksum_file}.tmp2" "$checksum_file"
|
mv "${checksum_file}.tmp2" "$checksum_file"
|
||||||
(cd "$(dirname "$f")" && "$checksum_cmd" -- "$github_base") >> "$checksum_file"
|
digest=$("$checksum_cmd" -- "$f" | awk '{print $1}')
|
||||||
|
printf '%s %s\n' "$digest" "$github_base" >> "$checksum_file"
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
|
|
||||||
|
|||||||
Generated
-4
@@ -3843,7 +3843,6 @@ dependencies = [
|
|||||||
"s3s",
|
"s3s",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"serial_test",
|
|
||||||
"sha2 0.11.0",
|
"sha2 0.11.0",
|
||||||
"suppaftp",
|
"suppaftp",
|
||||||
"time",
|
"time",
|
||||||
@@ -9804,7 +9803,6 @@ dependencies = [
|
|||||||
"rustfs-replication",
|
"rustfs-replication",
|
||||||
"rustfs-storage-api",
|
"rustfs-storage-api",
|
||||||
"s3s",
|
"s3s",
|
||||||
"serial_test",
|
|
||||||
"temp-env",
|
"temp-env",
|
||||||
"time",
|
"time",
|
||||||
"tokio",
|
"tokio",
|
||||||
@@ -9920,7 +9918,6 @@ dependencies = [
|
|||||||
"rustfs-config",
|
"rustfs-config",
|
||||||
"rustfs-io-metrics",
|
"rustfs-io-metrics",
|
||||||
"rustfs-utils",
|
"rustfs-utils",
|
||||||
"serial_test",
|
|
||||||
"temp-env",
|
"temp-env",
|
||||||
"tempfile",
|
"tempfile",
|
||||||
"tokio",
|
"tokio",
|
||||||
@@ -10293,7 +10290,6 @@ dependencies = [
|
|||||||
"s3s",
|
"s3s",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"serial_test",
|
|
||||||
"sha2 0.11.0",
|
"sha2 0.11.0",
|
||||||
"temp-env",
|
"temp-env",
|
||||||
"tempfile",
|
"tempfile",
|
||||||
|
|||||||
@@ -96,7 +96,6 @@ tokio-stream = { workspace = true }
|
|||||||
rustfs-madmin.workspace = true
|
rustfs-madmin.workspace = true
|
||||||
rustfs-filemeta.workspace = true
|
rustfs-filemeta.workspace = true
|
||||||
bytes = { workspace = true, features = ["serde"] }
|
bytes = { workspace = true, features = ["serde"] }
|
||||||
serial_test = { workspace = true }
|
|
||||||
aws-sdk-s3 = { workspace = true, default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
|
aws-sdk-s3 = { workspace = true, default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
|
||||||
aws-sdk-sts = { workspace = true, default-features = false, features = ["default-https-client", "rt-tokio"] }
|
aws-sdk-sts = { workspace = true, default-features = false, features = ["default-https-client", "rt-tokio"] }
|
||||||
aws-config = { workspace = true }
|
aws-config = { workspace = true }
|
||||||
|
|||||||
@@ -55,7 +55,6 @@ mod tests {
|
|||||||
use aws_sdk_s3::Client;
|
use aws_sdk_s3::Client;
|
||||||
use aws_sdk_s3::primitives::ByteStream;
|
use aws_sdk_s3::primitives::ByteStream;
|
||||||
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
|
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
|
||||||
use serial_test::serial;
|
|
||||||
use sha2::{Digest, Sha256};
|
use sha2::{Digest, Sha256};
|
||||||
use std::error::Error;
|
use std::error::Error;
|
||||||
use tokio::time::{Duration, timeout};
|
use tokio::time::{Duration, timeout};
|
||||||
@@ -269,7 +268,6 @@ mod tests {
|
|||||||
/// stripes) and a multipart object (3 parts × 5 MiB) must GET back as a
|
/// stripes) and a multipart object (3 parts × 5 MiB) must GET back as a
|
||||||
/// full, byte-identical body with the correct Content-Length. No early EOF.
|
/// full, byte-identical body with the correct Content-Length. No early EOF.
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn degraded_read_large_objects_with_one_disk_offline_return_full_body() -> TestResult {
|
async fn degraded_read_large_objects_with_one_disk_offline_return_full_body() -> TestResult {
|
||||||
init_logging();
|
init_logging();
|
||||||
info!("dist-13 (a): large-object degraded read with one of four disks offline");
|
info!("dist-13 (a): large-object degraded read with one of four disks offline");
|
||||||
@@ -335,7 +333,6 @@ mod tests {
|
|||||||
/// mid-stream — the exact window the fixes had to reconstruct through rather
|
/// mid-stream — the exact window the fixes had to reconstruct through rather
|
||||||
/// than truncate.
|
/// than truncate.
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn degraded_read_reconstructs_through_midstream_bitrot_within_quorum() -> TestResult {
|
async fn degraded_read_reconstructs_through_midstream_bitrot_within_quorum() -> TestResult {
|
||||||
init_logging();
|
init_logging();
|
||||||
info!("dist-13 (b): mid-stream bitrot within quorum must reconstruct a full body");
|
info!("dist-13 (b): mid-stream bitrot within quorum must reconstruct a full body");
|
||||||
@@ -393,7 +390,6 @@ mod tests {
|
|||||||
/// Content-Length. `get_checked` panics on that forbidden outcome, so this
|
/// Content-Length. `get_checked` panics on that forbidden outcome, so this
|
||||||
/// test fails loudly if the truncation bug ever returns.
|
/// test fails loudly if the truncation bug ever returns.
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn beyond_quorum_degraded_read_never_silently_truncates() -> TestResult {
|
async fn beyond_quorum_degraded_read_never_silently_truncates() -> TestResult {
|
||||||
init_logging();
|
init_logging();
|
||||||
info!("dist-13 (c): beyond-quorum degraded read must fail, never 200+truncated");
|
info!("dist-13 (c): beyond-quorum degraded read must fail, never 200+truncated");
|
||||||
|
|||||||
@@ -51,7 +51,6 @@ mod tests {
|
|||||||
use aws_sdk_s3::Client;
|
use aws_sdk_s3::Client;
|
||||||
use aws_sdk_s3::primitives::ByteStream;
|
use aws_sdk_s3::primitives::ByteStream;
|
||||||
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
|
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
|
||||||
use serial_test::serial;
|
|
||||||
use std::error::Error;
|
use std::error::Error;
|
||||||
use tokio::time::{Duration, timeout};
|
use tokio::time::{Duration, timeout};
|
||||||
use tracing::info;
|
use tracing::info;
|
||||||
@@ -129,7 +128,6 @@ mod tests {
|
|||||||
/// the body — and assert the server log names the object, at the log level a
|
/// the body — and assert the server log names the object, at the log level a
|
||||||
/// default deployment actually runs with.
|
/// default deployment actually runs with.
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn midstream_get_failure_is_logged_with_the_object_at_default_log_level() -> TestResult {
|
async fn midstream_get_failure_is_logged_with_the_object_at_default_log_level() -> TestResult {
|
||||||
init_logging();
|
init_logging();
|
||||||
info!("rustfs#4784: a mid-stream GET failure must name its object in the source log");
|
info!("rustfs#4784: a mid-stream GET failure must name its object in the source log");
|
||||||
|
|||||||
@@ -46,7 +46,6 @@ use prost::Message;
|
|||||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||||
use rustfs_signer::sign_v4;
|
use rustfs_signer::sign_v4;
|
||||||
use s3s::Body;
|
use s3s::Body;
|
||||||
use serial_test::serial;
|
|
||||||
use std::collections::BTreeMap;
|
use std::collections::BTreeMap;
|
||||||
use std::convert::Infallible;
|
use std::convert::Infallible;
|
||||||
use std::error::Error;
|
use std::error::Error;
|
||||||
@@ -1695,7 +1694,6 @@ fn assert_storage_layout(
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn four_node_inline_storage_and_get_boundaries() -> TestResult {
|
async fn four_node_inline_storage_and_get_boundaries() -> TestResult {
|
||||||
init_logging();
|
init_logging();
|
||||||
|
|
||||||
@@ -1767,7 +1765,6 @@ async fn four_node_inline_storage_and_get_boundaries() -> TestResult {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn four_node_empty_legacy_volumes_start_as_fresh() -> TestResult {
|
async fn four_node_empty_legacy_volumes_start_as_fresh() -> TestResult {
|
||||||
init_logging();
|
init_logging();
|
||||||
|
|
||||||
@@ -1805,7 +1802,6 @@ async fn four_node_empty_legacy_volumes_start_as_fresh() -> TestResult {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn four_node_inline_fallback_controls() -> TestResult {
|
async fn four_node_inline_fallback_controls() -> TestResult {
|
||||||
init_logging();
|
init_logging();
|
||||||
|
|
||||||
@@ -1870,7 +1866,6 @@ async fn four_node_inline_fallback_controls() -> TestResult {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn four_node_compressed_inline_fallback() -> TestResult {
|
async fn four_node_compressed_inline_fallback() -> TestResult {
|
||||||
init_logging();
|
init_logging();
|
||||||
|
|
||||||
@@ -1905,7 +1900,6 @@ async fn four_node_compressed_inline_fallback() -> TestResult {
|
|||||||
/// Multipart disk compression is live again, so a compression-enabled cluster classifies multipart objects as compressed and the roundtrip (full GET plus partNumber GET) must still return the original bytes.
|
/// Multipart disk compression is live again, so a compression-enabled cluster classifies multipart objects as compressed and the roundtrip (full GET plus partNumber GET) must still return the original bytes.
|
||||||
/// Reverting the multipart compression fix must fail this test.
|
/// Reverting the multipart compression fix must fail this test.
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn four_node_multipart_disk_compression_roundtrip() -> TestResult {
|
async fn four_node_multipart_disk_compression_roundtrip() -> TestResult {
|
||||||
init_logging();
|
init_logging();
|
||||||
|
|
||||||
@@ -1952,7 +1946,6 @@ async fn four_node_multipart_disk_compression_roundtrip() -> TestResult {
|
|||||||
/// read costs on the order of the covering part's block size against a ~5 MiB
|
/// read costs on the order of the covering part's block size against a ~5 MiB
|
||||||
/// object.
|
/// object.
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn four_node_compressed_multipart_tail_range_reads_are_bounded() -> TestResult {
|
async fn four_node_compressed_multipart_tail_range_reads_are_bounded() -> TestResult {
|
||||||
init_logging();
|
init_logging();
|
||||||
|
|
||||||
@@ -2019,7 +2012,6 @@ async fn four_node_compressed_multipart_tail_range_reads_are_bounded() -> TestRe
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn four_node_mixed_msgpack_compat_mode_preserves_fallback_controls() -> TestResult {
|
async fn four_node_mixed_msgpack_compat_mode_preserves_fallback_controls() -> TestResult {
|
||||||
init_logging();
|
init_logging();
|
||||||
|
|
||||||
@@ -2123,7 +2115,6 @@ async fn four_node_mixed_msgpack_compat_mode_preserves_fallback_controls() -> Te
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn four_node_add_tier_converges() -> TestResult {
|
async fn four_node_add_tier_converges() -> TestResult {
|
||||||
init_logging();
|
init_logging();
|
||||||
|
|
||||||
@@ -2142,7 +2133,6 @@ async fn four_node_add_tier_converges() -> TestResult {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn four_node_add_tier_converges_after_offline_node_restart_without_second_mutation() -> TestResult {
|
async fn four_node_add_tier_converges_after_offline_node_restart_without_second_mutation() -> TestResult {
|
||||||
init_logging();
|
init_logging();
|
||||||
|
|
||||||
@@ -2164,7 +2154,6 @@ async fn four_node_add_tier_converges_after_offline_node_restart_without_second_
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn four_node_manual_transition_job_status_survives_node_restart() -> TestResult {
|
async fn four_node_manual_transition_job_status_survives_node_restart() -> TestResult {
|
||||||
init_logging();
|
init_logging();
|
||||||
|
|
||||||
@@ -2239,7 +2228,6 @@ async fn four_node_manual_transition_job_status_survives_node_restart() -> TestR
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn four_node_manual_transition_distributed_admission_conflict_reports_status_and_backpressure() -> TestResult {
|
async fn four_node_manual_transition_distributed_admission_conflict_reports_status_and_backpressure() -> TestResult {
|
||||||
init_logging();
|
init_logging();
|
||||||
|
|
||||||
@@ -2381,7 +2369,6 @@ async fn four_node_manual_transition_distributed_admission_conflict_reports_stat
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
#[ignore = "manual #1508 evidence harness: starts a 4-node cluster, a remote tier, and an in-flight transition job"]
|
#[ignore = "manual #1508 evidence harness: starts a 4-node cluster, a remote tier, and an in-flight transition job"]
|
||||||
async fn four_node_manual_transition_rollout_non_empty_restart_readback() -> TestResult {
|
async fn four_node_manual_transition_rollout_non_empty_restart_readback() -> TestResult {
|
||||||
init_logging();
|
init_logging();
|
||||||
@@ -2486,7 +2473,6 @@ async fn four_node_manual_transition_rollout_non_empty_restart_readback() -> Tes
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn four_node_mixed_msgpack_compat_mode_preserves_fallback_controls_during_transition() -> TestResult {
|
async fn four_node_mixed_msgpack_compat_mode_preserves_fallback_controls_during_transition() -> TestResult {
|
||||||
init_logging();
|
init_logging();
|
||||||
|
|
||||||
@@ -2598,7 +2584,6 @@ async fn four_node_mixed_msgpack_compat_mode_preserves_fallback_controls_during_
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn four_node_transitioned_inline_fallback() -> TestResult {
|
async fn four_node_transitioned_inline_fallback() -> TestResult {
|
||||||
init_logging();
|
init_logging();
|
||||||
|
|
||||||
|
|||||||
@@ -19,7 +19,6 @@
|
|||||||
//! multipart upload behaviour.
|
//! multipart upload behaviour.
|
||||||
|
|
||||||
use crate::common::{TEST_BUCKET, init_logging};
|
use crate::common::{TEST_BUCKET, init_logging};
|
||||||
use serial_test::serial;
|
|
||||||
use tokio::time::{Duration, sleep};
|
use tokio::time::{Duration, sleep};
|
||||||
use tracing::{error, info};
|
use tracing::{error, info};
|
||||||
|
|
||||||
@@ -62,7 +61,6 @@ impl VaultKmsTestContext {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_vault_kms_end_to_end() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
async fn test_vault_kms_end_to_end() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||||
init_logging();
|
init_logging();
|
||||||
if skip_if_kms_admin_tool_unavailable("test_vault_kms_end_to_end") {
|
if skip_if_kms_admin_tool_unavailable("test_vault_kms_end_to_end") {
|
||||||
@@ -118,7 +116,6 @@ async fn test_vault_kms_end_to_end() -> Result<(), Box<dyn std::error::Error + S
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_vault_kms_key_isolation() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
async fn test_vault_kms_key_isolation() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||||
init_logging();
|
init_logging();
|
||||||
if skip_if_kms_admin_tool_unavailable("test_vault_kms_key_isolation") {
|
if skip_if_kms_admin_tool_unavailable("test_vault_kms_key_isolation") {
|
||||||
@@ -205,7 +202,6 @@ async fn test_vault_kms_key_isolation() -> Result<(), Box<dyn std::error::Error
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_vault_kms_large_file() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
async fn test_vault_kms_large_file() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||||
init_logging();
|
init_logging();
|
||||||
if skip_if_kms_admin_tool_unavailable("test_vault_kms_large_file") {
|
if skip_if_kms_admin_tool_unavailable("test_vault_kms_large_file") {
|
||||||
@@ -270,7 +266,6 @@ async fn test_vault_kms_large_file() -> Result<(), Box<dyn std::error::Error + S
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_vault_kms_multipart_upload() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
async fn test_vault_kms_multipart_upload() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||||
init_logging();
|
init_logging();
|
||||||
if skip_if_kms_admin_tool_unavailable("test_vault_kms_multipart_upload") {
|
if skip_if_kms_admin_tool_unavailable("test_vault_kms_multipart_upload") {
|
||||||
@@ -301,7 +296,6 @@ async fn test_vault_kms_multipart_upload() -> Result<(), Box<dyn std::error::Err
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_vault_kms_key_operations() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
async fn test_vault_kms_key_operations() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||||
init_logging();
|
init_logging();
|
||||||
if skip_if_kms_admin_tool_unavailable("test_vault_kms_key_operations") {
|
if skip_if_kms_admin_tool_unavailable("test_vault_kms_key_operations") {
|
||||||
|
|||||||
@@ -17,7 +17,6 @@
|
|||||||
use crate::common::{awscurl_delete, awscurl_put, init_logging};
|
use crate::common::{awscurl_delete, awscurl_put, init_logging};
|
||||||
use crate::policy::test_env::PolicyTestEnvironment;
|
use crate::policy::test_env::PolicyTestEnvironment;
|
||||||
use aws_sdk_s3::primitives::ByteStream;
|
use aws_sdk_s3::primitives::ByteStream;
|
||||||
use serial_test::serial;
|
|
||||||
use tracing::info;
|
use tracing::info;
|
||||||
|
|
||||||
/// Helper function to create a regular user with given credentials
|
/// Helper function to create a regular user with given credentials
|
||||||
@@ -122,7 +121,6 @@ async fn cleanup_user_and_policy(env: &PolicyTestEnvironment, username: &str, po
|
|||||||
|
|
||||||
/// Test AWS policy variables with single-value scenarios
|
/// Test AWS policy variables with single-value scenarios
|
||||||
#[tokio::test(flavor = "multi_thread")]
|
#[tokio::test(flavor = "multi_thread")]
|
||||||
#[serial]
|
|
||||||
#[ignore = "Starts a rustfs server; enable when running full E2E"]
|
#[ignore = "Starts a rustfs server; enable when running full E2E"]
|
||||||
pub async fn test_aws_policy_variables_single_value() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
pub async fn test_aws_policy_variables_single_value() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||||
test_aws_policy_variables_single_value_impl().await
|
test_aws_policy_variables_single_value_impl().await
|
||||||
@@ -275,7 +273,6 @@ pub async fn test_aws_policy_variables_single_value_impl_with_env(
|
|||||||
|
|
||||||
/// Test AWS policy variables with multi-value scenarios
|
/// Test AWS policy variables with multi-value scenarios
|
||||||
#[tokio::test(flavor = "multi_thread")]
|
#[tokio::test(flavor = "multi_thread")]
|
||||||
#[serial]
|
|
||||||
#[ignore = "Starts a rustfs server; enable when running full E2E"]
|
#[ignore = "Starts a rustfs server; enable when running full E2E"]
|
||||||
pub async fn test_aws_policy_variables_multi_value() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
pub async fn test_aws_policy_variables_multi_value() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||||
test_aws_policy_variables_multi_value_impl().await
|
test_aws_policy_variables_multi_value_impl().await
|
||||||
@@ -401,7 +398,6 @@ pub async fn test_aws_policy_variables_multi_value_impl_with_env(
|
|||||||
|
|
||||||
/// Test AWS policy variables with variable concatenation
|
/// Test AWS policy variables with variable concatenation
|
||||||
#[tokio::test(flavor = "multi_thread")]
|
#[tokio::test(flavor = "multi_thread")]
|
||||||
#[serial]
|
|
||||||
#[ignore = "Starts a rustfs server; enable when running full E2E"]
|
#[ignore = "Starts a rustfs server; enable when running full E2E"]
|
||||||
pub async fn test_aws_policy_variables_concatenation() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
pub async fn test_aws_policy_variables_concatenation() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||||
test_aws_policy_variables_concatenation_impl().await
|
test_aws_policy_variables_concatenation_impl().await
|
||||||
@@ -491,7 +487,6 @@ pub async fn test_aws_policy_variables_concatenation_impl_with_env(
|
|||||||
|
|
||||||
/// Test AWS policy variables with nested scenarios
|
/// Test AWS policy variables with nested scenarios
|
||||||
#[tokio::test(flavor = "multi_thread")]
|
#[tokio::test(flavor = "multi_thread")]
|
||||||
#[serial]
|
|
||||||
#[ignore = "Starts a rustfs server; enable when running full E2E"]
|
#[ignore = "Starts a rustfs server; enable when running full E2E"]
|
||||||
pub async fn test_aws_policy_variables_nested() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
pub async fn test_aws_policy_variables_nested() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||||
test_aws_policy_variables_nested_impl().await
|
test_aws_policy_variables_nested_impl().await
|
||||||
@@ -509,7 +504,6 @@ pub async fn test_aws_policy_variables_nested_impl() -> Result<(), Box<dyn std::
|
|||||||
|
|
||||||
/// Test AWS policy variables with STS temporary credentials
|
/// Test AWS policy variables with STS temporary credentials
|
||||||
#[tokio::test(flavor = "multi_thread")]
|
#[tokio::test(flavor = "multi_thread")]
|
||||||
#[serial]
|
|
||||||
#[ignore = "Starts a rustfs server; enable when running full E2E"]
|
#[ignore = "Starts a rustfs server; enable when running full E2E"]
|
||||||
pub async fn test_aws_policy_variables_sts() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
pub async fn test_aws_policy_variables_sts() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||||
test_aws_policy_variables_sts_impl().await
|
test_aws_policy_variables_sts_impl().await
|
||||||
@@ -705,7 +699,6 @@ pub async fn test_aws_policy_variables_sts_impl_with_env(
|
|||||||
|
|
||||||
/// Test AWS policy variables with deny scenarios
|
/// Test AWS policy variables with deny scenarios
|
||||||
#[tokio::test(flavor = "multi_thread")]
|
#[tokio::test(flavor = "multi_thread")]
|
||||||
#[serial]
|
|
||||||
#[ignore = "Starts a rustfs server; enable when running full E2E"]
|
#[ignore = "Starts a rustfs server; enable when running full E2E"]
|
||||||
pub async fn test_aws_policy_variables_deny() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
pub async fn test_aws_policy_variables_deny() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||||
test_aws_policy_variables_deny_impl().await
|
test_aws_policy_variables_deny_impl().await
|
||||||
|
|||||||
@@ -14,7 +14,6 @@
|
|||||||
|
|
||||||
use crate::common::init_logging;
|
use crate::common::init_logging;
|
||||||
use crate::policy::test_env::PolicyTestEnvironment;
|
use crate::policy::test_env::PolicyTestEnvironment;
|
||||||
use serial_test::serial;
|
|
||||||
use std::time::Instant;
|
use std::time::Instant;
|
||||||
use tokio::time::{Duration, sleep};
|
use tokio::time::{Duration, sleep};
|
||||||
use tracing::{error, info};
|
use tracing::{error, info};
|
||||||
@@ -213,7 +212,6 @@ impl PolicyTestSuite {
|
|||||||
|
|
||||||
/// Test suite
|
/// Test suite
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
#[ignore = "Connects to existing rustfs server"]
|
#[ignore = "Connects to existing rustfs server"]
|
||||||
async fn test_policy_critical_suite() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
async fn test_policy_critical_suite() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||||
let config = TestSuiteConfig {
|
let config = TestSuiteConfig {
|
||||||
|
|||||||
@@ -41,7 +41,6 @@ use reqwest::Client;
|
|||||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||||
use rustfs_signer::sign_v4;
|
use rustfs_signer::sign_v4;
|
||||||
use s3s::Body;
|
use s3s::Body;
|
||||||
use serial_test::serial;
|
|
||||||
use tokio::process::Command;
|
use tokio::process::Command;
|
||||||
use tracing::info;
|
use tracing::info;
|
||||||
|
|
||||||
@@ -821,7 +820,6 @@ pub async fn test_webdav_core_operations() -> Result<()> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_webdav_core_operations_direct() -> Result<()> {
|
async fn test_webdav_core_operations_direct() -> Result<()> {
|
||||||
test_webdav_core_operations().await
|
test_webdav_core_operations().await
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -27,7 +27,6 @@ mod tests {
|
|||||||
use aws_sdk_s3::Client;
|
use aws_sdk_s3::Client;
|
||||||
use aws_sdk_s3::primitives::ByteStream;
|
use aws_sdk_s3::primitives::ByteStream;
|
||||||
use aws_sdk_s3::types::{BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, VersioningConfiguration};
|
use aws_sdk_s3::types::{BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, VersioningConfiguration};
|
||||||
use serial_test::serial;
|
|
||||||
use sha2::{Digest, Sha256};
|
use sha2::{Digest, Sha256};
|
||||||
use std::collections::HashSet;
|
use std::collections::HashSet;
|
||||||
use std::error::Error;
|
use std::error::Error;
|
||||||
@@ -157,7 +156,6 @@ mod tests {
|
|||||||
/// content, degraded writes must succeed, and everything must still
|
/// content, degraded writes must succeed, and everything must still
|
||||||
/// verify after the disk returns.
|
/// verify after the disk returns.
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_degraded_read_write_with_one_disk_offline() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
async fn test_degraded_read_write_with_one_disk_offline() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||||
init_logging();
|
init_logging();
|
||||||
info!("Reliability: degraded read/write with one of four disks offline");
|
info!("Reliability: degraded read/write with one of four disks offline");
|
||||||
@@ -210,7 +208,6 @@ mod tests {
|
|||||||
/// bytes to a reader: per-shard bitrot checksums reject the bad shard and
|
/// bytes to a reader: per-shard bitrot checksums reject the bad shard and
|
||||||
/// the object is reconstructed from the remaining shards.
|
/// the object is reconstructed from the remaining shards.
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_bitrot_corrupted_shard_read_returns_correct_data() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
async fn test_bitrot_corrupted_shard_read_returns_correct_data() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||||
init_logging();
|
init_logging();
|
||||||
info!("Reliability: GET must read through a bitrot-corrupted shard");
|
info!("Reliability: GET must read through a bitrot-corrupted shard");
|
||||||
@@ -253,7 +250,6 @@ mod tests {
|
|||||||
/// heal, and require the replaced disk to be rebuilt and all content to
|
/// heal, and require the replaced disk to be rebuilt and all content to
|
||||||
/// verify against the sha256 manifest.
|
/// verify against the sha256 manifest.
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_fresh_disk_replacement_heals_after_sigkill_restart() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
async fn test_fresh_disk_replacement_heals_after_sigkill_restart() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||||
init_logging();
|
init_logging();
|
||||||
info!("Reliability: fresh-disk replacement heals after SIGKILL restart");
|
info!("Reliability: fresh-disk replacement heals after SIGKILL restart");
|
||||||
@@ -327,7 +323,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_versioned_shard_census_selects_each_version_data_dir() -> Result<(), Box<dyn Error + Send + Sync>> {
|
async fn test_versioned_shard_census_selects_each_version_data_dir() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||||
init_logging();
|
init_logging();
|
||||||
info!("Reliability: physical shard census selects the requested object version");
|
info!("Reliability: physical shard census selects the requested object version");
|
||||||
|
|||||||
@@ -29,7 +29,6 @@ mod tests {
|
|||||||
use aws_sdk_s3::primitives::ByteStream;
|
use aws_sdk_s3::primitives::ByteStream;
|
||||||
use aws_sdk_s3::types::{BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, VersioningConfiguration};
|
use aws_sdk_s3::types::{BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, VersioningConfiguration};
|
||||||
use http::Method;
|
use http::Method;
|
||||||
use serial_test::serial;
|
|
||||||
use sha2::{Digest, Sha256};
|
use sha2::{Digest, Sha256};
|
||||||
use std::collections::BTreeSet;
|
use std::collections::BTreeSet;
|
||||||
use std::error::Error;
|
use std::error::Error;
|
||||||
@@ -1061,7 +1060,6 @@ mod tests {
|
|||||||
/// Linux mount namespaces are per-thread; keep mount setup and process
|
/// Linux mount namespaces are per-thread; keep mount setup and process
|
||||||
/// spawning on one OS thread so child RustFS nodes inherit the test mounts.
|
/// spawning on one OS thread so child RustFS nodes inherit the test mounts.
|
||||||
#[tokio::test(flavor = "current_thread")]
|
#[tokio::test(flavor = "current_thread")]
|
||||||
#[serial]
|
|
||||||
#[ignore = "requires Linux root/CAP_SYS_ADMIN and RUSTFS_PRIVILEGED_REPLACEMENT_E2E=1"]
|
#[ignore = "requires Linux root/CAP_SYS_ADMIN and RUSTFS_PRIVILEGED_REPLACEMENT_E2E=1"]
|
||||||
async fn test_privileged_3x4_auto_replacement_rebuilds_ec8_plus_4_without_admin_heal()
|
async fn test_privileged_3x4_auto_replacement_rebuilds_ec8_plus_4_without_admin_heal()
|
||||||
-> Result<(), Box<dyn Error + Send + Sync>> {
|
-> Result<(), Box<dyn Error + Send + Sync>> {
|
||||||
@@ -1075,7 +1073,6 @@ mod tests {
|
|||||||
/// Linux mount namespaces are per-thread; keep mount setup and process
|
/// Linux mount namespaces are per-thread; keep mount setup and process
|
||||||
/// spawning on one OS thread so child RustFS nodes inherit the test mounts.
|
/// spawning on one OS thread so child RustFS nodes inherit the test mounts.
|
||||||
#[tokio::test(flavor = "current_thread")]
|
#[tokio::test(flavor = "current_thread")]
|
||||||
#[serial]
|
|
||||||
#[ignore = "requires Linux root/CAP_SYS_ADMIN and RUSTFS_PRIVILEGED_REPLACEMENT_E2E=1"]
|
#[ignore = "requires Linux root/CAP_SYS_ADMIN and RUSTFS_PRIVILEGED_REPLACEMENT_E2E=1"]
|
||||||
async fn test_privileged_3x4_auto_replacement_rebuilds_ec6_plus_6_without_admin_heal()
|
async fn test_privileged_3x4_auto_replacement_rebuilds_ec6_plus_6_without_admin_heal()
|
||||||
-> Result<(), Box<dyn Error + Send + Sync>> {
|
-> Result<(), Box<dyn Error + Send + Sync>> {
|
||||||
|
|||||||
@@ -41,6 +41,7 @@ const IAM_FORMAT_FILE_PATH: &str = "config/iam/format.json";
|
|||||||
const IAM_USERS_PREFIX: &str = "config/iam/users/";
|
const IAM_USERS_PREFIX: &str = "config/iam/users/";
|
||||||
const IAM_SERVICE_ACCOUNTS_PREFIX: &str = "config/iam/service-accounts/";
|
const IAM_SERVICE_ACCOUNTS_PREFIX: &str = "config/iam/service-accounts/";
|
||||||
const IAM_STS_PREFIX: &str = "config/iam/sts/";
|
const IAM_STS_PREFIX: &str = "config/iam/sts/";
|
||||||
|
const MINIO_GO_ZERO_TIME: OffsetDateTime = time::macros::datetime!(0001-01-01 00:00 UTC);
|
||||||
const IAM_GROUPS_PREFIX: &str = "config/iam/groups/";
|
const IAM_GROUPS_PREFIX: &str = "config/iam/groups/";
|
||||||
const IAM_POLICIES_PREFIX: &str = "config/iam/policies/";
|
const IAM_POLICIES_PREFIX: &str = "config/iam/policies/";
|
||||||
const IAM_POLICY_DB_PREFIX: &str = "config/iam/policydb/";
|
const IAM_POLICY_DB_PREFIX: &str = "config/iam/policydb/";
|
||||||
@@ -120,6 +121,15 @@ fn normalize_iam_config_blob(path: &str, data: &[u8]) -> std::result::Result<Opt
|
|||||||
if is_identity_path(path) {
|
if is_identity_path(path) {
|
||||||
let mut identity: UserIdentity =
|
let mut identity: UserIdentity =
|
||||||
serde_json::from_slice(data).map_err(|err| format!("parse IAM identity failed: {err}"))?;
|
serde_json::from_slice(data).map_err(|err| format!("parse IAM identity failed: {err}"))?;
|
||||||
|
if (path.starts_with(IAM_USERS_PREFIX) || path.starts_with(IAM_SERVICE_ACCOUNTS_PREFIX))
|
||||||
|
&& identity
|
||||||
|
.credentials
|
||||||
|
.expiration
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(|expiration| *expiration == MINIO_GO_ZERO_TIME || *expiration == OffsetDateTime::UNIX_EPOCH)
|
||||||
|
{
|
||||||
|
identity.credentials.expiration = None;
|
||||||
|
}
|
||||||
if identity.update_at.is_none() {
|
if identity.update_at.is_none() {
|
||||||
identity.update_at = Some(OffsetDateTime::now_utc());
|
identity.update_at = Some(OffsetDateTime::now_utc());
|
||||||
}
|
}
|
||||||
@@ -441,7 +451,10 @@ mod tests {
|
|||||||
use crate::bucket::replication::{
|
use crate::bucket::replication::{
|
||||||
BucketReplicationResyncStatus, ReplicationMigrationBridge, ResyncStatusType, TargetReplicationResyncStatus,
|
BucketReplicationResyncStatus, ReplicationMigrationBridge, ResyncStatusType, TargetReplicationResyncStatus,
|
||||||
};
|
};
|
||||||
|
use rustfs_policy::auth::UserIdentity;
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
|
use time::OffsetDateTime;
|
||||||
|
use time::format_description::well_known::Rfc3339;
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_normalize_policy_mapping_legacy_timestamp_and_fields() {
|
fn test_normalize_policy_mapping_legacy_timestamp_and_fields() {
|
||||||
@@ -493,6 +506,54 @@ mod tests {
|
|||||||
assert!(v.get("updatedAt").is_some(), "normalize should backfill updatedAt");
|
assert!(v.get("updatedAt").is_some(), "normalize should backfill updatedAt");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_normalize_minio_permanent_credential_expiration() {
|
||||||
|
let cases = [
|
||||||
|
("config/iam/users/alice/identity.json", "0001-01-01T00:00:00Z", true),
|
||||||
|
("config/iam/users/alice/identity.json", "1970-01-01T00:00:00Z", true),
|
||||||
|
("config/iam/service-accounts/svc/identity.json", "0001-01-01T00:00:00Z", true),
|
||||||
|
("config/iam/service-accounts/svc/identity.json", "1970-01-01T00:00:00Z", true),
|
||||||
|
("config/iam/service-accounts/svc/identity.json", "1970-01-01T00:00:00.000000001Z", false),
|
||||||
|
("config/iam/sts/temp/identity.json", "0001-01-01T00:00:00Z", false),
|
||||||
|
("config/iam/sts/temp/identity.json", "1970-01-01T00:00:00Z", false),
|
||||||
|
("config/iam/users/alice/identity.json", "1969-12-31T23:59:59Z", false),
|
||||||
|
("config/iam/users/alice/identity.json", "1970-01-01T00:00:00.000000001Z", false),
|
||||||
|
("config/iam/users/alice/identity.json", "0001-01-01T00:00:00.000000001Z", false),
|
||||||
|
("config/iam/users/alice/identity.json", "2030-01-01T00:00:00Z", false),
|
||||||
|
];
|
||||||
|
|
||||||
|
for (path, expiration, should_clear) in cases {
|
||||||
|
let input = serde_json::json!({
|
||||||
|
"version": 1,
|
||||||
|
"credentials": {
|
||||||
|
"accessKey": "test-access",
|
||||||
|
"secretKey": "test-secret",
|
||||||
|
"sessionToken": "test-session-token",
|
||||||
|
"parentUser": "test-parent",
|
||||||
|
"expiration": expiration,
|
||||||
|
}
|
||||||
|
});
|
||||||
|
let output = normalize_iam_config_blob(path, &serde_json::to_vec(&input).expect("serialize identity fixture"))
|
||||||
|
.expect("normalize should succeed")
|
||||||
|
.expect("identity path should be supported");
|
||||||
|
let identity: UserIdentity = serde_json::from_slice(&output).expect("deserialize normalized identity");
|
||||||
|
|
||||||
|
assert_eq!(identity.credentials.access_key, "test-access");
|
||||||
|
assert_eq!(identity.credentials.secret_key, "test-secret");
|
||||||
|
assert_eq!(identity.credentials.session_token, "test-session-token");
|
||||||
|
assert_eq!(identity.credentials.parent_user, "test-parent");
|
||||||
|
if should_clear {
|
||||||
|
assert_eq!(identity.credentials.expiration, None, "path: {path}, expiration: {expiration}");
|
||||||
|
} else {
|
||||||
|
assert_eq!(
|
||||||
|
identity.credentials.expiration,
|
||||||
|
Some(OffsetDateTime::parse(expiration, &Rfc3339).expect("parse expected expiration")),
|
||||||
|
"path: {path}, expiration: {expiration}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_normalize_bucket_meta_blob_resync_reencode() {
|
fn test_normalize_bucket_meta_blob_resync_reencode() {
|
||||||
let path = ".buckets/test/.replication/resync.bin";
|
let path = ".buckets/test/.replication/resync.bin";
|
||||||
|
|||||||
@@ -94,6 +94,9 @@ const DECOMMISSION_BUCKET_CONCURRENCY_DEFAULT_CAP: usize = 4;
|
|||||||
const DECOMMISSION_TARGET_CAPACITY_OVERHEAD_PERCENT: usize = 30;
|
const DECOMMISSION_TARGET_CAPACITY_OVERHEAD_PERCENT: usize = 30;
|
||||||
const DECOMMISSION_LISTING_MAX_ATTEMPTS: usize = 3;
|
const DECOMMISSION_LISTING_MAX_ATTEMPTS: usize = 3;
|
||||||
const DECOMMISSION_LISTING_RETRY_DELAY: std::time::Duration = std::time::Duration::from_secs(5);
|
const DECOMMISSION_LISTING_RETRY_DELAY: std::time::Duration = std::time::Duration::from_secs(5);
|
||||||
|
/// Background decommission walks must tolerate slow object migrations; the
|
||||||
|
/// stall timeout is the drive-health bound, not the total listing duration.
|
||||||
|
const DECOMMISSION_BACKGROUND_WALKDIR_STALL_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60);
|
||||||
|
|
||||||
pub const POOL_META_NAME: &str = "pool.bin";
|
pub const POOL_META_NAME: &str = "pool.bin";
|
||||||
pub const POOL_META_FORMAT: u16 = 1;
|
pub const POOL_META_FORMAT: u16 = 1;
|
||||||
@@ -5047,6 +5050,8 @@ impl SetDisks {
|
|||||||
path: bucket_info.prefix.clone(),
|
path: bucket_info.prefix.clone(),
|
||||||
recursive: true,
|
recursive: true,
|
||||||
min_disks: listing_quorum,
|
min_disks: listing_quorum,
|
||||||
|
skip_walkdir_total_timeout: true,
|
||||||
|
walkdir_stall_timeout: Some(DECOMMISSION_BACKGROUND_WALKDIR_STALL_TIMEOUT),
|
||||||
agreed: Some(Box::new(move |entry: MetaCacheEntry| Box::pin(cb1(entry)))),
|
agreed: Some(Box::new(move |entry: MetaCacheEntry| Box::pin(cb1(entry)))),
|
||||||
partial: Some(Box::new(move |entries: MetaCacheEntries, _: &[Option<DiskError>]| {
|
partial: Some(Box::new(move |entries: MetaCacheEntries, _: &[Option<DiskError>]| {
|
||||||
let resolver = resolver.clone();
|
let resolver = resolver.clone();
|
||||||
|
|||||||
@@ -23,7 +23,7 @@ use std::{
|
|||||||
io,
|
io,
|
||||||
path::{Component, Path, PathBuf},
|
path::{Component, Path, PathBuf},
|
||||||
sync::{Arc, LazyLock, Weak},
|
sync::{Arc, LazyLock, Weak},
|
||||||
time::Instant,
|
time::{Duration, Instant},
|
||||||
};
|
};
|
||||||
use tokio::fs;
|
use tokio::fs;
|
||||||
use tokio::sync::{
|
use tokio::sync::{
|
||||||
@@ -328,6 +328,9 @@ const ENV_DST_DIR_FSYNC_GROUP_COMMIT_ENABLE: &str = "RUSTFS_EXPERIMENTAL_DST_DIR
|
|||||||
const DEFAULT_DST_DIR_FSYNC_GROUP_COMMIT_ENABLE: bool = false;
|
const DEFAULT_DST_DIR_FSYNC_GROUP_COMMIT_ENABLE: bool = false;
|
||||||
const ENV_FILE_FDATASYNC_GROUP_COMMIT_ENABLE: &str = "RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_ENABLE";
|
const ENV_FILE_FDATASYNC_GROUP_COMMIT_ENABLE: &str = "RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_ENABLE";
|
||||||
const DEFAULT_FILE_FDATASYNC_GROUP_COMMIT_ENABLE: bool = false;
|
const DEFAULT_FILE_FDATASYNC_GROUP_COMMIT_ENABLE: bool = false;
|
||||||
|
const ENV_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS: &str = "RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS";
|
||||||
|
const DEFAULT_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS: u64 = 0;
|
||||||
|
const MAX_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS: u64 = 1_000;
|
||||||
#[cfg(not(test))]
|
#[cfg(not(test))]
|
||||||
const MAX_DST_DIR_FSYNC_GROUPS: usize = 1024;
|
const MAX_DST_DIR_FSYNC_GROUPS: usize = 1024;
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
@@ -354,6 +357,16 @@ static DST_DIR_FSYNC_GROUP_COMMIT_ENABLED: LazyLock<bool> = LazyLock::new(|| {
|
|||||||
static FILE_FDATASYNC_GROUP_COMMIT_ENABLED: LazyLock<bool> = LazyLock::new(|| {
|
static FILE_FDATASYNC_GROUP_COMMIT_ENABLED: LazyLock<bool> = LazyLock::new(|| {
|
||||||
rustfs_utils::get_env_bool(ENV_FILE_FDATASYNC_GROUP_COMMIT_ENABLE, DEFAULT_FILE_FDATASYNC_GROUP_COMMIT_ENABLE)
|
rustfs_utils::get_env_bool(ENV_FILE_FDATASYNC_GROUP_COMMIT_ENABLE, DEFAULT_FILE_FDATASYNC_GROUP_COMMIT_ENABLE)
|
||||||
});
|
});
|
||||||
|
fn file_fdatasync_group_commit_wait_duration(wait_micros: u64) -> Duration {
|
||||||
|
Duration::from_micros(wait_micros.min(MAX_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS))
|
||||||
|
}
|
||||||
|
|
||||||
|
static FILE_FDATASYNC_GROUP_COMMIT_WAIT: LazyLock<Duration> = LazyLock::new(|| {
|
||||||
|
file_fdatasync_group_commit_wait_duration(rustfs_utils::get_env_u64(
|
||||||
|
ENV_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS,
|
||||||
|
DEFAULT_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS,
|
||||||
|
))
|
||||||
|
});
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod dst_dir_fsync_group_commit_override {
|
mod dst_dir_fsync_group_commit_override {
|
||||||
@@ -402,6 +415,7 @@ mod file_fdatasync_group_commit_override {
|
|||||||
use std::sync::{Mutex, MutexGuard, PoisonError, RwLock};
|
use std::sync::{Mutex, MutexGuard, PoisonError, RwLock};
|
||||||
|
|
||||||
static OVERRIDE: RwLock<Option<bool>> = RwLock::new(None);
|
static OVERRIDE: RwLock<Option<bool>> = RwLock::new(None);
|
||||||
|
static WAIT_OVERRIDE_MICROS: RwLock<Option<u64>> = RwLock::new(None);
|
||||||
static SERIAL: Mutex<()> = Mutex::new(());
|
static SERIAL: Mutex<()> = Mutex::new(());
|
||||||
|
|
||||||
pub(crate) fn get() -> Option<bool> {
|
pub(crate) fn get() -> Option<bool> {
|
||||||
@@ -415,6 +429,7 @@ mod file_fdatasync_group_commit_override {
|
|||||||
impl Drop for OverrideGuard {
|
impl Drop for OverrideGuard {
|
||||||
fn drop(&mut self) {
|
fn drop(&mut self) {
|
||||||
*OVERRIDE.write().unwrap_or_else(PoisonError::into_inner) = None;
|
*OVERRIDE.write().unwrap_or_else(PoisonError::into_inner) = None;
|
||||||
|
*WAIT_OVERRIDE_MICROS.write().unwrap_or_else(PoisonError::into_inner) = None;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -423,6 +438,14 @@ mod file_fdatasync_group_commit_override {
|
|||||||
*OVERRIDE.write().unwrap_or_else(PoisonError::into_inner) = Some(enabled);
|
*OVERRIDE.write().unwrap_or_else(PoisonError::into_inner) = Some(enabled);
|
||||||
OverrideGuard { _serial: serial }
|
OverrideGuard { _serial: serial }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn set_wait_micros(wait_micros: u64) {
|
||||||
|
*WAIT_OVERRIDE_MICROS.write().unwrap_or_else(PoisonError::into_inner) = Some(wait_micros);
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn wait_micros() -> Option<u64> {
|
||||||
|
*WAIT_OVERRIDE_MICROS.read().unwrap_or_else(PoisonError::into_inner)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
@@ -430,6 +453,11 @@ pub(crate) fn set_file_fdatasync_group_commit_for_test(enabled: bool) -> file_fd
|
|||||||
file_fdatasync_group_commit_override::set(enabled)
|
file_fdatasync_group_commit_override::set(enabled)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
fn set_file_fdatasync_group_commit_wait_for_test(wait_micros: u64) {
|
||||||
|
file_fdatasync_group_commit_override::set_wait_micros(wait_micros);
|
||||||
|
}
|
||||||
|
|
||||||
fn file_fdatasync_group_commit_enabled() -> bool {
|
fn file_fdatasync_group_commit_enabled() -> bool {
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
if let Some(enabled) = file_fdatasync_group_commit_override::get() {
|
if let Some(enabled) = file_fdatasync_group_commit_override::get() {
|
||||||
@@ -439,6 +467,15 @@ fn file_fdatasync_group_commit_enabled() -> bool {
|
|||||||
*FILE_FDATASYNC_GROUP_COMMIT_ENABLED
|
*FILE_FDATASYNC_GROUP_COMMIT_ENABLED
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn file_fdatasync_group_commit_wait() -> Duration {
|
||||||
|
#[cfg(test)]
|
||||||
|
if let Some(wait_micros) = file_fdatasync_group_commit_override::wait_micros() {
|
||||||
|
return file_fdatasync_group_commit_wait_duration(wait_micros);
|
||||||
|
}
|
||||||
|
|
||||||
|
*FILE_FDATASYNC_GROUP_COMMIT_WAIT
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Clone, Eq, Hash, PartialEq)]
|
#[derive(Clone, Eq, Hash, PartialEq)]
|
||||||
struct DstDirFsyncGroupKey {
|
struct DstDirFsyncGroupKey {
|
||||||
canonical_path: PathBuf,
|
canonical_path: PathBuf,
|
||||||
@@ -934,6 +971,10 @@ async fn run_file_fdatasync_group_worker(group: Arc<FileFdatasyncGroup>) {
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
file_sync_probe::run_before_group_batch();
|
file_sync_probe::run_before_group_batch();
|
||||||
tokio::task::yield_now().await;
|
tokio::task::yield_now().await;
|
||||||
|
let wait = file_fdatasync_group_commit_wait();
|
||||||
|
if !wait.is_zero() {
|
||||||
|
tokio::time::sleep(wait).await;
|
||||||
|
}
|
||||||
let (batch, batch_file_count): (Vec<FileFdatasyncWaiter>, usize) = {
|
let (batch, batch_file_count): (Vec<FileFdatasyncWaiter>, usize) = {
|
||||||
let mut group_state = group.inner.lock();
|
let mut group_state = group.inner.lock();
|
||||||
let batch_file_count = group_state.pending_files;
|
let batch_file_count = group_state.pending_files;
|
||||||
@@ -6075,6 +6116,7 @@ mod tests {
|
|||||||
use std::sync::mpsc;
|
use std::sync::mpsc;
|
||||||
|
|
||||||
let _group_commit = set_file_fdatasync_group_commit_for_test(true);
|
let _group_commit = set_file_fdatasync_group_commit_for_test(true);
|
||||||
|
set_file_fdatasync_group_commit_wait_for_test(0);
|
||||||
clear_file_fdatasync_group_commit_for_test();
|
clear_file_fdatasync_group_commit_for_test();
|
||||||
let temp_dir = tempdir().expect("create temp dir");
|
let temp_dir = tempdir().expect("create temp dir");
|
||||||
let first_dir = temp_dir.path().join("first");
|
let first_dir = temp_dir.path().join("first");
|
||||||
@@ -6141,12 +6183,105 @@ mod tests {
|
|||||||
assert_eq!(file_fdatasync_group_commit_counts_for_test(), (0, 0, 0));
|
assert_eq!(file_fdatasync_group_commit_counts_for_test(), (0, 0, 0));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn file_fdatasync_group_commit_wait_duration_uses_default_and_cap() {
|
||||||
|
assert_eq!(DEFAULT_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS, 0);
|
||||||
|
assert_eq!(
|
||||||
|
file_fdatasync_group_commit_wait_duration(DEFAULT_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS),
|
||||||
|
Duration::ZERO
|
||||||
|
);
|
||||||
|
assert_eq!(file_fdatasync_group_commit_wait_duration(250), Duration::from_micros(250));
|
||||||
|
assert_eq!(
|
||||||
|
file_fdatasync_group_commit_wait_duration(MAX_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS),
|
||||||
|
Duration::from_micros(MAX_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS)
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
file_fdatasync_group_commit_wait_duration(u64::MAX),
|
||||||
|
Duration::from_micros(MAX_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "current_thread", start_paused = true)]
|
||||||
|
#[serial_test::serial(file_sync_probe)]
|
||||||
|
async fn file_fdatasync_group_commit_wait_budget_batches_late_follower() {
|
||||||
|
use std::sync::mpsc;
|
||||||
|
|
||||||
|
let _group_commit = set_file_fdatasync_group_commit_for_test(true);
|
||||||
|
let wait_budget_micros = 1_000;
|
||||||
|
let wait_budget = file_fdatasync_group_commit_wait_duration(wait_budget_micros);
|
||||||
|
set_file_fdatasync_group_commit_wait_for_test(wait_budget_micros);
|
||||||
|
clear_file_fdatasync_group_commit_for_test();
|
||||||
|
let temp_dir = tempdir().expect("create temp dir");
|
||||||
|
let first_dir = temp_dir.path().join("first");
|
||||||
|
let second_dir = temp_dir.path().join("second");
|
||||||
|
std::fs::create_dir(&first_dir).expect("create first dir");
|
||||||
|
std::fs::create_dir(&second_dir).expect("create second dir");
|
||||||
|
std::fs::write(first_dir.join("part.1"), b"first").expect("write first part");
|
||||||
|
std::fs::write(second_dir.join("part.1"), b"second").expect("write second part");
|
||||||
|
let _probe = file_sync_probe::set_blocking(temp_dir.path());
|
||||||
|
let (entered_tx, entered_rx) = mpsc::channel();
|
||||||
|
file_sync_probe::set_before_group_batch(move || {
|
||||||
|
entered_tx.send(()).expect("signal first file fdatasync group worker");
|
||||||
|
});
|
||||||
|
|
||||||
|
let limiter = file_sync_limiter();
|
||||||
|
let first_limiter = limiter.clone();
|
||||||
|
let first_path = first_dir.clone();
|
||||||
|
let first = tokio::spawn(async move { sync_dir_files_with_limiter(first_path, first_limiter).await });
|
||||||
|
tokio::task::spawn_blocking(move || entered_rx.recv_timeout(Duration::from_secs(30)))
|
||||||
|
.await
|
||||||
|
.expect("group worker hook waiter should run")
|
||||||
|
.expect("first file fdatasync group worker should start");
|
||||||
|
|
||||||
|
let second_limiter = limiter.clone();
|
||||||
|
let second_path = second_dir.clone();
|
||||||
|
let second = tokio::spawn(async move { sync_dir_files_with_limiter(second_path, second_limiter).await });
|
||||||
|
tokio::time::timeout(Duration::from_secs(30), async {
|
||||||
|
loop {
|
||||||
|
if file_fdatasync_group_commit_counts_for_test().1 == 2 {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
tokio::task::yield_now().await;
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("second waiter should enqueue during the configured wait budget");
|
||||||
|
tokio::time::advance(wait_budget).await;
|
||||||
|
tokio::task::yield_now().await;
|
||||||
|
file_sync_probe::wait_for_active(1).await;
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
file_sync_probe::group_batches(),
|
||||||
|
vec![2],
|
||||||
|
"configured wait budget should let a follower join the leader's batch"
|
||||||
|
);
|
||||||
|
file_sync_probe::release();
|
||||||
|
first
|
||||||
|
.await
|
||||||
|
.expect("join first wait-budget file sync")
|
||||||
|
.expect("first wait-budget file sync must succeed");
|
||||||
|
second
|
||||||
|
.await
|
||||||
|
.expect("join second wait-budget file sync")
|
||||||
|
.expect("second wait-budget file sync must succeed");
|
||||||
|
assert!(
|
||||||
|
fsync_dir_recorder::was_fsynced(&first_dir),
|
||||||
|
"first source directory must still be fsynced"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
fsync_dir_recorder::was_fsynced(&second_dir),
|
||||||
|
"second source directory must still be fsynced"
|
||||||
|
);
|
||||||
|
assert_eq!(file_fdatasync_group_commit_counts_for_test(), (0, 0, 0));
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||||
#[serial_test::serial(file_sync_probe)]
|
#[serial_test::serial(file_sync_probe)]
|
||||||
async fn file_fdatasync_group_commit_failure_fails_all_waiters_before_dir_fsync() {
|
async fn file_fdatasync_group_commit_failure_fails_all_waiters_before_dir_fsync() {
|
||||||
use std::sync::mpsc;
|
use std::sync::mpsc;
|
||||||
|
|
||||||
let _group_commit = set_file_fdatasync_group_commit_for_test(true);
|
let _group_commit = set_file_fdatasync_group_commit_for_test(true);
|
||||||
|
set_file_fdatasync_group_commit_wait_for_test(0);
|
||||||
clear_file_fdatasync_group_commit_for_test();
|
clear_file_fdatasync_group_commit_for_test();
|
||||||
let temp_dir = tempdir().expect("create temp dir");
|
let temp_dir = tempdir().expect("create temp dir");
|
||||||
let first_dir = temp_dir.path().join("first");
|
let first_dir = temp_dir.path().join("first");
|
||||||
|
|||||||
@@ -343,6 +343,23 @@ impl ECStore {
|
|||||||
let (decommission, rebalance) = tokio::join!(self.is_decommission_running(), self.is_rebalance_started());
|
let (decommission, rebalance) = tokio::join!(self.is_decommission_running(), self.is_rebalance_started());
|
||||||
decommission || rebalance
|
decommission || rebalance
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Returns whether scanner metadata may still be hidden by a local
|
||||||
|
/// data-movement state. Terminal failed/canceled decommission entries
|
||||||
|
/// remain suspended until an operator clears or retries them, so they are
|
||||||
|
/// a publication barrier even after the worker has stopped.
|
||||||
|
pub async fn scanner_data_usage_publication_blocked(&self) -> bool {
|
||||||
|
if self.scanner_data_movement_active().await {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
let pool_meta = self.pool_meta.read().await;
|
||||||
|
pool_meta.pools.iter().any(|pool| {
|
||||||
|
pool.decommission
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(|info| !info.queued && (info.failed || info.canceled))
|
||||||
|
})
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// impl Clone for ECStore {
|
// impl Clone for ECStore {
|
||||||
@@ -875,6 +892,7 @@ impl crate::storage_api_contracts::admin::StorageAdminApi for ECStore {
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
use crate::core::pools::{PoolDecommissionInfo, PoolStatus};
|
||||||
use crate::layout::endpoints::{Endpoints, PoolEndpoints, SetupType};
|
use crate::layout::endpoints::{Endpoints, PoolEndpoints, SetupType};
|
||||||
use crate::runtime::global::reset_local_disk_test_state;
|
use crate::runtime::global::reset_local_disk_test_state;
|
||||||
use crate::runtime::sources::{clear_local_disk_id_map_for_test, local_disk_path_by_id};
|
use crate::runtime::sources::{clear_local_disk_id_map_for_test, local_disk_path_by_id};
|
||||||
@@ -911,6 +929,72 @@ mod tests {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn scanner_data_usage_publication_blocks_active_and_unqueued_terminal_decommission() {
|
||||||
|
let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
|
||||||
|
let cases = [
|
||||||
|
(
|
||||||
|
"active",
|
||||||
|
PoolDecommissionInfo {
|
||||||
|
start_time: Some(OffsetDateTime::now_utc()),
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
true,
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"failed",
|
||||||
|
PoolDecommissionInfo {
|
||||||
|
failed: true,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
true,
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"canceled",
|
||||||
|
PoolDecommissionInfo {
|
||||||
|
canceled: true,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
true,
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"queued_failed",
|
||||||
|
PoolDecommissionInfo {
|
||||||
|
failed: true,
|
||||||
|
queued: true,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
false,
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"complete",
|
||||||
|
PoolDecommissionInfo {
|
||||||
|
complete: true,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
false,
|
||||||
|
),
|
||||||
|
("idle", PoolDecommissionInfo::default(), false),
|
||||||
|
];
|
||||||
|
|
||||||
|
for (name, decommission, expected) in cases {
|
||||||
|
*store.pool_meta.write().await = PoolMeta {
|
||||||
|
pools: vec![PoolStatus {
|
||||||
|
id: 0,
|
||||||
|
cmd_line: format!("scanner-publication-{name}"),
|
||||||
|
last_update: OffsetDateTime::now_utc(),
|
||||||
|
decommission: Some(decommission),
|
||||||
|
}],
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
assert_eq!(
|
||||||
|
store.scanner_data_usage_publication_blocked().await,
|
||||||
|
expected,
|
||||||
|
"unexpected scanner publication barrier state for {name}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// The object graph is the isolation carrier: two ECStore instances holding
|
// The object graph is the isolation carrier: two ECStore instances holding
|
||||||
// distinct contexts report independent erasure state through their real
|
// distinct contexts report independent erasure state through their real
|
||||||
// `&self` accessors — no cross-contamination.
|
// `&self` accessors — no cross-contamination.
|
||||||
|
|||||||
@@ -17,11 +17,11 @@
|
|||||||
//! All direct `rustfs_ecstore` facade imports used by tests in this crate
|
//! All direct `rustfs_ecstore` facade imports used by tests in this crate
|
||||||
//! must go through this module (architecture migration rule:
|
//! must go through this module (architecture migration rule:
|
||||||
//! `check_architecture_migration_rules.sh`). Keep the surface minimal —
|
//! `check_architecture_migration_rules.sh`). Keep the surface minimal —
|
||||||
//! only what the tests actually need to build a temp-disk ECStore fixture
|
//! only what the tests actually need to run storage-backed IAM scenarios.
|
||||||
//! and to flip the erasure setup type for lock-quorum fault injection.
|
|
||||||
|
|
||||||
#[allow(unused_imports)]
|
#[allow(unused_imports)]
|
||||||
pub(crate) mod fixture {
|
pub(crate) mod fixture {
|
||||||
|
pub(crate) use rustfs_ecstore::api::bucket::migration::try_migrate_iam_config;
|
||||||
pub(crate) use rustfs_ecstore::api::layout::SetupType;
|
pub(crate) use rustfs_ecstore::api::layout::SetupType;
|
||||||
|
|
||||||
// `update_erasure_type` is a write-side global facade entry. Its use is
|
// `update_erasure_type` is a write-side global facade entry. Its use is
|
||||||
|
|||||||
@@ -0,0 +1,182 @@
|
|||||||
|
// Copyright 2024 RustFS Team
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
|
||||||
|
mod ecstore_test_compat;
|
||||||
|
|
||||||
|
use ecstore_test_compat::fixture::try_migrate_iam_config;
|
||||||
|
use rustfs_credentials::{get_global_action_cred, init_global_action_credentials};
|
||||||
|
use rustfs_iam::store::object::{
|
||||||
|
IAM_CONFIG_POLICY_DB_SERVICE_ACCOUNTS_PREFIX, IAM_CONFIG_POLICY_DB_USERS_PREFIX, IAM_CONFIG_SERVICE_ACCOUNTS_PREFIX,
|
||||||
|
IAM_CONFIG_USERS_PREFIX, ObjectStore,
|
||||||
|
};
|
||||||
|
use rustfs_iam::store::{Store, UserType};
|
||||||
|
use rustfs_iam::utils::generate_jwt;
|
||||||
|
use rustfs_policy::auth::UserIdentity;
|
||||||
|
use serde_json::{Value, json};
|
||||||
|
use std::collections::HashMap;
|
||||||
|
|
||||||
|
const LEGACY_META_BUCKET: &str = ".minio.sys";
|
||||||
|
const REGULAR_USER: &str = "minio-user";
|
||||||
|
const SERVICE_ACCOUNT: &str = "minio-service-account";
|
||||||
|
|
||||||
|
async fn seed_legacy_iam_object(env: &rustfs_test_utils::TestECStoreEnv, path: &str, value: &Value) {
|
||||||
|
env.put_object_bytes(
|
||||||
|
LEGACY_META_BUCKET,
|
||||||
|
path,
|
||||||
|
serde_json::to_vec(value).expect("legacy IAM object must serialize"),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
|
||||||
|
fn assert_identity_fields(actual: &UserIdentity, expected: &Value) {
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::to_value(actual).expect("loaded identity must serialize"),
|
||||||
|
*expected,
|
||||||
|
"migration must preserve every credential field except expiration",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn assert_identity_survives(
|
||||||
|
store: &ObjectStore,
|
||||||
|
identity_path: &str,
|
||||||
|
name: &str,
|
||||||
|
user_type: UserType,
|
||||||
|
source: &Value,
|
||||||
|
expected_policy: &Value,
|
||||||
|
) {
|
||||||
|
let mut expected = source.clone();
|
||||||
|
expected["credentials"]["expiration"] = Value::Null;
|
||||||
|
|
||||||
|
let persisted: UserIdentity = store
|
||||||
|
.load_iam_config(identity_path)
|
||||||
|
.await
|
||||||
|
.expect("migrated identity must be persisted");
|
||||||
|
assert_identity_fields(&persisted, &expected);
|
||||||
|
|
||||||
|
for _ in 0..2 {
|
||||||
|
let actual = store
|
||||||
|
.load_user_identity(name, user_type)
|
||||||
|
.await
|
||||||
|
.expect("migrated permanent identity must remain loadable");
|
||||||
|
assert_identity_fields(&actual, &expected);
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut mappings = HashMap::new();
|
||||||
|
store
|
||||||
|
.load_mapped_policy(name, user_type, false, &mut mappings)
|
||||||
|
.await
|
||||||
|
.expect("loading the identity must not delete its policy mapping");
|
||||||
|
let actual_policy = mappings.get(name).expect("migrated policy mapping must exist");
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::to_value(actual_policy).expect("loaded policy mapping must serialize"),
|
||||||
|
*expected_policy,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "multi_thread")]
|
||||||
|
async fn minio_permanent_identities_survive_migration_and_repeated_iam_loads() {
|
||||||
|
if get_global_action_cred().is_none() {
|
||||||
|
init_global_action_credentials(Some("MINIOMIGRATIONROOT".to_string()), Some("minio-migration-root-secret".to_string()))
|
||||||
|
.expect("root credentials must initialize for JWT validation");
|
||||||
|
}
|
||||||
|
|
||||||
|
let temp_dir = tempfile::TempDir::with_prefix("rustfs_minio_iam_migration_").expect("temp directory must be created");
|
||||||
|
let env = rustfs_test_utils::TestECStoreEnv::builder()
|
||||||
|
.base_dir(temp_dir.path())
|
||||||
|
.init_bucket_metadata(false)
|
||||||
|
.build()
|
||||||
|
.await;
|
||||||
|
for disk_path in &env.disk_paths {
|
||||||
|
tokio::fs::create_dir_all(disk_path.join(LEGACY_META_BUCKET))
|
||||||
|
.await
|
||||||
|
.expect("legacy metadata volume must be created");
|
||||||
|
}
|
||||||
|
|
||||||
|
let regular_source = json!({
|
||||||
|
"version": 1,
|
||||||
|
"credentials": {
|
||||||
|
"accessKey": REGULAR_USER,
|
||||||
|
"secretKey": "regular-user-secret",
|
||||||
|
"sessionToken": "",
|
||||||
|
"expiration": "0001-01-01T00:00:00Z",
|
||||||
|
"status": "on",
|
||||||
|
"parentUser": "regular-parent",
|
||||||
|
"groups": ["engineering", "operations"],
|
||||||
|
"claims": {"tenant": "alpha"},
|
||||||
|
"name": "MinIO regular user",
|
||||||
|
"description": "migrated regular identity"
|
||||||
|
},
|
||||||
|
"updatedAt": "2025-03-07T12:00:00Z"
|
||||||
|
});
|
||||||
|
let service_claims = json!({"sa-policy": "inherited-policy", "tenant": "alpha"});
|
||||||
|
let service_secret = "service-account-secret";
|
||||||
|
let service_source = json!({
|
||||||
|
"version": 1,
|
||||||
|
"credentials": {
|
||||||
|
"accessKey": SERVICE_ACCOUNT,
|
||||||
|
"secretKey": service_secret,
|
||||||
|
"sessionToken": generate_jwt(&service_claims, service_secret).expect("service-account JWT must be generated"),
|
||||||
|
"expiration": "1970-01-01T00:00:00Z",
|
||||||
|
"status": "on",
|
||||||
|
"parentUser": REGULAR_USER,
|
||||||
|
"groups": ["service-accounts"],
|
||||||
|
"claims": service_claims,
|
||||||
|
"name": "MinIO service account",
|
||||||
|
"description": "migrated service identity"
|
||||||
|
},
|
||||||
|
"updatedAt": "2025-03-07T12:00:00Z"
|
||||||
|
});
|
||||||
|
let regular_policy_source = json!({"version": 1, "policy": "readwrite", "updatedAt": "2025-03-07T12:00:00Z"});
|
||||||
|
let service_policy_source = json!({"version": 1, "policy": "readonly", "updatedAt": "2025-03-07T12:00:00Z"});
|
||||||
|
|
||||||
|
let regular_identity_path = format!("{}{REGULAR_USER}/identity.json", IAM_CONFIG_USERS_PREFIX.as_str());
|
||||||
|
let service_identity_path = format!("{}{SERVICE_ACCOUNT}/identity.json", IAM_CONFIG_SERVICE_ACCOUNTS_PREFIX.as_str());
|
||||||
|
|
||||||
|
seed_legacy_iam_object(&env, ®ular_identity_path, ®ular_source).await;
|
||||||
|
seed_legacy_iam_object(&env, &service_identity_path, &service_source).await;
|
||||||
|
seed_legacy_iam_object(
|
||||||
|
&env,
|
||||||
|
&format!("{}{REGULAR_USER}.json", IAM_CONFIG_POLICY_DB_USERS_PREFIX.as_str()),
|
||||||
|
®ular_policy_source,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
seed_legacy_iam_object(
|
||||||
|
&env,
|
||||||
|
&format!("{}{SERVICE_ACCOUNT}.json", IAM_CONFIG_POLICY_DB_SERVICE_ACCOUNTS_PREFIX.as_str()),
|
||||||
|
&service_policy_source,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
try_migrate_iam_config(env.ecstore.clone(), None).await;
|
||||||
|
|
||||||
|
let store = ObjectStore::new(env.ecstore);
|
||||||
|
assert_identity_survives(
|
||||||
|
&store,
|
||||||
|
®ular_identity_path,
|
||||||
|
REGULAR_USER,
|
||||||
|
UserType::Reg,
|
||||||
|
®ular_source,
|
||||||
|
®ular_policy_source,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_identity_survives(
|
||||||
|
&store,
|
||||||
|
&service_identity_path,
|
||||||
|
SERVICE_ACCOUNT,
|
||||||
|
UserType::Svc,
|
||||||
|
&service_source,
|
||||||
|
&service_policy_source,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
@@ -69,7 +69,7 @@ uuid = { workspace = true, features = ["v4", "serde", "fast-rng", "macro-diagnos
|
|||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
metrics-util = { workspace = true, features = ["debugging"] }
|
metrics-util = { workspace = true, features = ["debugging"] }
|
||||||
proptest = "1"
|
proptest = "1"
|
||||||
serial_test.workspace = true
|
serial_test = { workspace = true }
|
||||||
temp-env.workspace = true
|
temp-env.workspace = true
|
||||||
tokio = { workspace = true, features = ["macros", "fs", "rt-multi-thread"] }
|
tokio = { workspace = true, features = ["macros", "fs", "rt-multi-thread"] }
|
||||||
|
|
||||||
|
|||||||
@@ -1564,7 +1564,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn abort_incomplete_multipart_upload_due_accepts_zero_days() {
|
async fn abort_incomplete_multipart_upload_due_accepts_zero_days() {
|
||||||
let initiated = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
|
let initiated = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
|
||||||
let lc = BucketLifecycleConfiguration {
|
let lc = BucketLifecycleConfiguration {
|
||||||
@@ -1625,7 +1624,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn predict_expiration_selects_closest_expiry_for_put_object() {
|
async fn predict_expiration_selects_closest_expiry_for_put_object() {
|
||||||
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
|
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
|
||||||
let lc = BucketLifecycleConfiguration {
|
let lc = BucketLifecycleConfiguration {
|
||||||
@@ -1872,7 +1870,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn empty_transition_vectors_are_not_active_or_due() {
|
async fn empty_transition_vectors_are_not_active_or_due() {
|
||||||
let lc = BucketLifecycleConfiguration {
|
let lc = BucketLifecycleConfiguration {
|
||||||
expiry_updated_at: None,
|
expiry_updated_at: None,
|
||||||
@@ -1938,7 +1935,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn eval_inner_keeps_latest_object_before_days_due() {
|
async fn eval_inner_keeps_latest_object_before_days_due() {
|
||||||
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
|
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
|
||||||
let lc = BucketLifecycleConfiguration {
|
let lc = BucketLifecycleConfiguration {
|
||||||
@@ -1972,7 +1968,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn eval_inner_transitions_latest_object_after_days_due() {
|
async fn eval_inner_transitions_latest_object_after_days_due() {
|
||||||
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
|
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
|
||||||
let lc = BucketLifecycleConfiguration {
|
let lc = BucketLifecycleConfiguration {
|
||||||
@@ -2010,7 +2005,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn eval_inner_transitions_latest_object_after_date_due() {
|
async fn eval_inner_transitions_latest_object_after_date_due() {
|
||||||
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
|
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
|
||||||
let transition_date = base_time - Duration::days(1);
|
let transition_date = base_time - Duration::days(1);
|
||||||
@@ -2050,7 +2044,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn eval_inner_selects_earliest_due_among_multiple_past_due_events() {
|
async fn eval_inner_selects_earliest_due_among_multiple_past_due_events() {
|
||||||
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
|
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
|
||||||
// Two enabled rules both yield a past-due DeleteAction and a third yields a
|
// Two enabled rules both yield a past-due DeleteAction and a third yields a
|
||||||
@@ -2164,7 +2157,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn eval_inner_expires_noncurrent_version_after_due() {
|
async fn eval_inner_expires_noncurrent_version_after_due() {
|
||||||
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
|
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
|
||||||
let lc = BucketLifecycleConfiguration {
|
let lc = BucketLifecycleConfiguration {
|
||||||
@@ -2202,7 +2194,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn eval_inner_skips_noncurrent_expiration_without_successor() {
|
async fn eval_inner_skips_noncurrent_expiration_without_successor() {
|
||||||
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).expect("valid fixed test timestamp");
|
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).expect("valid fixed test timestamp");
|
||||||
let lc = BucketLifecycleConfiguration {
|
let lc = BucketLifecycleConfiguration {
|
||||||
@@ -2238,7 +2229,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn eval_inner_missing_successor_does_not_skip_noncurrent_transition() {
|
async fn eval_inner_missing_successor_does_not_skip_noncurrent_transition() {
|
||||||
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).expect("valid fixed test timestamp");
|
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).expect("valid fixed test timestamp");
|
||||||
let lc = BucketLifecycleConfiguration {
|
let lc = BucketLifecycleConfiguration {
|
||||||
@@ -2281,7 +2271,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn eval_inner_noncurrent_expiration_one_day_respects_due_boundary() {
|
async fn eval_inner_noncurrent_expiration_one_day_respects_due_boundary() {
|
||||||
let successor_time = datetime!(2025-06-15 12:00:00 UTC);
|
let successor_time = datetime!(2025-06-15 12:00:00 UTC);
|
||||||
let due = expected_expiry_time(successor_time, 1);
|
let due = expected_expiry_time(successor_time, 1);
|
||||||
@@ -2323,7 +2312,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn eval_inner_expires_noncurrent_version_immediately_when_zero_days() {
|
async fn eval_inner_expires_noncurrent_version_immediately_when_zero_days() {
|
||||||
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
|
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
|
||||||
let lc = BucketLifecycleConfiguration {
|
let lc = BucketLifecycleConfiguration {
|
||||||
@@ -2361,7 +2349,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn eval_inner_transitions_noncurrent_version_after_due() {
|
async fn eval_inner_transitions_noncurrent_version_after_due() {
|
||||||
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
|
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
|
||||||
let lc = BucketLifecycleConfiguration {
|
let lc = BucketLifecycleConfiguration {
|
||||||
@@ -2437,7 +2424,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn evaluator_honors_newer_noncurrent_versions_retention_count() {
|
async fn evaluator_honors_newer_noncurrent_versions_retention_count() {
|
||||||
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
|
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
|
||||||
let lc = Arc::new(BucketLifecycleConfiguration {
|
let lc = Arc::new(BucketLifecycleConfiguration {
|
||||||
@@ -2726,7 +2712,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn expired_object_delete_marker_ignores_marker_with_noncurrent_versions_present() {
|
async fn expired_object_delete_marker_ignores_marker_with_noncurrent_versions_present() {
|
||||||
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
|
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
|
||||||
let lc = BucketLifecycleConfiguration {
|
let lc = BucketLifecycleConfiguration {
|
||||||
@@ -2803,7 +2788,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn expired_object_delete_marker_deletes_only_delete_marker_immediately() {
|
async fn expired_object_delete_marker_deletes_only_delete_marker_immediately() {
|
||||||
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
|
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
|
||||||
let lc = BucketLifecycleConfiguration {
|
let lc = BucketLifecycleConfiguration {
|
||||||
@@ -2881,7 +2865,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn expiration_days_deletes_only_expired_delete_marker_when_due() {
|
async fn expiration_days_deletes_only_expired_delete_marker_when_due() {
|
||||||
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
|
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
|
||||||
let lc = BucketLifecycleConfiguration {
|
let lc = BucketLifecycleConfiguration {
|
||||||
@@ -2932,7 +2915,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn expiration_days_uses_earliest_due_rule_for_expired_delete_marker() {
|
async fn expiration_days_uses_earliest_due_rule_for_expired_delete_marker() {
|
||||||
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
|
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
|
||||||
let make_rule = |id: &str, days| LifecycleRule {
|
let make_rule = |id: &str, days| LifecycleRule {
|
||||||
@@ -3263,7 +3245,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn del_marker_expiration_deletes_marker_and_older_versions_when_due() {
|
async fn del_marker_expiration_deletes_marker_and_older_versions_when_due() {
|
||||||
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).expect("fixed timestamp should be valid");
|
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).expect("fixed timestamp should be valid");
|
||||||
let lc = BucketLifecycleConfiguration {
|
let lc = BucketLifecycleConfiguration {
|
||||||
@@ -3303,7 +3284,6 @@ mod tests {
|
|||||||
// --- TASK-003 tests: Round up to next UTC processing boundary ---
|
// --- TASK-003 tests: Round up to next UTC processing boundary ---
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn expected_expiry_time_rounds_up_to_next_midnight_utc() {
|
fn expected_expiry_time_rounds_up_to_next_midnight_utc() {
|
||||||
with_default_ilm_process_time(|| {
|
with_default_ilm_process_time(|| {
|
||||||
// Object created at 2025-01-15T10:30:45Z, expire in 30 days
|
// Object created at 2025-01-15T10:30:45Z, expire in 30 days
|
||||||
@@ -3319,7 +3299,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn expected_expiry_time_immediate_expiry_returns_epoch() {
|
fn expected_expiry_time_immediate_expiry_returns_epoch() {
|
||||||
with_default_ilm_process_time(|| {
|
with_default_ilm_process_time(|| {
|
||||||
let mod_time = datetime!(2025-06-01 12:00:00 UTC);
|
let mod_time = datetime!(2025-06-01 12:00:00 UTC);
|
||||||
@@ -3329,7 +3308,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn expected_expiry_time_preserves_exact_midnight_boundary() {
|
fn expected_expiry_time_preserves_exact_midnight_boundary() {
|
||||||
with_default_ilm_process_time(|| {
|
with_default_ilm_process_time(|| {
|
||||||
let mod_time = datetime!(2025-03-01 00:00:00 UTC);
|
let mod_time = datetime!(2025-03-01 00:00:00 UTC);
|
||||||
@@ -3339,7 +3317,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn expected_expiry_time_rounds_end_of_day_to_following_midnight() {
|
fn expected_expiry_time_rounds_end_of_day_to_following_midnight() {
|
||||||
with_default_ilm_process_time(|| {
|
with_default_ilm_process_time(|| {
|
||||||
let mod_time = datetime!(2025-06-15 23:59:59 UTC);
|
let mod_time = datetime!(2025-06-15 23:59:59 UTC);
|
||||||
@@ -3349,7 +3326,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn expected_expiry_time_uses_canonical_process_time_boundary() {
|
fn expected_expiry_time_uses_canonical_process_time_boundary() {
|
||||||
let mod_time = datetime!(2025-01-15 10:30:45 UTC);
|
let mod_time = datetime!(2025-01-15 10:30:45 UTC);
|
||||||
|
|
||||||
@@ -3362,7 +3338,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn expected_expiry_time_uses_deprecated_process_time_alias() {
|
fn expected_expiry_time_uses_deprecated_process_time_alias() {
|
||||||
let mod_time = datetime!(2025-01-15 10:30:45 UTC);
|
let mod_time = datetime!(2025-01-15 10:30:45 UTC);
|
||||||
|
|
||||||
@@ -3375,7 +3350,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn expected_expiry_time_uses_default_boundary_when_process_time_is_zero_or_invalid() {
|
fn expected_expiry_time_uses_default_boundary_when_process_time_is_zero_or_invalid() {
|
||||||
let mod_time = datetime!(2025-01-15 10:30:45 UTC);
|
let mod_time = datetime!(2025-01-15 10:30:45 UTC);
|
||||||
|
|
||||||
@@ -3398,7 +3372,6 @@ mod tests {
|
|||||||
|
|
||||||
// (a) Default path (env unset) is byte-identical: one day == 86400s.
|
// (a) Default path (env unset) is byte-identical: one day == 86400s.
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn ilm_day_secs_defaults_to_86400_when_unset() {
|
fn ilm_day_secs_defaults_to_86400_when_unset() {
|
||||||
temp_env::with_var_unset(ENV_ILM_DEBUG_DAY_SECS, || {
|
temp_env::with_var_unset(ENV_ILM_DEBUG_DAY_SECS, || {
|
||||||
assert_eq!(ilm_day_secs(), DEFAULT_ILM_DAY_SECS);
|
assert_eq!(ilm_day_secs(), DEFAULT_ILM_DAY_SECS);
|
||||||
@@ -3427,7 +3400,6 @@ mod tests {
|
|||||||
|
|
||||||
// (b) End-to-end env read scales the day length.
|
// (b) End-to-end env read scales the day length.
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn ilm_day_secs_scales_when_env_set() {
|
fn ilm_day_secs_scales_when_env_set() {
|
||||||
temp_env::with_var(ENV_ILM_DEBUG_DAY_SECS, Some("2"), || {
|
temp_env::with_var(ENV_ILM_DEBUG_DAY_SECS, Some("2"), || {
|
||||||
assert_eq!(ilm_day_secs(), 2);
|
assert_eq!(ilm_day_secs(), 2);
|
||||||
@@ -3436,7 +3408,6 @@ mod tests {
|
|||||||
|
|
||||||
// (c) Invalid env value falls back to 86400.
|
// (c) Invalid env value falls back to 86400.
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn ilm_day_secs_falls_back_on_invalid_env() {
|
fn ilm_day_secs_falls_back_on_invalid_env() {
|
||||||
temp_env::with_var(ENV_ILM_DEBUG_DAY_SECS, Some("bogus"), || {
|
temp_env::with_var(ENV_ILM_DEBUG_DAY_SECS, Some("bogus"), || {
|
||||||
assert_eq!(ilm_day_secs(), DEFAULT_ILM_DAY_SECS);
|
assert_eq!(ilm_day_secs(), DEFAULT_ILM_DAY_SECS);
|
||||||
@@ -3449,7 +3420,6 @@ mod tests {
|
|||||||
// Deadline math scales: with a 1s day and PROCESS_TIME unset, a Days=1 rule is
|
// Deadline math scales: with a 1s day and PROCESS_TIME unset, a Days=1 rule is
|
||||||
// due 1s after mod_time (rounded up to the next 1s boundary => same instant).
|
// due 1s after mod_time (rounded up to the next 1s boundary => same instant).
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn expected_expiry_time_scales_with_debug_day_secs() {
|
fn expected_expiry_time_scales_with_debug_day_secs() {
|
||||||
let mod_time = datetime!(2025-01-15 10:30:45 UTC);
|
let mod_time = datetime!(2025-01-15 10:30:45 UTC);
|
||||||
temp_env::with_var(ENV_ILM_DEBUG_DAY_SECS, Some("1"), || {
|
temp_env::with_var(ENV_ILM_DEBUG_DAY_SECS, Some("1"), || {
|
||||||
@@ -3465,7 +3435,6 @@ mod tests {
|
|||||||
|
|
||||||
// days == 0 still yields the immediate-expiry sentinel regardless of the switch.
|
// days == 0 still yields the immediate-expiry sentinel regardless of the switch.
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn expected_expiry_time_zero_days_ignores_debug_day_secs() {
|
fn expected_expiry_time_zero_days_ignores_debug_day_secs() {
|
||||||
let mod_time = datetime!(2025-06-01 12:00:00 UTC);
|
let mod_time = datetime!(2025-06-01 12:00:00 UTC);
|
||||||
temp_env::with_var(ENV_ILM_DEBUG_DAY_SECS, Some("2"), || {
|
temp_env::with_var(ENV_ILM_DEBUG_DAY_SECS, Some("2"), || {
|
||||||
@@ -3476,7 +3445,6 @@ mod tests {
|
|||||||
// (③) Interaction with an explicit RUSTFS_ILM_PROCESS_TIME: the deadline offset
|
// (③) Interaction with an explicit RUSTFS_ILM_PROCESS_TIME: the deadline offset
|
||||||
// uses the accelerated day length, but the rounding boundary honors PROCESS_TIME.
|
// uses the accelerated day length, but the rounding boundary honors PROCESS_TIME.
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn expected_expiry_time_debug_day_secs_respects_explicit_process_time() {
|
fn expected_expiry_time_debug_day_secs_respects_explicit_process_time() {
|
||||||
let mod_time = datetime!(2025-01-15 10:30:00 UTC);
|
let mod_time = datetime!(2025-01-15 10:30:00 UTC);
|
||||||
// day == 10s, but round up to the next 60s (PROCESS_TIME) boundary.
|
// day == 10s, but round up to the next 60s (PROCESS_TIME) boundary.
|
||||||
@@ -3493,7 +3461,6 @@ mod tests {
|
|||||||
|
|
||||||
// (③) With the switch unset, an explicit PROCESS_TIME behaves exactly as before.
|
// (③) With the switch unset, an explicit PROCESS_TIME behaves exactly as before.
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn expected_expiry_time_unset_debug_day_secs_matches_legacy_process_time() {
|
fn expected_expiry_time_unset_debug_day_secs_matches_legacy_process_time() {
|
||||||
let mod_time = datetime!(2025-01-15 10:30:45 UTC);
|
let mod_time = datetime!(2025-01-15 10:30:45 UTC);
|
||||||
temp_env::with_var_unset(ENV_ILM_DEBUG_DAY_SECS, || {
|
temp_env::with_var_unset(ENV_ILM_DEBUG_DAY_SECS, || {
|
||||||
@@ -3521,7 +3488,6 @@ mod tests {
|
|||||||
|
|
||||||
// The abort-incomplete-multipart deadline path also scales through the switch.
|
// The abort-incomplete-multipart deadline path also scales through the switch.
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn abort_incomplete_multipart_due_scales_with_debug_day_secs() {
|
fn abort_incomplete_multipart_due_scales_with_debug_day_secs() {
|
||||||
use s3s::dto::AbortIncompleteMultipartUpload;
|
use s3s::dto::AbortIncompleteMultipartUpload;
|
||||||
let initiated = datetime!(2025-01-15 10:30:45 UTC);
|
let initiated = datetime!(2025-01-15 10:30:45 UTC);
|
||||||
@@ -3566,7 +3532,6 @@ mod tests {
|
|||||||
// (⑤ evaluator seam) A Days=1 rule fires under RUSTFS_ILM_DEBUG_DAY_SECS=1 once
|
// (⑤ evaluator seam) A Days=1 rule fires under RUSTFS_ILM_DEBUG_DAY_SECS=1 once
|
||||||
// `now` advances a few seconds past a mod_time only ~seconds in the past.
|
// `now` advances a few seconds past a mod_time only ~seconds in the past.
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn eval_inner_expires_days_one_rule_under_debug_day_secs() {
|
fn eval_inner_expires_days_one_rule_under_debug_day_secs() {
|
||||||
let lc = BucketLifecycleConfiguration {
|
let lc = BucketLifecycleConfiguration {
|
||||||
expiry_updated_at: None,
|
expiry_updated_at: None,
|
||||||
@@ -3615,7 +3580,6 @@ mod tests {
|
|||||||
|
|
||||||
// Absolute Date-based rules must NOT scale with the switch (regression guard).
|
// Absolute Date-based rules must NOT scale with the switch (regression guard).
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn eval_inner_date_rule_ignores_debug_day_secs() {
|
fn eval_inner_date_rule_ignores_debug_day_secs() {
|
||||||
let expiry_date = datetime!(2025-06-01 00:00:00 UTC);
|
let expiry_date = datetime!(2025-06-01 00:00:00 UTC);
|
||||||
let lc = BucketLifecycleConfiguration {
|
let lc = BucketLifecycleConfiguration {
|
||||||
@@ -3873,7 +3837,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn eval_inner_triggers_delete_all_versions_when_expired_object_all_versions_set() {
|
async fn eval_inner_triggers_delete_all_versions_when_expired_object_all_versions_set() {
|
||||||
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
|
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
|
||||||
let lc = BucketLifecycleConfiguration {
|
let lc = BucketLifecycleConfiguration {
|
||||||
@@ -3912,7 +3875,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn expired_object_all_versions_does_not_apply_to_current_delete_marker() {
|
async fn expired_object_all_versions_does_not_apply_to_current_delete_marker() {
|
||||||
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).expect("fixed timestamp should be valid");
|
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).expect("fixed timestamp should be valid");
|
||||||
let lc = BucketLifecycleConfiguration {
|
let lc = BucketLifecycleConfiguration {
|
||||||
@@ -3942,7 +3904,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn eval_inner_uses_delete_action_when_all_versions_not_set() {
|
async fn eval_inner_uses_delete_action_when_all_versions_not_set() {
|
||||||
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
|
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
|
||||||
let lc = BucketLifecycleConfiguration {
|
let lc = BucketLifecycleConfiguration {
|
||||||
@@ -4061,7 +4022,6 @@ mod tests {
|
|||||||
use super::*;
|
use super::*;
|
||||||
use proptest::prelude::*;
|
use proptest::prelude::*;
|
||||||
use s3s::dto::{NoncurrentVersionExpiration, Tag};
|
use s3s::dto::{NoncurrentVersionExpiration, Tag};
|
||||||
use serial_test::serial;
|
|
||||||
|
|
||||||
const DAY_SECS: i64 = 86400;
|
const DAY_SECS: i64 = 86400;
|
||||||
|
|
||||||
@@ -4292,7 +4252,6 @@ mod tests {
|
|||||||
/// combination, and must be deterministic: the same input
|
/// combination, and must be deterministic: the same input
|
||||||
/// evaluated twice yields an identical event.
|
/// evaluated twice yields an identical event.
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn eval_inner_never_panics_and_is_deterministic(
|
fn eval_inner_never_panics_and_is_deterministic(
|
||||||
rules in prop::collection::vec(arb_rule(), 0..4),
|
rules in prop::collection::vec(arb_rule(), 0..4),
|
||||||
obj in arb_object_opts(),
|
obj in arb_object_opts(),
|
||||||
@@ -4432,7 +4391,6 @@ mod tests {
|
|||||||
/// candidate set — earliest due wins, ties prefer delete-class —
|
/// candidate set — earliest due wins, ties prefer delete-class —
|
||||||
/// and must be `NoneAction` exactly when that set is empty.
|
/// and must be `NoneAction` exactly when that set is empty.
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn eval_inner_winner_matches_selection_oracle(
|
fn eval_inner_winner_matches_selection_oracle(
|
||||||
rules in prop::collection::vec(arb_selection_rule(), 0..5),
|
rules in prop::collection::vec(arb_selection_rule(), 0..5),
|
||||||
mod_off in 0i64..(2 * DAY_SECS),
|
mod_off in 0i64..(2 * DAY_SECS),
|
||||||
@@ -4486,7 +4444,6 @@ mod tests {
|
|||||||
/// non-decreasing in `days` (days == 0 maps to UNIX_EPOCH, below
|
/// non-decreasing in `days` (days == 0 maps to UNIX_EPOCH, below
|
||||||
/// any post-1970 deadline).
|
/// any post-1970 deadline).
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn expected_expiry_time_is_monotonic_in_days(
|
fn expected_expiry_time_is_monotonic_in_days(
|
||||||
mod_off in 0i64..(3650 * DAY_SECS),
|
mod_off in 0i64..(3650 * DAY_SECS),
|
||||||
d1 in 0i32..2000,
|
d1 in 0i32..2000,
|
||||||
@@ -4508,7 +4465,6 @@ mod tests {
|
|||||||
/// to the next whole-day boundary: the result is day-aligned, not
|
/// to the next whole-day boundary: the result is day-aligned, not
|
||||||
/// before `mod_time + days`, and less than one boundary beyond it.
|
/// before `mod_time + days`, and less than one boundary beyond it.
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn expected_expiry_time_lands_on_default_day_boundary(
|
fn expected_expiry_time_lands_on_default_day_boundary(
|
||||||
mod_off in 0i64..(3650 * DAY_SECS),
|
mod_off in 0i64..(3650 * DAY_SECS),
|
||||||
days in 1i32..2000,
|
days in 1i32..2000,
|
||||||
@@ -4526,7 +4482,6 @@ mod tests {
|
|||||||
/// to that boundary instead: aligned to it, never early, and less
|
/// to that boundary instead: aligned to it, never early, and less
|
||||||
/// than one boundary late.
|
/// than one boundary late.
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn expected_expiry_time_lands_on_explicit_process_boundary(
|
fn expected_expiry_time_lands_on_explicit_process_boundary(
|
||||||
mod_off in 0i64..(365 * DAY_SECS),
|
mod_off in 0i64..(365 * DAY_SECS),
|
||||||
days in 1i32..400,
|
days in 1i32..400,
|
||||||
|
|||||||
@@ -73,7 +73,6 @@ walkdir = { workspace = true }
|
|||||||
|
|
||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
criterion = { workspace = true, features = ["html_reports"] }
|
criterion = { workspace = true, features = ["html_reports"] }
|
||||||
serial_test = { workspace = true }
|
|
||||||
temp-env = { workspace = true, features = ["async_closure"] }
|
temp-env = { workspace = true, features = ["async_closure"] }
|
||||||
tempfile = { workspace = true }
|
tempfile = { workspace = true }
|
||||||
tokio = { workspace = true, features = ["test-util", "macros", "fs", "rt-multi-thread"] }
|
tokio = { workspace = true, features = ["test-util", "macros", "fs", "rt-multi-thread"] }
|
||||||
|
|||||||
@@ -1484,7 +1484,6 @@ mod tests {
|
|||||||
ENV_CAPACITY_SAMPLE_RATE, ENV_CAPACITY_STAT_TIMEOUT, ENV_CAPACITY_WRITE_FREQUENCY_THRESHOLD,
|
ENV_CAPACITY_SAMPLE_RATE, ENV_CAPACITY_STAT_TIMEOUT, ENV_CAPACITY_WRITE_FREQUENCY_THRESHOLD,
|
||||||
ENV_CAPACITY_WRITE_TRIGGER_DELAY,
|
ENV_CAPACITY_WRITE_TRIGGER_DELAY,
|
||||||
};
|
};
|
||||||
use serial_test::serial;
|
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||||
|
|
||||||
@@ -1669,7 +1668,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn test_config_getter_defaults() {
|
fn test_config_getter_defaults() {
|
||||||
for (env_var, getter, default, _, _) in config_getter_cases() {
|
for (env_var, getter, default, _, _) in config_getter_cases() {
|
||||||
temp_env::with_var(env_var, None::<&str>, || {
|
temp_env::with_var(env_var, None::<&str>, || {
|
||||||
@@ -1679,7 +1677,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn test_config_getter_env_overrides() {
|
fn test_config_getter_env_overrides() {
|
||||||
for (env_var, getter, _, override_value, expected) in config_getter_cases() {
|
for (env_var, getter, _, override_value, expected) in config_getter_cases() {
|
||||||
temp_env::with_var(env_var, Some(override_value), || {
|
temp_env::with_var(env_var, Some(override_value), || {
|
||||||
@@ -1689,7 +1686,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn test_zero_env_values_clamp_to_defaults() {
|
fn test_zero_env_values_clamp_to_defaults() {
|
||||||
// A zero threshold makes small disks report 0 bytes; a zero timeout
|
// A zero threshold makes small disks report 0 bytes; a zero timeout
|
||||||
// (with dynamic timeout off) makes every scan fail. Both must fall
|
// (with dynamic timeout off) makes every scan fail. Both must fall
|
||||||
@@ -1709,7 +1705,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_update_capacity_preserves_retrieval_metadata() {
|
async fn test_update_capacity_preserves_retrieval_metadata() {
|
||||||
let manager = HybridCapacityManager::from_env();
|
let manager = HybridCapacityManager::from_env();
|
||||||
|
|
||||||
@@ -1725,7 +1720,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_record_write_operation() {
|
async fn test_record_write_operation() {
|
||||||
let manager = HybridCapacityManager::from_env();
|
let manager = HybridCapacityManager::from_env();
|
||||||
|
|
||||||
@@ -1736,7 +1730,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_write_frequency_window() {
|
async fn test_write_frequency_window() {
|
||||||
let manager = HybridCapacityManager::from_env();
|
let manager = HybridCapacityManager::from_env();
|
||||||
|
|
||||||
@@ -1824,7 +1817,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn test_recent_write_count_ignores_future_buckets() {
|
fn test_recent_write_count_ignores_future_buckets() {
|
||||||
let record = WriteRecord::new();
|
let record = WriteRecord::new();
|
||||||
record.write_buckets[0].store(120, 3);
|
record.write_buckets[0].store(120, 3);
|
||||||
@@ -1838,7 +1830,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_needs_fast_update() {
|
async fn test_needs_fast_update() {
|
||||||
let manager = HybridCapacityManager::from_env();
|
let manager = HybridCapacityManager::from_env();
|
||||||
|
|
||||||
@@ -1855,7 +1846,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_cache_age_tracking() {
|
async fn test_cache_age_tracking() {
|
||||||
let manager = HybridCapacityManager::from_env();
|
let manager = HybridCapacityManager::from_env();
|
||||||
|
|
||||||
@@ -1875,7 +1865,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_data_source_tracking() {
|
async fn test_data_source_tracking() {
|
||||||
let manager = HybridCapacityManager::from_env();
|
let manager = HybridCapacityManager::from_env();
|
||||||
|
|
||||||
@@ -1891,7 +1880,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_needs_fast_update_waits_for_write_trigger_delay() {
|
async fn test_needs_fast_update_waits_for_write_trigger_delay() {
|
||||||
let manager = create_isolated_manager(HybridStrategyConfig {
|
let manager = create_isolated_manager(HybridStrategyConfig {
|
||||||
scheduled_update_interval: Duration::from_secs(60),
|
scheduled_update_interval: Duration::from_secs(60),
|
||||||
@@ -1922,7 +1910,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_needs_fast_update_respects_enable_write_trigger() {
|
async fn test_needs_fast_update_respects_enable_write_trigger() {
|
||||||
let manager = create_isolated_manager(HybridStrategyConfig {
|
let manager = create_isolated_manager(HybridStrategyConfig {
|
||||||
scheduled_update_interval: Duration::from_secs(60),
|
scheduled_update_interval: Duration::from_secs(60),
|
||||||
@@ -1949,7 +1936,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_concurrent_access() {
|
async fn test_concurrent_access() {
|
||||||
let manager = Arc::new(HybridCapacityManager::from_env());
|
let manager = Arc::new(HybridCapacityManager::from_env());
|
||||||
let mut handles = Vec::new();
|
let mut handles = Vec::new();
|
||||||
@@ -1976,7 +1962,6 @@ mod tests {
|
|||||||
// exact under heavy same-second contention or the frequency window (and the
|
// exact under heavy same-second contention or the frequency window (and the
|
||||||
// write-trigger decision) would undercount.
|
// write-trigger decision) would undercount.
|
||||||
#[tokio::test(flavor = "multi_thread", worker_threads = 8)]
|
#[tokio::test(flavor = "multi_thread", worker_threads = 8)]
|
||||||
#[serial]
|
|
||||||
async fn test_record_write_operation_lock_free_is_exact_under_contention() {
|
async fn test_record_write_operation_lock_free_is_exact_under_contention() {
|
||||||
let manager = Arc::new(HybridCapacityManager::from_env());
|
let manager = Arc::new(HybridCapacityManager::from_env());
|
||||||
let mut handles = Vec::new();
|
let mut handles = Vec::new();
|
||||||
@@ -2001,7 +1986,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_performance_overhead() {
|
async fn test_performance_overhead() {
|
||||||
let manager = Arc::new(HybridCapacityManager::from_env());
|
let manager = Arc::new(HybridCapacityManager::from_env());
|
||||||
let start = Instant::now();
|
let start = Instant::now();
|
||||||
@@ -2018,7 +2002,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_refresh_or_join_singleflight() {
|
async fn test_refresh_or_join_singleflight() {
|
||||||
let manager = Arc::new(HybridCapacityManager::from_env());
|
let manager = Arc::new(HybridCapacityManager::from_env());
|
||||||
let calls = Arc::new(AtomicUsize::new(0));
|
let calls = Arc::new(AtomicUsize::new(0));
|
||||||
@@ -2058,7 +2041,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_refresh_or_join_recovers_after_leader_cancellation() {
|
async fn test_refresh_or_join_recovers_after_leader_cancellation() {
|
||||||
let manager = Arc::new(HybridCapacityManager::from_env());
|
let manager = Arc::new(HybridCapacityManager::from_env());
|
||||||
|
|
||||||
@@ -2087,7 +2069,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_refresh_or_join_cancelled_leader_unblocks_joiner() {
|
async fn test_refresh_or_join_cancelled_leader_unblocks_joiner() {
|
||||||
let manager = Arc::new(HybridCapacityManager::from_env());
|
let manager = Arc::new(HybridCapacityManager::from_env());
|
||||||
|
|
||||||
@@ -2115,7 +2096,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_spawn_refresh_if_needed_deduplicates_background_refresh() {
|
async fn test_spawn_refresh_if_needed_deduplicates_background_refresh() {
|
||||||
let manager = Arc::new(HybridCapacityManager::from_env());
|
let manager = Arc::new(HybridCapacityManager::from_env());
|
||||||
let calls = Arc::new(AtomicUsize::new(0));
|
let calls = Arc::new(AtomicUsize::new(0));
|
||||||
@@ -2153,7 +2133,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_record_write_operation_with_scope_token_marks_dirty_disks() {
|
async fn test_record_write_operation_with_scope_token_marks_dirty_disks() {
|
||||||
let manager = create_isolated_manager(HybridStrategyConfig::default());
|
let manager = create_isolated_manager(HybridStrategyConfig::default());
|
||||||
let token = uuid::Uuid::new_v4();
|
let token = uuid::Uuid::new_v4();
|
||||||
@@ -2177,7 +2156,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_get_dirty_disks_drains_global_dirty_scope_registry() {
|
async fn test_get_dirty_disks_drains_global_dirty_scope_registry() {
|
||||||
let manager = create_isolated_manager(HybridStrategyConfig::default());
|
let manager = create_isolated_manager(HybridStrategyConfig::default());
|
||||||
record_global_dirty_scope(CapacityScope {
|
record_global_dirty_scope(CapacityScope {
|
||||||
@@ -2197,7 +2175,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_update_capacity_recomputes_total_from_disk_cache_for_subset_refresh() {
|
async fn test_update_capacity_recomputes_total_from_disk_cache_for_subset_refresh() {
|
||||||
let manager = create_isolated_manager(HybridStrategyConfig::default());
|
let manager = create_isolated_manager(HybridStrategyConfig::default());
|
||||||
|
|
||||||
@@ -2308,7 +2285,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_update_capacity_degraded_full_refresh_merges_cache_and_does_not_oscillate() {
|
async fn test_update_capacity_degraded_full_refresh_merges_cache_and_does_not_oscillate() {
|
||||||
let manager = create_isolated_manager(HybridStrategyConfig::default());
|
let manager = create_isolated_manager(HybridStrategyConfig::default());
|
||||||
|
|
||||||
@@ -2354,7 +2330,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_update_capacity_degraded_with_empty_per_disk_serves_merged_cache() {
|
async fn test_update_capacity_degraded_with_empty_per_disk_serves_merged_cache() {
|
||||||
let manager = create_isolated_manager(HybridStrategyConfig::default());
|
let manager = create_isolated_manager(HybridStrategyConfig::default());
|
||||||
manager.update_capacity(full_two_disk_update(), DataSource::RealTime).await;
|
manager.update_capacity(full_two_disk_update(), DataSource::RealTime).await;
|
||||||
@@ -2384,7 +2359,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_update_capacity_degraded_without_complete_cache_keeps_partial_sum() {
|
async fn test_update_capacity_degraded_without_complete_cache_keeps_partial_sum() {
|
||||||
let manager = create_isolated_manager(HybridStrategyConfig::default());
|
let manager = create_isolated_manager(HybridStrategyConfig::default());
|
||||||
|
|
||||||
@@ -2425,7 +2399,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_commit_keeps_dirty_marks_recorded_after_scan_start() {
|
async fn test_commit_keeps_dirty_marks_recorded_after_scan_start() {
|
||||||
let manager = create_isolated_manager(HybridStrategyConfig::default());
|
let manager = create_isolated_manager(HybridStrategyConfig::default());
|
||||||
let disk = scope_disk("node-a", "/tmp/disk-a");
|
let disk = scope_disk("node-a", "/tmp/disk-a");
|
||||||
@@ -2456,7 +2429,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_commit_clears_dirty_marks_recorded_before_scan_start() {
|
async fn test_commit_clears_dirty_marks_recorded_before_scan_start() {
|
||||||
let manager = create_isolated_manager(HybridStrategyConfig::default());
|
let manager = create_isolated_manager(HybridStrategyConfig::default());
|
||||||
let disk = scope_disk("node-a", "/tmp/disk-a");
|
let disk = scope_disk("node-a", "/tmp/disk-a");
|
||||||
@@ -2477,7 +2449,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_retain_dirty_disks_within_drops_ghost_entries() {
|
async fn test_retain_dirty_disks_within_drops_ghost_entries() {
|
||||||
let manager = create_isolated_manager(HybridStrategyConfig::default());
|
let manager = create_isolated_manager(HybridStrategyConfig::default());
|
||||||
let local = scope_disk("node-a", "/tmp/disk-a");
|
let local = scope_disk("node-a", "/tmp/disk-a");
|
||||||
@@ -2496,7 +2467,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_spawn_refresh_recovers_from_construction_panic() {
|
async fn test_spawn_refresh_recovers_from_construction_panic() {
|
||||||
let manager = create_isolated_manager(HybridStrategyConfig::default());
|
let manager = create_isolated_manager(HybridStrategyConfig::default());
|
||||||
|
|
||||||
@@ -2563,7 +2533,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test(start_paused = true)]
|
#[tokio::test(start_paused = true)]
|
||||||
#[serial]
|
|
||||||
async fn test_refresh_or_join_joiner_times_out_when_leader_wedges() {
|
async fn test_refresh_or_join_joiner_times_out_when_leader_wedges() {
|
||||||
let manager = create_isolated_manager(HybridStrategyConfig::default());
|
let manager = create_isolated_manager(HybridStrategyConfig::default());
|
||||||
|
|
||||||
@@ -2591,7 +2560,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_refresh_or_join_returns_cluster_total_for_dirty_subset() {
|
async fn test_refresh_or_join_returns_cluster_total_for_dirty_subset() {
|
||||||
let manager = create_isolated_manager(HybridStrategyConfig::default());
|
let manager = create_isolated_manager(HybridStrategyConfig::default());
|
||||||
|
|
||||||
@@ -2673,7 +2641,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_config_from_env() {
|
async fn test_config_from_env() {
|
||||||
let config = HybridStrategyConfig::from_env();
|
let config = HybridStrategyConfig::from_env();
|
||||||
|
|
||||||
@@ -2687,7 +2654,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_config_from_env_with_override() {
|
async fn test_config_from_env_with_override() {
|
||||||
temp_env::with_var(ENV_CAPACITY_SCHEDULED_INTERVAL, Some("600"), || {
|
temp_env::with_var(ENV_CAPACITY_SCHEDULED_INTERVAL, Some("600"), || {
|
||||||
let config = HybridStrategyConfig::from_env();
|
let config = HybridStrategyConfig::from_env();
|
||||||
|
|||||||
@@ -1069,7 +1069,6 @@ mod tests {
|
|||||||
#[cfg(unix)]
|
#[cfg(unix)]
|
||||||
use rustfs_config::ENV_CAPACITY_FOLLOW_SYMLINKS;
|
use rustfs_config::ENV_CAPACITY_FOLLOW_SYMLINKS;
|
||||||
use rustfs_config::{ENV_CAPACITY_MAX_FILES_THRESHOLD, ENV_CAPACITY_SAMPLE_RATE};
|
use rustfs_config::{ENV_CAPACITY_MAX_FILES_THRESHOLD, ENV_CAPACITY_SAMPLE_RATE};
|
||||||
use serial_test::serial;
|
|
||||||
|
|
||||||
/// Reference implementation using unbounded `u128` arithmetic, clamped to
|
/// Reference implementation using unbounded `u128` arithmetic, clamped to
|
||||||
/// `u64::MAX`, used as the source of truth for the sampling extrapolation.
|
/// `u64::MAX`, used as the source of truth for the sampling extrapolation.
|
||||||
@@ -1274,7 +1273,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_get_dir_size_async_nonexistent_directory() {
|
async fn test_get_dir_size_async_nonexistent_directory() {
|
||||||
let result = get_dir_size_async(Path::new("/nonexistent/path")).await;
|
let result = get_dir_size_async(Path::new("/nonexistent/path")).await;
|
||||||
assert!(result.is_err());
|
assert!(result.is_err());
|
||||||
@@ -1648,7 +1646,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_metadata_incomplete_aggregate_does_not_replace_disk_cache() {
|
async fn test_metadata_incomplete_aggregate_does_not_replace_disk_cache() {
|
||||||
use std::fs::File;
|
use std::fs::File;
|
||||||
use std::io::Write;
|
use std::io::Write;
|
||||||
@@ -1783,7 +1780,6 @@ mod tests {
|
|||||||
|
|
||||||
#[cfg(unix)]
|
#[cfg(unix)]
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_get_dir_size_async_ignores_symlink_targets_when_follow_disabled() {
|
async fn test_get_dir_size_async_ignores_symlink_targets_when_follow_disabled() {
|
||||||
use std::fs::File;
|
use std::fs::File;
|
||||||
use std::io::Write;
|
use std::io::Write;
|
||||||
@@ -1809,7 +1805,6 @@ mod tests {
|
|||||||
|
|
||||||
#[cfg(unix)]
|
#[cfg(unix)]
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_get_dir_size_async_counts_symlink_targets_when_follow_enabled() {
|
async fn test_get_dir_size_async_counts_symlink_targets_when_follow_enabled() {
|
||||||
use std::fs::File;
|
use std::fs::File;
|
||||||
use std::io::Write;
|
use std::io::Write;
|
||||||
|
|||||||
@@ -75,7 +75,7 @@ pub struct BucketReplicationBandwidthStats {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Default)]
|
#[derive(Debug, Clone, Default)]
|
||||||
pub struct BucketReplicationStats {
|
pub struct BucketReplicationMetricsSnapshot {
|
||||||
pub bucket: String,
|
pub bucket: String,
|
||||||
pub total_failed_bytes: u64,
|
pub total_failed_bytes: u64,
|
||||||
pub total_failed_count: u64,
|
pub total_failed_count: u64,
|
||||||
@@ -107,7 +107,7 @@ pub struct BucketReplicationStats {
|
|||||||
|
|
||||||
#[derive(Debug, Clone, Default)]
|
#[derive(Debug, Clone, Default)]
|
||||||
pub(crate) struct BucketReplicationRuntimeStats {
|
pub(crate) struct BucketReplicationRuntimeStats {
|
||||||
pub(crate) stats: BucketReplicationStats,
|
pub(crate) stats: BucketReplicationMetricsSnapshot,
|
||||||
pub(crate) target_flows: Vec<BucketReplicationTargetFlowStats>,
|
pub(crate) target_flows: Vec<BucketReplicationTargetFlowStats>,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -182,7 +182,7 @@ fn push_proxy_request_result_metrics(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn collect_bucket_replication_metrics(stats: &[BucketReplicationStats]) -> Vec<PrometheusMetric> {
|
pub fn collect_bucket_replication_metrics(stats: &[BucketReplicationMetricsSnapshot]) -> Vec<PrometheusMetric> {
|
||||||
if stats.is_empty() {
|
if stats.is_empty() {
|
||||||
return Vec::new();
|
return Vec::new();
|
||||||
}
|
}
|
||||||
@@ -572,7 +572,7 @@ mod tests {
|
|||||||
#[test]
|
#[test]
|
||||||
fn test_collect_bucket_replication_metrics() {
|
fn test_collect_bucket_replication_metrics() {
|
||||||
let stats = vec![BucketReplicationRuntimeStats {
|
let stats = vec![BucketReplicationRuntimeStats {
|
||||||
stats: BucketReplicationStats {
|
stats: BucketReplicationMetricsSnapshot {
|
||||||
bucket: "b1".to_string(),
|
bucket: "b1".to_string(),
|
||||||
total_failed_bytes: 64,
|
total_failed_bytes: 64,
|
||||||
total_failed_count: 2,
|
total_failed_count: 2,
|
||||||
@@ -876,7 +876,7 @@ mod tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_collect_bucket_replication_metrics_empty() {
|
fn test_collect_bucket_replication_metrics_empty() {
|
||||||
let stats: Vec<BucketReplicationStats> = Vec::new();
|
let stats: Vec<BucketReplicationMetricsSnapshot> = Vec::new();
|
||||||
let metrics = collect_bucket_replication_metrics(&stats);
|
let metrics = collect_bucket_replication_metrics(&stats);
|
||||||
assert!(metrics.is_empty());
|
assert!(metrics.is_empty());
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -48,7 +48,7 @@ pub(crate) use bucket_replication::{
|
|||||||
BucketReplicationTargetFlowStats, collect_bucket_replication_backlog_metrics, collect_bucket_replication_runtime_metrics,
|
BucketReplicationTargetFlowStats, collect_bucket_replication_backlog_metrics, collect_bucket_replication_runtime_metrics,
|
||||||
};
|
};
|
||||||
pub use bucket_replication::{
|
pub use bucket_replication::{
|
||||||
BucketReplicationBandwidthStats, BucketReplicationStats, BucketReplicationTargetStats,
|
BucketReplicationBandwidthStats, BucketReplicationMetricsSnapshot, BucketReplicationTargetStats,
|
||||||
collect_bucket_replication_bandwidth_metrics, collect_bucket_replication_metrics,
|
collect_bucket_replication_bandwidth_metrics, collect_bucket_replication_metrics,
|
||||||
};
|
};
|
||||||
pub use cluster::{ClusterStats, collect_cluster_metrics};
|
pub use cluster::{ClusterStats, collect_cluster_metrics};
|
||||||
@@ -68,8 +68,8 @@ pub(crate) use notification::collect_notification_runtime_metrics;
|
|||||||
pub use notification::{NotificationStats, collect_notification_metrics};
|
pub use notification::{NotificationStats, collect_notification_metrics};
|
||||||
pub(crate) use notification_target::{NotificationTargetRuntimeStats, collect_notification_target_runtime_metrics};
|
pub(crate) use notification_target::{NotificationTargetRuntimeStats, collect_notification_target_runtime_metrics};
|
||||||
pub use notification_target::{NotificationTargetStats, collect_notification_target_metrics};
|
pub use notification_target::{NotificationTargetStats, collect_notification_target_metrics};
|
||||||
|
pub use replication::{ReplicationMetricsSnapshot, collect_replication_metrics};
|
||||||
pub(crate) use replication::{ReplicationRuntimeStats, collect_replication_runtime_metrics};
|
pub(crate) use replication::{ReplicationRuntimeStats, collect_replication_runtime_metrics};
|
||||||
pub use replication::{ReplicationStats, collect_replication_metrics};
|
|
||||||
pub(crate) use request::{ApiRequestMetricSupport, ApiRequestStats, collect_request_metrics};
|
pub(crate) use request::{ApiRequestMetricSupport, ApiRequestStats, collect_request_metrics};
|
||||||
pub use resource::{ResourceStats, collect_resource_metrics};
|
pub use resource::{ResourceStats, collect_resource_metrics};
|
||||||
pub(crate) use scanner::{ScannerRuntimeStats, collect_scanner_runtime_metrics};
|
pub(crate) use scanner::{ScannerRuntimeStats, collect_scanner_runtime_metrics};
|
||||||
|
|||||||
@@ -22,7 +22,7 @@ use crate::metrics::schema::replication::*;
|
|||||||
|
|
||||||
/// Replication statistics.
|
/// Replication statistics.
|
||||||
#[derive(Debug, Clone, Default)]
|
#[derive(Debug, Clone, Default)]
|
||||||
pub struct ReplicationStats {
|
pub struct ReplicationMetricsSnapshot {
|
||||||
/// Average number of active replication workers
|
/// Average number of active replication workers
|
||||||
pub average_active_workers: f64,
|
pub average_active_workers: f64,
|
||||||
/// Average queued bytes since server start
|
/// Average queued bytes since server start
|
||||||
@@ -54,13 +54,13 @@ pub struct ReplicationStats {
|
|||||||
#[derive(Debug, Clone, Default)]
|
#[derive(Debug, Clone, Default)]
|
||||||
pub(crate) struct ReplicationRuntimeStats {
|
pub(crate) struct ReplicationRuntimeStats {
|
||||||
pub(crate) server: String,
|
pub(crate) server: String,
|
||||||
pub(crate) stats: ReplicationStats,
|
pub(crate) stats: ReplicationMetricsSnapshot,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Collects replication metrics from the given stats.
|
/// Collects replication metrics from the given stats.
|
||||||
///
|
///
|
||||||
/// Returns a vector of Prometheus metrics for replication statistics.
|
/// Returns a vector of Prometheus metrics for replication statistics.
|
||||||
pub fn collect_replication_metrics(stats: &ReplicationStats) -> Vec<PrometheusMetric> {
|
pub fn collect_replication_metrics(stats: &ReplicationMetricsSnapshot) -> Vec<PrometheusMetric> {
|
||||||
vec![
|
vec![
|
||||||
PrometheusMetric::from_descriptor(&REPLICATION_AVERAGE_ACTIVE_WORKERS_MD, stats.average_active_workers),
|
PrometheusMetric::from_descriptor(&REPLICATION_AVERAGE_ACTIVE_WORKERS_MD, stats.average_active_workers),
|
||||||
PrometheusMetric::from_descriptor(&REPLICATION_AVERAGE_QUEUED_BYTES_MD, stats.average_queued_bytes as f64),
|
PrometheusMetric::from_descriptor(&REPLICATION_AVERAGE_QUEUED_BYTES_MD, stats.average_queued_bytes as f64),
|
||||||
@@ -120,7 +120,7 @@ mod tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_collect_replication_metrics() {
|
fn test_collect_replication_metrics() {
|
||||||
let stats = ReplicationStats {
|
let stats = ReplicationMetricsSnapshot {
|
||||||
average_active_workers: 8.5,
|
average_active_workers: 8.5,
|
||||||
average_queued_bytes: 1024 * 1024 * 40,
|
average_queued_bytes: 1024 * 1024 * 40,
|
||||||
average_queued_count: 240,
|
average_queued_count: 240,
|
||||||
@@ -182,7 +182,7 @@ mod tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_collect_replication_metrics_default() {
|
fn test_collect_replication_metrics_default() {
|
||||||
let stats = ReplicationStats::default();
|
let stats = ReplicationMetricsSnapshot::default();
|
||||||
let metrics = collect_replication_metrics(&stats);
|
let metrics = collect_replication_metrics(&stats);
|
||||||
|
|
||||||
assert_eq!(metrics.len(), 13);
|
assert_eq!(metrics.len(), 13);
|
||||||
@@ -194,7 +194,7 @@ mod tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn replication_stats_struct_literal_keeps_legacy_fields() {
|
fn replication_stats_struct_literal_keeps_legacy_fields() {
|
||||||
let stats = ReplicationStats {
|
let stats = ReplicationMetricsSnapshot {
|
||||||
average_active_workers: 1.0,
|
average_active_workers: 1.0,
|
||||||
average_queued_bytes: 2,
|
average_queued_bytes: 2,
|
||||||
average_queued_count: 3,
|
average_queued_count: 3,
|
||||||
|
|||||||
@@ -2811,14 +2811,14 @@ mod tests {
|
|||||||
#[test]
|
#[test]
|
||||||
fn replication_proxy_bucket_keys_detect_removed_buckets() {
|
fn replication_proxy_bucket_keys_detect_removed_buckets() {
|
||||||
let previous = repl_proxy_bucket_live_keys(&[BucketReplicationRuntimeStats {
|
let previous = repl_proxy_bucket_live_keys(&[BucketReplicationRuntimeStats {
|
||||||
stats: crate::metrics::BucketReplicationStats {
|
stats: crate::metrics::BucketReplicationMetricsSnapshot {
|
||||||
bucket: "photos".to_string(),
|
bucket: "photos".to_string(),
|
||||||
..Default::default()
|
..Default::default()
|
||||||
},
|
},
|
||||||
..Default::default()
|
..Default::default()
|
||||||
}]);
|
}]);
|
||||||
let current = repl_proxy_bucket_live_keys(&[BucketReplicationRuntimeStats {
|
let current = repl_proxy_bucket_live_keys(&[BucketReplicationRuntimeStats {
|
||||||
stats: crate::metrics::BucketReplicationStats {
|
stats: crate::metrics::BucketReplicationMetricsSnapshot {
|
||||||
bucket: "logs".to_string(),
|
bucket: "logs".to_string(),
|
||||||
..Default::default()
|
..Default::default()
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -21,12 +21,12 @@
|
|||||||
use crate::metrics::collectors::scanner::{ScannerActiveBucketDriveStats, ScannerBucketDriveResultStats, ScannerSourceWorkStats};
|
use crate::metrics::collectors::scanner::{ScannerActiveBucketDriveStats, ScannerBucketDriveResultStats, ScannerSourceWorkStats};
|
||||||
use crate::metrics::collectors::{
|
use crate::metrics::collectors::{
|
||||||
ApiRequestMetricSupport, ApiRequestStats, BucketReplicationBacklogStats, BucketReplicationBandwidthStats,
|
ApiRequestMetricSupport, ApiRequestStats, BucketReplicationBacklogStats, BucketReplicationBandwidthStats,
|
||||||
BucketReplicationRuntimeStats, BucketReplicationStats, BucketReplicationTargetBacklogStats, BucketReplicationTargetFlowStats,
|
BucketReplicationMetricsSnapshot, BucketReplicationRuntimeStats, BucketReplicationTargetBacklogStats,
|
||||||
BucketReplicationTargetStats, BucketStats, BucketUsageStats, ClusterConfigStats, ClusterHealthStats, ClusterStats,
|
BucketReplicationTargetFlowStats, BucketReplicationTargetStats, BucketStats, BucketUsageStats, ClusterConfigStats,
|
||||||
ClusterUsageStats, CompressionClusterStats, CpuStats, DiskStats, DriveCountStats, DriveDetailedStats,
|
ClusterHealthStats, ClusterStats, ClusterUsageStats, CompressionClusterStats, CpuStats, DiskStats, DriveCountStats,
|
||||||
DriveRuntimeDetailedStats, ErasureSetStats, HostNetworkStats, IamStats, IlmActionTaskStats, IlmBackpressureStats,
|
DriveDetailedStats, DriveRuntimeDetailedStats, ErasureSetStats, HostNetworkStats, IamStats, IlmActionTaskStats,
|
||||||
IlmQueueTaskStats, IlmRuntimeStats, IlmStats, IlmTaskEventStats, MemoryStats, NetworkStats, ProcessStats, ProcessStatusType,
|
IlmBackpressureStats, IlmQueueTaskStats, IlmRuntimeStats, IlmStats, IlmTaskEventStats, MemoryStats, NetworkStats,
|
||||||
ReplicationStats, ResourceStats, ScannerRuntimeStats, ScannerStats,
|
ProcessStats, ProcessStatusType, ReplicationMetricsSnapshot, ResourceStats, ScannerRuntimeStats, ScannerStats,
|
||||||
};
|
};
|
||||||
use crate::metrics::runtime_sources::{ObsIlmRuntimeSnapshot, bucket_monitor_handle, iam_metrics_snapshot, ilm_runtime_snapshot};
|
use crate::metrics::runtime_sources::{ObsIlmRuntimeSnapshot, bucket_monitor_handle, iam_metrics_snapshot, ilm_runtime_snapshot};
|
||||||
use crate::metrics::{
|
use crate::metrics::{
|
||||||
@@ -266,7 +266,7 @@ fn bucket_replication_detail_from_snapshot(stats: ObsBucketReplicationStatsSnaps
|
|||||||
|
|
||||||
BucketReplicationRuntimeStats {
|
BucketReplicationRuntimeStats {
|
||||||
target_flows,
|
target_flows,
|
||||||
stats: BucketReplicationStats {
|
stats: BucketReplicationMetricsSnapshot {
|
||||||
bucket,
|
bucket,
|
||||||
total_failed_bytes: stats.total_failed_bytes,
|
total_failed_bytes: stats.total_failed_bytes,
|
||||||
total_failed_count: stats.total_failed_count,
|
total_failed_count: stats.total_failed_count,
|
||||||
@@ -298,7 +298,7 @@ fn bucket_replication_detail_from_snapshot(stats: ObsBucketReplicationStatsSnaps
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn obs_site_replication_stats() -> ReplicationStats {
|
async fn obs_site_replication_stats() -> ReplicationMetricsSnapshot {
|
||||||
let current_data_transfer_rate = obs_bucket_replication_bandwidth_stats()
|
let current_data_transfer_rate = obs_bucket_replication_bandwidth_stats()
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.flatten()
|
.flatten()
|
||||||
@@ -306,7 +306,7 @@ async fn obs_site_replication_stats() -> ReplicationStats {
|
|||||||
.sum::<f64>();
|
.sum::<f64>();
|
||||||
let stats = obs_replication_site_stats_snapshot(current_data_transfer_rate).await;
|
let stats = obs_replication_site_stats_snapshot(current_data_transfer_rate).await;
|
||||||
|
|
||||||
ReplicationStats {
|
ReplicationMetricsSnapshot {
|
||||||
average_active_workers: stats.average_active_workers,
|
average_active_workers: stats.average_active_workers,
|
||||||
average_queued_bytes: stats.average_queued_bytes,
|
average_queued_bytes: stats.average_queued_bytes,
|
||||||
average_queued_count: stats.average_queued_count,
|
average_queued_count: stats.average_queued_count,
|
||||||
@@ -648,7 +648,7 @@ pub fn collect_bucket_replication_bandwidth_stats() -> Vec<BucketReplicationBand
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Collect bucket and target level replication stats from the global replication runtime.
|
/// Collect bucket and target level replication stats from the global replication runtime.
|
||||||
pub async fn collect_bucket_replication_detail_stats() -> Vec<BucketReplicationStats> {
|
pub async fn collect_bucket_replication_detail_stats() -> Vec<BucketReplicationMetricsSnapshot> {
|
||||||
obs_bucket_replication_stats_snapshot()
|
obs_bucket_replication_stats_snapshot()
|
||||||
.await
|
.await
|
||||||
.into_iter()
|
.into_iter()
|
||||||
@@ -662,7 +662,7 @@ pub(crate) async fn collect_bucket_replication_stats_bundle()
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Collect site-level replication stats from the global replication runtime.
|
/// Collect site-level replication stats from the global replication runtime.
|
||||||
pub async fn collect_replication_stats() -> ReplicationStats {
|
pub async fn collect_replication_stats() -> ReplicationMetricsSnapshot {
|
||||||
obs_site_replication_stats().await
|
obs_site_replication_stats().await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -103,8 +103,7 @@ hex-simd.workspace = true
|
|||||||
|
|
||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
tracing-subscriber = { workspace = true, features = ["json", "env-filter", "time"] }
|
tracing-subscriber = { workspace = true, features = ["json", "env-filter", "time"] }
|
||||||
serial_test = { workspace = true }
|
temp-env = { workspace = true, features = ["async_closure"] }
|
||||||
temp-env = { workspace = true }
|
|
||||||
tempfile = { workspace = true }
|
tempfile = { workspace = true }
|
||||||
uuid = { workspace = true, features = ["v4", "serde", "fast-rng", "macro-diagnostics"] }
|
uuid = { workspace = true, features = ["v4", "serde", "fast-rng", "macro-diagnostics"] }
|
||||||
tokio = { workspace = true, features = ["test-util", "fs", "rt-multi-thread"] }
|
tokio = { workspace = true, features = ["test-util", "fs", "rt-multi-thread"] }
|
||||||
|
|||||||
@@ -599,10 +599,8 @@ impl ScannerConfigObjectDelete for ECStore {
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
use serial_test::serial;
|
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn runtime_tier_names_serves_cached_arc_within_ttl() {
|
async fn runtime_tier_names_serves_cached_arc_within_ttl() {
|
||||||
reset_tier_name_cache_for_test();
|
reset_tier_name_cache_for_test();
|
||||||
// The tier config manager is unconfigured in unit tests, so the
|
// The tier config manager is unconfigured in unit tests, so the
|
||||||
@@ -616,7 +614,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn foreground_read_guard_tracks_stream_lifetime() {
|
fn foreground_read_guard_tracks_stream_lifetime() {
|
||||||
reset_foreground_read_activity_for_test();
|
reset_foreground_read_activity_for_test();
|
||||||
assert_eq!(current_foreground_read_activity(), 0);
|
assert_eq!(current_foreground_read_activity(), 0);
|
||||||
@@ -630,7 +627,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn foreground_read_activity_keeps_larger_signal() {
|
fn foreground_read_activity_keeps_larger_signal() {
|
||||||
reset_foreground_read_activity_for_test();
|
reset_foreground_read_activity_for_test();
|
||||||
let _guard = ForegroundReadGuard::new();
|
let _guard = ForegroundReadGuard::new();
|
||||||
@@ -643,7 +639,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn scanner_runtime_guard_tracks_runtime_lifetime() {
|
fn scanner_runtime_guard_tracks_runtime_lifetime() {
|
||||||
reset_scanner_runtime_instances_for_test();
|
reset_scanner_runtime_instances_for_test();
|
||||||
assert!(!scanner_runtime_initialized());
|
assert!(!scanner_runtime_initialized());
|
||||||
|
|||||||
@@ -868,7 +868,6 @@ mod tests {
|
|||||||
SCANNER_CYCLE_MAX_DIRECTORIES, SCANNER_CYCLE_MAX_DURATION, SCANNER_CYCLE_MAX_OBJECTS, SCANNER_DELAY, SCANNER_IDLE_MODE,
|
SCANNER_CYCLE_MAX_DIRECTORIES, SCANNER_CYCLE_MAX_DURATION, SCANNER_CYCLE_MAX_OBJECTS, SCANNER_DELAY, SCANNER_IDLE_MODE,
|
||||||
SCANNER_SPEED, SCANNER_SUB_SYS, ScannerSpeed,
|
SCANNER_SPEED, SCANNER_SUB_SYS, ScannerSpeed,
|
||||||
};
|
};
|
||||||
use serial_test::serial;
|
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
use temp_env::{with_var, with_var_unset};
|
use temp_env::{with_var, with_var_unset};
|
||||||
@@ -916,7 +915,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn scanner_runtime_config_uses_persisted_values_when_env_is_unset() {
|
fn scanner_runtime_config_uses_persisted_values_when_env_is_unset() {
|
||||||
let config = server_config_with_scanner(&[
|
let config = server_config_with_scanner(&[
|
||||||
(SCANNER_SPEED, "slow"),
|
(SCANNER_SPEED, "slow"),
|
||||||
@@ -944,7 +942,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn scanner_runtime_config_normalizes_persisted_default_speed() {
|
fn scanner_runtime_config_normalizes_persisted_default_speed() {
|
||||||
let config = server_config_with_scanner(&[(SCANNER_SPEED, "default")]);
|
let config = server_config_with_scanner(&[(SCANNER_SPEED, "default")]);
|
||||||
|
|
||||||
@@ -960,7 +957,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn scanner_runtime_config_prefers_env_over_persisted_config() {
|
fn scanner_runtime_config_prefers_env_over_persisted_config() {
|
||||||
let config = server_config_with_scanner(&[(SCANNER_SPEED, "slowest"), (SCANNER_CYCLE, "600")]);
|
let config = server_config_with_scanner(&[(SCANNER_SPEED, "slowest"), (SCANNER_CYCLE, "600")]);
|
||||||
|
|
||||||
@@ -977,7 +973,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn scanner_runtime_config_prefers_heal_bitrot_cycle_over_scanner_compat_config() {
|
fn scanner_runtime_config_prefers_heal_bitrot_cycle_over_scanner_compat_config() {
|
||||||
let config = server_config_with_scanner_and_heal(&[(SCANNER_BITROT_CYCLE, "3600")], &[(HEAL_BITROT_CYCLE, "off")]);
|
let config = server_config_with_scanner_and_heal(&[(SCANNER_BITROT_CYCLE, "3600")], &[(HEAL_BITROT_CYCLE, "off")]);
|
||||||
|
|
||||||
@@ -990,7 +985,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn scanner_runtime_config_marks_scanner_bitrot_cycle_as_compat_source() {
|
fn scanner_runtime_config_marks_scanner_bitrot_cycle_as_compat_source() {
|
||||||
let config = server_config_with_scanner(&[(SCANNER_BITROT_CYCLE, "3600")]);
|
let config = server_config_with_scanner(&[(SCANNER_BITROT_CYCLE, "3600")]);
|
||||||
|
|
||||||
@@ -1007,7 +1001,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn scanner_runtime_config_normalizes_persisted_default_bitrot_cycles() {
|
fn scanner_runtime_config_normalizes_persisted_default_bitrot_cycles() {
|
||||||
let default_cycle = DEFAULT_HEAL_BITROT_CYCLE_SECS.to_string();
|
let default_cycle = DEFAULT_HEAL_BITROT_CYCLE_SECS.to_string();
|
||||||
for config in [
|
for config in [
|
||||||
@@ -1032,7 +1025,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn scanner_runtime_config_validation_rejects_invalid_persisted_speed_with_env_override() {
|
fn scanner_runtime_config_validation_rejects_invalid_persisted_speed_with_env_override() {
|
||||||
let config = server_config_with_scanner(&[(SCANNER_SPEED, "warp")]);
|
let config = server_config_with_scanner(&[(SCANNER_SPEED, "warp")]);
|
||||||
|
|
||||||
@@ -1066,7 +1058,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn scanner_runtime_config_uses_derived_delay_for_excessive_env_override() {
|
fn scanner_runtime_config_uses_derived_delay_for_excessive_env_override() {
|
||||||
let config = server_config_with_scanner(&[(SCANNER_SPEED, "slow")]);
|
let config = server_config_with_scanner(&[(SCANNER_SPEED, "slow")]);
|
||||||
|
|
||||||
@@ -1087,7 +1078,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn scanner_runtime_config_status_reports_value_sources() {
|
fn scanner_runtime_config_status_reports_value_sources() {
|
||||||
let config = server_config_with_scanner(&[(SCANNER_CYCLE_MAX_OBJECTS, "100"), (SCANNER_CACHE_SAVE_TIMEOUT, "5")]);
|
let config = server_config_with_scanner(&[(SCANNER_CYCLE_MAX_OBJECTS, "100"), (SCANNER_CACHE_SAVE_TIMEOUT, "5")]);
|
||||||
|
|
||||||
@@ -1108,7 +1098,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn applied_runtime_config_is_the_authoritative_scheduler_state() {
|
fn applied_runtime_config_is_the_authoritative_scheduler_state() {
|
||||||
let config = server_config_with_scanner(&[(SCANNER_CYCLE, "321")]);
|
let config = server_config_with_scanner(&[(SCANNER_CYCLE, "321")]);
|
||||||
|
|
||||||
@@ -1125,7 +1114,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn scanner_runtime_config_status_reports_persisted_pacing_overrides() {
|
fn scanner_runtime_config_status_reports_persisted_pacing_overrides() {
|
||||||
let config = server_config_with_scanner(&[("delay", "3.5"), ("max_wait", "7")]);
|
let config = server_config_with_scanner(&[("delay", "3.5"), ("max_wait", "7")]);
|
||||||
|
|
||||||
@@ -1147,7 +1135,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn scanner_runtime_config_status_prefers_env_pacing_overrides() {
|
fn scanner_runtime_config_status_prefers_env_pacing_overrides() {
|
||||||
let config = server_config_with_scanner(&[("delay", "3.5"), ("max_wait", "7")]);
|
let config = server_config_with_scanner(&[("delay", "3.5"), ("max_wait", "7")]);
|
||||||
|
|
||||||
@@ -1169,7 +1156,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn scanner_runtime_config_status_preserves_subsecond_max_wait() {
|
fn scanner_runtime_config_status_preserves_subsecond_max_wait() {
|
||||||
let config = server_config_with_scanner(&[(SCANNER_SPEED, "fast")]);
|
let config = server_config_with_scanner(&[(SCANNER_SPEED, "fast")]);
|
||||||
|
|
||||||
|
|||||||
+166
-78
@@ -1081,18 +1081,6 @@ async fn run_data_scanner_cycle(
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
let (sender, receiver) = mpsc::channel::<DataUsageInfo>(1);
|
let (sender, receiver) = mpsc::channel::<DataUsageInfo>(1);
|
||||||
let storeapi_clone = storeapi.clone();
|
|
||||||
let ctx_clone = ctx.clone();
|
|
||||||
let mut usage_persist_task = AbortOnDropHandle::new(tokio::spawn(async move {
|
|
||||||
store_data_usage_in_backend_with_outcome_for_epoch_and_baseline(
|
|
||||||
ctx_clone,
|
|
||||||
storeapi_clone,
|
|
||||||
receiver,
|
|
||||||
Some(leader_epoch),
|
|
||||||
Some(usage_persist_baseline),
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
}));
|
|
||||||
|
|
||||||
let done_cycle = Metrics::time(Metric::ScanCycle);
|
let done_cycle = Metrics::time(Metric::ScanCycle);
|
||||||
let cycle_budget = ScannerCycleBudget::new(ctx, cycle_budget_config);
|
let cycle_budget = ScannerCycleBudget::new(ctx, cycle_budget_config);
|
||||||
@@ -1107,47 +1095,78 @@ async fn run_data_scanner_cycle(
|
|||||||
scan_mode,
|
scan_mode,
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
|
let publication_defer_reason = match &scan_result {
|
||||||
|
Ok(result) => final_data_usage_publication_defer_reason(storeapi.as_ref(), result.status).await,
|
||||||
|
Err(_) => Some(ScannerCycleDeferReason::ActivityBaselineUnavailable),
|
||||||
|
};
|
||||||
let budget_elapsed = cycle_budget.budget_elapsed() && !ctx.is_cancelled();
|
let budget_elapsed = cycle_budget.budget_elapsed() && !ctx.is_cancelled();
|
||||||
let usage_persist_outcome = match wait_for_data_usage_persist_task(ctx, &mut usage_persist_task, usage_persist_timeout).await
|
let usage_persist_outcome = match publication_defer_reason {
|
||||||
{
|
Some(reason) => {
|
||||||
DataUsagePersistTaskResult::Completed(outcome) => outcome,
|
drop(receiver);
|
||||||
DataUsagePersistTaskResult::JoinFailed(err) => {
|
DataUsagePersistOutcome::Deferred(reason)
|
||||||
error!(
|
|
||||||
target: "rustfs::scanner",
|
|
||||||
event = EVENT_SCANNER_PERSIST_STATE,
|
|
||||||
component = LOG_COMPONENT_SCANNER,
|
|
||||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
|
||||||
cycle = cycle_info.current,
|
|
||||||
state = "usage_persist_task_failed",
|
|
||||||
error = %err,
|
|
||||||
"Scanner data usage persistence task failed"
|
|
||||||
);
|
|
||||||
DataUsagePersistOutcome::Failed
|
|
||||||
}
|
}
|
||||||
DataUsagePersistTaskResult::Cancelled => {
|
None => {
|
||||||
debug!(
|
// ScannerIO emits its complete or observational update only after
|
||||||
target: "rustfs::scanner",
|
// all set workers finish. Persist after the final activity fence;
|
||||||
event = EVENT_SCANNER_PERSIST_STATE,
|
// this also avoids blocking the scanner on a denied publication.
|
||||||
component = LOG_COMPONENT_SCANNER,
|
let storeapi_clone = storeapi.clone();
|
||||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
let ctx_clone = ctx.clone();
|
||||||
cycle = cycle_info.current,
|
let route_probe_store = storeapi.clone();
|
||||||
state = "usage_persist_task_cancelled",
|
let mut usage_persist_task = AbortOnDropHandle::new(tokio::spawn(async move {
|
||||||
"Scanner data usage persistence task cancelled"
|
store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe(
|
||||||
);
|
ctx_clone,
|
||||||
DataUsagePersistOutcome::Failed
|
storeapi_clone,
|
||||||
}
|
receiver,
|
||||||
DataUsagePersistTaskResult::TimedOut => {
|
Some(leader_epoch),
|
||||||
error!(
|
Some(usage_persist_baseline),
|
||||||
target: "rustfs::scanner",
|
move || {
|
||||||
event = EVENT_SCANNER_PERSIST_STATE,
|
let storeapi = route_probe_store.clone();
|
||||||
component = LOG_COMPONENT_SCANNER,
|
async move { storeapi.scanner_data_usage_publication_blocked().await }
|
||||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
},
|
||||||
cycle = cycle_info.current,
|
)
|
||||||
timeout = ?usage_persist_timeout,
|
.await
|
||||||
state = "usage_persist_task_timed_out",
|
}));
|
||||||
"Scanner data usage persistence task timed out"
|
match wait_for_data_usage_persist_task(ctx, &mut usage_persist_task, usage_persist_timeout).await {
|
||||||
);
|
DataUsagePersistTaskResult::Completed(outcome) => outcome,
|
||||||
DataUsagePersistOutcome::Failed
|
DataUsagePersistTaskResult::JoinFailed(err) => {
|
||||||
|
error!(
|
||||||
|
target: "rustfs::scanner",
|
||||||
|
event = EVENT_SCANNER_PERSIST_STATE,
|
||||||
|
component = LOG_COMPONENT_SCANNER,
|
||||||
|
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||||
|
cycle = cycle_info.current,
|
||||||
|
state = "usage_persist_task_failed",
|
||||||
|
error = %err,
|
||||||
|
"Scanner data usage persistence task failed"
|
||||||
|
);
|
||||||
|
DataUsagePersistOutcome::Failed
|
||||||
|
}
|
||||||
|
DataUsagePersistTaskResult::Cancelled => {
|
||||||
|
debug!(
|
||||||
|
target: "rustfs::scanner",
|
||||||
|
event = EVENT_SCANNER_PERSIST_STATE,
|
||||||
|
component = LOG_COMPONENT_SCANNER,
|
||||||
|
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||||
|
cycle = cycle_info.current,
|
||||||
|
state = "usage_persist_task_cancelled",
|
||||||
|
"Scanner data usage persistence task cancelled"
|
||||||
|
);
|
||||||
|
DataUsagePersistOutcome::Failed
|
||||||
|
}
|
||||||
|
DataUsagePersistTaskResult::TimedOut => {
|
||||||
|
error!(
|
||||||
|
target: "rustfs::scanner",
|
||||||
|
event = EVENT_SCANNER_PERSIST_STATE,
|
||||||
|
component = LOG_COMPONENT_SCANNER,
|
||||||
|
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||||
|
cycle = cycle_info.current,
|
||||||
|
timeout = ?usage_persist_timeout,
|
||||||
|
state = "usage_persist_task_timed_out",
|
||||||
|
"Scanner data usage persistence task timed out"
|
||||||
|
);
|
||||||
|
DataUsagePersistOutcome::Failed
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
let unresolved_heal_work = global_metrics().current_scan_cycle_has_unresolved_heal_work();
|
let unresolved_heal_work = global_metrics().current_scan_cycle_has_unresolved_heal_work();
|
||||||
@@ -1191,33 +1210,51 @@ async fn run_data_scanner_cycle(
|
|||||||
mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await;
|
mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await;
|
||||||
return ScannerCycleOutcome::Failed;
|
return ScannerCycleOutcome::Failed;
|
||||||
}
|
}
|
||||||
if let Some(required_cycle) = scan_cycle_result.required_cycle_floor() {
|
match scanner_cycle_pre_commit_outcome(scan_cycle_result.required_cycle_floor(), &usage_persist_outcome) {
|
||||||
warn!(
|
Some(ScannerCyclePreCommitOutcome::RecoverCacheCycle(required_cycle)) => {
|
||||||
target: "rustfs::scanner",
|
warn!(
|
||||||
event = EVENT_SCANNER_CYCLE_STATE,
|
target: "rustfs::scanner",
|
||||||
component = LOG_COMPONENT_SCANNER,
|
event = EVENT_SCANNER_CYCLE_STATE,
|
||||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
component = LOG_COMPONENT_SCANNER,
|
||||||
cycle = cycle_info.current,
|
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||||
required_cycle,
|
cycle = cycle_info.current,
|
||||||
state = "cache_cycle_ahead",
|
required_cycle,
|
||||||
"Scanner cycle is recovering to a newer durable cache generation"
|
state = "cache_cycle_ahead",
|
||||||
);
|
"Scanner cycle is recovering to a newer durable cache generation"
|
||||||
emit_scan_cycle_partial_with_source(cycle_start.elapsed(), ScanCyclePartialReason::Unknown, None);
|
);
|
||||||
return if persist_required_scanner_cycle_floor(
|
emit_scan_cycle_partial_with_source(cycle_start.elapsed(), ScanCyclePartialReason::Unknown, None);
|
||||||
ctx,
|
return if persist_required_scanner_cycle_floor(
|
||||||
storeapi.clone(),
|
ctx,
|
||||||
cycle_info,
|
storeapi.clone(),
|
||||||
cycle_revision,
|
cycle_info,
|
||||||
leader_epoch,
|
cycle_revision,
|
||||||
required_cycle,
|
leader_epoch,
|
||||||
&mut cycle_metrics_guard,
|
required_cycle,
|
||||||
)
|
&mut cycle_metrics_guard,
|
||||||
.await
|
)
|
||||||
{
|
.await
|
||||||
ScannerCycleOutcome::Partial
|
{
|
||||||
} else {
|
ScannerCycleOutcome::Partial
|
||||||
ScannerCycleOutcome::Failed
|
} else {
|
||||||
};
|
ScannerCycleOutcome::Failed
|
||||||
|
};
|
||||||
|
}
|
||||||
|
Some(ScannerCyclePreCommitOutcome::Deferred(reason)) => {
|
||||||
|
info!(
|
||||||
|
target: "rustfs::scanner",
|
||||||
|
event = EVENT_SCANNER_CYCLE_STATE,
|
||||||
|
component = LOG_COMPONENT_SCANNER,
|
||||||
|
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||||
|
cycle = cycle_info.current,
|
||||||
|
reason = reason.as_str(),
|
||||||
|
state = "deferred",
|
||||||
|
"Scanner cycle deferred before data usage publication"
|
||||||
|
);
|
||||||
|
emit_scan_cycle_deferred(cycle_start.elapsed());
|
||||||
|
mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await;
|
||||||
|
return ScannerCycleOutcome::Deferred(reason);
|
||||||
|
}
|
||||||
|
None => {}
|
||||||
}
|
}
|
||||||
if usage_persist_outcome == DataUsagePersistOutcome::Failed {
|
if usage_persist_outcome == DataUsagePersistOutcome::Failed {
|
||||||
error!(
|
error!(
|
||||||
@@ -2000,6 +2037,56 @@ impl Drop for ScannerScanModeGuard {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn final_data_usage_publication_defer_reason(
|
||||||
|
storeapi: &ECStore,
|
||||||
|
status: ScannerCycleStatus,
|
||||||
|
) -> Option<ScannerCycleDeferReason> {
|
||||||
|
match status {
|
||||||
|
ScannerCycleStatus::Complete | ScannerCycleStatus::Superseded => {
|
||||||
|
if storeapi.scanner_data_usage_publication_blocked().await {
|
||||||
|
return Some(ScannerCycleDeferReason::DataMovement);
|
||||||
|
}
|
||||||
|
if status == ScannerCycleStatus::Complete {
|
||||||
|
let distributed = storeapi.setup_is_dist_erasure().await;
|
||||||
|
match probe_scanner_activity(storeapi, distributed).await {
|
||||||
|
Ok(snapshot) if scanner_activity_allows_usage_publication(&snapshot) => None,
|
||||||
|
Ok(_) => Some(ScannerCycleDeferReason::DataMovement),
|
||||||
|
Err(_) => Some(ScannerCycleDeferReason::ActivityBaselineUnavailable),
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// A superseded cycle is explicitly observational and cannot
|
||||||
|
// replace the authoritative snapshot. It may still be
|
||||||
|
// persisted as a convergence baseline for the next cycle.
|
||||||
|
None
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ScannerCycleStatus::Deferred(reason) => Some(reason),
|
||||||
|
// Incomplete cycles do not publish a usage snapshot. Keep the
|
||||||
|
// decision permissive so existing partial-cycle handling remains
|
||||||
|
// unchanged if a future scanner path emits a bookkeeping update.
|
||||||
|
ScannerCycleStatus::Incomplete => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||||
|
enum ScannerCyclePreCommitOutcome {
|
||||||
|
RecoverCacheCycle(u64),
|
||||||
|
Deferred(ScannerCycleDeferReason),
|
||||||
|
}
|
||||||
|
|
||||||
|
fn scanner_cycle_pre_commit_outcome(
|
||||||
|
required_cycle_floor: Option<u64>,
|
||||||
|
usage_persist_outcome: &DataUsagePersistOutcome,
|
||||||
|
) -> Option<ScannerCyclePreCommitOutcome> {
|
||||||
|
// Keep the publication barrier fail-closed: `.bloomcycle.bin` uses the
|
||||||
|
// same routed writer and its floor must remain pending while data movement
|
||||||
|
// hides the source pool.
|
||||||
|
match usage_persist_outcome {
|
||||||
|
DataUsagePersistOutcome::Deferred(reason) => Some(ScannerCyclePreCommitOutcome::Deferred(*reason)),
|
||||||
|
_ => required_cycle_floor.map(ScannerCyclePreCommitOutcome::RecoverCacheCycle),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn scanner_cycle_completion_outcome(
|
fn scanner_cycle_completion_outcome(
|
||||||
scan_status: ScannerCycleStatus,
|
scan_status: ScannerCycleStatus,
|
||||||
usage_persist_outcome: DataUsagePersistOutcome,
|
usage_persist_outcome: DataUsagePersistOutcome,
|
||||||
@@ -2007,6 +2094,7 @@ fn scanner_cycle_completion_outcome(
|
|||||||
has_failed_dirty_usage: bool,
|
has_failed_dirty_usage: bool,
|
||||||
) -> ScannerCycleOutcome {
|
) -> ScannerCycleOutcome {
|
||||||
match (scan_status, usage_persist_outcome) {
|
match (scan_status, usage_persist_outcome) {
|
||||||
|
(_, DataUsagePersistOutcome::Deferred(reason)) => ScannerCycleOutcome::Deferred(reason),
|
||||||
(_, DataUsagePersistOutcome::Failed) => ScannerCycleOutcome::Failed,
|
(_, DataUsagePersistOutcome::Failed) => ScannerCycleOutcome::Failed,
|
||||||
(ScannerCycleStatus::Deferred(reason), DataUsagePersistOutcome::NoUpdate)
|
(ScannerCycleStatus::Deferred(reason), DataUsagePersistOutcome::NoUpdate)
|
||||||
if !has_dirty_usage && !has_failed_dirty_usage =>
|
if !has_dirty_usage && !has_failed_dirty_usage =>
|
||||||
|
|||||||
@@ -19,7 +19,6 @@ use crate::{
|
|||||||
ScannerObjectInfo as ObjectInfo, ScannerObjectOptions as ObjectOptions, ScannerPutObjReader as PutObjReader,
|
ScannerObjectInfo as ObjectInfo, ScannerObjectOptions as ObjectOptions, ScannerPutObjReader as PutObjReader,
|
||||||
init_bucket_metadata_sys_for_scanner_tests, init_ecstore_config_for_scanner_tests, init_local_disks_with_instance_ctx,
|
init_bucket_metadata_sys_for_scanner_tests, init_ecstore_config_for_scanner_tests, init_local_disks_with_instance_ctx,
|
||||||
};
|
};
|
||||||
use serial_test::serial;
|
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::io::Cursor;
|
use std::io::Cursor;
|
||||||
use std::task::Poll;
|
use std::task::Poll;
|
||||||
@@ -153,6 +152,7 @@ struct MemoryConfigStore {
|
|||||||
objects: Mutex<HashMap<String, Vec<u8>>>,
|
objects: Mutex<HashMap<String, Vec<u8>>>,
|
||||||
revisions: Mutex<HashMap<String, u64>>,
|
revisions: Mutex<HashMap<String, u64>>,
|
||||||
fail_put_number: Mutex<HashMap<String, usize>>,
|
fail_put_number: Mutex<HashMap<String, usize>>,
|
||||||
|
object_not_found_put_number: Mutex<HashMap<String, usize>>,
|
||||||
error_after_commit_put_number: Mutex<HashMap<String, usize>>,
|
error_after_commit_put_number: Mutex<HashMap<String, usize>>,
|
||||||
interleaving_puts: Mutex<HashMap<String, (usize, Vec<u8>)>>,
|
interleaving_puts: Mutex<HashMap<String, (usize, Vec<u8>)>>,
|
||||||
cancel_after_interleaving_puts: Mutex<HashMap<String, CancellationToken>>,
|
cancel_after_interleaving_puts: Mutex<HashMap<String, CancellationToken>>,
|
||||||
@@ -224,6 +224,9 @@ impl crate::storage_api::scanner_io::ObjectIO for MemoryConfigStore {
|
|||||||
if self.fail_put_number.lock().await.get(&key) == Some(&put_count) {
|
if self.fail_put_number.lock().await.get(&key) == Some(&put_count) {
|
||||||
return Err(EcstoreError::other("injected put failure"));
|
return Err(EcstoreError::other("injected put failure"));
|
||||||
}
|
}
|
||||||
|
if self.object_not_found_put_number.lock().await.get(&key) == Some(&put_count) {
|
||||||
|
return Err(EcstoreError::ObjectNotFound(bucket.to_string(), object.to_string()));
|
||||||
|
}
|
||||||
|
|
||||||
let interleaving_data = {
|
let interleaving_data = {
|
||||||
let mut interleaving_puts = self.interleaving_puts.lock().await;
|
let mut interleaving_puts = self.interleaving_puts.lock().await;
|
||||||
@@ -358,7 +361,6 @@ fn test_initial_scanner_delay_uses_configured_start_delay() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn test_initial_scanner_delay_uses_cycle_without_explicit_start_delay() {
|
fn test_initial_scanner_delay_uses_cycle_without_explicit_start_delay() {
|
||||||
with_var(ENV_SCANNER_CYCLE, Some("120"), || {
|
with_var(ENV_SCANNER_CYCLE, Some("120"), || {
|
||||||
crate::runtime_config::refresh_scanner_runtime_config_for_tests();
|
crate::runtime_config::refresh_scanner_runtime_config_for_tests();
|
||||||
@@ -405,7 +407,6 @@ fn test_initial_scanner_delay_keeps_delay_for_replication_without_buckets() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn test_scanner_cycle_max_duration_uses_env() {
|
fn test_scanner_cycle_max_duration_uses_env() {
|
||||||
with_var(ENV_SCANNER_CYCLE_MAX_DURATION_SECS, Some("42"), || {
|
with_var(ENV_SCANNER_CYCLE_MAX_DURATION_SECS, Some("42"), || {
|
||||||
assert_eq!(scanner_cycle_max_duration(), Some(Duration::from_secs(42)));
|
assert_eq!(scanner_cycle_max_duration(), Some(Duration::from_secs(42)));
|
||||||
@@ -413,7 +414,6 @@ fn test_scanner_cycle_max_duration_uses_env() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn test_scanner_cycle_max_duration_default_is_disabled() {
|
fn test_scanner_cycle_max_duration_default_is_disabled() {
|
||||||
with_var_unset(ENV_SCANNER_CYCLE_MAX_DURATION_SECS, || {
|
with_var_unset(ENV_SCANNER_CYCLE_MAX_DURATION_SECS, || {
|
||||||
assert_eq!(scanner_cycle_max_duration(), None);
|
assert_eq!(scanner_cycle_max_duration(), None);
|
||||||
@@ -457,7 +457,6 @@ async fn test_scanner_cycle_budget_drop_cancels_child_without_elapsed() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn test_scanner_cycle_budget_config_uses_work_budget_env() {
|
fn test_scanner_cycle_budget_config_uses_work_budget_env() {
|
||||||
with_var(ENV_SCANNER_CYCLE_MAX_OBJECTS, Some("100"), || {
|
with_var(ENV_SCANNER_CYCLE_MAX_OBJECTS, Some("100"), || {
|
||||||
with_var(ENV_SCANNER_CYCLE_MAX_DIRECTORIES, Some("25"), || {
|
with_var(ENV_SCANNER_CYCLE_MAX_DIRECTORIES, Some("25"), || {
|
||||||
@@ -469,7 +468,6 @@ fn test_scanner_cycle_budget_config_uses_work_budget_env() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn test_scanner_cycle_budget_config_disables_zero_work_budgets() {
|
fn test_scanner_cycle_budget_config_disables_zero_work_budgets() {
|
||||||
with_var(ENV_SCANNER_CYCLE_MAX_OBJECTS, Some("0"), || {
|
with_var(ENV_SCANNER_CYCLE_MAX_OBJECTS, Some("0"), || {
|
||||||
with_var(ENV_SCANNER_CYCLE_MAX_DIRECTORIES, Some("0"), || {
|
with_var(ENV_SCANNER_CYCLE_MAX_DIRECTORIES, Some("0"), || {
|
||||||
@@ -512,7 +510,6 @@ fn test_scan_cycle_partial_source_maps_budget_reason() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_mark_scan_cycle_idle_clears_published_cycle_state() {
|
async fn test_mark_scan_cycle_idle_clears_published_cycle_state() {
|
||||||
let mut cycle_info = CurrentCycle {
|
let mut cycle_info = CurrentCycle {
|
||||||
current: 12,
|
current: 12,
|
||||||
@@ -541,7 +538,6 @@ async fn test_mark_scan_cycle_idle_clears_published_cycle_state() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn scanner_cycle_metrics_guard_covers_published_first_cycle_lifetime() {
|
async fn scanner_cycle_metrics_guard_covers_published_first_cycle_lifetime() {
|
||||||
let cycle_started = Utc::now() - chrono::Duration::seconds(5);
|
let cycle_started = Utc::now() - chrono::Duration::seconds(5);
|
||||||
let mut cycle_info = CurrentCycle {
|
let mut cycle_info = CurrentCycle {
|
||||||
@@ -568,7 +564,6 @@ async fn scanner_cycle_metrics_guard_covers_published_first_cycle_lifetime() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn scanner_cycle_metrics_guard_keeps_active_cycle_published_during_finalization() {
|
async fn scanner_cycle_metrics_guard_keeps_active_cycle_published_during_finalization() {
|
||||||
let mut cycle_info = CurrentCycle {
|
let mut cycle_info = CurrentCycle {
|
||||||
current: 12,
|
current: 12,
|
||||||
@@ -593,7 +588,6 @@ async fn scanner_cycle_metrics_guard_keeps_active_cycle_published_during_finaliz
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn scanner_cycle_metrics_guard_drop_clears_activity() {
|
async fn scanner_cycle_metrics_guard_drop_clears_activity() {
|
||||||
let guard = ScannerCycleMetricsGuard::new(CurrentCycle {
|
let guard = ScannerCycleMetricsGuard::new(CurrentCycle {
|
||||||
current: 12,
|
current: 12,
|
||||||
@@ -611,7 +605,6 @@ async fn scanner_cycle_metrics_guard_drop_clears_activity() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn run_data_scanner_cycle_publishes_activity_for_owner_lifetime() {
|
async fn run_data_scanner_cycle_publishes_activity_for_owner_lifetime() {
|
||||||
let (_temp_dir, store) = setup_scanner_cycle_store().await;
|
let (_temp_dir, store) = setup_scanner_cycle_store().await;
|
||||||
let ctx = CancellationToken::new();
|
let ctx = CancellationToken::new();
|
||||||
@@ -662,7 +655,6 @@ async fn run_data_scanner_cycle_publishes_activity_for_owner_lifetime() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_finalize_partial_scan_cycle_advances_and_persists_counter() {
|
async fn test_finalize_partial_scan_cycle_advances_and_persists_counter() {
|
||||||
let store = Arc::new(MemoryConfigStore::default());
|
let store = Arc::new(MemoryConfigStore::default());
|
||||||
let ctx = CancellationToken::new();
|
let ctx = CancellationToken::new();
|
||||||
@@ -698,7 +690,6 @@ async fn test_finalize_partial_scan_cycle_advances_and_persists_counter() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn scanner_cycle_recovers_to_newer_durable_cache_floor() {
|
async fn scanner_cycle_recovers_to_newer_durable_cache_floor() {
|
||||||
let store = Arc::new(MemoryConfigStore::default());
|
let store = Arc::new(MemoryConfigStore::default());
|
||||||
let ctx = CancellationToken::new();
|
let ctx = CancellationToken::new();
|
||||||
@@ -738,7 +729,6 @@ async fn scanner_cycle_recovers_to_newer_durable_cache_floor() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn scanner_cycle_rejects_invalid_cache_floor() {
|
async fn scanner_cycle_rejects_invalid_cache_floor() {
|
||||||
let store = Arc::new(MemoryConfigStore::default());
|
let store = Arc::new(MemoryConfigStore::default());
|
||||||
let ctx = CancellationToken::new();
|
let ctx = CancellationToken::new();
|
||||||
@@ -1009,7 +999,6 @@ async fn scanner_usage_floor_fails_closed_on_corrupt_or_exhausted_usage_state()
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn scanner_usage_backup_uses_durable_cycle_cadence_across_tasks() {
|
async fn scanner_usage_backup_uses_durable_cycle_cadence_across_tasks() {
|
||||||
let store = Arc::new(MemoryConfigStore::default());
|
let store = Arc::new(MemoryConfigStore::default());
|
||||||
let ctx = CancellationToken::new();
|
let ctx = CancellationToken::new();
|
||||||
@@ -1083,7 +1072,6 @@ fn scanner_cycle_advance_fails_before_reserved_exhausted_value() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_finalize_partial_scan_cycle_reports_persist_failure() {
|
async fn test_finalize_partial_scan_cycle_reports_persist_failure() {
|
||||||
let store = Arc::new(MemoryConfigStore::default());
|
let store = Arc::new(MemoryConfigStore::default());
|
||||||
let ctx = CancellationToken::new();
|
let ctx = CancellationToken::new();
|
||||||
@@ -1107,7 +1095,6 @@ async fn test_finalize_partial_scan_cycle_reports_persist_failure() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_persist_scanner_cycle_state_reconciles_newer_winner() {
|
async fn test_persist_scanner_cycle_state_reconciles_newer_winner() {
|
||||||
let store = Arc::new(MemoryConfigStore::default());
|
let store = Arc::new(MemoryConfigStore::default());
|
||||||
let ctx = CancellationToken::new();
|
let ctx = CancellationToken::new();
|
||||||
@@ -1431,6 +1418,168 @@ async fn test_store_data_usage_in_backend_preserves_newer_snapshot() {
|
|||||||
assert_eq!(outcome, DataUsagePersistOutcome::Current);
|
assert_eq!(outcome, DataUsagePersistOutcome::Current);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_usage_save_object_not_found_defers_only_with_a_fresh_route_barrier() {
|
||||||
|
for (route_blocked, expected) in [
|
||||||
|
(true, DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement)),
|
||||||
|
(false, DataUsagePersistOutcome::Failed),
|
||||||
|
] {
|
||||||
|
let store = Arc::new(MemoryConfigStore::default());
|
||||||
|
let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str());
|
||||||
|
let baseline = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(10)), 1);
|
||||||
|
let baseline_data = serde_json::to_vec(&baseline).expect("baseline usage snapshot should encode");
|
||||||
|
store.objects.lock().await.insert(key.clone(), baseline_data.clone());
|
||||||
|
store.revisions.lock().await.insert(key.clone(), 1);
|
||||||
|
store.object_not_found_put_number.lock().await.insert(key.clone(), 1);
|
||||||
|
|
||||||
|
let (sender, receiver) = mpsc::channel(1);
|
||||||
|
sender
|
||||||
|
.send(complete_usage_with_bucket_count(
|
||||||
|
Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)),
|
||||||
|
2,
|
||||||
|
))
|
||||||
|
.await
|
||||||
|
.expect("new usage snapshot should enqueue");
|
||||||
|
drop(sender);
|
||||||
|
let probe_calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||||
|
let route_probe_calls = probe_calls.clone();
|
||||||
|
|
||||||
|
let outcome = store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe(
|
||||||
|
CancellationToken::new(),
|
||||||
|
store.clone(),
|
||||||
|
receiver,
|
||||||
|
None,
|
||||||
|
Some(DataUsagePersistBaseline {
|
||||||
|
data: Some(Bytes::from(baseline_data.clone())),
|
||||||
|
revision: DataUsageCacheRevision::Etag("memory-1".to_string()),
|
||||||
|
}),
|
||||||
|
move || {
|
||||||
|
let probe_calls = route_probe_calls.clone();
|
||||||
|
async move {
|
||||||
|
let call = probe_calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||||
|
route_blocked && call > 1
|
||||||
|
}
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
assert_eq!(outcome, expected);
|
||||||
|
assert_eq!(
|
||||||
|
probe_calls.load(std::sync::atomic::Ordering::SeqCst),
|
||||||
|
3,
|
||||||
|
"ObjectNotFound must be followed by a fresh route-barrier probe"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
store.objects.lock().await.get(&key),
|
||||||
|
Some(&baseline_data),
|
||||||
|
"a route failure must not replace the authoritative baseline"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_usage_save_route_barrier_prevents_missing_snapshot_creation() {
|
||||||
|
for observational in [false, true] {
|
||||||
|
let store = Arc::new(MemoryConfigStore::default());
|
||||||
|
let target_path = if observational {
|
||||||
|
DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str()
|
||||||
|
} else {
|
||||||
|
DATA_USAGE_OBJ_NAME_PATH.as_str()
|
||||||
|
};
|
||||||
|
let target_key = memory_config_key(RUSTFS_META_BUCKET, target_path);
|
||||||
|
let mut incoming = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)), 1);
|
||||||
|
incoming.usage_snapshot_converged = Some(!observational);
|
||||||
|
let (sender, receiver) = mpsc::channel(1);
|
||||||
|
sender.send(incoming).await.expect("usage snapshot should enqueue");
|
||||||
|
drop(sender);
|
||||||
|
|
||||||
|
let outcome = store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe(
|
||||||
|
CancellationToken::new(),
|
||||||
|
store.clone(),
|
||||||
|
receiver,
|
||||||
|
None,
|
||||||
|
Some(DataUsagePersistBaseline {
|
||||||
|
data: None,
|
||||||
|
revision: DataUsageCacheRevision::Missing,
|
||||||
|
}),
|
||||||
|
|| async { true },
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
assert_eq!(outcome, DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement));
|
||||||
|
assert!(!store.objects.lock().await.contains_key(&target_key));
|
||||||
|
assert_eq!(
|
||||||
|
store.put_counts.lock().await.get(&target_key),
|
||||||
|
None,
|
||||||
|
"the final pool-state fence must run before the first PUT"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_usage_route_barrier_precedes_durable_reconciliation() {
|
||||||
|
let store = Arc::new(MemoryConfigStore::default());
|
||||||
|
let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str());
|
||||||
|
let snapshot = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)), 1);
|
||||||
|
let snapshot_data = serde_json::to_vec(&snapshot).expect("usage snapshot should encode");
|
||||||
|
let (sender, receiver) = mpsc::channel(1);
|
||||||
|
sender.send(snapshot).await.expect("usage snapshot should enqueue");
|
||||||
|
drop(sender);
|
||||||
|
|
||||||
|
let outcome = store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe(
|
||||||
|
CancellationToken::new(),
|
||||||
|
store.clone(),
|
||||||
|
receiver,
|
||||||
|
None,
|
||||||
|
Some(DataUsagePersistBaseline {
|
||||||
|
data: Some(Bytes::from(snapshot_data)),
|
||||||
|
revision: DataUsageCacheRevision::Etag("memory-1".to_string()),
|
||||||
|
}),
|
||||||
|
|| async { true },
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
assert_eq!(outcome, DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement));
|
||||||
|
assert_eq!(store.put_counts.lock().await.get(&key), None);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_deferred_usage_save_keeps_last_real_save_metric() {
|
||||||
|
let metrics = global_metrics();
|
||||||
|
metrics.record_scanner_usage_save_result(ScannerUsageSaveResult::Success);
|
||||||
|
let before = metrics.report().await.usage_freshness;
|
||||||
|
|
||||||
|
let store = Arc::new(MemoryConfigStore::default());
|
||||||
|
let (sender, receiver) = mpsc::channel(1);
|
||||||
|
sender
|
||||||
|
.send(complete_usage_with_bucket_count(
|
||||||
|
Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)),
|
||||||
|
1,
|
||||||
|
))
|
||||||
|
.await
|
||||||
|
.expect("usage snapshot should enqueue");
|
||||||
|
drop(sender);
|
||||||
|
|
||||||
|
let outcome = store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe(
|
||||||
|
CancellationToken::new(),
|
||||||
|
store,
|
||||||
|
receiver,
|
||||||
|
None,
|
||||||
|
Some(DataUsagePersistBaseline {
|
||||||
|
data: None,
|
||||||
|
revision: DataUsageCacheRevision::Missing,
|
||||||
|
}),
|
||||||
|
|| async { true },
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
assert_eq!(outcome, DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement));
|
||||||
|
let after = metrics.report().await.usage_freshness;
|
||||||
|
assert_eq!(after.last_usage_save_result, before.last_usage_save_result);
|
||||||
|
assert_eq!(after.last_usage_save_result_code, before.last_usage_save_result_code);
|
||||||
|
assert_eq!(after.last_usage_save_unix_secs, before.last_usage_save_unix_secs);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn test_store_data_usage_in_backend_fences_interleaving_newer_writer() {
|
async fn test_store_data_usage_in_backend_fences_interleaving_newer_writer() {
|
||||||
let store = Arc::new(MemoryConfigStore::default());
|
let store = Arc::new(MemoryConfigStore::default());
|
||||||
@@ -2325,6 +2474,15 @@ async fn test_store_data_usage_in_backend_reports_missing_snapshot() {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_scanner_cycle_completion_prioritizes_persist_failure() {
|
fn test_scanner_cycle_completion_prioritizes_persist_failure() {
|
||||||
|
assert_eq!(
|
||||||
|
scanner_cycle_completion_outcome(
|
||||||
|
ScannerCycleStatus::Complete,
|
||||||
|
DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement),
|
||||||
|
true,
|
||||||
|
false,
|
||||||
|
),
|
||||||
|
ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement)
|
||||||
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
scanner_cycle_completion_outcome(
|
scanner_cycle_completion_outcome(
|
||||||
ScannerCycleStatus::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable),
|
ScannerCycleStatus::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable),
|
||||||
@@ -2422,7 +2580,33 @@ fn test_scanner_cycle_completion_prioritizes_persist_failure() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
fn scanner_cycle_cache_floor_stays_pending_during_deferred_usage_publication() {
|
||||||
|
for reason in [
|
||||||
|
ScannerCycleDeferReason::DataMovement,
|
||||||
|
ScannerCycleDeferReason::ActivityBaselineUnavailable,
|
||||||
|
] {
|
||||||
|
let deferred = DataUsagePersistOutcome::Deferred(reason);
|
||||||
|
assert_eq!(
|
||||||
|
scanner_cycle_pre_commit_outcome(Some(19), &deferred),
|
||||||
|
Some(ScannerCyclePreCommitOutcome::Deferred(reason)),
|
||||||
|
"a blocked publication must not persist the routed scanner cycle floor"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
scanner_cycle_pre_commit_outcome(None, &deferred),
|
||||||
|
Some(ScannerCyclePreCommitOutcome::Deferred(reason))
|
||||||
|
);
|
||||||
|
}
|
||||||
|
assert_eq!(
|
||||||
|
scanner_cycle_pre_commit_outcome(Some(19), &DataUsagePersistOutcome::Saved),
|
||||||
|
Some(ScannerCyclePreCommitOutcome::RecoverCacheCycle(19))
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
scanner_cycle_pre_commit_outcome(Some(19), &DataUsagePersistOutcome::Failed),
|
||||||
|
Some(ScannerCyclePreCommitOutcome::RecoverCacheCycle(19))
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
fn finalizing_a_saved_cycle_acknowledges_its_exact_dirty_snapshot() {
|
fn finalizing_a_saved_cycle_acknowledges_its_exact_dirty_snapshot() {
|
||||||
crate::scanner_io::clear_dirty_usage_bucket("photos");
|
crate::scanner_io::clear_dirty_usage_bucket("photos");
|
||||||
crate::scanner_io::record_dirty_usage_bucket("photos");
|
crate::scanner_io::record_dirty_usage_bucket("photos");
|
||||||
@@ -2448,6 +2632,22 @@ fn finalizing_a_saved_cycle_acknowledges_its_exact_dirty_snapshot() {
|
|||||||
assert!(!crate::scanner_io::dirty_usage_buckets_pending());
|
assert!(!crate::scanner_io::dirty_usage_buckets_pending());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn finalizing_a_deferred_usage_save_keeps_dirty_work_pending() {
|
||||||
|
crate::scanner_io::clear_dirty_usage_bucket("photos");
|
||||||
|
crate::scanner_io::record_dirty_usage_bucket("photos");
|
||||||
|
let dirty_snapshot = crate::scanner_io::dirty_usage_buckets_for_tests();
|
||||||
|
let deferred = crate::scanner_io::ScannerCycleResult::new(ScannerCycleStatus::Complete, Some(dirty_snapshot));
|
||||||
|
|
||||||
|
let (outcome, _, acknowledgements) =
|
||||||
|
finalize_scanner_cycle_result(deferred, DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement));
|
||||||
|
|
||||||
|
assert_eq!(outcome, ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement));
|
||||||
|
assert!(acknowledgements.is_empty());
|
||||||
|
assert!(crate::scanner_io::dirty_usage_buckets_pending());
|
||||||
|
crate::scanner_io::clear_dirty_usage_bucket("photos");
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn scanner_cycle_keeps_remote_pending_acknowledgement() {
|
async fn scanner_cycle_keeps_remote_pending_acknowledgement() {
|
||||||
let pending = remote_dirty_usage_acknowledgement_pending(7, 1, std::future::ready(Ok::<bool, std::io::Error>(true))).await;
|
let pending = remote_dirty_usage_acknowledgement_pending(7, 1, std::future::ready(Ok::<bool, std::io::Error>(true))).await;
|
||||||
@@ -2475,7 +2675,6 @@ async fn scanner_cycle_keeps_remote_pending_acknowledgement() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn finalizing_an_already_durable_cycle_acknowledges_its_exact_dirty_snapshot() {
|
fn finalizing_an_already_durable_cycle_acknowledges_its_exact_dirty_snapshot() {
|
||||||
crate::scanner_io::clear_dirty_usage_bucket("photos");
|
crate::scanner_io::clear_dirty_usage_bucket("photos");
|
||||||
crate::scanner_io::record_dirty_usage_bucket("photos");
|
crate::scanner_io::record_dirty_usage_bucket("photos");
|
||||||
@@ -2490,7 +2689,6 @@ fn finalizing_an_already_durable_cycle_acknowledges_its_exact_dirty_snapshot() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn finalizing_a_prior_same_cycle_snapshot_keeps_new_dirty_work_pending() {
|
fn finalizing_a_prior_same_cycle_snapshot_keeps_new_dirty_work_pending() {
|
||||||
crate::scanner_io::clear_dirty_usage_bucket("photos");
|
crate::scanner_io::clear_dirty_usage_bucket("photos");
|
||||||
crate::scanner_io::record_dirty_usage_bucket("photos");
|
crate::scanner_io::record_dirty_usage_bucket("photos");
|
||||||
@@ -2506,7 +2704,6 @@ fn finalizing_a_prior_same_cycle_snapshot_keeps_new_dirty_work_pending() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn finalizing_a_durable_superseded_snapshot_keeps_dirty_work_pending() {
|
fn finalizing_a_durable_superseded_snapshot_keeps_dirty_work_pending() {
|
||||||
crate::scanner_io::clear_dirty_usage_bucket("photos");
|
crate::scanner_io::clear_dirty_usage_bucket("photos");
|
||||||
crate::scanner_io::record_dirty_usage_bucket("photos");
|
crate::scanner_io::record_dirty_usage_bucket("photos");
|
||||||
@@ -2522,7 +2719,6 @@ fn finalizing_a_durable_superseded_snapshot_keeps_dirty_work_pending() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn data_usage_persist_wait_covers_cache_retries_and_backup() {
|
fn data_usage_persist_wait_covers_cache_retries_and_backup() {
|
||||||
with_var(rustfs_config::ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS, Some("7"), || {
|
with_var(rustfs_config::ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS, Some("7"), || {
|
||||||
crate::runtime_config::refresh_scanner_runtime_config_for_tests();
|
crate::runtime_config::refresh_scanner_runtime_config_for_tests();
|
||||||
@@ -2575,7 +2771,6 @@ async fn maintenance_feature_inspection_preserves_base_cycle_after_timeout() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test(start_paused = true)]
|
#[tokio::test(start_paused = true)]
|
||||||
#[serial]
|
|
||||||
async fn stable_maintenance_detection_preserves_base_cycle_after_timeout() {
|
async fn stable_maintenance_detection_preserves_base_cycle_after_timeout() {
|
||||||
let ctx = CancellationToken::new();
|
let ctx = CancellationToken::new();
|
||||||
|
|
||||||
@@ -2641,7 +2836,6 @@ async fn maintenance_feature_inspection_stops_on_cancellation() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn test_cycle_interval_prefers_explicit_cycle_override() {
|
fn test_cycle_interval_prefers_explicit_cycle_override() {
|
||||||
with_var(ENV_SCANNER_SPEED, Some("slowest"), || {
|
with_var(ENV_SCANNER_SPEED, Some("slowest"), || {
|
||||||
with_var(ENV_SCANNER_CYCLE, Some("42"), || {
|
with_var(ENV_SCANNER_CYCLE, Some("42"), || {
|
||||||
@@ -2651,7 +2845,6 @@ fn test_cycle_interval_prefers_explicit_cycle_override() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn test_cycle_interval_prefers_explicit_cycle_over_default_cycle() {
|
fn test_cycle_interval_prefers_explicit_cycle_over_default_cycle() {
|
||||||
let _guard = ScannerDefaultCycleGuard::set(TEST_DEFAULT_SCANNER_CYCLE_SECS);
|
let _guard = ScannerDefaultCycleGuard::set(TEST_DEFAULT_SCANNER_CYCLE_SECS);
|
||||||
|
|
||||||
@@ -2661,7 +2854,6 @@ fn test_cycle_interval_prefers_explicit_cycle_over_default_cycle() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn test_cycle_interval_uses_scanner_default_speed_override_when_unconfigured() {
|
fn test_cycle_interval_uses_scanner_default_speed_override_when_unconfigured() {
|
||||||
let _guard = ScannerDefaultSpeedGuard::set(ScannerSpeed::Slowest);
|
let _guard = ScannerDefaultSpeedGuard::set(ScannerSpeed::Slowest);
|
||||||
|
|
||||||
@@ -2671,7 +2863,6 @@ fn test_cycle_interval_uses_scanner_default_speed_override_when_unconfigured() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn test_cycle_interval_prefers_explicit_speed_over_default_speed_override() {
|
fn test_cycle_interval_prefers_explicit_speed_over_default_speed_override() {
|
||||||
let _guard = ScannerDefaultSpeedGuard::set(ScannerSpeed::Slowest);
|
let _guard = ScannerDefaultSpeedGuard::set(ScannerSpeed::Slowest);
|
||||||
|
|
||||||
@@ -2689,7 +2880,6 @@ fn test_cycle_interval_prefers_explicit_speed_over_default_speed_override() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn test_cycle_interval_uses_default_cycle_override_when_unconfigured() {
|
fn test_cycle_interval_uses_default_cycle_override_when_unconfigured() {
|
||||||
let _guard = ScannerDefaultCycleGuard::set(TEST_DEFAULT_SCANNER_CYCLE_SECS);
|
let _guard = ScannerDefaultCycleGuard::set(TEST_DEFAULT_SCANNER_CYCLE_SECS);
|
||||||
|
|
||||||
@@ -2853,7 +3043,6 @@ fn scanner_cycle_wait_plan_drives_growth_resets_and_bitrot_cap() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn scanner_cycle_schedule_status_reports_effective_backoff() {
|
fn scanner_cycle_schedule_status_reports_effective_backoff() {
|
||||||
record_scanner_cycle_schedule(Duration::from_millis(86_400_001), true, 2_048, true, 7);
|
record_scanner_cycle_schedule(Duration::from_millis(86_400_001), true, 2_048, true, 7);
|
||||||
|
|
||||||
@@ -3133,7 +3322,6 @@ fn dirty_usage_wakes_are_disabled_for_explicit_cycle_policy() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn clean_idle_cap_preserves_default_bitrot_coverage_window() {
|
fn clean_idle_cap_preserves_default_bitrot_coverage_window() {
|
||||||
let config = ScannerRuntimeConfig {
|
let config = ScannerRuntimeConfig {
|
||||||
bitrot_cycle: Some(Duration::from_secs(30 * 24 * 60 * 60)),
|
bitrot_cycle: Some(Duration::from_secs(30 * 24 * 60 * 60)),
|
||||||
@@ -3163,7 +3351,6 @@ fn clean_idle_cap_allows_policy_max_when_bitrot_is_disabled() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn clean_idle_cap_never_shortens_the_base_cycle() {
|
fn clean_idle_cap_never_shortens_the_base_cycle() {
|
||||||
let config = ScannerRuntimeConfig {
|
let config = ScannerRuntimeConfig {
|
||||||
bitrot_cycle: Some(Duration::from_secs(60)),
|
bitrot_cycle: Some(Duration::from_secs(60)),
|
||||||
@@ -3177,7 +3364,6 @@ fn clean_idle_cap_never_shortens_the_base_cycle() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn test_cycle_interval_keeps_default_cycle_with_explicit_speed() {
|
fn test_cycle_interval_keeps_default_cycle_with_explicit_speed() {
|
||||||
let _guard = ScannerDefaultCycleGuard::set(TEST_DEFAULT_SCANNER_CYCLE_SECS);
|
let _guard = ScannerDefaultCycleGuard::set(TEST_DEFAULT_SCANNER_CYCLE_SECS);
|
||||||
|
|
||||||
@@ -3195,7 +3381,6 @@ fn test_cycle_interval_keeps_default_cycle_with_explicit_speed() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn test_cycle_interval_prefers_explicit_start_delay_over_default_cycle() {
|
fn test_cycle_interval_prefers_explicit_start_delay_over_default_cycle() {
|
||||||
let _guard = ScannerDefaultCycleGuard::set(TEST_DEFAULT_SCANNER_CYCLE_SECS);
|
let _guard = ScannerDefaultCycleGuard::set(TEST_DEFAULT_SCANNER_CYCLE_SECS);
|
||||||
|
|
||||||
@@ -3209,7 +3394,6 @@ fn test_cycle_interval_prefers_explicit_start_delay_over_default_cycle() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn test_cycle_interval_supports_minio_speed_alias() {
|
fn test_cycle_interval_supports_minio_speed_alias() {
|
||||||
with_var_unset(ENV_SCANNER_SPEED, || {
|
with_var_unset(ENV_SCANNER_SPEED, || {
|
||||||
with_var_unset(ENV_SCANNER_CYCLE, || {
|
with_var_unset(ENV_SCANNER_CYCLE, || {
|
||||||
@@ -3223,7 +3407,6 @@ fn test_cycle_interval_supports_minio_speed_alias() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn test_cycle_interval_supports_minio_cycle_alias() {
|
fn test_cycle_interval_supports_minio_cycle_alias() {
|
||||||
with_var_unset(ENV_SCANNER_CYCLE, || {
|
with_var_unset(ENV_SCANNER_CYCLE, || {
|
||||||
with_var_unset(ENV_SCANNER_START_DELAY_SECS, || {
|
with_var_unset(ENV_SCANNER_START_DELAY_SECS, || {
|
||||||
@@ -3243,7 +3426,6 @@ fn test_randomized_cycle_delay_handles_small_start_delay() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_wait_for_next_scanner_cycle_wakes_for_dirty_usage() {
|
async fn test_wait_for_next_scanner_cycle_wakes_for_dirty_usage() {
|
||||||
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
||||||
|
|
||||||
@@ -3269,7 +3451,6 @@ async fn test_wait_for_next_scanner_cycle_wakes_for_dirty_usage() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_wait_for_next_scanner_cycle_sees_unattempted_dirty_usage() {
|
async fn test_wait_for_next_scanner_cycle_sees_unattempted_dirty_usage() {
|
||||||
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
||||||
let dirty_generation = crate::scanner_io::dirty_usage_generation();
|
let dirty_generation = crate::scanner_io::dirty_usage_generation();
|
||||||
@@ -3291,7 +3472,6 @@ async fn test_wait_for_next_scanner_cycle_sees_unattempted_dirty_usage() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test(start_paused = true)]
|
#[tokio::test(start_paused = true)]
|
||||||
#[serial]
|
|
||||||
async fn test_wait_for_next_scanner_cycle_retries_stable_dirty_usage_on_timer() {
|
async fn test_wait_for_next_scanner_cycle_retries_stable_dirty_usage_on_timer() {
|
||||||
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
||||||
crate::scanner_io::record_dirty_usage_bucket("photos");
|
crate::scanner_io::record_dirty_usage_bucket("photos");
|
||||||
@@ -3313,7 +3493,6 @@ async fn test_wait_for_next_scanner_cycle_retries_stable_dirty_usage_on_timer()
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test(start_paused = true)]
|
#[tokio::test(start_paused = true)]
|
||||||
#[serial]
|
|
||||||
async fn test_wait_for_next_scanner_cycle_can_defer_dirty_wakes_until_timer() {
|
async fn test_wait_for_next_scanner_cycle_can_defer_dirty_wakes_until_timer() {
|
||||||
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
||||||
let ctx = CancellationToken::new();
|
let ctx = CancellationToken::new();
|
||||||
@@ -3332,7 +3511,6 @@ async fn test_wait_for_next_scanner_cycle_can_defer_dirty_wakes_until_timer() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_wait_for_next_scanner_cycle_wakes_for_repeated_dirty_bucket() {
|
async fn test_wait_for_next_scanner_cycle_wakes_for_repeated_dirty_bucket() {
|
||||||
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
||||||
crate::scanner_io::record_dirty_usage_bucket("photos");
|
crate::scanner_io::record_dirty_usage_bucket("photos");
|
||||||
@@ -3358,7 +3536,6 @@ async fn test_wait_for_next_scanner_cycle_wakes_for_repeated_dirty_bucket() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_wait_for_next_scanner_cycle_reschedules_for_runtime_config() {
|
async fn test_wait_for_next_scanner_cycle_reschedules_for_runtime_config() {
|
||||||
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
||||||
let observed_generation = crate::runtime_config::scanner_runtime_config_generation();
|
let observed_generation = crate::runtime_config::scanner_runtime_config_generation();
|
||||||
@@ -3386,7 +3563,6 @@ async fn test_wait_for_next_scanner_cycle_reschedules_for_runtime_config() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_wait_for_next_scanner_cycle_reschedules_for_maintenance_change() {
|
async fn test_wait_for_next_scanner_cycle_reschedules_for_maintenance_change() {
|
||||||
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
||||||
let observed_generation = crate::scanner_io::scanner_maintenance_generation();
|
let observed_generation = crate::scanner_io::scanner_maintenance_generation();
|
||||||
@@ -3630,7 +3806,6 @@ fn scanner_activity_after_a_cycle_restores_the_base_interval() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test(start_paused = true)]
|
#[tokio::test(start_paused = true)]
|
||||||
#[serial]
|
|
||||||
async fn distributed_clean_idle_wait_wakes_at_base_interval_for_remote_activity() {
|
async fn distributed_clean_idle_wait_wakes_at_base_interval_for_remote_activity() {
|
||||||
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
||||||
let ctx = CancellationToken::new();
|
let ctx = CancellationToken::new();
|
||||||
@@ -3658,7 +3833,6 @@ async fn distributed_clean_idle_wait_wakes_at_base_interval_for_remote_activity(
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test(start_paused = true)]
|
#[tokio::test(start_paused = true)]
|
||||||
#[serial]
|
|
||||||
async fn superseded_retry_wait_defers_dirty_cluster_activity_until_timer() {
|
async fn superseded_retry_wait_defers_dirty_cluster_activity_until_timer() {
|
||||||
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
||||||
let ctx = CancellationToken::new();
|
let ctx = CancellationToken::new();
|
||||||
@@ -3686,7 +3860,6 @@ async fn superseded_retry_wait_defers_dirty_cluster_activity_until_timer() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test(start_paused = true)]
|
#[tokio::test(start_paused = true)]
|
||||||
#[serial]
|
|
||||||
async fn distributed_clean_idle_wait_blocks_backoff_for_unpropagated_maintenance() {
|
async fn distributed_clean_idle_wait_blocks_backoff_for_unpropagated_maintenance() {
|
||||||
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
||||||
let ctx = CancellationToken::new();
|
let ctx = CancellationToken::new();
|
||||||
@@ -3713,7 +3886,6 @@ async fn distributed_clean_idle_wait_blocks_backoff_for_unpropagated_maintenance
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test(start_paused = true)]
|
#[tokio::test(start_paused = true)]
|
||||||
#[serial]
|
|
||||||
async fn distributed_clean_idle_wait_fails_closed_when_a_peer_is_unverifiable() {
|
async fn distributed_clean_idle_wait_fails_closed_when_a_peer_is_unverifiable() {
|
||||||
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
||||||
let ctx = CancellationToken::new();
|
let ctx = CancellationToken::new();
|
||||||
@@ -3740,7 +3912,6 @@ async fn distributed_clean_idle_wait_fails_closed_when_a_peer_is_unverifiable()
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test(start_paused = true)]
|
#[tokio::test(start_paused = true)]
|
||||||
#[serial]
|
|
||||||
async fn distributed_clean_idle_wait_keeps_the_extended_deadline_when_peers_are_clean() {
|
async fn distributed_clean_idle_wait_keeps_the_extended_deadline_when_peers_are_clean() {
|
||||||
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
||||||
let ctx = CancellationToken::new();
|
let ctx = CancellationToken::new();
|
||||||
@@ -3768,7 +3939,6 @@ async fn distributed_clean_idle_wait_keeps_the_extended_deadline_when_peers_are_
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test(start_paused = true)]
|
#[tokio::test(start_paused = true)]
|
||||||
#[serial]
|
|
||||||
async fn scanner_activity_probe_wait_is_cancellation_aware() {
|
async fn scanner_activity_probe_wait_is_cancellation_aware() {
|
||||||
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
||||||
let ctx = CancellationToken::new();
|
let ctx = CancellationToken::new();
|
||||||
@@ -3799,7 +3969,6 @@ async fn scanner_activity_probe_wait_is_cancellation_aware() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test(start_paused = true)]
|
#[tokio::test(start_paused = true)]
|
||||||
#[serial]
|
|
||||||
async fn scanner_activity_probe_wait_stops_after_leader_lock_loss() {
|
async fn scanner_activity_probe_wait_stops_after_leader_lock_loss() {
|
||||||
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
||||||
let ctx = CancellationToken::new();
|
let ctx = CancellationToken::new();
|
||||||
@@ -3831,7 +4000,6 @@ async fn scanner_activity_probe_wait_stops_after_leader_lock_loss() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn test_get_cycle_scan_mode_runs_deep_until_selection_window_completes() {
|
fn test_get_cycle_scan_mode_runs_deep_until_selection_window_completes() {
|
||||||
with_var(ENV_SCANNER_BITROT_CYCLE_SECS, Some("3600"), || {
|
with_var(ENV_SCANNER_BITROT_CYCLE_SECS, Some("3600"), || {
|
||||||
let mode = get_cycle_scan_mode(10, 0, Some(Utc::now()), bitrot_scan_cycle());
|
let mode = get_cycle_scan_mode(10, 0, Some(Utc::now()), bitrot_scan_cycle());
|
||||||
@@ -3840,7 +4008,6 @@ fn test_get_cycle_scan_mode_runs_deep_until_selection_window_completes() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn test_get_cycle_scan_mode_respects_elapsed_bitrot_cycle() {
|
fn test_get_cycle_scan_mode_respects_elapsed_bitrot_cycle() {
|
||||||
with_var(ENV_SCANNER_BITROT_CYCLE_SECS, Some("3600"), || {
|
with_var(ENV_SCANNER_BITROT_CYCLE_SECS, Some("3600"), || {
|
||||||
let recent = Utc::now() - chrono::Duration::minutes(30);
|
let recent = Utc::now() - chrono::Duration::minutes(30);
|
||||||
@@ -3852,7 +4019,6 @@ fn test_get_cycle_scan_mode_respects_elapsed_bitrot_cycle() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn test_get_cycle_scan_mode_can_disable_periodic_deep_scan() {
|
fn test_get_cycle_scan_mode_can_disable_periodic_deep_scan() {
|
||||||
with_var(ENV_SCANNER_BITROT_CYCLE_SECS, Some("off"), || {
|
with_var(ENV_SCANNER_BITROT_CYCLE_SECS, Some("off"), || {
|
||||||
assert_eq!(get_cycle_scan_mode(1, 0, None, bitrot_scan_cycle()), HealScanMode::Normal);
|
assert_eq!(get_cycle_scan_mode(1, 0, None, bitrot_scan_cycle()), HealScanMode::Normal);
|
||||||
@@ -3860,7 +4026,6 @@ fn test_get_cycle_scan_mode_can_disable_periodic_deep_scan() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn test_background_heal_info_for_scan_start_marks_deep_active() {
|
fn test_background_heal_info_for_scan_start_marks_deep_active() {
|
||||||
let now = Utc::now();
|
let now = Utc::now();
|
||||||
let info =
|
let info =
|
||||||
@@ -3873,7 +4038,6 @@ fn test_background_heal_info_for_scan_start_marks_deep_active() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn test_background_heal_info_for_scan_start_keeps_deep_window_start() {
|
fn test_background_heal_info_for_scan_start_keeps_deep_window_start() {
|
||||||
with_var_unset(ENV_SCANNER_BITROT_CYCLE_SECS, || {
|
with_var_unset(ENV_SCANNER_BITROT_CYCLE_SECS, || {
|
||||||
let started_at = Utc::now();
|
let started_at = Utc::now();
|
||||||
|
|||||||
@@ -22,6 +22,10 @@ pub(super) enum DataUsagePersistOutcome {
|
|||||||
AlreadyDurable,
|
AlreadyDurable,
|
||||||
PriorCycleDurable,
|
PriorCycleDurable,
|
||||||
Saved,
|
Saved,
|
||||||
|
/// The metadata route is temporarily unavailable (for example while a
|
||||||
|
/// terminal decommission state keeps the source pool suspended). The
|
||||||
|
/// caller must retry without acknowledging dirty usage.
|
||||||
|
Deferred(ScannerCycleDeferReason),
|
||||||
Failed,
|
Failed,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -92,10 +96,33 @@ pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch(
|
|||||||
pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_baseline(
|
pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_baseline(
|
||||||
ctx: CancellationToken,
|
ctx: CancellationToken,
|
||||||
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
|
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
|
||||||
mut receiver: mpsc::Receiver<DataUsageInfo>,
|
receiver: mpsc::Receiver<DataUsageInfo>,
|
||||||
leader_epoch: Option<u64>,
|
leader_epoch: Option<u64>,
|
||||||
initial_baseline: Option<DataUsagePersistBaseline>,
|
initial_baseline: Option<DataUsagePersistBaseline>,
|
||||||
) -> DataUsagePersistOutcome {
|
) -> DataUsagePersistOutcome {
|
||||||
|
store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe(
|
||||||
|
ctx,
|
||||||
|
storeapi,
|
||||||
|
receiver,
|
||||||
|
leader_epoch,
|
||||||
|
initial_baseline,
|
||||||
|
|| async { false },
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe<F, Fut>(
|
||||||
|
ctx: CancellationToken,
|
||||||
|
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
|
||||||
|
mut receiver: mpsc::Receiver<DataUsageInfo>,
|
||||||
|
leader_epoch: Option<u64>,
|
||||||
|
initial_baseline: Option<DataUsagePersistBaseline>,
|
||||||
|
route_probe: F,
|
||||||
|
) -> DataUsagePersistOutcome
|
||||||
|
where
|
||||||
|
F: Fn() -> Fut + Send + Sync,
|
||||||
|
Fut: Future<Output = bool> + Send,
|
||||||
|
{
|
||||||
let mut outcome = DataUsagePersistOutcome::NoUpdate;
|
let mut outcome = DataUsagePersistOutcome::NoUpdate;
|
||||||
let mut next_baseline = initial_baseline;
|
let mut next_baseline = initial_baseline;
|
||||||
|
|
||||||
@@ -113,6 +140,19 @@ pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_basel
|
|||||||
} else {
|
} else {
|
||||||
DATA_USAGE_OBJ_NAME_PATH.as_str()
|
DATA_USAGE_OBJ_NAME_PATH.as_str()
|
||||||
};
|
};
|
||||||
|
if route_probe().await {
|
||||||
|
debug!(
|
||||||
|
target: "rustfs::scanner",
|
||||||
|
event = EVENT_SCANNER_PERSIST_STATE,
|
||||||
|
component = LOG_COMPONENT_SCANNER,
|
||||||
|
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||||
|
path = %target_path,
|
||||||
|
state = "publication_blocked_before_reconcile",
|
||||||
|
"Scanner data usage publication deferred by the pool-state fence"
|
||||||
|
);
|
||||||
|
outcome = DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
if observational && data_usage_info.usage_snapshot_authoritative_baseline.is_none() {
|
if observational && data_usage_info.usage_snapshot_authoritative_baseline.is_none() {
|
||||||
let authoritative_data = match next_baseline.as_ref() {
|
let authoritative_data = match next_baseline.as_ref() {
|
||||||
@@ -275,6 +315,18 @@ pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_basel
|
|||||||
if ctx.is_cancelled() {
|
if ctx.is_cancelled() {
|
||||||
break 'updates;
|
break 'updates;
|
||||||
}
|
}
|
||||||
|
if route_probe().await {
|
||||||
|
debug!(
|
||||||
|
target: "rustfs::scanner",
|
||||||
|
event = EVENT_SCANNER_PERSIST_STATE,
|
||||||
|
component = LOG_COMPONENT_SCANNER,
|
||||||
|
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||||
|
path = %target_path,
|
||||||
|
state = "publication_blocked_before_save",
|
||||||
|
"Scanner data usage publication deferred by the final pool-state fence"
|
||||||
|
);
|
||||||
|
break DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
|
||||||
|
}
|
||||||
|
|
||||||
let done_save = Metrics::time(Metric::SaveUsage);
|
let done_save = Metrics::time(Metric::SaveUsage);
|
||||||
let save_result = save_config_shared_with_preconditions(
|
let save_result = save_config_shared_with_preconditions(
|
||||||
@@ -313,6 +365,33 @@ pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_basel
|
|||||||
"Scanner data usage CAS conflict will be reconciled"
|
"Scanner data usage CAS conflict will be reconciled"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
Err(e @ EcstoreError::ObjectNotFound(_, _)) => {
|
||||||
|
let route_blocked = route_probe().await;
|
||||||
|
if route_blocked {
|
||||||
|
warn!(
|
||||||
|
target: "rustfs::scanner",
|
||||||
|
event = EVENT_SCANNER_PERSIST_STATE,
|
||||||
|
component = LOG_COMPONENT_SCANNER,
|
||||||
|
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||||
|
path = %target_path,
|
||||||
|
state = "publication_deferred",
|
||||||
|
error = %e,
|
||||||
|
"Scanner data usage route is blocked by data movement; retrying later"
|
||||||
|
);
|
||||||
|
break DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
|
||||||
|
}
|
||||||
|
error!(
|
||||||
|
target: "rustfs::scanner",
|
||||||
|
event = EVENT_SCANNER_PERSIST_STATE,
|
||||||
|
component = LOG_COMPONENT_SCANNER,
|
||||||
|
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||||
|
path = %target_path,
|
||||||
|
state = "save_failed",
|
||||||
|
error = %e,
|
||||||
|
"Scanner data usage save failed"
|
||||||
|
);
|
||||||
|
break DataUsagePersistOutcome::Failed;
|
||||||
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
error!(
|
error!(
|
||||||
target: "rustfs::scanner",
|
target: "rustfs::scanner",
|
||||||
@@ -370,6 +449,13 @@ pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_basel
|
|||||||
outcome = DataUsagePersistOutcome::Failed;
|
outcome = DataUsagePersistOutcome::Failed;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
DataUsagePersistOutcome::Deferred(reason) => {
|
||||||
|
// A deferred publication is an intentional retryable state, not a
|
||||||
|
// failed save. Keep the last real save result so admin freshness
|
||||||
|
// reporting does not turn a pool-recovery fence into a false error.
|
||||||
|
outcome = DataUsagePersistOutcome::Deferred(reason);
|
||||||
|
break 'updates;
|
||||||
|
}
|
||||||
DataUsagePersistOutcome::Saved => {
|
DataUsagePersistOutcome::Saved => {
|
||||||
if observational {
|
if observational {
|
||||||
invalidate_admin_data_usage_snapshot_cache().await;
|
invalidate_admin_data_usage_snapshot_cache().await;
|
||||||
|
|||||||
@@ -18,7 +18,6 @@ use super::*;
|
|||||||
use crate::storage_api::VersionPurgeStatusType;
|
use crate::storage_api::VersionPurgeStatusType;
|
||||||
use crate::{DiskOption, Endpoint, STORAGE_FORMAT_FILE, TierStats, new_disk, storageclass};
|
use crate::{DiskOption, Endpoint, STORAGE_FORMAT_FILE, TierStats, new_disk, storageclass};
|
||||||
use rustfs_filemeta::{FileInfo, FileMeta};
|
use rustfs_filemeta::{FileInfo, FileMeta};
|
||||||
use serial_test::serial;
|
|
||||||
use std::io::Write;
|
use std::io::Write;
|
||||||
#[cfg(unix)]
|
#[cfg(unix)]
|
||||||
use std::os::unix::fs::{PermissionsExt, symlink};
|
use std::os::unix::fs::{PermissionsExt, symlink};
|
||||||
@@ -356,7 +355,6 @@ impl Drop for TestGuard {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_should_skip_failed_respects_ttl() {
|
async fn test_should_skip_failed_respects_ttl() {
|
||||||
let (mut scanner, temp_dir) = build_test_scanner().await;
|
let (mut scanner, temp_dir) = build_test_scanner().await;
|
||||||
let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir);
|
let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir);
|
||||||
@@ -378,7 +376,6 @@ async fn test_should_skip_failed_respects_ttl() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_record_failed_ttl_zero_noop() {
|
async fn test_record_failed_ttl_zero_noop() {
|
||||||
let (mut scanner, temp_dir) = build_test_scanner().await;
|
let (mut scanner, temp_dir) = build_test_scanner().await;
|
||||||
let _guard = TestGuard::new(0, 100, &mut scanner, temp_dir);
|
let _guard = TestGuard::new(0, 100, &mut scanner, temp_dir);
|
||||||
@@ -467,7 +464,6 @@ fn test_should_account_replication_stats_only_for_live_object_versions() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_heal_replication_only_queues_pending_null_deletes() {
|
async fn test_heal_replication_only_queues_pending_null_deletes() {
|
||||||
async fn replication_skipped_count() -> u64 {
|
async fn replication_skipped_count() -> u64 {
|
||||||
global_metrics()
|
global_metrics()
|
||||||
@@ -716,7 +712,6 @@ async fn test_scanner_heal_admission_accounting_maps_deep_scan_to_bitrot() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn test_excessive_version_alert_thresholds_use_env() {
|
fn test_excessive_version_alert_thresholds_use_env() {
|
||||||
with_var(rustfs_config::ENV_SCANNER_ALERT_EXCESS_VERSIONS, Some("3"), || {
|
with_var(rustfs_config::ENV_SCANNER_ALERT_EXCESS_VERSIONS, Some("3"), || {
|
||||||
with_var(rustfs_config::ENV_SCANNER_ALERT_EXCESS_VERSION_SIZE, Some("100"), || {
|
with_var(rustfs_config::ENV_SCANNER_ALERT_EXCESS_VERSION_SIZE, Some("100"), || {
|
||||||
@@ -731,7 +726,6 @@ fn test_excessive_version_alert_thresholds_use_env() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn test_excessive_folders_threshold_uses_env() {
|
fn test_excessive_folders_threshold_uses_env() {
|
||||||
with_var(rustfs_config::ENV_SCANNER_ALERT_EXCESS_FOLDERS, Some("3"), || {
|
with_var(rustfs_config::ENV_SCANNER_ALERT_EXCESS_FOLDERS, Some("3"), || {
|
||||||
crate::runtime_config::refresh_scanner_runtime_config_for_tests();
|
crate::runtime_config::refresh_scanner_runtime_config_for_tests();
|
||||||
@@ -741,7 +735,6 @@ fn test_excessive_folders_threshold_uses_env() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn test_excessive_folders_threshold_default_supports_pbs_layout() {
|
fn test_excessive_folders_threshold_default_supports_pbs_layout() {
|
||||||
with_var_unset(rustfs_config::ENV_SCANNER_ALERT_EXCESS_FOLDERS, || {
|
with_var_unset(rustfs_config::ENV_SCANNER_ALERT_EXCESS_FOLDERS, || {
|
||||||
crate::runtime_config::refresh_scanner_runtime_config_for_tests();
|
crate::runtime_config::refresh_scanner_runtime_config_for_tests();
|
||||||
@@ -751,7 +744,6 @@ fn test_excessive_folders_threshold_default_supports_pbs_layout() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn test_scanner_yield_every_n_objects_uses_env() {
|
fn test_scanner_yield_every_n_objects_uses_env() {
|
||||||
with_var(rustfs_config::ENV_SCANNER_YIELD_EVERY_N_OBJECTS, Some("32"), || {
|
with_var(rustfs_config::ENV_SCANNER_YIELD_EVERY_N_OBJECTS, Some("32"), || {
|
||||||
crate::runtime_config::refresh_scanner_runtime_config_for_tests();
|
crate::runtime_config::refresh_scanner_runtime_config_for_tests();
|
||||||
@@ -761,7 +753,6 @@ fn test_scanner_yield_every_n_objects_uses_env() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn test_scanner_yield_every_n_objects_uses_default() {
|
fn test_scanner_yield_every_n_objects_uses_default() {
|
||||||
with_var_unset(rustfs_config::ENV_SCANNER_YIELD_EVERY_N_OBJECTS, || {
|
with_var_unset(rustfs_config::ENV_SCANNER_YIELD_EVERY_N_OBJECTS, || {
|
||||||
crate::runtime_config::refresh_scanner_runtime_config_for_tests();
|
crate::runtime_config::refresh_scanner_runtime_config_for_tests();
|
||||||
@@ -888,7 +879,6 @@ fn test_order_folders_for_resume_reports_stale_hint() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_record_failed_prunes_to_max_entries() {
|
async fn test_record_failed_prunes_to_max_entries() {
|
||||||
let (mut scanner, temp_dir) = build_test_scanner().await;
|
let (mut scanner, temp_dir) = build_test_scanner().await;
|
||||||
let _guard = TestGuard::new(1000, 2, &mut scanner, temp_dir);
|
let _guard = TestGuard::new(1000, 2, &mut scanner, temp_dir);
|
||||||
@@ -920,7 +910,6 @@ async fn test_record_failed_prunes_to_max_entries() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_prune_failed_objects_cache_drops_expired() {
|
async fn test_prune_failed_objects_cache_drops_expired() {
|
||||||
let (mut scanner, temp_dir) = build_test_scanner().await;
|
let (mut scanner, temp_dir) = build_test_scanner().await;
|
||||||
let _guard = TestGuard::new(5, 10, &mut scanner, temp_dir);
|
let _guard = TestGuard::new(5, 10, &mut scanner, temp_dir);
|
||||||
@@ -944,7 +933,6 @@ async fn test_prune_failed_objects_cache_drops_expired() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_prune_failed_objects_max_zero_keeps_fresh() {
|
async fn test_prune_failed_objects_max_zero_keeps_fresh() {
|
||||||
let (mut scanner, temp_dir) = build_test_scanner().await;
|
let (mut scanner, temp_dir) = build_test_scanner().await;
|
||||||
let _guard = TestGuard::new(60, 0, &mut scanner, temp_dir);
|
let _guard = TestGuard::new(60, 0, &mut scanner, temp_dir);
|
||||||
@@ -1701,7 +1689,6 @@ async fn test_heal_actions_returns_actual_size_without_inline_heal() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
#[cfg(unix)]
|
#[cfg(unix)]
|
||||||
async fn test_scan_folder_skips_unreadable_child_directory() {
|
async fn test_scan_folder_skips_unreadable_child_directory() {
|
||||||
let (mut scanner, temp_dir) = build_test_scanner().await;
|
let (mut scanner, temp_dir) = build_test_scanner().await;
|
||||||
@@ -1734,7 +1721,6 @@ async fn test_scan_folder_skips_unreadable_child_directory() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_scan_folder_exits_when_abandoned_child_listing_finishes() {
|
async fn test_scan_folder_exits_when_abandoned_child_listing_finishes() {
|
||||||
let (mut scanner, temp_dir) = build_test_scanner().await;
|
let (mut scanner, temp_dir) = build_test_scanner().await;
|
||||||
let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir.clone());
|
let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir.clone());
|
||||||
@@ -1813,7 +1799,6 @@ async fn test_scan_folder_exits_when_abandoned_child_listing_finishes() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_scan_folder_xl_meta_named_directory_uses_namespace_descent() {
|
async fn test_scan_folder_xl_meta_named_directory_uses_namespace_descent() {
|
||||||
let (mut scanner, temp_dir) = build_test_scanner().await;
|
let (mut scanner, temp_dir) = build_test_scanner().await;
|
||||||
let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir.clone());
|
let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir.clone());
|
||||||
@@ -1859,7 +1844,6 @@ async fn test_scan_folder_xl_meta_named_directory_uses_namespace_descent() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test(flavor = "current_thread")]
|
#[tokio::test(flavor = "current_thread")]
|
||||||
#[serial]
|
|
||||||
async fn test_scan_folder_corrupt_xl_meta_stops_erasure_data_dir_descent() {
|
async fn test_scan_folder_corrupt_xl_meta_stops_erasure_data_dir_descent() {
|
||||||
let logs = CapturedLogs::default();
|
let logs = CapturedLogs::default();
|
||||||
let subscriber = tracing_subscriber::fmt()
|
let subscriber = tracing_subscriber::fmt()
|
||||||
@@ -2021,7 +2005,6 @@ async fn test_scan_folder_corrupt_xl_meta_stops_erasure_data_dir_descent() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_scan_folder_missing_xl_meta_stops_erasure_data_dir_descent() {
|
async fn test_scan_folder_missing_xl_meta_stops_erasure_data_dir_descent() {
|
||||||
let (mut scanner, temp_dir) = build_test_scanner().await;
|
let (mut scanner, temp_dir) = build_test_scanner().await;
|
||||||
let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir.clone());
|
let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir.clone());
|
||||||
@@ -2099,7 +2082,6 @@ async fn test_scan_folder_missing_xl_meta_stops_erasure_data_dir_descent() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_scan_folder_uuid_namespace_part_name_directory_is_not_data_dir() {
|
async fn test_scan_folder_uuid_namespace_part_name_directory_is_not_data_dir() {
|
||||||
let (mut scanner, temp_dir) = build_test_scanner().await;
|
let (mut scanner, temp_dir) = build_test_scanner().await;
|
||||||
let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir.clone());
|
let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir.clone());
|
||||||
@@ -2161,7 +2143,6 @@ async fn test_scan_folder_uuid_namespace_part_name_directory_is_not_data_dir() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_scan_folder_non_erasure_metadata_keeps_namespace_descent() {
|
async fn test_scan_folder_non_erasure_metadata_keeps_namespace_descent() {
|
||||||
let (mut scanner, temp_dir) = build_test_scanner().await;
|
let (mut scanner, temp_dir) = build_test_scanner().await;
|
||||||
let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir.clone());
|
let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir.clone());
|
||||||
@@ -2203,7 +2184,6 @@ async fn test_scan_folder_non_erasure_metadata_keeps_namespace_descent() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_scan_folder_compacted_parent_sends_partial_update() {
|
async fn test_scan_folder_compacted_parent_sends_partial_update() {
|
||||||
let (mut scanner, temp_dir) = build_test_scanner().await;
|
let (mut scanner, temp_dir) = build_test_scanner().await;
|
||||||
let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir.clone());
|
let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir.clone());
|
||||||
@@ -2245,7 +2225,6 @@ async fn test_scan_folder_compacted_parent_sends_partial_update() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_scan_data_folder_cancelled_before_scan_clears_current_path() {
|
async fn test_scan_data_folder_cancelled_before_scan_clears_current_path() {
|
||||||
let (scanner, temp_dir) = build_test_scanner().await;
|
let (scanner, temp_dir) = build_test_scanner().await;
|
||||||
let _guard = TestGuard {
|
let _guard = TestGuard {
|
||||||
@@ -2290,7 +2269,6 @@ async fn test_scan_data_folder_cancelled_before_scan_clears_current_path() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_scan_data_folder_returns_partial_cache_on_budget_cancel() {
|
async fn test_scan_data_folder_returns_partial_cache_on_budget_cancel() {
|
||||||
let (mut scanner, temp_dir) = build_test_scanner().await;
|
let (mut scanner, temp_dir) = build_test_scanner().await;
|
||||||
let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir.clone());
|
let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir.clone());
|
||||||
@@ -2346,7 +2324,6 @@ async fn test_scan_data_folder_returns_partial_cache_on_budget_cancel() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_scan_data_folder_reports_invalid_checkpoint_ignored_once() {
|
async fn test_scan_data_folder_reports_invalid_checkpoint_ignored_once() {
|
||||||
let (scanner, temp_dir) = build_test_scanner().await;
|
let (scanner, temp_dir) = build_test_scanner().await;
|
||||||
let _guard = TestGuard {
|
let _guard = TestGuard {
|
||||||
@@ -2391,7 +2368,6 @@ async fn test_scan_data_folder_reports_invalid_checkpoint_ignored_once() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_scan_data_folder_resume_hint_prioritizes_next_existing_folder() {
|
async fn test_scan_data_folder_resume_hint_prioritizes_next_existing_folder() {
|
||||||
let (scanner, temp_dir) = build_test_scanner().await;
|
let (scanner, temp_dir) = build_test_scanner().await;
|
||||||
let _guard = TestGuard {
|
let _guard = TestGuard {
|
||||||
@@ -2465,7 +2441,6 @@ async fn test_scan_data_folder_resume_hint_prioritizes_next_existing_folder() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn scan_data_folder_missing_bucket_returns_partial() {
|
async fn scan_data_folder_missing_bucket_returns_partial() {
|
||||||
let (scanner, temp_dir) = build_test_scanner().await;
|
let (scanner, temp_dir) = build_test_scanner().await;
|
||||||
let _guard = TestGuard {
|
let _guard = TestGuard {
|
||||||
@@ -2517,7 +2492,6 @@ async fn scan_data_folder_missing_bucket_returns_partial() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn scan_data_folder_missing_scan_root_returns_partial() {
|
async fn scan_data_folder_missing_scan_root_returns_partial() {
|
||||||
let (scanner, temp_dir) = build_test_scanner().await;
|
let (scanner, temp_dir) = build_test_scanner().await;
|
||||||
tokio::fs::remove_dir_all(&temp_dir)
|
tokio::fs::remove_dir_all(&temp_dir)
|
||||||
@@ -2563,7 +2537,6 @@ async fn scan_data_folder_missing_scan_root_returns_partial() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_scan_data_folder_resume_hint_orders_across_new_and_existing_folders() {
|
async fn test_scan_data_folder_resume_hint_orders_across_new_and_existing_folders() {
|
||||||
let (scanner, temp_dir) = build_test_scanner().await;
|
let (scanner, temp_dir) = build_test_scanner().await;
|
||||||
let _guard = TestGuard {
|
let _guard = TestGuard {
|
||||||
@@ -2632,7 +2605,6 @@ async fn test_scan_data_folder_resume_hint_orders_across_new_and_existing_folder
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_scan_data_folder_partial_object_budget_accumulates_progress() {
|
async fn test_scan_data_folder_partial_object_budget_accumulates_progress() {
|
||||||
let (scanner, temp_dir) = build_test_scanner().await;
|
let (scanner, temp_dir) = build_test_scanner().await;
|
||||||
let _guard = TestGuard {
|
let _guard = TestGuard {
|
||||||
@@ -2715,7 +2687,6 @@ async fn test_scan_data_folder_partial_object_budget_accumulates_progress() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_partial_compacted_entry_does_not_carry_children() {
|
async fn test_partial_compacted_entry_does_not_carry_children() {
|
||||||
let (mut scanner, temp_dir) = build_test_scanner().await;
|
let (mut scanner, temp_dir) = build_test_scanner().await;
|
||||||
let _guard = TestGuard {
|
let _guard = TestGuard {
|
||||||
@@ -2761,7 +2732,6 @@ async fn test_partial_compacted_entry_does_not_carry_children() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_partial_entry_does_not_carry_missing_old_child() {
|
async fn test_partial_entry_does_not_carry_missing_old_child() {
|
||||||
let (mut scanner, temp_dir) = build_test_scanner().await;
|
let (mut scanner, temp_dir) = build_test_scanner().await;
|
||||||
let _guard = TestGuard {
|
let _guard = TestGuard {
|
||||||
@@ -2794,7 +2764,6 @@ async fn test_partial_entry_does_not_carry_missing_old_child() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_legacy_windows_cache_rebuilds_and_round_trips_portable_keys() {
|
async fn test_legacy_windows_cache_rebuilds_and_round_trips_portable_keys() {
|
||||||
let (scanner, temp_dir) = build_test_scanner().await;
|
let (scanner, temp_dir) = build_test_scanner().await;
|
||||||
let _guard = TestGuard {
|
let _guard = TestGuard {
|
||||||
@@ -2861,7 +2830,6 @@ async fn test_legacy_windows_cache_rebuilds_and_round_trips_portable_keys() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_scan_data_folder_success_clears_resume_hint() {
|
async fn test_scan_data_folder_success_clears_resume_hint() {
|
||||||
let (scanner, temp_dir) = build_test_scanner().await;
|
let (scanner, temp_dir) = build_test_scanner().await;
|
||||||
let _guard = TestGuard {
|
let _guard = TestGuard {
|
||||||
@@ -2904,7 +2872,6 @@ async fn test_scan_data_folder_success_clears_resume_hint() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn test_scan_data_folder_keeps_unresolved_objects_partial() {
|
async fn test_scan_data_folder_keeps_unresolved_objects_partial() {
|
||||||
let (scanner, temp_dir) = build_test_scanner().await;
|
let (scanner, temp_dir) = build_test_scanner().await;
|
||||||
let _guard = TestGuard {
|
let _guard = TestGuard {
|
||||||
@@ -2951,7 +2918,6 @@ async fn test_scan_data_folder_keeps_unresolved_objects_partial() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
#[cfg(unix)]
|
#[cfg(unix)]
|
||||||
async fn test_scan_folder_ignores_symlinked_child_directory() {
|
async fn test_scan_folder_ignores_symlinked_child_directory() {
|
||||||
let (mut scanner, temp_dir) = build_test_scanner().await;
|
let (mut scanner, temp_dir) = build_test_scanner().await;
|
||||||
|
|||||||
@@ -49,6 +49,25 @@ impl ScannerIOCycle for ECStore {
|
|||||||
) -> Result<ScannerCycleResult> {
|
) -> Result<ScannerCycleResult> {
|
||||||
let child_token = ctx.child_token();
|
let child_token = ctx.child_token();
|
||||||
|
|
||||||
|
// Check the local pool metadata before listing buckets. A failed or
|
||||||
|
// canceled decommission remains suspended after its worker exits, so
|
||||||
|
// starting a scan in that state could build a snapshot that cannot be
|
||||||
|
// routed to the authoritative metadata object.
|
||||||
|
if self.scanner_data_usage_publication_blocked().await {
|
||||||
|
debug!(
|
||||||
|
target: "rustfs::scanner::io",
|
||||||
|
event = EVENT_SCANNER_SET_STATE,
|
||||||
|
component = LOG_COMPONENT_SCANNER,
|
||||||
|
subsystem = LOG_SUBSYSTEM_IO,
|
||||||
|
state = "cycle_data_usage_route_blocked",
|
||||||
|
"Scanner cycle deferred while data usage metadata remains hidden by data movement"
|
||||||
|
);
|
||||||
|
return Ok(ScannerCycleResult::new(
|
||||||
|
ScannerCycleStatus::Deferred(ScannerCycleDeferReason::DataMovement),
|
||||||
|
None,
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
let distributed = self.setup_is_dist_erasure().await;
|
let distributed = self.setup_is_dist_erasure().await;
|
||||||
let activity_before = match scanner_activity_preflight(crate::scanner::probe_scanner_activity(self, distributed).await) {
|
let activity_before = match scanner_activity_preflight(crate::scanner::probe_scanner_activity(self, distributed).await) {
|
||||||
ScannerActivityPreflight::Ready(snapshot) => snapshot,
|
ScannerActivityPreflight::Ready(snapshot) => snapshot,
|
||||||
|
|||||||
@@ -17,7 +17,9 @@ use super::io_disk::tier_stats_template;
|
|||||||
use super::*;
|
use super::*;
|
||||||
use crate::scanner_budget::ScannerCycleBudgetConfig;
|
use crate::scanner_budget::ScannerCycleBudgetConfig;
|
||||||
use crate::scanner_folder::ScannerItem;
|
use crate::scanner_folder::ScannerItem;
|
||||||
use crate::storage_api::owner::{EcstoreRebalStatus, EcstoreRebalanceInfo, EcstoreRebalanceMeta, EcstoreRebalanceStats};
|
use crate::storage_api::owner::{
|
||||||
|
EcstorePoolDecommissionInfo, EcstoreRebalStatus, EcstoreRebalanceInfo, EcstoreRebalanceMeta, EcstoreRebalanceStats,
|
||||||
|
};
|
||||||
use crate::storage_api::scan::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions, ObjectIO as _};
|
use crate::storage_api::scan::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions, ObjectIO as _};
|
||||||
use crate::{
|
use crate::{
|
||||||
DiskOption, ECStore, Endpoint, EndpointServerPools, Endpoints, InstanceContext, PoolEndpoints, ScannerObjectOptions,
|
DiskOption, ECStore, Endpoint, EndpointServerPools, Endpoints, InstanceContext, PoolEndpoints, ScannerObjectOptions,
|
||||||
@@ -25,7 +27,6 @@ use crate::{
|
|||||||
init_local_disks_with_instance_ctx, new_disk, path2_bucket_object_with_base_path,
|
init_local_disks_with_instance_ctx, new_disk, path2_bucket_object_with_base_path,
|
||||||
};
|
};
|
||||||
use rustfs_filemeta::FileInfo;
|
use rustfs_filemeta::FileInfo;
|
||||||
use serial_test::serial;
|
|
||||||
use temp_env::with_var;
|
use temp_env::with_var;
|
||||||
use time::OffsetDateTime;
|
use time::OffsetDateTime;
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
@@ -101,7 +102,6 @@ async fn setup_two_pool_scanner_store() -> (tempfile::TempDir, Arc<ECStore>) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn scanner_cache_locks_block_same_source_workers() {
|
async fn scanner_cache_locks_block_same_source_workers() {
|
||||||
let (_temp_dir, store) = setup_two_pool_scanner_store().await;
|
let (_temp_dir, store) = setup_two_pool_scanner_store().await;
|
||||||
let set = &store.pools[0].disk_set[0];
|
let set = &store.pools[0].disk_set[0];
|
||||||
@@ -128,7 +128,6 @@ async fn scanner_cache_locks_block_same_source_workers() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn scanner_cache_locks_allow_cross_source_workers() {
|
async fn scanner_cache_locks_allow_cross_source_workers() {
|
||||||
let (_temp_dir, store) = setup_two_pool_scanner_store().await;
|
let (_temp_dir, store) = setup_two_pool_scanner_store().await;
|
||||||
let first_set = &store.pools[0].disk_set[0];
|
let first_set = &store.pools[0].disk_set[0];
|
||||||
@@ -147,7 +146,6 @@ async fn scanner_cache_locks_allow_cross_source_workers() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn scanner_cycle_is_deferred_while_rebalance_is_active() {
|
async fn scanner_cycle_is_deferred_while_rebalance_is_active() {
|
||||||
let (_temp_dir, store) = setup_two_pool_scanner_store().await;
|
let (_temp_dir, store) = setup_two_pool_scanner_store().await;
|
||||||
let mut pool_stats = vec![EcstoreRebalanceStats::default(); store.pools.len()];
|
let mut pool_stats = vec![EcstoreRebalanceStats::default(); store.pools.len()];
|
||||||
@@ -182,6 +180,38 @@ async fn scanner_cycle_is_deferred_while_rebalance_is_active() {
|
|||||||
assert!(receiver.recv().await.is_none(), "rebalance-deferred cycle must not publish usage");
|
assert!(receiver.recv().await.is_none(), "rebalance-deferred cycle must not publish usage");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn scanner_cycle_is_deferred_while_terminal_decommission_is_blocked() {
|
||||||
|
let (_temp_dir, store) = setup_two_pool_scanner_store().await;
|
||||||
|
for decommission in [
|
||||||
|
EcstorePoolDecommissionInfo {
|
||||||
|
failed: true,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
EcstorePoolDecommissionInfo {
|
||||||
|
canceled: true,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
] {
|
||||||
|
store.pool_meta.write().await.pools[0].decommission = Some(decommission);
|
||||||
|
assert!(store.scanner_data_usage_publication_blocked().await);
|
||||||
|
|
||||||
|
let ctx = CancellationToken::new();
|
||||||
|
let budget = ScannerCycleBudget::new(&ctx, ScannerCycleBudgetConfig::default());
|
||||||
|
let (updates, mut receiver) = mpsc::channel(1);
|
||||||
|
let result = tokio::time::timeout(
|
||||||
|
Duration::from_secs(30),
|
||||||
|
ScannerIOCycle::nsscanner_with_status(store.as_ref(), ctx, budget, updates, 1, 1, HealScanMode::Normal),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("terminal-decommission-deferred scanner cycle should finish")
|
||||||
|
.expect("terminal-decommission-deferred scanner cycle should succeed");
|
||||||
|
|
||||||
|
assert_eq!(result.status, ScannerCycleStatus::Deferred(ScannerCycleDeferReason::DataMovement));
|
||||||
|
assert!(receiver.recv().await.is_none(), "blocked cycle must not publish usage");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn data_usage_publish_fails_when_receiver_is_closed() {
|
async fn data_usage_publish_fails_when_receiver_is_closed() {
|
||||||
let (updates, receiver) = mpsc::channel(1);
|
let (updates, receiver) = mpsc::channel(1);
|
||||||
@@ -195,7 +225,6 @@ async fn data_usage_publish_fails_when_receiver_is_closed() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn multi_pool_scanner_cycle_publishes_combined_usage() {
|
async fn multi_pool_scanner_cycle_publishes_combined_usage() {
|
||||||
let (_temp_dir, store) = setup_two_pool_scanner_store().await;
|
let (_temp_dir, store) = setup_two_pool_scanner_store().await;
|
||||||
let bucket = format!("scanner-union-{}", Uuid::new_v4().simple());
|
let bucket = format!("scanner-union-{}", Uuid::new_v4().simple());
|
||||||
@@ -236,10 +265,13 @@ async fn multi_pool_scanner_cycle_publishes_combined_usage() {
|
|||||||
assert_eq!(bucket_usage.size, 11);
|
assert_eq!(bucket_usage.size, 11);
|
||||||
assert_eq!(usage.objects_total_count, 2);
|
assert_eq!(usage.objects_total_count, 2);
|
||||||
assert_eq!(usage.objects_total_size, 11);
|
assert_eq!(usage.objects_total_size, 11);
|
||||||
|
assert!(
|
||||||
|
receiver.recv().await.is_none(),
|
||||||
|
"a scanner cycle must publish at most one terminal usage snapshot"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
|
||||||
async fn multi_pool_scanner_cycle_zero_fills_bucket_absent_from_first_pool() {
|
async fn multi_pool_scanner_cycle_zero_fills_bucket_absent_from_first_pool() {
|
||||||
let (_temp_dir, store) = setup_two_pool_scanner_store().await;
|
let (_temp_dir, store) = setup_two_pool_scanner_store().await;
|
||||||
let bucket = format!("scanner-second-pool-{}", Uuid::new_v4().simple());
|
let bucket = format!("scanner-second-pool-{}", Uuid::new_v4().simple());
|
||||||
@@ -327,7 +359,6 @@ fn object_lock_config_enabled_accepts_enabled_only() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn dirty_usage_snapshot_clear_preserves_newer_generation() {
|
fn dirty_usage_snapshot_clear_preserves_newer_generation() {
|
||||||
clear_dirty_usage_buckets_for_tests();
|
clear_dirty_usage_buckets_for_tests();
|
||||||
record_dirty_usage_bucket("photos");
|
record_dirty_usage_bucket("photos");
|
||||||
@@ -342,7 +373,6 @@ fn dirty_usage_snapshot_clear_preserves_newer_generation() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn dirty_usage_generation_acknowledgement_preserves_newer_mutations() {
|
fn dirty_usage_generation_acknowledgement_preserves_newer_mutations() {
|
||||||
clear_dirty_usage_buckets_for_tests();
|
clear_dirty_usage_buckets_for_tests();
|
||||||
record_dirty_usage_bucket("photos");
|
record_dirty_usage_bucket("photos");
|
||||||
@@ -368,7 +398,6 @@ fn dirty_usage_generation_acknowledgement_preserves_newer_mutations() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn dirty_usage_generation_acknowledgement_rejects_stale_process_and_future_generation() {
|
fn dirty_usage_generation_acknowledgement_rejects_stale_process_and_future_generation() {
|
||||||
clear_dirty_usage_buckets_for_tests();
|
clear_dirty_usage_buckets_for_tests();
|
||||||
record_dirty_usage_bucket("photos");
|
record_dirty_usage_bucket("photos");
|
||||||
@@ -398,7 +427,6 @@ fn dirty_usage_generation_acknowledgement_rejects_stale_process_and_future_gener
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn dirty_usage_snapshot_detects_uncovered_generation() {
|
fn dirty_usage_snapshot_detects_uncovered_generation() {
|
||||||
clear_dirty_usage_buckets_for_tests();
|
clear_dirty_usage_buckets_for_tests();
|
||||||
record_dirty_usage_bucket("photos");
|
record_dirty_usage_bucket("photos");
|
||||||
@@ -423,7 +451,6 @@ fn generation_saturates_instead_of_wrapping() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn dirty_usage_snapshot_clears_a_stably_absent_bucket_after_durable_save() {
|
fn dirty_usage_snapshot_clears_a_stably_absent_bucket_after_durable_save() {
|
||||||
clear_dirty_usage_buckets_for_tests();
|
clear_dirty_usage_buckets_for_tests();
|
||||||
record_dirty_usage_bucket("photos");
|
record_dirty_usage_bucket("photos");
|
||||||
@@ -445,7 +472,6 @@ fn dirty_usage_snapshot_clears_a_stably_absent_bucket_after_durable_save() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn dirty_usage_snapshot_preserves_an_absent_bucket_recorded_after_listing_started() {
|
fn dirty_usage_snapshot_preserves_an_absent_bucket_recorded_after_listing_started() {
|
||||||
clear_dirty_usage_buckets_for_tests();
|
clear_dirty_usage_buckets_for_tests();
|
||||||
let generation_before_bucket_list = dirty_usage_generation();
|
let generation_before_bucket_list = dirty_usage_generation();
|
||||||
@@ -460,7 +486,6 @@ fn dirty_usage_snapshot_preserves_an_absent_bucket_recorded_after_listing_starte
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn deleting_a_clean_bucket_invalidates_an_inflight_usage_snapshot() {
|
fn deleting_a_clean_bucket_invalidates_an_inflight_usage_snapshot() {
|
||||||
clear_dirty_usage_buckets_for_tests();
|
clear_dirty_usage_buckets_for_tests();
|
||||||
let snapshot = snapshot_dirty_usage_buckets(&[bucket_info("photos")], dirty_usage_generation());
|
let snapshot = snapshot_dirty_usage_buckets(&[bucket_info("photos")], dirty_usage_generation());
|
||||||
@@ -474,7 +499,6 @@ fn deleting_a_clean_bucket_invalidates_an_inflight_usage_snapshot() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn deleting_a_bucket_during_listing_invalidates_the_resulting_usage_snapshot() {
|
fn deleting_a_bucket_during_listing_invalidates_the_resulting_usage_snapshot() {
|
||||||
clear_dirty_usage_buckets_for_tests();
|
clear_dirty_usage_buckets_for_tests();
|
||||||
let generation_before_bucket_list = dirty_usage_generation();
|
let generation_before_bucket_list = dirty_usage_generation();
|
||||||
@@ -488,7 +512,6 @@ fn deleting_a_bucket_during_listing_invalidates_the_resulting_usage_snapshot() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn scanner_maintenance_change_advances_generation_and_marks_usage_dirty() {
|
fn scanner_maintenance_change_advances_generation_and_marks_usage_dirty() {
|
||||||
clear_dirty_usage_buckets_for_tests();
|
clear_dirty_usage_buckets_for_tests();
|
||||||
let generation = scanner_maintenance_generation();
|
let generation = scanner_maintenance_generation();
|
||||||
@@ -501,7 +524,6 @@ fn scanner_maintenance_change_advances_generation_and_marks_usage_dirty() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn dirty_usage_clear_excludes_failed_buckets() {
|
fn dirty_usage_clear_excludes_failed_buckets() {
|
||||||
clear_dirty_usage_buckets_for_tests();
|
clear_dirty_usage_buckets_for_tests();
|
||||||
record_dirty_usage_bucket("photos");
|
record_dirty_usage_bucket("photos");
|
||||||
@@ -533,7 +555,6 @@ fn dirty_usage_clear_plan_excludes_cache_save_failures() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn dirty_usage_is_acknowledged_only_after_durable_usage_confirmation() {
|
fn dirty_usage_is_acknowledged_only_after_durable_usage_confirmation() {
|
||||||
clear_dirty_usage_buckets_for_tests();
|
clear_dirty_usage_buckets_for_tests();
|
||||||
record_dirty_usage_bucket("photos");
|
record_dirty_usage_bucket("photos");
|
||||||
@@ -551,7 +572,6 @@ fn dirty_usage_is_acknowledged_only_after_durable_usage_confirmation() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn clear_dirty_usage_bucket_removes_deleted_bucket_marker() {
|
fn clear_dirty_usage_bucket_removes_deleted_bucket_marker() {
|
||||||
clear_dirty_usage_buckets_for_tests();
|
clear_dirty_usage_buckets_for_tests();
|
||||||
record_dirty_usage_bucket("photos");
|
record_dirty_usage_bucket("photos");
|
||||||
@@ -878,35 +898,30 @@ async fn bucket_cache_pending_heal_reaches_cycle_maintenance_state() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn scanner_concurrency_limit_preserves_available_when_unconfigured() {
|
fn scanner_concurrency_limit_preserves_available_when_unconfigured() {
|
||||||
crate::reset_foreground_read_activity_for_test();
|
crate::reset_foreground_read_activity_for_test();
|
||||||
assert_eq!(scanner_concurrency_limit(0, 4), 4);
|
assert_eq!(scanner_concurrency_limit(0, 4), 4);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn scanner_concurrency_limit_caps_to_configured_value() {
|
fn scanner_concurrency_limit_caps_to_configured_value() {
|
||||||
crate::reset_foreground_read_activity_for_test();
|
crate::reset_foreground_read_activity_for_test();
|
||||||
assert_eq!(scanner_concurrency_limit(2, 4), 2);
|
assert_eq!(scanner_concurrency_limit(2, 4), 2);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn scanner_concurrency_limit_never_exceeds_available_work() {
|
fn scanner_concurrency_limit_never_exceeds_available_work() {
|
||||||
crate::reset_foreground_read_activity_for_test();
|
crate::reset_foreground_read_activity_for_test();
|
||||||
assert_eq!(scanner_concurrency_limit(8, 4), 4);
|
assert_eq!(scanner_concurrency_limit(8, 4), 4);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn scanner_concurrency_limit_handles_no_available_work() {
|
fn scanner_concurrency_limit_handles_no_available_work() {
|
||||||
crate::reset_foreground_read_activity_for_test();
|
crate::reset_foreground_read_activity_for_test();
|
||||||
assert_eq!(scanner_concurrency_limit(2, 0), 0);
|
assert_eq!(scanner_concurrency_limit(2, 0), 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn scanner_concurrency_limit_yields_to_foreground_reads() {
|
fn scanner_concurrency_limit_yields_to_foreground_reads() {
|
||||||
crate::reset_foreground_read_activity_for_test();
|
crate::reset_foreground_read_activity_for_test();
|
||||||
crate::set_foreground_read_activity(8);
|
crate::set_foreground_read_activity(8);
|
||||||
@@ -916,7 +931,6 @@ fn scanner_concurrency_limit_yields_to_foreground_reads() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn scanner_concurrency_limit_yields_to_streaming_reads() {
|
fn scanner_concurrency_limit_yields_to_streaming_reads() {
|
||||||
crate::reset_foreground_read_activity_for_test();
|
crate::reset_foreground_read_activity_for_test();
|
||||||
let _guard = crate::ForegroundReadGuard::new();
|
let _guard = crate::ForegroundReadGuard::new();
|
||||||
@@ -940,7 +954,6 @@ fn increment_atomic_usize_saturates_at_max() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn scanner_max_concurrent_set_scans_uses_env_cap() {
|
fn scanner_max_concurrent_set_scans_uses_env_cap() {
|
||||||
with_var(ENV_SCANNER_MAX_CONCURRENT_SET_SCANS, Some("2"), || {
|
with_var(ENV_SCANNER_MAX_CONCURRENT_SET_SCANS, Some("2"), || {
|
||||||
crate::runtime_config::refresh_scanner_runtime_config_for_tests();
|
crate::runtime_config::refresh_scanner_runtime_config_for_tests();
|
||||||
@@ -950,7 +963,6 @@ fn scanner_max_concurrent_set_scans_uses_env_cap() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn scanner_max_concurrent_disk_scans_uses_env_cap() {
|
fn scanner_max_concurrent_disk_scans_uses_env_cap() {
|
||||||
with_var(ENV_SCANNER_MAX_CONCURRENT_DISK_SCANS, Some("1"), || {
|
with_var(ENV_SCANNER_MAX_CONCURRENT_DISK_SCANS, Some("1"), || {
|
||||||
crate::runtime_config::refresh_scanner_runtime_config_for_tests();
|
crate::runtime_config::refresh_scanner_runtime_config_for_tests();
|
||||||
|
|||||||
@@ -258,7 +258,6 @@ impl SleepTimer {
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
use serial_test::serial;
|
|
||||||
use temp_env::{with_var, with_var_unset};
|
use temp_env::{with_var, with_var_unset};
|
||||||
|
|
||||||
struct ScannerDefaultSpeedGuard;
|
struct ScannerDefaultSpeedGuard;
|
||||||
@@ -326,7 +325,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn test_refresh_from_env_applies_speed_and_idle_mode_for_next_cycle() {
|
fn test_refresh_from_env_applies_speed_and_idle_mode_for_next_cycle() {
|
||||||
let prev_mode = SCANNER_IDLE_MODE.load(Ordering::Relaxed);
|
let prev_mode = SCANNER_IDLE_MODE.load(Ordering::Relaxed);
|
||||||
SCANNER_IDLE_MODE.store(true, Ordering::Relaxed);
|
SCANNER_IDLE_MODE.store(true, Ordering::Relaxed);
|
||||||
@@ -346,7 +344,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
fn test_refresh_from_env_uses_default_speed_override_when_speed_unset() {
|
fn test_refresh_from_env_uses_default_speed_override_when_speed_unset() {
|
||||||
let _guard = ScannerDefaultSpeedGuard::set(ScannerSpeed::Slowest);
|
let _guard = ScannerDefaultSpeedGuard::set(ScannerSpeed::Slowest);
|
||||||
let s = DynamicSleeper::new(ScannerSpeed::Default);
|
let s = DynamicSleeper::new(ScannerSpeed::Default);
|
||||||
@@ -362,7 +359,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test(start_paused = true)]
|
#[tokio::test(start_paused = true)]
|
||||||
#[serial]
|
|
||||||
async fn test_fastest_never_sleeps() {
|
async fn test_fastest_never_sleeps() {
|
||||||
let prev_mode = SCANNER_IDLE_MODE.load(Ordering::Relaxed);
|
let prev_mode = SCANNER_IDLE_MODE.load(Ordering::Relaxed);
|
||||||
SCANNER_IDLE_MODE.store(true, Ordering::Relaxed);
|
SCANNER_IDLE_MODE.store(true, Ordering::Relaxed);
|
||||||
@@ -376,7 +372,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test(start_paused = true)]
|
#[tokio::test(start_paused = true)]
|
||||||
#[serial]
|
|
||||||
async fn test_idle_mode_off_skips_sleep() {
|
async fn test_idle_mode_off_skips_sleep() {
|
||||||
let prev_mode = SCANNER_IDLE_MODE.load(Ordering::Relaxed);
|
let prev_mode = SCANNER_IDLE_MODE.load(Ordering::Relaxed);
|
||||||
SCANNER_IDLE_MODE.store(false, Ordering::Relaxed);
|
SCANNER_IDLE_MODE.store(false, Ordering::Relaxed);
|
||||||
|
|||||||
@@ -47,6 +47,8 @@ pub(crate) use rustfs_ecstore::api::bucket::versioning_sys::BucketVersioningSys
|
|||||||
pub(crate) use rustfs_ecstore::api::cache::{
|
pub(crate) use rustfs_ecstore::api::cache::{
|
||||||
ListPathRawOptions as EcstoreListPathRawOptions, list_path_raw as ecstore_list_path_raw,
|
ListPathRawOptions as EcstoreListPathRawOptions, list_path_raw as ecstore_list_path_raw,
|
||||||
};
|
};
|
||||||
|
#[cfg(test)]
|
||||||
|
pub(crate) use rustfs_ecstore::api::capacity::PoolDecommissionInfo as EcstorePoolDecommissionInfo;
|
||||||
pub(crate) use rustfs_ecstore::api::capacity::{
|
pub(crate) use rustfs_ecstore::api::capacity::{
|
||||||
is_reserved_or_invalid_bucket as ecstore_is_reserved_or_invalid_bucket, path2_bucket_object as ecstore_path2_bucket_object,
|
is_reserved_or_invalid_bucket as ecstore_is_reserved_or_invalid_bucket, path2_bucket_object as ecstore_path2_bucket_object,
|
||||||
path2_bucket_object_with_base_path as ecstore_path2_bucket_object_with_base_path,
|
path2_bucket_object_with_base_path as ecstore_path2_bucket_object_with_base_path,
|
||||||
@@ -127,9 +129,9 @@ pub(crate) mod owner {
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
pub(crate) use super::{
|
pub(crate) use super::{
|
||||||
EcstoreDiskOption, EcstoreDiskStore, EcstoreEndpoint, EcstoreEndpointServerPools, EcstoreEndpoints,
|
EcstoreDiskOption, EcstoreDiskStore, EcstoreEndpoint, EcstoreEndpointServerPools, EcstoreEndpoints,
|
||||||
EcstoreInstanceContext, EcstorePoolEndpoints, EcstoreRebalStatus, EcstoreRebalanceInfo, EcstoreRebalanceMeta,
|
EcstoreInstanceContext, EcstorePoolDecommissionInfo, EcstorePoolEndpoints, EcstoreRebalStatus, EcstoreRebalanceInfo,
|
||||||
EcstoreRebalanceStats, ecstore_config_init, ecstore_init_bucket_metadata_sys, ecstore_init_local_disks_with_instance_ctx,
|
EcstoreRebalanceMeta, EcstoreRebalanceStats, ecstore_config_init, ecstore_init_bucket_metadata_sys,
|
||||||
ecstore_new_disk,
|
ecstore_init_local_disks_with_instance_ctx, ecstore_new_disk,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -14,7 +14,6 @@
|
|||||||
|
|
||||||
#![recursion_limit = "256"]
|
#![recursion_limit = "256"]
|
||||||
|
|
||||||
use futures::FutureExt;
|
|
||||||
use rustfs_config::ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT;
|
use rustfs_config::ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT;
|
||||||
use rustfs_scanner::scanner_folder::ScannerItem;
|
use rustfs_scanner::scanner_folder::ScannerItem;
|
||||||
use rustfs_scanner::scanner_io::ScannerIODisk;
|
use rustfs_scanner::scanner_io::ScannerIODisk;
|
||||||
@@ -23,10 +22,8 @@ use rustfs_scanner::{
|
|||||||
scanner::init_data_scanner,
|
scanner::init_data_scanner,
|
||||||
};
|
};
|
||||||
use s3s::dto::RestoreRequest;
|
use s3s::dto::RestoreRequest;
|
||||||
use serial_test::serial;
|
|
||||||
use std::{
|
use std::{
|
||||||
collections::HashMap,
|
collections::HashMap,
|
||||||
env,
|
|
||||||
path::{Path, PathBuf},
|
path::{Path, PathBuf},
|
||||||
sync::{Arc, Once, OnceLock},
|
sync::{Arc, Once, OnceLock},
|
||||||
time::Duration,
|
time::Duration,
|
||||||
@@ -535,31 +532,15 @@ async fn wait_for_transition(ecstore: &Arc<ECStore>, bucket: &str, object: &str,
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// SAFETY: this helper is used only by `#[serial]` tests and runs under the single-threaded Tokio
|
// Run `test_fn` with `ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT`
|
||||||
// runtime (`worker_threads = 1`), so no concurrent test can mutate process environment during the
|
// set to `"1"` for its duration. `temp_env` serializes environment mutations
|
||||||
// `env::set_var` / `env::remove_var` window.
|
// globally, preventing data races when multiple tests run in parallel.
|
||||||
#[allow(unsafe_code)]
|
|
||||||
async fn with_forced_immediate_enqueue_timeout<F, Fut>(test_fn: F)
|
async fn with_forced_immediate_enqueue_timeout<F, Fut>(test_fn: F)
|
||||||
where
|
where
|
||||||
F: FnOnce() -> Fut,
|
F: FnOnce() -> Fut,
|
||||||
Fut: std::future::Future<Output = ()>,
|
Fut: std::future::Future<Output = ()>,
|
||||||
{
|
{
|
||||||
let original = env::var_os(ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT);
|
temp_env::async_with_vars([(ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT, Some("1"))], test_fn()).await;
|
||||||
unsafe {
|
|
||||||
env::set_var(ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT, "1");
|
|
||||||
}
|
|
||||||
let result = std::panic::AssertUnwindSafe(test_fn()).catch_unwind().await;
|
|
||||||
match original {
|
|
||||||
Some(value) => unsafe {
|
|
||||||
env::set_var(ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT, value);
|
|
||||||
},
|
|
||||||
None => unsafe {
|
|
||||||
env::remove_var(ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT);
|
|
||||||
},
|
|
||||||
}
|
|
||||||
if let Err(err) = result {
|
|
||||||
std::panic::resume_unwind(err);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
mod serial_tests {
|
mod serial_tests {
|
||||||
@@ -592,7 +573,6 @@ mod serial_tests {
|
|||||||
/// body (GET won) or a clean object/version-not-found (expiry won). A
|
/// body (GET won) or a clean object/version-not-found (expiry won). A
|
||||||
/// tier-fetch failure -- the #3491 symptom -- is never tolerated.
|
/// tier-fetch failure -- the #3491 symptom -- is never tolerated.
|
||||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||||
#[serial]
|
|
||||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-2)"]
|
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-2)"]
|
||||||
async fn test_expire_transitioned_object_never_races_concurrent_get() {
|
async fn test_expire_transitioned_object_never_races_concurrent_get() {
|
||||||
let (_disk_paths, ecstore) = setup_isolated_test_env(false).await;
|
let (_disk_paths, ecstore) = setup_isolated_test_env(false).await;
|
||||||
@@ -738,7 +718,6 @@ mod serial_tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||||
#[serial]
|
|
||||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial"]
|
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial"]
|
||||||
async fn rejected_transition_candidate_is_recovered_from_persisted_delete_journal() {
|
async fn rejected_transition_candidate_is_recovered_from_persisted_delete_journal() {
|
||||||
let (_disk_paths, ecstore) = setup_isolated_test_env(false).await;
|
let (_disk_paths, ecstore) = setup_isolated_test_env(false).await;
|
||||||
@@ -825,7 +804,6 @@ mod serial_tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||||
#[serial]
|
|
||||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial"]
|
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial"]
|
||||||
async fn cancelled_before_cleanup_store_resolution_persists_journal() {
|
async fn cancelled_before_cleanup_store_resolution_persists_journal() {
|
||||||
let (_disk_paths, ecstore) = setup_isolated_test_env(false).await;
|
let (_disk_paths, ecstore) = setup_isolated_test_env(false).await;
|
||||||
@@ -919,7 +897,6 @@ mod serial_tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||||
#[serial]
|
|
||||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial"]
|
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial"]
|
||||||
async fn rejected_transition_cleanup_durability_matrix() {
|
async fn rejected_transition_cleanup_durability_matrix() {
|
||||||
#[derive(Clone, Copy)]
|
#[derive(Clone, Copy)]
|
||||||
@@ -1059,7 +1036,6 @@ mod serial_tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
|
||||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
|
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
|
||||||
fn test_transition_and_restore_flows() {
|
fn test_transition_and_restore_flows() {
|
||||||
std::thread::Builder::new()
|
std::thread::Builder::new()
|
||||||
@@ -1385,7 +1361,6 @@ mod serial_tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||||
#[serial]
|
|
||||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
|
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
|
||||||
async fn test_scanner_enqueues_free_version_cleanup_for_stale_transitioned_object() {
|
async fn test_scanner_enqueues_free_version_cleanup_for_stale_transitioned_object() {
|
||||||
let (disk_paths, ecstore) = setup_isolated_test_env(false).await;
|
let (disk_paths, ecstore) = setup_isolated_test_env(false).await;
|
||||||
@@ -1446,7 +1421,6 @@ mod serial_tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||||
#[serial]
|
|
||||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
|
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
|
||||||
async fn test_scanner_cleanup_still_works_after_immediate_compensation_transition() {
|
async fn test_scanner_cleanup_still_works_after_immediate_compensation_transition() {
|
||||||
let (disk_paths, ecstore) = setup_isolated_test_env(false).await;
|
let (disk_paths, ecstore) = setup_isolated_test_env(false).await;
|
||||||
@@ -1504,7 +1478,6 @@ mod serial_tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||||
#[serial]
|
|
||||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
|
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
|
||||||
async fn test_existing_object_backfill_is_idempotent_after_immediate_compensation_transition() {
|
async fn test_existing_object_backfill_is_idempotent_after_immediate_compensation_transition() {
|
||||||
let (_disk_paths, ecstore) = setup_isolated_test_env(false).await;
|
let (_disk_paths, ecstore) = setup_isolated_test_env(false).await;
|
||||||
@@ -1547,7 +1520,6 @@ mod serial_tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||||
#[serial]
|
|
||||||
#[ignore = "FAILING on main: excluded from the serial ILM lane pending a fix, see rustfs/backlog#1148 (ilm-1 partial)"]
|
#[ignore = "FAILING on main: excluded from the serial ILM lane pending a fix, see rustfs/backlog#1148 (ilm-1 partial)"]
|
||||||
async fn test_noncurrent_expiry_still_works_after_immediate_compensation_transition() {
|
async fn test_noncurrent_expiry_still_works_after_immediate_compensation_transition() {
|
||||||
let (disk_paths, ecstore) = setup_isolated_test_env(true).await;
|
let (disk_paths, ecstore) = setup_isolated_test_env(true).await;
|
||||||
@@ -1631,7 +1603,6 @@ mod serial_tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||||
#[serial]
|
|
||||||
#[ignore = "FAILING on main: excluded from the serial ILM lane pending a fix, see rustfs/backlog#1148 (ilm-1 partial)"]
|
#[ignore = "FAILING on main: excluded from the serial ILM lane pending a fix, see rustfs/backlog#1148 (ilm-1 partial)"]
|
||||||
async fn test_noncurrent_transition_still_works_after_immediate_compensation_transition() {
|
async fn test_noncurrent_transition_still_works_after_immediate_compensation_transition() {
|
||||||
let (disk_paths, ecstore) = setup_isolated_test_env(true).await;
|
let (disk_paths, ecstore) = setup_isolated_test_env(true).await;
|
||||||
@@ -1714,7 +1685,6 @@ mod serial_tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||||
#[serial]
|
|
||||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
|
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
|
||||||
async fn test_modeled_versioned_delete_creates_delete_marker_after_immediate_compensation_transition() {
|
async fn test_modeled_versioned_delete_creates_delete_marker_after_immediate_compensation_transition() {
|
||||||
let (_disk_paths, ecstore) = setup_isolated_test_env(true).await;
|
let (_disk_paths, ecstore) = setup_isolated_test_env(true).await;
|
||||||
@@ -1762,7 +1732,6 @@ mod serial_tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||||
#[serial]
|
|
||||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
|
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
|
||||||
async fn test_modeled_delete_marker_cleanup_after_immediate_compensation_transition() {
|
async fn test_modeled_delete_marker_cleanup_after_immediate_compensation_transition() {
|
||||||
let (disk_paths, ecstore) = setup_isolated_test_env(true).await;
|
let (disk_paths, ecstore) = setup_isolated_test_env(true).await;
|
||||||
@@ -1839,7 +1808,6 @@ mod serial_tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||||
#[serial]
|
|
||||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
|
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
|
||||||
async fn test_scanner_expires_zero_day_current_version() {
|
async fn test_scanner_expires_zero_day_current_version() {
|
||||||
let (disk_paths, ecstore) = setup_isolated_test_env(false).await;
|
let (disk_paths, ecstore) = setup_isolated_test_env(false).await;
|
||||||
@@ -1866,7 +1834,6 @@ mod serial_tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||||
#[serial]
|
|
||||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
|
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
|
||||||
async fn test_put_object_immediately_enqueues_zero_day_current_expiry() {
|
async fn test_put_object_immediately_enqueues_zero_day_current_expiry() {
|
||||||
let (_disk_paths, ecstore) = setup_isolated_test_env(true).await;
|
let (_disk_paths, ecstore) = setup_isolated_test_env(true).await;
|
||||||
@@ -1904,7 +1871,6 @@ mod serial_tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||||
#[serial]
|
|
||||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
|
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
|
||||||
async fn test_scanner_expires_zero_day_noncurrent_version() {
|
async fn test_scanner_expires_zero_day_noncurrent_version() {
|
||||||
let (disk_paths, ecstore) = setup_isolated_test_env(false).await;
|
let (disk_paths, ecstore) = setup_isolated_test_env(false).await;
|
||||||
@@ -1971,7 +1937,6 @@ mod serial_tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||||
#[serial]
|
|
||||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
|
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
|
||||||
async fn test_put_object_immediately_enqueues_zero_day_noncurrent_expiry() {
|
async fn test_put_object_immediately_enqueues_zero_day_noncurrent_expiry() {
|
||||||
let (_disk_paths, ecstore) = setup_isolated_test_env(true).await;
|
let (_disk_paths, ecstore) = setup_isolated_test_env(true).await;
|
||||||
@@ -2032,7 +1997,6 @@ mod serial_tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||||
#[serial]
|
|
||||||
async fn test_background_scanner_expires_zero_day_current_version() {
|
async fn test_background_scanner_expires_zero_day_current_version() {
|
||||||
let (_disk_paths, ecstore) = setup_isolated_test_env(true).await;
|
let (_disk_paths, ecstore) = setup_isolated_test_env(true).await;
|
||||||
|
|
||||||
@@ -2056,7 +2020,6 @@ mod serial_tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||||
#[serial]
|
|
||||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
|
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
|
||||||
async fn test_background_scanner_expires_zero_day_current_version_for_exact_key_prefix() {
|
async fn test_background_scanner_expires_zero_day_current_version_for_exact_key_prefix() {
|
||||||
let (_disk_paths, ecstore) = setup_isolated_test_env(true).await;
|
let (_disk_paths, ecstore) = setup_isolated_test_env(true).await;
|
||||||
@@ -2122,7 +2085,6 @@ mod serial_tests {
|
|||||||
/// tier object is untouched (zero `remove` calls) -> GET streams from the
|
/// tier object is untouched (zero `remove` calls) -> GET streams from the
|
||||||
/// tier again -> a second restore succeeds.
|
/// tier again -> a second restore succeeds.
|
||||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||||
#[serial]
|
|
||||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-8)"]
|
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-8)"]
|
||||||
async fn test_restore_chain_local_read_expiry_keeps_remote_and_allows_re_restore() {
|
async fn test_restore_chain_local_read_expiry_keeps_remote_and_allows_re_restore() {
|
||||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||||
@@ -2254,7 +2216,6 @@ mod serial_tests {
|
|||||||
/// parts) must reassemble the exact part layout: part count and sizes,
|
/// parts) must reassemble the exact part layout: part count and sizes,
|
||||||
/// the multipart ETag, and byte-identical content across part boundaries.
|
/// the multipart ETag, and byte-identical content across part boundaries.
|
||||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||||
#[serial]
|
|
||||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-8)"]
|
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-8)"]
|
||||||
async fn test_multipart_restore_preserves_parts_and_etag() {
|
async fn test_multipart_restore_preserves_parts_and_etag() {
|
||||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||||
|
|||||||
@@ -3969,6 +3969,14 @@ fn delete_creates_delete_marker(opts: &ObjectOptions) -> bool {
|
|||||||
opts.version_id.is_none() && opts.versioned && !opts.version_suspended
|
opts.version_id.is_none() && opts.versioned && !opts.version_suspended
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// `DeleteObjects` is idempotent. A raw filesystem `NotFound` can cross the
|
||||||
|
/// distributed delete path instead of its usual typed missing-object error.
|
||||||
|
fn is_delete_objects_not_found(error: &EcstoreError) -> bool {
|
||||||
|
is_err_object_not_found(error)
|
||||||
|
|| is_err_version_not_found(error)
|
||||||
|
|| matches!(error, StorageError::Io(source) if source.kind() == std::io::ErrorKind::NotFound)
|
||||||
|
}
|
||||||
|
|
||||||
/// Bounded concurrency for the per-object pre-delete stat fanout in
|
/// Bounded concurrency for the per-object pre-delete stat fanout in
|
||||||
/// `execute_delete_objects` (backlog#929 / HP-8). Keeps the metadata reads for
|
/// `execute_delete_objects` (backlog#929 / HP-8). Keeps the metadata reads for
|
||||||
/// a 1000-key batch from serializing while capping the disk fanout pressure.
|
/// a 1000-key batch from serializing while capping the disk fanout pressure.
|
||||||
@@ -4030,6 +4038,27 @@ fn delete_response_version_id(version_id: Option<Uuid>, synthetic_version_id: bo
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn reduce_delete_objects_result<'a>(
|
||||||
|
object: &ObjectToDelete,
|
||||||
|
deleted: &'a StorageDeletedObject,
|
||||||
|
error: Option<&EcstoreError>,
|
||||||
|
synthetic_version_id: bool,
|
||||||
|
) -> Result<&'a StorageDeletedObject, s3s::dto::Error> {
|
||||||
|
match error {
|
||||||
|
None => Ok(deleted),
|
||||||
|
Some(error) if is_delete_objects_not_found(error) => Ok(deleted),
|
||||||
|
Some(error) => {
|
||||||
|
let api_error = ApiError::from(error.clone());
|
||||||
|
Err(s3s::dto::Error {
|
||||||
|
code: Some(api_error.code.as_str().to_string()),
|
||||||
|
key: Some(object.object_name.clone()),
|
||||||
|
message: Some(api_error.message),
|
||||||
|
version_id: delete_response_version_id(object.version_id, synthetic_version_id),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn resolve_put_object_extract_options(headers: &HeaderMap) -> S3Result<PutObjectExtractOptions> {
|
fn resolve_put_object_extract_options(headers: &HeaderMap) -> S3Result<PutObjectExtractOptions> {
|
||||||
let prefix = snowball_meta_value(headers, SNOWBALL_PREFIX_HEADER_KEYS, SNOWBALL_PREFIX_SUFFIX_LOWER)
|
let prefix = snowball_meta_value(headers, SNOWBALL_PREFIX_HEADER_KEYS, SNOWBALL_PREFIX_SUFFIX_LOWER)
|
||||||
.map(|value| normalize_snowball_prefix(&value))
|
.map(|value| normalize_snowball_prefix(&value))
|
||||||
@@ -8476,39 +8505,31 @@ impl DefaultObjectUsecase {
|
|||||||
for (i, err) in errs.iter().enumerate() {
|
for (i, err) in errs.iter().enumerate() {
|
||||||
let didx = object_to_delete_idx[i];
|
let didx = object_to_delete_idx[i];
|
||||||
|
|
||||||
if err.is_none()
|
match reduce_delete_objects_result(
|
||||||
|| err
|
&object_to_delete[i],
|
||||||
.clone()
|
&dobjs[i],
|
||||||
.is_some_and(|v| is_err_object_not_found(&v) || is_err_version_not_found(&v))
|
err.as_ref(),
|
||||||
{
|
delete_results[didx].synthetic_version_id,
|
||||||
delete_results[didx].delete_object = Some(dobjs[i].clone());
|
) {
|
||||||
let (versioned, version_suspended) = object_versioning[i];
|
Ok(deleted_object) => {
|
||||||
let creates_delete_marker = object_to_delete[i].version_id.is_none() && versioned && !version_suspended;
|
delete_results[didx].delete_object = Some(deleted_object.clone());
|
||||||
if creates_delete_marker {
|
let (versioned, version_suspended) = object_versioning[i];
|
||||||
record_bucket_delete_marker_memory(&bucket).await;
|
let creates_delete_marker = object_to_delete[i].version_id.is_none() && versioned && !version_suspended;
|
||||||
} else {
|
if creates_delete_marker {
|
||||||
let size = object_sizes[i].max(0) as u64;
|
record_bucket_delete_marker_memory(&bucket).await;
|
||||||
record_bucket_object_delete_memory(
|
} else {
|
||||||
&bucket,
|
let size = object_sizes[i].max(0) as u64;
|
||||||
size,
|
record_bucket_object_delete_memory(
|
||||||
existing_object_infos[i].is_some() && object_to_delete[i].version_id.is_none(),
|
&bucket,
|
||||||
)
|
size,
|
||||||
.await;
|
existing_object_infos[i].is_some() && object_to_delete[i].version_id.is_none(),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(error) => {
|
||||||
|
delete_results[didx].error = Some(error);
|
||||||
}
|
}
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
if let Some(err) = err.clone() {
|
|
||||||
let api_error = ApiError::from(err);
|
|
||||||
delete_results[didx].error = Some(s3s::dto::Error {
|
|
||||||
code: Some(api_error.code.as_str().to_string()),
|
|
||||||
key: Some(object_to_delete[i].object_name.clone()),
|
|
||||||
message: Some(api_error.message),
|
|
||||||
version_id: delete_response_version_id(
|
|
||||||
object_to_delete[i].version_id,
|
|
||||||
delete_results[didx].synthetic_version_id,
|
|
||||||
),
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -17692,6 +17713,35 @@ mod tests {
|
|||||||
assert_eq!(internal_version_id, None);
|
assert_eq!(internal_version_id, None);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn delete_objects_treats_raw_io_not_found_as_idempotent() {
|
||||||
|
assert!(is_delete_objects_not_found(&StorageError::FileNotFound));
|
||||||
|
assert!(is_delete_objects_not_found(&StorageError::Io(std::io::Error::from(
|
||||||
|
std::io::ErrorKind::NotFound,
|
||||||
|
))));
|
||||||
|
assert!(!is_delete_objects_not_found(&StorageError::Io(std::io::Error::from(
|
||||||
|
std::io::ErrorKind::PermissionDenied,
|
||||||
|
))));
|
||||||
|
assert!(!is_delete_objects_not_found(&StorageError::DiskNotFound));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn delete_objects_result_reducer_reports_raw_not_found_as_deleted() {
|
||||||
|
let object = ObjectToDelete {
|
||||||
|
object_name: "missing-key".to_string(),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
let deleted = StorageDeletedObject {
|
||||||
|
object_name: object.object_name.clone(),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
let error = StorageError::Io(std::io::Error::from(std::io::ErrorKind::NotFound));
|
||||||
|
|
||||||
|
let deleted = reduce_delete_objects_result(&object, &deleted, Some(&error), false)
|
||||||
|
.expect("raw not-found must produce a deleted result");
|
||||||
|
assert_eq!(deleted.object_name, "missing-key");
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn recursive_force_delete_requires_administrative_or_replica_context() {
|
fn recursive_force_delete_requires_administrative_or_replica_context() {
|
||||||
let mut headers = HeaderMap::new();
|
let mut headers = HeaderMap::new();
|
||||||
|
|||||||
@@ -1537,7 +1537,7 @@ fn process_connection(
|
|||||||
None
|
None
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
// ── Canonical Middleware Stack Order (outermost → innermost) ──
|
// ── Canonical External Middleware Stack Order (outermost → innermost) ──
|
||||||
// This order MUST be preserved across refactorings.
|
// This order MUST be preserved across refactorings.
|
||||||
// Only AddExtensionLayer (layers 1-2) are per-connection; most remaining layers are stateless.
|
// Only AddExtensionLayer (layers 1-2) are per-connection; most remaining layers are stateless.
|
||||||
//
|
//
|
||||||
@@ -1565,6 +1565,8 @@ fn process_connection(
|
|||||||
// 22. PublicHealthEndpointLayer — handles public health before s3s host parsing
|
// 22. PublicHealthEndpointLayer — handles public health before s3s host parsing
|
||||||
// 23. VirtualHostStyleHintLayer — actionable error for unroutable virtual-hosted-style (conditional)
|
// 23. VirtualHostStyleHintLayer — actionable error for unroutable virtual-hosted-style (conditional)
|
||||||
// 24. DoubleSlashListBucketsCompatLayer — rewrites `GET //` to `GET /` for ListBuckets (MinIO browser compat)
|
// 24. DoubleSlashListBucketsCompatLayer — rewrites `GET //` to `GET /` for ListBuckets (MinIO browser compat)
|
||||||
|
// The internode lane below intentionally keeps only the shared
|
||||||
|
// transport/auth/observability subset needed by `/rustfs/rpc/...`.
|
||||||
// ─────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────
|
||||||
let build_external_stack = |service| {
|
let build_external_stack = |service| {
|
||||||
ServiceBuilder::new()
|
ServiceBuilder::new()
|
||||||
@@ -1747,16 +1749,9 @@ fn process_connection(
|
|||||||
.layer(PropagateRequestIdLayer::x_request_id())
|
.layer(PropagateRequestIdLayer::x_request_id())
|
||||||
.layer(CompressionLayer::new().compress_when(PathAwareHttpCompressionPredicate::new(compression_config.clone())))
|
.layer(CompressionLayer::new().compress_when(PathAwareHttpCompressionPredicate::new(compression_config.clone())))
|
||||||
.option_layer(compression_config.enabled.then_some(PathCategoryInjectionLayer))
|
.option_layer(compression_config.enabled.then_some(PathCategoryInjectionLayer))
|
||||||
.layer(S3ErrorMessageCompatLayer)
|
// The internode lane only serves `/rustfs/rpc/...` gRPC requests.
|
||||||
.layer(IcebergRestErrorCompatLayer)
|
// Keep safety/observability layers above, but leave S3/REST
|
||||||
.layer(ObjectAttributesEtagFixLayer)
|
// compatibility rewrites on the external lane.
|
||||||
.layer(ConditionalCorsLayer::new())
|
|
||||||
.option_layer(if is_console { Some(RedirectLayer) } else { None })
|
|
||||||
.layer(BodylessStatusFixLayer)
|
|
||||||
.layer(HeadRequestBodyFixLayer)
|
|
||||||
.layer(PublicHealthEndpointLayer::new(Arc::clone(&server_ctx)))
|
|
||||||
.option_layer((!server_domains_configured && !is_console).then_some(VirtualHostStyleHintLayer))
|
|
||||||
.layer(DoubleSlashListBucketsCompatLayer)
|
|
||||||
.service(service)
|
.service(service)
|
||||||
};
|
};
|
||||||
let external_stack_service = build_external_stack(external_service);
|
let external_stack_service = build_external_stack(external_service);
|
||||||
|
|||||||
Reference in New Issue
Block a user