Compare commits

..

16 Commits

Author SHA1 Message Date
cxymds 8585d39b6b fix(scanner): reject unmeasured admin usage snapshots
Validate the persisted scanner baseline before identity filtering can mark
it as displayable. Cover two buckets with only one current identity and
adapt the backup catchup cache save calls to the merged API.
2026-11-21 18:45:34 +08:00
houseme bc78014592 Merge remote-tracking branch 'refs/remotes/origin/main' into feat/scanner-observability
# Conflicts:
#	crates/scanner/src/scanner_io/cache.rs

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
2026-11-21 05:22:28 +00:00
Chris An 781cf7360e Merge branch 'main' into feat/scanner-observability 2026-11-21 06:17:36 +08:00
cxymds d8406f9494 Merge branch 'main' into feat/scanner-observability 2026-11-20 09:33:37 +08:00
cxymds ed12c5320d Merge branch 'main' into feat/scanner-observability 2026-11-19 08:12:38 +08:00
cxymds eb57789f0f fix(scanner): require a measured baseline for admin usage overlays 2026-11-18 11:10:55 +08:00
cxymds 791b1bcd86 chore: sync main for gateway and multipart CI fixes 2026-11-18 11:10:55 +08:00
cxymds 0517566fad chore: sync main for scanner statistics review 2026-11-15 15:56:57 +08:00
cxymds 73895b37e8 test(admin): bind snapshot fixtures to current bucket identities 2026-11-15 14:49:54 +08:00
cxymds 00243c7e71 chore: sync main for scanner validation
Resolve the shared checkpoint fixture update by retaining production drain and the stronger carried-value, certificate, budget and acknowledgement assertions. Preserve upstream functionality and the validated scanner implementation.
2026-11-15 12:02:04 +08:00
cxymds 36bc1057ac fix(ci): give the multipart refusal fixture its storage stack 2026-11-15 11:25:22 +08:00
cxymds 32f7f70a29 feat(scanner): observe bucket reuse and production proof gates 2026-11-15 11:25:22 +08:00
cxymds 5d32827af5 fix(usage): validate cached statistics against bucket identity 2026-11-15 11:25:06 +08:00
cxymds 04ef836aa3 fix(scanner): cap superseded retries at the nominal cycle 2026-11-15 11:25:05 +08:00
cxymds 03850c1cd6 fix(scanner): preserve baselines during backup catchup
Require decoded-primary provenance before accepting an unchanged baseline after its backup catches up. Keep current-revision CAS and publication fences, and align the real checkpoint fixture with the bounded-cycle carry contract.
2026-11-15 11:24:42 +08:00
cxymds 654be96186 feat(scanner): expose scan work and reuse observations 2026-11-14 21:25:19 +08:00
119 changed files with 13036 additions and 11904 deletions
-1
View File
@@ -57,7 +57,6 @@ rustfs-scanner 1392 20
rustfs-scanner-metrics 90 0
rustfs-security-governance 20 0
rustfs-signer 34 0
rustfs-sse 145 0
rustfs-storage-api 34 0
rustfs-targets 612 26
rustfs-test-utils 1 0
+1 -1
View File
@@ -231,7 +231,7 @@ jobs:
# --all-targets links the same test binaries as the reader lane.
CARGO_BUILD_JOBS: "2"
run: |
cargo build -p rustfs -p rustfs-ecstore -p rustfs-sse --all-targets --features rio-v2
cargo build -p rustfs -p rustfs-ecstore --all-targets --features rio-v2
cargo build -p rustfs --bins --features rio-v2,e2e-test-hooks
# Readers: the swift and sftp legs of test-and-lint-protocols. Built in
+2 -2
View File
@@ -563,7 +563,7 @@ jobs:
run: chmod go-w "$(realpath "$HOME")"
- name: Run rio-v2 clippy lints
run: cargo clippy -p rustfs -p rustfs-ecstore -p rustfs-sse --all-targets --features rio-v2 -- -D warnings
run: cargo clippy -p rustfs -p rustfs-ecstore --all-targets --features rio-v2 -- -D warnings
- name: Run rio-v2 feature tests
shell: bash
@@ -580,7 +580,7 @@ jobs:
rm -f target/nextest/ci/junit.xml
./scripts/ci/resource_sampler.sh start nextest-rio-v2
trap './scripts/ci/resource_sampler.sh stop' EXIT
cargo nextest run --profile ci -p rustfs -p rustfs-ecstore -p rustfs-sse --features rio-v2 \
cargo nextest run --profile ci -p rustfs -p rustfs-ecstore --features rio-v2 \
2>&1 | tee artifacts/workspace/nextest.log
cargo test -p rustfs --doc --features rio-v2
+1 -1
View File
@@ -94,7 +94,7 @@ refactors.
|--------|--------------------------|----------------|
| Foundation | `checksums`, `common`, `config`, `data-usage`, `heal-contracts`, `scanner-metrics`, `utils` | Shared configuration, data-usage models, heal domain contracts, scanner telemetry types, utilities, and checksums. |
| I/O and storage | `concurrency`, `ecstore`, `filemeta`, `heal`, `io-core`, `io-metrics`, `lifecycle`, `lock`, `object-capacity`, `object-data-cache`, `replication`, `rio`, `rio-v2`, `s3-client`, `scanner`, `storage-api` | Erasure-coded object storage, metadata, recovery, lifecycle, replication, locking, cache, I/O pipelines, and the engine-side S3 client for remote tier/transition targets. |
| Security and identity | `credentials`, `crypto`, `iam`, `keystone`, `kms`, `policy`, `security-governance`, `signer`, `sse`, `tls-runtime`, `trusted-proxies` | Credentials, authentication, authorization, encryption, key management, server-side encryption key resolution, TLS, and security contracts. |
| Security and identity | `credentials`, `crypto`, `iam`, `keystone`, `kms`, `policy`, `security-governance`, `signer`, `tls-runtime`, `trusted-proxies` | Credentials, authentication, authorization, encryption, key management, TLS, and security contracts. |
| Protocols and contracts | `extension-schema`, `madmin`, `protos`, `protocols`, `s3-ops`, `s3-types`, `s3select-api`, `s3select-query` | Admin, inter-node, S3, S3 Select, and optional protocol contracts. |
| Operations and integration | `audit`, `notify`, `obs`, `targets`, `zip` | Auditing, observability, event delivery, notification targets, and archive support. |
| Test support | `e2e_test`, `test-utils` | End-to-end validation and shared test bootstrap utilities. |
Generated
+187 -212
View File
File diff suppressed because it is too large Load Diff
+19 -24
View File
@@ -60,7 +60,6 @@ members = [
"crates/security-governance", # Security governance contracts
"crates/extension-schema", # Extension schema contracts
"crates/signer", # client signer
"crates/sse", # S3 server-side encryption key resolution and metadata
"crates/storage-api", # Storage API contracts
"crates/test-utils", # Shared test bootstrap helpers (dev-dependency only)
"crates/targets", # Target-specific configurations and utilities
@@ -141,7 +140,6 @@ rustfs-scanner = { path = "crates/scanner", version = "1.0.5" }
rustfs-security-governance = { path = "crates/security-governance", version = "1.0.5" }
rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.5" }
rustfs-signer = { path = "crates/signer", version = "1.0.5" }
rustfs-sse = { path = "crates/sse", version = "1.0.5" }
rustfs-storage-api = { path = "crates/storage-api", version = "1.0.5" }
rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.5" }
rustfs-targets = { path = "crates/targets", version = "1.0.5" }
@@ -177,9 +175,9 @@ futures = "0.3.34"
futures-core = "0.3.34"
futures-lite = "2.6.1"
futures-util = "0.3.34"
pollster = "1.0.3"
pollster = "1.0.2"
pulsar = { default-features = false, version = "=6.9.0" }
lapin = { default-features = false, version = "4.13.0" }
lapin = { default-features = false, version = "4.12.3" }
hyper = { version = "1.12.0" }
h2 = { version = "0.4.20" }
hyper-rustls = { default-features = false, version = "0.27.10" }
@@ -218,7 +216,7 @@ csv-core = "0.1.13"
bytesize = "2.7.0"
byteorder = "1.5.0"
flatbuffers = "25.12.19"
form_urlencoded = "1.2.3"
form_urlencoded = "1.2.2"
prost = "0.14.6"
quick-xml = "0.42.0"
rmp = { version = "0.8.15" }
@@ -260,7 +258,7 @@ proptest = "1"
# Time and Date
chrono = { version = "0.4.45" }
humantime = "2.4.0"
jiff = { version = "1.0.2" }
jiff = { version = "1.0.1" }
time = { version = "0.3.55" }
# Database
@@ -280,15 +278,15 @@ atomic_enum = "0.3.0"
aws-config = { version = "1.13.3" }
aws-credential-types = { version = "1.3.1" }
aws-sdk-kms = { default-features = false, version = "1.133.0" }
aws-sdk-s3 = { default-features = false, version = "1.161.0" }
aws-sdk-sts = { default-features = false, version = "1.128.0" }
aws-sdk-s3 = { default-features = false, version = "1.160.0" }
aws-sdk-sts = { default-features = false, version = "1.127.0" }
aws-smithy-async = { version = "1.3.1" }
aws-smithy-http-client = { default-features = false, version = "1.5.3" }
aws-smithy-runtime-api = { version = "1.22.0" }
aws-smithy-types = { version = "1.9.4" }
base64-simd = "0.8.0"
brotli = "9.1.3"
clap = { version = "4.6.9" }
clap = { version = "4.6.8" }
const-str = { version = "1.1.0" }
convert_case = "0.12.0"
criterion = { version = "0.8" }
@@ -300,12 +298,12 @@ datafusion = { default-features = false, version = "55.3.0" }
derive_builder = "0.20.2"
ed25519-dalek = "3.0.0"
enumset = "1.1.15"
faster-hex = "2.0.0"
faster-hex = "1.0.0"
flate2 = "1.1.11"
glob = "0.3.4"
google-cloud-storage = "1.22.0"
google-cloud-auth = "1.19.0"
hashbrown = { version = "0.17.2" }
hashbrown = { version = "0.17.1" }
# Base32 for RFC 6238 TOTP shared secrets (RFC 4648 unpadded, the alphabet
# every authenticator app expects). Already in the graph transitively.
data-encoding = "2.11.1"
@@ -339,7 +337,7 @@ percent-encoding = "2.3.2"
# CLI needs its own QR encoder. No default features: the image/render backends
# pull in an image stack this only needs SVG and text output from.
qrcode-rs = { version = "2.2.0", default-features = false, features = ["std", "svg"] }
pin-project-lite = "0.2.18"
pin-project-lite = "0.2.17"
pretty_assertions = "1.4.1"
rand = { version = "0.10.4" }
ratelimit = "2.0.0"
@@ -362,7 +360,7 @@ legacy-s3s = { package = "s3s", git = "https://github.com/s3s-project/s3s", rev
serial_test = "4.0.1"
shadow-rs = { default-features = false, version = "2.0.0" }
siphasher = "1.0.4"
smallvec = { version = "1.16.6" }
smallvec = { version = "1.16.5" }
compact_str = "0.10.0"
snap = "1.1.2"
starshard = { version = "2.4.1" }
@@ -380,23 +378,23 @@ tracing-error = "0.2.1"
tracing-opentelemetry = { version = "0.34.0" }
tracing-subscriber = { version = "0.3.23" }
transform-stream = "0.3.1"
url = "2.5.9"
url = "2.5.8"
urlencoding = "2.1.3"
uuid = { version = "1.29.3" }
vaultrs = { version = "0.8.0" }
tar = "0.4.46"
walkdir = "2.5.1"
walkdir = "2.5.0"
winapi-util = "0.1.11"
windows = { version = "0.62.2" }
windows-sys = "0.61.2"
xxhash-rust = { version = "0.8.21" }
zip = "9.8.1"
zip = "9.7.0"
zstd = "0.14.4"
# Observability and Metrics
metrics = "0.24.6"
metrics-util = "0.20"
dial9 = { version = "0.5.9", default-features = false }
dial9 = { version = "0.5.8", default-features = false }
opentelemetry = { version = "0.33.1" }
opentelemetry-appender-tracing = { version = "0.33.1" }
opentelemetry-otlp = { version = "0.33.1" }
@@ -407,12 +405,12 @@ opentelemetry-stdout = { version = "0.33.1" }
pyroscope = { version = "2.1.5" }
# FTP and SFTP
libunftp = { version = "0.25.1" }
libunftp = { version = "0.25.0" }
unftp-core = "0.1.1"
suppaftp = { version = "13.0.0" }
rcgen = { version = "0.14.10", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
russh = { version = "0.65.4" }
russh-sftp = "4.0.0"
russh = { version = "0.65.2" }
russh-sftp = "3.0.1"
# WebDAV
dav-server = "0.12.0"
@@ -425,10 +423,7 @@ hotpath = { version = "0.32.1", default-features = false }
insta = { version = "1.49" }
[workspace.metadata.cargo-shear]
# The `#[path]` test modules under `rustfs/src/server/http/tests/` are linked by
# rustc (including the `http.rs` + `http/` sibling-directory rule), but cargo-shear
# resolves those nested `#[path]` attributes against the wrong base directory.
ignored-paths = ["rustfs/src/server/http/tests/**/*.rs"]
ignored = ["hotpath", "rustfs"]
[profile.dev]
# Full debuginfo roughly doubles compile+link time and produces multi-GB
+4 -3
View File
@@ -15,11 +15,12 @@ workspace = true
[features]
default = []
hotpath = ["rustfs-io-core/hotpath"]
hotpath-alloc = ["hotpath", "rustfs-io-core/hotpath-alloc"]
hotpath-cpu = ["hotpath", "rustfs-io-core/hotpath-cpu"]
hotpath = ["hotpath/hotpath", "hotpath/tokio", "rustfs-io-core/hotpath"]
hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc", "rustfs-io-core/hotpath-alloc"]
hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu", "rustfs-io-core/hotpath-cpu"]
[dependencies]
hotpath.workspace = true
# Internal crates
rustfs-io-core = { workspace = true }
serde = { workspace = true, features = ["derive"] }
@@ -171,6 +171,70 @@ impl DataUsageBucketObservation {
}
impl DataUsageInfo {
/// Hide values that cannot be bound to the current storage-owned identity.
///
/// `incarnations` contains canonical non-nil identities read from storage;
/// a missing entry means identity was unavailable. Neither legacy values
/// without identity nor metadata read failures establish confirmed zero.
/// Changed responses are display-only and cannot authorize quota or writes.
/// If retained counters overflow, all values remain unknown.
/// Returns the number of canonical values removed.
pub fn retain_current_bucket_usage_for_admin(&mut self, incarnations: &HashMap<String, String>) -> usize {
let original_count = self.buckets_usage.len();
let original_observation_count = self.usage_snapshot_bucket_observations.len();
let matches_current = |bucket: &String, usage: &BucketUsageInfo| {
incarnations.get(bucket).is_some_and(|current| {
!current.is_empty()
&& current != "00000000-0000-0000-0000-000000000000"
&& usage.bucket_incarnation.as_ref() == Some(current)
})
};
self.buckets_usage.retain(|bucket, usage| matches_current(bucket, usage));
self.usage_snapshot_bucket_observations.retain(|bucket, observation| {
matches_current(bucket, &observation.usage)
&& observation
.sources
.iter()
.all(|source| incarnations.get(bucket) == Some(&source.bucket_incarnation))
});
if original_count == self.buckets_usage.len()
&& original_observation_count == self.usage_snapshot_bucket_observations.len()
{
return 0;
}
let totals = self.buckets_usage.values().try_fold([0_u64; 4], |total, usage| {
Some([
total[0].checked_add(usage.size)?,
total[1].checked_add(usage.objects_count)?,
total[2].checked_add(usage.versions_count)?,
total[3].checked_add(usage.delete_markers_count)?,
])
});
let [size, objects, versions, markers] = totals.unwrap_or_else(|| {
self.buckets_usage.clear();
self.usage_snapshot_bucket_observations.clear();
[0; 4]
});
self.bucket_sizes = self
.buckets_usage
.iter()
.map(|(bucket, usage)| (bucket.clone(), usage.size))
.collect();
self.buckets_count = u64::try_from(self.buckets_usage.len()).unwrap_or(u64::MAX);
self.objects_total_size = size;
self.objects_total_count = objects;
self.versions_total_count = versions;
self.delete_markers_total_count = markers;
self.replication_info.clear();
self.tier_stats = None;
self.unknown_tier_stats = None;
self.usage_snapshot_complete = false;
self.usage_snapshot_partial = true;
self.usage_snapshot_converged = Some(false);
self.usage_snapshot_admin_projection = true;
original_count.saturating_sub(self.buckets_usage.len())
}
/// Overlay validated companions on an owned admin response only.
///
/// `incarnations` must come from current storage metadata, not the companion.
@@ -321,3 +321,110 @@ fn provenance_and_companions_change_candidate_bytes_and_round_trip_as_maps() {
let restored: DataUsageInfo = rmp_serde::from_slice(&named).expect("read named map with additive fields");
assert_eq!(restored, canonical);
}
#[test]
fn current_identity_filter_preserves_an_unchanged_complete_snapshot() {
let mut candidate = snapshot();
let original = candidate.clone();
assert_eq!(candidate.retain_current_bucket_usage_for_admin(&incarnations()), 0);
assert_eq!(candidate, original);
assert!(candidate.is_complete_bucket_usage_snapshot());
}
#[test]
fn current_identity_filter_hides_unbound_values_without_confirming_zero() {
for identity in [
None,
Some(""),
Some("corrupt"),
Some("00000000-0000-0000-0000-000000000000"),
Some("22222222-2222-4222-8222-222222222222"),
] {
let mut candidate = snapshot();
candidate.usage_snapshot_bucket_observations.clear();
candidate
.buckets_usage
.get_mut("bucket")
.expect("canonical")
.bucket_incarnation = identity.map(str::to_string);
assert_eq!(
candidate.retain_current_bucket_usage_for_admin(&incarnations()),
1,
"identity {identity:?}"
);
assert!(!candidate.buckets_usage.contains_key("bucket"));
assert!(!candidate.bucket_sizes.contains_key("bucket"));
assert_eq!(candidate.usage_snapshot_converged, Some(false));
assert!(candidate.usage_snapshot_admin_projection);
assert!(!candidate.is_complete_bucket_usage_snapshot());
assert!(!candidate.is_valid_partial_snapshot());
}
for current in ["", "00000000-0000-0000-0000-000000000000"] {
let mut candidate = snapshot();
candidate
.buckets_usage
.get_mut("bucket")
.expect("canonical")
.bucket_incarnation = Some(current.to_string());
assert_eq!(
candidate.retain_current_bucket_usage_for_admin(&HashMap::from([("bucket".to_string(), current.to_string())])),
1
);
assert!(candidate.usage_snapshot_admin_projection);
}
let mut candidate = snapshot();
assert_eq!(candidate.retain_current_bucket_usage_for_admin(&HashMap::new()), 1);
assert!(candidate.usage_snapshot_bucket_observations.is_empty());
assert_eq!(candidate.usage_snapshot_converged, Some(false));
}
#[test]
fn current_identity_filter_keeps_healthy_buckets_and_recalculates_partial_totals() {
let mut candidate = snapshot();
candidate.buckets_usage.insert("cold".to_string(), usage(7));
candidate.buckets_count = 2;
candidate.bucket_sizes.insert("cold".to_string(), 7);
candidate.calculate_totals();
candidate.replication_info.insert("target".to_string(), Default::default());
candidate.tier_stats = Some(Default::default());
candidate.unknown_tier_stats = Some(Default::default());
assert_eq!(
candidate.retain_current_bucket_usage_for_admin(&HashMap::from([("cold".to_string(), INCARNATION.to_string())])),
1
);
assert_eq!(candidate.buckets_usage.len(), 1);
assert_eq!(candidate.buckets_usage["cold"].size, 7);
assert_eq!(candidate.bucket_sizes, HashMap::from([("cold".to_string(), 7)]));
assert_eq!(
(
candidate.buckets_count,
candidate.objects_total_size,
candidate.objects_total_count,
candidate.versions_total_count
),
(1, 7, 7, 7)
);
assert!(candidate.replication_info.is_empty());
assert!(candidate.tier_stats.is_none());
assert!(candidate.unknown_tier_stats.is_none());
assert!(candidate.usage_snapshot_admin_projection);
assert_eq!(candidate.usage_snapshot_converged, Some(false));
}
#[test]
fn current_identity_filter_overflow_leaves_totals_unknown() {
let mut candidate = snapshot();
candidate.buckets_usage.insert("large".to_string(), usage(u64::MAX));
candidate.buckets_usage.insert("small".to_string(), usage(1));
candidate.buckets_count = 3;
let current = HashMap::from([
("large".to_string(), INCARNATION.to_string()),
("small".to_string(), INCARNATION.to_string()),
]);
assert_eq!(candidate.retain_current_bucket_usage_for_admin(&current), 3);
assert!(candidate.buckets_usage.is_empty());
assert!(candidate.bucket_sizes.is_empty());
assert_eq!(candidate.usage_snapshot_converged, Some(false));
assert!(candidate.usage_snapshot_admin_projection);
assert!(!candidate.is_complete_bucket_usage_snapshot());
}
+1
View File
@@ -183,6 +183,7 @@ rmp.workspace = true
rmp-serde.workspace = true
tokio-util = { workspace = true, features = ["io", "compat", "rt"] }
tokio-stream = { workspace = true, features = ["sync"] }
hmac = { workspace = true }
sha2 = { workspace = true }
hex-simd = { workspace = true }
tempfile.workspace = true
+2 -2
View File
@@ -454,8 +454,8 @@ pub mod data_usage {
DATA_USAGE_CACHE_NAME, apply_bucket_usage_memory_overlay, compute_bucket_usage,
init_compression_total_memory_from_backend, invalidate_admin_data_usage_snapshot_cache,
invalidate_data_usage_snapshot_cache, live_bucket_usage_computations, load_admin_data_usage_from_backend_cached,
load_bucket_usage_from_backend_cached, load_compression_total_from_memory, load_data_usage_from_backend,
load_data_usage_from_backend_cached, quota_object_size, record_bucket_delete_marker_memory,
load_admin_data_usage_with_memory_overlay, load_bucket_usage_from_backend_cached, load_compression_total_from_memory,
load_data_usage_from_backend, load_data_usage_from_backend_cached, quota_object_size, record_bucket_delete_marker_memory,
record_bucket_object_delete_memory, record_bucket_object_version_write_memory, record_bucket_object_write_memory,
record_bucket_object_write_unknown_previous_memory, record_compression_total_memory,
refresh_bucket_usage_from_object_layer, refresh_versioned_bucket_usage_from_object_layer,
+119 -21
View File
@@ -1463,22 +1463,39 @@ async fn load_admin_data_usage_from_backend(store: Arc<ECStore>) -> Result<DataU
let observed = load_observed_data_usage_snapshot(store.clone()).await.ok().flatten();
let (selected, selected_is_current_format) =
select_admin_data_usage_snapshot(authoritative, source.is_authoritative(), observed);
let mut response = normalize_loaded_data_usage(selected, selected_is_current_format).await.0;
Ok(normalize_loaded_data_usage(selected, selected_is_current_format).await.0)
}
async fn project_admin_data_usage_for_current_buckets(
store: Arc<ECStore>,
mut response: DataUsageInfo,
with_memory_overlay: bool,
) -> DataUsageInfo {
use futures::StreamExt as _;
if !(response.is_complete_bucket_usage_snapshot() || response.is_valid_partial_snapshot()) {
return DataUsageInfo::default();
}
let overlay_authoritative = with_memory_overlay && !runtime_sources::setup_is_dist_erasure().await;
let deadline = tokio::time::Instant::now() + ADMIN_USAGE_INCARNATION_TIMEOUT;
let mut incarnations = HashMap::new();
{
// Display-only identity reads must not serialize every bucket behind a
// slow metadata lock while holding the shared admin refresh permit.
// The common deadline bounds their total added latency; dropping this
// stream cancels all unfinished reads without spawning detached work.
// Own the names and store across awaits so this future remains Send
// when composed into spawned metrics collectors.
let buckets = response
.usage_snapshot_bucket_observations
// Identity reads happen after releasing snapshot and refresh locks,
// including on cache hits. Metadata transaction reads linearize each
// bucket against creation/deletion without relying on peer notices.
// The common deadline cancels unfinished work and bounds added latency.
let mut buckets = response
.buckets_usage
.keys()
.chain(response.usage_snapshot_bucket_observations.keys())
.cloned()
.collect::<Vec<_>>();
.collect::<HashSet<_>>();
if overlay_authoritative {
// Copy only names while holding the memory lock, then release it
// before storage I/O. A later new entry has no identity proof and
// will remain unknown in this response.
buckets.extend(memory_cache().read().await.keys().cloned());
}
let mut identities = futures::stream::iter(buckets)
.map(move |bucket| {
let store = store.clone();
@@ -1492,7 +1509,6 @@ async fn load_admin_data_usage_from_backend(store: Arc<ECStore>) -> Result<DataU
let Ok(Some((bucket, identity))) = tokio::time::timeout_at(deadline, identities.next()).await else {
break;
};
// Unavailable, timed-out or recreated buckets retain canonical values.
if let Ok(incarnation) = identity
&& !incarnation.is_nil()
{
@@ -1501,7 +1517,17 @@ async fn load_admin_data_usage_from_backend(store: Arc<ECStore>) -> Result<DataU
}
}
response.project_bucket_observations_for_admin(&incarnations);
Ok(response)
if apply_bucket_usage_memory_overlay_if_authoritative(&mut response, overlay_authoritative).await {
response.usage_snapshot_complete = false;
response.usage_snapshot_partial = true;
response.usage_snapshot_converged = Some(false);
response.usage_snapshot_admin_projection = true;
response.replication_info.clear();
response.tier_stats = None;
response.unknown_tier_stats = None;
}
response.retain_current_bucket_usage_for_admin(&incarnations);
response
}
fn discard_incomplete_bucket_usage(data_usage_info: &mut DataUsageInfo) {
@@ -1690,11 +1716,7 @@ async fn project_data_usage_from_backend_cached<T>(
}
}
/// Load the freshest structurally complete snapshot for authenticated admin
/// observability. A scan raced by namespace activity may be selected here, but
/// never by [`load_data_usage_from_backend_cached`], which remains the
/// converged source for quota admission.
pub async fn load_admin_data_usage_from_backend_cached(store: Arc<ECStore>) -> Result<DataUsageInfo, Error> {
async fn load_admin_data_usage_snapshot_cached(store: Arc<ECStore>) -> Result<DataUsageInfo, Error> {
let ttl = Duration::from_secs(DATA_USAGE_CACHE_TTL_SECS);
loop {
@@ -1743,6 +1765,38 @@ pub async fn load_admin_data_usage_from_backend_cached(store: Arc<ECStore>) -> R
}
}
/// Load cached scanner statistics for authenticated admin observability.
///
/// Each response verifies bucket identities against current storage metadata,
/// including cache hits. Missing, failed, or timed-out identity reads leave that
/// bucket's usage absent (unknown). Legacy values without an incarnation stay
/// unknown until a scanner snapshot certifies their identity. Per-bucket reads
/// linearize independently; this is not an atomic namespace snapshot.
/// Identity reads have a shared deadline and do not outlive cancellation. The
/// unchanged cached baseline remains independent of quota and persistence.
///
/// # Errors
/// Returns the cached snapshot loading error when no baseline can be loaded.
pub async fn load_admin_data_usage_from_backend_cached(store: Arc<ECStore>) -> Result<DataUsageInfo, Error> {
let response = load_admin_data_usage_snapshot_cached(store.clone()).await?;
Ok(project_admin_data_usage_for_current_buckets(store, response, false).await)
}
/// Load the admin view with the existing authoritative single-process overlay.
///
/// Only a complete or valid partial scanner snapshot can receive memory values.
/// The final identity check excludes statistics from retired or unverified buckets.
/// Distributed nodes continue to use scanner statistics without local totals.
/// Cancellation, unknown values, and per-bucket read semantics are the same as
/// [`load_admin_data_usage_from_backend_cached`]. No overlay is persisted.
///
/// # Errors
/// Returns the cached snapshot loading error when no baseline can be loaded.
pub async fn load_admin_data_usage_with_memory_overlay(store: Arc<ECStore>) -> Result<DataUsageInfo, Error> {
let response = load_admin_data_usage_snapshot_cached(store.clone()).await?;
Ok(project_admin_data_usage_for_current_buckets(store, response, true).await)
}
/// Invalidate the process-local persisted usage snapshot cache after a durable
/// scanner write.
pub async fn invalidate_data_usage_snapshot_cache() {
@@ -2805,16 +2859,17 @@ async fn replace_bucket_usage_memory_from_info_if_generation(data_usage_info: &D
*cache = next_cache;
}
async fn apply_bucket_usage_memory_overlay_if_authoritative(data_usage_info: &mut DataUsageInfo, authoritative: bool) {
async fn apply_bucket_usage_memory_overlay_if_authoritative(data_usage_info: &mut DataUsageInfo, authoritative: bool) -> bool {
if !authoritative {
return;
return false;
}
let cache = memory_cache().read().await;
if cache.is_empty() {
return;
return false;
}
let was_complete = data_usage_info.usage_snapshot_complete;
let persisted_update = data_usage_info.last_update;
let mut changed = false;
let mut added_bucket = false;
@@ -2851,6 +2906,7 @@ async fn apply_bucket_usage_memory_overlay_if_authoritative(data_usage_info: &mu
data_usage_info.buckets_count = data_usage_info.buckets_usage.len() as u64;
data_usage_info.calculate_totals();
}
changed || was_complete != data_usage_info.usage_snapshot_complete
}
pub async fn apply_bucket_usage_memory_overlay(data_usage_info: &mut DataUsageInfo) {
@@ -3744,6 +3800,23 @@ mod tests {
*cache_updating().write().await = false;
}
async fn bind_admin_usage_test_identities(store: &Arc<ECStore>, snapshot: &mut DataUsageInfo) {
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
for (bucket, usage) in &mut snapshot.buckets_usage {
store
.make_bucket(bucket, &crate::storage_api_contracts::bucket::MakeBucketOptions::default())
.await
.expect("create a real bucket for current admin identity validation");
usage.bucket_incarnation = Some(
store
.bucket_incarnation_id_from_disk(bucket)
.await
.expect("read the storage-owned fixture identity")
.to_string(),
);
}
}
fn data_usage_info_for_test(bucket: &str, objects_count: u64, size: u64, last_update: SystemTime) -> DataUsageInfo {
let mut info = DataUsageInfo {
last_update: Some(last_update),
@@ -4279,7 +4352,11 @@ mod tests {
use crate::config::com::save_config;
let (_dirs, store) = isolated_store_over_temp_disks().await;
let (authoritative, observed) = usage_snapshots_with_missing_observed_set();
let (mut authoritative, mut observed) = usage_snapshots_with_missing_observed_set();
bind_admin_usage_test_identities(&store, &mut authoritative).await;
for (bucket, usage) in &mut observed.buckets_usage {
usage.bucket_incarnation = authoritative.buckets_usage[bucket].bucket_incarnation.clone();
}
let primary_bytes = serde_json::to_vec(&authoritative).expect("encode complete usage");
let observed_bytes = serde_json::to_vec(&observed).expect("encode partial observation");
save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), primary_bytes.clone())
@@ -4328,6 +4405,11 @@ mod tests {
reconciled.scanner_epoch = Some(4);
reconciled.scanner_cycle = Some(12);
reconciled.usage_snapshot_converged = Some(true);
reconciled
.buckets_usage
.get_mut("large-bucket")
.expect("reconciled bucket")
.bucket_incarnation = authoritative.buckets_usage["large-bucket"].bucket_incarnation.clone();
save_config(
store.clone(),
DATA_USAGE_OBJ_NAME_PATH.as_str(),
@@ -4849,6 +4931,12 @@ mod tests {
backup.scanner_epoch = Some(4);
backup.scanner_cycle = Some(10);
backup.usage_snapshot_converged = Some(true);
bind_admin_usage_test_identities(&store, &mut current).await;
backup
.buckets_usage
.get_mut("existing")
.expect("backup bucket")
.bucket_incarnation = current.buckets_usage["existing"].bucket_incarnation.clone();
for (path, info) in [
(DATA_USAGE_OBJ_NAME_PATH.as_str(), &current),
(DATA_USAGE_OBJ_BACKUP_PATH.as_str(), &backup),
@@ -4871,6 +4959,11 @@ mod tests {
after_delete.scanner_epoch = current.scanner_epoch;
after_delete.scanner_cycle = current.scanner_cycle;
after_delete.usage_snapshot_converged = Some(true);
after_delete
.buckets_usage
.get_mut("existing")
.expect("post-delete bucket")
.bucket_incarnation = current.buckets_usage["existing"].bucket_incarnation.clone();
let mut stale_observation =
data_usage_info_for_test("retired-incarnation", 99, 999, SystemTime::UNIX_EPOCH + Duration::from_secs(4));
stale_observation.scanner_epoch = Some(4);
@@ -4932,6 +5025,7 @@ mod tests {
authoritative.scanner_epoch = Some(7);
authoritative.scanner_cycle = Some(10);
authoritative.usage_snapshot_converged = Some(true);
bind_admin_usage_test_identities(&store, &mut authoritative).await;
save_config(
store.clone(),
DATA_USAGE_OBJ_NAME_PATH.as_str(),
@@ -4946,6 +5040,10 @@ mod tests {
info.scanner_cycle = Some(cycle);
info.usage_snapshot_converged = Some(false);
info.usage_snapshot_authoritative_baseline = Some(authoritative.snapshot_identity());
info.buckets_usage
.get_mut("bucket")
.expect("observed bucket")
.bucket_incarnation = authoritative.buckets_usage["bucket"].bucket_incarnation.clone();
info
};
let first = observation(7, 11, 200, 11, 1_100);
@@ -221,7 +221,7 @@ async fn companion_admin_cache_isolated_from_quota_memory_and_persistence() {
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[serial]
async fn blocked_companion_identity_keeps_canonical_without_delaying_healthy_buckets() {
async fn blocked_companion_identity_stays_unknown_without_delaying_healthy_buckets() {
use crate::bucket::metadata_sys::{
acquire_metadata_transaction_guard, init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks,
};
@@ -280,8 +280,11 @@ async fn blocked_companion_identity_keeps_canonical_without_delaying_healthy_buc
.await
.expect("optional projection must return before the blocked metadata lock's long timeout")
.expect("canonical admin response stays available");
assert_eq!(response.buckets_usage[blocked].size, 100);
assert_eq!(response.buckets_usage[blocked].usage_value_state, Some(BucketUsageValueState::Carried));
assert!(
!response.buckets_usage.contains_key(blocked),
"an unavailable current identity cannot bind the canonical value"
);
assert!(!response.bucket_sizes.contains_key(blocked));
assert_eq!(response.buckets_usage[healthy].size, 250);
assert_eq!(response.buckets_usage[healthy].usage_value_state, Some(BucketUsageValueState::Observed));
assert!(response.usage_snapshot_admin_projection);
@@ -293,13 +296,12 @@ async fn blocked_companion_identity_keeps_canonical_without_delaying_healthy_buc
.is_ok()
);
drop(blocked_guard);
invalidate_data_usage_snapshot_cache().await;
let recovered = load_admin_data_usage_from_backend_cached(store.clone())
.await
.expect("refresh after lock recovery");
assert_eq!(
recovered.buckets_usage[blocked].size, 250,
"unavailable identity can recover on the next refresh"
"unavailable identity can recover on a cache hit"
);
assert_eq!(recovered.buckets_usage[healthy].size, 250);
let quota = load_data_usage_from_backend_cached(store)
@@ -308,3 +310,331 @@ async fn blocked_companion_identity_keeps_canonical_without_delaying_healthy_buc
assert_eq!(quota.buckets_usage[blocked].size, 100);
assert_eq!(quota.buckets_usage[healthy].size, 100);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[serial]
async fn cached_canonical_admin_usage_rechecks_current_identity_without_peer_invalidation() {
use crate::bucket::metadata::{load_bucket_incarnation, save_bucket_incarnation};
use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks};
use crate::config::com::save_config;
use crate::storage_api_contracts::bucket::MakeBucketOptions;
let (_dirs, store) = isolated_store_over_temp_disks().await;
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let bucket = "usage-recreated-cache";
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("create identity fixture");
let old_identity = store
.bucket_incarnation_id_from_disk(bucket)
.await
.expect("original identity");
let mut canonical = companion_snapshot(bucket, old_identity);
canonical.usage_snapshot_bucket_observations.clear();
canonical.usage_snapshot_converged = Some(true);
canonical.usage_snapshot_pending_buckets.clear();
canonical.usage_snapshot_pending_bucket_count = 0;
canonical
.buckets_usage
.get_mut(bucket)
.expect("canonical bucket")
.objects_count = 107;
canonical.calculate_totals();
let original_bytes = serde_json::to_vec(&canonical).expect("encode canonical old bucket");
save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), original_bytes.clone())
.await
.expect("persist canonical root");
invalidate_data_usage_snapshot_cache().await;
let first = load_admin_data_usage_from_backend_cached(store.clone())
.await
.expect("prime raw admin cache");
assert_eq!(first.buckets_usage[bucket].objects_count, 107);
assert!(first.is_complete_bucket_usage_snapshot());
let cache_generation = admin_data_usage_snapshot_generation();
let new_identity = uuid::Uuid::new_v4();
// Simulate the storage-owned identity changing on another node while this
// process has not received its invalidation notice. Do not clear the cache.
save_bucket_incarnation(store.clone(), bucket, new_identity)
.await
.expect("publish changed disk identity");
assert_eq!(
load_bucket_incarnation(store.clone(), bucket)
.await
.expect("read new sidecar"),
Some(new_identity)
);
let second = load_admin_data_usage_from_backend_cached(store.clone())
.await
.expect("read raw cache with fresh identity");
assert_eq!(admin_data_usage_snapshot_generation(), cache_generation);
assert!(
!second.buckets_usage.contains_key(bucket),
"same name cannot retain the predecessor's 107"
);
assert!(!second.bucket_sizes.contains_key(bucket));
assert_eq!(second.usage_snapshot_converged, Some(false));
assert!(second.usage_snapshot_admin_projection);
assert!(reject_admin_usage_projection(&second).is_err());
assert!(certified_bucket_usage(&second, bucket, new_identity).is_none());
assert!(parse_usage_snapshot(&serde_json::to_vec(&second).expect("encode display response")).is_err());
let quota = load_data_usage_from_backend_cached(store.clone())
.await
.expect("independent quota baseline");
assert_eq!(quota.buckets_usage[bucket].objects_count, 107);
assert_eq!(
read_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
.await
.expect("unchanged persisted root"),
original_bytes
);
// Rebinding a cached raw baseline is transient: recovery does not need to
// reload or permanently blank a healthy bucket's statistics.
save_bucket_incarnation(store.clone(), bucket, old_identity)
.await
.expect("restore fixture identity");
let recovered = load_admin_data_usage_from_backend_cached(store)
.await
.expect("recover on another cache hit");
assert_eq!(recovered.buckets_usage[bucket].objects_count, 107);
assert!(recovered.is_complete_bucket_usage_snapshot());
assert_eq!(admin_data_usage_snapshot_generation(), cache_generation);
invalidate_data_usage_snapshot_cache().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[serial]
async fn admin_memory_overlay_requires_a_measured_scanner_baseline() {
use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks};
use crate::config::com::save_config;
use crate::storage_api_contracts::bucket::MakeBucketOptions;
let (_dirs, store) = isolated_store_over_temp_disks().await;
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
assert!(!runtime_sources::setup_is_dist_erasure().await);
let bucket = "usage-unmeasured-overlay";
store.make_bucket(bucket, &MakeBucketOptions::default()).await.unwrap();
let incarnation = store.bucket_incarnation_id_from_disk(bucket).await.unwrap();
let mut baseline = data_usage_info_for_test(bucket, 1, 100, SystemTime::UNIX_EPOCH);
let usage = baseline.buckets_usage.get_mut(bucket).unwrap();
usage.bucket_incarnation = Some(incarnation.to_string());
let memory_usage = BucketUsageInfo {
size: 150,
..usage.clone()
};
clear_usage_memory_cache_for_test().await;
replace_bucket_usage_memory_from_authoritative(bucket, memory_usage, SystemTime::now()).await;
// A missing snapshot and a mismatched bucket count both leave usage unknown.
for (buckets_count, expected_size) in [(None, None), (Some(2), None), (Some(1), Some(150))] {
if let Some(count) = buckets_count {
baseline.buckets_count = count;
save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), serde_json::to_vec(&baseline).unwrap())
.await
.unwrap();
}
invalidate_data_usage_snapshot_cache().await;
let response = load_admin_data_usage_with_memory_overlay(store.clone()).await.unwrap();
assert_eq!(
response.buckets_usage.get(bucket).map(|usage| usage.size),
expected_size,
"baseline count {buckets_count:?}"
);
assert_eq!(response.bucket_sizes.get(bucket).copied(), expected_size);
assert_eq!(response.objects_total_size, expected_size.unwrap_or_default());
assert_eq!(response.usage_snapshot_admin_projection, expected_size.is_some());
assert!(!response.is_complete_bucket_usage_snapshot());
assert!(!response.is_valid_partial_snapshot());
}
clear_usage_memory_cache_for_test().await;
invalidate_data_usage_snapshot_cache().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[serial]
async fn admin_identity_filter_does_not_validate_an_unmeasured_partial_snapshot() {
use crate::bucket::metadata_sys::test_support::isolated_store_over_temp_disks;
use crate::config::com::save_config;
let (_dirs, store) = isolated_store_over_temp_disks().await;
let current = "usage-partial-current";
let stale = "usage-partial-stale";
let mut baseline = data_usage_info_for_test(current, 5, 100, SystemTime::UNIX_EPOCH);
baseline.buckets_usage.insert(
stale.to_owned(),
BucketUsageInfo {
size: 200,
objects_count: 10,
..Default::default()
},
);
baseline.bucket_sizes.insert(stale.to_owned(), 200);
baseline.buckets_count = 2;
baseline.calculate_totals();
baseline.usage_snapshot_complete = false;
baseline.usage_snapshot_partial = true;
baseline.usage_snapshot_converged = Some(false);
baseline.scanner_epoch = Some(2);
baseline.scanner_cycle = Some(9);
baseline.usage_snapshot_set_states = vec![DataUsageSnapshotSetState {
scanner_epoch: baseline.scanner_epoch,
scanner_cycle: baseline.scanner_cycle,
scan_plan_digest: Some([1; 32]),
..Default::default()
}];
bind_admin_usage_test_identities(&store, &mut baseline).await;
let current_incarnation = baseline.buckets_usage[current].bucket_incarnation.clone();
assert_ne!(current_incarnation, baseline.buckets_usage[stale].bucket_incarnation);
baseline
.buckets_usage
.get_mut(stale)
.expect("stale bucket fixture")
.bucket_incarnation = current_incarnation;
for (case, snapshot, expected_size) in [
(
"missing set provenance",
DataUsageInfo {
usage_snapshot_set_states: Vec::new(),
..baseline.clone()
},
None,
),
(
"mismatched bucket count",
DataUsageInfo {
buckets_count: 3,
..baseline.clone()
},
None,
),
("valid partial snapshot", baseline, Some(100)),
] {
save_config(
store.clone(),
DATA_USAGE_OBJ_NAME_PATH.as_str(),
serde_json::to_vec(&snapshot).expect("encode partial snapshot"),
)
.await
.expect("persist partial snapshot");
invalidate_admin_data_usage_snapshot_cache().await;
let response = load_admin_data_usage_from_backend_cached(store.clone())
.await
.expect("load admin usage through the identity filter");
assert_eq!(response.buckets_usage.get(current).map(|usage| usage.size), expected_size, "{case}");
assert_eq!(response.bucket_sizes.get(current).copied(), expected_size, "{case}");
assert!(!response.buckets_usage.contains_key(stale), "{case}");
assert!(!response.bucket_sizes.contains_key(stale), "{case}");
assert_eq!(response.objects_total_size, expected_size.unwrap_or_default(), "{case}");
assert_eq!(response.usage_snapshot_admin_projection, expected_size.is_some(), "{case}");
}
invalidate_admin_data_usage_snapshot_cache().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[serial]
async fn admin_memory_overlay_cannot_reintroduce_a_predecessor_identity() {
use crate::bucket::metadata::save_bucket_incarnation;
use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks};
use crate::config::com::save_config;
use crate::storage_api_contracts::bucket::MakeBucketOptions;
let (_dirs, store) = isolated_store_over_temp_disks().await;
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let bucket = "usage-recreated-overlay";
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("create overlay identity fixture");
let old_identity = store
.bucket_incarnation_id_from_disk(bucket)
.await
.expect("original identity");
let mut canonical = companion_snapshot(bucket, old_identity);
canonical.usage_snapshot_bucket_observations.clear();
save_config(
store.clone(),
DATA_USAGE_OBJ_NAME_PATH.as_str(),
serde_json::to_vec(&canonical).expect("encode canonical root"),
)
.await
.expect("persist overlay fixture");
invalidate_data_usage_snapshot_cache().await;
let cached_usage = canonical.buckets_usage[bucket].clone();
replace_bucket_usage_memory_from_authoritative(bucket, cached_usage, SystemTime::now()).await;
let new_identity = uuid::Uuid::new_v4();
save_bucket_incarnation(store.clone(), bucket, new_identity)
.await
.expect("publish a replacement identity");
let response = load_admin_data_usage_with_memory_overlay(store.clone())
.await
.expect("project after memory overlay");
assert!(!response.buckets_usage.contains_key(bucket));
assert_eq!(response.usage_snapshot_converged, Some(false));
assert!(response.usage_snapshot_admin_projection);
clear_usage_memory_cache_for_test().await;
invalidate_data_usage_snapshot_cache().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[serial]
async fn admin_overlay_flags_only_change_keeps_identity_bound_cold_usage() {
use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks};
use crate::config::com::save_config;
use crate::storage_api_contracts::bucket::MakeBucketOptions;
let (_dirs, store) = isolated_store_over_temp_disks().await;
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
assert!(
!runtime_sources::setup_is_dist_erasure().await,
"exercise the production single-process overlay"
);
let cold = "usage-overlay-cold";
let unverified = "usage-overlay-unverified";
for bucket in [cold, unverified] {
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("create overlay fixture bucket");
}
let identity = store
.bucket_incarnation_id_from_disk(cold)
.await
.expect("cold bucket identity");
let mut canonical = companion_snapshot(cold, identity);
canonical.usage_snapshot_bucket_observations.clear();
canonical.usage_snapshot_converged = Some(true);
save_config(
store.clone(),
DATA_USAGE_OBJ_NAME_PATH.as_str(),
serde_json::to_vec(&canonical).expect("encode cold canonical value"),
)
.await
.expect("persist cold baseline");
clear_usage_memory_cache_for_test().await;
invalidate_data_usage_snapshot_cache().await;
memory_cache().write().await.insert(
unverified.to_string(),
cached_bucket_usage_from_backend(BucketUsageInfo::default(), SystemTime::UNIX_EPOCH, false),
);
record_bucket_object_write_memory(unverified, None, 42).await;
{
let cache = memory_cache().read().await;
assert!(cache[unverified].dirty);
assert!(!cache[unverified].authoritative);
}
let response = load_admin_data_usage_with_memory_overlay(store)
.await
.expect("identity-filtered flags-only overlay");
assert_eq!(response.buckets_usage[cold].size, 100);
assert_eq!(response.objects_total_size, 100);
assert_eq!(response.bucket_sizes[cold], 100);
assert!(!response.buckets_usage.contains_key(unverified));
assert!(response.usage_snapshot_admin_projection);
assert_eq!(response.usage_snapshot_converged, Some(false));
assert!(!response.is_complete_bucket_usage_snapshot());
assert!(!response.is_valid_partial_snapshot());
clear_usage_memory_cache_for_test().await;
invalidate_data_usage_snapshot_cache().await;
}
@@ -1033,16 +1033,26 @@ mod tests {
#[tokio::test]
#[serial]
async fn server_info_reads_complete_observation_without_promoting_authority() {
use crate::bucket::metadata_sys::test_support::isolated_store_over_temp_disks;
use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks};
use crate::config::com::{read_config, save_config};
use crate::data_usage::{
DATA_USAGE_OBJ_NAME_PATH, DATA_USAGE_OBSERVED_OBJ_NAME_PATH, invalidate_data_usage_snapshot_cache,
load_data_usage_from_backend_cached,
};
use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions};
use rustfs_data_usage::{BucketUsageInfo, DataUsageInfo};
use std::time::{Duration, SystemTime};
let (_dirs, store) = isolated_store_over_temp_disks().await;
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
store
.make_bucket("bucket", &MakeBucketOptions::default())
.await
.expect("create a real observed bucket");
let incarnation = store
.bucket_incarnation_id_from_disk("bucket")
.await
.expect("read current storage-owned incarnation");
let primary = DataUsageInfo {
last_update: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(1)),
scanner_epoch: Some(1),
@@ -1066,6 +1076,7 @@ mod tests {
versions_count: 4,
delete_markers_count: 1,
size: 6,
bucket_incarnation: Some(incarnation.to_string()),
..Default::default()
},
)]
@@ -1117,6 +1128,46 @@ mod tests {
invalidate_data_usage_snapshot_cache().await;
}
#[test]
fn identity_filtered_usage_remains_unknown_in_server_info() {
let mut info = rustfs_data_usage::DataUsageInfo {
last_update: Some(std::time::SystemTime::UNIX_EPOCH),
buckets_count: 1,
objects_total_count: 107,
objects_total_size: 1024,
buckets_usage: [(
"bucket".to_string(),
rustfs_data_usage::BucketUsageInfo {
objects_count: 107,
size: 1024,
bucket_incarnation: Some("11111111-1111-4111-8111-111111111111".to_string()),
..Default::default()
},
)]
.into_iter()
.collect(),
usage_snapshot_complete: true,
..Default::default()
};
let current =
std::collections::HashMap::from([("bucket".to_string(), "22222222-2222-4222-8222-222222222222".to_string())]);
assert_eq!(info.retain_current_bucket_usage_for_admin(&current), 1);
assert!(info.buckets_usage.is_empty());
assert!(info.usage_snapshot_admin_projection);
assert_eq!(info.usage_snapshot_converged, Some(false));
let mut buckets = rustfs_madmin::Buckets::default();
let mut objects = rustfs_madmin::Objects::default();
let mut versions = rustfs_madmin::Versions::default();
let mut delete_markers = rustfs_madmin::DeleteMarkers::default();
let mut usage = rustfs_madmin::Usage::default();
apply_data_usage_result(Ok(info), &mut buckets, &mut objects, &mut versions, &mut delete_markers, &mut usage);
assert_eq!(buckets.error.as_deref(), Some(DATA_USAGE_UNAVAILABLE_ERROR));
assert_eq!(objects.error.as_deref(), Some(DATA_USAGE_UNAVAILABLE_ERROR));
assert_eq!(versions.error.as_deref(), Some(DATA_USAGE_UNAVAILABLE_ERROR));
assert_eq!(delete_markers.error.as_deref(), Some(DATA_USAGE_UNAVAILABLE_ERROR));
assert_eq!(usage.error.as_deref(), Some(DATA_USAGE_UNAVAILABLE_ERROR));
}
#[test]
fn incomplete_data_usage_is_unavailable_in_server_info() {
let mut buckets = rustfs_madmin::Buckets::default();
+4 -3
View File
@@ -29,11 +29,12 @@ workspace = true
[features]
default = []
hotpath = ["rustfs-io-metrics/hotpath"]
hotpath-alloc = ["hotpath", "rustfs-io-metrics/hotpath-alloc"]
hotpath-cpu = ["hotpath", "rustfs-io-metrics/hotpath-cpu"]
hotpath = ["hotpath/hotpath", "hotpath/tokio", "rustfs-io-metrics/hotpath"]
hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc", "rustfs-io-metrics/hotpath-alloc"]
hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu", "rustfs-io-metrics/hotpath-cpu"]
[dependencies]
hotpath.workspace = true
bytes = { workspace = true, features = ["serde"] }
thiserror = { workspace = true }
tokio = { workspace = true, features = ["io-util", "fs", "sync", "rt-multi-thread"] }
+1 -2
View File
@@ -6,8 +6,7 @@ Applies to `crates/kms/`.
When changing key-management behavior, verify compatibility with:
- `crates/sse/` (SSE key resolution; consumes this crate on the data path)
- `rustfs/src/storage/sse.rs` (binds `crates/sse/` to the running server)
- `rustfs/src/storage/sse.rs`
- `rustfs/src/admin/handlers/kms.rs`
- `rustfs/src/admin/handlers/kms_dynamic.rs`
- `rustfs/src/admin/handlers/kms_keys.rs`
+7 -13
View File
@@ -410,10 +410,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
}
};
let handle = self.allocate_handle(HandleState::Dir(cursor))?;
Ok(Handle {
id,
handle: handle.into(),
})
Ok(Handle { id, handle })
}
/// READDIR body shared with the Handler trait wrapper. Removes the
@@ -669,11 +666,8 @@ mod tests {
assert_eq!(auth.continuation_token, call.continuation_token);
}
assert_eq!(authorized[2].continuation_token.as_ref().map(|value| value.as_str()), Some("next-page"));
let denied = with_test_auth_override(
|_, _, prefix| prefix != Some("private/"),
driver.readdir_inner(2, String::from_utf8(handle.handle.into_bytes()).expect("utf8 handle")),
)
.await;
let denied =
with_test_auth_override(|_, _, prefix| prefix != Some("private/"), driver.readdir_inner(2, handle.handle)).await;
assert!(denied.is_err(), "policy change must deny the next READDIR");
assert_eq!(backend.list_objects_calls().len(), 2, "denied READDIR cannot reach storage");
}
@@ -722,7 +716,7 @@ mod tests {
};
let handle_id = driver.allocate_handle(HandleState::Dir(cursor)).expect("allocate");
let readdir_fut = driver.readdir(1, handle_id.clone().into());
let readdir_fut = driver.readdir(1, handle_id.clone());
with_test_auth_override(|_, _, _| true, async {
tokio::select! {
@@ -764,7 +758,7 @@ mod tests {
// token, no skipped entries).
backend.clear_stall_list_objects_v2();
backend.queue_list_objects_v2_ok_empty();
let result = with_test_auth_override(|_, _, _| true, driver.readdir(2, handle_id.into())).await;
let result = with_test_auth_override(|_, _, _| true, driver.readdir(2, handle_id)).await;
// Empty page returns Eof per readdir's empty-Name-to-Eof translation.
let err = result.expect_err("re-issued READDIR against an empty listing must return Eof, not Ok");
assert!(
@@ -791,7 +785,7 @@ mod tests {
let handle_id = driver.allocate_handle(HandleState::Dir(cursor)).expect("allocate");
let (result, captured) =
capture_tracing_at(Level::ERROR, with_test_auth_override(|_, _, _| true, driver.readdir(7, handle_id.into()))).await;
capture_tracing_at(Level::ERROR, with_test_auth_override(|_, _, _| true, driver.readdir(7, handle_id))).await;
let err = result.expect_err("exhausted cursor must return Eof");
assert!(matches!(StatusCode::from(err), StatusCode::Eof));
assert!(
@@ -824,7 +818,7 @@ mod tests {
let handle_id = driver.allocate_handle(HandleState::Dir(cursor)).expect("allocate");
let (result, captured) =
capture_tracing_at(Level::ERROR, with_test_auth_override(|_, _, _| true, driver.readdir(8, handle_id.into()))).await;
capture_tracing_at(Level::ERROR, with_test_auth_override(|_, _, _| true, driver.readdir(8, handle_id))).await;
let err = result.expect_err("backend error must propagate as Err");
assert!(!matches!(StatusCode::from(err), StatusCode::Eof), "backend error must not be Eof");
assert!(
+17 -30
View File
@@ -33,9 +33,7 @@ use super::write::{build_write_tombstone, fstat_reported_size, rejects_excl_or_t
use crate::common::client::s3::StorageBackend;
use crate::common::gateway::{AuthorizationError, S3Action};
use crate::common::session::SessionContext;
use russh_sftp::protocol::{
Attrs, Data, File, FileAttributes, Handle, Name, OpenFlags, Packet, SftpHandle, Status, StatusCode, Version,
};
use russh_sftp::protocol::{Attrs, Data, File, FileAttributes, Handle, Name, OpenFlags, Packet, Status, StatusCode, Version};
use rustfs_credentials::Credentials;
use rustfs_gateway_types::dto::{AbortMultipartUploadInput, CopyObjectInput, ListObjectsV2Input};
use rustfs_gateway_types::{BucketName, ObjectKey, UploadIdClaim};
@@ -210,12 +208,6 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
}
}
/// russh-sftp 4 hands the Handler opaque `SftpHandle` bytes; the table
/// keys remain the UTF-8 UUID strings this driver mints.
fn handle_table_key(handle: SftpHandle) -> Result<String, SftpError> {
String::from_utf8(handle.into_bytes()).map_err(|_| SftpError::code(StatusCode::Failure))
}
/// Generate a fresh UUID v4 handle, insert the given state into the
/// per-session handle table, and return the handle string. Enforces
/// self.handles_per_session before any UUID generation. Cap-exceeded
@@ -502,9 +494,8 @@ impl<S: StorageBackend + Send + Sync + 'static> russh_sftp::server::Handler for
/// Streaming returns (next_part_number - 1) * part_size + buffer
/// length. Failed returns the size recorded at the most recent
/// successful write. An unknown handle returns Failure.
#[tracing::instrument(level = "debug", skip(self), fields(id, handle = ?handle), err(Debug))]
async fn fstat(&mut self, id: u32, handle: SftpHandle) -> Result<Attrs, Self::Error> {
let handle = Self::handle_table_key(handle)?;
#[tracing::instrument(level = "debug", skip(self), fields(id, handle = %handle), err(Debug))]
async fn fstat(&mut self, id: u32, handle: String) -> Result<Attrs, Self::Error> {
self.session_diag.stamp();
let part_size = self.part_size;
let result = self.with_handle_ref(&handle, |state| match state {
@@ -550,9 +541,8 @@ impl<S: StorageBackend + Send + Sync + 'static> russh_sftp::server::Handler for
/// exhaustion, so it is normal control flow on this handler. The
/// instrument attribute therefore omits err(Debug) and non-Eof
/// failures are surfaced via the explicit error log below.
#[tracing::instrument(level = "debug", skip(self), fields(id, handle = ?handle))]
async fn readdir(&mut self, id: u32, handle: SftpHandle) -> Result<Name, Self::Error> {
let handle = Self::handle_table_key(handle)?;
#[tracing::instrument(level = "debug", skip(self), fields(id, handle = %handle))]
async fn readdir(&mut self, id: u32, handle: String) -> Result<Name, Self::Error> {
self.session_diag.stamp();
let result = self.readdir_inner(id, handle.clone()).await;
self.session_diag.stamp();
@@ -627,9 +617,8 @@ impl<S: StorageBackend + Send + Sync + 'static> russh_sftp::server::Handler for
/// read-past-end-of-file, so it is normal control flow on this
/// handler. The instrument attribute therefore omits err(Debug) and
/// non-Eof failures are surfaced via the explicit error log below.
#[tracing::instrument(level = "debug", skip(self), fields(id, handle = ?handle, offset, len))]
async fn read(&mut self, id: u32, handle: SftpHandle, offset: u64, len: u32) -> Result<Data, Self::Error> {
let handle = Self::handle_table_key(handle)?;
#[tracing::instrument(level = "debug", skip(self), fields(id, handle = %handle, offset, len))]
async fn read(&mut self, id: u32, handle: String, offset: u64, len: u32) -> Result<Data, Self::Error> {
self.session_diag.stamp();
let result = self.read_inner(id, handle.clone(), offset, len).await;
self.session_diag.stamp();
@@ -661,9 +650,8 @@ impl<S: StorageBackend + Send + Sync + 'static> russh_sftp::server::Handler for
///
/// A missing handle is treated as Ok to tolerate clients that
/// double-close on session teardown.
#[tracing::instrument(level = "info", skip(self), fields(id, handle = ?handle), err(Debug))]
async fn close(&mut self, id: u32, handle: SftpHandle) -> Result<Status, Self::Error> {
let handle = Self::handle_table_key(handle)?;
#[tracing::instrument(level = "info", skip(self), fields(id, handle = %handle), err(Debug))]
async fn close(&mut self, id: u32, handle: String) -> Result<Status, Self::Error> {
self.session_diag.stamp();
let removed = self.handles.remove(&handle);
let Some(HandleState::Write {
@@ -767,9 +755,8 @@ impl<S: StorageBackend + Send + Sync + 'static> russh_sftp::server::Handler for
/// to Streaming by issuing CreateMultipartUpload. A Failed handle
/// rejects every subsequent write with the same status that caused
/// the failure.
#[tracing::instrument(level = "debug", skip(self, data), fields(id, handle = ?handle, offset, len = data.len()), err(Debug))]
async fn write(&mut self, id: u32, handle: SftpHandle, offset: u64, data: Vec<u8>) -> Result<Status, Self::Error> {
let handle = Self::handle_table_key(handle)?;
#[tracing::instrument(level = "debug", skip(self, data), fields(id, handle = %handle, offset, len = data.len()), err(Debug))]
async fn write(&mut self, id: u32, handle: String, offset: u64, data: Vec<u8>) -> Result<Status, Self::Error> {
self.session_diag.stamp();
self.enforce_server_readonly()?;
@@ -996,8 +983,8 @@ impl<S: StorageBackend + Send + Sync + 'static> russh_sftp::server::Handler for
/// SSH_FXP_FSETSTAT, SFTP Internet Draft section 6.6. Same rationale
/// as setstat: S3 cannot honour POSIX attributes, and clients use
/// FSETSTAT during transfers to stamp the in-flight handle.
#[tracing::instrument(level = "debug", skip(self, _attrs), fields(id, handle = ?_handle), err(Debug))]
async fn fsetstat(&mut self, id: u32, _handle: SftpHandle, _attrs: FileAttributes) -> Result<Status, Self::Error> {
#[tracing::instrument(level = "debug", skip(self, _attrs), fields(id, handle = %_handle), err(Debug))]
async fn fsetstat(&mut self, id: u32, _handle: String, _attrs: FileAttributes) -> Result<Status, Self::Error> {
self.enforce_server_readonly()?;
Ok(ok_status(id))
}
@@ -1267,7 +1254,7 @@ mod tests {
let handle_id = driver
.allocate_handle(file_handle("bucket", "k", 1234, attrs.clone()))
.expect("allocate");
let out = driver.fstat(4, handle_id.into()).await.expect("fstat on File must succeed");
let out = driver.fstat(4, handle_id).await.expect("fstat on File must succeed");
assert_eq!(out.attrs.size, Some(1234));
assert_eq!(out.attrs.mtime, Some(1_700_000_000));
}
@@ -1280,7 +1267,7 @@ mod tests {
part_buffer: vec![0u8; 4096],
};
let handle_id = driver.allocate_handle(write_handle("bucket", "k", phase)).expect("allocate");
let out = driver.fstat(5, handle_id.into()).await.expect("fstat on Write must succeed");
let out = driver.fstat(5, handle_id).await.expect("fstat on Write must succeed");
assert_eq!(
out.attrs.size,
Some(4096),
@@ -1296,7 +1283,7 @@ mod tests {
.allocate_handle(file_handle("bucket", "k", 0, FileAttributes::default()))
.expect("allocate");
let status = driver
.fsetstat(6, handle_id.into(), FileAttributes::default())
.fsetstat(6, handle_id, FileAttributes::default())
.await
.expect("fsetstat must succeed on any attrs");
assert!(matches!(status.status_code, StatusCode::Ok));
@@ -1430,7 +1417,7 @@ mod tests {
let handle_id = driver
.allocate_handle(file_handle("bucket", "k", 0, FileAttributes::default()))
.expect("allocate");
let result = driver.fsetstat(10, handle_id.into(), FileAttributes::default()).await;
let result = driver.fsetstat(10, handle_id, FileAttributes::default()).await;
match result {
Err(err) => assert!(matches!(err.0, StatusCode::PermissionDenied)),
Ok(_) => panic!("fsetstat must error in read-only mode"),
+19 -26
View File
@@ -69,10 +69,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
attrs,
read_cache,
})?;
Ok(Handle {
id,
handle: handle.into(),
})
Ok(Handle { id, handle })
}
/// Body of the SSH_FXP_READ handler. Returns up to len bytes starting
@@ -259,7 +256,7 @@ mod tests {
.allocate_handle(file_handle("bucket", "secret.txt", 4096, FileAttributes::default()))
.expect("allocate read handle");
driver.try_populate_read_cache(&handle, 0, vec![42; 4096]);
let error = with_test_auth_override(|_, _, _| true, driver.read(1, handle.into(), 0, 1024))
let error = with_test_auth_override(|_, _, _| true, driver.read(1, handle, 0, 1024))
.await
.expect_err("backend policy denial must prevent cached content disclosure");
assert!(matches!(StatusCode::from(error), StatusCode::PermissionDenied));
@@ -273,7 +270,7 @@ mod tests {
.allocate_handle(file_handle("bucket", "k", 100, FileAttributes::default()))
.expect("allocate");
let err = driver
.read(1, handle_id.into(), 0, 0)
.read(1, handle_id, 0, 0)
.await
.expect_err("len=0 must return BadMessage");
assert!(matches!(StatusCode::from(err), StatusCode::BadMessage));
@@ -287,7 +284,7 @@ mod tests {
.allocate_handle(file_handle("bucket", "k", 10, FileAttributes::default()))
.expect("allocate");
let err = driver
.read(2, handle_id.into(), 10, 4)
.read(2, handle_id, 10, 4)
.await
.expect_err("offset==size must return Eof");
assert!(matches!(StatusCode::from(err), StatusCode::Eof));
@@ -302,7 +299,7 @@ mod tests {
.allocate_handle(file_handle("bucket", "k", 5, FileAttributes::default()))
.expect("allocate");
let data = with_test_auth_override(|_, _, _| true, driver.read(3, handle_id.into(), 0, 1024))
let data = with_test_auth_override(|_, _, _| true, driver.read(3, handle_id, 0, 1024))
.await
.expect("read must succeed");
assert_eq!(data.data, b"hello".to_vec());
@@ -320,7 +317,7 @@ mod tests {
.allocate_handle(file_handle("bucket", "k", 10, FileAttributes::default()))
.expect("allocate");
let (result, captured) = capture_tracing_at(Level::ERROR, async { driver.read(11, handle_id.into(), 10, 4).await }).await;
let (result, captured) = capture_tracing_at(Level::ERROR, async { driver.read(11, handle_id, 10, 4).await }).await;
let err = result.expect_err("offset==size must return Eof");
assert!(matches!(StatusCode::from(err), StatusCode::Eof));
assert!(
@@ -345,11 +342,8 @@ mod tests {
.allocate_handle(file_handle("bucket", "k", 1024, FileAttributes::default()))
.expect("allocate");
let (result, captured) = capture_tracing_at(
Level::ERROR,
with_test_auth_override(|_, _, _| true, driver.read(12, handle_id.into(), 0, 256)),
)
.await;
let (result, captured) =
capture_tracing_at(Level::ERROR, with_test_auth_override(|_, _, _| true, driver.read(12, handle_id, 0, 256))).await;
let err = result.expect_err("backend error must propagate as Err");
assert!(!matches!(StatusCode::from(err), StatusCode::Eof), "backend error must not be Eof");
assert!(
@@ -385,7 +379,7 @@ mod tests {
let start = Instant::now();
let outcome = tokio::time::timeout(
Duration::from_secs(10),
with_test_auth_override(|_, _, _| true, driver.read(14, handle_id.into(), 0, 4096)),
with_test_auth_override(|_, _, _| true, driver.read(14, handle_id, 0, 4096)),
)
.await;
let elapsed = start.elapsed();
@@ -431,7 +425,7 @@ mod tests {
let mut assembled: Vec<u8> = Vec::with_capacity(object_size as usize);
let mut reads: u32 = 0;
while offset < object_size {
let data = with_test_auth_override(|_, _, _| true, driver.read(20 + reads, handle_id.clone().into(), offset, chunk))
let data = with_test_auth_override(|_, _, _| true, driver.read(20 + reads, handle_id.clone(), offset, chunk))
.await
.expect("read inside the cached window must succeed without a second backend call");
assert!(!data.data.is_empty(), "non-empty hit");
@@ -467,21 +461,21 @@ mod tests {
.expect("allocate");
// First read fetches from the backend and populates window 1.
let r1 = with_test_auth_override(|_, _, _| true, driver.read(30, handle_id.clone().into(), 0, 1024))
let r1 = with_test_auth_override(|_, _, _| true, driver.read(30, handle_id.clone(), 0, 1024))
.await
.expect("first read must succeed");
assert!(r1.data.iter().all(|b| *b == 0xAA), "first read must come from window 1");
// Second read inside the cached chunk is served from the
// buffer. No second backend call yet.
let r2 = with_test_auth_override(|_, _, _| true, driver.read(31, handle_id.clone().into(), 1024, 1024))
let r2 = with_test_auth_override(|_, _, _| true, driver.read(31, handle_id.clone(), 1024, 1024))
.await
.expect("mid-window read must succeed from cache");
assert!(r2.data.iter().all(|b| *b == 0xAA), "mid-window read still in window 1");
// Reading at offset == window falls outside the cached chunk
// and triggers the second backend call.
let r3 = with_test_auth_override(|_, _, _| true, driver.read(32, handle_id.clone().into(), window, 1024))
let r3 = with_test_auth_override(|_, _, _| true, driver.read(32, handle_id.clone(), window, 1024))
.await
.expect("read at offset=window must succeed via second backend call");
assert!(r3.data.iter().all(|b| *b == 0xBB), "read at window boundary must come from window 2");
@@ -489,7 +483,7 @@ mod tests {
// A read inside the second cached chunk is served from the
// buffer. The queue is empty by now, so any third backend
// call would fail.
let r4 = with_test_auth_override(|_, _, _| true, driver.read(33, handle_id.into(), window + 1024, 1024))
let r4 = with_test_auth_override(|_, _, _| true, driver.read(33, handle_id, window + 1024, 1024))
.await
.expect("mid-window-2 read must succeed from cache");
assert!(r4.data.iter().all(|b| *b == 0xBB), "mid-window-2 read still in window 2");
@@ -517,13 +511,13 @@ mod tests {
.expect("allocate");
// Populate window 1 with a full read.
let _ = with_test_auth_override(|_, _, _| true, driver.read(40, handle_id.clone().into(), 0, window as u32))
let _ = with_test_auth_override(|_, _, _| true, driver.read(40, handle_id.clone(), 0, window as u32))
.await
.expect("populate window 1");
// Ask for 256 bytes starting 64 bytes before window end. Only
// 64 bytes are in the window. The driver must return 64.
let edge = with_test_auth_override(|_, _, _| true, driver.read(41, handle_id.into(), window - 64, 256))
let edge = with_test_auth_override(|_, _, _| true, driver.read(41, handle_id, window - 64, 256))
.await
.expect("partial-hit read must succeed");
assert_eq!(edge.data.len(), 64, "partial hit must return only the in-window portion");
@@ -558,10 +552,9 @@ mod tests {
for i in 0..read_count {
let offset = (chunk_size as u64) * (i as u64);
let data =
with_test_auth_override(|_, _, _| true, driver.read(50 + i, handle_id.clone().into(), offset, chunk_size as u32))
.await
.expect("each read must succeed via the backend");
let data = with_test_auth_override(|_, _, _| true, driver.read(50 + i, handle_id.clone(), offset, chunk_size as u32))
.await
.expect("each read must succeed via the backend");
assert_eq!(data.data.len(), chunk_size, "read must return full requested length");
let expected_byte = (i + 1) as u8;
assert!(
+4 -7
View File
@@ -336,10 +336,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
open_attrs,
phase: WritePhase::Buffering { part_buffer: Vec::new() },
})?;
Ok(Handle {
id,
handle: handle.into(),
})
Ok(Handle { id, handle })
}
/// Upload the buffered bytes to S3 with a single PutObject call.
@@ -2108,7 +2105,7 @@ mod tests {
)
.await
.expect("WRITE | CREATE | TRUNCATE on a missing file must allocate a handle");
assert!(!handle.handle.as_bytes().is_empty());
assert!(!handle.handle.is_empty());
assert!(backend.head_object_calls().is_empty(), "the non-EXCL accept path must not HEAD");
}
@@ -2133,7 +2130,7 @@ mod tests {
)
.await
.expect("WRITE | CREATE | TRUNCATE on an existing file must allocate a handle");
assert!(!handle.handle.as_bytes().is_empty());
assert!(!handle.handle.is_empty());
assert!(backend.head_object_calls().is_empty(), "the non-EXCL accept path must not HEAD");
}
@@ -2180,7 +2177,7 @@ mod tests {
)
.await
.expect("EXCL on a missing key must allow creation");
assert!(!handle.handle.as_bytes().is_empty());
assert!(!handle.handle.is_empty());
assert_eq!(backend.head_object_calls().len(), 1, "EXCL must HEAD exactly once");
}
+5
View File
@@ -38,6 +38,8 @@ default = []
# event. Requires `RUSTFLAGS="--cfg hyper_unstable_tracing"`.
diagnostic-internode-http2-ping = ["dep:hyper", "rustfs-io-metrics/diagnostic-internode-http2-ping"]
hotpath = [
"hotpath/hotpath",
"hotpath/tokio",
"rustfs-common/hotpath",
"rustfs-config/hotpath",
"rustfs-io-metrics/hotpath",
@@ -46,6 +48,7 @@ hotpath = [
]
hotpath-alloc = [
"hotpath",
"hotpath/hotpath-alloc",
"rustfs-common/hotpath-alloc",
"rustfs-config/hotpath-alloc",
"rustfs-io-metrics/hotpath-alloc",
@@ -54,6 +57,7 @@ hotpath-alloc = [
]
hotpath-cpu = [
"hotpath",
"hotpath/hotpath-cpu",
"rustfs-common/hotpath-cpu",
"rustfs-config/hotpath-cpu",
"rustfs-io-metrics/hotpath-cpu",
@@ -62,6 +66,7 @@ hotpath-cpu = [
]
[dependencies]
hotpath.workspace = true
rustfs-common.workspace = true
rustfs-heal-contracts.workspace = true
rustfs-io-metrics.workspace = true
+5
View File
@@ -31,24 +31,29 @@ workspace = true
[features]
default = []
hotpath = [
"hotpath/hotpath",
"hotpath/tokio",
"rustfs-rio/hotpath",
"rustfs-utils/hotpath",
"rustfs-filemeta/hotpath",
]
hotpath-alloc = [
"hotpath",
"hotpath/hotpath-alloc",
"rustfs-rio/hotpath-alloc",
"rustfs-utils/hotpath-alloc",
"rustfs-filemeta/hotpath-alloc",
]
hotpath-cpu = [
"hotpath",
"hotpath/hotpath-cpu",
"rustfs-rio/hotpath-cpu",
"rustfs-utils/hotpath-cpu",
"rustfs-filemeta/hotpath-cpu",
]
[dependencies]
hotpath.workspace = true
aes-gcm = { workspace = true, features = ["rand_core"] }
bytes = { workspace = true, features = ["serde"] }
chacha20poly1305.workspace = true
+2 -4
View File
@@ -32,7 +32,6 @@ workspace = true
internode-transport = []
default = []
hotpath = [
"dep:hotpath",
"hotpath/hotpath",
"hotpath/tokio",
"hotpath/futures",
@@ -60,8 +59,7 @@ hotpath-cpu = [
]
[dependencies]
# Feature-gated in src (measure_block!); optional so default builds do not warn.
hotpath = { workspace = true, optional = true }
hotpath.workspace = true
arc-swap.workspace = true
tokio = { workspace = true, features = ["io-util", "macros", "net", "rt-multi-thread", "sync", "time"] }
rand = { workspace = true, features = ["serde"] }
@@ -81,6 +79,7 @@ rustfs-io-metrics.workspace = true
rustfs-tls-runtime.workspace = true
rustfs-utils = { workspace = true, features = ["io", "hash", "compress"] }
serde_json = { workspace = true, features = ["raw_value"] }
md-5 = { workspace = true }
hmac.workspace = true
minlz.workspace = true
tracing.workspace = true
@@ -92,7 +91,6 @@ xxhash-rust = { workspace = true, features = ["xxh64", "xxh3"] }
hex-simd.workspace = true
[dev-dependencies]
md-5 = { workspace = true }
faster-hex.workspace = true
rcgen.workspace = true
tokio-rustls.workspace = true
+1
View File
@@ -15,6 +15,7 @@
pub mod init_time;
pub mod last_minute;
pub mod metrics;
pub mod scan_observation;
pub mod startup;
pub mod usage_candidate;
+40 -1
View File
@@ -13,6 +13,9 @@
// limitations under the License.
use crate::last_minute::{AccElem, LastMinuteLatency};
use crate::scan_observation::{
ScannerMetadataReadGuard, ScannerMetadataReadOutcome, ScannerScanObservation, ScannerScanObservationSnapshot,
};
use crate::usage_candidate::{
ScannerUsageCandidateCount, ScannerUsageCandidateKind, ScannerUsageCandidateOutcome, USAGE_CANDIDATE_COUNTER_SLOTS,
};
@@ -977,6 +980,7 @@ impl ScannerUsagePublicationObservation {
pub struct Metrics {
startup: crate::startup::ScannerStartupObservation,
scan_observation: ScannerScanObservation,
operations: Vec<AtomicU64>,
latency: Vec<LockedLastMinuteLatency>,
actions: Vec<AtomicU64>,
@@ -1631,6 +1635,9 @@ pub struct ScannerMetricsReport {
pub lifecycle_transition: ScannerLifecycleTransitionSnapshot,
#[serde(default)]
pub usage_freshness: ScannerUsageFreshnessSnapshot,
/// Node-local, process-lifetime work observations. Older nodes have no evidence.
#[serde(default)]
pub scan_observation: ScannerScanObservationSnapshot,
/// Process-lifetime candidate-construction results. Missing on older reports
/// means unobserved, not accepted or healthy. Publication is tracked separately.
#[serde(default)]
@@ -2125,9 +2132,38 @@ impl Metrics {
&self.startup
}
pub fn scan_observation(&self) -> &ScannerScanObservation {
&self.scan_observation
}
/// Start after scanner node admission, before polling the disk read future.
/// Disk-permit and RPC waits remain included; decoding is excluded. Dropping
/// an unfinished guard observes cancellation, not physical I/O completion.
pub fn start_scanner_metadata_read(&self) -> ScannerMetadataReadGuard<'_> {
self.scan_observation.start_metadata_read();
ScannerMetadataReadGuard {
metrics: self,
started: std::time::Instant::now(),
outcome: ScannerMetadataReadOutcome::Cancelled,
bytes: 0,
}
}
pub(crate) fn finish_scanner_metadata_read(&self, outcome: ScannerMetadataReadOutcome, bytes: u64, elapsed: Duration) {
let bytes = if outcome == ScannerMetadataReadOutcome::Success {
bytes
} else {
0
};
self.scan_observation.finish_metadata_read(outcome, bytes, elapsed);
self.operations[Metric::ReadMetadata as usize].fetch_add(1, Ordering::Relaxed);
self.latency[Metric::ReadMetadata as usize].add_size(elapsed, bytes);
}
pub fn new() -> Self {
Self {
startup: crate::startup::ScannerStartupObservation::default(),
scan_observation: ScannerScanObservation::default(),
operations: (0..Metric::Last as usize).map(|_| AtomicU64::new(0)).collect(),
// Each slot gets its own fresh LockedLastMinuteLatency so that
// different metrics never accidentally share state.
@@ -2733,6 +2769,8 @@ impl Metrics {
/// walked the whole namespace, so an operator can tell why scoped
/// convergence is not happening.
pub fn record_scanner_scope_fallback(&self, reason: impl Into<String>) {
let reason = reason.into();
self.scan_observation.record_scope_fallback(&reason);
self.scanner_scope_fallback_total.fetch_add(1, Ordering::Relaxed);
self.scanner_scope_fallback_last_unix_secs
.store(unix_now_secs(), Ordering::Relaxed);
@@ -2740,7 +2778,7 @@ impl Metrics {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
};
*last_reason = reason.into();
*last_reason = reason;
}
pub fn record_scanner_usage_durable_success(&self) {
@@ -3875,6 +3913,7 @@ impl Metrics {
failed: self.scanner_transition_failed.load(Ordering::Relaxed),
};
m.usage_candidates = self.usage_candidate_counts();
m.scan_observation = self.scan_observation.snapshot();
let usage_save_result = ScannerUsageSaveResult::from_code(self.scanner_usage_last_save_result.load(Ordering::Relaxed));
m.usage_freshness = ScannerUsageFreshnessSnapshot {
dirty_pending_buckets: self.scanner_dirty_usage_pending_buckets.load(Ordering::Relaxed),
@@ -0,0 +1,675 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Node-local, process-lifetime observations of scanner work. Counters are
//! independently loaded, so concurrent reports need not balance exactly.
//! These observations never grant cache reuse or publication authority.
use crate::metrics::Metrics;
use serde::{Deserialize, Serialize};
use std::{
collections::BTreeMap,
sync::atomic::{AtomicU64, Ordering},
time::{Duration, Instant},
};
pub const PREFIX_REUSE_BLOCKERS: [&str; 8] = [
"missing_producer_identity",
"missing_durable_journal_replay",
"restart_gap",
"generation_gap",
"overflow",
"stale_ack_generation",
"missing_cold_zero_walk_oracle",
"distributed_without_peer_invalidation",
];
const SCOPE_FALLBACK_REASONS: [&str; 6] = [
"dirty_snapshot_incomplete",
"dirty_generation_exhausted",
"dirty_bucket_overflow",
"no_listed_dirty_buckets",
"baseline_unavailable",
"peer_scope_unavailable",
];
/// A storage read result, independent of later metadata decoding or ILM work.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum ScannerMetadataReadOutcome {
Success,
NotFound,
Error,
Cancelled,
}
/// The existing path that actually copied a cached subtree instead of walking it.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum ScannerCacheReuseKind {
CleanPrefix,
CoveredNamespace,
SampledNamespace,
ClassifiedNamespace,
}
impl ScannerCacheReuseKind {
const ALL: [Self; 4] = [
Self::CleanPrefix,
Self::CoveredNamespace,
Self::SampledNamespace,
Self::ClassifiedNamespace,
];
fn label(self) -> &'static str {
match self {
Self::CleanPrefix => "clean_prefix",
Self::CoveredNamespace => "covered_namespace",
Self::SampledNamespace => "sampled_namespace",
Self::ClassifiedNamespace => "classified_namespace",
}
}
}
/// Completed bucket-level reuse work, independent of durable publication.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum ScannerBucketReuseKind {
ScopedColdBucket,
CurrentRoot,
}
impl ScannerBucketReuseKind {
const ALL: [Self; 2] = [Self::ScopedColdBucket, Self::CurrentRoot];
fn label(self) -> &'static str {
match self {
Self::ScopedColdBucket => "scoped_cold_bucket",
Self::CurrentRoot => "current_root",
}
}
}
/// A production folder gate, not a diagnostic preflight.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum ScannerCacheReuseGate {
CleanPrefix,
CoveredNamespace,
NamespaceSampling,
}
impl ScannerCacheReuseGate {
const ALL: [Self; 3] = [Self::CleanPrefix, Self::CoveredNamespace, Self::NamespaceSampling];
fn label(self) -> &'static str {
match self {
Self::CleanPrefix => "clean_prefix",
Self::CoveredNamespace => "covered_namespace",
Self::NamespaceSampling => "namespace_sampling",
}
}
}
/// The first existing condition reached by a folder reuse gate.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum ScannerCacheReuseDecision {
MissingScope,
NoResumeSweep,
DeepScan,
ForwardSweep,
ActiveReplication,
NotRoot,
CompactedParent,
MissingCachedChild,
InvalidPrefix,
DirtyPrefix,
ErasureHealthRequired,
MissingCoverage,
CachedFailures,
UncompactedChild,
SelectedForWalk,
Eligible,
}
impl ScannerCacheReuseDecision {
const ALL: [Self; 16] = [
Self::MissingScope,
Self::NoResumeSweep,
Self::DeepScan,
Self::ForwardSweep,
Self::ActiveReplication,
Self::NotRoot,
Self::CompactedParent,
Self::MissingCachedChild,
Self::InvalidPrefix,
Self::DirtyPrefix,
Self::ErasureHealthRequired,
Self::MissingCoverage,
Self::CachedFailures,
Self::UncompactedChild,
Self::SelectedForWalk,
Self::Eligible,
];
fn label(self) -> &'static str {
match self {
Self::MissingScope => "missing_scope",
Self::NoResumeSweep => "no_resume_sweep",
Self::DeepScan => "deep_scan",
Self::ForwardSweep => "forward_sweep",
Self::ActiveReplication => "active_replication",
Self::NotRoot => "not_root",
Self::CompactedParent => "compacted_parent",
Self::MissingCachedChild => "missing_cached_child",
Self::InvalidPrefix => "invalid_prefix",
Self::DirtyPrefix => "dirty_prefix",
Self::ErasureHealthRequired => "erasure_health_required",
Self::MissingCoverage => "missing_coverage",
Self::CachedFailures => "cached_failures",
Self::UncompactedChild => "uncompacted_child",
Self::SelectedForWalk => "selected_for_walk",
Self::Eligible => "eligible",
}
}
fn not_evaluated(self) -> bool {
matches!(
self,
Self::MissingScope | Self::NoResumeSweep | Self::DeepScan | Self::ForwardSweep | Self::ActiveReplication
)
}
}
/// Scope admission and end-of-cycle evidence are separate observations. An
/// activated completion proof does not mean this cycle used prefix reuse.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum ScannerPrefixReuseStage {
ScopeResolution,
CycleCompletion,
}
/// The first existing baseline validation check that rejected a snapshot.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum ScannerBaselineRejection {
Missing,
Decode,
Incomplete,
Partial,
Marks,
Epoch,
SourceCount,
PlanMissing,
PlanMismatch,
SourceIndex,
SourceMembership,
SourceDuplicate,
SetIncomplete,
SetTombstone,
SetEpoch,
SetCycle,
SetPlan,
SourceCoverage,
AuthoritativeIdentity,
NotNewer,
}
impl ScannerBaselineRejection {
const ALL: [Self; 20] = [
Self::Missing,
Self::Decode,
Self::Incomplete,
Self::Partial,
Self::Marks,
Self::Epoch,
Self::SourceCount,
Self::PlanMissing,
Self::PlanMismatch,
Self::SourceIndex,
Self::SourceMembership,
Self::SourceDuplicate,
Self::SetIncomplete,
Self::SetTombstone,
Self::SetEpoch,
Self::SetCycle,
Self::SetPlan,
Self::SourceCoverage,
Self::AuthoritativeIdentity,
Self::NotNewer,
];
fn label(self) -> &'static str {
match self {
Self::Missing => "missing",
Self::Decode => "decode",
Self::Incomplete => "incomplete",
Self::Partial => "partial",
Self::Marks => "marks",
Self::Epoch => "epoch",
Self::SourceCount => "source_count",
Self::PlanMissing => "plan_missing",
Self::PlanMismatch => "plan_mismatch",
Self::SourceIndex => "source_index",
Self::SourceMembership => "source_membership",
Self::SourceDuplicate => "source_duplicate",
Self::SetIncomplete => "set_incomplete",
Self::SetTombstone => "set_tombstone",
Self::SetEpoch => "set_epoch",
Self::SetCycle => "set_cycle",
Self::SetPlan => "set_plan",
Self::SourceCoverage => "source_coverage",
Self::AuthoritativeIdentity => "authoritative_identity",
Self::NotNewer => "not_newer",
}
}
}
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
#[serde(default)]
pub struct ScannerMetadataReadSnapshot {
pub attempts: u64,
pub succeeded: u64,
pub not_found: u64,
pub errors: u64,
pub cancelled: u64,
/// Bytes returned by successful storage reads, including corrupt metadata.
pub bytes: u64,
/// Elapsed read-future time, including failed and cancelled reads.
pub elapsed_nanos: u64,
}
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
#[serde(default)]
pub struct ScannerCacheReuseSnapshot {
/// Completed subtree reuse decisions; not cache entries or objects skipped.
pub count: u64,
/// Time spent copying and attaching the reused subtree, excluding listing.
pub elapsed_nanos: u64,
}
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
#[serde(default)]
pub struct ScannerBucketReuseSnapshot {
/// Cold bucket copies or current roots successfully handed to the collector.
pub count: u64,
/// Cold copy/attachment time or current-root channel-send time; excludes validation.
pub elapsed_nanos: u64,
}
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
#[serde(default)]
pub struct ScannerCacheReuseDecisionSnapshot {
/// Policy prerequisites that stopped the gate before cache proof checks.
pub not_evaluated: BTreeMap<String, u64>,
/// First failed cache/work condition; later conditions are not evaluated.
pub blocked: BTreeMap<String, u64>,
/// Admitted candidates, which may still need EC classification before copying.
pub eligible: u64,
}
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
#[serde(default)]
pub struct ScannerPrefixReuseSnapshot {
pub evaluated: u64,
pub activated: u64,
/// All failed checks are counted; one evaluation can have several blockers.
pub blockers: BTreeMap<String, u64>,
}
/// Missing on an older node means unobserved, not zero work or healthy reuse.
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
#[serde(default)]
pub struct ScannerScanObservationSnapshot {
pub metadata_reads: ScannerMetadataReadSnapshot,
pub cache_reuse: BTreeMap<String, ScannerCacheReuseSnapshot>,
pub bucket_reuse: BTreeMap<String, ScannerBucketReuseSnapshot>,
pub reuse_decisions: BTreeMap<String, ScannerCacheReuseDecisionSnapshot>,
/// Clean root children still walked because EC scans also check health.
pub erasure_clean_prefix_walks: u64,
pub prefix_scope_resolution: ScannerPrefixReuseSnapshot,
pub prefix_cycle_completion: ScannerPrefixReuseSnapshot,
pub scope_fallbacks: BTreeMap<String, u64>,
/// Rejections only for cycles with a baseline_unavailable scope fallback.
pub authoritative_baseline_rejections: BTreeMap<String, u64>,
pub observed_baseline_rejections: BTreeMap<String, u64>,
}
#[derive(Default)]
pub struct ScannerScanObservation {
metadata: [AtomicU64; 7],
cache_count: [AtomicU64; 4],
cache_nanos: [AtomicU64; 4],
bucket_count: [AtomicU64; 2],
bucket_nanos: [AtomicU64; 2],
reuse_decisions: [[AtomicU64; ScannerCacheReuseDecision::ALL.len()]; 3],
erasure_clean_prefix_walks: AtomicU64,
prefix_evaluated: [AtomicU64; 2],
prefix_activated: [AtomicU64; 2],
prefix_blockers: [[AtomicU64; PREFIX_REUSE_BLOCKERS.len()]; 2],
scope_fallbacks: [AtomicU64; SCOPE_FALLBACK_REASONS.len()],
baseline_rejections: [[AtomicU64; ScannerBaselineRejection::ALL.len()]; 2],
}
fn duration_nanos(duration: Duration) -> u64 {
u64::try_from(duration.as_nanos()).unwrap_or(u64::MAX)
}
fn labelled_counts(labels: impl IntoIterator<Item = &'static str>, counters: &[AtomicU64]) -> BTreeMap<String, u64> {
labels
.into_iter()
.zip(counters)
.map(|(label, count)| (label.to_owned(), count.load(Ordering::Relaxed)))
.collect()
}
impl ScannerScanObservation {
pub(crate) fn start_metadata_read(&self) {
self.metadata[0].fetch_add(1, Ordering::Relaxed);
}
pub(crate) fn finish_metadata_read(&self, outcome: ScannerMetadataReadOutcome, bytes: u64, elapsed: Duration) {
self.metadata[1 + outcome as usize].fetch_add(1, Ordering::Relaxed);
if outcome == ScannerMetadataReadOutcome::Success {
self.metadata[5].fetch_add(bytes, Ordering::Relaxed);
}
self.metadata[6].fetch_add(duration_nanos(elapsed), Ordering::Relaxed);
}
pub fn record_cache_reuse(&self, kind: ScannerCacheReuseKind, elapsed: Duration) {
self.cache_count[kind as usize].fetch_add(1, Ordering::Relaxed);
self.cache_nanos[kind as usize].fetch_add(duration_nanos(elapsed), Ordering::Relaxed);
}
/// Records completed reuse work only; it grants no publication authority.
pub fn record_bucket_reuse(&self, kind: ScannerBucketReuseKind, count: u64, elapsed: Duration) {
self.bucket_count[kind as usize].fetch_add(count, Ordering::Relaxed);
self.bucket_nanos[kind as usize].fetch_add(duration_nanos(elapsed), Ordering::Relaxed);
}
/// Records one first-condition decision without inspecting later prerequisites.
pub fn record_cache_reuse_decision(&self, gate: ScannerCacheReuseGate, decision: ScannerCacheReuseDecision) {
self.reuse_decisions[gate as usize][decision as usize].fetch_add(1, Ordering::Relaxed);
}
pub fn record_erasure_clean_prefix_walk(&self) {
self.erasure_clean_prefix_walks.fetch_add(1, Ordering::Relaxed);
}
pub fn record_prefix_reuse(&self, stage: ScannerPrefixReuseStage, activated: bool, blockers: &[Option<&str>; 8]) {
let stage = stage as usize;
self.prefix_evaluated[stage].fetch_add(1, Ordering::Relaxed);
if activated {
self.prefix_activated[stage].fetch_add(1, Ordering::Relaxed);
}
for blocker in blockers.iter().flatten() {
if let Some(index) = PREFIX_REUSE_BLOCKERS.iter().position(|label| label == blocker) {
self.prefix_blockers[stage][index].fetch_add(1, Ordering::Relaxed);
}
}
}
pub(crate) fn record_scope_fallback(&self, reason: &str) {
if let Some(index) = SCOPE_FALLBACK_REASONS.iter().position(|label| *label == reason) {
self.scope_fallbacks[index].fetch_add(1, Ordering::Relaxed);
}
}
pub fn record_baseline_rejection(&self, authoritative: ScannerBaselineRejection, observed: Option<ScannerBaselineRejection>) {
self.baseline_rejections[0][authoritative as usize].fetch_add(1, Ordering::Relaxed);
if let Some(observed) = observed {
self.baseline_rejections[1][observed as usize].fetch_add(1, Ordering::Relaxed);
}
}
pub fn snapshot(&self) -> ScannerScanObservationSnapshot {
let prefix = |stage: ScannerPrefixReuseStage| ScannerPrefixReuseSnapshot {
evaluated: self.prefix_evaluated[stage as usize].load(Ordering::Relaxed),
activated: self.prefix_activated[stage as usize].load(Ordering::Relaxed),
blockers: labelled_counts(PREFIX_REUSE_BLOCKERS, &self.prefix_blockers[stage as usize]),
};
ScannerScanObservationSnapshot {
metadata_reads: ScannerMetadataReadSnapshot {
attempts: self.metadata[0].load(Ordering::Relaxed),
succeeded: self.metadata[1].load(Ordering::Relaxed),
not_found: self.metadata[2].load(Ordering::Relaxed),
errors: self.metadata[3].load(Ordering::Relaxed),
cancelled: self.metadata[4].load(Ordering::Relaxed),
bytes: self.metadata[5].load(Ordering::Relaxed),
elapsed_nanos: self.metadata[6].load(Ordering::Relaxed),
},
cache_reuse: ScannerCacheReuseKind::ALL
.into_iter()
.map(|kind| {
(
kind.label().to_owned(),
ScannerCacheReuseSnapshot {
count: self.cache_count[kind as usize].load(Ordering::Relaxed),
elapsed_nanos: self.cache_nanos[kind as usize].load(Ordering::Relaxed),
},
)
})
.collect(),
bucket_reuse: ScannerBucketReuseKind::ALL
.into_iter()
.map(|kind| {
(
kind.label().to_owned(),
ScannerBucketReuseSnapshot {
count: self.bucket_count[kind as usize].load(Ordering::Relaxed),
elapsed_nanos: self.bucket_nanos[kind as usize].load(Ordering::Relaxed),
},
)
})
.collect(),
reuse_decisions: ScannerCacheReuseGate::ALL
.into_iter()
.map(|gate| {
let counts = &self.reuse_decisions[gate as usize];
let decisions = |not_evaluated| {
ScannerCacheReuseDecision::ALL
.into_iter()
.filter(|decision| {
*decision != ScannerCacheReuseDecision::Eligible && decision.not_evaluated() == not_evaluated
})
.map(|decision| (decision.label().to_owned(), counts[decision as usize].load(Ordering::Relaxed)))
.collect()
};
(
gate.label().to_owned(),
ScannerCacheReuseDecisionSnapshot {
not_evaluated: decisions(true),
blocked: decisions(false),
eligible: counts[ScannerCacheReuseDecision::Eligible as usize].load(Ordering::Relaxed),
},
)
})
.collect(),
erasure_clean_prefix_walks: self.erasure_clean_prefix_walks.load(Ordering::Relaxed),
prefix_scope_resolution: prefix(ScannerPrefixReuseStage::ScopeResolution),
prefix_cycle_completion: prefix(ScannerPrefixReuseStage::CycleCompletion),
scope_fallbacks: labelled_counts(SCOPE_FALLBACK_REASONS, &self.scope_fallbacks),
authoritative_baseline_rejections: labelled_counts(
ScannerBaselineRejection::ALL.map(ScannerBaselineRejection::label),
&self.baseline_rejections[0],
),
observed_baseline_rejections: labelled_counts(
ScannerBaselineRejection::ALL.map(ScannerBaselineRejection::label),
&self.baseline_rejections[1],
),
}
}
}
/// Dropping an unfinished read records cancellation. The guard observes the
/// read future only; it cannot cancel it or change its storage result.
#[must_use]
pub struct ScannerMetadataReadGuard<'a> {
pub(crate) metrics: &'a Metrics,
pub(crate) started: Instant,
pub(crate) outcome: ScannerMetadataReadOutcome,
pub(crate) bytes: u64,
}
impl ScannerMetadataReadGuard<'_> {
pub fn finish(mut self, outcome: ScannerMetadataReadOutcome, bytes: u64) {
self.outcome = outcome;
self.bytes = bytes;
}
}
impl Drop for ScannerMetadataReadGuard<'_> {
fn drop(&mut self) {
self.metrics
.finish_scanner_metadata_read(self.outcome, self.bytes, self.started.elapsed());
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::Arc;
#[tokio::test]
async fn metadata_read_reports_storage_results_and_existing_read_metric() {
let metrics = Metrics::new();
metrics
.start_scanner_metadata_read()
.finish(ScannerMetadataReadOutcome::Success, 127);
metrics
.start_scanner_metadata_read()
.finish(ScannerMetadataReadOutcome::Success, 0);
metrics
.start_scanner_metadata_read()
.finish(ScannerMetadataReadOutcome::NotFound, 0);
metrics
.start_scanner_metadata_read()
.finish(ScannerMetadataReadOutcome::Error, 0);
let report = metrics.report().await;
let reads = &report.scan_observation.metadata_reads;
assert_eq!(
(reads.attempts, reads.succeeded, reads.not_found, reads.errors, reads.cancelled),
(4, 2, 1, 1, 0)
);
assert_eq!(reads.bytes, 127);
assert_eq!(report.life_time_ops["read_metadata"], 4);
assert_eq!(report.last_minute.actions["read_metadata"].count, 4);
assert_eq!(report.last_minute.actions["read_metadata"].bytes, 127);
}
#[tokio::test]
async fn metadata_read_cancellation_balances_an_actually_polled_future_once() {
let metrics = Arc::new(Metrics::new());
let worker_metrics = Arc::clone(&metrics);
let (started, admitted) = tokio::sync::oneshot::channel();
let worker = tokio::spawn(async move {
let read = worker_metrics.start_scanner_metadata_read();
started.send(()).expect("observe started read");
std::future::pending::<()>().await;
read.finish(ScannerMetadataReadOutcome::Success, 99);
});
tokio::time::timeout(Duration::from_secs(5), admitted)
.await
.expect("read starts within deadline")
.expect("read started");
let in_flight = metrics.scan_observation().snapshot().metadata_reads;
assert_eq!((in_flight.attempts, in_flight.cancelled, in_flight.succeeded), (1, 0, 0));
worker.abort();
assert!(worker.await.expect_err("read task was aborted").is_cancelled());
let report = metrics.report().await;
let reads = &report.scan_observation.metadata_reads;
assert_eq!((reads.attempts, reads.cancelled, reads.succeeded, reads.bytes), (1, 1, 0, 0));
assert_eq!(report.life_time_ops["read_metadata"], 1);
assert_eq!(report.last_minute.actions["read_metadata"].count, 1);
assert_eq!(
Duration::from_nanos(reads.elapsed_nanos).as_secs(),
report.last_minute.actions["read_metadata"].acc_time
);
}
#[tokio::test]
async fn observation_report_keeps_stages_reasons_and_copy_costs_separate() {
let metrics = Metrics::new();
let observation = metrics.scan_observation();
observation.record_cache_reuse(ScannerCacheReuseKind::CleanPrefix, Duration::from_nanos(321));
observation.record_bucket_reuse(ScannerBucketReuseKind::ScopedColdBucket, 2, Duration::from_nanos(654));
observation.record_bucket_reuse(ScannerBucketReuseKind::CurrentRoot, 1, Duration::from_nanos(987));
observation.record_cache_reuse_decision(ScannerCacheReuseGate::CleanPrefix, ScannerCacheReuseDecision::MissingScope);
observation
.record_cache_reuse_decision(ScannerCacheReuseGate::CleanPrefix, ScannerCacheReuseDecision::ErasureHealthRequired);
observation.record_cache_reuse_decision(ScannerCacheReuseGate::CoveredNamespace, ScannerCacheReuseDecision::Eligible);
observation.record_erasure_clean_prefix_walk();
observation.record_prefix_reuse(
ScannerPrefixReuseStage::ScopeResolution,
false,
&[
Some("missing_producer_identity"),
Some("missing_durable_journal_replay"),
None,
None,
None,
None,
None,
None,
],
);
observation.record_prefix_reuse(ScannerPrefixReuseStage::CycleCompletion, true, &[None; 8]);
metrics.record_scanner_scope_fallback("baseline_unavailable");
observation.record_baseline_rejection(ScannerBaselineRejection::Epoch, Some(ScannerBaselineRejection::Missing));
let report = metrics.report().await;
let snapshot = &report.scan_observation;
assert_eq!(snapshot.cache_reuse["clean_prefix"].count, 1);
assert_eq!(snapshot.cache_reuse["clean_prefix"].elapsed_nanos, 321);
assert_eq!(snapshot.cache_reuse["sampled_namespace"].count, 0);
assert_eq!(snapshot.bucket_reuse["scoped_cold_bucket"].count, 2);
assert_eq!(snapshot.bucket_reuse["scoped_cold_bucket"].elapsed_nanos, 654);
assert_eq!(snapshot.bucket_reuse["current_root"].count, 1);
assert_eq!(snapshot.bucket_reuse["current_root"].elapsed_nanos, 987);
assert_eq!(snapshot.reuse_decisions["clean_prefix"].not_evaluated["missing_scope"], 1);
assert_eq!(snapshot.reuse_decisions["clean_prefix"].blocked["erasure_health_required"], 1);
assert_eq!(snapshot.reuse_decisions["covered_namespace"].eligible, 1);
let mut prior_observations = serde_json::to_value(snapshot).expect("encode observations");
let fields = prior_observations.as_object_mut().expect("observation object");
fields.remove("bucket_reuse");
fields.remove("reuse_decisions");
let prior: ScannerScanObservationSnapshot =
serde_json::from_value(prior_observations).expect("read earlier observations");
assert!(prior.bucket_reuse.is_empty());
assert!(prior.reuse_decisions.is_empty());
assert_eq!(prior.cache_reuse["clean_prefix"].count, 1);
assert_eq!(snapshot.erasure_clean_prefix_walks, 1);
assert_eq!(
(snapshot.prefix_scope_resolution.evaluated, snapshot.prefix_scope_resolution.activated),
(1, 0)
);
assert_eq!(snapshot.prefix_scope_resolution.blockers["missing_producer_identity"], 1);
assert_eq!(snapshot.prefix_scope_resolution.blockers["missing_durable_journal_replay"], 1);
assert_eq!(
(snapshot.prefix_cycle_completion.evaluated, snapshot.prefix_cycle_completion.activated),
(1, 1)
);
assert_eq!(
snapshot.scope_fallbacks["baseline_unavailable"],
report.usage_freshness.scope_fallback_total
);
assert_eq!(snapshot.authoritative_baseline_rejections["epoch"], 1);
assert_eq!(snapshot.observed_baseline_rejections["missing"], 1);
let mut legacy = serde_json::to_value(&report).expect("encode report");
legacy
.as_object_mut()
.expect("report is an object")
.remove("scan_observation");
let decoded: crate::metrics::ScannerMetricsReport = serde_json::from_value(legacy).expect("read legacy report");
assert!(decoded.scan_observation.cache_reuse.is_empty(), "legacy counters are unobserved");
assert_eq!(decoded.scan_observation.metadata_reads.attempts, 0);
}
}
+26
View File
@@ -206,6 +206,32 @@ pub(crate) struct DataUsageCacheRevisions {
backup: Option<DataUsageCacheRevision>,
}
/// In-memory provenance of the decoded cache and the revisions observed by its loader.
#[derive(Clone, Debug)]
pub(crate) struct DataUsageCacheBaselineRevisions {
revisions: DataUsageCacheRevisions,
loaded_from_main: bool,
}
impl DataUsageCacheBaselineRevisions {
pub(crate) fn revisions(&self) -> &DataUsageCacheRevisions {
&self.revisions
}
/// A backup may catch up to an unchanged primary without changing the decoded baseline.
/// Both reads must have decoded that primary; an ETag on a corrupt primary is insufficient.
pub(crate) fn permits_unchanged_baseline_since(&self, initial: &Self) -> bool {
if self.revisions == initial.revisions {
return true;
}
self.loaded_from_main
&& initial.loaded_from_main
&& matches!(&initial.revisions.main, DataUsageCacheRevision::Etag(etag) if !etag.is_empty())
&& self.revisions.main == initial.revisions.main
&& self.revisions.backup.as_ref() == Some(&self.revisions.main)
}
}
pub static DATA_USAGE_BUCKET: LazyLock<String> =
LazyLock::new(|| format!("{RUSTFS_META_BUCKET}{SLASH_SEPARATOR}{BUCKET_META_PREFIX}"));
@@ -71,6 +71,7 @@ struct DataUsageCacheLoadResult {
cache: DataUsageCache,
main_revision: DataUsageCacheRevision,
backup_revision: Option<DataUsageCacheRevision>,
loaded_from_main: bool,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
@@ -107,6 +108,18 @@ impl DataUsageCache {
store: Arc<S>,
name: &str,
) -> StorageResult<DataUsageCacheRevisions> {
self.load_with_baseline_revisions(store, name)
.await
.map(|baseline| baseline.revisions)
}
/// Loads cache revisions with provenance for unchanged-primary backup reconciliation.
/// Missing or rebuilt caches and a decoded backup do not establish a decoded primary.
pub(crate) async fn load_with_baseline_revisions<S: ScannerObjectIO>(
&mut self,
store: Arc<S>,
name: &str,
) -> StorageResult<DataUsageCacheBaselineRevisions> {
let backup_name = format!("{name}.bkp");
let backup_path = path_join_buf(&[BUCKET_META_PREFIX, &backup_name]);
let loaded = Self::load_cache(store.clone(), name).await?;
@@ -134,7 +147,10 @@ impl DataUsageCache {
let main = loaded.main_revision;
*self = loaded.cache;
Ok(DataUsageCacheRevisions { main, backup })
Ok(DataUsageCacheBaselineRevisions {
revisions: DataUsageCacheRevisions { main, backup },
loaded_from_main: loaded.loaded_from_main,
})
}
pub(crate) async fn read_revisions<S: ScannerObjectIO>(store: Arc<S>, name: &str) -> StorageResult<DataUsageCacheRevisions> {
@@ -157,10 +173,13 @@ impl DataUsageCache {
cache,
revision: Some(main_revision),
} => {
// Legacy-location fallback carries Missing for the primary path.
let loaded_from_main = matches!(&main_revision, DataUsageCacheRevision::Etag(_));
return Ok(DataUsageCacheLoadResult {
cache: *cache,
main_revision,
backup_revision: None,
loaded_from_main,
});
}
DataUsageCacheLoadAttempt::Loaded { revision: None, .. } => {
@@ -198,6 +217,7 @@ impl DataUsageCache {
cache: *cache,
main_revision,
backup_revision: Some(backup_revision),
loaded_from_main: false,
});
}
DataUsageCacheLoadState::Loaded
@@ -221,6 +241,7 @@ impl DataUsageCache {
main_revision: main_revision
.ok_or_else(|| Error::other(format!("scanner cache missing state has no revision: {name}")))?,
backup_revision,
loaded_from_main: false,
});
}
(DataUsageCacheLoadState::Corrupt, DataUsageCacheLoadState::Missing)
@@ -240,6 +261,7 @@ impl DataUsageCache {
main_revision: main_revision
.ok_or_else(|| Error::other(format!("scanner cache corrupt state has no revision: {name}")))?,
backup_revision,
loaded_from_main: false,
});
}
_ => {}
@@ -772,6 +772,102 @@ async fn corrupt_primary_cache_loads_valid_backup() {
assert_eq!(store.puts.load(Ordering::SeqCst), 0);
}
#[test]
fn baseline_revisions_allow_only_a_decoded_unchanged_primary_backup_catchup() {
let etag = |value: &str| DataUsageCacheRevision::Etag(value.to_owned());
let initial = DataUsageCacheBaselineRevisions {
revisions: DataUsageCacheRevisions {
main: etag("primary"),
backup: Some(etag("old-backup")),
},
loaded_from_main: true,
};
assert!(initial.permits_unchanged_baseline_since(&initial));
let mut caught_up = initial.clone();
caught_up.revisions.backup = Some(etag("primary"));
assert!(caught_up.permits_unchanged_baseline_since(&initial));
assert!(
!initial.permits_unchanged_baseline_since(&caught_up),
"backup rollback changes the baseline"
);
for backup in [None, Some(DataUsageCacheRevision::Missing)] {
let mut missing_backup = initial.clone();
missing_backup.revisions.backup = backup;
assert!(caught_up.permits_unchanged_baseline_since(&missing_backup));
assert!(!missing_backup.permits_unchanged_baseline_since(&initial));
}
for (initial_from_main, current_from_main) in [(false, true), (true, false), (false, false)] {
let mut baseline = initial.clone();
baseline.loaded_from_main = initial_from_main;
let mut current = caught_up.clone();
current.loaded_from_main = current_from_main;
assert!(
!current.permits_unchanged_baseline_since(&baseline),
"backup fallback grants no primary proof"
);
current.revisions = baseline.revisions.clone();
assert!(
current.permits_unchanged_baseline_since(&baseline),
"exact revisions retain existing admission"
);
}
let mut changed_main = caught_up.clone();
changed_main.revisions.main = etag("new-primary");
changed_main.revisions.backup = Some(etag("new-primary"));
assert!(!changed_main.permits_unchanged_baseline_since(&initial));
let mut changed_backup = caught_up;
changed_backup.revisions.backup = Some(etag("unrelated-backup"));
assert!(!changed_backup.permits_unchanged_baseline_since(&initial));
for main in [DataUsageCacheRevision::Missing, etag("")] {
let mut baseline = initial.clone();
baseline.revisions.main = main.clone();
let mut current = baseline.clone();
current.revisions.backup = Some(main);
assert!(
!current.permits_unchanged_baseline_since(&baseline),
"missing or empty primary ETag is not proof"
);
}
}
#[tokio::test]
async fn baseline_revisions_record_the_actual_decoded_cache_source() {
let mut expected = DataUsageCache::default();
expected.info.name = "decoded".to_owned();
let valid = expected.marshal_msg().expect("encode valid cache");
for (main, backup, from_main, name) in [
(valid.clone(), Some(valid.clone()), true, "decoded"),
(b"corrupt".to_vec(), Some(valid.clone()), false, "decoded"),
(b"corrupt".to_vec(), None, false, ""),
(Vec::new(), None, false, ""),
] {
let store = Arc::new(CacheReadStore::new(CacheReadBody::Bytes(main), backup));
let mut loaded = DataUsageCache::default();
let baseline = loaded
.load_with_baseline_revisions(store.clone(), DATA_USAGE_CACHE_NAME)
.await
.expect("load actual cache source");
assert_eq!(baseline.loaded_from_main, from_main);
assert_eq!(loaded.info.name, name);
let mut raw_loaded = DataUsageCache::default();
let raw = raw_loaded
.load_with_revisions(store, DATA_USAGE_CACHE_NAME)
.await
.expect("load compatible raw revisions");
assert_eq!(baseline.revisions(), &raw, "raw loader retains its revision contract");
}
let mut legacy_store = CacheReadStore::new(CacheReadBody::Bytes(Vec::new()), None);
legacy_store.legacy = Some(valid);
let mut legacy_cache = DataUsageCache::default();
let legacy = legacy_cache
.load_with_baseline_revisions(Arc::new(legacy_store), "legacy-cache")
.await
.expect("load cache from the legacy location");
assert_eq!(legacy_cache.info.name, "decoded");
assert!(!legacy.loaded_from_main);
assert_eq!(legacy.revisions.main, DataUsageCacheRevision::Missing);
}
#[tokio::test]
async fn corrupt_cache_without_valid_backup_rebuilds_with_cas_revisions() {
for backup in [None, Some(b"also-not-msgpack".to_vec())] {
+8 -9
View File
@@ -248,13 +248,13 @@ const CLEAN_IDLE_BACKOFF_FACTOR: u32 = 2;
/// otherwise idle instance whose clean-idle backoff had doubled a 60 s
/// interval), which defeats the fast path it is meant to protect.
///
/// The exponential growth in [`ScannerRetryBackoff::retry_interval`] is
/// what protects against a persistently hot bucket driving an unbroken
/// full-scan loop, so it can start small: 5 s, 10 s, 20 s … capped by
/// [`SCANNER_RETRY_MAX_INTERVAL`]. A one-off race recovers in seconds; a
/// genuinely hot bucket still reaches minute-scale backoff within a handful of
/// cycles. Preflight deferrals use the same bounded schedule so a temporarily
/// unavailable peer cannot drive a tight retry loop.
/// The exponential growth in [`ScannerRetryBackoff::retry_interval`] protects
/// against a persistently hot bucket driving an unbroken full-scan loop, so
/// retries can start small: 5 s, 10 s, 20 s … Superseded retries cap this
/// schedule at the ordinary cycle interval and preserve an operator-selected
/// nominal cadence. Preflight deferrals retain the generic
/// [`SCANNER_RETRY_MAX_INTERVAL`] cap so a temporarily unavailable peer cannot
/// drive a tight retry loop.
const SCANNER_RETRY_BASE_INTERVAL: Duration = Duration::from_secs(5);
const SCANNER_RETRY_MAX_INTERVAL: Duration = Duration::from_secs(30 * 60);
/// Probe cadence while data movement pauses the scanner. Recovery retries use
@@ -3528,8 +3528,7 @@ where
));
let mut wait_plan =
scanner_cycle_wait_plan(&runtime_config, clean_idle_backoff, backoff_enabled, randomized_cycle_delay_for);
let superseded_retry_interval =
scanner_superseded_retry_interval(superseded_backoff, &runtime_config, maintenance_features);
let superseded_retry_interval = scanner_superseded_retry_interval(superseded_backoff, &runtime_config);
let deferred_retry_interval = deferred_backoff.retry_interval(runtime_config.cycle_interval);
let scope_fallback_retry_interval = scanner_scope_fallback_retry_interval(scope_fallback_backoff, &runtime_config);
let mut convergence_retry_interval = superseded_retry_interval
+7 -10
View File
@@ -192,19 +192,16 @@ impl ScannerRetryBackoff {
pub(super) fn scanner_superseded_retry_interval(
backoff: ScannerRetryBackoff,
runtime_config: &ScannerRuntimeConfig,
maintenance_features: ScannerMaintenanceFeatures,
) -> Option<Duration> {
let mut retry_interval = backoff.retry_interval(runtime_config.cycle_interval)?;
// Preserve an operator-selected duty-cycle floor during convergence.
let cycle_interval = runtime_config.cycle_interval.max(Duration::from_secs(1));
let retry_interval = backoff.retry_interval(cycle_interval)?;
// Preserve an operator-selected nominal cadence during convergence.
if runtime_config.cycle_interval_source != ScannerRuntimeConfigSource::Default {
retry_interval = retry_interval.max(runtime_config.cycle_interval);
return Some(cycle_interval);
}
// A complete but non-authoritative walk must not postpone lifecycle or
// replication evaluation beyond its ordinary maintenance cadence.
if maintenance_features.needs_regular_cycle() {
retry_interval = retry_interval.min(runtime_config.cycle_interval.max(Duration::from_secs(1)));
}
Some(retry_interval)
// A complete but non-authoritative walk must not postpone convergence
// beyond the ordinary cadence, even without lifecycle or replication.
Some(retry_interval.min(cycle_interval))
}
/// Pacing for the cycle after a full-namespace scope fallback while dirty
+73 -38
View File
@@ -8947,7 +8947,7 @@ fn clean_idle_backoff_grows_to_cap() {
}
#[test]
fn superseded_retry_backoff_grows_caps_and_resets_after_convergence() {
fn retry_backoff_grows_caps_and_resets_after_convergence() {
let mut backoff = ScannerRetryBackoff::default();
assert_eq!(backoff.retry_interval(Duration::from_secs(24 * 60 * 60)), None);
@@ -8971,7 +8971,7 @@ fn superseded_retry_backoff_grows_caps_and_resets_after_convergence() {
}
#[test]
fn superseded_retry_backoff_respects_a_faster_configured_cycle() {
fn retry_backoff_respects_a_faster_configured_cycle() {
let mut backoff = ScannerRetryBackoff::default();
backoff.record_retryable_cycle(true);
@@ -8983,7 +8983,7 @@ fn superseded_retry_backoff_respects_a_faster_configured_cycle() {
}
#[test]
fn superseded_retry_backoff_grows_from_the_default_cycle() {
fn retry_backoff_grows_from_the_default_cycle() {
let mut backoff = ScannerRetryBackoff::default();
// The first race after a write burst retries in seconds, not a whole
// cycle, while repeated supersedes still climb toward the cap.
@@ -9003,7 +9003,7 @@ fn superseded_retry_preserves_explicit_cycle_cadence() {
..Default::default()
};
assert_eq!(
scanner_superseded_retry_interval(backoff, &default_config, ScannerMaintenanceFeatures::default()),
scanner_superseded_retry_interval(backoff, &default_config),
Some(SCANNER_RETRY_BASE_INTERVAL),
"the default adaptive cadence must retain fast convergence"
);
@@ -9013,24 +9013,24 @@ fn superseded_retry_preserves_explicit_cycle_cadence() {
ScannerRuntimeConfigSource::Config,
ScannerRuntimeConfigSource::ScannerCompatConfig,
] {
let runtime_config = ScannerRuntimeConfig {
cycle_interval: Duration::from_secs(6 * 60 * 60),
cycle_interval_source: source,
..Default::default()
};
for features in [
ScannerMaintenanceFeatures::default(),
ScannerMaintenanceFeatures {
lifecycle: true,
for cycle_secs in [1, 3, 600, 6 * 60 * 60] {
let runtime_config = ScannerRuntimeConfig {
cycle_interval: Duration::from_secs(cycle_secs),
cycle_interval_source: source,
..Default::default()
},
] {
assert_eq!(
scanner_superseded_retry_interval(backoff, &runtime_config, features),
Some(runtime_config.cycle_interval),
"{source:?} cycle cadence must not be shortened by convergence retries"
);
};
let mut backoff = ScannerRetryBackoff::default();
assert_eq!(scanner_superseded_retry_interval(backoff, &runtime_config), None);
for _ in 0..40 {
backoff.record_retryable_cycle(true);
assert_eq!(
scanner_superseded_retry_interval(backoff, &runtime_config),
Some(runtime_config.cycle_interval),
"{source:?} cycle cadence must hold throughout convergence retries"
);
}
backoff.record_retryable_cycle(false);
assert_eq!(scanner_superseded_retry_interval(backoff, &runtime_config), None);
}
}
}
@@ -9051,7 +9051,7 @@ fn superseded_retry_does_not_delay_required_maintenance() {
for consecutive_cycles in 1..=12 {
backoff.record_retryable_cycle(true);
assert_eq!(
scanner_superseded_retry_interval(backoff, &config, features),
scanner_superseded_retry_interval(backoff, &config),
Some(config.cycle_interval),
"superseded cycle {consecutive_cycles} must preserve regular maintenance"
);
@@ -9137,19 +9137,50 @@ fn scope_fallback_retry_preserves_explicit_and_fast_cycle_cadence() {
}
#[test]
fn superseded_retry_only_caps_regular_maintenance() {
fn superseded_retry_caps_at_the_default_cycle_and_resets_after_convergence() {
let config = ScannerRuntimeConfig {
cycle_interval: Duration::from_secs(60),
..Default::default()
};
let mut backoff = ScannerRetryBackoff::default();
for _ in 0..6 {
assert_eq!(scanner_superseded_retry_interval(backoff, &config), None);
for expected in [5, 10, 20, 40, 60, 60, 60, 60] {
backoff.record_retryable_cycle(true);
assert_eq!(scanner_superseded_retry_interval(backoff, &config), Some(Duration::from_secs(expected)));
}
backoff.consecutive_cycles = u32::MAX;
assert_eq!(scanner_superseded_retry_interval(backoff, &config), Some(config.cycle_interval));
backoff.record_retryable_cycle(true);
assert_eq!(scanner_superseded_retry_interval(backoff, &config), Some(config.cycle_interval));
backoff.record_retryable_cycle(false);
assert_eq!(scanner_superseded_retry_interval(backoff, &config), None);
}
#[test]
fn superseded_retry_caps_fast_cycles_without_changing_the_generic_retry_cap() {
let mut backoff = ScannerRetryBackoff::default();
for _ in 0..40 {
backoff.record_retryable_cycle(true);
}
assert_eq!(
scanner_superseded_retry_interval(backoff, &config, ScannerMaintenanceFeatures::default()),
Some(Duration::from_secs(160)),
backoff.retry_interval(Duration::from_secs(60)),
Some(Duration::from_secs(30 * 60)),
"deferred and transient retries must retain the generic cap"
);
for cycle_secs in [0, 1, 3, 60, 6 * 60 * 60] {
let config = ScannerRuntimeConfig {
cycle_interval: Duration::from_secs(cycle_secs),
..Default::default()
};
assert_eq!(
scanner_superseded_retry_interval(backoff, &config),
Some(Duration::from_secs(cycle_secs.max(1))),
);
}
}
#[test]
fn superseded_retry_timer_keeps_required_maintenance_full_scans() {
for features in [
ScannerMaintenanceFeatures {
lifecycle: true,
@@ -9164,28 +9195,23 @@ fn superseded_retry_only_caps_regular_maintenance() {
..Default::default()
},
] {
assert_eq!(scanner_superseded_retry_interval(backoff, &config, features), Some(config.cycle_interval));
assert_eq!(scanner_superseded_retry_interval(ScannerRetryBackoff::default(), &config, features), None);
assert!(features.requires_full_scan(Some(7), 7, ScannerCycleWakeReason::Timer, true));
}
}
#[tokio::test(start_paused = true)]
#[serial]
async fn superseded_maintenance_wait_keeps_convergence_fencing() {
async fn superseded_retry_wait_keeps_convergence_fencing_at_the_default_cycle() {
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
let config = ScannerRuntimeConfig {
cycle_interval: Duration::from_secs(60),
..Default::default()
};
let features = ScannerMaintenanceFeatures {
lifecycle: true,
..Default::default()
};
let mut backoff = ScannerRetryBackoff::default();
for _ in 0..12 {
backoff.record_retryable_cycle(true);
}
let interval = scanner_superseded_retry_interval(backoff, &config, features);
let interval = scanner_superseded_retry_interval(backoff, &config);
assert_eq!(interval, Some(config.cycle_interval));
let generations = ScannerCycleObservedGenerations::for_wait(
&config,
@@ -9200,19 +9226,28 @@ async fn superseded_maintenance_wait_keeps_convergence_fencing() {
let mut seen = Some(BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))]));
let changed = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 8, 3))]);
let started = tokio::time::Instant::now();
let reason = wait_for_next_scanner_cycle_with_activity(
let mut wait = Box::pin(wait_for_next_scanner_cycle_with_activity(
&ctx,
interval.expect("maintenance must retain its convergence interval"),
interval.expect("superseded retry must retain its convergence interval"),
Some(Duration::from_secs(1)),
&mut seen,
generations,
|| false,
|| std::future::ready(Ok(changed.clone())),
)
.await;
));
assert!(matches!(futures::poll!(&mut wait), Poll::Pending));
crate::scanner_io::record_dirty_usage_bucket("photos");
assert!(matches!(futures::poll!(&mut wait), Poll::Pending));
for _ in 0..3 {
tokio::time::advance(Duration::from_secs(15)).await;
crate::scanner_io::record_dirty_usage_bucket("photos");
assert!(matches!(futures::poll!(&mut wait), Poll::Pending));
}
let reason = wait.await;
assert_eq!(reason, ScannerCycleWakeReason::Timer);
assert_eq!(started.elapsed(), config.cycle_interval);
assert_eq!(seen, Some(changed));
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
}
#[test]
+105 -44
View File
@@ -59,6 +59,7 @@ use rustfs_scanner_metrics::metrics::{
CloseDiskGuard, IlmAction, Metric, Metrics, ScannerReplicationRepairKind, ScannerSourceWorkUpdate, ScannerWorkSource,
UpdateCurrentPathFn, current_path_updater, global_metrics,
};
use rustfs_scanner_metrics::scan_observation::{ScannerCacheReuseDecision, ScannerCacheReuseGate, ScannerCacheReuseKind};
use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf};
use sha2::{Digest, Sha256};
use time::OffsetDateTime;
@@ -1060,38 +1061,51 @@ impl FolderScanner {
child_hash: &DataUsageHash,
abandoned_children: &DataUsageHashMap,
) -> bool {
let Some(prefix_scan_scope) = &self.prefix_scan_scope else {
return false;
let decision = if let Some(prefix_scan_scope) = &self.prefix_scan_scope {
if folder.parent.is_some() || folder.name != self.old_cache.info.name {
ScannerCacheReuseDecision::NotRoot
} else if into.compacted {
ScannerCacheReuseDecision::CompactedParent
} else if !abandoned_children.contains(&child_hash.0) {
ScannerCacheReuseDecision::MissingCachedChild
} else if let Some(entry) = child
.name
.strip_prefix(folder.name.as_str())
.and_then(|entry| entry.strip_prefix('/'))
{
if entry.is_empty() || entry.contains('/') {
ScannerCacheReuseDecision::InvalidPrefix
} else if prefix_scan_scope.contains(entry) {
ScannerCacheReuseDecision::DirtyPrefix
} else if self.is_erasure_mode {
// EC scans must still perform independent object-health work.
global_metrics().scan_observation().record_erasure_clean_prefix_walk();
ScannerCacheReuseDecision::ErasureHealthRequired
} else {
ScannerCacheReuseDecision::Eligible
}
} else {
ScannerCacheReuseDecision::InvalidPrefix
}
} else {
ScannerCacheReuseDecision::MissingScope
};
// Erasure-mode usage scans also perform probabilistic object-health
// work. Reusing a clean subtree here would silently suppress that
// independent maintenance path, so prefix reuse is limited to the
// non-erasure data-usage scanner.
if self.is_erasure_mode
|| folder.parent.is_some()
|| folder.name != self.old_cache.info.name
|| into.compacted
|| !abandoned_children.contains(&clone_cache_value(&child_hash.0))
{
return false;
}
let Some(entry) = child
.name
.strip_prefix(folder.name.as_str())
.and_then(|entry| entry.strip_prefix('/'))
else {
return false;
};
!entry.is_empty() && !entry.contains('/') && !prefix_scan_scope.contains(entry)
global_metrics()
.scan_observation()
.record_cache_reuse_decision(ScannerCacheReuseGate::CleanPrefix, decision);
decision == ScannerCacheReuseDecision::Eligible
}
fn reuse_clean_root_child(&mut self, child: &CachedFolder, child_hash: &DataUsageHash, into: &mut DataUsageEntry) {
let started = Instant::now();
self.new_cache.copy_with_children(&self.old_cache, child_hash, &child.parent);
if let Some(updates) = &mut self.usage_updates {
updates.cache.copy_with_children(&self.old_cache, child_hash, &child.parent);
}
into.add_child(child_hash);
global_metrics()
.scan_observation()
.record_cache_reuse(ScannerCacheReuseKind::CleanPrefix, started.elapsed());
}
fn failed_retry_suppressed(&self, path: &str) -> bool {
@@ -1278,14 +1292,60 @@ impl FolderScanner {
}
fn can_reuse_covered_namespace(&self, name: &str, compacted: bool) -> bool {
self.old_cache.info.scan_progress.is_some()
&& !compacted
&& self
.resume_frontier
.as_deref()
.is_some_and(|frontier| scan_frontier_covers_path(name, frontier))
&& self.old_cache.find(name).is_some()
&& !self.resume_subtree_has_failures(name)
let decision = if self.old_cache.info.scan_progress.is_none() {
ScannerCacheReuseDecision::NoResumeSweep
} else if compacted {
ScannerCacheReuseDecision::CompactedParent
} else if !self
.resume_frontier
.as_deref()
.is_some_and(|frontier| scan_frontier_covers_path(name, frontier))
{
ScannerCacheReuseDecision::MissingCoverage
} else if self.old_cache.find(name).is_none() {
ScannerCacheReuseDecision::MissingCachedChild
} else if self.resume_subtree_has_failures(name) {
ScannerCacheReuseDecision::CachedFailures
} else {
ScannerCacheReuseDecision::Eligible
};
global_metrics()
.scan_observation()
.record_cache_reuse_decision(ScannerCacheReuseGate::CoveredNamespace, decision);
decision == ScannerCacheReuseDecision::Eligible
}
// None preserves work that cannot use sampling; false requests this cycle's rescan.
fn sampled_namespace_reuse(&self, hash: &DataUsageHash, compacted: bool, forward_sweep: bool) -> Option<bool> {
let decision = if self.scan_mode == HealScanMode::Deep {
ScannerCacheReuseDecision::DeepScan
} else if forward_sweep {
ScannerCacheReuseDecision::ForwardSweep
} else if self
.old_cache
.info
.replication
.as_ref()
.is_some_and(|config| config.has_active_rules("", true))
{
ScannerCacheReuseDecision::ActiveReplication
} else if compacted {
ScannerCacheReuseDecision::CompactedParent
} else if !self.old_cache.is_compacted(hash) {
ScannerCacheReuseDecision::UncompactedChild
} else if hash.mod_(self.old_cache.info.next_cycle as u32, data_usage_update_dir_cycles()) {
ScannerCacheReuseDecision::SelectedForWalk
} else {
ScannerCacheReuseDecision::Eligible
};
global_metrics()
.scan_observation()
.record_cache_reuse_decision(ScannerCacheReuseGate::NamespaceSampling, decision);
match decision {
ScannerCacheReuseDecision::Eligible => Some(true),
ScannerCacheReuseDecision::SelectedForWalk => Some(false),
_ => None,
}
}
fn resume_subtree_has_failures(&self, folder: &str) -> bool {
@@ -2077,7 +2137,11 @@ impl FolderScanner {
// The caller stitches this cached child into its
// parent; do not clone a growing child set into the
// unused provisional accumulator as well.
let started = Instant::now();
self.new_cache.copy_with_children(&self.old_cache, &this_hash, &folder.parent);
global_metrics()
.scan_observation()
.record_cache_reuse(ScannerCacheReuseKind::ClassifiedNamespace, started.elapsed());
self.budget.record_entries_visited(pending_entry_progress);
self.finish_raw_enumeration_parent(&folder.name);
done_folder();
@@ -2664,8 +2728,12 @@ impl FolderScanner {
FolderScanMode::Namespace
};
if child_mode == FolderScanMode::Namespace && reuse_covered {
let started = Instant::now();
self.new_cache.copy_with_children(&self.old_cache, &h, &folder_item.parent);
into.add_child(&h);
global_metrics()
.scan_observation()
.record_cache_reuse(ScannerCacheReuseKind::CoveredNamespace, started.elapsed());
continue;
}
@@ -2705,22 +2773,15 @@ impl FolderScanner {
}
FolderScanSource::Existing => {
// Usage sampling is not proof that a Deep check ran.
if self.scan_mode != HealScanMode::Deep
&& !forward_sweep
&& !self
.old_cache
.info
.replication
.as_ref()
.is_some_and(|config| config.has_active_rules("", true))
&& !into.compacted
&& self.old_cache.is_compacted(&h)
{
let next_cycle = self.old_cache.info.next_cycle as u32;
if !h.mod_(next_cycle, data_usage_update_dir_cycles()) {
if let Some(reuse) = self.sampled_namespace_reuse(&h, into.compacted, forward_sweep) {
if reuse {
if child_mode == FolderScanMode::Namespace {
let started = Instant::now();
self.new_cache.copy_with_children(&self.old_cache, &h, &folder_item.parent);
into.add_child(&h);
global_metrics()
.scan_observation()
.record_cache_reuse(ScannerCacheReuseKind::SampledNamespace, started.elapsed());
self.record_scan_resume_hint(&folder_item.name);
continue;
}
+191
View File
@@ -3103,6 +3103,112 @@ async fn test_scan_folder_non_erasure_metadata_keeps_namespace_descent() {
assert_eq!(budget.reason(), Some(crate::scanner_budget::ScannerCycleBudgetReason::Directories));
}
#[tokio::test]
#[serial]
async fn cache_reuse_gates_report_only_the_first_production_condition() {
let (mut scanner, temp_dir) = build_test_scanner().await;
let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir);
scanner.old_cache.info.name = "bucket".to_string();
scanner.old_cache.replace("bucket", "", DataUsageEntry::default());
scanner.old_cache.replace(
"bucket/child",
"bucket",
DataUsageEntry {
compacted: true,
objects: 2,
..Default::default()
},
);
let hash = hash_path("bucket/child");
let folder = CachedFolder {
name: "bucket".to_string(),
parent: None,
object_heal_prob_div: 1,
};
let child = CachedFolder {
name: "bucket/child".to_string(),
parent: Some(hash_path("bucket")),
object_heal_prob_div: 1,
};
let abandoned = HashSet::from([hash.0.clone()]);
let before = global_metrics().scan_observation().snapshot();
assert!(!scanner.should_reuse_clean_root_child(&folder, &DataUsageEntry::default(), &child, &hash, &abandoned));
let missing_scope = global_metrics().scan_observation().snapshot();
assert_eq!(
missing_scope.reuse_decisions["clean_prefix"].not_evaluated["missing_scope"],
before.reuse_decisions["clean_prefix"].not_evaluated["missing_scope"] + 1
);
scanner.prefix_scan_scope = ScannerBucketPrefixScanScope::from_dirty_top_level_entries(HashSet::from(["dirty".to_string()]));
scanner.is_erasure_mode = true;
assert!(!scanner.should_reuse_clean_root_child(&folder, &DataUsageEntry::default(), &child, &hash, &abandoned));
let erasure = global_metrics().scan_observation().snapshot();
assert_eq!(
erasure.reuse_decisions["clean_prefix"].blocked["erasure_health_required"],
before.reuse_decisions["clean_prefix"].blocked["erasure_health_required"] + 1
);
assert_eq!(erasure.erasure_clean_prefix_walks, before.erasure_clean_prefix_walks + 1);
assert!(!scanner.can_reuse_covered_namespace("bucket/child", true));
scanner.old_cache.info.scan_progress = Some(crate::DataUsageScanProgress {
started_plan: crate::DataUsageScanPlanDigest([1; 32]),
requested_plan: crate::DataUsageScanPlanDigest([1; 32]),
});
assert!(!scanner.can_reuse_covered_namespace("bucket/child", true));
assert!(!scanner.can_reuse_covered_namespace("bucket/child", false));
scanner.resume_frontier = Some("bucket/child".to_string());
scanner.resume_failed_paths = vec!["bucket/child/failed".to_string()];
assert!(!scanner.can_reuse_covered_namespace("bucket/child", false));
scanner.resume_failed_paths.clear();
assert!(scanner.can_reuse_covered_namespace("bucket/child", false));
let covered = global_metrics().scan_observation().snapshot();
let reasons = &covered.reuse_decisions["covered_namespace"];
let prior = &before.reuse_decisions["covered_namespace"];
assert_eq!(reasons.not_evaluated["no_resume_sweep"], prior.not_evaluated["no_resume_sweep"] + 1);
assert_eq!(reasons.blocked["compacted_parent"], prior.blocked["compacted_parent"] + 1);
assert_eq!(reasons.blocked["missing_coverage"], prior.blocked["missing_coverage"] + 1);
assert_eq!(reasons.blocked["cached_failures"], prior.blocked["cached_failures"] + 1);
assert_eq!(reasons.eligible, prior.eligible + 1);
with_var(ENV_DATA_USAGE_UPDATE_DIR_CYCLES, Some("16"), || {
scanner.scan_mode = HealScanMode::Deep;
assert_eq!(scanner.sampled_namespace_reuse(&hash, true, true), None);
scanner.scan_mode = HealScanMode::Normal;
assert_eq!(scanner.sampled_namespace_reuse(&hash, true, true), None);
let replication: ReplicationConfig = serde_json::from_value(serde_json::json!({
"config": { "role": "", "rules": [{ "destination": { "bucket": "arn:test" }, "id": "status", "prefix": "", "priority": 1, "status": "Enabled" }] },
"remotes": crate::storage_api::EcstoreBucketTargets::default(),
})).expect("active replication fixture");
scanner.old_cache.info.replication = Some(Arc::new(replication));
assert_eq!(scanner.sampled_namespace_reuse(&hash, true, false), None);
scanner.old_cache.info.replication = None;
assert_eq!(scanner.sampled_namespace_reuse(&hash, true, false), None);
let rescan_cycle = (0..16)
.find(|cycle| hash.mod_(*cycle, 16))
.expect("hash is visited in one sampling cycle");
scanner.old_cache.info.next_cycle = u64::from(rescan_cycle);
assert_eq!(scanner.sampled_namespace_reuse(&hash, false, false), Some(false));
scanner.old_cache.info.next_cycle = u64::from((rescan_cycle + 1) % 16);
assert_eq!(scanner.sampled_namespace_reuse(&hash, false, false), Some(true));
});
let after = global_metrics().scan_observation().snapshot();
let reasons = &after.reuse_decisions["namespace_sampling"];
let prior = &before.reuse_decisions["namespace_sampling"];
for label in ["deep_scan", "forward_sweep", "active_replication"] {
assert_eq!(
reasons.not_evaluated[label],
prior.not_evaluated[label] + 1,
"first sampling condition {label}"
);
}
assert_eq!(reasons.blocked["compacted_parent"], prior.blocked["compacted_parent"] + 1);
assert_eq!(reasons.blocked["selected_for_walk"], prior.blocked["selected_for_walk"] + 1);
assert_eq!(reasons.eligible, prior.eligible + 1);
for (kind, reuse) in &after.cache_reuse {
assert_eq!(reuse.count, before.cache_reuse[kind].count, "gate admission alone does not copy {kind}");
}
}
#[tokio::test]
#[serial]
async fn scoped_root_scan_reuses_clean_top_level_entries_and_rescans_dirty_entries() {
@@ -3144,12 +3250,21 @@ async fn scoped_root_scan_reuses_clean_top_level_entries_and_rescans_dirty_entri
parent: None,
object_heal_prob_div: 1,
};
let before = global_metrics().scan_observation().snapshot();
let mut root = DataUsageEntry::default();
scanner
.scan_folder(CancellationToken::new(), folder, &mut root)
.await
.expect("scoped root scan should finish successfully");
let observed = global_metrics().scan_observation().snapshot();
assert_eq!(observed.cache_reuse["clean_prefix"].count, before.cache_reuse["clean_prefix"].count + 1);
assert_eq!(observed.erasure_clean_prefix_walks, before.erasure_clean_prefix_walks);
assert_eq!(
observed.reuse_decisions["clean_prefix"].eligible,
before.reuse_decisions["clean_prefix"].eligible + 1
);
let clean = scanner
.new_cache
.size_recursive("bucket/clean")
@@ -3167,6 +3282,78 @@ async fn scoped_root_scan_reuses_clean_top_level_entries_and_rescans_dirty_entri
assert_eq!((bucket.size, bucket.objects), (17, 3));
}
#[tokio::test]
#[serial]
async fn scoped_cache_observations_distinguish_checkpoint_and_periodic_sampling() {
for covered in [true, false] {
let (mut scanner, temp_dir) = build_test_scanner().await;
let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir.clone());
tokio::fs::create_dir_all(temp_dir.join("bucket/child"))
.await
.expect("create cached namespace");
scanner.old_cache.info.name = "bucket".to_string();
scanner.new_cache.info.name = "bucket".to_string();
scanner.old_cache.replace("bucket", "", DataUsageEntry::default());
scanner.old_cache.replace(
"bucket/child",
"bucket",
DataUsageEntry {
size: 17,
objects: 5,
compacted: true,
..Default::default()
},
);
let kind = if covered {
scanner.old_cache.info.scan_progress = Some(crate::DataUsageScanProgress {
started_plan: crate::DataUsageScanPlanDigest([1; 32]),
requested_plan: crate::DataUsageScanPlanDigest([1; 32]),
});
scanner.resume_frontier = Some("bucket/child".to_string());
"covered_namespace"
} else {
let hash = hash_path("bucket/child");
let cycle = (0..data_usage_update_dir_cycles())
.find(|cycle| !hash.mod_(*cycle, data_usage_update_dir_cycles()))
.expect("a cycle samples other work");
scanner.old_cache.info.next_cycle = u64::from(cycle);
"sampled_namespace"
};
let before = global_metrics().scan_observation().snapshot();
let mut root = DataUsageEntry::default();
scanner
.scan_folder(
CancellationToken::new(),
CachedFolder {
name: "bucket".to_string(),
parent: None,
object_heal_prob_div: 1,
},
&mut root,
)
.await
.expect("cached namespace reuses accounting");
let summary = scanner.new_cache.size_recursive("bucket").expect("complete bucket totals");
assert_eq!((summary.size, summary.objects), (17, 5));
let after = global_metrics().scan_observation().snapshot();
for label in [
"clean_prefix",
"covered_namespace",
"sampled_namespace",
"classified_namespace",
] {
assert_eq!(
after.cache_reuse[label].count,
before.cache_reuse[label].count + u64::from(label == kind),
"reuse reason {label}"
);
}
assert_eq!(after.metadata_reads.attempts, before.metadata_reads.attempts);
let gate = if covered { "covered_namespace" } else { "namespace_sampling" };
assert_eq!(after.reuse_decisions[gate].eligible, before.reuse_decisions[gate].eligible + 1);
}
}
async fn scan_hot_cold_segment_fixture(scoped: bool) -> (DataUsageEntry, Vec<String>) {
let (mut scanner, temp_dir) = build_test_scanner().await;
let visits = metadata_visits::observe(&mut scanner);
@@ -3278,6 +3465,7 @@ async fn scoped_root_scan_preserves_erasure_health_walks() {
parent: None,
object_heal_prob_div: 1,
};
let before = global_metrics().scan_observation().snapshot();
let mut root = DataUsageEntry::default();
scanner
.scan_folder(CancellationToken::new(), folder, &mut root)
@@ -3289,6 +3477,9 @@ async fn scoped_root_scan_preserves_erasure_health_walks() {
.size_recursive("bucket/clean")
.expect("erasure scan should visit the clean entry");
assert_eq!((clean.size, clean.objects), (0, 0));
let observed = global_metrics().scan_observation().snapshot();
assert_eq!(observed.erasure_clean_prefix_walks, before.erasure_clean_prefix_walks + 1);
assert_eq!(observed.cache_reuse["clean_prefix"].count, before.cache_reuse["clean_prefix"].count);
}
#[tokio::test]
@@ -774,6 +774,7 @@ async fn erasure_candidate_covered_namespace_reuses_totals_after_classification(
},
);
scanner.budget = budget.clone();
let before = global_metrics().scan_observation().snapshot();
scanner
.scan_folder(
budget.token(),
@@ -790,6 +791,15 @@ async fn erasure_candidate_covered_namespace_reuses_totals_after_classification(
assert_eq!(budget.progress(), (0, 1), "only the parent consumes namespace quota");
assert_eq!(charged_ops(&root), 4, "list both directories once without re-reading covered metadata");
assert_eq!(scanner.new_cache.find(&folder.name).expect("reused candidate").objects, 1);
let observed = global_metrics().scan_observation().snapshot();
assert_eq!(
observed.cache_reuse["classified_namespace"].count,
before.cache_reuse["classified_namespace"].count + 1
);
assert_eq!(
observed.metadata_reads.attempts, before.metadata_reads.attempts,
"classification retains the existing zero metadata read path"
);
}
#[tokio::test]
@@ -869,6 +879,7 @@ async fn erasure_candidate_preserves_compacted_namespace_sampling_after_classifi
},
);
scanner.budget = budget.clone();
let before = global_metrics().scan_observation().snapshot();
scanner
.scan_folder(
budget.token(),
@@ -889,6 +900,15 @@ async fn erasure_candidate_preserves_compacted_namespace_sampling_after_classifi
"sampling still requires full current classification, without metadata re-read"
);
assert_eq!(scanner.new_cache.find(&folder.name).expect("sampled candidate").objects, 1);
let observed = global_metrics().scan_observation().snapshot();
assert_eq!(
observed.cache_reuse["classified_namespace"].count,
before.cache_reuse["classified_namespace"].count + 1
);
assert_eq!(
observed.metadata_reads.attempts, before.metadata_reads.attempts,
"classification retains the existing zero metadata read path"
);
}
#[tokio::test]
+133 -46
View File
@@ -40,6 +40,7 @@ use rustfs_lock::{LockError, NamespaceLockGuard};
use rustfs_scanner_metrics::metrics::{
Metric, Metrics, emit_scan_bucket_drive_complete, emit_scan_bucket_drive_partial, global_metrics,
};
use rustfs_scanner_metrics::scan_observation::{PREFIX_REUSE_BLOCKERS, ScannerBaselineRejection, ScannerPrefixReuseStage};
use rustfs_utils::path::path_join_buf;
use sha2::{Digest as _, Sha256};
use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
@@ -112,16 +113,7 @@ pub(crate) const SCANNER_SEGMENT_ACTIVATION_PROOF_INPUTS: [&str; 9] = [
"durable_dirty_producer_journal",
"ack_generation_guard",
];
pub(crate) const SCANNER_SEGMENT_ACTIVATION_FAIL_CLOSED_CHECKS: [&str; 8] = [
"missing_producer_identity",
"missing_durable_journal_replay",
"restart_gap",
"generation_gap",
"overflow",
"stale_ack_generation",
"missing_cold_zero_walk_oracle",
"distributed_without_peer_invalidation",
];
pub(crate) const SCANNER_SEGMENT_ACTIVATION_FAIL_CLOSED_CHECKS: [&str; 8] = PREFIX_REUSE_BLOCKERS;
pub type DirtyUsageBuckets = HashMap<String, u64>;
@@ -197,6 +189,7 @@ struct VerifiedRemoteDirtyUsage {
}
struct ScannerBucketScopeResolutionResult {
segment_reuse_activation_preflight: Option<ScannerSegmentReuseActivationPreflight>,
scope: ScannerBucketScanScope,
remote_dirty_usage_acknowledgements: Vec<crate::scanner::ScannerDirtyUsageAcknowledgement>,
distributed_segment_invalidation_evidence: Option<DistributedSegmentInvalidationEvidence>,
@@ -236,6 +229,13 @@ pub(crate) struct ScannerSegmentReuseActivationPreflight {
}
impl ScannerSegmentReuseActivationPreflight {
fn record_observation(&self, stage: ScannerPrefixReuseStage) {
global_metrics().scan_observation().record_prefix_reuse(
stage,
self.scanner_segment_reuse_activated,
&self.fail_closed_blockers,
);
}
#[cfg(test)]
pub(crate) fn fail_closed_blockers(&self) -> impl Iterator<Item = &'static str> + '_ {
self.fail_closed_blockers.iter().filter_map(|blocker| *blocker)
@@ -319,6 +319,7 @@ fn resolve_remote_dirty_usage_scope(
baseline_proof: ScannerCacheBaselineProof<'_>,
) -> ScannerBucketScopeResolutionResult {
let default_result = |scope: ScannerBucketScanScope| ScannerBucketScopeResolutionResult {
segment_reuse_activation_preflight: None,
scope,
remote_dirty_usage_acknowledgements: Vec::new(),
distributed_segment_invalidation_evidence: None,
@@ -410,6 +411,7 @@ fn resolve_remote_dirty_usage_scope(
};
ScannerBucketScopeResolutionResult {
segment_reuse_activation_preflight: Some(segment_reuse_activation_preflight),
scope,
remote_dirty_usage_acknowledgements: scoped_acknowledgements,
distributed_segment_invalidation_evidence,
@@ -427,43 +429,81 @@ fn complete_scanner_cache_snapshot_plan_digest(
proof: ScannerCacheBaselineProof<'_>,
expected_converged: Option<bool>,
) -> Option<DataUsageScanPlanDigest> {
validate_scanner_cache_snapshot_plan_digest(snapshot, proof, expected_converged).ok()
}
fn validate_scanner_cache_snapshot_plan_digest(
snapshot: &DataUsageInfo,
proof: ScannerCacheBaselineProof<'_>,
expected_converged: Option<bool>,
) -> std::result::Result<DataUsageScanPlanDigest, ScannerBaselineRejection> {
// The observational mark disqualifies only the authoritative slot: an
// observed candidate carries it by construction.
let marks_accepted = match expected_converged {
Some(expected) => snapshot.usage_snapshot_converged == Some(expected),
None => snapshot.usage_snapshot_converged.is_some() && !snapshot.usage_snapshot_observational,
};
if !snapshot.is_complete_bucket_usage_snapshot()
|| snapshot.usage_snapshot_partial
|| !marks_accepted
|| snapshot.scanner_epoch != Some(proof.leader_epoch)
|| snapshot.usage_snapshot_set_states.len() != proof.expected_sources.len()
{
return None;
if !snapshot.is_complete_bucket_usage_snapshot() {
return Err(ScannerBaselineRejection::Incomplete);
}
if snapshot.usage_snapshot_partial {
return Err(ScannerBaselineRejection::Partial);
}
if !marks_accepted {
return Err(ScannerBaselineRejection::Marks);
}
if snapshot.scanner_epoch != Some(proof.leader_epoch) {
return Err(ScannerBaselineRejection::Epoch);
}
if snapshot.usage_snapshot_set_states.len() != proof.expected_sources.len() {
return Err(ScannerBaselineRejection::SourceCount);
}
// Completed maintenance also covers ordinary usage. Keep its exact stored
// proof for cache reuse, and reject mixtures of different set work proofs.
let baseline_plan_digest = DataUsageScanPlanDigest(snapshot.usage_snapshot_set_states.first()?.scan_plan_digest?);
let baseline_plan_digest = DataUsageScanPlanDigest(
snapshot
.usage_snapshot_set_states
.first()
.and_then(|state| state.scan_plan_digest)
.ok_or(ScannerBaselineRejection::PlanMissing)?,
);
if !scanner_plan_digest_is_work_variant(proof.scan_plan_digest, baseline_plan_digest) {
return None;
return Err(ScannerBaselineRejection::PlanMismatch);
}
let mut states = HashSet::with_capacity(snapshot.usage_snapshot_set_states.len());
for state in &snapshot.usage_snapshot_set_states {
let source = DataUsageCacheSource::new(usize::try_from(state.pool_index).ok()?, usize::try_from(state.set_index).ok()?);
if !proof.expected_sources.contains(&source)
|| !states.insert(source)
|| !state.complete
|| state.tombstone
|| state.scanner_epoch != Some(proof.leader_epoch)
|| state.scanner_cycle.is_none_or(|cycle| cycle > proof.want_cycle)
|| state.scan_plan_digest != Some(baseline_plan_digest.0)
{
return None;
let source = DataUsageCacheSource::new(
usize::try_from(state.pool_index).map_err(|_| ScannerBaselineRejection::SourceIndex)?,
usize::try_from(state.set_index).map_err(|_| ScannerBaselineRejection::SourceIndex)?,
);
if !proof.expected_sources.contains(&source) {
return Err(ScannerBaselineRejection::SourceMembership);
}
if !states.insert(source) {
return Err(ScannerBaselineRejection::SourceDuplicate);
}
if !state.complete {
return Err(ScannerBaselineRejection::SetIncomplete);
}
if state.tombstone {
return Err(ScannerBaselineRejection::SetTombstone);
}
if state.scanner_epoch != Some(proof.leader_epoch) {
return Err(ScannerBaselineRejection::SetEpoch);
}
if state.scanner_cycle.is_none_or(|cycle| cycle > proof.want_cycle) {
return Err(ScannerBaselineRejection::SetCycle);
}
if state.scan_plan_digest != Some(baseline_plan_digest.0) {
return Err(ScannerBaselineRejection::SetPlan);
}
}
(states == *proof.expected_sources).then_some(baseline_plan_digest)
if states != *proof.expected_sources {
return Err(ScannerBaselineRejection::SourceCoverage);
}
Ok(baseline_plan_digest)
}
/// Whether `digest` is `structural` under any work flag: the same bucket
@@ -493,10 +533,35 @@ fn scanner_baseline_pending_buckets(proof: &ScannerCacheBaselineProof<'_>) -> Ve
}
fn complete_scanner_cache_baseline_plan_digest(proof: ScannerCacheBaselineProof<'_>) -> Option<DataUsageScanPlanDigest> {
let authoritative = serde_json::from_slice::<DataUsageInfo>(proof.authoritative_data?).ok()?;
if let Some(validated_digest) = complete_scanner_cache_snapshot_plan_digest(&authoritative, proof, None) {
return Some(validated_digest);
}
validate_scanner_cache_baseline_plan_digest(proof).ok()
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
struct ScannerBaselineValidationFailure {
authoritative: ScannerBaselineRejection,
observed: Option<ScannerBaselineRejection>,
}
fn validate_scanner_cache_baseline_plan_digest(
proof: ScannerCacheBaselineProof<'_>,
) -> std::result::Result<DataUsageScanPlanDigest, ScannerBaselineValidationFailure> {
let authoritative_error = |authoritative| ScannerBaselineValidationFailure {
authoritative,
observed: None,
};
let authoritative_data = proof
.authoritative_data
.ok_or_else(|| authoritative_error(ScannerBaselineRejection::Missing))?;
let authoritative = serde_json::from_slice::<DataUsageInfo>(authoritative_data)
.map_err(|_| authoritative_error(ScannerBaselineRejection::Decode))?;
let authoritative_rejection = match validate_scanner_cache_snapshot_plan_digest(&authoritative, proof, None) {
Ok(validated_digest) => return Ok(validated_digest),
Err(reason) => reason,
};
let observed_error = |observed| ScannerBaselineValidationFailure {
authoritative: authoritative_rejection,
observed: Some(observed),
};
// A complete but superseded observation may reuse its per-set cache only
// when it was explicitly tied to the durable authoritative baseline. It
@@ -506,14 +571,18 @@ fn complete_scanner_cache_baseline_plan_digest(proof: ScannerCacheBaselineProof<
&& !authoritative.usage_snapshot_observational)
|| crate::scanner::data_usage_info_is_bootstrap_pending(&authoritative);
if !authoritative_has_identity {
return None;
return Err(observed_error(ScannerBaselineRejection::AuthoritativeIdentity));
}
let observed = serde_json::from_slice::<DataUsageInfo>(proof.observed_candidate_data?).ok()?;
let observed_data = proof
.observed_candidate_data
.ok_or_else(|| observed_error(ScannerBaselineRejection::Missing))?;
let observed =
serde_json::from_slice::<DataUsageInfo>(observed_data).map_err(|_| observed_error(ScannerBaselineRejection::Decode))?;
if !observed_data_usage_is_newer(&observed, &authoritative) {
return None;
return Err(observed_error(ScannerBaselineRejection::NotNewer));
}
complete_scanner_cache_snapshot_plan_digest(&observed, proof, Some(false))
validate_scanner_cache_snapshot_plan_digest(&observed, proof, Some(false)).map_err(observed_error)
}
fn scanner_segment_reuse_baseline_producer_evidence(
@@ -658,6 +727,12 @@ impl ScannerScopeFallbackReason {
}
}
#[derive(Clone, Copy)]
struct ScannerScopeFallback {
reason: ScannerScopeFallbackReason,
baseline_rejection: Option<ScannerBaselineValidationFailure>,
}
/// Classify a resolved full-namespace scope against the preconditions
/// `resolve_scanner_bucket_scan_scope` checks, in the same order. Returns
/// `None` when the cycle was required to walk everything or was scoped.
@@ -668,18 +743,24 @@ fn scanner_scope_fallback_reason(
all_buckets: &[BucketInfo],
baseline_proof: ScannerCacheBaselineProof<'_>,
distributed: bool,
) -> Option<ScannerScopeFallbackReason> {
) -> Option<ScannerScopeFallback> {
let fallback = |reason| {
Some(ScannerScopeFallback {
reason,
baseline_rejection: None,
})
};
if !scope_eligible || !resolved_scope.is_default() {
return None;
}
if dirty_usage_snapshot.generation == u64::MAX {
return Some(ScannerScopeFallbackReason::DirtyGenerationExhausted);
return fallback(ScannerScopeFallbackReason::DirtyGenerationExhausted);
}
if !dirty_usage_snapshot.covers_listed_pending {
return Some(ScannerScopeFallbackReason::DirtySnapshotIncomplete);
return fallback(ScannerScopeFallbackReason::DirtySnapshotIncomplete);
}
if dirty_usage_snapshot.buckets.len() > crate::SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES {
return Some(ScannerScopeFallbackReason::DirtyBucketOverflow);
return fallback(ScannerScopeFallbackReason::DirtyBucketOverflow);
}
// Peers may contribute dirty buckets the local snapshot does not hold.
if !distributed {
@@ -689,13 +770,19 @@ fn scanner_scope_fallback_reason(
.keys()
.any(|bucket| listed.contains(bucket.as_str()))
{
return Some(ScannerScopeFallbackReason::NoListedDirtyBuckets);
return fallback(ScannerScopeFallbackReason::NoListedDirtyBuckets);
}
}
if complete_scanner_cache_baseline_plan_digest(baseline_proof).is_none() {
return Some(ScannerScopeFallbackReason::BaselineUnavailable);
if let Err(failure) = validate_scanner_cache_baseline_plan_digest(baseline_proof) {
return Some(ScannerScopeFallback {
reason: ScannerScopeFallbackReason::BaselineUnavailable,
baseline_rejection: Some(failure),
});
}
distributed.then_some(ScannerScopeFallbackReason::PeerScopeUnavailable)
distributed.then_some(ScannerScopeFallback {
reason: ScannerScopeFallbackReason::PeerScopeUnavailable,
baseline_rejection: None,
})
}
fn scanner_segment_reuse_activation_preflight() -> ScannerSegmentReuseActivationPreflight {
+11 -7
View File
@@ -13,7 +13,7 @@
// limitations under the License.
use super::*;
/// scanner cache locks and the cache snapshot persist/publish path.
use crate::data_usage_define::clone_cache_value;
use crate::data_usage_define::{DataUsageCacheBaselineRevisions, clone_cache_value};
use crate::{ScannerConfigObjectDelete, ScannerObjectIO};
use rustfs_scanner_metrics::usage_candidate::{
ScannerUsageCandidateKind as CandidateKind, ScannerUsageCandidateOutcome as CandidateOutcome,
@@ -1570,7 +1570,7 @@ pub(super) async fn persist_and_publish_cache_snapshot(
store: Arc<SetDisks>,
updates: &mpsc::Sender<DataUsageCache>,
mut cache_snapshot: DataUsageCache,
initial_revisions: Option<&DataUsageCacheRevisions>,
initial_revisions: Option<&DataUsageCacheBaselineRevisions>,
cache_cycle_floor: &AtomicU64,
expected_publication_epoch: u64,
) -> Option<SystemTime> {
@@ -1612,7 +1612,10 @@ pub(super) async fn persist_and_publish_cache_snapshot(
};
let mut persisted = DataUsageCache::default();
let revisions = match persisted.load_with_revisions(store.clone(), DATA_USAGE_CACHE_NAME).await {
let current_baseline = match persisted
.load_with_baseline_revisions(store.clone(), DATA_USAGE_CACHE_NAME)
.await
{
Ok(revisions) => revisions,
Err(err) => {
error!(
@@ -1634,6 +1637,7 @@ pub(super) async fn persist_and_publish_cache_snapshot(
return None;
}
};
let revisions = current_baseline.revisions();
let Some(scan_plan_digest) = cache_snapshot.info.scan_plan_digest else {
log_missing_set_snapshot_metadata(&store, &cache_snapshot.info, "plan_digest");
return None;
@@ -1697,9 +1701,9 @@ pub(super) async fn persist_and_publish_cache_snapshot(
{
cache_snapshot = persisted;
} else {
// A later execution may have completed while this scan was walking.
// Only replace the cache revision from which this scan started.
if initial_revisions != Some(&revisions) {
// Another execution must not advance the baseline. A backup catching
// up to the same decoded primary changes no baseline content.
if !initial_revisions.is_some_and(|initial| current_baseline.permits_unchanged_baseline_since(initial)) {
warn!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
@@ -1741,7 +1745,7 @@ pub(super) async fn persist_and_publish_cache_snapshot(
.save_with_revisions_for_epoch(
store.clone(),
DATA_USAGE_CACHE_NAME,
&revisions,
revisions,
expected_publication_epoch,
CancellationToken::new(),
)
+48 -3
View File
@@ -14,6 +14,7 @@
use super::*;
/// ScannerIOCache implementation for SetDisks: bucket ordering, worker fan-out, merge, and publish.
use crate::data_usage_define::clone_cache_value;
use rustfs_scanner_metrics::scan_observation::{ScannerBucketReuseKind, ScannerScanObservation};
pub(super) async fn collect_completed_bucket_results(
ctx: CancellationToken,
@@ -142,6 +143,32 @@ pub(super) struct PreparedScopedSetScan {
pub(super) buckets: Vec<BucketInfo>,
pub(super) cache: DataUsageCache,
pub(super) cold_bucket_reuse_proof: Option<ScopedColdBucketReuseProof>,
cold_bucket_reuse_count: u64,
cold_bucket_reuse_elapsed: Duration,
}
impl PreparedScopedSetScan {
pub(super) fn record_cold_bucket_reuse(&self, observation: &ScannerScanObservation) {
observation.record_bucket_reuse(
ScannerBucketReuseKind::ScopedColdBucket,
self.cold_bucket_reuse_count,
self.cold_bucket_reuse_elapsed,
);
}
}
/// Counts only roots delivered to the collector; cancellation or channel failure is not reuse.
pub(super) async fn send_current_cache_root_entry(
bucket_result_tx: &mpsc::Sender<DataUsageEntryInfo>,
root: DataUsageEntryInfo,
cache: &DataUsageCache,
pending_maintenance_work: &AtomicBool,
observation: &ScannerScanObservation,
) -> std::result::Result<(), ScannerError> {
let started = Instant::now();
send_cache_root_entry(bucket_result_tx, root, cache, pending_maintenance_work).await?;
observation.record_bucket_reuse(ScannerBucketReuseKind::CurrentRoot, 1, started.elapsed());
Ok(())
}
pub(super) struct ScopedColdBucketReuseProof {
@@ -225,6 +252,8 @@ pub(super) fn prepare_scoped_set_scan(
};
cache.replace(DATA_USAGE_ROOT, "", DataUsageEntry::default());
let mut current_bucket_names = HashSet::with_capacity(all_buckets.len());
let mut cold_bucket_reuse_count = 0u64;
let mut cold_bucket_reuse_elapsed = Duration::ZERO;
for bucket in all_buckets {
if !current_bucket_names.insert(bucket.name.as_str()) {
return None;
@@ -233,11 +262,14 @@ pub(super) fn prepare_scoped_set_scan(
cache.info.scan_bucket_value_origins.remove(&bucket.name);
cache.replace(&bucket.name, DATA_USAGE_ROOT, DataUsageEntry::default());
} else {
let started = Instant::now();
cache.copy_with_children(
old_cache,
&rustfs_data_usage::hash_path(&bucket.name),
&Some(rustfs_data_usage::hash_path(DATA_USAGE_ROOT)),
);
cold_bucket_reuse_count = cold_bucket_reuse_count.saturating_add(1);
cold_bucket_reuse_elapsed = cold_bucket_reuse_elapsed.saturating_add(started.elapsed());
}
}
@@ -249,6 +281,8 @@ pub(super) fn prepare_scoped_set_scan(
.collect(),
cache,
cold_bucket_reuse_proof,
cold_bucket_reuse_count,
cold_bucket_reuse_elapsed,
})
}
@@ -426,7 +460,10 @@ impl ScannerIOCache for SetDisks {
.ok_or_else(|| StorageError::other("scanner cache publication is blocked by data movement"))?,
};
let mut old_cache = DataUsageCache::default();
let initial_revisions = match old_cache.load_with_revisions(self.clone(), DATA_USAGE_CACHE_NAME).await {
let initial_revisions = match old_cache
.load_with_baseline_revisions(self.clone(), DATA_USAGE_CACHE_NAME)
.await
{
Ok(revisions) => Some(revisions),
Err(e) => {
warn!(
@@ -556,6 +593,8 @@ impl ScannerIOCache for SetDisks {
})
});
let mut scoped_cache = scoped_scan.map(|mut prepared| {
// The copies are completed work even if this turn later fails publication.
prepared.record_cold_bucket_reuse(global_metrics().scan_observation());
buckets = prepared.buckets;
prepared.cache.info.scan_coverage_digest = Some(bucket_coverage_digest);
prepared.cache
@@ -1560,8 +1599,14 @@ impl ScannerIOCache for SetDisks {
}
record_completed_usage_watch(&usage_watch_completions_clone, source, &bucket.name, bucket_usage_watch.as_ref()).await;
if let Err(e) =
send_cache_root_entry(&bucket_result_tx_clone, *root, &cache, &pending_maintenance_work_clone)
.await
send_current_cache_root_entry(
&bucket_result_tx_clone,
*root,
&cache,
&pending_maintenance_work_clone,
global_metrics().scan_observation(),
)
.await
{
record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
error!(
+20 -3
View File
@@ -357,6 +357,7 @@ where
S: ScannerStorage,
{
let default_result = |scope: ScannerBucketScanScope| ScannerBucketScopeResolutionResult {
segment_reuse_activation_preflight: None,
scope,
remote_dirty_usage_acknowledgements: Vec::new(),
distributed_segment_invalidation_evidence: None,
@@ -506,7 +507,7 @@ where
distributed,
resolution.baseline_proof,
);
default_result(scoped_scan_scope_from_dirty_buckets(
let scope = scoped_scan_scope_from_dirty_buckets(
resolution.requested_scope,
dirty_buckets,
(!distributed).then_some(resolution.dirty_usage_snapshot.scopes.as_ref()),
@@ -514,7 +515,11 @@ where
segment_reuse_activation_preflight.scanner_segment_reuse_activated,
resolution.all_buckets,
resolution.baseline_proof,
))
);
ScannerBucketScopeResolutionResult {
segment_reuse_activation_preflight: Some(segment_reuse_activation_preflight),
..default_result(scope)
}
}
/// Runs once between the bucket listing and the dirty-usage snapshot, the
@@ -775,6 +780,9 @@ where
},
)
.await;
if let Some(preflight) = scope_resolution.segment_reuse_activation_preflight {
preflight.record_observation(ScannerPrefixReuseStage::ScopeResolution);
}
let remote_dirty_usage_acknowledgements = scope_resolution.remote_dirty_usage_acknowledgements;
let distributed_segment_invalidation_evidence = scope_resolution.distributed_segment_invalidation_evidence;
let scan_scope = scope_resolution.scope;
@@ -786,8 +794,14 @@ where
baseline_proof,
distributed,
);
if let Some(reason) = scope_fallback {
if let Some(fallback) = scope_fallback {
let reason = fallback.reason;
global_metrics().record_scanner_scope_fallback(reason.as_str());
if let Some(failure) = fallback.baseline_rejection {
global_metrics()
.scan_observation()
.record_baseline_rejection(failure.authoritative, failure.observed);
}
tracing::info!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_SET_STATE,
@@ -801,6 +815,7 @@ where
"Scanner cycle walks every bucket because its dirty-bucket scope was not admitted"
);
}
let scope_fallback = scope_fallback.map(|fallback| fallback.reason);
let segment_invalidation_proof = dirty_usage_producer_evidence(&dirty_usage_snapshot).segment_invalidation_proof();
#[cfg(test)]
if let Some(observer) = resolved_scope_observer {
@@ -877,6 +892,7 @@ where
None,
false,
);
segment_reuse_activation_preflight.record_observation(ScannerPrefixReuseStage::CycleCompletion);
return Ok(ScannerCycleResult::new(status, dirty_usage_clear)
.with_publication_epoch(publication_epoch)
.with_activity_digest(activity_digest)
@@ -1224,6 +1240,7 @@ where
distributed_segment_invalidation_evidence,
cold_zero_walk_oracle,
);
segment_reuse_activation_preflight.record_observation(ScannerPrefixReuseStage::CycleCompletion);
let pending_maintenance_work = pending_maintenance_work_for_cycle(&pending_maintenance_work, &results);
let observed_cycle_floor = cache_cycle_floor.load(Ordering::Acquire);
let required_cycle_floor = (observed_cycle_floor > want_cycle).then_some(observed_cycle_floor);
+15 -2
View File
@@ -15,6 +15,7 @@ use super::*;
use crate::UNKNOWN_TIER;
/// ScannerIODisk implementation for Disk: get_size and the per-disk bucket scan.
use crate::data_usage_define::clone_cache_value;
use rustfs_scanner_metrics::scan_observation::ScannerMetadataReadOutcome;
///
/// Seed [`SizeSummary::tier_stats`] from the cached tier-name list.
@@ -62,8 +63,20 @@ impl ScannerIODisk for Disk {
ensure_scanner_node_operation_active(cancellation.as_ref())?;
let metadata_object_path = item.object_path();
let metadata_result =
crate::workload_admission::with_scanner_metadata_io(self.read_metadata(&item.bucket, &metadata_object_path)).await;
let metadata_result = crate::workload_admission::with_scanner_metadata_io(async {
let read = global_metrics().start_scanner_metadata_read();
let result = self.read_metadata(&item.bucket, &metadata_object_path).await;
let (outcome, bytes) = match &result {
Ok(data) => (ScannerMetadataReadOutcome::Success, u64::try_from(data.len()).unwrap_or(u64::MAX)),
Err(e) if DiskError::is_err_object_not_found(e) || DiskError::is_err_version_not_found(e) => {
(ScannerMetadataReadOutcome::NotFound, 0)
}
Err(_) => (ScannerMetadataReadOutcome::Error, 0),
};
read.finish(outcome, bytes);
result
})
.await;
ensure_scanner_node_operation_active(cancellation.as_ref())?;
let data = match metadata_result {
Ok(data) => data,
+564 -7
View File
@@ -27,10 +27,10 @@ use crate::storage_api::owner::{
};
use crate::storage_api::scan::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions, ObjectIO as _};
use crate::{
DATA_USAGE_BLOOM_NAME_PATH, DiskOption, ECStore, Endpoint, EndpointServerPools, Endpoints, InstanceContext, PoolEndpoints,
ScannerObjectOptions, ScannerPutObjReader, UNKNOWN_TIER, init_bucket_metadata_sys_for_scanner_tests,
init_ecstore_config_for_scanner_tests, init_local_disks_with_instance_ctx, new_disk, path2_bucket_object_with_base_path,
save_config,
BUCKET_META_PREFIX, DATA_USAGE_BLOOM_NAME_PATH, DiskOption, ECStore, Endpoint, EndpointServerPools, Endpoints,
InstanceContext, PoolEndpoints, ScannerObjectOptions, ScannerPutObjReader, UNKNOWN_TIER,
init_bucket_metadata_sys_for_scanner_tests, init_ecstore_config_for_scanner_tests, init_local_disks_with_instance_ctx,
new_disk, path2_bucket_object_with_base_path, save_config,
};
use rustfs_concurrency::{
AdmissionState, WorkloadAdmissionRegistrySnapshot, WorkloadAdmissionSnapshot, WorkloadAdmissionSnapshotProvider,
@@ -104,6 +104,7 @@ fn scanner_activity_preflight_defers_a_temporarily_offline_peer() {
#[test]
fn scanner_segment_reuse_activation_preflight_reports_release_gate_inputs() {
let before = global_metrics().scan_observation().snapshot();
let preflight = scanner_segment_reuse_activation_preflight();
assert!(preflight.production_activation);
@@ -124,6 +125,9 @@ fn scanner_segment_reuse_activation_preflight_reports_release_gate_inputs() {
"distributed_without_peer_invalidation",
]
);
let after = global_metrics().scan_observation().snapshot();
assert_eq!(after.prefix_scope_resolution.evaluated, before.prefix_scope_resolution.evaluated);
assert_eq!(after.prefix_cycle_completion.evaluated, before.prefix_cycle_completion.evaluated);
}
#[test]
@@ -2487,6 +2491,260 @@ async fn set_cache_send_failure_logs_receiver_and_snapshot_without_changing_retu
assert_set_cache_context(set_cache_event(&events, "publish_failed"), Some((0, 0)));
}
#[tokio::test]
#[serial]
async fn set_snapshot_accepts_backup_catchup_to_the_same_decoded_primary() {
use crate::data_usage_define::read_config_with_revision;
let (_temp_dir, store, cache, epoch) = set_cache_log_fixture().await;
let set = Arc::clone(&store.pools[0].disk_set[0]);
let primary_path = path_join_buf(&[BUCKET_META_PREFIX, DATA_USAGE_CACHE_NAME]);
let backup_path = format!("{primary_path}.bkp");
let (primary_body, _) = read_config_with_revision(Arc::clone(&set), &primary_path)
.await
.expect("read decoded primary bytes");
let primary_body = primary_body.expect("primary exists");
let mut older = cache.clone();
older.info.scan_execution_digest = Some(DataUsageScanPlanDigest([8; 32]));
save_config(Arc::clone(&set), &backup_path, older.marshal_msg().expect("encode older backup"))
.await
.expect("seed a lagging backup");
let mut loaded = DataUsageCache::default();
let initial = loaded
.load_with_baseline_revisions(Arc::clone(&set), DATA_USAGE_CACHE_NAME)
.await
.expect("capture primary with lagging backup");
save_config(Arc::clone(&set), &backup_path, primary_body)
.await
.expect("backup catches up without changing primary");
let current = loaded
.load_with_baseline_revisions(Arc::clone(&set), DATA_USAGE_CACHE_NAME)
.await
.expect("capture the reconciled backup");
assert_ne!(initial.revisions(), current.revisions());
assert!(current.permits_unchanged_baseline_since(&initial));
let mut candidate = cache;
candidate.info.next_cycle += 1;
candidate.info.scan_execution_digest = Some(DataUsageScanPlanDigest([3; 32]));
candidate.replace(
"photos",
DATA_USAGE_ROOT,
DataUsageEntry {
size: 20,
objects: 1,
..Default::default()
},
);
let cycle = candidate.info.next_cycle;
let cycle_floor = AtomicU64::new(cycle);
let (tx, mut rx) = mpsc::channel(1);
assert!(
persist_and_publish_cache_snapshot(Arc::clone(&set), &tx, candidate, Some(&initial), &cycle_floor, epoch)
.await
.is_some(),
"content-preserving backup catchup must not veto the new snapshot"
);
let published = rx.try_recv().expect("new snapshot reached the collector");
assert_eq!(published.info.next_cycle, cycle);
assert_eq!(published.find("photos").expect("new bucket entry").size, 20);
let (primary, primary_revision) = read_config_with_revision(Arc::clone(&set), &primary_path)
.await
.expect("read new primary");
let (backup, backup_revision) = read_config_with_revision(set, &backup_path).await.expect("read new backup");
assert_eq!(primary, backup, "backup CAS must use the reconciled current revision");
assert_eq!(primary_revision, backup_revision);
let persisted = DataUsageCache::unmarshal(&primary.expect("new primary body")).expect("decode new primary");
assert_eq!(persisted.info.next_cycle, cycle);
assert_eq!(persisted.find("photos").expect("persisted new entry").size, 20);
}
#[tokio::test]
#[serial]
async fn set_snapshot_backup_catchup_keeps_revision_provenance_and_epoch_fences() {
use crate::data_usage_define::read_config_with_revision;
for case in ["different_backup", "different_primary", "corrupt_primary", "changed_epoch"] {
let (_temp_dir, store, cache, epoch) = set_cache_log_fixture().await;
let set = Arc::clone(&store.pools[0].disk_set[0]);
let primary_path = path_join_buf(&[BUCKET_META_PREFIX, DATA_USAGE_CACHE_NAME]);
let backup_path = format!("{primary_path}.bkp");
let (primary_body, _) = read_config_with_revision(Arc::clone(&set), &primary_path)
.await
.expect("read original primary");
let primary_body = primary_body.expect("original primary exists");
let mut older = cache.clone();
older.info.scan_execution_digest = Some(DataUsageScanPlanDigest([8; 32]));
save_config(Arc::clone(&set), &backup_path, older.marshal_msg().expect("encode lagging backup"))
.await
.expect("seed lagging backup");
if case == "corrupt_primary" {
save_config(Arc::clone(&set), &primary_path, b"corrupt-primary".to_vec())
.await
.expect("inject primary corruption before the initial load");
}
let mut loaded = DataUsageCache::default();
let initial = loaded
.load_with_baseline_revisions(Arc::clone(&set), DATA_USAGE_CACHE_NAME)
.await
.expect("capture original baseline source and revisions");
let mut replacement = cache.clone();
replacement.info.scan_execution_digest = Some(DataUsageScanPlanDigest([9; 32]));
match case {
"different_backup" => {
save_config(Arc::clone(&set), &backup_path, replacement.marshal_msg().expect("encode changed backup"))
.await
.expect("backup changes to different contents");
}
"different_primary" => {
let body = replacement.marshal_msg().expect("encode changed primary");
save_config(Arc::clone(&set), &primary_path, body.clone())
.await
.expect("primary advances");
save_config(Arc::clone(&set), &backup_path, body)
.await
.expect("backup follows changed primary");
}
"corrupt_primary" => {
save_config(Arc::clone(&set), &backup_path, b"corrupt-primary".to_vec())
.await
.expect("backup matches the corrupt primary ETag without a decoded primary proof");
}
"changed_epoch" => {
save_config(Arc::clone(&set), &backup_path, primary_body)
.await
.expect("backup catches up before epoch refusal");
}
_ => unreachable!("fixed test cases"),
}
let before_primary = read_config_with_revision(Arc::clone(&set), &primary_path)
.await
.expect("current primary");
let before_backup = read_config_with_revision(Arc::clone(&set), &backup_path)
.await
.expect("current backup");
let mut candidate = cache;
candidate.info.scan_execution_digest = Some(DataUsageScanPlanDigest([3; 32]));
let cycle_floor = AtomicU64::new(candidate.info.next_cycle);
let requested_epoch = if case == "changed_epoch" {
epoch.checked_add(1).expect("fixture epoch can advance")
} else {
epoch
};
let (tx, mut rx) = mpsc::channel(1);
let (result, events) = capture_set_cache_logs(persist_and_publish_cache_snapshot(
Arc::clone(&set),
&tx,
candidate,
Some(&initial),
&cycle_floor,
requested_epoch,
))
.await;
assert!(result.is_none(), "{case} must still refuse publication");
assert!(
matches!(rx.try_recv(), Err(mpsc::error::TryRecvError::Empty)),
"{case} forwards no snapshot"
);
let expected_state = if case == "changed_epoch" {
"save_failed"
} else {
"scan_baseline_revision_changed"
};
assert_set_cache_context(set_cache_event(&events, expected_state), Some((0, 0)));
assert_eq!(
read_config_with_revision(Arc::clone(&set), &primary_path)
.await
.expect("protected primary"),
before_primary,
"{case} does not overwrite primary"
);
assert_eq!(
read_config_with_revision(set, &backup_path).await.expect("protected backup"),
before_backup,
"{case} does not overwrite backup"
);
}
}
#[tokio::test]
#[serial]
async fn set_cache_cas_preserves_updates_after_backup_catchup_validation() {
use crate::data_usage_define::read_config_with_revision;
for changed_copy in ["primary", "backup"] {
let (_temp_dir, store, cache, epoch) = set_cache_log_fixture().await;
let set = Arc::clone(&store.pools[0].disk_set[0]);
let primary_path = path_join_buf(&[BUCKET_META_PREFIX, DATA_USAGE_CACHE_NAME]);
let backup_path = format!("{primary_path}.bkp");
let (primary_body, _) = read_config_with_revision(Arc::clone(&set), &primary_path)
.await
.expect("read initial primary");
let mut lagging = cache.clone();
lagging.info.scan_execution_digest = Some(DataUsageScanPlanDigest([8; 32]));
save_config(Arc::clone(&set), &backup_path, lagging.marshal_msg().expect("lagging backup bytes"))
.await
.expect("seed backup lag");
let mut loaded = DataUsageCache::default();
let initial = loaded
.load_with_baseline_revisions(Arc::clone(&set), DATA_USAGE_CACHE_NAME)
.await
.expect("read initial baseline");
save_config(Arc::clone(&set), &backup_path, primary_body.expect("initial primary exists"))
.await
.expect("finish backup catchup");
let current = loaded
.load_with_baseline_revisions(Arc::clone(&set), DATA_USAGE_CACHE_NAME)
.await
.expect("read reconciled baseline");
assert!(current.permits_unchanged_baseline_since(&initial));
let changed_path = if changed_copy == "primary" {
&primary_path
} else {
&backup_path
};
let mut concurrent = cache.clone();
concurrent.info.scan_execution_digest = Some(DataUsageScanPlanDigest([9; 32]));
let concurrent_body = concurrent.marshal_msg().expect("encode concurrent update");
save_config(Arc::clone(&set), changed_path, concurrent_body.clone())
.await
.expect("copy changes after the baseline validation");
let mut candidate = cache;
candidate.info.next_cycle += 1;
candidate.info.scan_execution_digest = Some(DataUsageScanPlanDigest([3; 32]));
let result = candidate
.save_with_revisions_for_epoch(
Arc::clone(&set),
DATA_USAGE_CACHE_NAME,
current.revisions(),
epoch,
CancellationToken::new(),
)
.await;
if changed_copy == "primary" {
assert!(result.is_err(), "primary CAS refuses the intervening update");
} else {
result.expect("primary remains durable when the best-effort backup CAS refuses an update");
}
let (protected, _) = read_config_with_revision(Arc::clone(&set), changed_path)
.await
.expect("read protected concurrent copy");
assert_eq!(
protected,
Some(concurrent_body),
"{changed_copy} CAS must not overwrite the concurrent update"
);
let (primary, _) = read_config_with_revision(set, &primary_path)
.await
.expect("read resulting primary");
let primary = DataUsageCache::unmarshal(&primary.expect("primary exists")).expect("primary remains decodable");
assert_eq!(
primary.info.scan_execution_digest,
Some(DataUsageScanPlanDigest([if changed_copy == "primary" { 9 } else { 3 }; 32])),
);
}
}
#[tokio::test]
#[serial]
async fn set_snapshot_reuse_requires_execution_identity_and_fences_stale_writers() {
@@ -2501,7 +2759,7 @@ async fn set_snapshot_reuse_requires_execution_identity_and_fences_stale_writers
.expect("seed legacy set cache");
let mut persisted = DataUsageCache::default();
let initial = persisted
.load_with_revisions(Arc::clone(&set), DATA_USAGE_CACHE_NAME)
.load_with_baseline_revisions(Arc::clone(&set), DATA_USAGE_CACHE_NAME)
.await
.expect("capture the shared starting revision");
let mut fresh = legacy.clone();
@@ -2527,7 +2785,7 @@ async fn set_snapshot_reuse_requires_execution_identity_and_fences_stale_writers
assert_eq!(published.find("photos").expect("published bucket").size, 20);
assert_eq!(published.info.scan_execution_digest, fresh.info.scan_execution_digest);
let current = persisted
.load_with_revisions(Arc::clone(&set), DATA_USAGE_CACHE_NAME)
.load_with_baseline_revisions(Arc::clone(&set), DATA_USAGE_CACHE_NAME)
.await
.expect("capture the current revision for the unidentified execution");
@@ -2819,6 +3077,130 @@ fn scoped_scan_requires_a_complete_authoritative_baseline_with_exact_set_provena
);
}
#[test]
fn scoped_scan_baseline_rejections_preserve_the_existing_first_failed_check() {
let source = DataUsageCacheSource::new(1, 2);
let expected_sources = HashSet::from([source]);
let digest = DataUsageScanPlanDigest([9; 32]);
let baseline = complete_usage_baseline(source, digest, 7, 11);
let snapshot: DataUsageInfo = serde_json::from_slice(&baseline).expect("decode complete baseline");
let proof = ScannerCacheBaselineProof {
authoritative_data: Some(&baseline),
observed_candidate_data: None,
expected_sources: &expected_sources,
leader_epoch: 11,
want_cycle: 8,
scan_plan_digest: digest,
};
type RejectionCase = (ScannerBaselineRejection, fn(&mut DataUsageInfo));
let cases: [RejectionCase; 13] = [
(ScannerBaselineRejection::Incomplete, |s| s.usage_snapshot_complete = false),
(ScannerBaselineRejection::Partial, |s| s.usage_snapshot_partial = true),
(ScannerBaselineRejection::Marks, |s| s.usage_snapshot_converged = None),
(ScannerBaselineRejection::Epoch, |s| s.scanner_epoch = Some(10)),
(ScannerBaselineRejection::SourceCount, |s| s.usage_snapshot_set_states.clear()),
(ScannerBaselineRejection::PlanMissing, |s| {
s.usage_snapshot_set_states[0].scan_plan_digest = None
}),
(ScannerBaselineRejection::PlanMismatch, |s| {
s.usage_snapshot_set_states[0].scan_plan_digest = Some([8; 32])
}),
(ScannerBaselineRejection::SourceMembership, |s| {
s.usage_snapshot_set_states[0].set_index = 3
}),
(ScannerBaselineRejection::SetIncomplete, |s| {
s.usage_snapshot_set_states[0].complete = false
}),
(ScannerBaselineRejection::SetTombstone, |s| {
s.usage_snapshot_set_states[0].tombstone = true
}),
(ScannerBaselineRejection::SetEpoch, |s| {
s.usage_snapshot_set_states[0].scanner_epoch = None
}),
(ScannerBaselineRejection::SetCycle, |s| {
s.usage_snapshot_set_states[0].scanner_cycle = Some(9)
}),
(ScannerBaselineRejection::SetCycle, |s| {
s.usage_snapshot_set_states[0].scanner_cycle = None
}),
];
assert_eq!(validate_scanner_cache_snapshot_plan_digest(&snapshot, proof, None), Ok(digest));
for (expected, mutate) in cases {
let mut invalid = snapshot.clone();
mutate(&mut invalid);
assert_eq!(validate_scanner_cache_snapshot_plan_digest(&invalid, proof, None), Err(expected));
assert_eq!(complete_scanner_cache_snapshot_plan_digest(&invalid, proof, None), None);
}
let mut invalid = snapshot.clone();
invalid.usage_snapshot_partial = true;
invalid.scanner_epoch = Some(10);
assert_eq!(
validate_scanner_cache_snapshot_plan_digest(&invalid, proof, None),
Err(ScannerBaselineRejection::Partial)
);
let two_sources = HashSet::from([source, DataUsageCacheSource::new(1, 3)]);
let mut two_sets = snapshot.clone();
two_sets
.usage_snapshot_set_states
.push(two_sets.usage_snapshot_set_states[0].clone());
let two_set_proof = ScannerCacheBaselineProof {
expected_sources: &two_sources,
..proof
};
assert_eq!(
validate_scanner_cache_snapshot_plan_digest(&two_sets, two_set_proof, None),
Err(ScannerBaselineRejection::SourceDuplicate)
);
two_sets.usage_snapshot_set_states[1].set_index = 3;
two_sets.usage_snapshot_set_states[1].scan_plan_digest = Some([8; 32]);
assert_eq!(
validate_scanner_cache_snapshot_plan_digest(&two_sets, two_set_proof, None),
Err(ScannerBaselineRejection::SetPlan)
);
let undecodable = Bytes::from_static(b"not-json");
for (data, expected) in [
(None, ScannerBaselineRejection::Missing),
(Some(&undecodable), ScannerBaselineRejection::Decode),
] {
assert_eq!(
validate_scanner_cache_baseline_plan_digest(ScannerCacheBaselineProof {
authoritative_data: data,
..proof
}),
Err(ScannerBaselineValidationFailure {
authoritative: expected,
observed: None
})
);
}
let mut wrong_epoch = snapshot;
wrong_epoch.scanner_epoch = Some(10);
let wrong_epoch = Bytes::from(serde_json::to_vec(&wrong_epoch).expect("encode stale epoch baseline"));
let stale_proof = ScannerCacheBaselineProof {
authoritative_data: Some(&wrong_epoch),
..proof
};
assert_eq!(
validate_scanner_cache_baseline_plan_digest(stale_proof),
Err(ScannerBaselineValidationFailure {
authoritative: ScannerBaselineRejection::Epoch,
observed: Some(ScannerBaselineRejection::Missing),
})
);
assert_eq!(
validate_scanner_cache_baseline_plan_digest(ScannerCacheBaselineProof {
observed_candidate_data: Some(&undecodable),
..stale_proof
}),
Err(ScannerBaselineValidationFailure {
authoritative: ScannerBaselineRejection::Epoch,
observed: Some(ScannerBaselineRejection::Decode)
})
);
}
#[test]
fn scoped_scan_accepts_only_a_complete_observation_tied_to_the_authoritative_baseline() {
let source = DataUsageCacheSource::new(1, 2);
@@ -3252,6 +3634,12 @@ fn distributed_segment_reuse_activation_keeps_remote_dirty_buckets_at_bucket_sco
result.scope.prefix_scope_for("photos").is_some(),
"durable local producer proof may activate local segment reuse after distributed ACK capability evidence"
);
assert!(
result
.segment_reuse_activation_preflight
.expect("actual scope proof is retained for observation")
.scanner_segment_reuse_activated
);
assert!(
result.scope.prefix_scope_for("archive").is_none(),
"remote dirty usage has only bucket-granularity evidence and must not be narrowed to a local prefix"
@@ -3670,6 +4058,10 @@ fn scoped_set_scan_rebuilds_selected_buckets_and_drops_deleted_buckets() {
.checked_flatten("stable")
.expect("selected bucket placeholder should exist");
assert_eq!((stable.size, stable.objects), (0, 0));
let observation = rustfs_scanner_metrics::scan_observation::ScannerScanObservation::default();
prepared.record_cold_bucket_reuse(&observation);
assert_eq!(observation.snapshot().bucket_reuse["scoped_cold_bucket"].count, 0);
assert!(prepared.cache.find("stable/prefix").is_none());
assert_eq!(prepared.cache.find("dirty").map(|entry| (entry.size, entry.objects)), Some((0, 0)));
assert!(prepared.cache.find("deleted").is_none());
@@ -3681,6 +4073,124 @@ fn scoped_set_scan_rebuilds_selected_buckets_and_drops_deleted_buckets() {
assert_eq!(prepared.cache.info.lkg_scan_plan_digest, Some(baseline_digest));
}
#[test]
fn cold_bucket_reuse_observation_rejects_unbound_or_failed_preparation() {
let baseline_digest = DataUsageScanPlanDigest([1; 32]);
let mut old_cache = complete_set_usage_cache(&[("stable", 10), ("dirty", 20)], baseline_digest);
old_cache.info.scan_bucket_incarnations = test_bucket_incarnations(&["stable", "dirty"]);
let incarnations = old_cache.info.scan_bucket_incarnations.clone();
let buckets = vec![
bucket_info_with_created_time("stable"),
bucket_info_with_created_time("dirty"),
];
let scope = ScannerBucketScanScope {
selected_buckets: Some(Arc::new(HashSet::from(["dirty".to_string()]))),
selected_bucket_prefixes: None,
baseline_scan_plan_digest: Some(baseline_digest),
};
let generation = ScannerSetCacheGeneration {
want_cycle: 8,
leader_epoch: 11,
tier_registry_generation: 13,
source: DataUsageCacheSource::new(1, 2),
scan_plan_digest: DataUsageScanPlanDigest([2; 32]),
};
let observation = rustfs_scanner_metrics::scan_observation::ScannerScanObservation::default();
let prepared = prepare_scoped_set_scan(&old_cache, &buckets, &buckets, &scope, generation, Some(&incarnations))
.expect("bound complete cache permits a cold bucket copy");
prepared.record_cold_bucket_reuse(&observation);
assert_eq!(observation.snapshot().bucket_reuse["scoped_cold_bucket"].count, 1);
for case in ["incomplete", "duplicate", "nil_incarnation"] {
let mut invalid_cache = old_cache.clone();
let mut invalid_buckets = buckets.clone();
let mut invalid_incarnations = incarnations.clone();
match case {
"incomplete" => invalid_cache.info.snapshot_complete = false,
"duplicate" => invalid_buckets.push(buckets[0].clone()),
"nil_incarnation" => {
invalid_incarnations.insert("stable".to_string(), Uuid::nil());
}
_ => unreachable!("fixed test cases"),
}
let prepared = prepare_scoped_set_scan(
&invalid_cache,
&invalid_buckets,
&invalid_buckets,
&scope,
generation,
Some(&invalid_incarnations),
);
assert!(prepared.is_none(), "{case} must not return a reusable prepared cache");
if let Some(prepared) = prepared {
prepared.record_cold_bucket_reuse(&observation);
}
assert_eq!(
observation.snapshot().bucket_reuse["scoped_cold_bucket"].count,
1,
"{case} does not count completed reuse"
);
}
}
#[tokio::test]
async fn current_root_reuse_counts_only_successful_collector_delivery() {
use rustfs_scanner_metrics::scan_observation::ScannerScanObservation;
let observation = ScannerScanObservation::default();
let cache = DataUsageCache::default();
let pending = AtomicBool::new(false);
let root = || DataUsageEntryInfo {
name: "bucket".to_string(),
parent: DATA_USAGE_ROOT.to_string(),
entry: DataUsageEntry {
size: 10,
objects: 2,
..Default::default()
},
bucket_incarnation: Some(Uuid::from_u128(7)),
tier_registry_generation: None,
value_origin: None,
};
let (sender, mut receiver) = mpsc::channel(1);
super::io_cache::send_current_cache_root_entry(&sender, root(), &cache, &pending, &observation)
.await
.expect("current root is delivered to the collector");
let delivered = receiver.recv().await.expect("collector received the root");
assert_eq!((delivered.entry.size, delivered.entry.objects), (10, 2));
assert_eq!(observation.snapshot().bucket_reuse["current_root"].count, 1);
drop(receiver);
assert!(
super::io_cache::send_current_cache_root_entry(&sender, root(), &cache, &pending, &observation)
.await
.is_err()
);
assert_eq!(
observation.snapshot().bucket_reuse["current_root"].count,
1,
"closed collector is not completed reuse"
);
let (sender, _receiver) = mpsc::channel(1);
sender.send(root()).await.expect("fill bounded collector channel");
{
let mut send = Box::pin(super::io_cache::send_current_cache_root_entry(
&sender,
root(),
&cache,
&pending,
&observation,
));
assert!(futures::poll!(send.as_mut()).is_pending(), "full collector keeps reuse unfinished");
}
assert_eq!(
observation.snapshot().bucket_reuse["current_root"].count,
1,
"dropping unfinished delivery is not reuse"
);
assert_eq!(observation.snapshot().bucket_reuse["scoped_cold_bucket"].count, 0);
}
#[test]
fn scoped_set_scan_reuses_unselected_buckets_with_matching_incarnations() {
let baseline_digest = DataUsageScanPlanDigest([1; 32]);
@@ -3722,6 +4232,12 @@ fn scoped_set_scan_reuses_unselected_buckets_with_matching_incarnations() {
)
.expect("matching bucket incarnations should authorize cold bucket reuse");
let observation = rustfs_scanner_metrics::scan_observation::ScannerScanObservation::default();
prepared.record_cold_bucket_reuse(&observation);
let observed = observation.snapshot();
assert_eq!(observed.bucket_reuse["scoped_cold_bucket"].count, 1);
assert_eq!(observed.bucket_reuse["current_root"].count, 0);
assert!(observed.cache_reuse.values().all(|reuse| reuse.count == 0));
assert_eq!(prepared.buckets.iter().map(|bucket| bucket.name.as_str()).collect::<Vec<_>>(), ["dirty"]);
let stable = prepared
.cache
@@ -4727,12 +5243,36 @@ async fn get_size_treats_missing_metadata_as_skip_file() {
debug: false,
};
let before = global_metrics().scan_observation().snapshot().metadata_reads;
let err = disk
.get_size(item)
.get_size(item.clone())
.await
.expect_err("missing metadata should be skipped instead of reported as a scanner failure");
assert!(matches!(err, StorageError::Io(ref io) if io.to_string() == SCANNER_SKIP_FILE_ERROR));
let missing = global_metrics().scan_observation().snapshot().metadata_reads;
assert_eq!(missing.attempts, before.attempts + 1);
assert_eq!(missing.not_found, before.not_found + 1);
assert_eq!(missing.bytes, before.bytes);
#[cfg(unix)]
{
tokio::fs::create_dir_all(&object_dir)
.await
.expect("recreate object directory");
std::os::unix::fs::symlink(STORAGE_FORMAT_FILE, &metadata_path).expect("inject a metadata symlink loop");
let err = disk
.get_size(item)
.await
.expect_err("symlink loop must remain a read failure");
assert!(is_scanner_metadata_transient_error(&err), "read error: {err:?}");
let failed = global_metrics().scan_observation().snapshot().metadata_reads;
assert_eq!(failed.attempts, missing.attempts + 1);
assert_eq!(failed.errors, missing.errors + 1);
assert_eq!(failed.succeeded, before.succeeded);
assert_eq!(failed.bytes, before.bytes);
}
let _ = tokio::fs::remove_dir_all(&temp_dir).await;
}
@@ -4783,6 +5323,7 @@ async fn get_size_marks_corrupt_metadata_for_heal() {
debug: false,
};
let before = global_metrics().scan_observation().snapshot().metadata_reads;
for contents in [b"".as_slice(), b"not-valid-filemeta".as_slice()] {
tokio::fs::write(&metadata_path, contents)
.await
@@ -4794,6 +5335,16 @@ async fn get_size_marks_corrupt_metadata_for_heal() {
assert!(is_scanner_metadata_corrupt_error(&err));
}
let reads = global_metrics().scan_observation().snapshot().metadata_reads;
assert_eq!(reads.attempts, before.attempts + 2);
assert_eq!(
reads.succeeded,
before.succeeded + 2,
"storage reads succeed before decoding rejects corruption"
);
assert_eq!(reads.errors, before.errors);
assert_eq!(reads.bytes, before.bytes + 18);
let _ = tokio::fs::remove_dir_all(&temp_dir).await;
}
@@ -4866,11 +5417,17 @@ async fn get_size_counts_delete_markers_separately_from_versions() {
debug: false,
};
let before = global_metrics().scan_observation().snapshot().metadata_reads;
let expected_bytes = tokio::fs::metadata(&metadata_path).await.expect("stat stored metadata").len();
let summary = disk.get_size(item).await.expect("scanner should read versioned metadata");
assert_eq!(summary.versions, 2);
assert_eq!(summary.delete_markers, 1);
assert_eq!(summary.total_size, 30);
let reads = global_metrics().scan_observation().snapshot().metadata_reads;
assert_eq!(reads.attempts, before.attempts + 1);
assert_eq!(reads.succeeded, before.succeeded + 1);
assert_eq!(reads.bytes, before.bytes + expected_bytes);
let _ = tokio::fs::remove_dir_all(&temp_dir).await;
}
@@ -183,9 +183,11 @@ async fn dirty_entry_removed_during_listing_still_forces_a_full_scan() {
);
assert_eq!(second.buckets_usage["hot-bucket"].objects_count, 2);
assert_eq!(second.buckets_usage["cold-bucket"].objects_count, 2);
let freshness = global_metrics().report().await.usage_freshness;
let report = global_metrics().report().await;
let freshness = report.usage_freshness;
assert_eq!(freshness.scope_fallback_total, fallbacks_before + 1);
assert_eq!(freshness.last_scope_fallback_reason, "dirty_snapshot_incomplete");
assert_eq!(report.scan_observation.scope_fallbacks["dirty_snapshot_incomplete"], 1);
clear_dirty_usage_buckets_for_tests();
}
@@ -199,11 +201,53 @@ async fn required_full_scan_is_not_reported_as_a_scope_fallback() {
wait_for_namespace_commit_tails(store.as_ref()).await;
super::io_cycle::set_bucket_listing_window_hook_for_tests(|| record_dirty_usage_bucket("created-after-listing"));
let before = global_metrics().scan_observation().snapshot();
let (result, _, scope) = run_cycle(&store, 2, Some(baseline), true).await;
assert!(scope.is_default());
assert_eq!(result.pending_dirty_usage_buckets(), 1);
assert_eq!(result.scope_fallback(), None, "scheduled maintenance walks are not degradations");
assert_eq!(result.completed_scope_fallback(), None);
let after = global_metrics().scan_observation().snapshot();
assert_eq!(after.scope_fallbacks, before.scope_fallbacks);
assert_eq!(after.prefix_scope_resolution.evaluated, before.prefix_scope_resolution.evaluated);
assert_eq!(after.prefix_cycle_completion.evaluated, before.prefix_cycle_completion.evaluated + 1);
clear_dirty_usage_buckets_for_tests();
}
#[tokio::test]
#[serial]
async fn baseline_unavailable_cycle_records_real_scope_and_baseline_rejections_once() {
let (_temp_dir, store) = setup_local_scanner_store(1, 1).await;
let _baseline = converged_baseline(&store).await;
put_fixture_object(&store, "hot-bucket", "second").await;
record_dirty_usage_bucket("hot-bucket");
wait_for_namespace_commit_tails(store.as_ref()).await;
let before = global_metrics().scan_observation().snapshot();
let (result, snapshot, scope) = run_cycle(&store, 2, Some(Bytes::from_static(b"not-json")), false).await;
assert!(scope.is_default());
assert_eq!(result.status, ScannerCycleStatus::Complete);
assert_eq!(result.scope_fallback(), Some(ScannerScopeFallbackReason::BaselineUnavailable));
assert_eq!(snapshot.buckets_usage["hot-bucket"].objects_count, 2);
let after = global_metrics().scan_observation().snapshot();
assert_eq!(
after.scope_fallbacks["baseline_unavailable"],
before.scope_fallbacks["baseline_unavailable"] + 1
);
assert_eq!(
after.authoritative_baseline_rejections["decode"],
before.authoritative_baseline_rejections["decode"] + 1
);
assert_eq!(
after.observed_baseline_rejections, before.observed_baseline_rejections,
"an undecodable authority never admits an observation"
);
assert_eq!(after.prefix_scope_resolution.evaluated, before.prefix_scope_resolution.evaluated + 1);
assert_eq!(after.prefix_scope_resolution.activated, before.prefix_scope_resolution.activated);
assert_eq!(
after.prefix_scope_resolution.blockers["missing_cold_zero_walk_oracle"],
before.prefix_scope_resolution.blockers["missing_cold_zero_walk_oracle"] + 1
);
assert_eq!(after.prefix_cycle_completion.evaluated, before.prefix_cycle_completion.evaluated + 1);
clear_dirty_usage_buckets_for_tests();
}
@@ -1,12 +1,12 @@
// Copyright 2026 RustFS Team
// SPDX-License-Identifier: Apache-2.0
//! A bucket whose walk stops early without exhausting the cycle budget (here
//! a sweep resumed under a newer plan, the `mixed_coverage` case of
//! rustfs/rustfs#8494) no longer makes the whole cycle `Incomplete`. The set
//! publishes the bucket's last certified value, the cycle completes with the
//! bucket pending, and the next cycle walks it again. A bucket that never
//! certified keeps the cycle incomplete.
//! A bucket whose walk stops early can publish its last certified value as
//! pending. This fixture tests usage-scan status, not the outer scanner cycle
//! outcome: budget exhaustion may leave complete validated usage while the
//! coordinator records `Partial`. A continuation under a newer plan exercises
//! `mixed_coverage` (rustfs/rustfs#8494). A bucket that never certified keeps
//! the usage scan incomplete.
use super::*;
@@ -38,8 +38,7 @@ async fn run_cycle(store: &Arc<ECStore>, cycle: u64, options: CycleOptions) -> (
..Default::default()
},
);
// These cycles persist real bucket checkpoints and source caches. Keep
// CI storage latency inside the production drain window after a count stop.
// Real checkpoint commits need the runtime drain grace after the count budget stops the walk.
budget.use_production_drain_grace_for_tests();
let (updates, mut receiver) = mpsc::channel(1);
let result = tokio::time::timeout(
@@ -49,7 +48,7 @@ async fn run_cycle(store: &Arc<ECStore>, cycle: u64, options: CycleOptions) -> (
ScannerCycleRequest {
owner_ctx: ctx.clone(),
ctx: budget.token(),
budget,
budget: Arc::clone(&budget),
updates,
want_cycle: cycle,
leader_epoch: LEADER_EPOCH,
@@ -69,6 +68,13 @@ async fn run_cycle(store: &Arc<ECStore>, cycle: u64, options: CycleOptions) -> (
.await
.expect("cycle should finish within the test deadline")
.expect("cycle should succeed");
if options.max_objects.is_some() {
assert_eq!(
budget.reason(),
Some(crate::scanner_budget::ScannerCycleBudgetReason::Objects),
"a count-limited fixture must stop for its object budget"
);
}
let snapshot = receiver.recv().await;
assert!(receiver.recv().await.is_none(), "a cycle publishes at most one terminal snapshot");
(result, snapshot)
@@ -125,9 +131,8 @@ async fn partial_bucket_with_a_certified_value_completes_the_cycle_as_pending()
assert!(result.clear_verified_usage().is_empty());
assert!(dirty_usage_buckets_for_tests().is_empty());
// Cycle 3: the object budget stops the hot bucket's walk and leaves a
// checkpoint under this cycle's plan. Its certified value is carried, so
// budget exhaustion leaves the bucket pending without blocking the cycle.
// Cycle 3: the budget stops the walk and saves a checkpoint. Usage can
// carry the certified value as pending; the outer coordinator records Partial.
for object in ["p4/third", "p5/third"] {
put_fixture_object(&store, "hot-bucket", object).await;
}
@@ -144,12 +149,38 @@ async fn partial_bucket_with_a_certified_value_completes_the_cycle_as_pending()
},
)
.await;
let third = third.expect("a carried partial cycle publishes");
assert_eq!(result.status, ScannerCycleStatus::Complete);
assert_eq!(
result.status,
ScannerCycleStatus::Complete,
"certified carried usage remains complete after a budget stop"
);
let third = third.expect("validated carried usage publishes after the budget stop");
assert_eq!(result.pending_dirty_usage_buckets(), 1);
assert!(!third.usage_snapshot_observational);
assert_eq!(third.usage_snapshot_converged, Some(false));
assert_eq!(third.usage_snapshot_pending_buckets, vec!["hot-bucket".to_string()]);
assert_eq!(third.buckets_usage["hot-bucket"].objects_count, 4);
assert_eq!(third.usage_snapshot_pending_bucket_count, 1);
assert!(third.is_complete_bucket_usage_snapshot());
let carried_hot = &third.buckets_usage["hot-bucket"];
let certified_hot = &second.buckets_usage["hot-bucket"];
assert_eq!(carried_hot.usage_value_state, Some(rustfs_data_usage::BucketUsageValueState::Carried));
assert!(certified_hot.usage_certification.is_some(), "cycle 2 certified the prior hot value");
assert_eq!(
carried_hot.usage_certification, certified_hot.usage_certification,
"carrying must not mint a new certificate"
);
assert_eq!(carried_hot.objects_count, 4);
assert_eq!(carried_hot.size, certified_hot.size);
assert_eq!(third.buckets_usage["cold-bucket"].objects_count, 1);
assert_eq!(third.buckets_usage["idle-bucket"].objects_count, 1);
assert_eq!(third.objects_total_count, 6);
// Only hot was dirty in this scoped scan; it received no fresh certification or ACK.
assert!(result.clear_verified_usage().is_empty());
assert_eq!(
dirty_usage_buckets_for_tests().keys().cloned().collect::<Vec<_>>(),
vec!["hot-bucket".to_string()],
"the budget-stopped hot bucket retains its dirty responsibility"
);
let set = Arc::clone(&store.pools[0].disk_set[0]);
let mut interrupted = DataUsageCache::default();
interrupted
@@ -15,30 +15,6 @@
use super::*;
use crate::data_usage_define::{DATA_USAGE_OBJ_NAME_PATH, read_config_with_revision};
// Two-pool EC fixtures keep many scan futures live at once and overflow
// Linux libtest's default 2 MiB test-thread stack. Run each case on a
// dedicated 32 MiB thread (same approach as scoped_entry_fallback).
fn run_cohort_test<F, Fut>(thread_name: &'static str, test_fn: F)
where
F: FnOnce() -> Fut + Send + 'static,
Fut: std::future::Future<Output = ()> + 'static,
{
let handle = std::thread::Builder::new()
.name(thread_name.to_string())
.stack_size(32 * 1024 * 1024)
.spawn(move || {
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.expect("service cohort test runtime should build");
runtime.block_on(test_fn());
})
.expect("service cohort test thread should spawn");
if let Err(payload) = handle.join() {
std::panic::resume_unwind(payload);
}
}
async fn create_cohort_bucket(store: &ECStore, bucket: &str) {
create_cohort_bucket_objects(store, bucket, 1).await;
}
@@ -128,16 +104,9 @@ async fn run_cohort_cycle(
(result, usage)
}
#[test]
#[tokio::test]
#[serial]
fn service_cohort_production_dispatch_services_waiters_across_sources() {
run_cohort_test(
"cohort-prod-disp",
service_cohort_production_dispatch_services_waiters_across_sources_case,
);
}
async fn service_cohort_production_dispatch_services_waiters_across_sources_case() {
async fn service_cohort_production_dispatch_services_waiters_across_sources() {
let (_dir, store) = setup_two_pool_scanner_store().await;
clear_dirty_usage_buckets_for_tests();
let hot = format!("a-hot-{}", Uuid::new_v4().simple());
@@ -228,16 +197,9 @@ async fn service_cohort_production_dispatch_services_waiters_across_sources_case
clear_dirty_usage_buckets_for_tests();
}
#[test]
#[tokio::test]
#[serial]
fn service_cohort_flat_bucket_budget_does_not_publish_unscanned_small_bucket() {
run_cohort_test(
"cohort-flat-bud",
service_cohort_flat_bucket_budget_does_not_publish_unscanned_small_bucket_case,
);
}
async fn service_cohort_flat_bucket_budget_does_not_publish_unscanned_small_bucket_case() {
async fn service_cohort_flat_bucket_budget_does_not_publish_unscanned_small_bucket() {
let (_dir, store) = setup_two_pool_scanner_store().await;
clear_dirty_usage_buckets_for_tests();
let flat = format!("a-flat-{}", Uuid::new_v4().simple());
@@ -321,16 +283,9 @@ async fn service_cohort_flat_bucket_budget_does_not_publish_unscanned_small_buck
clear_dirty_usage_buckets_for_tests();
}
#[test]
#[tokio::test]
#[serial]
fn service_cohort_fresh_complete_aggregate_preserves_reordered_sources() {
run_cohort_test(
"cohort-fresh-agg",
service_cohort_fresh_complete_aggregate_preserves_reordered_sources_case,
);
}
async fn service_cohort_fresh_complete_aggregate_preserves_reordered_sources_case() {
async fn service_cohort_fresh_complete_aggregate_preserves_reordered_sources() {
let (_dir, store) = setup_two_pool_scanner_store().await;
clear_dirty_usage_buckets_for_tests();
for bucket in ["cohort-first", "cohort-second"] {
@@ -374,16 +329,9 @@ async fn service_cohort_fresh_complete_aggregate_preserves_reordered_sources_cas
clear_dirty_usage_buckets_for_tests();
}
#[test]
#[tokio::test]
#[serial]
fn service_cohort_cancelled_dispatch_does_not_consume_waiters_or_leak_permits() {
run_cohort_test(
"cohort-cancelled",
service_cohort_cancelled_dispatch_does_not_consume_waiters_or_leak_permits_case,
);
}
async fn service_cohort_cancelled_dispatch_does_not_consume_waiters_or_leak_permits_case() {
async fn service_cohort_cancelled_dispatch_does_not_consume_waiters_or_leak_permits() {
let (_dir, store) = setup_two_pool_scanner_store().await;
clear_dirty_usage_buckets_for_tests();
let bucket = format!("cancel-{}", Uuid::new_v4().simple());
@@ -412,16 +360,9 @@ async fn service_cohort_cancelled_dispatch_does_not_consume_waiters_or_leak_perm
/// N=17, q=4: every bucket reaches a worker within ceil(17/4)=5 rounds while
/// one bucket stays hot and another fails forever, with leader-local state
/// discarded after each round as a restart would.
#[test]
#[tokio::test]
#[serial]
fn cold_bucket_service_bound_survives_restart_with_hot_and_failing_bucket() {
run_cohort_test(
"cold-svc-bound",
cold_bucket_service_bound_survives_restart_with_hot_and_failing_bucket_case,
);
}
async fn cold_bucket_service_bound_survives_restart_with_hot_and_failing_bucket_case() {
async fn cold_bucket_service_bound_survives_restart_with_hot_and_failing_bucket() {
const BUCKETS: usize = 17;
const QUANTA: u64 = 4;
const ROUND_BOUND: usize = 5;
@@ -493,16 +434,9 @@ fn quantum_budget(quantum: crate::scanner_budget::ScannerBucketQuantum) -> Arc<S
)
}
#[test]
#[tokio::test]
#[serial]
fn source_turn_dispatch_rotates_real_sources_and_retains_completed_roots() {
run_cohort_test(
"src-turn-disp",
source_turn_dispatch_rotates_real_sources_and_retains_completed_roots_case,
);
}
async fn source_turn_dispatch_rotates_real_sources_and_retains_completed_roots_case() {
async fn source_turn_dispatch_rotates_real_sources_and_retains_completed_roots() {
let mut evidence = None;
temp_env::async_with_vars([(ENV_SCANNER_MAX_CONCURRENT_SET_SCANS, Some("1"))], async {
crate::runtime_config::refresh_scanner_runtime_config_for_tests();
@@ -569,16 +503,9 @@ async fn source_turn_dispatch_rotates_real_sources_and_retains_completed_roots_c
);
}
#[test]
#[tokio::test]
#[serial]
fn source_turn_zero_new_frontier_stops_requeuing_without_starving_other_sources() {
run_cohort_test(
"src-turn-zero",
source_turn_zero_new_frontier_stops_requeuing_without_starving_other_sources_case,
);
}
async fn source_turn_zero_new_frontier_stops_requeuing_without_starving_other_sources_case() {
async fn source_turn_zero_new_frontier_stops_requeuing_without_starving_other_sources() {
temp_env::async_with_vars([(ENV_SCANNER_MAX_CONCURRENT_SET_SCANS, Some("1"))], async {
crate::runtime_config::refresh_scanner_runtime_config_for_tests();
let (_dir, store) = setup_two_pool_scanner_store().await;
@@ -606,13 +533,9 @@ async fn source_turn_zero_new_frontier_stops_requeuing_without_starving_other_so
crate::runtime_config::refresh_scanner_runtime_config_for_tests();
}
#[test]
#[tokio::test]
#[serial]
fn fair_prefix_quantum_spent_bucket_returns_behind_waiting_buckets() {
run_cohort_test("fair-spent-qm", fair_prefix_quantum_spent_bucket_returns_behind_waiting_buckets_case);
}
async fn fair_prefix_quantum_spent_bucket_returns_behind_waiting_buckets_case() {
async fn fair_prefix_quantum_spent_bucket_returns_behind_waiting_buckets() {
let (_dir, store) = setup_local_scanner_store(1, 4).await;
clear_dirty_usage_buckets_for_tests();
create_cohort_bucket_objects(&store, "a-flat", 6).await;
@@ -642,13 +565,9 @@ async fn fair_prefix_quantum_spent_bucket_returns_behind_waiting_buckets_case()
clear_dirty_usage_buckets_for_tests();
}
#[test]
#[tokio::test]
#[serial]
fn fair_prefix_quantum_without_durable_motion_ends_the_bucket_turn() {
run_cohort_test("fair-no-motion", fair_prefix_quantum_without_durable_motion_ends_the_bucket_turn_case);
}
async fn fair_prefix_quantum_without_durable_motion_ends_the_bucket_turn_case() {
async fn fair_prefix_quantum_without_durable_motion_ends_the_bucket_turn() {
let (_dir, store) = setup_local_scanner_store(1, 4).await;
clear_dirty_usage_buckets_for_tests();
create_cohort_bucket_objects(&store, "a-flat", 6).await;
@@ -671,16 +590,9 @@ async fn fair_prefix_quantum_without_durable_motion_ends_the_bucket_turn_case()
/// A one-object share stops inside its first child, so the frontier never
/// moves. Requeuing that bucket would spin until the cycle deadline.
#[test]
#[tokio::test]
#[serial]
fn fair_prefix_quantum_repeated_share_without_a_new_frontier_stops_requeuing() {
run_cohort_test(
"fair-rep-share",
fair_prefix_quantum_repeated_share_without_a_new_frontier_stops_requeuing_case,
);
}
async fn fair_prefix_quantum_repeated_share_without_a_new_frontier_stops_requeuing_case() {
async fn fair_prefix_quantum_repeated_share_without_a_new_frontier_stops_requeuing() {
let (_dir, store) = setup_local_scanner_store(1, 4).await;
clear_dirty_usage_buckets_for_tests();
create_cohort_bucket_objects(&store, "a-flat", 3).await;
@@ -701,16 +613,9 @@ async fn fair_prefix_quantum_repeated_share_without_a_new_frontier_stops_requeui
clear_dirty_usage_buckets_for_tests();
}
#[test]
#[tokio::test]
#[serial]
fn cold_bucket_service_saved_order_keeps_coverage_and_rejects_older_owners() {
run_cohort_test(
"cold-saved-ord",
cold_bucket_service_saved_order_keeps_coverage_and_rejects_older_owners_case,
);
}
async fn cold_bucket_service_saved_order_keeps_coverage_and_rejects_older_owners_case() {
async fn cold_bucket_service_saved_order_keeps_coverage_and_rejects_older_owners() {
let (_dir, store) = setup_local_scanner_store(1, 4).await;
clear_dirty_usage_buckets_for_tests();
create_cohort_bucket(&store, "cold-only").await;
-40
View File
@@ -1,40 +0,0 @@
# SSE Crate Instructions
Applies to `crates/sse/`.
## Boundary
- This crate sits between the S3 layer and the storage engine and depends on neither. Its `rustfs-*` dependencies are pinned by `scripts/check_architecture_migration_rules.sh` to `rustfs-config`, `rustfs-kms`, `rustfs-policy`, `rustfs-s3-types`, and `rustfs-utils`; see [crate boundaries](../../docs/architecture/crate-boundaries.md).
- Anything the crate needs from the running server arrives through `SseHost` (`src/host.rs`), passed as the first argument of every entry point that consults the server. Do not add a global that stands in for the host: a caller must not be able to reach key selection without naming the server it speaks for.
- Translation to and from storage-engine types (`ObjectEncryptionResolver`, `WriteEncryption`, `MultipartCompletionAuthorization`) and request types belongs in `rustfs/src/storage/sse.rs`, not here.
- Report failures as `SseError`, classified with the S3 error code the caller answers with. A failure that did not originate from the request goes through `SseError::internal`, which keeps its detail out of the response.
## Security
- Never log plaintext customer keys, decrypted key material, data-key envelopes, nonces, or SSE request payloads, including through error strings and `Debug` output.
- KMS audit tags may carry only the fields documented on `KmsRequestAuditState`.
- The `test-util` feature exposes authorization overrides and DEK-provider injection. It is a dev-dependency feature only; never enable it from a normal dependency.
## Change Coordination
When changing key resolution, metadata, or error classification, verify compatibility with:
- `rustfs/src/storage/sse.rs` (the host implementation and the engine-boundary adapters)
- `crates/kms/`
- `crates/utils/src/http/object_encryption_keys.rs` (the stored-metadata key vocabulary)
- `rustfs/src/kms_rekey.rs` and `rustfs/src/admin/handlers/kms*.rs`
## Testing
Run both builds; the `rio-v2` feature swaps the object-key scheme and compiles different code paths:
```bash
cargo nextest run -p rustfs-sse --no-tests fail
cargo nextest run -p rustfs-sse --features rio-v2 --no-tests fail
```
The adapter tests live with the adapter:
```bash
cargo nextest run -p rustfs --no-tests fail -E 'test(/^storage::sse::/)'
```
-70
View File
@@ -1,70 +0,0 @@
# Copyright 2024 RustFS Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
[package]
name = "rustfs-sse"
edition.workspace = true
license.workspace = true
repository.workspace = true
rust-version.workspace = true
version.workspace = true
homepage.workspace = true
description = "S3 server-side encryption (SSE-S3, SSE-KMS, SSE-C) key resolution and metadata for RustFS."
keywords = ["sse", "encryption", "s3", "kms", "rustfs"]
categories = ["cryptography"]
[lints]
workspace = true
[features]
default = []
# MinIO-compatible sealed object keys (DARE). Mirrors the `rio-v2` feature of
# the server binary, which enables both together.
rio-v2 = ["dep:chacha20poly1305", "dep:hmac", "dep:sha2"]
# Test seams for dependents: per-principal authorization overrides, DEK
# provider injection and metadata fixtures. Never enable in a release build.
test-util = []
[dependencies]
# The only RustFS crates this one may reach. It sits between the S3 layer and
# the storage engine and must not depend on either: what it needs from the
# running server arrives through `SseHost`.
rustfs-config = { workspace = true }
rustfs-kms = { workspace = true }
rustfs-policy = { workspace = true }
rustfs-s3-types = { workspace = true }
rustfs-utils = { workspace = true, features = ["http", "path"] }
aes-gcm = { workspace = true, features = ["rand_core", "zeroize"] }
async-trait = { workspace = true }
base64-simd = { workspace = true }
chacha20poly1305 = { workspace = true, optional = true }
hmac = { workspace = true, optional = true }
http = { workspace = true }
md-5 = { workspace = true }
rand = { workspace = true }
serde = { workspace = true, features = ["derive"] }
serde_json = { workspace = true }
sha2 = { workspace = true, optional = true }
tracing = { workspace = true }
[dev-dependencies]
# The fake AWS KMS endpoint the backend-selection tests talk to.
axum = { workspace = true }
temp-env = { workspace = true, features = ["async_closure"] }
tempfile = { workspace = true }
tokio = { workspace = true, features = ["macros", "net", "rt-multi-thread", "sync", "time"] }
[lib]
doctest = false
-176
View File
@@ -1,176 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! The error every SSE operation reports, already classified for the S3 data path.
use rustfs_kms::{KmsError, KmsUnavailableError};
use rustfs_s3_types::S3ErrorCode;
type BoxError = Box<dyn std::error::Error + Send + Sync>;
/// S3 error code a KMS key that does not exist is reported under.
pub const KMS_KEY_NOT_FOUND_ERROR_CODE: &str = "KMS.NotFoundException";
pub(crate) const INTERNAL_ERROR_MESSAGE: &str = "We encountered an internal error, please try again.";
const SERVICE_UNAVAILABLE_MESSAGE: &str = "The service is unavailable. Please retry.";
const ACCESS_DENIED_MESSAGE: &str = "Access Denied.";
/// A failed SSE operation, carrying the S3 error code and message the caller answers with.
///
/// `source` keeps the underlying failure for logs and for classification by
/// callers; it is never part of the response.
#[derive(Debug)]
pub struct SseError {
pub code: S3ErrorCode,
pub message: String,
pub source: Option<BoxError>,
}
impl SseError {
/// Classify a failure that did not originate from the request itself.
///
/// A [`KmsUnavailableError`] is a retryable `ServiceUnavailable`, a
/// [`KmsError`] follows [`data_plane_kms_error`], and everything else is an
/// `InternalError` whose detail stays in `source` and out of the response.
pub fn internal(error: impl Into<BoxError>) -> Self {
let source = error.into();
let (code, message) = if source.downcast_ref::<KmsUnavailableError>().is_some() {
(S3ErrorCode::ServiceUnavailable, SERVICE_UNAVAILABLE_MESSAGE.to_string())
} else if let Some(classified) = source.downcast_ref::<KmsError>().and_then(data_plane_kms_error) {
classified
} else {
(S3ErrorCode::InternalError, INTERNAL_ERROR_MESSAGE.to_string())
};
Self {
code,
message,
source: Some(source),
}
}
}
impl std::fmt::Display for SseError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(&self.message)
}
}
impl std::error::Error for SseError {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
self.source.as_deref().map(|source| source as _)
}
}
/// Map a KMS failure that surfaced on the S3 data path to its S3 error code.
///
/// The contract deliberately differs from the admin key-lifecycle handlers:
/// there a key id is the resource being addressed, so a missing key is `404`.
/// Here the key id arrives inside a request header or a bucket default, so a
/// key that is missing, disabled, or otherwise unusable is configuration the
/// caller has to correct — AWS answers `400`, and reporting `500` instead both
/// misfiles the failure as a server fault and makes SDKs back off and retry a
/// request that cannot succeed.
///
/// `None` keeps the caller's fallthrough, which is the `500` that integrity
/// faults — damaged, unreadable, or unknown-format key material — must keep.
///
/// Messages either name what the caller asked for or stay generic; detail that
/// belongs to the deployment rather than the request stays in `source`, which
/// the caller attaches and the logs retain.
pub fn data_plane_kms_error(error: &KmsError) -> Option<(S3ErrorCode, String)> {
use KmsError as Kms;
match error {
Kms::KeyNotFound { key_id } => Some((
S3ErrorCode::Custom(KMS_KEY_NOT_FOUND_ERROR_CODE.into()),
format!("KMS key not found: {key_id}"),
)),
Kms::KeyVersionNotFound { key_id, version } => Some((
S3ErrorCode::Custom(KMS_KEY_NOT_FOUND_ERROR_CODE.into()),
format!("KMS key version {version} not found for key {key_id}"),
)),
// The key exists but its state forbids the operation (disabled,
// pending deletion). AWS treats an invalid key state as a request
// error, not a server fault.
Kms::InvalidOperation { .. } => Some((S3ErrorCode::InvalidRequest, error.to_string())),
// A policy decision needs a human, so it must be distinguishable from
// the transient classes an SDK retries.
Kms::AccessDenied { .. } => Some((S3ErrorCode::AccessDenied, ACCESS_DENIED_MESSAGE.to_string())),
// Request-side faults: what was asked for cannot be served as asked.
Kms::ContextMismatch { .. }
| Kms::InvalidKey { .. }
| Kms::ValidationError { .. }
| Kms::UnsupportedAlgorithm { .. }
| Kms::InvalidKeySize { .. } => Some((S3ErrorCode::InvalidRequest, error.to_string())),
// A deployment whose KMS configuration cannot serve the request (for
// example no key named and no default). Actionable, but the detail
// describes the deployment, so it stays out of the response body.
Kms::ConfigurationError { .. } => {
Some((S3ErrorCode::InvalidRequest, "The KMS configuration cannot serve this request".to_string()))
}
// Transient: worth retrying, and must be counted against availability
// rather than against the caller. `IoError` belongs here because it is
// how a backend reports that its key store itself was unreachable —
// rustfs/rustfs#7470 separated that from a missing key precisely so the
// two stop looking alike, and leaving it on the 500 fallthrough would
// erase that distinction again at the S3 boundary.
Kms::BackendError { .. }
| Kms::IoError { .. }
| Kms::OperationTimedOut { .. }
| Kms::OperationCancelled { .. }
| Kms::CredentialsUnavailable { .. }
| Kms::CacheError { .. } => Some((S3ErrorCode::ServiceUnavailable, SERVICE_UNAVAILABLE_MESSAGE.to_string())),
// A permanent gap in the configured backend, never a missing resource.
Kms::UnsupportedCapability { .. } => Some((S3ErrorCode::NotImplemented, error.to_string())),
_ => None,
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn internal_keeps_request_free_failures_generic() {
let error = SseError::internal("Failed to decode IV: bad base64");
assert_eq!(error.code, S3ErrorCode::InternalError);
assert_eq!(error.message, INTERNAL_ERROR_MESSAGE);
assert!(
error
.source
.as_ref()
.is_some_and(|source| source.to_string().contains("Failed to decode IV"))
);
}
#[test]
fn internal_reports_an_unavailable_kms_as_retryable() {
let error = SseError::internal(KmsUnavailableError);
assert_eq!(error.code, S3ErrorCode::ServiceUnavailable);
assert_eq!(error.message, SERVICE_UNAVAILABLE_MESSAGE);
}
#[test]
fn internal_classifies_kms_errors_for_the_data_path() {
let missing = SseError::internal(KmsError::key_not_found("absent-key"));
assert_eq!(missing.code, S3ErrorCode::Custom(KMS_KEY_NOT_FOUND_ERROR_CODE.into()));
assert_eq!(missing.message, "KMS key not found: absent-key");
let damaged = SseError::internal(KmsError::cryptographic_error("decrypt", "authentication failed"));
assert_eq!(damaged.code, S3ErrorCode::InternalError);
assert_eq!(damaged.message, INTERNAL_ERROR_MESSAGE);
}
}
-85
View File
@@ -1,85 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! What SSE needs from the server it runs in.
//!
//! This crate reaches the running server only through [`SseHost`]: it holds no
//! handle to the storage engine, to IAM, or to the bucket metadata system. Every
//! entry point that consults the server takes the host as its first argument, so
//! a caller cannot reach key selection without saying which server it speaks for.
use crate::{SseError, SseKmsPrincipal};
use async_trait::async_trait;
use rustfs_kms::KmsServiceManager;
use rustfs_kms::service::ObjectEncryptionService;
use rustfs_policy::policy::action::KmsAction;
use std::sync::Arc;
/// The server capabilities SSE key resolution depends on.
#[async_trait]
pub trait SseHost: Send + Sync {
/// The KMS object-encryption service currently serving, if one is.
///
/// Resolved on every call: the KMS backend can be reconfigured while the
/// server runs, so a service handle must not outlive the operation it serves.
async fn encryption_service(&self) -> Option<Arc<ObjectEncryptionService>>;
/// The KMS service manager, when the server has one.
///
/// Only its status is read, to tell a configured KMS that is not serving
/// (retryable) from a deployment that never configured one (a request error).
fn kms_service_manager(&self) -> Option<Arc<KmsServiceManager>>;
/// The bucket's default-encryption configuration.
///
/// `Ok(None)` means the bucket has no encryption configuration.
///
/// # Errors
///
/// Any failure to read the configuration, already classified for the S3
/// response. A write must not proceed as if the bucket had no default when
/// the configuration could not be read.
async fn bucket_encryption(&self, bucket: &str) -> Result<Option<BucketEncryption>, SseError>;
/// The authorizer deciding per-key KMS permissions on the data path.
fn kms_key_authorizer(&self) -> &dyn KmsKeyAuthorizer;
}
/// A bucket's stored encryption configuration, reduced to what a write consults.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct BucketEncryption {
/// The default-encryption rule of the first configuration rule, if it has one.
pub default_rule: Option<BucketDefaultEncryption>,
}
/// The `ApplyServerSideEncryptionByDefault` element of a bucket encryption rule.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct BucketDefaultEncryption {
/// The stored algorithm, exactly as persisted (`AES256`, `aws:kms`, or an unknown value).
pub sse_algorithm: String,
pub kms_master_key_id: Option<String>,
}
/// Decides whether a principal may use a specific KMS key on the data path.
///
/// Behind a trait so tests can pin the allow/deny contract of the SSE call sites
/// without a running IAM subsystem.
#[async_trait]
pub trait KmsKeyAuthorizer: Send + Sync {
/// # Errors
///
/// When the decision itself could not be made, for example because the
/// policy subsystem is not ready. A denial is `Ok(false)`.
async fn is_allowed(&self, principal: &SseKmsPrincipal, action: KmsAction, key_id: &str) -> Result<bool, SseError>;
}
File diff suppressed because it is too large Load Diff
-70
View File
@@ -1,70 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Fixtures for tests of SSE call sites, in this crate and in its dependents.
use crate::{KmsKeyAuthorizer, SseError, SseKmsPrincipal};
use async_trait::async_trait;
use base64_simd::STANDARD as BASE64_STANDARD;
use rustfs_policy::policy::action::KmsAction;
use rustfs_utils::http::headers::AMZ_ENCRYPTION_KMS;
use rustfs_utils::http::object_encryption_keys::{
INTERNAL_ENCRYPTION_ALGORITHM_HEADER, INTERNAL_ENCRYPTION_IV_HEADER, INTERNAL_ENCRYPTION_KEY_HEADER,
INTERNAL_ENCRYPTION_KEY_ID_HEADER,
};
use std::collections::HashMap;
use std::sync::{Arc, Mutex, PoisonError};
/// An authorizer that answers every request the same way and records what it was asked.
pub struct RecordingKmsKeyAuthorizer {
allowed: bool,
calls: Mutex<Vec<(KmsAction, String)>>,
}
impl RecordingKmsKeyAuthorizer {
pub fn new(allowed: bool) -> Arc<Self> {
Arc::new(Self {
allowed,
calls: Mutex::new(Vec::new()),
})
}
/// The `(action, key id)` pairs this authorizer was asked about, in order.
pub fn calls(&self) -> Vec<(KmsAction, String)> {
self.calls.lock().unwrap_or_else(PoisonError::into_inner).clone()
}
}
#[async_trait]
impl KmsKeyAuthorizer for RecordingKmsKeyAuthorizer {
async fn is_allowed(&self, _principal: &SseKmsPrincipal, action: KmsAction, key_id: &str) -> Result<bool, SseError> {
self.calls
.lock()
.unwrap_or_else(PoisonError::into_inner)
.push((action, key_id.to_string()));
Ok(self.allowed)
}
}
/// Stored metadata of an SSE-KMS object wrapped under the key `finance-key`.
pub fn sse_kms_object_metadata() -> HashMap<String, String> {
HashMap::from([
("x-amz-server-side-encryption".to_string(), AMZ_ENCRYPTION_KMS.to_string()),
("x-amz-server-side-encryption-aws-kms-key-id".to_string(), "finance-key".to_string()),
(INTERNAL_ENCRYPTION_KEY_ID_HEADER.to_string(), "finance-key".to_string()),
(INTERNAL_ENCRYPTION_KEY_HEADER.to_string(), BASE64_STANDARD.encode_to_string([7u8; 48])),
(INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode_to_string([9u8; 12])),
(INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(), "aws:kms".to_string()),
])
}
+8 -3
View File
@@ -30,11 +30,12 @@ doctest = false
[features]
default = []
hotpath = ["rustfs-rio/hotpath"]
hotpath-alloc = ["hotpath", "rustfs-rio/hotpath-alloc"]
hotpath-cpu = ["hotpath", "rustfs-rio/hotpath-cpu"]
hotpath = ["hotpath/hotpath", "hotpath/tokio", "rustfs-rio/hotpath"]
hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc", "rustfs-rio/hotpath-alloc"]
hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu", "rustfs-rio/hotpath-cpu"]
[dependencies]
hotpath.workspace = true
async-compression = { workspace = true, features = [
"tokio",
"bzip2",
@@ -48,6 +49,10 @@ rustfs-rio.workspace = true
tokio = { workspace = true, features = ["io-util", "macros", "rt"] }
thiserror = { workspace = true }
[target.'cfg(target_os = "macos")'.dependencies]
# Keep XZ decoding self-contained in macOS binaries instead of linking Homebrew liblzma.
liblzma = { version = "0.4.9", features = ["static"] }
[dev-dependencies]
rustfs-tokio-tar = { workspace = true }
futures = { workspace = true }
+5 -7
View File
@@ -29,16 +29,14 @@ zdict_builder = ["zstd_fixed/zdict_builder"]
zstdmt = ["zstd_fixed/zstdmt"]
[dev-dependencies]
# Enables zstd-safe's `seekable` feature for tests/security.rs. The test imports
# `zstd_safe::seekable` directly so cargo-shear treats this as a used dependency.
zstd-safe = { version = "8.3.0", features = ["seekable"] }
toml = "1.2.2"
toml = "1.2.1"
semver = "1.0.28"
apache-avro.workspace = true
arrow-array = "60.2.0"
arrow-ipc = { version = "60.2.0", features = ["zstd"] }
arrow-schema = "60.2.0"
parquet = { version = "60.2.0", default-features = false, features = ["arrow", "zstd"] }
arrow-array = "60.1.0"
arrow-ipc = { version = "60.1.0", features = ["zstd"] }
arrow-schema = "60.1.0"
parquet = { version = "60.1.0", default-features = false, features = ["arrow", "zstd"] }
kafka-protocol = { version = "0.18.0", default-features = false, features = ["zstd"] }
bytes.workspace = true
+1 -1
View File
@@ -86,7 +86,7 @@ fn corrupt_legacy_frame_cannot_emit_bytes_when_writer_is_dropped() {
#[test]
fn seekable_offsets_past_eof_keep_vec_length_within_capacity() {
use zstd_safe::seekable::{Seekable, SeekableCStream};
use zstd::zstd_safe::seekable::{Seekable, SeekableCStream};
let source = b"seekable frame fixture with enough data for several independent frames";
let mut encoded = [0; 1024];
-2
View File
@@ -30,11 +30,9 @@ Contract crates stay below implementation crates. Forbidden edges:
| `security-governance -> rustfs` | Governance contracts stay below the binary crate |
| `extension-schema -> rustfs` | The extension schema is consumed by the binary, never the reverse |
| `extension-schema -> ecstore` | The extension schema must not reach storage internals |
| `sse -> ecstore`, `sse -> iam`, `sse -> rustfs` | SSE key resolution reaches the running server only through its `SseHost` port |
- `rustfs-storage-api` exposes storage-facing replication status/state contracts only through `crates/storage-api/src/replication.rs`, so its temporary dependency on `rustfs-filemeta` wire types stays centralized and no `rustfs-replication` / `rustfs-storage-api` cycle appears.
- Leaf crates (`config`, `credentials`, `crypto`, `io-metrics`, `madmin`) may not depend on other `rustfs-*` crates, in either TOML spelling, except the adjudicated edges pinned in the guard's leaf allowlist: `io-metrics -> rustfs-s3-ops` (pure contract crates sharing the `S3Operation` vocabulary) and `madmin -> rustfs-signer` (the SigV4-signed admin SDK client). A new leaf exception must be a pure contract dependency (types and enums only, no I/O, no globals, no non-contract internal dependencies) and land together with its allowlist entry.
- `rustfs-sse` sits between the S3 layer and the storage engine and depends on neither. Its internal dependencies are pinned in the same guard to `rustfs-config`, `rustfs-kms`, `rustfs-policy`, `rustfs-s3-types`, and `rustfs-utils`; the KMS service, a bucket's default encryption, and key-permission decisions arrive through `rustfs_sse::SseHost`, which `rustfs/src/storage/sse.rs` implements. That file also owns every translation across the engine boundary (`ObjectEncryptionResolver`, `WriteEncryption`, `MultipartCompletionAuthorization`), so neither side names the other's types.
- Compile-time source reads follow the same direction: `include_str!` / `include!` of a `.rs` file must not resolve outside the including crate (`scripts/check_layer_dependencies.sh`). Shared source-text expectations belong in a contract surface such as `rustfs_protos::compat_manifest` (`crates/protos/src/compat_manifest.rs`) and are asserted by each owning crate.
## ECStore Access Boundary
+1 -1
View File
@@ -47,7 +47,7 @@ RustFS-side statics that matter architecturally because other modules are tempte
| `KEYSTONE_AUTH`, `KEYSTONE_MAPPER`, `KEYSTONE_CONFIG` | `rustfs/src/auth_keystone.rs` | Keystone provider, mapper, and config stay private to the Keystone owner. |
| `DEADLOCK_DETECTOR` | `rustfs/src/storage/deadlock_detector.rs` | Detector lifecycle stays private to the storage deadlock detector. |
| `CONCURRENCY_MANAGER` | `rustfs/src/storage/concurrency/manager.rs` | Storage concurrency scheduler state stays inside the concurrency owner. |
| `GLOBAL_KMS_DEK_PROVIDER`, `GLOBAL_SSE_DEK_PROVIDER` | `crates/sse/src/lib.rs` | DEK provider caches stay private to the SSE owner. |
| `GLOBAL_KMS_DEK_PROVIDER`, `GLOBAL_SSE_DEK_PROVIDER` | `rustfs/src/storage/sse.rs` | DEK provider caches stay private to the SSE owner. |
| `ECSTORE_EVENT_DISPATCH_HOOK` | `rustfs/src/server/event.rs` | Event bridge registration goes through the storage facade. |
| `AUDIT_MODULE_ENABLED`, `NOTIFY_MODULE_ENABLED` | `rustfs/src/module_switches.rs` | Module toggles are read through module-switch helpers; `MODULE_SWITCH_RMW_LOCK` (`rustfs/src/server/module_switch.rs`) serializes persisted updates. |
| `RUNTIME_CONFIG_RELOAD_MUTEX` | `rustfs/src/admin/service/config.rs` | Serializes dynamic config reload fanout. |
+3 -3
View File
@@ -1,7 +1,7 @@
# KMS Bulk Rekey Job Contract
**Use this when:** changing the bulk envelope re-wrap sweep (`rustfs/src/kms_rekey.rs`), its admin endpoints (`rustfs/src/admin/handlers/kms_rekey.rs`), the re-wrap primitive, or anything that decides which objects a rekey may touch.
**Source of truth:** `rustfs/src/kms_rekey.rs`, `rustfs/src/admin/handlers/kms_rekey.rs`, `rewrap_object_encryption_metadata` in `crates/sse/src/lib.rs`, `KmsManager::rewrap_data_key` / `KmsManager::describe_data_key_wrapping` in `crates/kms/src/manager.rs`, `put_object_metadata` in `crates/ecstore/src/set_disk/ops/object.rs`.
**Source of truth:** `rustfs/src/kms_rekey.rs`, `rustfs/src/admin/handlers/kms_rekey.rs`, `rewrap_object_encryption_metadata` in `rustfs/src/storage/sse.rs`, `KmsManager::rewrap_data_key` / `KmsManager::describe_data_key_wrapping` in `crates/kms/src/manager.rs`, `put_object_metadata` in `crates/ecstore/src/set_disk/ops/object.rs`.
The bulk rekey job re-wraps stored data-key envelopes under the current key-encryption key (KEK) without rewriting object bodies. This document is the acceptance bar; where the shipped v1 sweep deliberately narrows it, [Implementation Status](#implementation-status-v1-sweep) records the deviation.
@@ -52,7 +52,7 @@ Idempotency comes from object metadata itself: the envelope's state is the targe
### Reading the wrapping KEK version
There is no key-version metadata key. Object metadata carries the key id (`x-rustfs-encryption-key-id`) and the sealed blob under `x-rustfs-encryption-key`; `DecryptResponse` in `crates/kms/src/types.rs` does not report a version either. The version is recoverable because the sealed blob is structured: for every backend that builds one, the ciphertext is the JSON of `DataKeyEnvelope` (`crates/kms/src/encryption/dek.rs`), and the read path already discriminates on it via `is_data_key_envelope` in `crates/sse/src/lib.rs`.
There is no key-version metadata key. Object metadata carries the key id (`x-rustfs-encryption-key-id`) and the sealed blob under `x-rustfs-encryption-key`; `DecryptResponse` in `crates/kms/src/types.rs` does not report a version either. The version is recoverable because the sealed blob is structured: for every backend that builds one, the ciphertext is the JSON of `DataKeyEnvelope` (`crates/kms/src/encryption/dek.rs`), and the read path already discriminates on it via `is_data_key_envelope` in `rustfs/src/storage/sse.rs`.
| Backend | Rotates | Where the wrapping version lives | Recoverable by a scan |
|---|---|---|---|
@@ -95,7 +95,7 @@ Replication destinations are unresolved: propagation depends on `rustfs/backlog#
Three properties of `put_object_metadata` (`crates/ecstore/src/set_disk/ops/object.rs`) constrain the re-wrap write:
- **The merge is additive; it cannot remove keys.** Overwriting a key that keeps its name is safe; a re-wrap that changes *which* keys describe the envelope leaves the old keys behind. This is a live hazard: `parse_minio_managed_sealed_key` in `crates/sse/src/lib.rs` selects the MinIO decrypt branch on the mere presence of the MinIO seal-algorithm header, so a RustFS-native envelope written onto MinIO-compatible headers without neutralizing them steers reads down the stale branch. Any envelope-shape change must neutralize superseded keys in the same write.
- **The merge is additive; it cannot remove keys.** Overwriting a key that keeps its name is safe; a re-wrap that changes *which* keys describe the envelope leaves the old keys behind. This is a live hazard: `parse_minio_managed_sealed_key` in `rustfs/src/storage/sse.rs` selects the MinIO decrypt branch on the mere presence of the MinIO seal-algorithm header, so a RustFS-native envelope written onto MinIO-compatible headers without neutralizing them steers reads down the stale branch. Any envelope-shape change must neutralize superseded keys in the same write.
- **Object-lock retention is enforced before the merge.** `check_object_lock_retention_update` (`crates/ecstore/src/set_disk/mod.rs`) runs first; rekey inherits its decision and must not acquire a bypass.
- **`mod_time` is preserved unless the caller sets it.** The re-wrap path leaves it unset so age-based lifecycle rules are not perturbed.
@@ -1,13 +1,13 @@
# MinIO On-Disk Format Interoperability
**Use this when:** deciding whether a MinIO drive set, bucket-metadata blob, or SSE object can be read or imported by a given RustFS build, or before touching any constant or codec listed under Version Anchors.
**Source of truth:** `crates/filemeta/src/filemeta.rs`, `crates/filemeta/src/filemeta/codec.rs`, `crates/ecstore/src/bucket/metadata.rs`, `crates/ecstore/src/bucket/migration.rs`, `crates/sse/src/lib.rs`, `rustfs/Cargo.toml` `[features]`, `.github/workflows/ci.yml`, `.github/workflows/minio-interop.yml`.
**Source of truth:** `crates/filemeta/src/filemeta.rs`, `crates/filemeta/src/filemeta/codec.rs`, `crates/ecstore/src/bucket/metadata.rs`, `crates/ecstore/src/bucket/migration.rs`, `rustfs/src/storage/sse.rs`, `rustfs/Cargo.toml` `[features]`, `.github/workflows/ci.yml`, `.github/workflows/minio-interop.yml`.
This is an interop contract, not a plan. Migration is one-way (MinIO to RustFS). Erasure-coding internals are owned by [erasure-coding.md](erasure-coding.md); this document owns the interop claim, the fixture evidence, and the out-of-scope list.
## Scope Matrix By Build Variant
Build variants are the `rustfs` crate features in `rustfs/Cargo.toml`: `default`, `full`, and `rio-v2` (which enables `rustfs-ecstore/rio-v2` and `rustfs-sse/rio-v2`, and pulls in `crates/rio-v2`). `rio-v2` is absent from both `default` and `full`.
Build variants are the `rustfs` crate features in `rustfs/Cargo.toml`: `default`, `full`, and `rio-v2` (which enables `rustfs-ecstore/rio-v2` and pulls in `crates/rio-v2`). `rio-v2` is absent from both `default` and `full`.
| MinIO artifact | `default` / `full` build | `rio-v2` build | Notes |
|---|:--:|:--:|---|
@@ -81,7 +81,7 @@ The `xl.meta` around a MinIO SSE object parses in every build, so such objects l
### Seams
The cryptography (DARE v2 stream format, object-key derivation, sealing) was never the gap. Three metadata seams above it rejected MinIO objects; all are closed in `rio-v2` builds. Symbols are in `crates/sse/src/lib.rs` unless noted.
The cryptography (DARE v2 stream format, object-key derivation, sealing) was never the gap. Three metadata seams above it rejected MinIO objects; all are closed in `rio-v2` builds. Symbols are in `rustfs/src/storage/sse.rs` unless noted.
| Seam | Resolution |
|---|---|
@@ -113,10 +113,10 @@ Inventory the source first: bucket default encryption means objects can be encry
| Fact | Value |
|---|---|
| Shipping status | Ships in no default build: absent from `default` and `full` in `rustfs/Cargo.toml`; released binaries and container images never include it. Enable with `--features rio-v2`. |
| Pull-request coverage | `test-and-lint-rio-v2` in `.github/workflows/ci.yml`: clippy plus `cargo nextest` for `rustfs`, `rustfs-ecstore`, and `rustfs-sse` with `--features rio-v2`. This is the cfg-seam guard; it keeps the feature compiling and its unit suite green on every pull request. |
| Pull-request coverage | `test-and-lint-rio-v2` in `.github/workflows/ci.yml`: clippy plus `cargo nextest` for `rustfs` and `rustfs-ecstore` with `--features rio-v2`. This is the cfg-seam guard; it keeps the feature compiling and its unit suite green on every pull request. |
| Full-suite lane | `build-rustfs-debug-binary-rio-v2` and `e2e-tests-rio-v2` in `ci.yml` run only on the nightly schedule and manual dispatch (`cache-warm.yml` keeps the `ci-feat-rio` cache warm so the scheduled build fits its timeout). |
| Interop evidence | `.github/workflows/minio-interop.yml` (nightly plus manual) regenerates real MinIO backend trees via `crates/rio-v2/tests/minio_fixture_lab/` and runs the `#[ignore]` reader tests in `rustfs/src/storage/minio_generated_read_test.rs` with `--features rio-v2`. Its freshness is tracked in `.github/scheduled-validations.json`. |
| Promote-or-delete condition | The variant stays dormant until one of two things happens. **Promote**: a release commits to shipping MinIO SSE migration as a supported capability; then `rio-v2` joins `default`/`full`, the scheduled lanes run on every pull request, and this section is rewritten. **Delete**: no release commits to it and the scheduled lanes are not kept green; then the feature flag, `crates/rio-v2`, the cfg seams in `crates/sse/src/lib.rs`, the three `ci.yml` jobs, the `cache-warm.yml` warm step, `minio-interop.yml`, and its `scheduled-validations.json` entry are removed in one change. Either outcome must update `ARCHITECTURE.md` and the `ci.yml` job comments that cite this section. |
| Promote-or-delete condition | The variant stays dormant until one of two things happens. **Promote**: a release commits to shipping MinIO SSE migration as a supported capability; then `rio-v2` joins `default`/`full`, the scheduled lanes run on every pull request, and this section is rewritten. **Delete**: no release commits to it and the scheduled lanes are not kept green; then the feature flag, `crates/rio-v2`, the cfg seams in `rustfs/src/storage/sse.rs`, the three `ci.yml` jobs, the `cache-warm.yml` warm step, `minio-interop.yml`, and its `scheduled-validations.json` entry are removed in one change. Either outcome must update `ARCHITECTURE.md` and the `ci.yml` job comments that cite this section. |
## Fixture Evidence
+1 -1
View File
@@ -1,7 +1,7 @@
# AWS KMS backend: setup, verification, and operations
**Use this when:** connecting RustFS to AWS KMS for the first time, verifying an existing AWS-backed deployment, upgrading one from 1.0.4 or 1.0.5, or deciding how to run and protect the AWS keys RustFS depends on.
**Source of truth:** `crates/kms/src/backends/aws.rs` (backend behavior and its deviations), `crates/kms/src/config.rs` (`ENV_KMS_AWS_*`), `crates/kms/src/policy.rs` (retry, deadline and circuit breaker), `crates/sse/src/lib.rs` (SSE data path). 中文版: [kms-aws-backend-guide.zh-CN.md](kms-aws-backend-guide.zh-CN.md).
**Source of truth:** `crates/kms/src/backends/aws.rs` (backend behavior and its deviations), `crates/kms/src/config.rs` (`ENV_KMS_AWS_*`), `crates/kms/src/policy.rs` (retry, deadline and circuit breaker), `rustfs/src/storage/sse.rs` (SSE data path). 中文版: [kms-aws-backend-guide.zh-CN.md](kms-aws-backend-guide.zh-CN.md).
The statements about behavior on this page were observed against the real AWS KMS service (region `eu-north-1`) and, for failures AWS cannot be asked to produce on demand, against a loopback stand-in that speaks the AWS KMS wire protocol. Where a statement is guidance that was not exercised, it says so. For the contract-level differences between AWS and the other backends, see [KMS backend security properties](kms-backend-security.md#aws-kms-deviations-from-the-shared-backend-contract).
@@ -1,7 +1,7 @@
# AWS KMS 后端:接入、验证与运维
**适用场景:** 首次把 RustFS 接入 AWS KMS、验证已有的 AWS 后端部署、从 1.0.4 或 1.0.5 升级,或者规划如何运维并保护 RustFS 所依赖的 AWS 密钥。
**事实来源:** `crates/kms/src/backends/aws.rs`(后端行为及其差异)、`crates/kms/src/config.rs`(`ENV_KMS_AWS_*`)、`crates/kms/src/policy.rs`(重试、超时与熔断)、`crates/sse/src/lib.rs`(SSE 数据面)。English: [kms-aws-backend-guide.md](kms-aws-backend-guide.md)。
**事实来源:** `crates/kms/src/backends/aws.rs`(后端行为及其差异)、`crates/kms/src/config.rs`(`ENV_KMS_AWS_*`)、`crates/kms/src/policy.rs`(重试、超时与熔断)、`rustfs/src/storage/sse.rs`(SSE 数据面)。English: [kms-aws-backend-guide.md](kms-aws-backend-guide.md)。
本页关于行为的描述,均来自对真实 AWS KMS 服务(区域 `eu-north-1`)的实测;对于无法让 AWS 按需产生的故障,则使用一个实现了 AWS KMS 线上协议的本机替身来实测。属于未经实测的建议之处,文中会明确标注。AWS 与其它后端在契约层面的差异,见 [KMS backend security properties](kms-backend-security.md#aws-kms-deviations-from-the-shared-backend-contract)。
@@ -902,6 +902,42 @@ done
The `aggregated.scanner` payload preserves the same scanner progress, checkpoint, pacing, source work, maintenance control, lifecycle expiry, and lifecycle transition fields used by the local scanner status, but only for the responding node; `by_host.*.scanner` keeps that node's host view. Compare the per-node artifacts externally to find old active paths, partial checkpoints, pacing pressure, source-level control pressure, or downstream queue admission problems across the deployment.
## Reading Cached Bucket Statistics
Admin usage reads cache the selected scanner baseline for the existing 30-second TTL, then verify every returned bucket against uncached storage-owned incarnation metadata. Identity reads use at most four concurrent operations and a shared two-second deadline after releasing snapshot/refresh locks. Each bucket read linearizes independently against metadata transactions; the response is not an atomic snapshot of the entire namespace. Delayed peer invalidation cannot authorize a predecessor's statistics for a recreated bucket.
Missing, failed, timed-out, nil, mismatched, or legacy absent identities leave the bucket value absent (unknown), rather than returning the predecessor's value or a confirmed zero. Legacy statistics become visible after a scanner snapshot supplies the current identity. Account-info and usage display apply the existing single-process memory overlay before the final identity filter; distributed local counters are not promoted. When a projection omits values or changes an overlay, totals are partial and `usage_snapshot_converged=false`, and per-tier/replication aggregates are unavailable. Verified cold bucket values remain visible. Such response-only projections cannot be persisted, used for quota admission, or acknowledge dirty work.
## Reading Scanner Work Observations
`GET /rustfs/admin/v3/scanner/status` exposes `metrics.scan_observation` for the responding node. Capture timestamped snapshots on every node, including the scanner leader, and compare their deltas over the same workload window. These process-lifetime counters reset on restart; a missing field on an older node means unobserved. They are loaded independently and may briefly disagree while operations finish. This telemetry does not change scan cadence, cached usage reads, quota authority, dirty acknowledgement, or maintenance requirements.
| Field | Meaning |
|---|---|
| `metadata_reads.attempts` | Disk metadata read futures started after scanner node admission. Invalid paths and operations cancelled before starting the read do not count. |
| `metadata_reads.succeeded`, `not_found`, `errors`, `cancelled` | Storage outcomes, including object/version disappearance and dropped read futures. Successful storage reads can still contain corrupt metadata; decoding and lifecycle failures do not become read errors. Cancellation observes the future, not proof that physical I/O stopped. |
| `metadata_reads.bytes` | Total returned buffer lengths for successful reads, including inline data and corrupt metadata. Failed reads contribute no known bytes; this is not physical disk traffic. |
| `metadata_reads.elapsed_nanos` | Sum of read-future elapsed time, including disk-permit or RPC waits, failures, and cancellation. Folder pacing, scanner node admission, decoding, and lifecycle work are outside this interval. Concurrent reads overlap, so the sum is not cycle wall time. |
| `cache_reuse.clean_prefix` | Clean root subtrees copied under an admitted prefix scope by the non-erasure scanner. |
| `cache_reuse.covered_namespace` | Namespace subtrees copied from validated checkpoint coverage. |
| `cache_reuse.sampled_namespace` | Cached namespace subtrees copied when the existing periodic sampling selected other work. |
| `cache_reuse.classified_namespace` | EC candidate namespace subtrees copied only after the current listing finished classifying them. Health/classification work still runs. |
| `cache_reuse.*.count`, `elapsed_nanos` | Completed subtree reuse decisions and time spent copying/attaching their cache entries. Counts are not objects skipped or a cache hit ratio; times exclude classification/listing and are not total cache maintenance cost. |
| `bucket_reuse.scoped_cold_bucket` | Whole cold buckets copied and attached to a successfully prepared scoped set cache. Count is copied buckets, not plans selected; later turn publication can still fail. Time covers copying/attachment only. |
| `bucket_reuse.current_root` | Existing current bucket roots successfully sent to the local collector after watch, namespace lease and publication epoch checks. Time covers channel send only; failure or cancellation contributes no completed reuse. Collector acceptance and durable publication remain separate. |
| `reuse_decisions.*.not_evaluated`, `blocked`, `eligible` | First conditions reached by production clean-prefix, checkpoint-coverage and namespace-sampling gates. Policy exclusions (`missing_scope`, `no_resume_sweep`, `deep_scan`, `forward_sweep`, `active_replication`) precede cache proof checks. `blocked` reports the first failed work/cache condition; later checks are not repeated for diagnostics. `eligible` does not certify a copy: an EC candidate can still require classification. |
| `erasure_clean_prefix_walks` | Clean root children that still require walking because EC scans perform independent object-health work. |
| `prefix_scope_resolution.evaluated`, `activated`, `blockers` | Proof evaluations retained by the final scope resolver, admitted proof gates, and every failed prerequisite. An admitted proof does not guarantee a reusable child exists; actual subtree copies appear in `cache_reuse`. Startup/required full scans, early scope rejection, and discarded peer/ACK resolutions do not count here. |
| `prefix_cycle_completion.evaluated`, `activated`, `blockers` | End-of-cycle proof evaluations for subsequent reuse. An activated completion proof does not mean this cycle reused prefixes or that its later publication/persistence succeeded. One evaluation can increment several blocker counters. |
| `scope_fallbacks` | Per-reason breakdown of the existing `usage_freshness.scope_fallback_total`; required full scans are excluded. |
| `authoritative_baseline_rejections`, `observed_baseline_rejections` | First failed validation checks for `baseline_unavailable` fallback cycles, preserving the authoritative rejection when the observed fallback also fails. Missing/undecodable authority never reaches observed validation; `authoritative_identity` means it could not anchor an observed fallback. |
`freshness` describes the responding node's completed-cycle age, not whether every bucket has incorporated new writes. Read `metrics.usage_freshness` for node-local live pending work, guardrail vetoes, durable success and publication state; read `metrics.usage_publication` for the confirmed snapshot's pending count and publication class/outcome, and `cycle_schedule` for effective retry intervals and consecutive superseded cycles. Missing or unknown publication data does not establish zero pending work. A follower or a single node's empty dirty map cannot establish cluster-wide convergence.
Whole cold bucket copies, current-root reuse and the four subtree categories have different prerequisites and are not interchangeable hit rates. EC clean prefixes retain health walks. Deep or required full work must run, but a completed cache matching that exact work digest may satisfy a retry of the same cycle. A normal workload need not exercise current-root reuse or every subtree category; use controlled tests for positive emission evidence and capture real gate reasons when a VM window has zero copies.
The existing `life_time_ops.read_metadata` and `last_minute.actions.read_metadata` now include finished and cancelled read attempts; the latter reports count, accumulated whole seconds, and returned bytes in its existing window. Use `metadata_reads.elapsed_nanos` for subsecond read costs. Use read-count and byte deltas to assess work growth, and elapsed-time deltas to assess read-future cost. Compare those with reused-subtree deltas, scope fallback reasons, cycle wall time, pacing waits, and source work before attributing freshness delays to metadata I/O or proposing a scan optimization.
## Reading Lifecycle Transition Status
`metrics.lifecycle_transition`:
+1 -3
View File
@@ -80,7 +80,7 @@ e2e-test-hooks = ["rustfs-ecstore/e2e-test-hooks", "rustfs-scanner/e2e-test-hook
connect-e2e-short-credentials = []
# Builds the dedicated rustfs-cli-e2e target with a build-time public enrollment root.
offline-enrollment-e2e-root = []
rio-v2 = ["rustfs-ecstore/rio-v2", "rustfs-sse/rio-v2"]
rio-v2 = ["rustfs-ecstore/rio-v2"]
# Linux-only cuObject S3-over-RDMA GET into client (GPU) memory; needs libcuobjserver.
cuobject = ["dep:rustfs-cuobject", "rustfs-cuobject/server"]
# NIXL/UCX transport for S3-over-RDMA; replaces cuobject when both are enabled.
@@ -281,7 +281,6 @@ rustfs-rio = { workspace = true }
rustfs-s3-client = { workspace = true }
rustfs-s3-types = { workspace = true, features = ["compat-s3s"] }
rustfs-s3-ops = { workspace = true }
rustfs-sse = { workspace = true }
rustfs-security-governance = { workspace = true }
rustfs-data-usage = { workspace = true }
rustfs-s3select-api = { workspace = true }
@@ -425,7 +424,6 @@ h2.workspace = true
rustfs-targets = { workspace = true, features = ["test-support"] }
uuid = { workspace = true, features = ["v4", "v5", "fast-rng", "macro-diagnostics"] }
rustfs-iam = { workspace = true, features = ["test-util"] }
rustfs-sse = { workspace = true, features = ["test-util"] }
serial_test = { workspace = true }
tempfile = { workspace = true }
aws-config = { workspace = true }
+2 -3
View File
@@ -20,7 +20,7 @@ use crate::admin::service::caller_identity::oidc_profile_fields;
use crate::admin::storage_api::bucket::versioning_sys::BucketVersioningSys;
use crate::admin::storage_api::contract::admin::StorageAdminApi;
use crate::admin::storage_api::contract::bucket::{BucketOperations, BucketOptions};
use crate::admin::storage_api::data_usage::{apply_bucket_usage_memory_overlay, load_admin_data_usage_from_backend_cached};
use crate::admin::storage_api::data_usage::load_admin_data_usage_with_memory_overlay;
use crate::admin::storage_api::metadata_sys;
use crate::auth::get_condition_values;
use crate::server::ADMIN_PREFIX;
@@ -278,8 +278,7 @@ impl Operation for AccountInfoHandler {
// process-local absolute counters to cluster-wide bucket totals.
// This path never triggers a live full-version listing
// (rustfs/backlog#1306); freshness is owned by the scanner.
let mut data_usage_info = map_data_usage_result(load_admin_data_usage_from_backend_cached(store.clone()).await)?;
apply_bucket_usage_memory_overlay(&mut data_usage_info).await;
let data_usage_info = map_data_usage_result(load_admin_data_usage_with_memory_overlay(store.clone()).await)?;
account_info.usage_snapshot_converged = data_usage_info.usage_snapshot_converged;
for bucket in buckets.iter() {
+12 -8
View File
@@ -8617,14 +8617,18 @@ impl Operation for SiteReplicationRepairHandler {
.to_string(),
None => Uuid::new_v4().to_string(),
};
let (status, headers, body) = execute_site_replication_repair(SiteReplicationRepairExecutionRequest {
let response = execute_site_replication_repair(SiteReplicationRepairExecutionRequest {
local_peer,
preflight_token: supplied_token.to_string(),
operation_id,
signing_key,
})
.await?;
Ok(AdminResponse::with_headers(status, body, headers))
// The repair executor still answers with the s3s response type; the
// router reads the status from the output tuple, so only the tuple and
// the headers carry over to the owned response.
let (status, body) = response.output;
Ok(AdminResponse::with_headers(status, body.into(), response.headers))
}
}
@@ -13917,18 +13921,18 @@ mod tests {
env!("CARGO_MANIFEST_DIR"),
"/../crates/ecstore/tests/fixtures/bucket-config-goldens/lifecycle/minio.in.xml"
));
// Answer of s3s 5761ddfe (`Timestamp::parse(DateTime)` on this document's
// ExpiryUpdatedAt). The owned axis must keep that instant.
let legacy_axis =
parse_lifecycle_date("2026-07-07T15:58:57.337315Z").expect("the legacy expiry axis is the minio.in.xml value");
let mut old_reader = s3s::xml::Deserializer::new(raw);
let old: s3s::dto::BucketLifecycleConfiguration =
s3s::xml::Deserialize::deserialize(&mut old_reader).expect("read the actual old lifecycle expiry axis");
old_reader.expect_eof().expect("the old expiry reader consumes the document");
let old_axis = OffsetDateTime::from(old.expiry_updated_at.expect("the old config contains its expiry axis"));
let new = deserialize(raw).expect("read the persisted lifecycle expiry axis");
let new_axis = local_lifecycle_staleness_axis(Some(&new), OffsetDateTime::UNIX_EPOCH)
.expect("the parsed lifecycle axis is readable");
let incoming = parse_lifecycle_date("2026-07-07T15:58:57.337100Z").expect("the incoming expiry axis is valid");
assert_eq!(new_axis, legacy_axis, "the parsed axis must keep the legacy submillisecond instant");
assert_eq!(
is_stale_update(new_axis, Some(incoming)),
is_stale_update(legacy_axis, Some(incoming)),
is_stale_update(old_axis, Some(incoming)),
"a peer update older than the full-precision local axis must remain stale"
);
}
-26
View File
@@ -1021,32 +1021,6 @@ impl<T> S3Router<T> {
}
}
/// Dispatches one legacy admin request. Callers keep their assertions and do not name s3s.
/// Free function (not an inherent impl) so the `OwnedOperation` bound stays `pub(crate)`.
#[cfg(test)]
pub(crate) async fn call_legacy<T: OwnedOperation>(
router: &S3Router<T>,
method: Method,
uri: &str,
extensions: http::Extensions,
) -> s3s::S3Result<S3Response<Body>> {
S3Route::call(
router,
S3Request {
input: Body::empty(),
method,
uri: uri.parse().expect("test URI"),
headers: HeaderMap::new(),
extensions,
credentials: None,
region: None,
service: None,
trailing_headers: None,
},
)
.await
}
impl<T> Default for S3Router<T> {
fn default() -> Self {
Self::new(false)
+23 -15
View File
@@ -141,13 +141,15 @@ mod tests {
object_store_from_envelope, publish_test_app_context,
};
use crate::admin::operation::{AdminRequest, AdminResponse, Operation};
use crate::admin::router::{S3Router, call_legacy};
use crate::app::{RequestEnvelope, RequestHead};
use crate::admin::router::S3Router;
use crate::app::RequestEnvelope;
use hyper::{Method, StatusCode};
use matchit::Params;
use rustfs_iam::{store::object::ObjectStore, sys::IamSys};
use rustfs_kms::KmsServiceManager;
use rustfs_s3_types::{S3ErrorCode, s3_error};
use s3s::route::S3Route;
use s3s::{Body, S3Request};
use std::sync::Arc;
struct UnreadyIam;
@@ -190,18 +192,22 @@ mod tests {
Arc::new(AppContext::new(context.object_store(), Arc::new(UnreadyIam), Arc::new(TestKms)))
}
fn envelope(extensions: http::Extensions) -> RequestEnvelope {
RequestEnvelope::from_parts(
RequestHead {
method: Method::GET,
uri: "/context-probe".parse().expect("test URI"),
headers: http::HeaderMap::new(),
credentials: None,
region: None,
service: None,
},
fn request(extensions: http::Extensions) -> S3Request<Body> {
S3Request {
input: Body::empty(),
method: Method::GET,
uri: "/context-probe".parse().expect("test URI"),
headers: http::HeaderMap::new(),
extensions,
)
credentials: None,
region: None,
service: None,
trailing_headers: None,
}
}
fn envelope(extensions: http::Extensions) -> RequestEnvelope {
RequestEnvelope::from_s3s(&request(extensions))
}
#[tokio::test]
@@ -323,13 +329,15 @@ mod tests {
.insert(Method::GET, "/context-probe", ContextDependentAdminRoute)
.expect("register test route");
let err = call_legacy(&router, Method::GET, "/context-probe", http::Extensions::new())
let err = router
.call(request(http::Extensions::new()))
.await
.expect_err("empty slot must fail closed");
assert_eq!(err.code(), &S3ErrorCode::ServiceUnavailable.into());
assert!(slot.install(distinct_context(&ambient)));
let response = call_legacy(&router, Method::GET, "/context-probe", http::Extensions::new())
let response = router
.call(request(http::Extensions::new()))
.await
.expect("installed slot must serve request");
assert_eq!(response.status, Some(StatusCode::NO_CONTENT));
+6 -4
View File
@@ -945,10 +945,6 @@ pub(crate) static ERR_TIER_RESERVED_NAME: AdminErrorRef = AdminErrorRef(|| &ecst
pub(crate) mod data_usage {
use std::sync::Arc;
pub(crate) async fn apply_bucket_usage_memory_overlay(data_usage_info: &mut rustfs_data_usage::DataUsageInfo) {
crate::storage::storage_api::ecstore_data_usage::apply_bucket_usage_memory_overlay(data_usage_info).await;
}
pub(crate) async fn load_data_usage_from_backend(
store: Arc<crate::storage::storage_api::ECStore>,
) -> Result<rustfs_data_usage::DataUsageInfo, crate::storage::storage_api::StorageError> {
@@ -960,6 +956,12 @@ pub(crate) mod data_usage {
) -> Result<rustfs_data_usage::DataUsageInfo, crate::storage::storage_api::StorageError> {
crate::storage::storage_api::ecstore_data_usage::load_admin_data_usage_from_backend_cached(store).await
}
pub(crate) async fn load_admin_data_usage_with_memory_overlay(
store: Arc<crate::storage::storage_api::ECStore>,
) -> Result<rustfs_data_usage::DataUsageInfo, crate::storage::storage_api::StorageError> {
crate::storage::storage_api::ecstore_data_usage::load_admin_data_usage_with_memory_overlay(store).await
}
}
pub(crate) mod access {
+65 -22
View File
@@ -21,9 +21,7 @@ use super::storage_api::admin_usecase::capacity::{
};
use super::storage_api::admin_usecase::contract::StorageAdminApi;
use super::storage_api::admin_usecase::contract::bucket::{BucketOperations as _, BucketOptions};
use super::storage_api::admin_usecase::data_usage::{
apply_bucket_usage_memory_overlay, load_admin_data_usage_from_backend_cached,
};
use super::storage_api::admin_usecase::data_usage::load_admin_data_usage_with_memory_overlay;
use super::storage_api::admin_usecase::{ECStore, EndpointServerPools};
use crate::app::runtime_sources::{
AppContext, current_app_context, current_endpoints_handle, current_object_store_handle_for_context,
@@ -272,7 +270,7 @@ impl DefaultAdminUsecase {
info.buckets_usage.retain(|bucket, _| current_buckets.contains(bucket));
info.bucket_sizes.retain(|bucket, _| current_buckets.contains(bucket));
if info.buckets_usage.is_empty() && !current_buckets.is_empty() {
if info.buckets_usage.is_empty() && !current_buckets.is_empty() && !info.usage_snapshot_admin_projection {
*info = DataUsageInfo::default();
return;
}
@@ -287,19 +285,14 @@ impl DefaultAdminUsecase {
}
}
/// Overlay single-process counts on a persisted snapshot and narrow it
/// for display.
///
/// Whether the snapshot measured anything is decided before the overlay.
/// The overlay clears `usage_snapshot_complete` when it holds counts it
/// cannot reconcile with the snapshot, such as writes into a bucket that
/// has no verified baseline. That makes the totals unconverged, not the
/// scanned buckets unusable; treating it as structural incompleteness
/// blanked usage for every bucket for as long as one such bucket kept
/// receiving writes, because each write restarts its scanner fence.
async fn present_data_usage_snapshot(mut info: DataUsageInfo, buckets: impl IntoIterator<Item = String>) -> DataUsageInfo {
let snapshot_measured = info.is_complete_bucket_usage_snapshot() || info.is_valid_partial_snapshot();
apply_bucket_usage_memory_overlay(&mut info).await;
/// Keep already identity-bound display values without accepting a display
/// projection as a persistable or quota-authoritative snapshot.
fn present_identity_bound_data_usage_snapshot(
mut info: DataUsageInfo,
buckets: impl IntoIterator<Item = String>,
) -> DataUsageInfo {
let snapshot_measured =
info.usage_snapshot_admin_projection || info.is_complete_bucket_usage_snapshot() || info.is_valid_partial_snapshot();
Self::narrow_data_usage_snapshot_to_measured_buckets(&mut info, snapshot_measured, buckets);
info
}
@@ -345,7 +338,7 @@ impl DefaultAdminUsecase {
/// This path never triggers a live full-version listing
/// (rustfs/backlog#1306); freshness is owned by the scanner.
pub(crate) async fn query_data_usage_info_with_store(store: Arc<ECStore>) -> AdminUsecaseResult<DataUsageInfo> {
let info = Self::map_data_usage_load_result(load_admin_data_usage_from_backend_cached(store.clone()).await)?;
let info = Self::map_data_usage_load_result(load_admin_data_usage_with_memory_overlay(store.clone()).await)?;
let buckets = store
.list_bucket(&BucketOptions {
cached: true,
@@ -354,7 +347,7 @@ impl DefaultAdminUsecase {
})
.await
.map_err(|_| Self::app_error(S3ErrorCode::InternalError, "list_bucket failed"))?;
let mut info = Self::present_data_usage_snapshot(info, buckets.into_iter().map(|bucket| bucket.name)).await;
let mut info = Self::present_identity_bound_data_usage_snapshot(info, buckets.into_iter().map(|bucket| bucket.name));
let storage_info = StorageAdminApi::storage_info(store.as_ref()).await;
@@ -979,6 +972,52 @@ mod tests {
assert_eq!(info.buckets_count, 0);
}
#[test]
fn identity_bound_partial_display_keeps_valid_buckets_and_unknown_state() {
use std::collections::HashMap;
let mut projected = DataUsageInfo {
last_update: Some(std::time::SystemTime::UNIX_EPOCH),
usage_snapshot_admin_projection: true,
usage_snapshot_partial: true,
usage_snapshot_converged: Some(false),
buckets_count: 1,
objects_total_count: 7,
objects_total_size: 70,
buckets_usage: HashMap::from([(
"cold".to_string(),
rustfs_data_usage::BucketUsageInfo {
size: 70,
objects_count: 7,
..Default::default()
},
)]),
bucket_sizes: HashMap::from([("cold".to_string(), 70)]),
..Default::default()
};
projected = DefaultAdminUsecase::present_identity_bound_data_usage_snapshot(
projected,
["cold".to_string(), "recreated".to_string()],
);
assert_eq!(projected.buckets_usage["cold"].objects_count, 7);
assert!(!projected.buckets_usage.contains_key("recreated"));
assert_eq!(projected.objects_total_count, 7);
assert_eq!(projected.usage_snapshot_converged, Some(false));
assert!(!projected.is_complete_bucket_usage_snapshot());
assert!(!projected.is_valid_partial_snapshot());
projected.buckets_usage.clear();
projected.bucket_sizes.clear();
projected.calculate_totals();
projected.buckets_count = 0;
let unknown = DefaultAdminUsecase::present_identity_bound_data_usage_snapshot(projected, ["recreated".to_string()]);
assert!(unknown.buckets_usage.is_empty());
assert!(unknown.usage_snapshot_admin_projection);
assert_eq!(unknown.usage_snapshot_converged, Some(false));
assert!(!unknown.is_complete_bucket_usage_snapshot());
assert_eq!(unknown.last_update, Some(std::time::SystemTime::UNIX_EPOCH));
}
/// A write into a bucket without a verified baseline leaves process-local
/// counts the overlay cannot reconcile, so it clears
/// `usage_snapshot_complete`. That must flag the response non-converged,
@@ -1009,11 +1048,15 @@ mod tests {
crate::app::storage_api::test::data_usage::record_bucket_object_write_memory(unverified_bucket, None, 1_024).await;
let info = DefaultAdminUsecase::present_data_usage_snapshot(
// Exercise the production overlay, followed by the same identity-bound
// presentation function used by the admin query.
crate::app::storage_api::test::data_usage::apply_bucket_usage_memory_overlay(&mut persisted).await;
persisted.usage_snapshot_admin_projection = true;
persisted.usage_snapshot_converged = Some(false);
let info = DefaultAdminUsecase::present_identity_bound_data_usage_snapshot(
persisted,
[measured_bucket.to_string(), unverified_bucket.to_string()],
)
.await;
);
assert_eq!(
info.buckets_usage
+13 -12
View File
@@ -1320,22 +1320,23 @@ mod tests {
};
for (index, expected_key) in ["a&local", "z-local"].into_iter().enumerate() {
let request_marker = input.marker.clone().unwrap_or_default();
let list_req = S3Request {
input: input.clone(),
method: http::Method::GET,
uri: http::Uri::from_static("/"),
headers: HeaderMap::new(),
extensions: http::Extensions::new(),
credentials: None,
region: None,
service: None,
trailing_headers: None,
};
let envelope = crate::app::RequestEnvelope::from_s3s(&list_req);
let gateway_input =
rustfs_gateway_types::compat::s3s_5761ddfe::generated::ops::list_objects::input_from_s3s(
input.clone(),
list_req.input,
)
.expect("convert ListObjectsInput");
let envelope = crate::app::RequestEnvelope::from_parts(
crate::app::RequestHead {
method: http::Method::GET,
uri: http::Uri::from_static("/"),
headers: HeaderMap::new(),
credentials: None,
region: None,
service: None,
},
http::Extensions::new(),
);
let gateway_output = tokio::time::timeout(
Duration::from_secs(10),
DefaultBucketUsecase::from_global().execute_list_objects(envelope, gateway_input),
+3 -2
View File
@@ -509,7 +509,7 @@ impl DefaultBucketUsecase {
#[cfg(test)]
mod tests {
use super::*;
use crate::app::bucket_usecase::test_support::{build_request_with_req_info, owned_envelope};
use crate::app::bucket_usecase::test_support::{build_request, build_request_with_req_info};
use crate::app::storage_api::s3::dto::CreateBucketInput;
use http::Method;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
@@ -1005,10 +1005,11 @@ mod tests {
#[tokio::test]
async fn execute_list_buckets_returns_internal_error_when_store_uninitialized() {
let req = build_request((), Method::GET);
let usecase = DefaultBucketUsecase::without_context();
let err = usecase
.execute_list_buckets(owned_envelope(Method::GET), gateway_dto::ListBucketsInput::default())
.execute_list_buckets(RequestEnvelope::from_s3s(&req), gateway_dto::ListBucketsInput::default())
.await
.unwrap_err();
assert_eq!(err.code(), &rustfs_s3_types::S3ErrorCode::InternalError);
+6 -5
View File
@@ -366,7 +366,7 @@ impl DefaultBucketUsecase {
#[cfg(test)]
mod tests {
use super::*;
use crate::app::bucket_usecase::test_support::{build_request_with_req_info, owned_envelope, s3_op_total};
use crate::app::bucket_usecase::test_support::{build_request, build_request_with_req_info, s3_op_total};
use http::Method;
#[test]
@@ -399,10 +399,11 @@ mod tests {
/// The envelope of a request whose access phase installed `req_info`, or none.
fn envelope(req_info: Option<ReqInfo>) -> RequestEnvelope {
match req_info {
Some(req_info) => RequestEnvelope::from_s3s(&build_request_with_req_info((), Method::PUT, req_info)),
None => owned_envelope(Method::PUT),
}
let req = match req_info {
Some(req_info) => build_request_with_req_info((), Method::PUT, req_info),
None => build_request((), Method::PUT),
};
RequestEnvelope::from_s3s(&req)
}
#[test]
+6 -5
View File
@@ -203,7 +203,7 @@ impl DefaultBucketUsecase {
#[cfg(test)]
mod tests {
use super::*;
use crate::app::bucket_usecase::test_support::owned_envelope;
use crate::app::bucket_usecase::test_support::build_request;
use crate::app::storage_api::s3::dto::{self, PutBucketEncryptionInput, ServerSideEncryption};
use http::Method;
use rustfs_gateway_types::persistence::{
@@ -292,19 +292,20 @@ mod tests {
.build()
.unwrap();
let prepared = crate::storage::persisted_from_s3s(&input.server_side_encryption_configuration);
let req = build_request(input, Method::PUT);
let prepared = crate::storage::persisted_from_s3s(&req.input.server_side_encryption_configuration);
let usecase = DefaultBucketUsecase::without_context();
let err = usecase
.execute_put_bucket_encryption(owned_envelope(Method::PUT), &input.bucket, prepared.map_err(Into::into))
.execute_put_bucket_encryption(RequestEnvelope::from_s3s(&req), &req.input.bucket, prepared.map_err(Into::into))
.await
.unwrap_err();
assert_eq!(err.code(), &S3ErrorCode::InternalError);
let err = usecase
.execute_put_bucket_encryption(
owned_envelope(Method::PUT),
&input.bucket,
RequestEnvelope::from_s3s(&req),
&req.input.bucket,
Ok(PersistedBucketEncryptionConfiguration { rules: vec![] }),
)
.await
+4 -3
View File
@@ -405,7 +405,7 @@ mod tests {
// waiter again on the single-thread test runtime.
assert_eq!(admission.available_permits(), 1);
}
use crate::app::bucket_usecase::test_support::{build_request, owned_envelope};
use crate::app::bucket_usecase::test_support::build_request;
use crate::app::gating_test_env::{run_large_stack_test, shared_gating_ambient};
use crate::app::runtime_sources::ServerContextSlot;
use crate::app::storage_api::bucket_usecase::helper::MutationCompletionProbe;
@@ -775,12 +775,13 @@ mod tests {
.build()
.unwrap();
let req = build_request(input, Method::PUT);
let usecase = DefaultBucketUsecase::without_context();
let err = usecase
.execute_put_bucket_lifecycle_configuration(
owned_envelope(Method::PUT),
&input.bucket,
RequestEnvelope::from_s3s(&req),
&req.input.bucket,
PreparedLifecycleConfiguration::Missing,
)
.await
@@ -159,7 +159,7 @@ fn list_object_versions_output(
#[cfg(test)]
mod tests {
use super::*;
use crate::app::bucket_usecase::test_support::owned_envelope;
use crate::app::bucket_usecase::test_support::build_request;
use rustfs_gateway_types::BucketName;
use rustfs_s3_types::S3ErrorCode;
@@ -319,7 +319,7 @@ mod tests {
};
let err = DefaultBucketUsecase::without_context()
.execute_list_object_versions(owned_envelope(http::Method::GET), input)
.execute_list_object_versions(RequestEnvelope::from_s3s(&build_request((), http::Method::GET)), input)
.await
.expect_err("a negative page size");
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
@@ -333,7 +333,7 @@ mod tests {
};
let err = DefaultBucketUsecase::without_context()
.execute_list_object_versions(owned_envelope(http::Method::GET), input)
.execute_list_object_versions(RequestEnvelope::from_s3s(&build_request((), http::Method::GET)), input)
.await
.expect_err("no store");
assert_eq!(err.code(), &S3ErrorCode::InternalError);
@@ -372,12 +372,13 @@ fn list_objects_next_marker(info: &ListObjectsV2Info, cursor: Option<&str>) -> O
#[cfg(test)]
mod tests {
use super::*;
use crate::app::bucket_usecase::test_support::owned_envelope;
use crate::app::bucket_usecase::test_support::build_request;
use rustfs_gateway_types::BucketName;
#[test]
#[serial_test::serial]
fn list_apis_report_storage_read_quorum_as_unavailable() {
use crate::app::bucket_usecase::test_support::owned_envelope;
use crate::app::gating_test_env::{run_large_stack_test, shared_gating_ambient, shared_gating_ecstore};
run_large_stack_test("list-quorum-surface", || async {
let usecase = DefaultBucketUsecase::with_context(Some(shared_gating_ambient().await));
@@ -484,7 +485,7 @@ mod tests {
}
fn envelope() -> RequestEnvelope {
owned_envelope(http::Method::GET)
RequestEnvelope::from_s3s(&build_request((), http::Method::GET))
}
#[test]
+3 -12
View File
@@ -18,7 +18,7 @@ use super::*;
use crate::app::storage_api::bucket_usecase::contract::list::ListOperations as _;
use crate::app::storage_api::s3::dto::{
CommonPrefix, DeleteMarkerEntry, EncodingType, ListObjectVersionsInput, ListObjectVersionsOutput, ListObjectsV2Input,
ListObjectsV2Output, MetadataEntry, Object, ObjectStorageClass, ObjectVersion, ObjectVersionStorageClass, Owner, Timestamp,
ListObjectsV2Output, MetadataEntry, Object, ObjectStorageClass, ObjectVersion, ObjectVersionStorageClass, Timestamp,
UserMetadata,
};
use crate::app::storage_api::s3::xml::{SerResult, SerializeContent};
@@ -373,15 +373,6 @@ async fn collect_list_objects_metadata_permissions(
Ok(permissions)
}
/// List metadata XML still carries the legacy owner shape. The id and display name are the owned identity.
fn legacy_list_owner() -> Owner {
let owner = super::rustfs_owner();
Owner {
display_name: owner.display_name,
id: owner.id,
}
}
fn build_list_object_versions_metadata_output(
object_infos: ListObjectVersionsInfo,
bucket: &str,
@@ -389,7 +380,7 @@ fn build_list_object_versions_metadata_output(
encoding_type: Option<&EncodingType>,
permissions: &HashMap<String, ObjectMetadataPermissions>,
) -> ListObjectVersionsMetadataOutput {
let owner = legacy_list_owner();
let owner = rustfs_owner();
let common_prefixes = object_infos
.prefixes
.into_iter()
@@ -517,7 +508,7 @@ fn build_list_objects_v2_metadata_output(
fetch_owner: bool,
permissions: &HashMap<String, ObjectMetadataPermissions>,
) -> ListObjectsV2MetadataOutput {
let owner = legacy_list_owner();
let owner = rustfs_owner();
let contents = object_infos
.objects
+4 -2
View File
@@ -56,7 +56,7 @@ impl DefaultBucketUsecase {
#[cfg(test)]
mod tests {
use super::*;
use crate::app::bucket_usecase::test_support::{build_request, owned_envelope};
use crate::app::bucket_usecase::test_support::build_request;
use crate::app::bucket_usecase::{BucketOperations, DeleteBucketOptions, MakeBucketOptions};
use crate::app::gating_test_env::{run_large_stack_test, shared_gating_ambient};
use crate::app::runtime_sources::ServerContextSlot;
@@ -75,8 +75,10 @@ mod tests {
let context = shared_gating_ambient().await;
let bucket = format!("logging-missing-{}", uuid::Uuid::new_v4().simple());
let usecase = DefaultBucketUsecase::with_context(Some(context));
let request = build_request((), Method::PUT);
let envelope = RequestEnvelope::from_s3s(&request);
let error = usecase
.execute_put_bucket_logging(owned_envelope(Method::PUT), &bucket, || {
.execute_put_bucket_logging(envelope, &bucket, || {
Err(rustfs_s3_types::s3_error!(MalformedXML, "invalid logging configuration"))
})
.await
+7 -4
View File
@@ -243,15 +243,16 @@ impl DefaultBucketUsecase {
#[cfg(test)]
mod tests {
use super::*;
use crate::app::bucket_usecase::test_support::owned_envelope;
use crate::app::bucket_usecase::test_support::build_request;
use http::Method;
#[tokio::test]
async fn execute_delete_bucket_policy_returns_internal_error_when_store_uninitialized() {
let req = build_request((), Method::DELETE);
let usecase = DefaultBucketUsecase::without_context();
let err = usecase
.execute_delete_bucket_policy(owned_envelope(Method::DELETE), "test-bucket")
.execute_delete_bucket_policy(RequestEnvelope::from_s3s(&req), "test-bucket")
.await
.unwrap_err();
assert_eq!(err.code(), &rustfs_s3_types::S3ErrorCode::InternalError);
@@ -279,10 +280,11 @@ mod tests {
bucket: bucket("test-bucket"),
..Default::default()
};
let req = build_request((), Method::GET);
let usecase = DefaultBucketUsecase::without_context();
let err = usecase
.execute_get_bucket_policy_status(owned_envelope(Method::GET), input)
.execute_get_bucket_policy_status(RequestEnvelope::from_s3s(&req), input)
.await
.unwrap_err();
assert_eq!(err.code(), &rustfs_s3_types::S3ErrorCode::InternalError);
@@ -290,10 +292,11 @@ mod tests {
#[tokio::test]
async fn execute_put_bucket_policy_returns_internal_error_when_store_uninitialized() {
let req = build_request((), Method::PUT);
let usecase = DefaultBucketUsecase::without_context();
let err = usecase
.execute_put_bucket_policy(owned_envelope(Method::PUT), "test-bucket", "{}")
.execute_put_bucket_policy(RequestEnvelope::from_s3s(&req), "test-bucket", "{}")
.await
.unwrap_err();
assert_eq!(err.code(), &rustfs_s3_types::S3ErrorCode::InternalError);
@@ -112,7 +112,7 @@ impl DefaultBucketUsecase {
#[cfg(test)]
mod tests {
use super::*;
use crate::app::bucket_usecase::test_support::{build_request, owned_envelope};
use crate::app::bucket_usecase::test_support::build_request;
use crate::app::gating_test_env::{run_large_stack_test, shared_gating_ambient};
use crate::app::runtime_sources::ServerContextSlot;
use crate::app::storage_api::bucket_usecase::helper::MutationCompletionProbe;
@@ -278,7 +278,7 @@ mod tests {
#[tokio::test]
async fn execute_delete_public_access_block_returns_internal_error_when_store_uninitialized() {
let envelope = owned_envelope(Method::DELETE);
let envelope = RequestEnvelope::from_s3s(&build_request((), Method::DELETE));
let usecase = DefaultBucketUsecase::without_context();
let err = usecase
.execute_delete_public_access_block(envelope, "test-bucket")
@@ -296,7 +296,7 @@ mod tests {
#[tokio::test]
async fn execute_put_public_access_block_returns_internal_error_when_store_uninitialized() {
let envelope = owned_envelope(Method::PUT);
let envelope = RequestEnvelope::from_s3s(&build_request((), Method::PUT));
let usecase = DefaultBucketUsecase::without_context();
let err = usecase
.execute_put_public_access_block(envelope, "test-bucket", Ok(PersistedPublicAccessBlockConfiguration::default()))
+4 -3
View File
@@ -138,7 +138,7 @@ impl DefaultBucketUsecase {
#[cfg(test)]
mod tests {
use super::*;
use crate::app::bucket_usecase::test_support::{build_request, owned_envelope};
use crate::app::bucket_usecase::test_support::build_request;
use crate::app::storage_api::s3::dto::{self, PutBucketTaggingInput, Tag, Tagging};
use http::Method;
use rustfs_s3_types::S3ErrorCode;
@@ -249,10 +249,11 @@ mod tests {
.build()
.unwrap();
let req = build_request(input, Method::PUT);
let usecase = DefaultBucketUsecase::without_context();
let err = usecase
.execute_put_bucket_tagging(owned_envelope(Method::PUT), &input.bucket, PersistedTagging::default())
.execute_put_bucket_tagging(RequestEnvelope::from_s3s(&req), &req.input.bucket, PersistedTagging::default())
.await
.unwrap_err();
assert_eq!(err.code(), &S3ErrorCode::InternalError);
@@ -266,7 +267,7 @@ mod tests {
(vec![tag; MAX_BUCKET_TAGS + 1], "Cannot have more than 50 tags per bucket"),
] {
let err = usecase
.execute_put_bucket_tagging(owned_envelope(Method::PUT), &input.bucket, PersistedTagging { tag_set })
.execute_put_bucket_tagging(RequestEnvelope::from_s3s(&req), &req.input.bucket, PersistedTagging { tag_set })
.await
.unwrap_err();
assert_eq!(err.code(), &S3ErrorCode::InvalidTag);
+1 -11
View File
@@ -47,17 +47,7 @@ pub(super) fn gateway_bucket(name: &str) -> rustfs_gateway_types::BucketName {
/// The envelope of a request that carries nothing but `method`.
pub(super) fn owned_envelope(method: Method) -> RequestEnvelope {
RequestEnvelope::from_parts(
crate::app::RequestHead {
method,
uri: Uri::from_static("/"),
headers: HeaderMap::new(),
credentials: None,
region: None,
service: None,
},
Extensions::new(),
)
RequestEnvelope::from_s3s(&build_request((), method))
}
pub(super) fn build_request_with_req_info<T>(input: T, method: Method, req_info: ReqInfo) -> S3Request<T> {
+5 -4
View File
@@ -81,7 +81,7 @@ impl DefaultBucketUsecase {
mod tests {
use super::super::{BUCKET_ACCELERATE_CONFIG, BUCKET_REQUEST_PAYMENT_CONFIG};
use super::*;
use crate::app::bucket_usecase::test_support::{build_request, build_request_with_req_info, owned_envelope};
use crate::app::bucket_usecase::test_support::{build_request, build_request_with_req_info};
use crate::app::gating_test_env::{run_large_stack_test, shared_gating_ambient};
use crate::app::runtime_sources::ServerContextSlot;
use crate::app::storage_api::object_usecase::helper::MutationCompletionProbe;
@@ -190,21 +190,22 @@ mod tests {
let ambient = shared_gating_ambient().await;
let usecase = DefaultBucketUsecase::with_context(Some(ambient));
let bucket = format!("core-config-missing-{}", uuid::Uuid::new_v4().simple());
let request = build_request((), http::Method::PUT);
let mut converted = [false; 3];
let accelerate = usecase
.execute_put_bucket_accelerate_configuration(owned_envelope(http::Method::PUT), &bucket, || {
.execute_put_bucket_accelerate_configuration(RequestEnvelope::from_s3s(&request), &bucket, || {
converted[0] = true;
Err(rustfs_s3_types::s3_error!(MalformedXML, "invalid acceleration"))
})
.await;
let payment = usecase
.execute_put_bucket_request_payment(owned_envelope(http::Method::PUT), &bucket, || {
.execute_put_bucket_request_payment(RequestEnvelope::from_s3s(&request), &bucket, || {
converted[1] = true;
Err(rustfs_s3_types::s3_error!(MalformedXML, "invalid request payment"))
})
.await;
let website = usecase
.execute_put_bucket_website(owned_envelope(http::Method::PUT), &bucket, || {
.execute_put_bucket_website(RequestEnvelope::from_s3s(&request), &bucket, || {
converted[2] = true;
Err(rustfs_s3_types::s3_error!(MalformedXML, "invalid website"))
})
@@ -108,6 +108,17 @@ async fn persist_scanner_snapshot(ecstore: Arc<ECStore>, info: DataUsageInfo) {
rustfs_scanner::scanner::store_data_usage_in_backend(CancellationToken::new(), ecstore, receiver).await;
}
async fn bind_current_bucket_incarnations(ecstore: &ECStore, info: &mut DataUsageInfo) {
for (bucket, usage) in &mut info.buckets_usage {
let incarnation = ecstore
.bucket_incarnation_id_from_disk(bucket)
.await
.expect("load the current incarnation for the covered snapshot");
assert!(!incarnation.is_nil(), "covered snapshot must bind a non-nil bucket identity");
usage.bucket_incarnation = Some(incarnation.to_string());
}
}
#[tokio::test]
#[serial]
async fn data_usage_endpoint_serves_snapshot_without_live_listing() {
@@ -174,6 +185,7 @@ async fn data_usage_endpoint_serves_snapshot_without_live_listing() {
seeded_info.bucket_sizes.insert(overlay_bucket.clone(), 0);
seeded_info.buckets_count = u64::try_from(seeded_info.buckets_usage.len()).expect("test bucket count should fit into u64");
seeded_info.calculate_totals();
bind_current_bucket_incarnations(&ecstore, &mut seeded_info).await;
persist_scanner_snapshot(ecstore.clone(), seeded_info).await;
record_bucket_object_write_memory(&overlay_bucket, None, 512).await;
@@ -341,6 +353,7 @@ async fn data_usage_endpoint_treats_legacy_partial_stats_as_unknown_until_covere
completed.bucket_sizes.insert(small_bucket.clone(), 57_344);
completed.buckets_count = u64::try_from(completed.buckets_usage.len()).expect("test bucket count should fit into u64");
completed.calculate_totals();
bind_current_bucket_incarnations(&ecstore, &mut completed).await;
store_data_usage_in_backend(completed, ecstore.clone())
.await
.expect("persist complete covered snapshot");
@@ -52,19 +52,8 @@ async fn object_lock_handlers_schedule_replication() {
.expect("Failed to create object-lock bucket");
upload_test_object(&ecstore, bucket.as_str(), object, b"worm payload").await;
let envelope = || {
RequestEnvelope::from_parts(
crate::app::RequestHead {
method: Method::PUT,
uri: Uri::from_static("/"),
headers: HeaderMap::new(),
credentials: None,
region: None,
service: None,
},
Extensions::new(),
)
};
let req = build_request((), Method::PUT);
let envelope = || RequestEnvelope::from_s3s(&req);
// A bucket without a replication configuration can skip the decision.
MUST_REPLICATE_OBJECT_CALLS.store(0, Ordering::SeqCst);
@@ -22,17 +22,7 @@ const RESTORE_SUSPENDED_WAIT_TIMEOUT: Duration = Duration::from_secs(15);
async fn suspend_test_bucket(bucket: &str) {
DefaultBucketUsecase::from_global()
.execute_put_bucket_versioning(
crate::app::RequestEnvelope::from_parts(
crate::app::RequestHead {
method: Method::PUT,
uri: Uri::from_static("/"),
headers: HeaderMap::new(),
credentials: None,
region: None,
service: None,
},
Extensions::new(),
),
crate::app::RequestEnvelope::from_s3s(&build_request((), Method::PUT)),
bucket,
Ok(rustfs_gateway_types::persistence::PersistedVersioningConfiguration {
status: Some("Suspended".to_owned()),
+7 -20
View File
@@ -62,7 +62,7 @@ use super::storage_api::multipart_usecase::s3_api::multipart::{
};
use super::storage_api::multipart_usecase::set_disk::is_valid_storage_class;
use super::storage_api::multipart_usecase::sse::{
DecryptionRequest, EncryptionKeyKind, EncryptionRequest, PrepareEncryptionRequest, SseKmsPrincipal, SseKmsPrincipalExt,
DecryptionRequest, EncryptionKeyKind, EncryptionRequest, PrepareEncryptionRequest, SseKmsPrincipal,
apply_bucket_default_lock_retention, authorize_sse_kms_multipart_write, authorize_sse_kms_object_read,
build_ssec_read_headers, encryption_material_to_metadata, extract_server_side_encryption_from_headers,
extract_ssec_params_from_headers, extract_ssekms_context_from_headers, get_buffer_size_opt_in,
@@ -545,17 +545,10 @@ mod tests {
.key(key.to_owned())
.build()
.expect("valid create request DTO");
let error = match make_usecase()
.execute_create_multipart_upload(
owned_envelope(Method::POST),
super::create_values::CreateMultipartUploadFields::from_legacy(create),
None,
)
let error = make_usecase()
.execute_create_multipart_upload_from_s3s(build_request(create, Method::POST))
.await
{
Err(error) => crate::app::storage_api::s3::S3Error::from(error),
Ok(_) => panic!("reserved key must be rejected before accessing storage"),
};
.expect_err("reserved key must be rejected before accessing storage");
assert_eq!(error.code(), &crate::app::storage_api::s3::S3ErrorCode::InvalidArgument);
let complete = CompleteMultipartUploadInput::builder()
@@ -565,15 +558,9 @@ mod tests {
.multipart_upload(Some(CompletedMultipartUpload::default()))
.build()
.expect("valid complete request DTO");
let error =
match rustfs_gateway_types::compat::s3s_5761ddfe::generated::ops::complete_multipart_upload::input_from_s3s(
complete,
) {
Err(error) => crate::app::dto_values::seam_error(error),
Ok(input) => Box::pin(make_usecase().execute_complete_multipart_upload(owned_envelope(Method::POST), input))
.await
.expect_err("uploads created before the fix must not publish reserved keys"),
};
let error = Box::pin(make_usecase().complete_for_s3s_request(build_request(complete, Method::POST)))
.await
.expect_err("uploads created before the fix must not publish reserved keys");
assert_eq!(error.code(), &rustfs_s3_types::S3ErrorCode::InvalidArgument);
}
}
@@ -24,7 +24,8 @@ use std::sync::atomic::{AtomicUsize, Ordering};
#[tokio::test]
async fn execute_abort_multipart_upload_returns_internal_error_when_store_uninitialized() {
let envelope = owned_envelope(Method::DELETE);
let request = build_request((), Method::DELETE);
let envelope = RequestEnvelope::from_s3s(&request);
let before = s3_op_total(S3Operation::AbortMultipartUpload);
let target = AbortMultipartUploadTarget::Legacy {
bucket: "bucket".to_owned(),
+38 -233
View File
@@ -943,72 +943,21 @@ mod tests {
#[tokio::test]
async fn execute_complete_multipart_upload_rejects_missing_parts_payload() {
use rustfs_gateway_types::compat::s3s_5761ddfe::generated::ops::complete_multipart_upload;
let input = CompleteMultipartUploadInput::builder()
.bucket("bucket".to_string())
.key("object".to_string())
.upload_id("upload-id".to_string())
.build()
.unwrap();
let mut s3s_input = input;
let has_upload = s3s_input.multipart_upload.is_some();
s3s_input.multipart_upload.get_or_insert_with(Default::default);
let tags: Vec<Option<String>> = s3s_input
.multipart_upload
.iter_mut()
.flat_map(|upload| upload.parts.iter_mut().flatten())
.map(|part| {
part.part_number.get_or_insert(0);
part.e_tag.take().map(|tag| tag.value().to_owned())
})
.collect();
for checksum in [
&mut s3s_input.checksum_crc32,
&mut s3s_input.checksum_crc32c,
&mut s3s_input.checksum_crc64nvme,
&mut s3s_input.checksum_md5,
&mut s3s_input.checksum_sha1,
&mut s3s_input.checksum_sha256,
&mut s3s_input.checksum_sha512,
&mut s3s_input.checksum_xxhash128,
&mut s3s_input.checksum_xxhash3,
&mut s3s_input.checksum_xxhash64,
] {
*checksum = None;
}
let err = match complete_multipart_upload::input_from_s3s(s3s_input) {
Err(error) => crate::app::dto_values::seam_error(error),
Ok(mut owned) => {
for (part, tag) in owned.multipart_upload.parts.iter_mut().zip(tags) {
part.e_tag = tag.map(rustfs_gateway_types::ETag::from_stored);
}
let taken = std::mem::take(&mut owned.multipart_upload);
let parts = has_upload.then_some(taken);
let result = match parts {
Some(parts) => {
owned.multipart_upload = parts;
Box::pin(make_usecase().execute_complete_multipart_upload(owned_envelope(Method::POST), owned)).await
}
None => {
Box::pin(make_usecase().execute_complete_multipart_upload_parts(
owned_envelope(Method::POST),
owned,
None,
))
.await
}
};
result.unwrap_err()
}
};
let req = build_request(input, Method::POST);
let err = Box::pin(make_usecase().complete_for_s3s_request(req)).await.unwrap_err();
assert_eq!(err.code(), &S3ErrorCode::InvalidPart);
}
#[tokio::test]
async fn execute_complete_multipart_upload_rejects_missing_parts_list() {
use crate::app::storage_api::s3::xml::Deserialize as _;
use rustfs_gateway_types::compat::s3s_5761ddfe::generated::ops::complete_multipart_upload;
let mut deserializer = crate::app::storage_api::s3::xml::Deserializer::new(b"<CompleteMultipartUpload/>");
let multipart_upload =
@@ -1025,57 +974,11 @@ mod tests {
.multipart_upload(Some(multipart_upload))
.build()
.expect("complete multipart input should build");
let mut s3s_input = input;
let has_upload = s3s_input.multipart_upload.is_some();
s3s_input.multipart_upload.get_or_insert_with(Default::default);
let tags: Vec<Option<String>> = s3s_input
.multipart_upload
.iter_mut()
.flat_map(|upload| upload.parts.iter_mut().flatten())
.map(|part| {
part.part_number.get_or_insert(0);
part.e_tag.take().map(|tag| tag.value().to_owned())
})
.collect();
for checksum in [
&mut s3s_input.checksum_crc32,
&mut s3s_input.checksum_crc32c,
&mut s3s_input.checksum_crc64nvme,
&mut s3s_input.checksum_md5,
&mut s3s_input.checksum_sha1,
&mut s3s_input.checksum_sha256,
&mut s3s_input.checksum_sha512,
&mut s3s_input.checksum_xxhash128,
&mut s3s_input.checksum_xxhash3,
&mut s3s_input.checksum_xxhash64,
] {
*checksum = None;
}
let err = match complete_multipart_upload::input_from_s3s(s3s_input) {
Err(error) => crate::app::dto_values::seam_error(error),
Ok(mut owned) => {
for (part, tag) in owned.multipart_upload.parts.iter_mut().zip(tags) {
part.e_tag = tag.map(rustfs_gateway_types::ETag::from_stored);
}
let taken = std::mem::take(&mut owned.multipart_upload);
let parts = has_upload.then_some(taken);
let result = match parts {
Some(parts) => {
owned.multipart_upload = parts;
Box::pin(make_usecase().execute_complete_multipart_upload(owned_envelope(Method::POST), owned)).await
}
None => {
Box::pin(make_usecase().execute_complete_multipart_upload_parts(
owned_envelope(Method::POST),
owned,
None,
))
.await
}
};
result.expect_err("missing parts list must be rejected")
}
};
let req = build_request(input, Method::POST);
let err = Box::pin(make_usecase().complete_for_s3s_request(req))
.await
.expect_err("missing parts list must be rejected");
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
}
@@ -1221,9 +1124,10 @@ mod tests {
crate::app::gating_test_env::run_large_stack_test("ssec-completion-checksums", || async {
let (_store, bucket) =
crate::app::gating_test_env::durable_quota_test_bucket("ssec-completion-checksums", 16_384).await;
let versioning_req = build_request((), Method::PUT);
crate::app::bucket_usecase::DefaultBucketUsecase::from_global()
.execute_put_bucket_versioning(
super::super::test_support::owned_envelope(Method::PUT),
RequestEnvelope::from_s3s(&versioning_req),
&bucket,
Ok(rustfs_gateway_types::persistence::PersistedVersioningConfiguration {
status: Some("Enabled".to_owned()),
@@ -1400,13 +1304,16 @@ mod tests {
let bucket_usecase = crate::app::bucket_usecase::DefaultBucketUsecase::from_global();
// Listings carry neither the SSE-C key nor a per-object decryption request.
let versions_input = ListObjectVersionsInput {
bucket: bucket.to_string(),
prefix: Some(key.to_string()),
..Default::default()
};
let versions_env = super::super::test_support::owned_envelope(Method::GET);
let versions_in = list_object_versions::input_from_s3s(versions_input).expect("convert ListObjectVersionsInput");
let versions_req = build_request(
ListObjectVersionsInput {
bucket: bucket.to_string(),
prefix: Some(key.to_string()),
..Default::default()
},
Method::GET,
);
let versions_env = RequestEnvelope::from_s3s(&versions_req);
let versions_in = list_object_versions::input_from_s3s(versions_req.input).expect("convert ListObjectVersionsInput");
let versions = list_object_versions::output_to_s3s(
bucket_usecase
.execute_list_object_versions(versions_env, versions_in)
@@ -1421,13 +1328,16 @@ mod tests {
assert_eq!(entries[0].e_tag.as_ref(), Some(etag));
assert_xml_etag(&versions, etag);
let v1_input = ListObjectsInput {
bucket: bucket.to_string(),
prefix: Some(key.to_string()),
..Default::default()
};
let v1_env = super::super::test_support::owned_envelope(Method::GET);
let v1_in = list_objects::input_from_s3s(v1_input).expect("convert ListObjectsInput");
let v1_req = build_request(
ListObjectsInput {
bucket: bucket.to_string(),
prefix: Some(key.to_string()),
..Default::default()
},
Method::GET,
);
let v1_env = RequestEnvelope::from_s3s(&v1_req);
let v1_in = list_objects::input_from_s3s(v1_req.input).expect("convert ListObjectsInput");
let v1 = list_objects::output_to_s3s(
bucket_usecase
.execute_list_objects(v1_env, v1_in)
@@ -1501,6 +1411,7 @@ mod tests {
.await
.expect("create versioned Object Lock bucket");
let key = "kms-encrypted";
let enc_req = build_request((), Method::PUT);
let enc_cfg = crate::app::storage_api::s3::dto::ServerSideEncryptionConfiguration {
rules: vec![crate::app::storage_api::s3::dto::ServerSideEncryptionRule {
apply_server_side_encryption_by_default: Some(
@@ -1514,7 +1425,7 @@ mod tests {
};
let prepared = crate::storage::persisted_from_s3s(&enc_cfg).map_err(Into::into);
crate::app::bucket_usecase::DefaultBucketUsecase::from_global()
.execute_put_bucket_encryption(super::super::test_support::owned_envelope(Method::PUT), &bucket, prepared)
.execute_put_bucket_encryption(RequestEnvelope::from_s3s(&enc_req), &bucket, prepared)
.await
.expect("configure bucket-default SSE-KMS");
let usecase = DefaultMultipartUsecase::from_global();
@@ -2240,8 +2151,6 @@ mod tests {
#[tokio::test]
async fn execute_complete_multipart_upload_allows_duplicate_part_numbers_by_using_last_occurrence() {
use rustfs_gateway_types::compat::s3s_5761ddfe::generated::ops::complete_multipart_upload;
let multipart_upload = CompletedMultipartUpload {
parts: Some(vec![
CompletedPart {
@@ -2261,67 +2170,14 @@ mod tests {
.multipart_upload(Some(multipart_upload))
.build()
.unwrap();
let mut s3s_input = input;
let has_upload = s3s_input.multipart_upload.is_some();
s3s_input.multipart_upload.get_or_insert_with(Default::default);
let tags: Vec<Option<String>> = s3s_input
.multipart_upload
.iter_mut()
.flat_map(|upload| upload.parts.iter_mut().flatten())
.map(|part| {
part.part_number.get_or_insert(0);
part.e_tag.take().map(|tag| tag.value().to_owned())
})
.collect();
for checksum in [
&mut s3s_input.checksum_crc32,
&mut s3s_input.checksum_crc32c,
&mut s3s_input.checksum_crc64nvme,
&mut s3s_input.checksum_md5,
&mut s3s_input.checksum_sha1,
&mut s3s_input.checksum_sha256,
&mut s3s_input.checksum_sha512,
&mut s3s_input.checksum_xxhash128,
&mut s3s_input.checksum_xxhash3,
&mut s3s_input.checksum_xxhash64,
] {
*checksum = None;
}
let err = match complete_multipart_upload::input_from_s3s(s3s_input) {
Err(error) => crate::app::dto_values::seam_error(error),
Ok(mut owned) => {
for (part, tag) in owned.multipart_upload.parts.iter_mut().zip(tags) {
part.e_tag = tag.map(rustfs_gateway_types::ETag::from_stored);
}
let taken = std::mem::take(&mut owned.multipart_upload);
let parts = has_upload.then_some(taken);
let result = match parts {
Some(parts) => {
owned.multipart_upload = parts;
Box::pin(make_usecase().execute_complete_multipart_upload(owned_envelope(Method::POST), owned)).await
}
None => {
Box::pin(make_usecase().execute_complete_multipart_upload_parts(
owned_envelope(Method::POST),
owned,
None,
))
.await
}
};
match result {
Err(error) => error,
Ok(_output) => panic!("duplicate part numbers must not succeed before the order check"),
}
}
};
let req = build_request(input, Method::POST);
let err = Box::pin(make_usecase().complete_for_s3s_request(req)).await.unwrap_err();
assert_ne!(err.code(), &S3ErrorCode::InvalidPartOrder);
}
#[tokio::test]
async fn execute_complete_multipart_upload_rejects_out_of_order_parts() {
use rustfs_gateway_types::compat::s3s_5761ddfe::generated::ops::complete_multipart_upload;
let multipart_upload = CompletedMultipartUpload {
parts: Some(vec![
CompletedPart {
@@ -2341,60 +2197,9 @@ mod tests {
.multipart_upload(Some(multipart_upload))
.build()
.unwrap();
let mut s3s_input = input;
let has_upload = s3s_input.multipart_upload.is_some();
s3s_input.multipart_upload.get_or_insert_with(Default::default);
let tags: Vec<Option<String>> = s3s_input
.multipart_upload
.iter_mut()
.flat_map(|upload| upload.parts.iter_mut().flatten())
.map(|part| {
part.part_number.get_or_insert(0);
part.e_tag.take().map(|tag| tag.value().to_owned())
})
.collect();
for checksum in [
&mut s3s_input.checksum_crc32,
&mut s3s_input.checksum_crc32c,
&mut s3s_input.checksum_crc64nvme,
&mut s3s_input.checksum_md5,
&mut s3s_input.checksum_sha1,
&mut s3s_input.checksum_sha256,
&mut s3s_input.checksum_sha512,
&mut s3s_input.checksum_xxhash128,
&mut s3s_input.checksum_xxhash3,
&mut s3s_input.checksum_xxhash64,
] {
*checksum = None;
}
let err = match complete_multipart_upload::input_from_s3s(s3s_input) {
Err(error) => crate::app::dto_values::seam_error(error),
Ok(mut owned) => {
for (part, tag) in owned.multipart_upload.parts.iter_mut().zip(tags) {
part.e_tag = tag.map(rustfs_gateway_types::ETag::from_stored);
}
let taken = std::mem::take(&mut owned.multipart_upload);
let parts = has_upload.then_some(taken);
let result = match parts {
Some(parts) => {
owned.multipart_upload = parts;
Box::pin(make_usecase().execute_complete_multipart_upload(owned_envelope(Method::POST), owned)).await
}
None => {
Box::pin(make_usecase().execute_complete_multipart_upload_parts(
owned_envelope(Method::POST),
owned,
None,
))
.await
}
};
match result {
Err(error) => error,
Ok(_output) => panic!("out-of-order parts must be rejected"),
}
}
};
let req = build_request(input, Method::POST);
let err = Box::pin(make_usecase().complete_for_s3s_request(req)).await.unwrap_err();
assert_eq!(err.code(), &S3ErrorCode::InvalidPartOrder);
}
+8 -16
View File
@@ -545,17 +545,10 @@ mod tests {
.object_lock_retain_until_date(date.map(Timestamp::from))
.build()
.expect("direct fixture DTO");
let error = match make_usecase()
.execute_create_multipart_upload(
owned_envelope(Method::POST),
CreateMultipartUploadFields::from_legacy(input),
None,
)
let error = make_usecase()
.execute_create_multipart_upload_from_s3s(build_request(input, Method::POST))
.await
{
Err(error) => crate::app::storage_api::s3::S3Error::from(error),
Ok(_) => panic!("direct fixture does not create an upload"),
};
.expect_err("direct fixture does not create an upload");
assert_eq!(
(error.code().as_str(), error.message()),
(expected.0, Some(expected.1)),
@@ -597,13 +590,12 @@ mod tests {
.storage_class(Some(StorageClass::from("invalid".to_string())))
.build()
.unwrap();
let err = match make_usecase()
.execute_create_multipart_upload(owned_envelope(Method::POST), CreateMultipartUploadFields::from_legacy(input), None)
let req = build_request(input, Method::POST);
let err = make_usecase()
.execute_create_multipart_upload_from_s3s(req)
.await
{
Err(error) => crate::app::storage_api::s3::S3Error::from(error),
Ok(_) => panic!("invalid storage class must be rejected"),
};
.unwrap_err();
assert_eq!(err.code(), &S3ErrorCode::InvalidStorageClass);
}
+2 -2
View File
@@ -295,7 +295,7 @@ mod tests {
async fn list_uploads(input: gateway_dto::ListMultipartUploadsInput) -> rustfs_s3_types::S3Error {
make_usecase()
.execute_list_multipart_uploads(owned_envelope(Method::GET), input)
.execute_list_multipart_uploads(RequestEnvelope::from_s3s(&build_request((), Method::GET)), input)
.await
.expect_err("refused")
}
@@ -393,7 +393,7 @@ mod tests {
max_parts,
};
make_usecase()
.execute_list_parts(owned_envelope(Method::GET), fields)
.execute_list_parts(RequestEnvelope::from_s3s(&build_request((), Method::GET)), fields)
.await
.expect_err("refused")
}
@@ -14,7 +14,7 @@
//! ListParts execution against real multipart metadata and the native claim exchange.
use super::super::test_support::{Method, build_request, owned_envelope};
use super::super::test_support::{Method, build_request};
use super::*;
use crate::app::gating_test_env::{run_large_stack_test, shared_gating_ambient, shared_gating_ecstore_and_disk_paths};
use crate::app::storage_api::multipart_usecase::access::{AuthorizationRequest, load_bucket_generation_from_store};
@@ -81,7 +81,7 @@ fn list_parts_resolves_real_owners_and_retains_storage_errors() {
let key = "fixture/one";
let foreign_key = "fixture/two";
let opts = ObjectOptions::default();
let envelope = || owned_envelope(Method::GET);
let envelope = || RequestEnvelope::from_s3s(&build_request((), Method::GET));
for name in [&bucket, &foreign_bucket] {
store
.make_bucket(name, &MakeBucketOptions::default())
@@ -18,7 +18,6 @@ use super::*;
use crate::app::storage_api::s3::S3Request;
pub(super) use crate::app::storage_api::s3::dto::CompletedMultipartUpload;
pub(super) use crate::app::storage_api::s3::dto::StorageClass;
use crate::app::{RequestEnvelope, RequestHead};
pub(super) use http::Extensions;
pub(super) use http::Method;
pub(super) use rustfs_filemeta::ObjectPartInfo;
@@ -51,21 +50,6 @@ pub(super) fn build_request<T>(input: T, method: Method) -> S3Request<T> {
}
}
/// The envelope of a request that carries nothing but `method`.
pub(super) fn owned_envelope(method: Method) -> RequestEnvelope {
RequestEnvelope::from_parts(
RequestHead {
method,
uri: Uri::from_static("/"),
headers: HeaderMap::new(),
credentials: None,
region: None,
service: None,
},
Extensions::new(),
)
}
pub(super) fn make_usecase() -> DefaultMultipartUsecase {
DefaultMultipartUsecase::without_context()
}
+13 -29
View File
@@ -382,8 +382,7 @@ impl DefaultMultipartUsecase {
content_size: actual_size,
principal: None,
}
.validate_multipart_ssec(&fi.user_defined)
.map_err(ApiError::from)?;
.validate_multipart_ssec(&fi.user_defined)?;
}
let (requested_sse, requested_kms_key_id) = if has_ssec {
let ssec_material = sse_decryption(DecryptionRequest {
@@ -742,13 +741,9 @@ mod tests {
.part_number(1)
.build()
.unwrap();
let err = match make_usecase()
.execute_upload_part_with_audit(owned_envelope(Method::PUT), UploadPartWriteInput::from_legacy(input, None))
.await
{
Err(error) => crate::app::storage_api::s3::S3Error::from(error),
Ok(_) => panic!("missing body must be rejected"),
};
let req = build_request(input, Method::PUT);
let err = make_usecase().execute_upload_part(req).await.unwrap_err();
assert_eq!(err.code(), &LegacyErrorCode::IncompleteBody);
}
@@ -770,13 +765,9 @@ mod tests {
.content_length(Some(declared))
.build()
.unwrap();
let err = match make_usecase()
.execute_upload_part_with_audit(owned_envelope(Method::PUT), UploadPartWriteInput::from_legacy(input, None))
.await
{
Err(error) => crate::app::storage_api::s3::S3Error::from(error),
Ok(_) => panic!("declared part must be rejected"),
};
let req = build_request(input, Method::PUT);
let err = make_usecase().execute_upload_part(req).await.unwrap_err();
if expect_too_large {
assert_eq!(err.code(), &LegacyErrorCode::EntityTooLarge, "declared {declared}");
assert_eq!(polls.load(std::sync::atomic::Ordering::SeqCst), 0, "body must not be polled");
@@ -796,13 +787,9 @@ mod tests {
.part_number(part_number)
.build()
.unwrap();
let err = match make_usecase()
.execute_upload_part_with_audit(owned_envelope(Method::PUT), UploadPartWriteInput::from_legacy(input, None))
.await
{
Err(error) => crate::app::storage_api::s3::S3Error::from(error),
Ok(_) => panic!("invalid part number must be rejected"),
};
let req = build_request(input, Method::PUT);
let err = make_usecase().execute_upload_part(req).await.unwrap_err();
assert_eq!(err.code(), &LegacyErrorCode::InvalidArgument);
assert_eq!(err.message(), Some("partNumber must be between 1 and 10000"));
}
@@ -984,13 +971,10 @@ mod tests {
.build()
.expect("upload part input should build");
let err = match usecase
.execute_upload_part_with_audit(owned_envelope(Method::PUT), UploadPartWriteInput::from_legacy(input, None))
let err = usecase
.execute_upload_part(build_request(input, Method::PUT))
.await
{
Err(error) => crate::app::storage_api::s3::S3Error::from(error),
Ok(_) => panic!("full foreground write admission should reject before body ingest"),
};
.expect_err("full foreground write admission should reject before body ingest");
assert_eq!(err.code(), &LegacyErrorCode::SlowDown);
drop(held);
}
@@ -14,7 +14,6 @@
//! Upload target ownership and refusal order against real multipart state.
use super::super::super::test_support::owned_envelope;
use super::*;
use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions};
use rustfs_gateway_types::{BucketName, ObjectKey, UploadIdClaim};
@@ -102,7 +101,7 @@ async fn native_claim_checks_order_ownership_and_one_active_lookup() {
Body::from(Bytes::from_static(b"abc"))
}
});
let envelope = owned_envelope(Method::PUT);
let envelope = RequestEnvelope::from_s3s(&build_request((), Method::PUT));
LOOKUP
.scope((Cell::new(0), mode), async {
let result = usecase.execute_upload_part_with_audit(envelope, input).await;
@@ -212,7 +211,7 @@ fn native_copy_part_rejects_unproved_owner_without_changing_parts() {
copy_source_sse_customer_key: None,
copy_source_sse_customer_key_md5: None,
};
let envelope = owned_envelope(Method::PUT);
let envelope = RequestEnvelope::from_s3s(&build_request((), Method::PUT));
LOOKUP
.scope((Cell::new(0), mode), async {
let result = Box::pin(usecase.execute_upload_part_copy_with_audit(envelope, input)).await;
@@ -313,8 +313,7 @@ impl DefaultMultipartUsecase {
content_size: 0,
principal: None,
}
.validate_multipart_ssec(&mp_info.user_defined)
.map_err(ApiError::from)?;
.validate_multipart_ssec(&mp_info.user_defined)?;
let src_versioned = BucketVersioningSys::prefix_enabled(&src_bucket, &src_key).await;
let src_version_suspended = BucketVersioningSys::prefix_suspended(&src_bucket, &src_key).await;
@@ -562,9 +561,6 @@ impl DefaultMultipartUsecase {
mod tests {
use super::super::test_support::*;
use super::*;
use crate::app::multipart_usecase::UploadPartTarget;
use crate::app::object_usecase::CopySourceCondition;
use crate::app::storage_api::s3::dto::ETag;
#[tokio::test]
async fn execute_upload_part_copy_returns_internal_error_when_store_uninitialized() {
@@ -580,37 +576,9 @@ mod tests {
.upload_id("upload-id".to_string())
.build()
.unwrap();
let condition = |value| match value {
crate::app::storage_api::s3::dto::ETagCondition::Any => CopySourceCondition::Any,
crate::app::storage_api::s3::dto::ETagCondition::ETag(ETag::Strong(value)) => CopySourceCondition::Strong(value),
crate::app::storage_api::s3::dto::ETagCondition::ETag(ETag::Weak(_)) => CopySourceCondition::Weak,
};
let err = match Box::pin(make_usecase().execute_upload_part_copy_with_audit(
owned_envelope(Method::PUT),
UploadPartCopyWriteInput {
target: UploadPartTarget::Legacy {
bucket: input.bucket,
key: input.key,
upload_id: input.upload_id,
},
part_number: input.part_number,
copy_source: rustfs_gateway_types::compat::s3s_5761ddfe::copy_source::from_s3s(input.copy_source),
copy_source_range: input.copy_source_range,
copy_source_if_match: input.copy_source_if_match.map(&condition),
copy_source_if_none_match: input.copy_source_if_none_match.map(&condition),
sse_customer_algorithm: input.sse_customer_algorithm,
sse_customer_key: input.sse_customer_key,
sse_customer_key_md5: input.sse_customer_key_md5,
copy_source_sse_customer_algorithm: input.copy_source_sse_customer_algorithm,
copy_source_sse_customer_key: input.copy_source_sse_customer_key,
copy_source_sse_customer_key_md5: input.copy_source_sse_customer_key_md5,
},
))
.await
{
Err(error) => crate::app::storage_api::s3::S3Error::from(error),
Ok(_) => panic!("uninitialized store must be an internal error"),
};
let req = build_request(input, Method::PUT);
let err = Box::pin(make_usecase().execute_upload_part_copy(req)).await.unwrap_err();
assert_eq!(err.code(), &crate::app::storage_api::s3::S3ErrorCode::InternalError);
}
@@ -629,37 +597,9 @@ mod tests {
.upload_id("upload-id".to_string())
.build()
.unwrap();
let condition = |value| match value {
crate::app::storage_api::s3::dto::ETagCondition::Any => CopySourceCondition::Any,
crate::app::storage_api::s3::dto::ETagCondition::ETag(ETag::Strong(value)) => CopySourceCondition::Strong(value),
crate::app::storage_api::s3::dto::ETagCondition::ETag(ETag::Weak(_)) => CopySourceCondition::Weak,
};
let err = match Box::pin(make_usecase().execute_upload_part_copy_with_audit(
owned_envelope(Method::PUT),
UploadPartCopyWriteInput {
target: UploadPartTarget::Legacy {
bucket: input.bucket,
key: input.key,
upload_id: input.upload_id,
},
part_number: input.part_number,
copy_source: rustfs_gateway_types::compat::s3s_5761ddfe::copy_source::from_s3s(input.copy_source),
copy_source_range: input.copy_source_range,
copy_source_if_match: input.copy_source_if_match.map(&condition),
copy_source_if_none_match: input.copy_source_if_none_match.map(&condition),
sse_customer_algorithm: input.sse_customer_algorithm,
sse_customer_key: input.sse_customer_key,
sse_customer_key_md5: input.sse_customer_key_md5,
copy_source_sse_customer_algorithm: input.copy_source_sse_customer_algorithm,
copy_source_sse_customer_key: input.copy_source_sse_customer_key,
copy_source_sse_customer_key_md5: input.copy_source_sse_customer_key_md5,
},
))
.await
{
Err(error) => crate::app::storage_api::s3::S3Error::from(error),
Ok(_) => panic!("invalid part number must be rejected"),
};
let req = build_request(input, Method::PUT);
let err = Box::pin(make_usecase().execute_upload_part_copy(req)).await.unwrap_err();
assert_eq!(err.code(), &crate::app::storage_api::s3::S3ErrorCode::InvalidArgument);
assert_eq!(err.message(), Some("partNumber must be between 1 and 10000"));
}
@@ -76,38 +76,9 @@ fn upload_part_copy_shares_foreground_admission_with_upload_part() {
.any(|label| label.key() == "size_bucket" && label.value() == source_size_bucket)
})
};
let saturated = input.clone();
let condition = |value| match value {
crate::app::storage_api::s3::dto::ETagCondition::Any => CopySourceCondition::Any,
crate::app::storage_api::s3::dto::ETagCondition::ETag(ETag::Strong(value)) => CopySourceCondition::Strong(value),
crate::app::storage_api::s3::dto::ETagCondition::ETag(ETag::Weak(_)) => CopySourceCondition::Weak,
};
let error = match Box::pin(usecase.execute_upload_part_copy_with_audit(
owned_envelope(Method::PUT),
UploadPartCopyWriteInput {
target: UploadPartTarget::Legacy {
bucket: saturated.bucket,
key: saturated.key,
upload_id: saturated.upload_id,
},
part_number: saturated.part_number,
copy_source: rustfs_gateway_types::compat::s3s_5761ddfe::copy_source::from_s3s(saturated.copy_source),
copy_source_range: saturated.copy_source_range,
copy_source_if_match: saturated.copy_source_if_match.map(&condition),
copy_source_if_none_match: saturated.copy_source_if_none_match.map(&condition),
sse_customer_algorithm: saturated.sse_customer_algorithm,
sse_customer_key: saturated.sse_customer_key,
sse_customer_key_md5: saturated.sse_customer_key_md5,
copy_source_sse_customer_algorithm: saturated.copy_source_sse_customer_algorithm,
copy_source_sse_customer_key: saturated.copy_source_sse_customer_key,
copy_source_sse_customer_key_md5: saturated.copy_source_sse_customer_key_md5,
},
))
.await
{
Err(error) => crate::app::storage_api::s3::S3Error::from(error),
Ok(_) => panic!("copy must not bypass the occupied UploadPart pool"),
};
let error = Box::pin(usecase.execute_upload_part_copy(build_request(input.clone(), Method::PUT)))
.await
.expect_err("copy must not bypass the occupied UploadPart pool");
assert_eq!(error.code(), &S3ErrorCode::SlowDown);
assert!(!source_reader_observed(), "saturated copy must not construct a source body reader");
let parts = store
@@ -194,37 +165,9 @@ fn upload_part_copy_shares_foreground_admission_with_upload_part() {
.upload_id(deleted_upload.upload_id.clone())
.build()
.expect("deleted-source copy request");
let condition = |value| match value {
crate::app::storage_api::s3::dto::ETagCondition::Any => CopySourceCondition::Any,
crate::app::storage_api::s3::dto::ETagCondition::ETag(ETag::Strong(value)) => CopySourceCondition::Strong(value),
crate::app::storage_api::s3::dto::ETagCondition::ETag(ETag::Weak(_)) => CopySourceCondition::Weak,
};
let error = match Box::pin(usecase.execute_upload_part_copy_with_audit(
owned_envelope(Method::PUT),
UploadPartCopyWriteInput {
target: UploadPartTarget::Legacy {
bucket: input.bucket,
key: input.key,
upload_id: input.upload_id,
},
part_number: input.part_number,
copy_source: rustfs_gateway_types::compat::s3s_5761ddfe::copy_source::from_s3s(input.copy_source),
copy_source_range: input.copy_source_range,
copy_source_if_match: input.copy_source_if_match.map(&condition),
copy_source_if_none_match: input.copy_source_if_none_match.map(&condition),
sse_customer_algorithm: input.sse_customer_algorithm,
sse_customer_key: input.sse_customer_key,
sse_customer_key_md5: input.sse_customer_key_md5,
copy_source_sse_customer_algorithm: input.copy_source_sse_customer_algorithm,
copy_source_sse_customer_key: input.copy_source_sse_customer_key,
copy_source_sse_customer_key_md5: input.copy_source_sse_customer_key_md5,
},
))
.await
{
Err(error) => crate::app::storage_api::s3::S3Error::from(error),
Ok(_) => panic!("deleted source must fail before conditions or admission"),
};
let error = Box::pin(usecase.execute_upload_part_copy(build_request(input, Method::PUT)))
.await
.expect_err("deleted source must fail before conditions or admission");
assert_eq!(error.code(), &expected);
}
});
@@ -268,32 +211,7 @@ fn upload_part_copy_range_error_and_cancellation_release_foreground_permit() {
.upload_id(cancelled_upload.upload_id.clone())
.build()
.expect("cancelled copy request");
let condition = |value| match value {
crate::app::storage_api::s3::dto::ETagCondition::Any => CopySourceCondition::Any,
crate::app::storage_api::s3::dto::ETagCondition::ETag(ETag::Strong(value)) => CopySourceCondition::Strong(value),
crate::app::storage_api::s3::dto::ETagCondition::ETag(ETag::Weak(_)) => CopySourceCondition::Weak,
};
let mut cancelled_copy = Box::pin(usecase.execute_upload_part_copy_with_audit(
owned_envelope(Method::PUT),
UploadPartCopyWriteInput {
target: UploadPartTarget::Legacy {
bucket: cancelled_input.bucket,
key: cancelled_input.key,
upload_id: cancelled_input.upload_id,
},
part_number: cancelled_input.part_number,
copy_source: rustfs_gateway_types::compat::s3s_5761ddfe::copy_source::from_s3s(cancelled_input.copy_source),
copy_source_range: cancelled_input.copy_source_range,
copy_source_if_match: cancelled_input.copy_source_if_match.map(&condition),
copy_source_if_none_match: cancelled_input.copy_source_if_none_match.map(&condition),
sse_customer_algorithm: cancelled_input.sse_customer_algorithm,
sse_customer_key: cancelled_input.sse_customer_key,
sse_customer_key_md5: cancelled_input.sse_customer_key_md5,
copy_source_sse_customer_algorithm: cancelled_input.copy_source_sse_customer_algorithm,
copy_source_sse_customer_key: cancelled_input.copy_source_sse_customer_key,
copy_source_sse_customer_key_md5: cancelled_input.copy_source_sse_customer_key_md5,
},
));
let mut cancelled_copy = Box::pin(usecase.execute_upload_part_copy(build_request(cancelled_input, Method::PUT)));
tokio::time::timeout(Duration::from_secs(10), async {
tokio::select! {
biased;
@@ -302,10 +220,7 @@ fn upload_part_copy_range_error_and_cancellation_release_foreground_permit() {
tokio::task::yield_now().await;
}
} => {}
result = &mut cancelled_copy => match result {
Ok(_) => panic!("copy completed before cancellation"),
Err(error) => panic!("copy completed before cancellation: {error:?}"),
},
result = &mut cancelled_copy => panic!("copy completed before cancellation: {result:?}"),
}
})
.await
@@ -345,39 +260,17 @@ fn upload_part_copy_range_error_and_cancellation_release_foreground_permit() {
.upload_id(upload.upload_id.clone())
.build()
.expect("range request");
let response = Box::pin(usecase.execute_upload_part_copy_with_audit(
owned_envelope(Method::PUT),
UploadPartCopyWriteInput {
target: UploadPartTarget::Legacy {
bucket: input.bucket,
key: input.key,
upload_id: input.upload_id,
},
part_number: input.part_number,
copy_source: rustfs_gateway_types::compat::s3s_5761ddfe::copy_source::from_s3s(input.copy_source),
copy_source_range: input.copy_source_range,
copy_source_if_match: input.copy_source_if_match.map(&condition),
copy_source_if_none_match: input.copy_source_if_none_match.map(&condition),
sse_customer_algorithm: input.sse_customer_algorithm,
sse_customer_key: input.sse_customer_key,
sse_customer_key_md5: input.sse_customer_key_md5,
copy_source_sse_customer_algorithm: input.copy_source_sse_customer_algorithm,
copy_source_sse_customer_key: input.copy_source_sse_customer_key,
copy_source_sse_customer_key_md5: input.copy_source_sse_customer_key_md5,
},
))
.await;
let response = Box::pin(usecase.execute_upload_part_copy(build_request(input, Method::PUT))).await;
assert_eq!(
manager.put_object_admission_snapshot().active,
Some(0),
"copy releases its permit on every outcome"
);
if capped {
let error = match response {
Err(error) => crate::app::storage_api::s3::S3Error::from(error),
Ok(_) => panic!("range exceeds destination size limit"),
};
assert_eq!(error.code(), &S3ErrorCode::EntityTooLarge);
assert_eq!(
response.expect_err("range exceeds destination size limit").code(),
&S3ErrorCode::EntityTooLarge
);
let parts = store
.list_object_parts(&bucket, "object", &upload.upload_id, None, 1000, &ObjectOptions::default())
.await
@@ -388,16 +281,16 @@ fn upload_part_copy_range_error_and_cancellation_release_foreground_permit() {
.await
.expect("abort capped session");
} else {
let copied = match response {
Ok(output) => output,
Err(error) => panic!("range copy after failed request released its source and permit: {error:?}"),
};
let copied = response.expect("range copy after failed request released its source and permit");
let etag = copied
.output
.copy_part_result
.expect("range result")
.e_tag
.as_deref()
.map(crate::storage::storage_api::to_s3s_xml_etag)
.expect("range ETag");
let etag = etag.value().trim_matches('"').to_owned();
.expect("range ETag")
.value()
.trim_matches('"')
.to_owned();
Arc::clone(&store)
.complete_multipart_upload(
&bucket,
+2 -11
View File
@@ -258,17 +258,8 @@ mod tests {
sse_customer_key: None,
sse_customer_key_md5: None,
};
let envelope = RequestEnvelope::from_parts(
crate::app::RequestHead {
method: http::Method::GET,
uri: http::Uri::from_static("/"),
headers: http::HeaderMap::new(),
credentials: None,
region: None,
service: None,
},
http::Extensions::new(),
);
let request = build_request((), http::Method::GET);
let envelope = RequestEnvelope::from_s3s(&request);
let error = DefaultObjectUsecase::without_context()
.execute_get_object_attributes(envelope, fields)
.await
+5 -5
View File
@@ -108,11 +108,11 @@ use super::storage_api::object_usecase::set_disk::{
get_lock_acquire_timeout, get_object_disk_read_timeout, is_valid_storage_class,
};
use super::storage_api::object_usecase::sse::{
DecryptionRequest, EncryptionMaterialExt, EncryptionRequest, SseKmsPrincipal, SseKmsPrincipalExt,
apply_bucket_default_lock_retention, authorize_sse_kms_object_read, bucket_default_write_sse, build_ssec_read_headers,
classify_sse_read_response, encryption_material_to_metadata, extract_server_side_encryption_from_headers,
extract_ssec_params_from_headers, extract_ssekms_context_from_headers, get_buffer_size_opt_in,
load_bucket_object_lock_config_state, map_get_object_reader_error, sse_encryption, validate_bucket_object_lock_enabled_state,
DecryptionRequest, EncryptionRequest, SseKmsPrincipal, apply_bucket_default_lock_retention, authorize_sse_kms_object_read,
bucket_default_write_sse, build_ssec_read_headers, classify_sse_read_response, encryption_material_to_metadata,
extract_server_side_encryption_from_headers, extract_ssec_params_from_headers, extract_ssekms_context_from_headers,
get_buffer_size_opt_in, load_bucket_object_lock_config_state, map_get_object_reader_error, sse_encryption,
validate_bucket_object_lock_enabled_state,
};
use super::storage_api::object_usecase::storage_class as storageclass;
use super::storage_api::object_usecase::timeout_wrapper::{GetObjectTimeoutPolicy, RequestTimeoutWrapper};
+1 -3
View File
@@ -2,9 +2,7 @@
use super::storage_api::select_object::StorageError;
use super::storage_api::select_object::options::get_opts;
use super::storage_api::select_object::request_context::spawn_traced;
use super::storage_api::select_object::sse::{
SseKmsPrincipal, SseKmsPrincipalExt, authorize_sse_kms_object_read, project_sse_read_response_headers,
};
use super::storage_api::select_object::sse::{SseKmsPrincipal, authorize_sse_kms_object_read, project_sse_read_response_headers};
use super::storage_api::select_object::{
StoragePrepareSelectObjectSnapshotError, StorageSelectObjectSnapshot, get_validated_store, validate_sse_headers_for_read,
validate_ssec_for_read,
+6 -5
View File
@@ -125,14 +125,15 @@ pub(crate) mod data_usage {
pub(crate) use crate::storage::storage_api::ecstore_data_usage::quota_object_size;
#[cfg(test)]
pub(crate) async fn apply_bucket_usage_memory_overlay(data_usage_info: &mut rustfs_data_usage::DataUsageInfo) {
crate::storage::storage_api::ecstore_data_usage::apply_bucket_usage_memory_overlay(data_usage_info).await;
}
pub(crate) async fn load_admin_data_usage_from_backend_cached(
pub(crate) async fn load_admin_data_usage_with_memory_overlay(
store: Arc<crate::storage::storage_api::ECStore>,
) -> Result<rustfs_data_usage::DataUsageInfo, crate::storage::storage_api::StorageError> {
crate::storage::storage_api::ecstore_data_usage::load_admin_data_usage_from_backend_cached(store).await
crate::storage::storage_api::ecstore_data_usage::load_admin_data_usage_with_memory_overlay(store).await
}
pub(crate) async fn load_bucket_usage_from_backend_cached(
@@ -1675,9 +1676,9 @@ pub(crate) mod sse {
validate_bucket_object_lock_enabled_state,
};
pub(crate) use crate::storage::storage_api::sse_consumer::{
EncryptionKeyKind, EncryptionMaterialExt, SseKmsPrincipalExt, bucket_default_write_sse, build_ssec_read_headers,
encryption_material_to_metadata, extract_ssec_params_from_headers, extract_ssekms_context_from_headers,
map_get_object_reader_error, mark_encrypted_multipart_metadata, multipart_completion_authorization, stored_ssec_metadata,
EncryptionKeyKind, bucket_default_write_sse, build_ssec_read_headers, encryption_material_to_metadata,
extract_ssec_params_from_headers, extract_ssekms_context_from_headers, map_get_object_reader_error,
mark_encrypted_multipart_metadata, multipart_completion_authorization, stored_ssec_metadata,
};
}
@@ -2461,8 +2461,9 @@ mod tests {
let addr = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
let (socket, _) = listener.accept().await.unwrap();
let fallback =
tower::service_fn(|_| async { Ok::<_, Infallible>(Response::new(rustfs_s3_types::Body::empty())) });
let fallback = tower::service_fn(|_| async {
Ok::<_, Infallible>(Response::new(crate::storage_api::server::http::rpc::Body::empty()))
});
server_http1::Builder::new()
.serve_connection(
TokioIo::new(socket),

Some files were not shown because too many files have changed in this diff Show More