mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-22 16:02:46 +00:00
Compare commits
16 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 27035536f7 | |||
| 78b35c3ea8 | |||
| a9ff9e09b0 | |||
| bcfebbc1e9 | |||
| aff99bbdb2 | |||
| 856a270717 | |||
| 3c39a94388 | |||
| 0deff550aa | |||
| f41b022017 | |||
| 66d727279b | |||
| a8b4367c5f | |||
| b9d3aea1a5 | |||
| f034e3701b | |||
| 67e29c2143 | |||
| d6677ac005 | |||
| 40f4e5e1e2 |
@@ -57,7 +57,6 @@ rustfs-scanner 1392 20
|
||||
rustfs-scanner-metrics 90 0
|
||||
rustfs-security-governance 20 0
|
||||
rustfs-signer 34 0
|
||||
rustfs-sse 145 0
|
||||
rustfs-storage-api 34 0
|
||||
rustfs-targets 612 26
|
||||
rustfs-test-utils 1 0
|
||||
|
||||
@@ -231,7 +231,7 @@ jobs:
|
||||
# --all-targets links the same test binaries as the reader lane.
|
||||
CARGO_BUILD_JOBS: "2"
|
||||
run: |
|
||||
cargo build -p rustfs -p rustfs-ecstore -p rustfs-sse --all-targets --features rio-v2
|
||||
cargo build -p rustfs -p rustfs-ecstore --all-targets --features rio-v2
|
||||
cargo build -p rustfs --bins --features rio-v2,e2e-test-hooks
|
||||
|
||||
# Readers: the swift and sftp legs of test-and-lint-protocols. Built in
|
||||
|
||||
@@ -563,7 +563,7 @@ jobs:
|
||||
run: chmod go-w "$(realpath "$HOME")"
|
||||
|
||||
- name: Run rio-v2 clippy lints
|
||||
run: cargo clippy -p rustfs -p rustfs-ecstore -p rustfs-sse --all-targets --features rio-v2 -- -D warnings
|
||||
run: cargo clippy -p rustfs -p rustfs-ecstore --all-targets --features rio-v2 -- -D warnings
|
||||
|
||||
- name: Run rio-v2 feature tests
|
||||
shell: bash
|
||||
@@ -580,7 +580,7 @@ jobs:
|
||||
rm -f target/nextest/ci/junit.xml
|
||||
./scripts/ci/resource_sampler.sh start nextest-rio-v2
|
||||
trap './scripts/ci/resource_sampler.sh stop' EXIT
|
||||
cargo nextest run --profile ci -p rustfs -p rustfs-ecstore -p rustfs-sse --features rio-v2 \
|
||||
cargo nextest run --profile ci -p rustfs -p rustfs-ecstore --features rio-v2 \
|
||||
2>&1 | tee artifacts/workspace/nextest.log
|
||||
cargo test -p rustfs --doc --features rio-v2
|
||||
|
||||
|
||||
+1
-1
@@ -94,7 +94,7 @@ refactors.
|
||||
|--------|--------------------------|----------------|
|
||||
| Foundation | `checksums`, `common`, `config`, `data-usage`, `heal-contracts`, `scanner-metrics`, `utils` | Shared configuration, data-usage models, heal domain contracts, scanner telemetry types, utilities, and checksums. |
|
||||
| I/O and storage | `concurrency`, `ecstore`, `filemeta`, `heal`, `io-core`, `io-metrics`, `lifecycle`, `lock`, `object-capacity`, `object-data-cache`, `replication`, `rio`, `rio-v2`, `s3-client`, `scanner`, `storage-api` | Erasure-coded object storage, metadata, recovery, lifecycle, replication, locking, cache, I/O pipelines, and the engine-side S3 client for remote tier/transition targets. |
|
||||
| Security and identity | `credentials`, `crypto`, `iam`, `keystone`, `kms`, `policy`, `security-governance`, `signer`, `sse`, `tls-runtime`, `trusted-proxies` | Credentials, authentication, authorization, encryption, key management, server-side encryption key resolution, TLS, and security contracts. |
|
||||
| Security and identity | `credentials`, `crypto`, `iam`, `keystone`, `kms`, `policy`, `security-governance`, `signer`, `tls-runtime`, `trusted-proxies` | Credentials, authentication, authorization, encryption, key management, TLS, and security contracts. |
|
||||
| Protocols and contracts | `extension-schema`, `madmin`, `protos`, `protocols`, `s3-ops`, `s3-types`, `s3select-api`, `s3select-query` | Admin, inter-node, S3, S3 Select, and optional protocol contracts. |
|
||||
| Operations and integration | `audit`, `notify`, `obs`, `targets`, `zip` | Auditing, observability, event delivery, notification targets, and archive support. |
|
||||
| Test support | `e2e_test`, `test-utils` | End-to-end validation and shared test bootstrap utilities. |
|
||||
|
||||
@@ -116,7 +116,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
- Four-layer regression-prevention tests guard against silent feature deletion: compile-time module assertion, module-presence unit test, cross-module `Protocol` enum assertion, end-to-end SSH banner test against the running binary
|
||||
|
||||
### Changed
|
||||
- **Startup reports the state of `RUSTFS_SSE_S3_MASTER_KEY`**: the server now checks the local SSE-S3 master key when it starts and logs one `sse_local_master_key` event with `state` `loaded`, `not_set`, or `invalid`. An unusable value used to surface only as a `400 InvalidRequest` on the first SSE-S3 request served without a KMS; it is now logged as an error at startup with the reason. Startup still succeeds and request behavior is unchanged. The line "Using RUSTFS_SSE_S3_MASTER_KEY for SSE-S3 (KMS not configured)", logged when the key was first used, is replaced by this event.
|
||||
- **Scanner bucket walkers are no longer tied to the disk count**: a new scanner key `walkers_per_disk` (`RUSTFS_SCANNER_WALKERS_PER_DISK`, `1` through `16`, default `1`) spawns that many bucket walkers per local disk of an erasure set, so a single-disk deployment can walk several buckets at once instead of one bucket, one object at a time. The walkers of one disk share its scanner admission, so a peer still sees one scan session per physical disk, and remote disks keep one walker. The walker count is what `max_concurrent_disk_scans` caps and the disk count stays the floor; the `disk_concurrency_budget` log line now reports `walkers_per_disk`, `walkers`, and the `reason` for the effective limit. The default keeps today's I/O profile because a local disk cannot yet report whether it is rotational; the design intends a higher default on flash once it can (`docs/architecture/scanner-per-bucket-convergence.md`). As groundwork for pacing, the raw-enumeration page writer inserts into its building page in place and commits pages without copying them, and a regression test pins the cost of recording 20,000 entries in one prefix so the quadratic bookkeeping removed by [#8114](https://github.com/rustfs/rustfs/pull/8114) (rustfs#8113) cannot return. No persisted format, protocol or admin API changes beyond the additional config key, which older releases preserve and ignore.
|
||||
- **Encryption and KMS work merged since `1.0.0-rc.5`** (entries were missing from this section):
|
||||
- **Persisted KMS configuration secrets** are sealed field-by-field with `RUSTFS_KMS_CONFIG_SECRET`. **When the variable is unset the secrets are persisted in cleartext and the server only warns** (`persisted KMS configuration carries cleartext secrets`); it never refuses the write. Set it, identically, on every node, and re-save the configuration to seal an existing one.
|
||||
|
||||
Generated
+3
-28
@@ -9959,7 +9959,6 @@ dependencies = [
|
||||
"rustfs-scanner-metrics",
|
||||
"rustfs-security-governance",
|
||||
"rustfs-signer",
|
||||
"rustfs-sse",
|
||||
"rustfs-storage-api",
|
||||
"rustfs-targets",
|
||||
"rustfs-test-utils",
|
||||
@@ -11374,6 +11373,8 @@ dependencies = [
|
||||
"rustfs-storage-api",
|
||||
"rustfs-test-utils",
|
||||
"rustfs-utils",
|
||||
"rustix",
|
||||
"same-file",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"serial_test",
|
||||
@@ -11387,6 +11388,7 @@ dependencies = [
|
||||
"tracing",
|
||||
"tracing-subscriber",
|
||||
"uuid",
|
||||
"winapi-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -11429,33 +11431,6 @@ dependencies = [
|
||||
"url",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-sse"
|
||||
version = "1.0.5"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"async-trait",
|
||||
"axum",
|
||||
"base64-simd",
|
||||
"chacha20poly1305",
|
||||
"hmac 0.13.0",
|
||||
"http 1.5.0",
|
||||
"md-5",
|
||||
"rand 0.10.4",
|
||||
"rustfs-config",
|
||||
"rustfs-kms",
|
||||
"rustfs-policy",
|
||||
"rustfs-s3-types",
|
||||
"rustfs-utils",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha2 0.11.1",
|
||||
"temp-env",
|
||||
"tempfile",
|
||||
"tokio",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-storage-api"
|
||||
version = "1.0.5"
|
||||
|
||||
@@ -60,7 +60,6 @@ members = [
|
||||
"crates/security-governance", # Security governance contracts
|
||||
"crates/extension-schema", # Extension schema contracts
|
||||
"crates/signer", # client signer
|
||||
"crates/sse", # S3 server-side encryption key resolution and metadata
|
||||
"crates/storage-api", # Storage API contracts
|
||||
"crates/test-utils", # Shared test bootstrap helpers (dev-dependency only)
|
||||
"crates/targets", # Target-specific configurations and utilities
|
||||
@@ -141,7 +140,6 @@ rustfs-scanner = { path = "crates/scanner", version = "1.0.5" }
|
||||
rustfs-security-governance = { path = "crates/security-governance", version = "1.0.5" }
|
||||
rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.5" }
|
||||
rustfs-signer = { path = "crates/signer", version = "1.0.5" }
|
||||
rustfs-sse = { path = "crates/sse", version = "1.0.5" }
|
||||
rustfs-storage-api = { path = "crates/storage-api", version = "1.0.5" }
|
||||
rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.5" }
|
||||
rustfs-targets = { path = "crates/targets", version = "1.0.5" }
|
||||
|
||||
@@ -1235,23 +1235,66 @@ impl Serialize for DataUsageEntry {
|
||||
where
|
||||
S: serde::Serializer,
|
||||
{
|
||||
self.serialize_fields(serializer, &self.children, &self.replication_stats, &self.all_tier_stats)
|
||||
}
|
||||
}
|
||||
|
||||
struct OrderedUsageEntry<'a>(&'a DataUsageEntry);
|
||||
|
||||
impl Serialize for OrderedUsageEntry<'_> {
|
||||
fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
|
||||
let mut children: Vec<_> = self.0.children.iter().collect();
|
||||
children.sort_unstable();
|
||||
// MessagePack decoders accept struct fields as positional sequences.
|
||||
// Include the nested target and tier maps in stable order as well.
|
||||
let replication = self.0.replication_stats.as_ref().map(|stats| {
|
||||
let ReplicationAllStats {
|
||||
targets,
|
||||
replica_size,
|
||||
replica_count,
|
||||
} = stats;
|
||||
let targets: std::collections::BTreeMap<_, _> = targets.iter().collect();
|
||||
(targets, replica_size, replica_count)
|
||||
});
|
||||
let tiers = self.0.all_tier_stats.as_ref().map(|stats| {
|
||||
let AllTierStats { tiers } = stats;
|
||||
let tiers: std::collections::BTreeMap<_, _> = tiers.iter().collect();
|
||||
(tiers,)
|
||||
});
|
||||
self.0.serialize_fields(serializer, &children, &replication, &tiers)
|
||||
}
|
||||
}
|
||||
|
||||
impl DataUsageEntry {
|
||||
/// Stable MessagePack encoding for independently reusable scanner checkpoint pages.
|
||||
pub fn ordered_encoding(&self) -> impl Serialize + '_ {
|
||||
OrderedUsageEntry(self)
|
||||
}
|
||||
|
||||
fn serialize_fields<S: serde::Serializer, C: Serialize, R: Serialize, T: Serialize>(
|
||||
&self,
|
||||
serializer: S,
|
||||
children: &C,
|
||||
replication: &R,
|
||||
tiers: &T,
|
||||
) -> Result<S::Ok, S::Error> {
|
||||
// Keep entries map-encoded so older readers can ignore fields appended
|
||||
// by newer scanner versions during rolling upgrades. The derived
|
||||
// (array) encoding made any appended field a decode error for them.
|
||||
let mut state = serializer.serialize_map(Some(
|
||||
11 + usize::from(self.unknown_tier_stats.is_some()) + usize::from(self.tier_accounting_proof.is_some()),
|
||||
))?;
|
||||
state.serialize_entry("children", &self.children)?;
|
||||
state.serialize_entry("children", children)?;
|
||||
state.serialize_entry("size", &self.size)?;
|
||||
state.serialize_entry("objects", &self.objects)?;
|
||||
state.serialize_entry("versions", &self.versions)?;
|
||||
state.serialize_entry("delete_markers", &self.delete_markers)?;
|
||||
state.serialize_entry("obj_sizes", &self.obj_sizes)?;
|
||||
state.serialize_entry("obj_versions", &self.obj_versions)?;
|
||||
state.serialize_entry("replication_stats", &self.replication_stats)?;
|
||||
state.serialize_entry("replication_stats", replication)?;
|
||||
state.serialize_entry("compacted", &self.compacted)?;
|
||||
state.serialize_entry("failed_objects", &self.failed_objects)?;
|
||||
state.serialize_entry("all_tier_stats", &self.all_tier_stats)?;
|
||||
state.serialize_entry("all_tier_stats", tiers)?;
|
||||
// Keep the legacy no-unknown shape byte-for-byte stable. Once unknown
|
||||
// accounting exists, append its map field before the optional proof.
|
||||
if let Some(unknown_tier_stats) = self.unknown_tier_stats.as_ref() {
|
||||
@@ -2534,6 +2577,32 @@ mod tests {
|
||||
entry
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ordered_checkpoint_entry_round_trips_and_stays_identical_after_reload() {
|
||||
let mut entry = DataUsageEntry::default();
|
||||
let mut replication = ReplicationAllStats {
|
||||
replica_size: 998,
|
||||
replica_count: 3,
|
||||
..Default::default()
|
||||
};
|
||||
let mut tiers = AllTierStats::default();
|
||||
for index in 1..=32 {
|
||||
let key = format!("source-{index}");
|
||||
entry.children.insert(key.clone());
|
||||
replication.targets.insert(key.clone(), ReplicationTargetUsage::default());
|
||||
tiers.tiers.insert(key, TierStats::default());
|
||||
}
|
||||
entry.replication_stats = Some(replication);
|
||||
entry.all_tier_stats = Some(tiers);
|
||||
let bytes = rmp_serde::to_vec(&entry.ordered_encoding()).expect("encode ordered checkpoint");
|
||||
let reloaded: DataUsageEntry = rmp_serde::from_slice(&bytes).expect("read with the existing codec");
|
||||
assert_eq!(reloaded.children, entry.children);
|
||||
let replicas = reloaded.replication_stats.as_ref().expect("replication stats");
|
||||
assert_eq!((replicas.targets.len(), replicas.replica_size, replicas.replica_count), (32, 998, 3));
|
||||
assert_eq!(reloaded.all_tier_stats.as_ref().expect("tier stats").tiers.len(), 32);
|
||||
assert_eq!(rmp_serde::to_vec(&reloaded.ordered_encoding()).expect("encode reloaded entry"), bytes);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_stats_survive_entry_merge() {
|
||||
let mut left = tier_entry(
|
||||
|
||||
@@ -105,6 +105,9 @@ mod replication_target_matrix_test;
|
||||
#[cfg(test)]
|
||||
mod data_usage_test;
|
||||
|
||||
#[cfg(test)]
|
||||
mod scanner_wide_directory_test;
|
||||
|
||||
// KMS-specific test modules
|
||||
#[cfg(test)]
|
||||
mod kms;
|
||||
|
||||
@@ -0,0 +1,208 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::path::Path;
|
||||
use std::time::{Duration, SystemTime};
|
||||
|
||||
use aws_sdk_s3::{Client, primitives::ByteStream};
|
||||
use futures::{StreamExt, TryStreamExt, stream};
|
||||
use rustfs_data_usage::{BucketUsageValueState, DataUsageInfo};
|
||||
use serde::{Deserialize, de::DeserializeOwned};
|
||||
use tokio::time::{Instant, sleep, timeout};
|
||||
|
||||
use crate::common::{RustFSTestEnvironment, signed_request};
|
||||
use crate::storage_api::scanner_peer_bucket::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET};
|
||||
|
||||
type TestResult<T = ()> = Result<T, Box<dyn std::error::Error + Send + Sync>>;
|
||||
|
||||
const BUCKET: &str = "wide-directory-usage";
|
||||
const WIDTH: usize = 512;
|
||||
// The per-disk token bucket allows an initial burst of one second. Even the
|
||||
// first pass can read at most (1 + 3) * 96 = 384 of these 512 root entries.
|
||||
const SCAN_ENV: &[(&str, &str)] = &[
|
||||
("RUST_LOG", "warn,rustfs::scanner::io=debug,rustfs::scanner::data_usage=debug"),
|
||||
("RUSTFS_SCANNER_ENABLED", "true"),
|
||||
("RUSTFS_SCANNER_START_DELAY_SECS", "0"),
|
||||
("RUSTFS_SCANNER_CYCLE", "1"),
|
||||
("RUSTFS_SCANNER_CYCLE_MAX_DURATION_SECS", "3"),
|
||||
("RUSTFS_SCANNER_SPEED", "fastest"),
|
||||
("RUSTFS_SCANNER_DISK_IO_BUDGET_IOPS", "96"),
|
||||
("RUSTFS_SCANNER_DISK_IO_BUDGET_BYTES_PER_SEC", "off"),
|
||||
];
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct Status {
|
||||
metrics: Metrics,
|
||||
disk_io_budget: Vec<DiskBudget>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct Metrics {
|
||||
current_cycle_active: bool,
|
||||
active_paths: Vec<String>,
|
||||
cycle_max_duration_seconds: f64,
|
||||
partial_cycles_runtime: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct DiskBudget {
|
||||
iops: Option<u64>,
|
||||
}
|
||||
|
||||
async fn admin<T: DeserializeOwned>(env: &RustFSTestEnvironment, resource: &str) -> TestResult<T> {
|
||||
let url = format!("{}/rustfs/admin/v3/{resource}", env.url);
|
||||
let response = signed_request(http::Method::GET, &url, &env.access_key, &env.secret_key, None, None).await?;
|
||||
Ok(response.error_for_status()?.json().await?)
|
||||
}
|
||||
|
||||
async fn inventory(client: &Client) -> TestResult<BTreeMap<String, u64>> {
|
||||
let mut pages = client.list_objects_v2().bucket(BUCKET).max_keys(37).into_paginator().send();
|
||||
let mut sizes = BTreeMap::new();
|
||||
while let Some(page) = pages.next().await {
|
||||
for object in page?.contents() {
|
||||
let key = object.key().ok_or("S3 inventory omitted object key")?;
|
||||
let size = u64::try_from(object.size().ok_or("S3 inventory omitted object size")?)?;
|
||||
assert!(sizes.insert(key.to_owned(), size).is_none(), "duplicate inventory key {key}");
|
||||
}
|
||||
}
|
||||
Ok(sizes)
|
||||
}
|
||||
|
||||
async fn scan_status(env: &RustFSTestEnvironment) -> TestResult<Status> {
|
||||
let status: Status = admin(env, "scanner/status").await?;
|
||||
assert_eq!(status.metrics.cycle_max_duration_seconds, 3.0, "effective cycle budget");
|
||||
assert!(status.disk_io_budget.iter().all(|disk| disk.iops == Some(96)), "effective disk budget");
|
||||
Ok(status)
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn certifies_wide_directory_after_mid_scan_process_restart() -> TestResult {
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
let temporary_log = tempfile::NamedTempFile::new()?;
|
||||
let server_log = env
|
||||
.capture_log_path
|
||||
.get_or_insert_with(|| temporary_log.path().display().to_string())
|
||||
.clone();
|
||||
let bucket_metadata = Path::new(&env.temp_dir)
|
||||
.join(RUSTFS_META_BUCKET)
|
||||
.join(BUCKET_META_PREFIX)
|
||||
.join(BUCKET);
|
||||
// Seed a cold bucket without allowing any earlier, larger-budget sweep to count it.
|
||||
env.start_rustfs_server_without_cleanup_with_env(&[("RUSTFS_SCANNER_ENABLED", "false")])
|
||||
.await?;
|
||||
let client = env.create_s3_client();
|
||||
client.create_bucket().bucket(BUCKET).send().await?;
|
||||
stream::iter(0..WIDTH)
|
||||
.map(Ok)
|
||||
.try_for_each_concurrent(12, |i| {
|
||||
let client = &client;
|
||||
async move {
|
||||
client
|
||||
.put_object()
|
||||
.bucket(BUCKET)
|
||||
.key(format!("object-{i:064x}"))
|
||||
.body(ByteStream::from(vec![42; 1 + i % 67]))
|
||||
.send()
|
||||
.await
|
||||
.map(|_| ())
|
||||
}
|
||||
})
|
||||
.await?;
|
||||
let expected = inventory(&client).await?;
|
||||
assert_eq!(expected.len(), WIDTH);
|
||||
|
||||
env.restart_server_preserving_data(vec![], SCAN_ENV).await?;
|
||||
let first_pid = env.process.as_ref().ok_or("missing scanner process")?.id();
|
||||
timeout(Duration::from_secs(60), async {
|
||||
loop {
|
||||
let status = scan_status(&env).await?;
|
||||
if status.metrics.partial_cycles_runtime >= 1
|
||||
&& bucket_metadata.join(".usage-cache.bin").join("xl.meta").is_file()
|
||||
&& bucket_metadata.join(".usage-cache.bin.pages").is_dir()
|
||||
&& status.metrics.current_cycle_active
|
||||
&& status
|
||||
.metrics
|
||||
.active_paths
|
||||
.iter()
|
||||
.any(|path| path.split('/').any(|part| part == BUCKET))
|
||||
{
|
||||
let log = tokio::fs::read_to_string(&server_log).await?;
|
||||
if !log
|
||||
.lines()
|
||||
.any(|line| line.contains("partial_saved_not_published") && line.contains(BUCKET))
|
||||
{
|
||||
sleep(Duration::from_millis(200)).await;
|
||||
continue;
|
||||
}
|
||||
assert!(!status.disk_io_budget.is_empty(), "scanner never admitted disk I/O");
|
||||
break Ok::<_, Box<dyn std::error::Error + Send + Sync>>(());
|
||||
}
|
||||
sleep(Duration::from_millis(200)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(|_| "scanner did not re-enter the bucket after exhausting the fixed runtime budget")??;
|
||||
let partial: DataUsageInfo = admin(&env, "datausageinfo").await?;
|
||||
assert!(
|
||||
partial
|
||||
.buckets_usage
|
||||
.get(BUCKET)
|
||||
.is_none_or(|value| value.usage_certification.is_none()),
|
||||
"the first process certified the bucket before the injected crash"
|
||||
);
|
||||
|
||||
env.stop_server();
|
||||
// The terminal save has returned and the old process can no longer write.
|
||||
assert!(
|
||||
bucket_metadata.join(".usage-cache.bin").join("xl.meta").is_file()
|
||||
&& bucket_metadata.join(".usage-cache.bin.pages").is_dir(),
|
||||
"the interrupted process must have committed a paged checkpoint"
|
||||
);
|
||||
let restart_log_offset = usize::try_from(tokio::fs::metadata(&server_log).await?.len())?;
|
||||
let restarted_at = SystemTime::now();
|
||||
env.start_rustfs_server_without_cleanup_with_env(SCAN_ENV).await?;
|
||||
let second_pid = env.process.as_ref().ok_or("missing restarted scanner process")?.id();
|
||||
assert_ne!(first_pid, second_pid);
|
||||
let deadline = Instant::now() + Duration::from_secs(180);
|
||||
loop {
|
||||
let usage: DataUsageInfo = admin(&env, "datausageinfo").await?;
|
||||
let status = scan_status(&env).await?;
|
||||
if let Some(bucket) = usage.buckets_usage.get(BUCKET)
|
||||
&& bucket.usage_value_state == Some(BucketUsageValueState::Certified)
|
||||
{
|
||||
let log = tokio::fs::read_to_string(&server_log).await?;
|
||||
let first_admission = log
|
||||
.get(restart_log_offset..)
|
||||
.ok_or("server log truncated across restart")?
|
||||
.lines()
|
||||
.filter_map(|line| serde_json::from_str::<serde_json::Value>(line).ok())
|
||||
.find(|record| record["state"] == "bucket_checkpoint_prepared" && record["bucket"] == BUCKET)
|
||||
.ok_or("no checkpoint admission in the restarted process")?;
|
||||
assert_eq!(first_admission["resume_position"], true, "restarted scanner discarded the checkpoint");
|
||||
assert!(usage.is_complete_bucket_usage_snapshot());
|
||||
let certificate = bucket
|
||||
.usage_certification
|
||||
.ok_or("missing certificate for the measured bucket")?;
|
||||
assert!(certificate.certified_at >= restarted_at, "stale pre-restart certificate");
|
||||
assert!(status.metrics.partial_cycles_runtime > 0, "new process must cross a cycle deadline");
|
||||
assert!(!status.disk_io_budget.is_empty(), "restarted scanner never admitted disk I/O");
|
||||
assert_eq!(inventory(&client).await?, expected, "durable S3 inventory changed across restart");
|
||||
let count = u64::try_from(expected.len())?;
|
||||
let bytes = expected.values().sum::<u64>();
|
||||
assert_eq!((bucket.objects_count, bucket.size), (count, bytes));
|
||||
assert_eq!((usage.objects_total_count, usage.objects_total_size), (count, bytes));
|
||||
eprintln!(
|
||||
"PIDs {first_pid}/{second_pid}, runtime-limited cycles since restart {}, certificate epoch {} cycle {}, S3 and admin {} objects/{} bytes",
|
||||
status.metrics.partial_cycles_runtime, certificate.epoch, certificate.cycle, count, bytes
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
assert!(
|
||||
Instant::now() < deadline,
|
||||
"no certified bucket after restart: {status:?}, value={:?}",
|
||||
usage.buckets_usage.get(BUCKET)
|
||||
);
|
||||
sleep(Duration::from_millis(500)).await;
|
||||
}
|
||||
}
|
||||
@@ -158,6 +158,7 @@ fn to_madmin_scanner_metrics(metrics: rustfs_scanner_metrics::metrics::ScannerMe
|
||||
last_cycle_lifecycle_transition_actions: metrics.last_cycle_lifecycle_transition_actions,
|
||||
last_cycle_partial_source: metrics.last_cycle_partial_source,
|
||||
last_cycle_partial_source_code: metrics.last_cycle_partial_source_code,
|
||||
last_cycle_partial_stage: metrics.last_cycle_partial_stage,
|
||||
last_cycle_duration_seconds: metrics.last_cycle_duration_seconds,
|
||||
last_cycle_objects_scanned: metrics.last_cycle_objects_scanned,
|
||||
last_cycle_directories_scanned: metrics.last_cycle_directories_scanned,
|
||||
@@ -685,6 +686,7 @@ mod test {
|
||||
current_started,
|
||||
last_cycle_partial_source: "usage".to_string(),
|
||||
last_cycle_partial_source_code: 1,
|
||||
last_cycle_partial_stage: Some("object_budget".into()),
|
||||
cycle_timeout_total: 3,
|
||||
cycle_recovery_required_total: 2,
|
||||
cycle_last_progress_age: 17,
|
||||
@@ -700,6 +702,7 @@ mod test {
|
||||
assert_eq!(scanner.current_started, current_started);
|
||||
assert_eq!(scanner.last_cycle_partial_source, "usage");
|
||||
assert_eq!(scanner.last_cycle_partial_source_code, 1);
|
||||
assert_eq!(scanner.last_cycle_partial_stage.as_deref(), Some("object_budget"));
|
||||
assert_eq!(scanner.cycle_timeout_total, 3);
|
||||
assert_eq!(scanner.cycle_recovery_required_total, 2);
|
||||
assert_eq!(scanner.cycle_last_progress_age, 17);
|
||||
|
||||
@@ -6,8 +6,7 @@ Applies to `crates/kms/`.
|
||||
|
||||
When changing key-management behavior, verify compatibility with:
|
||||
|
||||
- `crates/sse/` (SSE key resolution; consumes this crate on the data path)
|
||||
- `rustfs/src/storage/sse.rs` (binds `crates/sse/` to the running server)
|
||||
- `rustfs/src/storage/sse.rs`
|
||||
- `rustfs/src/admin/handlers/kms.rs`
|
||||
- `rustfs/src/admin/handlers/kms_dynamic.rs`
|
||||
- `rustfs/src/admin/handlers/kms_keys.rs`
|
||||
|
||||
@@ -709,6 +709,8 @@ pub struct ScannerMetrics {
|
||||
pub last_cycle_partial_source: String,
|
||||
#[serde(rename = "last_cycle_partial_source_code", default)]
|
||||
pub last_cycle_partial_source_code: u64,
|
||||
#[serde(rename = "last_cycle_partial_stage", default, skip_serializing_if = "Option::is_none")]
|
||||
pub last_cycle_partial_stage: Option<String>,
|
||||
#[serde(rename = "last_cycle_duration_seconds", default)]
|
||||
pub last_cycle_duration_seconds: f64,
|
||||
#[serde(rename = "last_cycle_objects_scanned", default)]
|
||||
@@ -853,6 +855,7 @@ impl ScannerMetrics {
|
||||
self.last_cycle_partial_reason_code = other.last_cycle_partial_reason_code;
|
||||
self.last_cycle_partial_source = other.last_cycle_partial_source.clone();
|
||||
self.last_cycle_partial_source_code = other.last_cycle_partial_source_code;
|
||||
self.last_cycle_partial_stage.clone_from(&other.last_cycle_partial_stage);
|
||||
self.last_cycle_duration_seconds = other.last_cycle_duration_seconds;
|
||||
self.throttle_idle_mode_enabled = other.throttle_idle_mode_enabled;
|
||||
self.throttle_sleep_factor = other.throttle_sleep_factor;
|
||||
@@ -2138,6 +2141,7 @@ mod tests {
|
||||
collected_at: collected_at + SignedDuration::from_secs(1),
|
||||
last_cycle_partial_source: "lifecycle".to_string(),
|
||||
last_cycle_partial_source_code: 2,
|
||||
last_cycle_partial_stage: Some("object_budget".into()),
|
||||
pacing_pressure: ScannerPacingPressureSnapshot {
|
||||
primary_pressure: "queued_scans".to_string(),
|
||||
current_queued_scans: 4,
|
||||
@@ -2162,6 +2166,7 @@ mod tests {
|
||||
|
||||
assert_eq!(scanner.last_cycle_partial_source, "lifecycle");
|
||||
assert_eq!(scanner.last_cycle_partial_source_code, 2);
|
||||
assert_eq!(scanner.last_cycle_partial_stage.as_deref(), Some("object_budget"));
|
||||
assert_eq!(scanner.pacing_pressure.primary_pressure, "queued_scans");
|
||||
assert_eq!(scanner.pacing_pressure.current_queued_scans, 4);
|
||||
assert_eq!(scanner.pacing_pressure.current_active_scans, 3);
|
||||
|
||||
@@ -282,19 +282,6 @@ impl ScannerWorkSource {
|
||||
}
|
||||
}
|
||||
|
||||
fn from_code(code: u8) -> Option<Self> {
|
||||
match code {
|
||||
1 => Some(Self::Usage),
|
||||
2 => Some(Self::Lifecycle),
|
||||
3 => Some(Self::BucketReplication),
|
||||
4 => Some(Self::SiteReplication),
|
||||
5 => Some(Self::Heal),
|
||||
6 => Some(Self::Bitrot),
|
||||
7 => Some(Self::Alerts),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn all() -> &'static [Self] {
|
||||
&Self::ALL
|
||||
}
|
||||
@@ -975,6 +962,34 @@ impl ScannerUsagePublicationObservation {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Default)]
|
||||
struct LastScanCycle {
|
||||
sequence: u64,
|
||||
result: u8,
|
||||
partial_reason: ScanCyclePartialReason,
|
||||
partial_source: Option<ScannerWorkSource>,
|
||||
partial_stage: Option<&'static str>,
|
||||
duration_millis: u64,
|
||||
}
|
||||
|
||||
/// Updates only this observation if a later state write fails. A supervisor
|
||||
/// cancellation leaves the already recorded partial result and budget visible.
|
||||
pub struct PartialCycleObservation<'a> {
|
||||
latest: &'a Mutex<LastScanCycle>,
|
||||
sequence: u64,
|
||||
}
|
||||
|
||||
impl PartialCycleObservation<'_> {
|
||||
pub fn observe_state_write(self, persisted: bool) {
|
||||
if !persisted {
|
||||
let mut last = self.latest.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
if last.sequence == self.sequence {
|
||||
last.partial_stage = Some("cycle_state_not_durable");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct Metrics {
|
||||
startup: crate::startup::ScannerStartupObservation,
|
||||
operations: Vec<AtomicU64>,
|
||||
@@ -1020,11 +1035,8 @@ pub struct Metrics {
|
||||
scanner_leader_lock_held: AtomicBool,
|
||||
scanner_leader_lock_last_error: RwLock<String>,
|
||||
scanner_leader_lock_last_update_unix_secs: AtomicU64,
|
||||
last_scan_cycle_result: AtomicU8,
|
||||
last_scan_cycle_partial_reason: AtomicU8,
|
||||
last_scan_cycle_partial_source: AtomicU8,
|
||||
last_scan_cycle: Mutex<LastScanCycle>,
|
||||
last_scan_cycle_end_unix_secs: AtomicU64,
|
||||
last_scan_cycle_duration_millis: AtomicU64,
|
||||
last_scan_cycle_objects_scanned: AtomicU64,
|
||||
last_scan_cycle_directories_scanned: AtomicU64,
|
||||
/// Lifetime count of object versions walked by the data scanner, counted
|
||||
@@ -1140,7 +1152,6 @@ pub struct Metrics {
|
||||
partial_scan_cycles: AtomicU64,
|
||||
}
|
||||
|
||||
const SCAN_CYCLE_RESULT_UNKNOWN: u8 = 0;
|
||||
const SCAN_CYCLE_RESULT_SUCCESS: u8 = 1;
|
||||
const SCAN_CYCLE_RESULT_ERROR: u8 = 2;
|
||||
const SCAN_CYCLE_RESULT_PARTIAL: u8 = 3;
|
||||
@@ -1172,12 +1183,12 @@ impl ScanCyclePartialReason {
|
||||
}
|
||||
}
|
||||
|
||||
fn from_code(code: u8) -> Self {
|
||||
match code {
|
||||
1 => Self::Runtime,
|
||||
2 => Self::Objects,
|
||||
3 => Self::Directories,
|
||||
_ => Self::Unknown,
|
||||
fn budget_stage(self) -> Option<&'static str> {
|
||||
match self {
|
||||
Self::Runtime => Some("runtime_budget"),
|
||||
Self::Objects => Some("object_budget"),
|
||||
Self::Directories => Some("directory_budget"),
|
||||
Self::Unknown => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1577,6 +1588,9 @@ pub struct ScannerMetricsReport {
|
||||
pub last_cycle_partial_source: String,
|
||||
#[serde(default)]
|
||||
pub last_cycle_partial_source_code: u64,
|
||||
/// First observed blocker for this partial cycle, including the exact budget when known.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub last_cycle_partial_stage: Option<String>,
|
||||
pub last_cycle_duration_seconds: f64,
|
||||
#[serde(default)]
|
||||
pub last_cycle_objects_scanned: u64,
|
||||
@@ -2071,8 +2085,18 @@ pub fn emit_scan_cycle_partial_with_source(
|
||||
reason: ScanCyclePartialReason,
|
||||
source: Option<ScannerWorkSource>,
|
||||
) {
|
||||
global_metrics().record_scan_cycle_partial_with_source(duration, reason, source);
|
||||
emit_scan_cycle_partial_with_stage(duration, reason, source, reason.budget_stage());
|
||||
}
|
||||
|
||||
pub fn emit_scan_cycle_partial_with_stage(
|
||||
duration: Duration,
|
||||
reason: ScanCyclePartialReason,
|
||||
source: Option<ScannerWorkSource>,
|
||||
stage: Option<&'static str>,
|
||||
) -> PartialCycleObservation<'static> {
|
||||
let observation = global_metrics().record_scan_cycle_partial_with_stage(duration, reason, source, stage);
|
||||
metrics::counter!(OTEL_SCANNER_CYCLES, "result" => SCAN_CYCLE_RESULT_PARTIAL_LABEL).increment(1);
|
||||
observation
|
||||
}
|
||||
|
||||
pub fn emit_scan_cycle_superseded(duration: Duration) {
|
||||
@@ -2173,11 +2197,8 @@ impl Metrics {
|
||||
scanner_leader_lock_held: AtomicBool::new(false),
|
||||
scanner_leader_lock_last_error: RwLock::new(String::new()),
|
||||
scanner_leader_lock_last_update_unix_secs: AtomicU64::new(0),
|
||||
last_scan_cycle_result: AtomicU8::new(SCAN_CYCLE_RESULT_UNKNOWN),
|
||||
last_scan_cycle_partial_reason: AtomicU8::new(ScanCyclePartialReason::Unknown as u8),
|
||||
last_scan_cycle_partial_source: AtomicU8::new(0),
|
||||
last_scan_cycle: Mutex::new(LastScanCycle::default()),
|
||||
last_scan_cycle_end_unix_secs: AtomicU64::new(0),
|
||||
last_scan_cycle_duration_millis: AtomicU64::new(0),
|
||||
last_scan_cycle_objects_scanned: AtomicU64::new(0),
|
||||
last_scan_cycle_directories_scanned: AtomicU64::new(0),
|
||||
lifetime_versions_scanned: AtomicU64::new(0),
|
||||
@@ -3143,36 +3164,43 @@ impl Metrics {
|
||||
self.failed_scan_cycles.fetch_add(1, Ordering::Relaxed);
|
||||
SCAN_CYCLE_RESULT_ERROR
|
||||
};
|
||||
self.record_scanner_cycle_end_time();
|
||||
self.last_scan_cycle_result.store(result, Ordering::Relaxed);
|
||||
self.last_scan_cycle_partial_reason
|
||||
.store(ScanCyclePartialReason::Unknown as u8, Ordering::Relaxed);
|
||||
self.last_scan_cycle_partial_source.store(0, Ordering::Relaxed);
|
||||
self.last_scan_cycle_duration_millis
|
||||
.store(duration_millis_saturated(duration), Ordering::Relaxed);
|
||||
self.record_last_cycle(
|
||||
duration,
|
||||
LastScanCycle {
|
||||
result,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
pub fn record_scan_cycle_superseded(&self, duration: Duration) {
|
||||
self.record_scanner_cycle_end_time();
|
||||
self.superseded_scan_cycles.fetch_add(1, Ordering::Relaxed);
|
||||
self.last_scan_cycle_result
|
||||
.store(SCAN_CYCLE_RESULT_SUPERSEDED, Ordering::Relaxed);
|
||||
self.last_scan_cycle_partial_reason
|
||||
.store(ScanCyclePartialReason::Unknown as u8, Ordering::Relaxed);
|
||||
self.last_scan_cycle_partial_source.store(0, Ordering::Relaxed);
|
||||
self.last_scan_cycle_duration_millis
|
||||
.store(duration_millis_saturated(duration), Ordering::Relaxed);
|
||||
self.record_last_cycle(
|
||||
duration,
|
||||
LastScanCycle {
|
||||
result: SCAN_CYCLE_RESULT_SUPERSEDED,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
pub fn record_scan_cycle_deferred(&self, duration: Duration) {
|
||||
self.record_last_cycle(
|
||||
duration,
|
||||
LastScanCycle {
|
||||
result: SCAN_CYCLE_RESULT_DEFERRED,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
fn record_last_cycle(&self, duration: Duration, mut cycle: LastScanCycle) -> u64 {
|
||||
self.record_scanner_cycle_end_time();
|
||||
self.last_scan_cycle_result
|
||||
.store(SCAN_CYCLE_RESULT_DEFERRED, Ordering::Relaxed);
|
||||
self.last_scan_cycle_partial_reason
|
||||
.store(ScanCyclePartialReason::Unknown as u8, Ordering::Relaxed);
|
||||
self.last_scan_cycle_partial_source.store(0, Ordering::Relaxed);
|
||||
self.last_scan_cycle_duration_millis
|
||||
.store(duration_millis_saturated(duration), Ordering::Relaxed);
|
||||
cycle.duration_millis = duration_millis_saturated(duration);
|
||||
let mut latest = self.last_scan_cycle.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
cycle.sequence = latest.sequence.wrapping_add(1);
|
||||
*latest = cycle;
|
||||
cycle.sequence
|
||||
}
|
||||
|
||||
pub fn record_scan_cycle_partial(&self, duration: Duration, reason: ScanCyclePartialReason) {
|
||||
@@ -3185,7 +3213,16 @@ impl Metrics {
|
||||
reason: ScanCyclePartialReason,
|
||||
source: Option<ScannerWorkSource>,
|
||||
) {
|
||||
self.record_scanner_cycle_end_time();
|
||||
self.record_scan_cycle_partial_with_stage(duration, reason, source, reason.budget_stage());
|
||||
}
|
||||
|
||||
fn record_scan_cycle_partial_with_stage(
|
||||
&self,
|
||||
duration: Duration,
|
||||
reason: ScanCyclePartialReason,
|
||||
source: Option<ScannerWorkSource>,
|
||||
stage: Option<&'static str>,
|
||||
) -> PartialCycleObservation<'_> {
|
||||
self.partial_scan_cycles.fetch_add(1, Ordering::Relaxed);
|
||||
match reason {
|
||||
ScanCyclePartialReason::Unknown => &self.partial_scan_cycles_unknown,
|
||||
@@ -3194,18 +3231,25 @@ impl Metrics {
|
||||
ScanCyclePartialReason::Directories => &self.partial_scan_cycles_directories,
|
||||
}
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
self.last_scan_cycle_result
|
||||
.store(SCAN_CYCLE_RESULT_PARTIAL, Ordering::Relaxed);
|
||||
self.last_scan_cycle_partial_reason.store(reason as u8, Ordering::Relaxed);
|
||||
self.last_scan_cycle_partial_source
|
||||
.store(source.map(ScannerWorkSource::code).unwrap_or_default(), Ordering::Relaxed);
|
||||
if let Some(source) = source
|
||||
&& let Some(cycles) = self.partial_scan_cycles_by_source.get(source.index())
|
||||
{
|
||||
cycles.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
self.last_scan_cycle_duration_millis
|
||||
.store(duration_millis_saturated(duration), Ordering::Relaxed);
|
||||
let sequence = self.record_last_cycle(
|
||||
duration,
|
||||
LastScanCycle {
|
||||
result: SCAN_CYCLE_RESULT_PARTIAL,
|
||||
partial_reason: reason,
|
||||
partial_source: source,
|
||||
partial_stage: stage,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
PartialCycleObservation {
|
||||
latest: &self.last_scan_cycle,
|
||||
sequence,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn start_scan_cycle_work(&self) -> ScanCycleWorkSnapshot {
|
||||
@@ -3794,19 +3838,19 @@ impl Metrics {
|
||||
m.current_disk_scan_concurrency_limit = disk_scan_concurrency_limit;
|
||||
m.current_disk_bucket_scans_queued = disk_bucket_scans_queued;
|
||||
m.current_disk_bucket_scans_active = disk_bucket_scans_active;
|
||||
let last_cycle_result = self.last_scan_cycle_result.load(Ordering::Relaxed);
|
||||
m.last_cycle_result = scan_cycle_result_label(last_cycle_result).to_string();
|
||||
m.last_cycle_result_code = last_cycle_result as u64;
|
||||
let partial_reason = ScanCyclePartialReason::from_code(self.last_scan_cycle_partial_reason.load(Ordering::Relaxed));
|
||||
m.last_cycle_partial_reason = partial_reason.as_str().to_string();
|
||||
m.last_cycle_partial_reason_code = partial_reason as u8 as u64;
|
||||
let partial_source = ScannerWorkSource::from_code(self.last_scan_cycle_partial_source.load(Ordering::Relaxed));
|
||||
m.last_cycle_partial_source = partial_source
|
||||
let last = *self.last_scan_cycle.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
m.last_cycle_result = scan_cycle_result_label(last.result).to_string();
|
||||
m.last_cycle_result_code = last.result.into();
|
||||
m.last_cycle_partial_reason = last.partial_reason.as_str().to_string();
|
||||
m.last_cycle_partial_reason_code = last.partial_reason as u64;
|
||||
m.last_cycle_partial_source = last
|
||||
.partial_source
|
||||
.map(ScannerWorkSource::as_str)
|
||||
.unwrap_or(SCAN_CYCLE_RESULT_UNKNOWN_LABEL)
|
||||
.to_string();
|
||||
m.last_cycle_partial_source_code = partial_source.map(|source| source.code().into()).unwrap_or_default();
|
||||
m.last_cycle_duration_seconds = self.last_scan_cycle_duration_millis.load(Ordering::Relaxed) as f64 / 1000.0;
|
||||
m.last_cycle_partial_source_code = last.partial_source.map(|source| source.code().into()).unwrap_or_default();
|
||||
m.last_cycle_partial_stage = last.partial_stage.map(str::to_owned);
|
||||
m.last_cycle_duration_seconds = last.duration_millis as f64 / 1000.0;
|
||||
m.last_cycle_objects_scanned = self.last_scan_cycle_objects_scanned.load(Ordering::Relaxed);
|
||||
m.last_cycle_directories_scanned = self.last_scan_cycle_directories_scanned.load(Ordering::Relaxed);
|
||||
m.last_cycle_bucket_drive_scans = self.last_scan_cycle_bucket_drive_scans.load(Ordering::Relaxed);
|
||||
@@ -5223,6 +5267,56 @@ mod tests {
|
||||
assert_eq!(report.partial_cycles, 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delayed_partial_state_write_can_only_update_its_own_observation() {
|
||||
let metrics = Metrics::new();
|
||||
let record = || {
|
||||
metrics.record_scan_cycle_partial_with_stage(
|
||||
Duration::from_secs(1),
|
||||
ScanCyclePartialReason::Runtime,
|
||||
None,
|
||||
Some("runtime_budget"),
|
||||
)
|
||||
};
|
||||
let old = record();
|
||||
let current = record();
|
||||
assert_eq!(
|
||||
metrics.report().await.partial_cycles,
|
||||
2,
|
||||
"both stops are visible before either state write finishes"
|
||||
);
|
||||
old.observe_state_write(false);
|
||||
assert_eq!(metrics.report().await.last_cycle_partial_stage.as_deref(), Some("runtime_budget"));
|
||||
current.observe_state_write(false);
|
||||
let failed = metrics.report().await;
|
||||
assert_eq!(failed.last_cycle_partial_stage.as_deref(), Some("cycle_state_not_durable"));
|
||||
assert_eq!(failed.partial_cycles, 2);
|
||||
assert_eq!(failed.last_cycle_partial_reason, "runtime");
|
||||
let superseded = record();
|
||||
metrics.record_scan_cycle_complete(true, Duration::ZERO);
|
||||
superseded.observe_state_write(false);
|
||||
assert!(metrics.report().await.last_cycle_partial_stage.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn partial_stage_is_reported_with_the_last_result_and_cleared_on_completion() {
|
||||
let metrics = Metrics::new();
|
||||
metrics.record_scan_cycle_partial(Duration::from_secs(1), ScanCyclePartialReason::Directories);
|
||||
let partial = serde_json::to_value(metrics.report().await).unwrap();
|
||||
assert_eq!(partial["last_cycle_partial_reason"], "directories");
|
||||
assert_eq!(partial["last_cycle_partial_stage"], "directory_budget");
|
||||
metrics.record_scan_cycle_complete(true, Duration::from_secs(1));
|
||||
let finished = serde_json::to_value(metrics.report().await).unwrap();
|
||||
assert!(
|
||||
!finished
|
||||
.as_object()
|
||||
.expect("serialized report")
|
||||
.contains_key("last_cycle_partial_stage")
|
||||
);
|
||||
let legacy: ScannerMetricsReport = serde_json::from_value(finished).expect("older reports omit the stage");
|
||||
assert_eq!(legacy.last_cycle_partial_stage, None);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn report_tracks_partial_scan_cycle_without_failed_increment() {
|
||||
let metrics = Metrics::new();
|
||||
|
||||
@@ -73,6 +73,7 @@ hotpath-cpu = [
|
||||
]
|
||||
|
||||
[dependencies]
|
||||
tempfile.workspace = true
|
||||
hotpath.workspace = true
|
||||
rustfs-config = { workspace = true, features = ["server-config-model"] }
|
||||
rustfs-common = { workspace = true }
|
||||
@@ -108,6 +109,13 @@ uuid = { workspace = true, features = ["v4", "serde", "fast-rng"] }
|
||||
bytes.workspace = true
|
||||
hex-simd.workspace = true
|
||||
|
||||
[target.'cfg(any(target_os = "linux", target_os = "macos"))'.dependencies]
|
||||
rustix = { workspace = true, features = ["fs"] }
|
||||
|
||||
[target.'cfg(windows)'.dependencies]
|
||||
same-file.workspace = true
|
||||
winapi-util.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
opentelemetry-proto.workspace = true
|
||||
opentelemetry_sdk.workspace = true
|
||||
@@ -118,7 +126,6 @@ rustfs-test-utils.workspace = true
|
||||
tracing-subscriber = { workspace = true, features = ["json", "env-filter", "time"] }
|
||||
serial_test = { workspace = true }
|
||||
temp-env = { workspace = true, features = ["async_closure"] }
|
||||
tempfile = { workspace = true }
|
||||
uuid = { workspace = true, features = ["v4", "serde", "fast-rng", "macro-diagnostics"] }
|
||||
tokio = { workspace = true, features = ["test-util", "fs", "rt-multi-thread"] }
|
||||
# Test-only: pins the emitted scanner alert wire names against the canonical
|
||||
|
||||
@@ -200,10 +200,25 @@ pub(crate) async fn read_config_revision<S: ScannerObjectIO>(store: Arc<S>, path
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
#[derive(Clone, Debug, Eq)]
|
||||
pub(crate) struct DataUsageCacheRevisions {
|
||||
main: DataUsageCacheRevision,
|
||||
backup: Option<DataUsageCacheRevision>,
|
||||
damaged_pages: DamagedPages,
|
||||
}
|
||||
|
||||
// Admissions compare CAS tokens. A reader's local repair hint does not
|
||||
// indicate that either committed object changed while its locks were released.
|
||||
impl PartialEq for DataUsageCacheRevisions {
|
||||
fn eq(&self, other: &Self) -> bool {
|
||||
self.main == other.main && self.backup == other.backup
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
||||
struct DamagedPages {
|
||||
main: bool,
|
||||
backup: bool,
|
||||
}
|
||||
|
||||
pub static DATA_USAGE_BUCKET: LazyLock<String> =
|
||||
@@ -576,10 +591,31 @@ pub(crate) struct DataUsageDurableProgress {
|
||||
identity: Option<DataUsageScanIdentity>,
|
||||
started_plan: Option<DataUsageScanPlanDigest>,
|
||||
frontier: Option<String>,
|
||||
dormant: HashMap<String, String>,
|
||||
raw: HashMap<String, crate::raw_page_index::RawEnumerationDurableProgress>,
|
||||
compacted: Vec<crate::scanner_folder::DirectoryProgress>,
|
||||
listings: Vec<crate::scanner_folder::DirectoryReadProgress>,
|
||||
active_source: Option<String>,
|
||||
}
|
||||
|
||||
impl DataUsageDurableProgress {
|
||||
fn source_frontier(&self, source: Option<&str>) -> Option<&str> {
|
||||
if source == self.active_source.as_deref() {
|
||||
self.frontier.as_deref()
|
||||
} else {
|
||||
source.and_then(|source| self.dormant.get(source)).map(String::as_str)
|
||||
}
|
||||
}
|
||||
|
||||
fn covers_on(&self, source: Option<&str>, path: &str) -> bool {
|
||||
self.source_frontier(source)
|
||||
.is_some_and(|frontier| scan_frontier_covers_path(path, frontier))
|
||||
|| self
|
||||
.compacted
|
||||
.iter()
|
||||
.any(|done| Some(done.physical.endpoint.as_str()) == source && done.covers(path))
|
||||
}
|
||||
|
||||
pub(crate) fn advanced_from(&self, previous: &Self) -> bool {
|
||||
if self.bucket != previous.bucket
|
||||
|| self.source.is_none()
|
||||
@@ -591,31 +627,69 @@ impl DataUsageDurableProgress {
|
||||
{
|
||||
return false;
|
||||
}
|
||||
let mut advanced = match (&previous.frontier, &self.frontier) {
|
||||
(Some(before), Some(after)) => {
|
||||
if !scan_frontier_covers_path(before, after) {
|
||||
return false;
|
||||
let active = self.active_source.as_deref();
|
||||
let old = previous.active_source.as_deref();
|
||||
let switched = old.is_some() && active != old;
|
||||
// A frontier can move from active to dormant without earning a turn.
|
||||
// Retention and forward movement are compared only on the same source.
|
||||
if previous.frontier.as_deref().is_some_and(|path| !self.covers_on(old, path))
|
||||
|| previous
|
||||
.dormant
|
||||
.iter()
|
||||
.any(|(source, path)| !self.covers_on(Some(source), path))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
let mut advanced = self.frontier.as_deref().is_some_and(|now| {
|
||||
previous
|
||||
.source_frontier(active)
|
||||
.is_none_or(|before| before != now && scan_frontier_covers_path(before, now))
|
||||
});
|
||||
if switched {
|
||||
// Legacy raw-name pages are local to their writer. The physical
|
||||
// root position must still exist for that writer across rotation.
|
||||
if !previous.raw.is_empty()
|
||||
&& !previous.listings.iter().any(|before| {
|
||||
before.parent == self.bucket
|
||||
&& Some(before.physical.endpoint.as_str()) == old
|
||||
&& self.listings.iter().any(|after| after.extends(before))
|
||||
})
|
||||
{
|
||||
return false;
|
||||
}
|
||||
} else {
|
||||
for (parent, before) in &previous.raw {
|
||||
match self.raw.get(parent) {
|
||||
Some(after) if after.extends(before) => {}
|
||||
None if self.covers_on(active, parent) => {}
|
||||
_ => return false,
|
||||
}
|
||||
before != after
|
||||
}
|
||||
(Some(_), None) => return false,
|
||||
(None, Some(_)) => true,
|
||||
(None, None) => false,
|
||||
};
|
||||
for (parent, before) in &previous.raw {
|
||||
match self.raw.get(parent) {
|
||||
Some(after) if after.extends(before) => {}
|
||||
None if self
|
||||
.frontier
|
||||
.as_ref()
|
||||
.is_some_and(|frontier| scan_frontier_covers_path(parent, frontier)) => {}
|
||||
_ => return false,
|
||||
advanced |= self.raw.iter().any(|(parent, after)| {
|
||||
after.committed_entries() > previous.raw.get(parent).map_or(0, |before| before.committed_entries())
|
||||
});
|
||||
}
|
||||
for before in &previous.listings {
|
||||
if !self.listings.iter().any(|after| after.extends(before))
|
||||
&& !self.covers_on(Some(&before.physical.endpoint), &before.parent)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
for (parent, after) in &self.raw {
|
||||
let before = previous.raw.get(parent).map_or(0, |value| value.committed_entries());
|
||||
advanced |= after.committed_entries() > before;
|
||||
for before in &previous.compacted {
|
||||
if !self.compacted.iter().any(|after| after.extends(before))
|
||||
&& !self.covers_on(Some(&before.physical.endpoint), &before.parent)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
advanced |= self.listings.iter().any(|after| {
|
||||
Some(after.physical.endpoint.as_str()) == active && !previous.listings.iter().any(|before| before.extends(after))
|
||||
});
|
||||
advanced |= self.compacted.iter().any(|after| {
|
||||
Some(after.physical.endpoint.as_str()) == active && !previous.compacted.iter().any(|before| before.extends(after))
|
||||
});
|
||||
|
||||
advanced
|
||||
}
|
||||
}
|
||||
@@ -905,6 +979,14 @@ pub struct DataUsageCacheInfo {
|
||||
pub scan_raw_enumeration_cursor: Option<DataUsageRawEnumerationCursor>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub scan_raw_enumeration_page_index: Option<RawEnumerationPageIndex>,
|
||||
/// Source-bound native listing positions and fully committed compacted work.
|
||||
#[serde(default)]
|
||||
pub scan_directory_work: Vec<crate::scanner_folder::DirectoryCheckpoint>,
|
||||
/// Physical source of the regular linked tree and its frontier.
|
||||
#[serde(default)]
|
||||
pub scan_directory_source: Option<String>,
|
||||
#[serde(default)]
|
||||
pub scan_source_trees: std::collections::BTreeMap<String, Arc<source_tree::SourceTree>>,
|
||||
/// Indexes of the other directories on the active walk path. A restart
|
||||
/// must replay every wide ancestor before it reaches the selected one, and
|
||||
/// recharging those replays would stop a fixed budget from ever advancing.
|
||||
@@ -994,6 +1076,9 @@ impl Serialize for DataUsageCacheInfo {
|
||||
+ usize::from(self.scan_raw_enumeration_cursor.is_some())
|
||||
+ usize::from(self.scan_raw_enumeration_page_index.is_some())
|
||||
+ usize::from(!self.scan_raw_enumeration_path_page_indexes.is_empty())
|
||||
+ usize::from(!self.scan_directory_work.is_empty())
|
||||
+ usize::from(self.scan_directory_source.is_some())
|
||||
+ usize::from(!self.scan_source_trees.is_empty())
|
||||
+ usize::from(self.scan_identity.is_some())
|
||||
+ usize::from(self.scan_progress.is_some())
|
||||
+ usize::from(self.scan_usage_watch.is_some())
|
||||
@@ -1029,6 +1114,15 @@ impl Serialize for DataUsageCacheInfo {
|
||||
if let Some(index) = &self.scan_raw_enumeration_page_index {
|
||||
state.serialize_entry("scan_raw_enumeration_page_index", index)?;
|
||||
}
|
||||
if !self.scan_source_trees.is_empty() {
|
||||
state.serialize_entry("scan_source_trees", &self.scan_source_trees)?;
|
||||
}
|
||||
if let Some(source) = &self.scan_directory_source {
|
||||
state.serialize_entry("scan_directory_source", source)?;
|
||||
}
|
||||
if !self.scan_directory_work.is_empty() {
|
||||
state.serialize_entry("scan_directory_work", &self.scan_directory_work)?;
|
||||
}
|
||||
if !self.scan_raw_enumeration_path_page_indexes.is_empty() {
|
||||
state.serialize_entry("scan_raw_enumeration_path_page_indexes", &self.scan_raw_enumeration_path_page_indexes)?;
|
||||
}
|
||||
@@ -1236,6 +1330,25 @@ impl DataUsageCache {
|
||||
identity: self.info.scan_identity,
|
||||
started_plan: self.info.scan_progress.map(|progress| progress.started_plan),
|
||||
frontier: self.validated_scan_frontier().map(str::to_owned),
|
||||
dormant: self
|
||||
.info
|
||||
.scan_source_trees
|
||||
.iter()
|
||||
.filter_map(|(source, tree)| tree.frontier(&self.info).map(|last| (source.clone(), last.to_owned())))
|
||||
.collect(),
|
||||
active_source: self.info.scan_directory_source.clone(),
|
||||
listings: self
|
||||
.info
|
||||
.scan_directory_work
|
||||
.iter()
|
||||
.filter_map(|d| d.read_progress(&self.info))
|
||||
.collect(),
|
||||
compacted: self
|
||||
.info
|
||||
.scan_directory_work
|
||||
.iter()
|
||||
.filter_map(|d| d.progress(&self.info))
|
||||
.collect(),
|
||||
raw: self
|
||||
.info
|
||||
.scan_raw_enumeration_page_index
|
||||
@@ -1249,7 +1362,8 @@ impl DataUsageCache {
|
||||
}
|
||||
|
||||
pub(crate) fn has_scan_position(&self) -> bool {
|
||||
self.info.scan_resume_after.is_some()
|
||||
!self.info.scan_directory_work.is_empty()
|
||||
|| self.info.scan_resume_after.is_some()
|
||||
|| self.info.scan_checkpoint.is_some()
|
||||
|| self.info.scan_raw_enumeration_cursor.is_some()
|
||||
|| self.info.scan_raw_enumeration_page_index.is_some()
|
||||
@@ -1319,7 +1433,8 @@ impl DataUsageCache {
|
||||
&& self.info.scan_identity == Some(identity)
|
||||
&& (self.validated_scan_frontier().is_some()
|
||||
|| self.validated_raw_enumeration_cursor().is_some()
|
||||
|| self.validated_raw_enumeration_page_index().is_some());
|
||||
|| self.validated_raw_enumeration_page_index().is_some()
|
||||
|| self.info.scan_directory_work.iter().any(|d| d.is_bound_to(&self.info)));
|
||||
let handoff_frontier = cross_epoch_checkpoint
|
||||
.then(|| self.validated_scan_frontier().map(clone_cache_value))
|
||||
.flatten();
|
||||
@@ -1404,7 +1519,10 @@ impl DataUsageCache {
|
||||
.take(MAX_DATA_USAGE_CACHE_DEPTH)
|
||||
.filter(|index| self.raw_enumeration_page_index_is_valid(index))
|
||||
.collect();
|
||||
let directory_work = std::mem::take(&mut self.info.scan_directory_work);
|
||||
self.info.scan_directory_work = directory_work.into_iter().filter(|d| d.is_bound_to(&self.info)).collect();
|
||||
let cursor_is_valid = !self.has_scan_position()
|
||||
|| !self.info.scan_directory_work.is_empty()
|
||||
|| self.validated_scan_frontier().is_some()
|
||||
|| self.info.scan_raw_enumeration_cursor.is_some()
|
||||
|| self.info.scan_raw_enumeration_page_index.is_some();
|
||||
@@ -1447,6 +1565,9 @@ impl DataUsageCache {
|
||||
self.info.scan_raw_enumeration_cursor = None;
|
||||
self.info.scan_raw_enumeration_page_index = None;
|
||||
self.info.scan_raw_enumeration_path_page_indexes.clear();
|
||||
self.info.scan_directory_work.clear();
|
||||
self.info.scan_directory_source = None;
|
||||
self.info.scan_source_trees.clear();
|
||||
self.info.scan_coverage_receipt = None;
|
||||
}
|
||||
// Old readers do not understand coverage sweeps. An absent plan makes
|
||||
@@ -2192,6 +2313,7 @@ impl DataUsageCache {
|
||||
}
|
||||
|
||||
mod persistence;
|
||||
mod source_tree;
|
||||
|
||||
#[derive(Default, Clone)]
|
||||
struct Inner {
|
||||
|
||||
@@ -12,6 +12,9 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
mod pages;
|
||||
mod scan_load;
|
||||
|
||||
use super::*;
|
||||
use bytes::Bytes;
|
||||
use std::sync::Arc;
|
||||
@@ -71,6 +74,7 @@ struct DataUsageCacheLoadResult {
|
||||
cache: DataUsageCache,
|
||||
main_revision: DataUsageCacheRevision,
|
||||
backup_revision: Option<DataUsageCacheRevision>,
|
||||
damaged_pages: DamagedPages,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
@@ -107,34 +111,42 @@ impl DataUsageCache {
|
||||
store: Arc<S>,
|
||||
name: &str,
|
||||
) -> StorageResult<DataUsageCacheRevisions> {
|
||||
let backup_name = format!("{name}.bkp");
|
||||
let backup_path = path_join_buf(&[BUCKET_META_PREFIX, &backup_name]);
|
||||
let loaded = Self::load_cache(store.clone(), name).await?;
|
||||
let backup = match loaded.backup_revision {
|
||||
Some(revision) => Some(revision),
|
||||
None => match read_config_revision(store, &backup_path).await {
|
||||
Ok(revision) => Some(revision),
|
||||
Err(err) => {
|
||||
counter!(METRIC_CACHE_BACKUP_REVISION_FAILURE_TOTAL).increment(1);
|
||||
debug!(
|
||||
target: "rustfs::scanner::data_usage",
|
||||
event = EVENT_SCANNER_CACHE_LOAD_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_CACHE,
|
||||
cache_name = %name,
|
||||
backup_path = %backup_path,
|
||||
state = "backup_revision_unavailable",
|
||||
error = %err,
|
||||
"Scanner cache backup revision lookup failed"
|
||||
);
|
||||
None
|
||||
}
|
||||
},
|
||||
None => Self::backup_revision(store, name).await,
|
||||
};
|
||||
let main = loaded.main_revision;
|
||||
*self = loaded.cache;
|
||||
|
||||
Ok(DataUsageCacheRevisions { main, backup })
|
||||
Ok(DataUsageCacheRevisions {
|
||||
main,
|
||||
backup,
|
||||
damaged_pages: loaded.damaged_pages,
|
||||
})
|
||||
}
|
||||
|
||||
async fn backup_revision<S: ScannerObjectIO>(store: Arc<S>, name: &str) -> Option<DataUsageCacheRevision> {
|
||||
let backup_name = format!("{name}.bkp");
|
||||
let backup_path = path_join_buf(&[BUCKET_META_PREFIX, &backup_name]);
|
||||
match read_config_revision(store, &backup_path).await {
|
||||
Ok(revision) => Some(revision),
|
||||
Err(err) => {
|
||||
counter!(METRIC_CACHE_BACKUP_REVISION_FAILURE_TOTAL).increment(1);
|
||||
debug!(
|
||||
target: "rustfs::scanner::data_usage",
|
||||
event = EVENT_SCANNER_CACHE_LOAD_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_CACHE,
|
||||
cache_name = %name,
|
||||
backup_path = %backup_path,
|
||||
state = "backup_revision_unavailable",
|
||||
error = %err,
|
||||
"Scanner cache backup revision lookup failed"
|
||||
);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn read_revisions<S: ScannerObjectIO>(store: Arc<S>, name: &str) -> StorageResult<DataUsageCacheRevisions> {
|
||||
@@ -143,7 +155,11 @@ impl DataUsageCache {
|
||||
let backup_path = path_join_buf(&[BUCKET_META_PREFIX, &backup_name]);
|
||||
let main = read_config_revision(store.clone(), &main_path).await?;
|
||||
let backup = Some(read_config_revision(store, &backup_path).await?);
|
||||
Ok(DataUsageCacheRevisions { main, backup })
|
||||
Ok(DataUsageCacheRevisions {
|
||||
main,
|
||||
backup,
|
||||
damaged_pages: DamagedPages::default(),
|
||||
})
|
||||
}
|
||||
|
||||
async fn load_cache<S: ScannerObjectIO>(store: Arc<S>, name: &str) -> StorageResult<DataUsageCacheLoadResult> {
|
||||
@@ -161,6 +177,7 @@ impl DataUsageCache {
|
||||
cache: *cache,
|
||||
main_revision,
|
||||
backup_revision: None,
|
||||
damaged_pages: DamagedPages::default(),
|
||||
});
|
||||
}
|
||||
DataUsageCacheLoadAttempt::Loaded { revision: None, .. } => {
|
||||
@@ -185,6 +202,11 @@ impl DataUsageCache {
|
||||
let backup_name = format!("{name}.bkp");
|
||||
let backup_attempt = Self::try_load_inner(store.clone(), &backup_name, Duration::from_secs(30)).await?;
|
||||
let backup_revision = backup_attempt.revision();
|
||||
if backup_revision.is_some()
|
||||
&& let Some(DataUsageCacheRevision::Etag(etag)) = &main_revision
|
||||
{
|
||||
scan_load::PendingBackup::finish(&store, name, etag);
|
||||
}
|
||||
let backup_state = match backup_attempt {
|
||||
DataUsageCacheLoadAttempt::Loaded {
|
||||
cache,
|
||||
@@ -198,6 +220,10 @@ impl DataUsageCache {
|
||||
cache: *cache,
|
||||
main_revision,
|
||||
backup_revision: Some(backup_revision),
|
||||
damaged_pages: DamagedPages {
|
||||
main: main_state == DataUsageCacheLoadState::Corrupt,
|
||||
backup: false,
|
||||
},
|
||||
});
|
||||
}
|
||||
DataUsageCacheLoadState::Loaded
|
||||
@@ -221,6 +247,10 @@ impl DataUsageCache {
|
||||
main_revision: main_revision
|
||||
.ok_or_else(|| Error::other(format!("scanner cache missing state has no revision: {name}")))?,
|
||||
backup_revision,
|
||||
damaged_pages: DamagedPages {
|
||||
main: main_state == DataUsageCacheLoadState::Corrupt,
|
||||
backup: backup_state == DataUsageCacheLoadState::Corrupt,
|
||||
},
|
||||
});
|
||||
}
|
||||
(DataUsageCacheLoadState::Corrupt, DataUsageCacheLoadState::Missing)
|
||||
@@ -240,6 +270,10 @@ impl DataUsageCache {
|
||||
main_revision: main_revision
|
||||
.ok_or_else(|| Error::other(format!("scanner cache corrupt state has no revision: {name}")))?,
|
||||
backup_revision,
|
||||
damaged_pages: DamagedPages {
|
||||
main: main_state == DataUsageCacheLoadState::Corrupt,
|
||||
backup: backup_state == DataUsageCacheLoadState::Corrupt,
|
||||
},
|
||||
});
|
||||
}
|
||||
_ => {}
|
||||
@@ -295,10 +329,35 @@ impl DataUsageCache {
|
||||
.filter(|etag| !etag.is_empty())
|
||||
.cloned()
|
||||
.map(DataUsageCacheRevision::Etag);
|
||||
if let Some(DataUsageCacheRevision::Etag(etag)) = &revision
|
||||
&& scan_load::PendingBackup::needs_backup(&store, load_name, etag)
|
||||
{
|
||||
return Ok((None, revision, true));
|
||||
}
|
||||
match reader.read_all().await {
|
||||
Ok(data) => match DataUsageCache::unmarshal(&data) {
|
||||
Ok(data) => match Box::pin(pages::decode(store.clone(), load_name, &data)).await {
|
||||
Ok(cache) => Ok((Some(cache), revision, false)),
|
||||
Err(_) => Ok((None, revision, true)),
|
||||
Err(err)
|
||||
if matches!(err, StorageError::Io(ref io) if io.kind() == std::io::ErrorKind::InvalidData)
|
||||
|| matches!(err, StorageError::FileNotFound | StorageError::ObjectNotFound(..))
|
||||
|| (pages::is_manifest(&data) && pages::unreadable_page(&err)) =>
|
||||
{
|
||||
// A concurrent manifest commit can retire the generation
|
||||
// we were reading. Retry the current root before treating
|
||||
// missing pages as corruption or falling back to a backup.
|
||||
if pages::is_manifest(&data) {
|
||||
if Some(read_config_revision(store.clone(), &path).await?) != revision {
|
||||
return Err(StorageError::PreconditionFailed);
|
||||
}
|
||||
if !load_name.ends_with(".bkp")
|
||||
&& let Some(DataUsageCacheRevision::Etag(etag)) = &revision
|
||||
{
|
||||
scan_load::PendingBackup::begin(&store, load_name, etag);
|
||||
}
|
||||
}
|
||||
Ok((None, revision, true))
|
||||
}
|
||||
Err(err) => Err(err),
|
||||
},
|
||||
Err(e) => {
|
||||
// Read error
|
||||
@@ -652,20 +711,44 @@ impl DataUsageCache {
|
||||
expected_epoch: Option<u64>,
|
||||
) -> StorageResult<()> {
|
||||
let serialize_started = Instant::now();
|
||||
let mut buf = Vec::new();
|
||||
self.serialize(&mut rmp_serde::Serializer::new(&mut buf))?;
|
||||
// Primary, backup, and retries read the same immutable encoded body.
|
||||
let buf = Bytes::from(buf);
|
||||
counter!("rustfs_scanner_cache_serialized_bytes_total").increment(u64::try_from(buf.len()).unwrap_or(u64::MAX));
|
||||
let paged = matches!((revisions, expected_epoch), (Some(_), Some(_))) && pages::eligible(self, name);
|
||||
let encoded = if paged { Some(pages::encode(self)?) } else { None };
|
||||
let inline = if paged {
|
||||
Bytes::new()
|
||||
} else {
|
||||
let mut buf = Vec::new();
|
||||
self.serialize(&mut rmp_serde::Serializer::new(&mut buf))?;
|
||||
Bytes::from(buf)
|
||||
};
|
||||
counter!("rustfs_scanner_cache_serialized_bytes_total").increment(
|
||||
encoded
|
||||
.as_ref()
|
||||
.map_or_else(|| u64::try_from(inline.len()).unwrap_or(u64::MAX), pages::EncodedPages::length),
|
||||
);
|
||||
histogram!("rustfs_scanner_cache_serialize_seconds").record(serialize_started.elapsed().as_secs_f64());
|
||||
let timeout_duration = Self::cache_save_timeout();
|
||||
|
||||
let (primary_buf, primary_retired) = match (&encoded, revisions, expected_epoch) {
|
||||
(Some(encoded), Some(revisions), Some(epoch)) => {
|
||||
Box::pin(pages::prepare(
|
||||
self,
|
||||
store.clone(),
|
||||
name,
|
||||
&revisions.main,
|
||||
encoded,
|
||||
epoch,
|
||||
revisions.damaged_pages.main,
|
||||
))
|
||||
.await?
|
||||
}
|
||||
_ => (inline.clone(), [None, None]),
|
||||
};
|
||||
let path = path_join_buf(&[BUCKET_META_PREFIX, name]);
|
||||
let primary_save_started = Instant::now();
|
||||
Self::save_path_with_retry(
|
||||
store.clone(),
|
||||
&path,
|
||||
&buf,
|
||||
&primary_buf,
|
||||
timeout_duration,
|
||||
DATA_USAGE_CACHE_SAVE_RETRIES,
|
||||
revisions.map(|revisions| revisions.main.clone()),
|
||||
@@ -673,6 +756,7 @@ impl DataUsageCache {
|
||||
)
|
||||
.await?;
|
||||
|
||||
pages::reclaim_retired(store.clone(), name, primary_retired, expected_epoch).await;
|
||||
let backup_name = format!("{name}.bkp");
|
||||
let backup_path = path_join_buf(&[BUCKET_META_PREFIX, &backup_name]);
|
||||
let backup_timeout_duration = Self::backup_cache_save_timeout(timeout_duration, primary_save_started.elapsed());
|
||||
@@ -680,17 +764,36 @@ impl DataUsageCache {
|
||||
if revisions.is_some() && backup_revision.is_none() {
|
||||
return Ok(());
|
||||
}
|
||||
if let Err(e) = Self::save_path_with_retry(
|
||||
store,
|
||||
&backup_path,
|
||||
&buf,
|
||||
backup_timeout_duration,
|
||||
DATA_USAGE_CACHE_BACKUP_SAVE_RETRIES,
|
||||
backup_revision,
|
||||
expected_epoch,
|
||||
)
|
||||
.await
|
||||
{
|
||||
let save_backup = async {
|
||||
let (backup_buf, retired) = match (&encoded, &backup_revision, expected_epoch) {
|
||||
(Some(encoded), Some(revision), Some(epoch)) => {
|
||||
Box::pin(pages::prepare(
|
||||
self,
|
||||
store.clone(),
|
||||
&backup_name,
|
||||
revision,
|
||||
encoded,
|
||||
epoch,
|
||||
revisions.is_some_and(|revisions| revisions.damaged_pages.backup),
|
||||
))
|
||||
.await?
|
||||
}
|
||||
_ => (inline, [None, None]),
|
||||
};
|
||||
Self::save_path_with_retry(
|
||||
store.clone(),
|
||||
&backup_path,
|
||||
&backup_buf,
|
||||
backup_timeout_duration,
|
||||
DATA_USAGE_CACHE_BACKUP_SAVE_RETRIES,
|
||||
backup_revision,
|
||||
expected_epoch,
|
||||
)
|
||||
.await?;
|
||||
pages::reclaim_retired(store, &backup_name, retired, expected_epoch).await;
|
||||
Ok::<_, StorageError>(())
|
||||
};
|
||||
if let Err(e) = save_backup.await {
|
||||
warn!(
|
||||
target: "rustfs::scanner::data_usage",
|
||||
event = EVENT_SCANNER_CACHE_SAVE_STATE,
|
||||
|
||||
@@ -0,0 +1,646 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
mod restore;
|
||||
mod save;
|
||||
mod staging;
|
||||
|
||||
use super::*;
|
||||
use futures::future::try_join_all;
|
||||
use sha2::Digest as _;
|
||||
use std::collections::{BTreeMap, HashSet};
|
||||
use std::io::Write;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
const PAGE_MIN: usize = 8 * 1024;
|
||||
const PAGE_MAX: usize = 64 * 1024;
|
||||
const FANOUT: usize = 128;
|
||||
const IO_CONCURRENCY: usize = 4;
|
||||
const PREFIX: &[u8] = b"\x83\xa7pagesV1";
|
||||
type Digest = [u8; 32];
|
||||
pub(super) type RetiredBundles = [Option<uuid::Uuid>; 2];
|
||||
|
||||
/// Older readers see the committed compact bucket or an incomplete empty tree.
|
||||
#[derive(Deserialize)]
|
||||
struct PageEnvelope {
|
||||
#[serde(rename = "pagesV1")]
|
||||
header: Header,
|
||||
info: DataUsageCacheInfo,
|
||||
#[serde(rename = "cache")]
|
||||
_cache: serde::de::IgnoredAny,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
struct Header {
|
||||
body: Payload,
|
||||
retired: Option<uuid::Uuid>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
abandoned: Option<uuid::Uuid>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
enum Payload {
|
||||
Inline,
|
||||
Pages(Manifest),
|
||||
}
|
||||
|
||||
impl Header {
|
||||
fn manifest(&self) -> Option<&Manifest> {
|
||||
match &self.body {
|
||||
Payload::Pages(manifest) => Some(manifest),
|
||||
Payload::Inline => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn retired_bundles(&self) -> RetiredBundles {
|
||||
[self.retired, self.abandoned]
|
||||
}
|
||||
|
||||
fn is_valid(&self) -> bool {
|
||||
let live = self.manifest().map(|manifest| manifest.bundle);
|
||||
self.manifest()
|
||||
.is_none_or(|manifest| !manifest.bundle.is_nil() && manifest.length != 0 && manifest.level <= 9)
|
||||
&& (self.abandoned.is_none() || self.abandoned != self.retired)
|
||||
&& self
|
||||
.retired_bundles()
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.all(|id| !id.is_nil() && Some(id) != live)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
|
||||
struct Manifest {
|
||||
bundle: uuid::Uuid,
|
||||
finished: bool,
|
||||
cycle: u64,
|
||||
length: u64,
|
||||
level: u8,
|
||||
root: Digest,
|
||||
#[serde(default)]
|
||||
bundle_bytes: u64,
|
||||
#[serde(default)]
|
||||
bundle_commits: u16,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct Archive {
|
||||
blocks: BTreeMap<Digest, Bytes>,
|
||||
leaves: Vec<Digest>,
|
||||
buffer: Vec<u8>,
|
||||
rolling: u64,
|
||||
length: u64,
|
||||
}
|
||||
|
||||
fn invalid(message: &str) -> Error {
|
||||
Error::Io(std::io::Error::new(std::io::ErrorKind::InvalidData, message.to_owned()))
|
||||
}
|
||||
|
||||
pub(super) fn unreadable_page(error: &StorageError) -> bool {
|
||||
matches!(error, StorageError::FileCorrupt)
|
||||
|| matches!(error, StorageError::Io(io) if matches!(io.kind(), std::io::ErrorKind::InvalidData | std::io::ErrorKind::UnexpectedEof))
|
||||
}
|
||||
|
||||
fn digest(data: &[u8]) -> Digest {
|
||||
Sha256::digest(data).into()
|
||||
}
|
||||
|
||||
fn bundle_prefix(name: &str, bundle: uuid::Uuid) -> String {
|
||||
format!("{BUCKET_META_PREFIX}/{name}.pages/{bundle}/")
|
||||
}
|
||||
|
||||
fn block_path(name: &str, bundle: uuid::Uuid, block: &Digest) -> String {
|
||||
let hash = hex_simd::encode_to_string(block, hex_simd::AsciiCase::Lower);
|
||||
format!("{}{hash}", bundle_prefix(name, bundle))
|
||||
}
|
||||
|
||||
fn bucket_cache_name(name: &str) -> Option<&str> {
|
||||
let (bucket, leaf) = name.split_once('/')?;
|
||||
(!crate::is_reserved_or_invalid_bucket(bucket, true) && leaf == DATA_USAGE_CACHE_NAME).then_some(name)
|
||||
}
|
||||
|
||||
// The high bits of a multiplicative hash of each byte give a rolling
|
||||
// content-defined boundary. Insertion before a sealed run need not shift the
|
||||
// offsets of every following page. Page length is bounded for every input.
|
||||
const fn byte_gear(byte: u8) -> u64 {
|
||||
let mut x = (byte as u64).wrapping_add(0x9e37_79b9_7f4a_7c15);
|
||||
x = (x ^ (x >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9);
|
||||
x = (x ^ (x >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb);
|
||||
x ^ (x >> 31)
|
||||
}
|
||||
|
||||
const GEAR: [u64; 256] = {
|
||||
let mut table = [0; 256];
|
||||
let mut byte = 0u8;
|
||||
loop {
|
||||
table[byte as usize] = byte_gear(byte);
|
||||
if byte == u8::MAX {
|
||||
break;
|
||||
}
|
||||
byte += 1;
|
||||
}
|
||||
table
|
||||
};
|
||||
|
||||
impl Archive {
|
||||
fn intern(&mut self, bytes: Vec<u8>) -> Digest {
|
||||
let key = digest(&bytes);
|
||||
self.blocks.entry(key).or_insert_with(|| Bytes::from(bytes));
|
||||
key
|
||||
}
|
||||
|
||||
fn seal(&mut self) {
|
||||
if !self.buffer.is_empty() {
|
||||
let bytes = std::mem::take(&mut self.buffer);
|
||||
let key = self.intern(bytes);
|
||||
self.leaves.push(key);
|
||||
self.rolling = 0;
|
||||
}
|
||||
}
|
||||
|
||||
fn finish(&mut self, bundle: uuid::Uuid) -> StorageResult<Manifest> {
|
||||
self.seal();
|
||||
let mut children = std::mem::take(&mut self.leaves);
|
||||
let mut level = 0;
|
||||
while children.len() > 1 {
|
||||
children = children
|
||||
.chunks(FANOUT)
|
||||
.map(|chunk| {
|
||||
let bytes = chunk.iter().flatten().copied().collect();
|
||||
self.intern(bytes)
|
||||
})
|
||||
.collect();
|
||||
level += 1;
|
||||
}
|
||||
Ok(Manifest {
|
||||
bundle,
|
||||
finished: false,
|
||||
cycle: 0,
|
||||
length: self.length,
|
||||
level,
|
||||
root: children.pop().ok_or_else(|| invalid("empty scanner checkpoint"))?,
|
||||
bundle_bytes: 0,
|
||||
bundle_commits: 0,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl Write for Archive {
|
||||
fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
|
||||
self.length = self
|
||||
.length
|
||||
.checked_add(u64::try_from(bytes.len()).map_err(std::io::Error::other)?)
|
||||
.ok_or_else(|| std::io::Error::other("scanner checkpoint is too large"))?;
|
||||
let mut start = 0;
|
||||
for (offset, byte) in bytes.iter().enumerate() {
|
||||
self.rolling = self.rolling.wrapping_shl(1).wrapping_add(GEAR[usize::from(*byte)]);
|
||||
let end = offset + 1;
|
||||
let len = self.buffer.len() + (end - start);
|
||||
if len == PAGE_MAX || (len >= PAGE_MIN && self.rolling & 0x3fff == 0) {
|
||||
self.buffer.extend_from_slice(&bytes[start..end]);
|
||||
self.seal();
|
||||
start = end;
|
||||
}
|
||||
}
|
||||
self.buffer.extend_from_slice(&bytes[start..]);
|
||||
Ok(bytes.len())
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> std::io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) struct EncodedPages {
|
||||
archive: Archive,
|
||||
root: Manifest,
|
||||
inline: Option<Bytes>,
|
||||
}
|
||||
|
||||
impl EncodedPages {
|
||||
pub(super) fn length(&self) -> u64 {
|
||||
self.archive.length
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn encode(cache: &DataUsageCache) -> StorageResult<EncodedPages> {
|
||||
let mut archive = Archive::default();
|
||||
archive.write_all(&[0x92])?;
|
||||
cache.info.serialize(&mut rmp_serde::Serializer::new(&mut archive))?;
|
||||
archive.seal();
|
||||
let count = u32::try_from(cache.cache.len()).map_err(|_| invalid("too many scanner checkpoint entries"))?;
|
||||
// A wide root's growing child set must not shift every subtree page.
|
||||
let mut entries: Vec<_> = cache.cache.iter().collect();
|
||||
entries.sort_unstable_by(|a, b| a.0.cmp(b.0));
|
||||
let mut header = Vec::new();
|
||||
if count < 16 {
|
||||
header.push(0x80 | u8::try_from(count).map_err(|_| invalid("invalid fixed map length"))?);
|
||||
} else if let Ok(n) = u16::try_from(count) {
|
||||
header.push(0xde);
|
||||
header.extend_from_slice(&n.to_be_bytes());
|
||||
} else {
|
||||
header.push(0xdf);
|
||||
header.extend_from_slice(&count.to_be_bytes());
|
||||
}
|
||||
archive.write_all(&header)?;
|
||||
for (name, entry) in entries {
|
||||
let mut serializer = rmp_serde::Serializer::new(&mut archive);
|
||||
name.serialize(&mut serializer)?;
|
||||
entry.ordered_encoding().serialize(&mut serializer)?;
|
||||
if name == &cache.info.name {
|
||||
archive.seal();
|
||||
}
|
||||
}
|
||||
archive.seal();
|
||||
let inline = if archive.length <= PAGE_MIN as u64 {
|
||||
let mut buf = Vec::new();
|
||||
for key in &archive.leaves {
|
||||
buf.extend_from_slice(
|
||||
archive
|
||||
.blocks
|
||||
.get(key)
|
||||
.ok_or_else(|| invalid("missing encoded scanner page"))?,
|
||||
);
|
||||
}
|
||||
Some(Bytes::from(buf))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let root = archive.finish(uuid::Uuid::nil())?;
|
||||
Ok(EncodedPages { archive, root, inline })
|
||||
}
|
||||
|
||||
pub(super) fn is_manifest(bytes: &[u8]) -> bool {
|
||||
bytes.starts_with(PREFIX)
|
||||
}
|
||||
|
||||
fn read_envelope(bytes: &[u8], bucket: &str) -> StorageResult<PageEnvelope> {
|
||||
let head: PageEnvelope = rmp_serde::from_slice(bytes).map_err(|_| invalid("invalid scanner checkpoint manifest"))?;
|
||||
if head.info.name != bucket || !head.header.is_valid() {
|
||||
return Err(invalid("invalid scanner checkpoint identity or size"));
|
||||
}
|
||||
Ok(head)
|
||||
}
|
||||
|
||||
async fn read_block<S: ScannerObjectIO>(store: Arc<S>, name: &str, bundle: uuid::Uuid, key: Digest) -> StorageResult<Bytes> {
|
||||
let path = block_path(name, bundle, &key);
|
||||
let mut reader = store
|
||||
.get_object_reader(
|
||||
RUSTFS_META_BUCKET,
|
||||
&path,
|
||||
None,
|
||||
HeaderMap::new(),
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
if reader.object_info.size < 1 || usize::try_from(reader.object_info.size).map_or(true, |size| size > PAGE_MAX) {
|
||||
return Err(invalid("invalid scanner checkpoint page length"));
|
||||
}
|
||||
let bytes = Bytes::from(reader.read_all().await?);
|
||||
if bytes.len() > PAGE_MAX || digest(&bytes) != key {
|
||||
return Err(invalid("scanner checkpoint page checksum mismatch"));
|
||||
}
|
||||
Ok(bytes)
|
||||
}
|
||||
|
||||
async fn committed_keys<S: ScannerObjectIO>(
|
||||
store: Arc<S>,
|
||||
name: &str,
|
||||
manifest: &Manifest,
|
||||
candidate: &Archive,
|
||||
) -> StorageResult<HashSet<Digest>> {
|
||||
let mut keys = vec![manifest.root];
|
||||
let mut known = HashSet::from([manifest.root]);
|
||||
for _ in 0..manifest.level {
|
||||
let parents = std::mem::take(&mut keys);
|
||||
for batch in parents.chunks(IO_CONCURRENCY) {
|
||||
let mut reads = Vec::new();
|
||||
let mut nodes = Vec::with_capacity(batch.len());
|
||||
for &key in batch {
|
||||
// The committed parent authenticates the table digest.
|
||||
// Unchanged tables are already available in the candidate.
|
||||
if let Some(node) = candidate.blocks.get(&key) {
|
||||
nodes.push(node.clone());
|
||||
} else {
|
||||
reads.push(read_block(store.clone(), name, manifest.bundle, key));
|
||||
}
|
||||
}
|
||||
nodes.extend(try_join_all(reads).await?);
|
||||
for node in nodes {
|
||||
if node.is_empty() || node.len() > FANOUT * 32 || node.len() % 32 != 0 {
|
||||
return Err(invalid("invalid scanner checkpoint page table"));
|
||||
}
|
||||
for &key in node.as_chunks::<32>().0 {
|
||||
keys.push(key);
|
||||
known.insert(key);
|
||||
}
|
||||
if u64::try_from(keys.len()).unwrap_or(u64::MAX) > manifest.length.div_ceil(PAGE_MIN as u64).saturating_add(3) {
|
||||
return Err(invalid("scanner checkpoint contains excess page references"));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(known)
|
||||
}
|
||||
|
||||
pub(super) async fn decode<S: ScannerObjectIO>(store: Arc<S>, load_name: &str, bytes: &[u8]) -> StorageResult<DataUsageCache> {
|
||||
if !is_manifest(bytes) {
|
||||
return DataUsageCache::unmarshal(bytes).map_err(|_| invalid("invalid scanner cache"));
|
||||
}
|
||||
let name = load_name.strip_suffix(".bkp").unwrap_or(load_name);
|
||||
let bucket = bucket_cache_name(name)
|
||||
.and_then(|name| name.split_once('/'))
|
||||
.ok_or_else(|| invalid("misplaced scanner checkpoint"))?
|
||||
.0;
|
||||
let head = read_envelope(bytes, bucket)?;
|
||||
let Some(manifest) = head.header.manifest() else {
|
||||
return DataUsageCache::unmarshal(bytes).map_err(|_| invalid("invalid inline scanner checkpoint"));
|
||||
};
|
||||
let cache = restore::restore(store, load_name, manifest).await?;
|
||||
if cache.info.name != bucket
|
||||
|| cache.info.snapshot_complete != manifest.finished
|
||||
|| cache.info.next_cycle != manifest.cycle
|
||||
|| cache.info.scan_identity != head.info.scan_identity
|
||||
|| cache.info.source != head.info.source
|
||||
|| cache.info.leader_epoch != head.info.leader_epoch
|
||||
|| cache.info.next_cycle != head.info.next_cycle
|
||||
{
|
||||
return Err(invalid("scanner checkpoint scope changed"));
|
||||
}
|
||||
Ok(cache)
|
||||
}
|
||||
|
||||
async fn previous_header<S: ScannerObjectIO>(
|
||||
store: Arc<S>,
|
||||
name: &str,
|
||||
revision: &DataUsageCacheRevision,
|
||||
) -> StorageResult<Option<Header>> {
|
||||
if *revision == DataUsageCacheRevision::Missing {
|
||||
return Ok(None);
|
||||
}
|
||||
let path = path_join_buf(&[BUCKET_META_PREFIX, name]);
|
||||
let mut reader = store
|
||||
.get_object_reader(
|
||||
RUSTFS_META_BUCKET,
|
||||
&path,
|
||||
None,
|
||||
HeaderMap::new(),
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
if reader
|
||||
.object_info
|
||||
.etag
|
||||
.as_ref()
|
||||
.map(|etag| DataUsageCacheRevision::Etag(etag.clone()))
|
||||
.as_ref()
|
||||
!= Some(revision)
|
||||
{
|
||||
return Err(StorageError::PreconditionFailed);
|
||||
}
|
||||
// Header is the first value in the envelope. A large compatibility
|
||||
// projection or legacy cache does not require reading its body here.
|
||||
let mut prefix = Vec::new();
|
||||
(&mut reader).take(1024).read_to_end(&mut prefix).await?;
|
||||
if !is_manifest(&prefix) {
|
||||
return Ok(None);
|
||||
}
|
||||
let mut deserializer = rmp_serde::Deserializer::new(&prefix[PREFIX.len()..]);
|
||||
// A damaged old header cannot participate in reuse or reclamation.
|
||||
// Rebuilding uses the CAS revision obtained by the normal loader.
|
||||
Ok(Header::deserialize(&mut deserializer).ok().filter(Header::is_valid))
|
||||
}
|
||||
|
||||
fn build_bundle(name: &str, revision: &DataUsageCacheRevision, cache: &DataUsageCache) -> uuid::Uuid {
|
||||
// Before the first root CAS, retries and competing builders populate the
|
||||
// same staging namespace. Individual page names are immutable digests.
|
||||
let mut hash = Sha256::new();
|
||||
hash.update(b"rustfs-scanner-checkpoint-v1\0");
|
||||
hash.update(name.as_bytes());
|
||||
if let Some(identity) = cache.info.scan_identity {
|
||||
hash.update(identity.bucket_incarnation.as_bytes());
|
||||
}
|
||||
if let DataUsageCacheRevision::Etag(etag) = revision {
|
||||
hash.update(etag.as_bytes());
|
||||
}
|
||||
let mut id = [0; 16];
|
||||
id.copy_from_slice(&hash.finalize()[..16]);
|
||||
uuid::Uuid::from_bytes(id)
|
||||
}
|
||||
|
||||
pub(super) fn eligible(cache: &DataUsageCache, name: &str) -> bool {
|
||||
bucket_cache_name(name).is_some()
|
||||
&& cache.info.scan_identity.is_some()
|
||||
&& cache.info.name == name.split('/').next().unwrap_or("")
|
||||
}
|
||||
|
||||
pub(super) async fn prepare<S: ScannerObjectIO + ScannerConfigObjectDelete>(
|
||||
cache: &DataUsageCache,
|
||||
store: Arc<S>,
|
||||
name: &str,
|
||||
revision: &DataUsageCacheRevision,
|
||||
encoded: &EncodedPages,
|
||||
epoch: u64,
|
||||
damaged: bool,
|
||||
) -> StorageResult<(Bytes, RetiredBundles)> {
|
||||
let previous_header = previous_header(store.clone(), name, revision).await?;
|
||||
let previous = previous_header.as_ref().and_then(Header::manifest);
|
||||
let [mut retired, mut abandoned] = previous_header.as_ref().map_or([None, None], Header::retired_bundles);
|
||||
for pending in [&mut retired, &mut abandoned] {
|
||||
if let Some(id) = *pending
|
||||
&& reclaim(store.clone(), name, id, epoch).await.is_ok()
|
||||
{
|
||||
*pending = None;
|
||||
}
|
||||
}
|
||||
|
||||
// A marker lives beside a new bundle's pages and commits before any of
|
||||
// them. We can rediscover that bundle even if the writer never committed
|
||||
// its manifest or retained its in-memory write receipts.
|
||||
let candidate = build_bundle(name, revision, cache);
|
||||
let marker = format!("{}staging", bundle_prefix(name, candidate));
|
||||
let has_marker = match read_config_revision(store.clone(), &marker).await {
|
||||
Ok(revision) => revision != DataUsageCacheRevision::Missing,
|
||||
Err(error) if unreadable_page(&error) => true,
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
let allow_rotation = retired.is_none() && abandoned.is_none();
|
||||
let inline = encoded.inline.as_ref().filter(|_| previous.is_none() || allow_rotation);
|
||||
let live_bytes = save::block_bytes(encoded.archive.blocks.values());
|
||||
let reuse = match previous {
|
||||
Some(head) if inline.is_none() && !damaged && !(head.finished && head.cycle != cache.info.next_cycle) => {
|
||||
save::reuse(store.clone(), name, head, &encoded.archive, live_bytes).await?
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
if (inline.is_some() || reuse.is_some()) && has_marker && previous.is_none_or(|head| head.bundle != candidate) {
|
||||
if abandoned.is_some_and(|pending| pending != candidate) {
|
||||
return Err(Error::other(
|
||||
"cannot abandon scanner checkpoint before reclaiming its previous staging bundle",
|
||||
));
|
||||
}
|
||||
abandoned = Some(candidate);
|
||||
}
|
||||
|
||||
if let Some(inline) = inline {
|
||||
retired = retired.or_else(|| previous.map(|head| head.bundle));
|
||||
let header = Header {
|
||||
body: Payload::Inline,
|
||||
retired,
|
||||
abandoned,
|
||||
};
|
||||
let cleanup = header.retired_bundles();
|
||||
if cleanup == [None, None] {
|
||||
return Ok((inline.clone(), cleanup));
|
||||
}
|
||||
let bytes = rmp_serde::to_vec_named(&CompatibilityEnvelope {
|
||||
header: &header,
|
||||
info: &cache.info,
|
||||
cache: &cache.cache,
|
||||
})
|
||||
.map_err(|_| invalid("failed to encode inline scanner checkpoint"))?;
|
||||
return Ok((Bytes::from(bytes), cleanup));
|
||||
}
|
||||
|
||||
let verify_backup = name.ends_with(".bkp");
|
||||
let (bundle, known, bundle_bytes, bundle_commits) = if let Some((head, known, total)) = reuse {
|
||||
(head.bundle, known, total, head.bundle_commits + 1)
|
||||
} else {
|
||||
if !allow_rotation {
|
||||
// Keep the readable generations and their cleanup obligations.
|
||||
return Err(Error::other("cannot rotate scanner checkpoint before reclaiming its retired bundle"));
|
||||
}
|
||||
retired = previous.map(|head| head.bundle);
|
||||
(candidate, HashSet::new(), live_bytes, 1)
|
||||
};
|
||||
let reserve_candidate = || async {
|
||||
if !has_marker {
|
||||
DataUsageCache::save_path_with_retry(
|
||||
store.clone(),
|
||||
&marker,
|
||||
&Bytes::from_static(b"scanner checkpoint staging"),
|
||||
DataUsageCache::cache_save_timeout(),
|
||||
DATA_USAGE_CACHE_SAVE_RETRIES,
|
||||
Some(DataUsageCacheRevision::Missing),
|
||||
Some(epoch),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
Ok::<(), StorageError>(())
|
||||
};
|
||||
if bundle == candidate {
|
||||
reserve_candidate().await?;
|
||||
}
|
||||
let mut manifest = encoded.root.clone();
|
||||
manifest.bundle = bundle;
|
||||
manifest.finished = cache.info.snapshot_complete;
|
||||
manifest.cycle = cache.info.next_cycle;
|
||||
manifest.bundle_bytes = bundle_bytes;
|
||||
manifest.bundle_commits = bundle_commits;
|
||||
|
||||
if let Err(error) = save::stage(store.clone(), name, revision, epoch, &manifest, &encoded.archive, &known).await {
|
||||
// A corrupt XL header need not yield an ETag for a conditional repair.
|
||||
// Rebuild this backup under a fresh namespace; its old root stays visible
|
||||
// until the manifest CAS. Other I/O failures remain retryable in place.
|
||||
if !(verify_backup && manifest.bundle_commits > 1 && allow_rotation && unreadable_page(&error)) {
|
||||
return Err(error);
|
||||
}
|
||||
retired = Some(manifest.bundle);
|
||||
abandoned = None;
|
||||
manifest.bundle = candidate;
|
||||
reserve_candidate().await?;
|
||||
manifest.bundle_bytes = live_bytes;
|
||||
manifest.bundle_commits = 1;
|
||||
save::stage(store.clone(), name, revision, epoch, &manifest, &encoded.archive, &HashSet::new()).await?;
|
||||
}
|
||||
let header = Header {
|
||||
body: Payload::Pages(manifest),
|
||||
retired,
|
||||
abandoned,
|
||||
};
|
||||
let cleanup = header.retired_bundles();
|
||||
let projection = |info: &DataUsageCacheInfo, cache: &BTreeMap<String, DataUsageEntry>| {
|
||||
rmp_serde::to_vec_named(&CompatibilityEnvelope {
|
||||
header: &header,
|
||||
info,
|
||||
cache,
|
||||
})
|
||||
.map_err(|_| invalid("failed to encode scanner checkpoint root"))
|
||||
};
|
||||
if cache.info.snapshot_complete {
|
||||
// Older readers can serve compact totals when the compatibility
|
||||
// projection fits. The manifest remains bounded even with many
|
||||
// pending repairs or a large lifecycle configuration.
|
||||
let mut root = cache
|
||||
.checked_flatten_complete_scope(&cache.info.name)
|
||||
.ok_or_else(|| invalid("disconnected complete bucket checkpoint"))?;
|
||||
root.compacted = true;
|
||||
let bytes = projection(&cache.info, &BTreeMap::from([(cache.info.name.clone(), root)]))?;
|
||||
if bytes.len() <= PAGE_MIN {
|
||||
return Ok((Bytes::from(bytes), cleanup));
|
||||
}
|
||||
}
|
||||
let info = DataUsageCacheInfo {
|
||||
name: cache.info.name.clone(),
|
||||
source: cache.info.source,
|
||||
scan_identity: cache.info.scan_identity,
|
||||
leader_epoch: cache.info.leader_epoch,
|
||||
next_cycle: cache.info.next_cycle,
|
||||
..Default::default()
|
||||
};
|
||||
let bytes = projection(&info, &BTreeMap::new())?;
|
||||
Ok((Bytes::from(bytes), cleanup))
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct CompatibilityEnvelope<'a, T> {
|
||||
#[serde(rename = "pagesV1")]
|
||||
header: &'a Header,
|
||||
info: &'a DataUsageCacheInfo,
|
||||
cache: &'a T,
|
||||
}
|
||||
|
||||
pub(super) async fn reclaim_retired<S: ScannerObjectIO + ScannerConfigObjectDelete>(
|
||||
store: Arc<S>,
|
||||
name: &str,
|
||||
bundles: RetiredBundles,
|
||||
epoch: Option<u64>,
|
||||
) {
|
||||
if let Some(epoch) = epoch {
|
||||
for bundle in bundles.into_iter().flatten() {
|
||||
let _ = reclaim(store.clone(), name, bundle, epoch).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn reclaim<S: ScannerObjectIO + ScannerConfigObjectDelete>(
|
||||
store: Arc<S>,
|
||||
name: &str,
|
||||
bundle: uuid::Uuid,
|
||||
epoch: u64,
|
||||
) -> StorageResult<()> {
|
||||
let _admission = scanner_publication_admission_for_epoch(store.clone(), epoch)
|
||||
.await
|
||||
.ok_or_else(cache_publication_epoch_changed)?;
|
||||
let prefix = bundle_prefix(name, bundle);
|
||||
let opts = ObjectOptions {
|
||||
delete_prefix: true,
|
||||
..Default::default()
|
||||
};
|
||||
match timeout(
|
||||
DataUsageCache::cache_save_timeout(),
|
||||
store.delete_config_object(RUSTFS_META_BUCKET, &prefix, opts),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(Ok(_)) | Ok(Err(StorageError::FileNotFound | StorageError::ObjectNotFound(..))) => Ok(()),
|
||||
Ok(Err(error)) => Err(error),
|
||||
Err(_) => Err(Error::other("retired scanner checkpoint cleanup timed out")),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,207 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use super::super::scan_load::park_restore;
|
||||
use super::*;
|
||||
use std::any::Any;
|
||||
use std::fs::File;
|
||||
use std::io::{Seek, SeekFrom};
|
||||
use std::sync::{Mutex, Weak};
|
||||
use tokio::time::Instant as WallInstant;
|
||||
|
||||
const MEMORY_LIMIT: u64 = 1024 * 1024;
|
||||
const PARKED_LIMIT: usize = 32;
|
||||
|
||||
enum Buffer {
|
||||
Memory(Vec<u8>),
|
||||
File(Arc<Mutex<File>>),
|
||||
}
|
||||
|
||||
struct Position {
|
||||
pending: Vec<(Digest, u8)>,
|
||||
length: u64,
|
||||
leaves: u64,
|
||||
nodes: u64,
|
||||
buffer: Buffer,
|
||||
writer: Option<tokio::task::JoinHandle<std::io::Result<()>>>,
|
||||
decoder: Option<tokio::task::JoinHandle<StorageResult<DataUsageCache>>>,
|
||||
}
|
||||
|
||||
struct Suspended {
|
||||
owner: Weak<dyn Any + Send + Sync>,
|
||||
name: String,
|
||||
manifest: Manifest,
|
||||
last_polled: WallInstant,
|
||||
position: Position,
|
||||
}
|
||||
|
||||
struct Recovery(Option<Suspended>);
|
||||
|
||||
static PARKED: LazyLock<Mutex<Vec<Suspended>>> = LazyLock::new(|| Mutex::new(Vec::new()));
|
||||
|
||||
impl Recovery {
|
||||
async fn open<S: ScannerObjectIO>(store: &Arc<S>, name: &str, manifest: &Manifest) -> StorageResult<Self> {
|
||||
let erased: Arc<dyn Any + Send + Sync> = store.clone();
|
||||
let owner = Arc::downgrade(&erased);
|
||||
let resumed = {
|
||||
let mut parked = PARKED.lock().unwrap_or_else(|error| error.into_inner());
|
||||
parked.retain(|job| {
|
||||
job.owner.strong_count() != 0 && (!job.owner.ptr_eq(&owner) || job.name != name || job.manifest == *manifest)
|
||||
});
|
||||
parked
|
||||
.iter()
|
||||
.position(|job| job.owner.ptr_eq(&owner) && job.name == name && job.manifest == *manifest)
|
||||
.map(|index| parked.swap_remove(index))
|
||||
};
|
||||
if let Some(mut job) = resumed {
|
||||
job.last_polled = WallInstant::now();
|
||||
return Ok(Self(Some(job)));
|
||||
}
|
||||
let buffer = if manifest.length <= MEMORY_LIMIT {
|
||||
Buffer::Memory(Vec::new())
|
||||
} else {
|
||||
let file = tokio::task::spawn_blocking(tempfile::tempfile)
|
||||
.await
|
||||
.map_err(Error::other)??;
|
||||
Buffer::File(Arc::new(Mutex::new(file)))
|
||||
};
|
||||
Ok(Self(Some(Suspended {
|
||||
owner,
|
||||
name: name.to_owned(),
|
||||
manifest: manifest.clone(),
|
||||
last_polled: WallInstant::now(),
|
||||
position: Position {
|
||||
pending: vec![(manifest.root, manifest.level)],
|
||||
length: 0,
|
||||
leaves: 0,
|
||||
nodes: 0,
|
||||
buffer,
|
||||
writer: None,
|
||||
decoder: None,
|
||||
},
|
||||
})))
|
||||
}
|
||||
}
|
||||
|
||||
// Park only verified bytes and traversal state. Outstanding metadata reads
|
||||
// and their admissions are dropped with the waiter. A local spool write or
|
||||
// decode may finish after cancellation and is joined before advancing again.
|
||||
impl Drop for Recovery {
|
||||
fn drop(&mut self) {
|
||||
let Some(mut job) = self.0.take() else { return };
|
||||
if job.position.nodes == 0 && job.position.leaves == 0 {
|
||||
return;
|
||||
}
|
||||
let mut parked = PARKED.lock().unwrap_or_else(|error| error.into_inner());
|
||||
job.last_polled = WallInstant::now();
|
||||
if let Some(previous) = parked
|
||||
.iter_mut()
|
||||
.find(|previous| previous.owner.ptr_eq(&job.owner) && previous.name == job.name && previous.manifest == job.manifest)
|
||||
{
|
||||
// Readers of an immutable root traverse the same ordered prefix.
|
||||
// Keep the furthest admission regardless of cancellation order.
|
||||
if job.position.verified_progress() > previous.position.verified_progress() {
|
||||
std::mem::swap(previous, &mut job);
|
||||
}
|
||||
} else {
|
||||
let displaced = park_restore(&mut parked, PARKED_LIMIT, job, |job| job.last_polled);
|
||||
drop(parked);
|
||||
drop(displaced);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Position {
|
||||
fn verified_progress(&self) -> (u64, bool) {
|
||||
(self.nodes + self.leaves, self.decoder.is_some())
|
||||
}
|
||||
|
||||
async fn restore<S: ScannerObjectIO>(
|
||||
&mut self,
|
||||
store: Arc<S>,
|
||||
name: &str,
|
||||
manifest: &Manifest,
|
||||
) -> StorageResult<DataUsageCache> {
|
||||
let max_leaves = manifest.length.div_ceil(PAGE_MIN as u64).saturating_add(3);
|
||||
loop {
|
||||
// Retain the local write across cancellation. The verified page already
|
||||
// owns its offset and must not consume another storage read next round.
|
||||
if let Some(writer) = &mut self.writer {
|
||||
writer.await.map_err(Error::other)??;
|
||||
self.writer = None;
|
||||
}
|
||||
let Some(&(key, level)) = self.pending.last() else { break };
|
||||
if level != 0 {
|
||||
let node = read_block(store.clone(), name, manifest.bundle, key).await?;
|
||||
if node.is_empty()
|
||||
|| node.len() > FANOUT * 32
|
||||
|| node.len() % 32 != 0
|
||||
|| self.nodes >= max_leaves.saturating_mul(u64::from(manifest.level))
|
||||
{
|
||||
return Err(invalid("invalid scanner checkpoint restore table"));
|
||||
}
|
||||
self.pending.pop();
|
||||
for &child in node.as_chunks::<32>().0.iter().rev() {
|
||||
self.pending.push((child, level - 1));
|
||||
}
|
||||
self.nodes += 1;
|
||||
continue;
|
||||
}
|
||||
let block = read_block(store.clone(), name, manifest.bundle, key).await?;
|
||||
let next = self
|
||||
.length
|
||||
.checked_add(u64::try_from(block.len()).map_err(|_| invalid("invalid restore length"))?)
|
||||
.filter(|len| *len <= manifest.length)
|
||||
.ok_or_else(|| invalid("scanner checkpoint exceeds its committed length"))?;
|
||||
if self.leaves >= max_leaves {
|
||||
return Err(invalid("too many scanner checkpoint leaves"));
|
||||
}
|
||||
match &mut self.buffer {
|
||||
Buffer::Memory(bytes) => bytes.extend_from_slice(&block),
|
||||
Buffer::File(file) => {
|
||||
let file = file.clone();
|
||||
let offset = self.length;
|
||||
self.writer = Some(tokio::task::spawn_blocking(move || {
|
||||
let mut file = file.lock().unwrap_or_else(|error| error.into_inner());
|
||||
file.seek(SeekFrom::Start(offset))?;
|
||||
file.write_all(&block)
|
||||
}));
|
||||
}
|
||||
}
|
||||
self.pending.pop();
|
||||
self.length = next;
|
||||
self.leaves += 1;
|
||||
}
|
||||
if self.length != manifest.length {
|
||||
return Err(invalid("incomplete scanner checkpoint"));
|
||||
}
|
||||
let job = self.decoder.get_or_insert_with(|| {
|
||||
let buffer = std::mem::replace(&mut self.buffer, Buffer::Memory(Vec::new()));
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let result = match buffer {
|
||||
Buffer::Memory(bytes) => rmp_serde::from_slice(&bytes),
|
||||
Buffer::File(file) => {
|
||||
let mut file = file.lock().unwrap_or_else(|error| error.into_inner());
|
||||
file.seek(SeekFrom::Start(0))?;
|
||||
rmp_serde::from_read(std::io::BufReader::new(&mut *file))
|
||||
}
|
||||
};
|
||||
result.map_err(|_| invalid("invalid scanner checkpoint payload"))
|
||||
})
|
||||
});
|
||||
job.await.map_err(Error::other)?
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn restore<S: ScannerObjectIO>(store: Arc<S>, name: &str, manifest: &Manifest) -> StorageResult<DataUsageCache> {
|
||||
let mut job = Recovery::open(&store, name, manifest).await?;
|
||||
let result = job
|
||||
.0
|
||||
.as_mut()
|
||||
.ok_or_else(|| invalid("missing checkpoint restore state"))?
|
||||
.position
|
||||
.restore(store, name, manifest)
|
||||
.await;
|
||||
job.0.take();
|
||||
result
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use super::*;
|
||||
|
||||
// A losing writer can stage blocks without publishing its accounting. Even
|
||||
// tiny or identical successful checkpoints periodically retire these orphans.
|
||||
const MAX_BUNDLE_COMMITS: u16 = 256;
|
||||
|
||||
pub(super) fn block_bytes<'a>(blocks: impl Iterator<Item = &'a Bytes>) -> u64 {
|
||||
blocks.fold(0u64, |sum, bytes| sum.saturating_add(u64::try_from(bytes.len()).unwrap_or(u64::MAX)))
|
||||
}
|
||||
|
||||
pub(super) async fn reuse<'a, S: ScannerObjectIO>(
|
||||
store: Arc<S>,
|
||||
name: &str,
|
||||
head: &'a Manifest,
|
||||
candidate: &Archive,
|
||||
live: u64,
|
||||
) -> StorageResult<Option<(&'a Manifest, HashSet<Digest>, u64)>> {
|
||||
// Legacy manifests did not account for unreachable blocks in this bundle.
|
||||
if head.bundle_bytes == 0 || head.bundle_commits == 0 || head.bundle_commits >= MAX_BUNDLE_COMMITS {
|
||||
return Ok(None);
|
||||
}
|
||||
let known = match committed_keys(store, name, head, candidate).await {
|
||||
Ok(known) => known,
|
||||
Err(StorageError::FileNotFound | StorageError::ObjectNotFound(..)) => return Ok(None),
|
||||
Err(error) if unreadable_page(&error) => return Ok(None),
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
// Count reachable allocation once at bundle creation and newly staged
|
||||
// bytes thereafter. A small delta stays incremental even across cycles.
|
||||
let added = block_bytes(
|
||||
candidate
|
||||
.blocks
|
||||
.iter()
|
||||
.filter_map(|(key, bytes)| (!known.contains(key)).then_some(bytes)),
|
||||
);
|
||||
let total = head.bundle_bytes.checked_add(added);
|
||||
Ok(total
|
||||
.filter(|&total| total <= live.saturating_mul(2))
|
||||
.map(|total| (head, known, total)))
|
||||
}
|
||||
|
||||
/// Returns the conditional write necessary to restore this content-addressed
|
||||
/// backup page, or None when the bytes in storage already match the candidate.
|
||||
async fn backup_revision<S: ScannerObjectIO>(
|
||||
store: &S,
|
||||
path: &str,
|
||||
expected: &Bytes,
|
||||
) -> StorageResult<Option<DataUsageCacheRevision>> {
|
||||
let mut reader = match store
|
||||
.get_object_reader(
|
||||
RUSTFS_META_BUCKET,
|
||||
path,
|
||||
None,
|
||||
HeaderMap::new(),
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(reader) => reader,
|
||||
Err(StorageError::FileNotFound | StorageError::ObjectNotFound(..)) => return Ok(Some(DataUsageCacheRevision::Missing)),
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
if usize::try_from(reader.object_info.size).ok() == Some(expected.len()) {
|
||||
let mut bytes = Vec::with_capacity(expected.len());
|
||||
(&mut reader).take(PAGE_MAX as u64 + 1).read_to_end(&mut bytes).await?;
|
||||
if bytes == expected.as_ref() {
|
||||
return Ok(None);
|
||||
}
|
||||
}
|
||||
let etag = reader
|
||||
.object_info
|
||||
.etag
|
||||
.ok_or_else(|| invalid("damaged scanner backup page has no revision"))?;
|
||||
Ok(Some(DataUsageCacheRevision::Etag(etag)))
|
||||
}
|
||||
|
||||
pub(super) async fn stage<S: ScannerObjectIO + ScannerConfigObjectDelete>(
|
||||
store: Arc<S>,
|
||||
name: &str,
|
||||
revision: &DataUsageCacheRevision,
|
||||
epoch: u64,
|
||||
manifest: &Manifest,
|
||||
archive: &Archive,
|
||||
known: &HashSet<Digest>,
|
||||
) -> StorageResult<()> {
|
||||
let verify_backup = name.ends_with(".bkp");
|
||||
let staging = staging::Staging::open(&store, name, manifest.bundle, epoch, revision);
|
||||
let mut writes = Vec::with_capacity(IO_CONCURRENCY);
|
||||
for (key, bytes) in &archive.blocks {
|
||||
let staged = staging.contains(key);
|
||||
if !verify_backup && (known.contains(key) || staged) {
|
||||
continue;
|
||||
}
|
||||
let path = block_path(name, manifest.bundle, key);
|
||||
let store = store.clone();
|
||||
let bytes = bytes.clone();
|
||||
let key = *key;
|
||||
let receipts = &staging;
|
||||
writes.push(async move {
|
||||
let needed = if verify_backup && (manifest.bundle_commits > 1 || staged) {
|
||||
// A healthy primary load does not establish the integrity of its
|
||||
// independent backup. Check reused bytes and repair only damaged pages.
|
||||
match timeout(DataUsageCache::cache_save_timeout(), backup_revision(store.as_ref(), &path, &bytes))
|
||||
.await
|
||||
.map_err(|_| Error::other("scanner backup page validation timed out"))??
|
||||
{
|
||||
Some(revision) => revision,
|
||||
None => return Ok(()),
|
||||
}
|
||||
} else {
|
||||
DataUsageCacheRevision::Missing
|
||||
};
|
||||
DataUsageCache::save_path_with_retry(
|
||||
store,
|
||||
&path,
|
||||
&bytes,
|
||||
DataUsageCache::cache_save_timeout(),
|
||||
DATA_USAGE_CACHE_SAVE_RETRIES,
|
||||
Some(needed),
|
||||
Some(epoch),
|
||||
)
|
||||
.await?;
|
||||
receipts.record(key);
|
||||
Ok::<(), Error>(())
|
||||
});
|
||||
if writes.len() == IO_CONCURRENCY {
|
||||
try_join_all(std::mem::take(&mut writes)).await?;
|
||||
}
|
||||
}
|
||||
try_join_all(writes).await?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use super::*;
|
||||
use std::any::Any;
|
||||
use std::sync::{Mutex, Weak};
|
||||
use std::time::Instant as WallInstant;
|
||||
|
||||
/// A manifest commit can be interrupted after any group of immutable PUTs.
|
||||
/// Reuse only receipts from the same store, epoch and predecessor revision.
|
||||
struct Attempt {
|
||||
store: Weak<dyn Any + Send + Sync>,
|
||||
path: String,
|
||||
bundle: uuid::Uuid,
|
||||
epoch: u64,
|
||||
revision: DataUsageCacheRevision,
|
||||
seen: Arc<Mutex<HashSet<Digest>>>,
|
||||
last_used: WallInstant,
|
||||
}
|
||||
|
||||
const MAX_ATTEMPTS: usize = 32;
|
||||
static ATTEMPTS: LazyLock<Mutex<Vec<Attempt>>> = LazyLock::new(|| Mutex::new(Vec::new()));
|
||||
|
||||
pub(super) struct Staging(Arc<Mutex<HashSet<Digest>>>);
|
||||
|
||||
impl Staging {
|
||||
pub(super) fn open<S: ScannerObjectIO>(
|
||||
store: &Arc<S>,
|
||||
path: &str,
|
||||
bundle: uuid::Uuid,
|
||||
epoch: u64,
|
||||
revision: &DataUsageCacheRevision,
|
||||
) -> Self {
|
||||
let store: Arc<dyn Any + Send + Sync> = store.clone();
|
||||
let owner = Arc::downgrade(&store);
|
||||
let mut attempts = ATTEMPTS.lock().unwrap_or_else(|p| p.into_inner());
|
||||
attempts.retain(|job| job.store.strong_count() != 0 && job.last_used.elapsed() < Duration::from_secs(3600));
|
||||
if let Some(attempt) = attempts.iter_mut().find(|attempt| {
|
||||
attempt.store.ptr_eq(&owner)
|
||||
&& attempt.path == path
|
||||
&& attempt.bundle == bundle
|
||||
&& attempt.epoch == epoch
|
||||
&& &attempt.revision == revision
|
||||
}) {
|
||||
attempt.last_used = WallInstant::now();
|
||||
return Self(attempt.seen.clone());
|
||||
}
|
||||
// A successfully committed predecessor naturally retires the receipts
|
||||
// for its previous ETag. Concurrent older writers retain their own Arc.
|
||||
attempts.retain(|attempt| !attempt.store.ptr_eq(&owner) || attempt.path != path);
|
||||
if attempts.len() >= MAX_ATTEMPTS
|
||||
&& let Some((index, _)) = attempts.iter().enumerate().min_by_key(|(_, job)| job.last_used)
|
||||
{
|
||||
attempts.swap_remove(index);
|
||||
}
|
||||
let seen = Arc::new(Mutex::new(HashSet::new()));
|
||||
attempts.push(Attempt {
|
||||
store: owner,
|
||||
path: path.to_owned(),
|
||||
bundle,
|
||||
epoch,
|
||||
revision: revision.clone(),
|
||||
seen: seen.clone(),
|
||||
last_used: WallInstant::now(),
|
||||
});
|
||||
Self(seen)
|
||||
}
|
||||
|
||||
pub(super) fn contains(&self, key: &Digest) -> bool {
|
||||
self.0.lock().unwrap_or_else(|p| p.into_inner()).contains(key)
|
||||
}
|
||||
|
||||
pub(super) fn record(&self, key: Digest) {
|
||||
self.0.lock().unwrap_or_else(|p| p.into_inner()).insert(key);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,208 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use super::*;
|
||||
use crate::scanner_budget::ScannerCycleBudget;
|
||||
use std::any::Any;
|
||||
use std::sync::{Mutex, Weak};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
// Keep only a failed root's identity while its independent backup is being
|
||||
// recovered. A fresh primary reader still obtains the current ETag every time.
|
||||
pub(super) struct PendingBackup {
|
||||
owner: Weak<dyn Any + Send + Sync>,
|
||||
name: String,
|
||||
failed_etag: String,
|
||||
last_polled: tokio::time::Instant,
|
||||
}
|
||||
|
||||
static PENDING_BACKUPS: Mutex<Vec<PendingBackup>> = Mutex::new(Vec::new());
|
||||
const MAX_PENDING_BACKUPS: usize = 32;
|
||||
|
||||
// Age permits reclamation under contention. A scanner whose configured
|
||||
// interval exceeds this age can still resume while slots remain available.
|
||||
pub(super) fn park_restore<T>(
|
||||
slots: &mut Vec<T>,
|
||||
limit: usize,
|
||||
value: T,
|
||||
last_polled: impl Fn(&T) -> tokio::time::Instant,
|
||||
) -> Option<T> {
|
||||
if slots.len() < limit {
|
||||
slots.push(value);
|
||||
return None;
|
||||
}
|
||||
if let Some(oldest) = slots.iter_mut().min_by_key(|slot| last_polled(slot))
|
||||
&& last_polled(oldest).elapsed() >= Duration::from_secs(3600)
|
||||
{
|
||||
return Some(std::mem::replace(oldest, value));
|
||||
}
|
||||
Some(value)
|
||||
}
|
||||
|
||||
impl PendingBackup {
|
||||
fn owner<S: ScannerObjectIO>(store: &Arc<S>) -> Weak<dyn Any + Send + Sync> {
|
||||
let erased: Arc<dyn Any + Send + Sync> = store.clone();
|
||||
Arc::downgrade(&erased)
|
||||
}
|
||||
|
||||
pub(super) fn needs_backup<S: ScannerObjectIO>(store: &Arc<S>, name: &str, etag: &str) -> bool {
|
||||
let owner = Self::owner(store);
|
||||
let mut pending = PENDING_BACKUPS.lock().unwrap_or_else(|error| error.into_inner());
|
||||
let Some(index) = pending
|
||||
.iter()
|
||||
.position(|phase| phase.owner.ptr_eq(&owner) && phase.name == name)
|
||||
else {
|
||||
return false;
|
||||
};
|
||||
if pending[index].failed_etag == etag {
|
||||
pending[index].last_polled = tokio::time::Instant::now();
|
||||
true
|
||||
} else {
|
||||
pending.swap_remove(index);
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn begin<S: ScannerObjectIO>(store: &Arc<S>, name: &str, etag: &str) {
|
||||
let owner = Self::owner(store);
|
||||
let mut pending = PENDING_BACKUPS.lock().unwrap_or_else(|error| error.into_inner());
|
||||
pending.retain(|phase| phase.owner.strong_count() > 0 && !(phase.owner.ptr_eq(&owner) && phase.name == name));
|
||||
let phase = Self {
|
||||
owner,
|
||||
name: name.to_owned(),
|
||||
failed_etag: etag.to_owned(),
|
||||
last_polled: tokio::time::Instant::now(),
|
||||
};
|
||||
let _ = park_restore(&mut pending, MAX_PENDING_BACKUPS, phase, |phase| phase.last_polled);
|
||||
}
|
||||
|
||||
pub(super) fn finish<S: ScannerObjectIO>(store: &Arc<S>, name: &str, etag: &str) {
|
||||
let owner = Self::owner(store);
|
||||
PENDING_BACKUPS
|
||||
.lock()
|
||||
.unwrap_or_else(|error| error.into_inner())
|
||||
.retain(|phase| !(phase.owner.ptr_eq(&owner) && phase.name == name && phase.failed_etag == etag));
|
||||
}
|
||||
}
|
||||
|
||||
struct DecodedCheckpoint {
|
||||
owner: Weak<dyn Any + Send + Sync>,
|
||||
name: String,
|
||||
parked_at: std::time::Instant,
|
||||
cache: DataUsageCache,
|
||||
revisions: DataUsageCacheRevisions,
|
||||
}
|
||||
|
||||
// The scanner may finish a cold restore with too little time for the walk.
|
||||
// Move that exact value to the next admission. Only one decoded bucket can
|
||||
// occupy the handoff and it never owns storage, a permit, or a publication fence.
|
||||
static DECODED: Mutex<Option<DecodedCheckpoint>> = Mutex::new(None);
|
||||
|
||||
impl DataUsageCache {
|
||||
pub(crate) async fn load_for_scan<S: ScannerObjectIO>(
|
||||
&mut self,
|
||||
store: Arc<S>,
|
||||
name: &str,
|
||||
ctx: &CancellationToken,
|
||||
budget: &ScannerCycleBudget,
|
||||
) -> StorageResult<Option<DataUsageCacheRevisions>> {
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = ctx.cancelled() => Ok(None),
|
||||
result = crate::storage_api::ecstore_with_background_disk_io(
|
||||
ctx.clone(),
|
||||
self.load_for_scan_window(store, name, budget),
|
||||
) => result,
|
||||
}
|
||||
}
|
||||
|
||||
async fn load_for_scan_window<S: ScannerObjectIO>(
|
||||
&mut self,
|
||||
store: Arc<S>,
|
||||
name: &str,
|
||||
budget: &ScannerCycleBudget,
|
||||
) -> StorageResult<Option<DataUsageCacheRevisions>> {
|
||||
let erased: Arc<dyn Any + Send + Sync> = store.clone();
|
||||
let owner = Arc::downgrade(&erased);
|
||||
let is_ours = |ready: &DecodedCheckpoint| ready.owner.ptr_eq(&owner) && ready.name == name;
|
||||
let cached = {
|
||||
let mut decoded = DECODED.lock().unwrap_or_else(|error| error.into_inner());
|
||||
// Give a retired or deleted bucket's slot back to other active
|
||||
// readers. A matching admission can consume it even after a long
|
||||
// scheduled interval, subject to the revision checks below.
|
||||
decoded.take_if(|ready| {
|
||||
ready.owner.strong_count() == 0 || (!is_ours(ready) && ready.parked_at.elapsed() >= Duration::from_secs(3600))
|
||||
});
|
||||
decoded.as_ref().is_some_and(is_ours)
|
||||
};
|
||||
if cached {
|
||||
let path = path_join_buf(&[BUCKET_META_PREFIX, name]);
|
||||
let main = read_config_revision(store.clone(), &path).await;
|
||||
let backup = Self::backup_revision(store.clone(), name).await;
|
||||
let ready = DECODED
|
||||
.lock()
|
||||
.unwrap_or_else(|error| error.into_inner())
|
||||
.take_if(|ready| is_ours(ready));
|
||||
if let (Some(ready), Ok(main)) = (ready, main)
|
||||
&& ready.revisions.main == main
|
||||
&& ready.revisions.backup == backup
|
||||
{
|
||||
*self = ready.cache;
|
||||
return Ok(Some(ready.revisions));
|
||||
}
|
||||
}
|
||||
|
||||
let started = tokio::time::Instant::now();
|
||||
// Obtain the optional backup head before the expensive decode. A budget
|
||||
// stop after the decoder finishes must not discard the materialized tree
|
||||
// while waiting for an unrelated backup lookup.
|
||||
let backup = Self::backup_revision(store.clone(), name).await;
|
||||
let loaded = Box::pin(Self::load_cache(store, name)).await?;
|
||||
let revisions = DataUsageCacheRevisions {
|
||||
main: loaded.main_revision,
|
||||
backup: loaded.backup_revision.or(backup),
|
||||
damaged_pages: loaded.damaged_pages,
|
||||
};
|
||||
*self = loaded.cache;
|
||||
if self.info.scan_identity.is_some()
|
||||
&& !self.cache.is_empty()
|
||||
&& budget
|
||||
.remaining_config()
|
||||
.max_duration
|
||||
.is_some_and(|remaining| started.elapsed() > remaining)
|
||||
{
|
||||
let mut slot = DECODED.lock().unwrap_or_else(|error| error.into_inner());
|
||||
if slot.as_ref().is_none_or(|ready| ready.owner.strong_count() == 0) {
|
||||
*slot = Some(DecodedCheckpoint {
|
||||
owner,
|
||||
name: name.to_owned(),
|
||||
parked_at: std::time::Instant::now(),
|
||||
cache: std::mem::take(self),
|
||||
revisions,
|
||||
});
|
||||
return Ok(None);
|
||||
}
|
||||
}
|
||||
Ok(Some(revisions))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn only_reclaim_idle_restore_slots_under_pressure() {
|
||||
let first = tokio::time::Instant::now();
|
||||
tokio::time::advance(Duration::from_secs(7200)).await;
|
||||
let second = tokio::time::Instant::now();
|
||||
let mut slots = vec![first];
|
||||
assert_eq!(park_restore(&mut slots, 2, second, |&time| time), None);
|
||||
assert_eq!(slots, [first, second]);
|
||||
let new = second + Duration::from_secs(1);
|
||||
assert_eq!(park_restore(&mut slots, 2, new, |&time| time), Some(first));
|
||||
assert_eq!(slots, [new, second]);
|
||||
assert_eq!(park_restore(&mut slots, 2, new, |&time| time), Some(new));
|
||||
assert_eq!(slots, [new, second], "active recoveries retain both slots");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use super::*;
|
||||
|
||||
/// A linked tree paused on a different physical replica. It contains only
|
||||
/// traversal fields, so dormant sources cannot recursively contain each other.
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
pub struct SourceTree {
|
||||
bucket: String,
|
||||
identity: DataUsageScanIdentity,
|
||||
started_plan: DataUsageScanPlanDigest,
|
||||
#[serde(serialize_with = "ordered_entries")]
|
||||
entries: HashMap<String, DataUsageEntry>,
|
||||
resume_after: Option<String>,
|
||||
checkpoint: Option<DataUsageScanCheckpoint>,
|
||||
coverage: Option<DataUsageScanCoverageReceipt>,
|
||||
cursor: Option<DataUsageRawEnumerationCursor>,
|
||||
index: Option<RawEnumerationPageIndex>,
|
||||
ancestors: Vec<RawEnumerationPageIndex>,
|
||||
}
|
||||
|
||||
impl SourceTree {
|
||||
/// An inactive receipt may preserve an existing source position. Only an
|
||||
/// active, verified traversal can contribute new progress.
|
||||
pub(super) fn frontier(&self, info: &DataUsageCacheInfo) -> Option<&str> {
|
||||
if self.bucket != info.name
|
||||
|| Some(self.identity) != info.scan_identity
|
||||
|| Some(self.started_plan) != info.scan_progress.map(|p| p.started_plan)
|
||||
{
|
||||
return None;
|
||||
}
|
||||
self.coverage.as_ref().map(|receipt| receipt.through.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
// Dormant trees live in the checkpoint header stream. A cold process has
|
||||
// fresh HashMap seeds; preserve stable bytes for immutable Merkle pages.
|
||||
fn ordered_entries<S: serde::Serializer>(
|
||||
entries: &HashMap<String, DataUsageEntry>,
|
||||
serializer: S,
|
||||
) -> std::result::Result<S::Ok, S::Error> {
|
||||
let mut sorted = entries.iter().collect::<Vec<_>>();
|
||||
sorted.sort_unstable_by(|a, b| a.0.cmp(b.0));
|
||||
serializer.collect_map(sorted.into_iter().map(|(name, entry)| (name, entry.ordered_encoding())))
|
||||
}
|
||||
|
||||
impl DataUsageCache {
|
||||
pub(crate) fn activate_directory_source(&mut self, source: String, endpoints: &HashSet<String>) {
|
||||
let (Some(identity), Some(progress)) = (self.info.scan_identity, self.info.scan_progress) else {
|
||||
return;
|
||||
};
|
||||
let previous = self.info.scan_directory_source.replace(source.clone());
|
||||
// Every production endpoint has an erasure-set slot. A disk replacement
|
||||
// cannot retain a backup under the same slot or under a retired endpoint.
|
||||
let current_location = directory_source_location(&source);
|
||||
self.info.scan_directory_work.retain(|work| {
|
||||
work.endpoint() == source
|
||||
|| endpoints.contains(directory_source_location(work.endpoint()))
|
||||
&& directory_source_location(work.endpoint()) != current_location
|
||||
});
|
||||
self.info.scan_source_trees.retain(|key, tree| {
|
||||
tree.bucket == self.info.name
|
||||
&& tree.identity == identity
|
||||
&& tree.started_plan == progress.started_plan
|
||||
&& (key == &source
|
||||
|| endpoints.contains(directory_source_location(key)) && directory_source_location(key) != current_location)
|
||||
});
|
||||
let Some(previous) = previous else {
|
||||
// Pre-source checkpoints identify an erasure set but never recorded
|
||||
// which disk supplied the partial tree. The same keys can have
|
||||
// different metadata on another replica. Keep the certified usage,
|
||||
// repair obligations and observations outside this traversal.
|
||||
if self.has_scan_position() {
|
||||
self.clear_active_directory_traversal();
|
||||
}
|
||||
return;
|
||||
};
|
||||
if previous == source {
|
||||
return;
|
||||
}
|
||||
let entries = std::mem::take(&mut self.cache);
|
||||
let linked = entries.get(&hash_path(&self.info.name).0).is_some_and(|root| !root.compacted);
|
||||
let previous_tree = SourceTree {
|
||||
bucket: self.info.name.clone(),
|
||||
identity,
|
||||
started_plan: progress.started_plan,
|
||||
entries,
|
||||
resume_after: self.info.scan_resume_after.take(),
|
||||
checkpoint: self.info.scan_checkpoint.take(),
|
||||
coverage: self.info.scan_coverage_receipt.take(),
|
||||
cursor: self.info.scan_raw_enumeration_cursor.take(),
|
||||
index: self.info.scan_raw_enumeration_page_index.take(),
|
||||
ancestors: std::mem::take(&mut self.info.scan_raw_enumeration_path_page_indexes),
|
||||
};
|
||||
if linked
|
||||
&& endpoints.contains(directory_source_location(&previous))
|
||||
&& directory_source_location(&previous) != current_location
|
||||
{
|
||||
self.info.scan_source_trees.insert(previous, Arc::new(previous_tree));
|
||||
}
|
||||
if let Some(tree) = self.info.scan_source_trees.remove(&source) {
|
||||
let tree = Arc::unwrap_or_clone(tree);
|
||||
self.cache = tree.entries;
|
||||
self.info.scan_resume_after = tree.resume_after;
|
||||
self.info.scan_coverage_receipt = tree.coverage;
|
||||
self.info.scan_checkpoint = tree.checkpoint;
|
||||
self.info.scan_raw_enumeration_cursor = tree.cursor;
|
||||
self.info.scan_raw_enumeration_page_index = tree.index;
|
||||
self.info.scan_raw_enumeration_path_page_indexes = tree.ancestors;
|
||||
// A dormant tree must provide the same coverage/graph guarantees
|
||||
// required of an active checkpoint before its child links are used.
|
||||
if self.checked_flatten_complete_scope(&self.info.name).is_none()
|
||||
|| self.info.scan_coverage_receipt.is_some() && self.validated_scan_frontier().is_none()
|
||||
{
|
||||
self.clear_active_directory_traversal();
|
||||
}
|
||||
}
|
||||
}
|
||||
fn clear_active_directory_traversal(&mut self) {
|
||||
self.cache.clear();
|
||||
self.info.scan_resume_after = None;
|
||||
self.info.scan_checkpoint = None;
|
||||
self.info.scan_coverage_receipt = None;
|
||||
self.info.scan_raw_enumeration_cursor = None;
|
||||
self.info.scan_raw_enumeration_page_index = None;
|
||||
self.info.scan_raw_enumeration_path_page_indexes.clear();
|
||||
}
|
||||
}
|
||||
|
||||
fn directory_source_location(source: &str) -> &str {
|
||||
source.split_once('|').map_or(source, |(_, endpoint)| endpoint)
|
||||
}
|
||||
@@ -653,6 +653,7 @@ async fn cache_save_reconciles_an_ambiguous_committed_write() {
|
||||
cache.replace("bucket", "", DataUsageEntry::default());
|
||||
let revisions = DataUsageCacheRevisions {
|
||||
main: DataUsageCacheRevision::Missing,
|
||||
damaged_pages: DamagedPages::default(),
|
||||
backup: None,
|
||||
};
|
||||
|
||||
@@ -2857,6 +2858,7 @@ async fn test_save_backup_budget_tracks_primary_cost() {
|
||||
let store = Arc::new(store);
|
||||
let revisions = DataUsageCacheRevisions {
|
||||
main: DataUsageCacheRevision::Missing,
|
||||
damaged_pages: DamagedPages::default(),
|
||||
backup: Some(DataUsageCacheRevision::Etag("backup-etag".to_string())),
|
||||
};
|
||||
DataUsageCache::default()
|
||||
@@ -2909,6 +2911,7 @@ async fn test_checkpoint_budget_covers_retries_and_reconciliation() {
|
||||
let store = Arc::new(store);
|
||||
let revisions = DataUsageCacheRevisions {
|
||||
main: DataUsageCacheRevision::Missing,
|
||||
damaged_pages: DamagedPages::default(),
|
||||
backup: Some(DataUsageCacheRevision::Etag("backup-etag".to_string())),
|
||||
};
|
||||
let started = Instant::now();
|
||||
|
||||
@@ -88,6 +88,7 @@ async fn check_save(store: BackupFallbackStore, revisions: Option<DataUsageCache
|
||||
fn revisions(backup: bool) -> DataUsageCacheRevisions {
|
||||
DataUsageCacheRevisions {
|
||||
main: DataUsageCacheRevision::Missing,
|
||||
damaged_pages: DamagedPages::default(),
|
||||
backup: backup.then_some(DataUsageCacheRevision::Missing),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
use crate::data_usage_define::clone_cache_value;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sha2::{Digest as _, Sha256};
|
||||
use std::sync::Arc;
|
||||
use thiserror::Error;
|
||||
|
||||
mod writer;
|
||||
@@ -58,7 +59,7 @@ pub struct RawEnumerationPage {
|
||||
parent: String,
|
||||
page_index: u64,
|
||||
entries_start: u64,
|
||||
entries: Vec<String>,
|
||||
entries: Arc<[String]>,
|
||||
terminal: bool,
|
||||
digest: [u8; 32],
|
||||
}
|
||||
@@ -518,32 +519,26 @@ impl RawEnumerationPage {
|
||||
}
|
||||
|
||||
fn new(parent: &str, building: RawEnumerationPageBuilder) -> Self {
|
||||
let digest = raw_page_digest(parent, &building);
|
||||
let digest = raw_page_digest(parent, building.page_index, building.entries_start, building.terminal, &building.entries);
|
||||
Self {
|
||||
version: RAW_PAGE_INDEX_VERSION,
|
||||
parent: clone_cache_value(parent),
|
||||
page_index: building.page_index,
|
||||
entries_start: building.entries_start,
|
||||
entries: building.entries,
|
||||
entries: building.entries.into(),
|
||||
terminal: building.terminal,
|
||||
digest,
|
||||
}
|
||||
}
|
||||
|
||||
fn validate(&self, parent: &str, page_index: u64, entries_start: u64) -> Result<(), RawEnumerationPageIndexError> {
|
||||
let builder = RawEnumerationPageBuilder {
|
||||
page_index: self.page_index,
|
||||
entries_start: self.entries_start,
|
||||
entries: clone_cache_value(&self.entries),
|
||||
terminal: self.terminal,
|
||||
};
|
||||
if self.version != RAW_PAGE_INDEX_VERSION
|
||||
|| self.parent != parent
|
||||
|| self.page_index != page_index
|
||||
|| self.entries_start != entries_start
|
||||
|| self.entries.is_empty()
|
||||
|| !entries_are_normalized(&self.entries)
|
||||
|| self.digest != raw_page_digest(parent, &builder)
|
||||
|| self.digest != raw_page_digest(parent, self.page_index, self.entries_start, self.terminal, &self.entries)
|
||||
{
|
||||
return Err(RawEnumerationPageIndexError::CorruptIndex);
|
||||
}
|
||||
@@ -587,16 +582,16 @@ fn entry_sets_match(left: &[String], right: &[String]) -> bool {
|
||||
left.len() == right.len() && entries_contain_all(left, right)
|
||||
}
|
||||
|
||||
fn raw_page_digest(parent: &str, building: &RawEnumerationPageBuilder) -> [u8; 32] {
|
||||
fn raw_page_digest(parent: &str, page_index: u64, entries_start: u64, terminal: bool, entries: &[String]) -> [u8; 32] {
|
||||
#[cfg(test)]
|
||||
RAW_PAGE_DIGEST_ENTRIES.with(|count| count.set(count.get().saturating_add(building.entries.len())));
|
||||
RAW_PAGE_DIGEST_ENTRIES.with(|count| count.set(count.get().saturating_add(entries.len())));
|
||||
let mut digest = Sha256::new();
|
||||
update_digest(&mut digest, b"version", &RAW_PAGE_INDEX_VERSION.to_le_bytes());
|
||||
update_digest(&mut digest, b"parent", parent.as_bytes());
|
||||
update_digest(&mut digest, b"page_index", &building.page_index.to_le_bytes());
|
||||
update_digest(&mut digest, b"entries_start", &building.entries_start.to_le_bytes());
|
||||
update_digest(&mut digest, b"terminal", &[u8::from(building.terminal)]);
|
||||
for entry in &building.entries {
|
||||
update_digest(&mut digest, b"page_index", &page_index.to_le_bytes());
|
||||
update_digest(&mut digest, b"entries_start", &entries_start.to_le_bytes());
|
||||
update_digest(&mut digest, b"terminal", &[u8::from(terminal)]);
|
||||
for entry in entries {
|
||||
update_digest(&mut digest, b"entry", entry.as_bytes());
|
||||
}
|
||||
digest.finalize().into()
|
||||
|
||||
@@ -64,6 +64,15 @@ impl RawEnumerationPageWriter {
|
||||
})
|
||||
}
|
||||
|
||||
/// A parked native stream delivers the unread tail first. Incomplete
|
||||
/// membership can immediately append that tail; terminal page verification
|
||||
/// still requires observing the entire current directory.
|
||||
pub(crate) fn continue_native_stream(&mut self) {
|
||||
if !self.inner.complete && !self.inner.building.as_ref().is_some_and(|page| page.terminal) {
|
||||
self.observations = self.revalidate_after_entries;
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn indexed_entry_count(&self) -> usize {
|
||||
self.indexed.len()
|
||||
}
|
||||
|
||||
@@ -963,9 +963,14 @@ async fn scan_and_persist_local_bucket_with_store(
|
||||
}
|
||||
})?;
|
||||
let mut cache = DataUsageCache::default();
|
||||
let mut revisions = cache.load_with_revisions(set.clone(), &cache_name).await.map_err(|err| {
|
||||
RemoteScannerServerError::worker(format!("remote namespace scanner cache load or revision lookup failed: {err}"))
|
||||
})?;
|
||||
let Some(mut revisions) = Box::pin(cache.load_for_scan(set.clone(), &cache_name, &ctx, &budget))
|
||||
.await
|
||||
.map_err(|err| {
|
||||
RemoteScannerServerError::worker(format!("remote namespace scanner cache load or revision lookup failed: {err}"))
|
||||
})?
|
||||
else {
|
||||
return Ok(RemoteScannerFrameResult::Partial);
|
||||
};
|
||||
let expected_incarnation = cache
|
||||
.info
|
||||
.scan_identity
|
||||
|
||||
@@ -1198,6 +1198,43 @@ async fn terminal_frame_without_eof_still_obeys_semantic_stall_timeout() {
|
||||
assert!(error.to_string().contains("no semantic progress"));
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn heartbeat_frames_without_discovery_do_not_extend_semantic_deadline() {
|
||||
let request_id = Uuid::new_v4();
|
||||
let writer_auth = FrameAuthenticator::for_test(request_id);
|
||||
let reader_auth = FrameAuthenticator::for_test(request_id);
|
||||
let (mut writer, reader) = tokio::io::duplex(4096);
|
||||
let heartbeats = tokio::spawn(async move {
|
||||
let mut sequence = 0;
|
||||
loop {
|
||||
write_frame(
|
||||
&mut writer,
|
||||
&writer_auth,
|
||||
&mut sequence,
|
||||
&RemoteScannerFrame::with_phase(
|
||||
RemoteScannerProgress::default(),
|
||||
RemoteScannerPhase::Scanning,
|
||||
RemoteScannerFrameResult::Progress,
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("deliver authenticated heartbeat");
|
||||
writer.flush().await.expect("flush heartbeat");
|
||||
tokio::time::sleep(Duration::from_secs(4)).await;
|
||||
}
|
||||
});
|
||||
let owner = CancellationToken::new();
|
||||
let budget = ScannerCycleBudget::new(&owner, ScannerCycleBudgetConfig::default());
|
||||
let started = Instant::now();
|
||||
let error = consume_remote_scanner_stream(reader, owner, budget, "bucket", TEST_SOURCE, TEST_PLAN_DIGEST, reader_auth)
|
||||
.await
|
||||
.expect_err("heartbeats without work cannot keep the scanner alive");
|
||||
assert!(error.to_string().contains("no semantic progress"));
|
||||
assert_eq!(started.elapsed(), NS_SCANNER_SEMANTIC_STALL_TIMEOUT);
|
||||
heartbeats.abort();
|
||||
let _ = heartbeats.await;
|
||||
}
|
||||
|
||||
struct ResetAfterFrame {
|
||||
bytes: Vec<u8>,
|
||||
position: usize,
|
||||
|
||||
+115
-32
@@ -34,8 +34,9 @@ use crate::runtime_config::{
|
||||
use crate::scanner_budget::{ScannerCycleBudget, ScannerCycleBudgetConfig, ScannerCycleBudgetReason};
|
||||
use crate::scanner_folder::{data_usage_update_dir_cycles, heal_object_select_prob};
|
||||
use crate::scanner_io::{
|
||||
ScannerCycleDeferReason, ScannerCycleResult, ScannerCycleStatus, dirty_usage_bucket_notified, dirty_usage_buckets_pending,
|
||||
dirty_usage_generation, scanner_dirty_usage_state, scanner_maintenance_changed, scanner_maintenance_generation,
|
||||
ScannerCycleDeferReason, ScannerCycleResult, ScannerCycleStatus, ScannerServiceCohort, dirty_usage_bucket_notified,
|
||||
dirty_usage_buckets_pending, dirty_usage_generation, scanner_dirty_usage_state, scanner_maintenance_changed,
|
||||
scanner_maintenance_generation,
|
||||
};
|
||||
use crate::sleeper::{SCANNER_SLEEPER, set_scanner_default_speed};
|
||||
use crate::{DataUsageInfo, ScannerActivityGuard, ScannerError, ScannerRuntimeGuard};
|
||||
@@ -55,7 +56,7 @@ use rustfs_lock::{NamespaceLockGuard, error::LockError};
|
||||
use rustfs_scanner_metrics::metrics::{
|
||||
CurrentCycle, Metric, Metrics, RemoteScanProgress, ScanCyclePartialReason, ScanCycleWorkSnapshot,
|
||||
ScannerUsagePublicationObservation, ScannerUsageSaveResult, ScannerWorkSource, emit_scan_cycle_complete,
|
||||
emit_scan_cycle_deferred, emit_scan_cycle_partial_with_source, emit_scan_cycle_superseded, global_metrics,
|
||||
emit_scan_cycle_deferred, emit_scan_cycle_partial_with_stage, emit_scan_cycle_superseded, global_metrics,
|
||||
};
|
||||
use rustfs_scanner_metrics::startup::ScannerStartupToken;
|
||||
use serde::{Deserialize, Serialize};
|
||||
@@ -1212,6 +1213,8 @@ where
|
||||
}
|
||||
|
||||
let mut transient_backoff = ScannerRetryBackoff::default();
|
||||
// Only this supervisor, for this storage instance, owns resumable service.
|
||||
let mut service_continuation = None;
|
||||
let mut recovery_retry_count = 0_u32;
|
||||
loop {
|
||||
if ctx_clone.is_cancelled() {
|
||||
@@ -1226,6 +1229,7 @@ where
|
||||
startup_features,
|
||||
startup_maintenance_generation,
|
||||
startup,
|
||||
&mut service_continuation,
|
||||
)
|
||||
.await;
|
||||
if let Err(e) = &run_result {
|
||||
@@ -1773,11 +1777,26 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
/// Scheduling and mutation witnesses retained across a clean local deadline.
|
||||
/// Only an immediately succeeding leadership claim can resume this cohort.
|
||||
struct ScannerServiceContinuation {
|
||||
cohort: Arc<StdMutex<ScannerServiceCohort>>,
|
||||
fenced_epoch: u64,
|
||||
fenced_revision: DataUsageCacheRevision,
|
||||
}
|
||||
|
||||
impl ScannerServiceContinuation {
|
||||
fn matches_claim(&self, previous_revision: &DataUsageCacheRevision, claimed_epoch: u64) -> bool {
|
||||
&self.fenced_revision == previous_revision && self.fenced_epoch.checked_add(1) == Some(claimed_epoch)
|
||||
}
|
||||
}
|
||||
|
||||
struct ScannerCycleDeadlineState<'a> {
|
||||
cycle_info: &'a mut CurrentCycle,
|
||||
cycle_revision: &'a mut DataUsageCacheRevision,
|
||||
leader_epoch: &'a mut u64,
|
||||
cycle_budget: &'a ScannerCycleBudget,
|
||||
service_cohort: Arc<StdMutex<ScannerServiceCohort>>,
|
||||
allow_bootstrap_pending: bool,
|
||||
}
|
||||
|
||||
@@ -1791,7 +1810,8 @@ async fn handle_scanner_cycle_deadline<Store>(
|
||||
state: ScannerCycleDeadlineState<'_>,
|
||||
worker_stopped: bool,
|
||||
guard: &mut NamespaceLockGuard,
|
||||
) where
|
||||
) -> Option<ScannerServiceContinuation>
|
||||
where
|
||||
Store: ScannerObjectIO + ScannerConfigObjectDelete,
|
||||
{
|
||||
let fenced = fence_scanner_epoch_after_cycle_timeout(
|
||||
@@ -1821,8 +1841,14 @@ async fn handle_scanner_cycle_deadline<Store>(
|
||||
global_metrics().record_scanner_cycle_timeout(recovery_required, state.cycle_budget.progress_age());
|
||||
// Stop renewing before releasing the lease. A new leader can then claim the
|
||||
// higher persisted generation instead of inheriting the expired worker.
|
||||
let may_resume = !recovery_required && !guard.is_lock_lost() && !ctx.is_cancelled();
|
||||
guard.release();
|
||||
global_metrics().set_cycle(None).await;
|
||||
may_resume.then(|| ScannerServiceContinuation {
|
||||
cohort: state.service_cohort,
|
||||
fenced_epoch: *state.leader_epoch,
|
||||
fenced_revision: state.cycle_revision.clone(),
|
||||
})
|
||||
}
|
||||
|
||||
async fn mark_scan_cycle_idle(cycle_info: &mut CurrentCycle, cycle_metrics_guard: &mut ScannerCycleMetricsGuard) {
|
||||
@@ -2460,13 +2486,20 @@ where
|
||||
event = EVENT_SCANNER_CYCLE_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
cycle = cycle_info.current,
|
||||
required_cycle,
|
||||
state = "cache_cycle_ahead",
|
||||
partial_stage = "cache_cycle_ahead",
|
||||
cycle = cycle_info.current,
|
||||
leader_epoch,
|
||||
required_cycle,
|
||||
partial_cause = "cache_cycle_ahead",
|
||||
"Scanner cycle is recovering to a newer durable cache generation"
|
||||
);
|
||||
emit_scan_cycle_partial_with_source(cycle_start.elapsed(), ScanCyclePartialReason::Unknown, None);
|
||||
let partial = emit_scan_cycle_partial_with_stage(
|
||||
cycle_start.elapsed(),
|
||||
ScanCyclePartialReason::Unknown,
|
||||
None,
|
||||
Some("cache_cycle_ahead"),
|
||||
);
|
||||
let persisted = persist_required_scanner_cycle_floor_for_epoch(
|
||||
ctx,
|
||||
storeapi.clone(),
|
||||
@@ -2480,6 +2513,7 @@ where
|
||||
},
|
||||
)
|
||||
.await;
|
||||
partial.observe_state_write(persisted);
|
||||
return if persisted {
|
||||
cycle_budget.mark_cycle_state_persisted();
|
||||
ScannerCycleOutcome::Partial
|
||||
@@ -2530,18 +2564,7 @@ where
|
||||
}
|
||||
|
||||
usage_publication_result.restrict_outcome(usage_persist_outcome);
|
||||
let partial_cause = if scan_cycle_result.has_partial_bucket_progress() && !scan_cycle_result.has_usage_update() {
|
||||
"saved_bucket_progress"
|
||||
} else if scan_cycle_result.has_usage_update()
|
||||
&& matches!(
|
||||
scan_cycle_result.status,
|
||||
ScannerCycleStatus::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable)
|
||||
)
|
||||
{
|
||||
"activity_unverified_observation"
|
||||
} else {
|
||||
"incomplete_coverage"
|
||||
};
|
||||
let (partial_cause, partial_stage) = partial_cycle_diagnostic(&scan_cycle_result);
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
let durable_observation = scan_cycle_result
|
||||
.publication_expectation()
|
||||
@@ -2620,25 +2643,35 @@ where
|
||||
);
|
||||
}
|
||||
if cycle_budget.budget_elapsed() {
|
||||
let budget_reason = cycle_budget.reason();
|
||||
let budget_stage = match budget_reason {
|
||||
Some(ScannerCycleBudgetReason::Runtime) => Some("runtime_budget"),
|
||||
Some(ScannerCycleBudgetReason::Objects) => Some("object_budget"),
|
||||
Some(ScannerCycleBudgetReason::Directories) => Some("directory_budget"),
|
||||
Some(ScannerCycleBudgetReason::Entries) => Some("entry_budget"),
|
||||
None => None,
|
||||
};
|
||||
warn!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_CYCLE_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
state = "budget_reached",
|
||||
partial_stage = budget_stage,
|
||||
cycle = cycle_info.current,
|
||||
leader_epoch,
|
||||
duration = ?now.elapsed(),
|
||||
reason = ?cycle_budget.reason(),
|
||||
reason = ?budget_reason,
|
||||
max_duration = ?cycle_budget.max_duration(),
|
||||
max_objects = ?cycle_budget.max_objects(),
|
||||
max_directories = ?cycle_budget.max_directories(),
|
||||
state = "budget_reached",
|
||||
"Scanner cycle budget reached"
|
||||
);
|
||||
let budget_reason = cycle_budget.reason();
|
||||
emit_scan_cycle_partial_with_source(
|
||||
let partial = emit_scan_cycle_partial_with_stage(
|
||||
cycle_start.elapsed(),
|
||||
scan_cycle_partial_reason(budget_reason),
|
||||
scan_cycle_partial_source(budget_reason),
|
||||
budget_stage,
|
||||
);
|
||||
let persisted = finalize_partial_scan_cycle_for_epoch(
|
||||
ctx,
|
||||
@@ -2650,6 +2683,7 @@ where
|
||||
publication_epoch,
|
||||
)
|
||||
.await;
|
||||
partial.observe_state_write(persisted);
|
||||
return if persisted {
|
||||
cycle_budget.mark_cycle_state_persisted();
|
||||
ScannerCycleOutcome::Partial
|
||||
@@ -2683,29 +2717,56 @@ where
|
||||
return ScannerCycleOutcome::Failed;
|
||||
}
|
||||
ScannerCycleOutcome::Partial => {
|
||||
if ctx.is_cancelled() {
|
||||
let owner_cancelled = ctx.is_cancelled();
|
||||
let partial_stage = if owner_cancelled { "owner_cancelled" } else { partial_stage };
|
||||
if owner_cancelled {
|
||||
debug!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_CYCLE_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
cycle = cycle_info.current,
|
||||
state = "incomplete_cancelled",
|
||||
cycle = cycle_info.current,
|
||||
leader_epoch,
|
||||
partial_stage,
|
||||
"Scanner cycle stopped before a complete usage snapshot was produced"
|
||||
);
|
||||
} else {
|
||||
} else if partial_cause == "saved_bucket_progress"
|
||||
&& matches!(partial_stage, "bucket_checkpoint_saved" | "bucket_walk_incomplete")
|
||||
{
|
||||
warn!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_CYCLE_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
cycle = cycle_info.current,
|
||||
state = "incomplete",
|
||||
cycle = cycle_info.current,
|
||||
leader_epoch,
|
||||
partial_cause,
|
||||
"Scanner cycle ended without a complete usage snapshot"
|
||||
partial_stage,
|
||||
"Scanner saved bucket progress before completing coverage"
|
||||
);
|
||||
} else {
|
||||
error!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_CYCLE_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
state = "incomplete",
|
||||
cycle = cycle_info.current,
|
||||
leader_epoch,
|
||||
partial_cause,
|
||||
partial_stage,
|
||||
persist_outcome = ?usage_persist_outcome,
|
||||
"Scanner cycle could not certify complete usage"
|
||||
);
|
||||
}
|
||||
emit_scan_cycle_partial_with_source(cycle_start.elapsed(), ScanCyclePartialReason::Unknown, None);
|
||||
let partial = emit_scan_cycle_partial_with_stage(
|
||||
cycle_start.elapsed(),
|
||||
ScanCyclePartialReason::Unknown,
|
||||
None,
|
||||
Some(partial_stage),
|
||||
);
|
||||
let persisted = finalize_partial_scan_cycle_for_epoch(
|
||||
ctx,
|
||||
storeapi.clone(),
|
||||
@@ -2716,6 +2777,7 @@ where
|
||||
publication_epoch,
|
||||
)
|
||||
.await;
|
||||
partial.observe_state_write(persisted);
|
||||
return if persisted {
|
||||
cycle_budget.mark_cycle_state_persisted();
|
||||
ScannerCycleOutcome::Partial
|
||||
@@ -2909,6 +2971,7 @@ where
|
||||
maintenance_features,
|
||||
maintenance_generation,
|
||||
runtime_guard.startup_token(),
|
||||
&mut None,
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -2995,10 +3058,12 @@ async fn run_data_scanner_with_maintenance_state<S>(
|
||||
mut maintenance_features: ScannerMaintenanceFeatures,
|
||||
mut maintenance_generation_seen: Option<u64>,
|
||||
startup: ScannerStartupToken,
|
||||
continuation: &mut Option<ScannerServiceContinuation>,
|
||||
) -> Result<(), ScannerError>
|
||||
where
|
||||
S: ScannerStorage,
|
||||
{
|
||||
let previous = continuation.take();
|
||||
reset_scanner_cycle_schedule();
|
||||
// Normal scanner entry is an attempt, not evidence of leadership, coverage, or publication.
|
||||
global_metrics().startup().record_attempt(startup);
|
||||
@@ -3108,7 +3173,6 @@ where
|
||||
let mut deferred_backoff = ScannerRetryBackoff::default();
|
||||
let mut scope_fallback_backoff = ScannerRetryBackoff::default();
|
||||
let scope_fallback_slot = Arc::new(ScannerScopeFallbackSlot::default());
|
||||
let service_cohort = Arc::new(StdMutex::new(crate::scanner_io::ScannerServiceCohort::default()));
|
||||
let initial_runtime_config = resolve_scanner_runtime_config();
|
||||
if clean_idle_topology_supported && maintenance_generation_seen.is_none() {
|
||||
let Some((features, generation)) = detect_stable_scanner_maintenance_features(&ctx, &storeapi).await else {
|
||||
@@ -3269,6 +3333,7 @@ where
|
||||
finish_scanner_leader_iteration(guard.is_lock_lost(), "stopped", String::new()).await;
|
||||
return Ok(());
|
||||
}
|
||||
let previous_revision = cycle_revision.clone();
|
||||
let claim_ctx = ctx.child_token();
|
||||
let leadership_claimed = await_scanner_cycle_with_lock_fence(
|
||||
&claim_ctx,
|
||||
@@ -3318,6 +3383,10 @@ where
|
||||
finish_scanner_leader_iteration(false, "epoch_claim_failed", "leadership epoch claim failed".to_string()).await;
|
||||
return Ok(());
|
||||
}
|
||||
let service_cohort = previous
|
||||
.filter(|previous| previous.matches_claim(&previous_revision, leader_epoch))
|
||||
.map(|previous| previous.cohort)
|
||||
.unwrap_or_default();
|
||||
if rebuild_from_quarantine {
|
||||
record_scanner_cycle_state_rebuilt_from_quarantine(cycle_info.next, leader_epoch);
|
||||
}
|
||||
@@ -3402,7 +3471,7 @@ where
|
||||
return Ok(());
|
||||
}
|
||||
ScannerCycleWaitOutcome::Deadline { worker_stopped } => {
|
||||
handle_scanner_cycle_deadline(
|
||||
*continuation = handle_scanner_cycle_deadline(
|
||||
&ctx,
|
||||
storeapi.clone(),
|
||||
ScannerCycleDeadlineState {
|
||||
@@ -3410,6 +3479,7 @@ where
|
||||
cycle_revision: &mut cycle_revision,
|
||||
leader_epoch: &mut leader_epoch,
|
||||
cycle_budget: &cycle_budget,
|
||||
service_cohort,
|
||||
allow_bootstrap_pending: allow_usage_floor_bootstrap_pending,
|
||||
},
|
||||
worker_stopped,
|
||||
@@ -3747,7 +3817,7 @@ where
|
||||
return Ok(());
|
||||
}
|
||||
ScannerCycleWaitOutcome::Deadline { worker_stopped } => {
|
||||
handle_scanner_cycle_deadline(
|
||||
*continuation = handle_scanner_cycle_deadline(
|
||||
&ctx,
|
||||
storeapi.clone(),
|
||||
ScannerCycleDeadlineState {
|
||||
@@ -3755,6 +3825,7 @@ where
|
||||
cycle_revision: &mut cycle_revision,
|
||||
leader_epoch: &mut leader_epoch,
|
||||
cycle_budget: &cycle_budget,
|
||||
service_cohort,
|
||||
allow_bootstrap_pending: allow_usage_floor_bootstrap_pending,
|
||||
},
|
||||
worker_stopped,
|
||||
@@ -4334,6 +4405,18 @@ fn finalize_scanner_cycle_result(
|
||||
)
|
||||
}
|
||||
|
||||
fn partial_cycle_diagnostic(scan: &crate::scanner_io::ScannerCycleResult) -> (&'static str, &'static str) {
|
||||
if scan.has_partial_bucket_progress() && !scan.has_usage_update() {
|
||||
("saved_bucket_progress", scan.partial_stage.unwrap_or("bucket_checkpoint_saved"))
|
||||
} else if scan.has_usage_update()
|
||||
&& scan.status == ScannerCycleStatus::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable)
|
||||
{
|
||||
("activity_unverified_observation", "activity_baseline_unverified")
|
||||
} else {
|
||||
("incomplete_coverage", scan.partial_stage.unwrap_or("root_coverage_incomplete"))
|
||||
}
|
||||
}
|
||||
|
||||
fn scanner_cycle_completion_outcome_for_result(
|
||||
scan_cycle_result: &crate::scanner_io::ScannerCycleResult,
|
||||
usage_persist_outcome: DataUsagePersistOutcome,
|
||||
|
||||
@@ -52,9 +52,11 @@ const TEST_DEFAULT_SCANNER_CYCLE_SECS: u64 = 24 * 60 * 60;
|
||||
mod cycle_budget_publication;
|
||||
pub(super) mod cycle_persist_failure;
|
||||
mod gate_self_heal;
|
||||
mod paged_checkpoint;
|
||||
mod quota_reset_preservation;
|
||||
|
||||
mod recovery_control;
|
||||
mod restore_deadline;
|
||||
mod scoped_ack_publication;
|
||||
mod wakeup_metrics;
|
||||
|
||||
@@ -773,7 +775,8 @@ async fn cycle_budget_deadline_handler_fences_and_releases_guard() {
|
||||
);
|
||||
budget.mark_cycle_state_persisted();
|
||||
|
||||
handle_scanner_cycle_deadline(
|
||||
let service_cohort = Arc::new(StdMutex::new(ScannerServiceCohort::default()));
|
||||
let continuation = handle_scanner_cycle_deadline(
|
||||
&ctx,
|
||||
store.clone(),
|
||||
ScannerCycleDeadlineState {
|
||||
@@ -781,12 +784,24 @@ async fn cycle_budget_deadline_handler_fences_and_releases_guard() {
|
||||
cycle_revision: &mut cycle_revision,
|
||||
leader_epoch: &mut leader_epoch,
|
||||
cycle_budget: &budget,
|
||||
service_cohort: service_cohort.clone(),
|
||||
allow_bootstrap_pending: false,
|
||||
},
|
||||
true,
|
||||
&mut guard,
|
||||
)
|
||||
.await;
|
||||
.await
|
||||
.expect("a fully drained and fenced deadline retains the live cohort across supervisor iterations");
|
||||
assert!(Arc::ptr_eq(&continuation.cohort, &service_cohort));
|
||||
assert!(continuation.matches_claim(&cycle_revision, 3));
|
||||
assert!(
|
||||
!continuation.matches_claim(&cycle_revision, 4),
|
||||
"an intervening owner ends the continuation"
|
||||
);
|
||||
assert!(
|
||||
!continuation.matches_claim(&DataUsageCacheRevision::Missing, 3),
|
||||
"a replaced cursor ends the continuation"
|
||||
);
|
||||
|
||||
assert!(guard.is_released());
|
||||
assert_eq!(leader_epoch, 2, "deadline handler should claim the next epoch");
|
||||
@@ -989,6 +1004,7 @@ struct MemoryConfigStore {
|
||||
insert_after_gets: Mutex<HashMap<String, Vec<u8>>>,
|
||||
read_errors: Mutex<HashMap<String, EcstoreError>>,
|
||||
delayed_gets: Mutex<HashMap<String, Duration>>,
|
||||
checkpoint_page_delay_ms: AtomicUsize,
|
||||
non_regular_objects: Mutex<HashSet<String>>,
|
||||
fail_put_number: Mutex<HashMap<String, usize>>,
|
||||
/// Fail every put whose number is at least the stored value.
|
||||
@@ -1000,6 +1016,7 @@ struct MemoryConfigStore {
|
||||
cancel_after_successful_puts: Mutex<HashMap<String, (usize, CancellationToken)>>,
|
||||
replace_after_successful_puts: Mutex<HashMap<String, (usize, Vec<u8>)>>,
|
||||
error_after_commit_deletes: Mutex<HashSet<String>>,
|
||||
fail_prefix_deletes: AtomicBool,
|
||||
cancel_after_deletes: Mutex<HashMap<String, CancellationToken>>,
|
||||
pause_next_publication_admission: Mutex<Option<(Arc<tokio::sync::Notify>, Arc<tokio::sync::Notify>)>>,
|
||||
put_counts: Mutex<HashMap<String, usize>>,
|
||||
@@ -1046,6 +1063,17 @@ impl crate::storage_api::scanner_io::ObjectIO for MemoryConfigStore {
|
||||
.await
|
||||
.push(crate::storage_api::ecstore_diagnostic::current_read_version_entry_source());
|
||||
let key = memory_config_key(bucket, object);
|
||||
let _serial_page_read = if object.contains(".pages/") {
|
||||
restore_deadline::admit_page(object).await
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if object.contains(".pages/") {
|
||||
let delay = self.checkpoint_page_delay_ms.load(Ordering::Relaxed);
|
||||
if delay != 0 {
|
||||
tokio::time::sleep(Duration::from_millis(delay as u64)).await;
|
||||
}
|
||||
}
|
||||
if let Some(error) = self.read_errors.lock().await.get(&key).cloned() {
|
||||
return Err(error);
|
||||
}
|
||||
@@ -5003,6 +5031,15 @@ impl crate::ScannerConfigObjectDelete for MemoryConfigStore {
|
||||
async fn delete_config_object(&self, bucket: &str, object: &str, opts: ObjectOptions) -> EcstoreResult<ObjectInfo> {
|
||||
let key = memory_config_key(bucket, object);
|
||||
let mut objects = self.objects.lock().await;
|
||||
if opts.delete_prefix && !opts.delete_prefix_object {
|
||||
if self.fail_prefix_deletes.load(Ordering::Relaxed) {
|
||||
return Err(EcstoreError::other("injected prefix deletion failure"));
|
||||
}
|
||||
assert!(opts.http_preconditions.is_none(), "fixture prefix deletion cannot condition one object");
|
||||
objects.retain(|object, _| !object.starts_with(&key));
|
||||
self.revisions.lock().await.retain(|object, _| !object.starts_with(&key));
|
||||
return Ok(ObjectInfo::default());
|
||||
}
|
||||
if !objects.contains_key(&key) {
|
||||
return Err(EcstoreError::FileNotFound);
|
||||
}
|
||||
|
||||
@@ -253,6 +253,9 @@ async fn assert_coordinator_partial_progress(resume_mixed_plan: bool, fail_cycle
|
||||
.await;
|
||||
assert!(matches!(outcome, ScannerCycleOutcome::Partial));
|
||||
assert_eq!(budget.reason(), Some(ScannerCycleBudgetReason::Objects));
|
||||
let report = global_metrics().report().await;
|
||||
assert_eq!(report.last_cycle_partial_reason, "objects");
|
||||
assert_eq!(report.last_cycle_partial_stage.as_deref(), Some("object_budget"));
|
||||
assert!(budget.progress().0 > 0);
|
||||
assert!(budget.cycle_state_persisted());
|
||||
assert_eq!(cycle.next, 353);
|
||||
@@ -410,6 +413,16 @@ async fn assert_coordinator_partial_progress(resume_mixed_plan: bool, fail_cycle
|
||||
super::cycle_persist_failure::fail_saves(1, false, SCANNER_PERSIST_CAS_RETRIES + 1)
|
||||
}
|
||||
});
|
||||
use tracing::instrument::WithSubscriber as _;
|
||||
let error_log = tempfile::NamedTempFile::new().expect("error log");
|
||||
let subscriber = tracing_subscriber::fmt()
|
||||
.json()
|
||||
.without_time()
|
||||
.with_ansi(false)
|
||||
.with_max_level(tracing::Level::ERROR)
|
||||
.with_writer(error_log.reopen().expect("error log writer"))
|
||||
.finish();
|
||||
let _interest_guard = tracing::Dispatch::new(tracing::subscriber::NoSubscriber::default());
|
||||
let tail_outcome = run_data_scanner_cycle_with_budget(
|
||||
&ctx,
|
||||
&store,
|
||||
@@ -423,7 +436,23 @@ async fn assert_coordinator_partial_progress(resume_mixed_plan: bool, fail_cycle
|
||||
scope_fallback: None,
|
||||
},
|
||||
)
|
||||
.with_subscriber(subscriber)
|
||||
.await;
|
||||
let error_events = std::fs::read_to_string(error_log.path()).expect("read error log");
|
||||
let event = error_events
|
||||
.lines()
|
||||
.map(|line| serde_json::from_str::<serde_json::Value>(line).expect("JSON error event"))
|
||||
.find(|event| event["fields"]["state"] == "incomplete" && event["fields"]["cycle"] == 353)
|
||||
.expect("unbudgeted incomplete observation should be visible at error level");
|
||||
assert_eq!(event["fields"]["leader_epoch"], 1);
|
||||
assert_eq!(event["fields"]["partial_stage"], "mixed_plan_recertification_pending");
|
||||
let report = global_metrics().report().await;
|
||||
assert_eq!(report.last_cycle_partial_reason, "unknown");
|
||||
if fail_cycle_save == Some(false) {
|
||||
assert_eq!(report.last_cycle_partial_stage.as_deref(), Some("cycle_state_not_durable"));
|
||||
} else {
|
||||
assert_eq!(report.last_cycle_partial_stage.as_deref(), Some("mixed_plan_recertification_pending"));
|
||||
}
|
||||
assert!(!tail_budget.budget_elapsed(), "this is not the budget-exhaustion fast path");
|
||||
checkpoint.load(set, &cache_name).await.expect("reload the saved mixed tail");
|
||||
assert!(!checkpoint.info.snapshot_complete);
|
||||
@@ -527,6 +556,22 @@ async fn assert_coordinator_partial_progress(resume_mixed_plan: bool, fail_cycle
|
||||
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn partial_diagnostics_keep_the_activity_fence_distinct_from_checkpoint_progress() {
|
||||
let observation =
|
||||
ScannerCycleResult::new(ScannerCycleStatus::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable), None)
|
||||
.with_usage_update_emitted(true);
|
||||
assert_eq!(
|
||||
partial_cycle_diagnostic(&observation),
|
||||
("activity_unverified_observation", "activity_baseline_unverified")
|
||||
);
|
||||
let checkpoint = ScannerCycleResult::new(ScannerCycleStatus::Incomplete, None).with_partial_bucket_progress(true);
|
||||
assert_eq!(
|
||||
partial_cycle_diagnostic(&checkpoint),
|
||||
("saved_bucket_progress", "bucket_checkpoint_saved")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn partial_progress_does_not_mask_missing_updates_or_persistence_failure() {
|
||||
for (persisted, expected) in [
|
||||
|
||||
@@ -0,0 +1,475 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use super::*;
|
||||
use crate::BUCKET_META_PREFIX;
|
||||
|
||||
mod save_review;
|
||||
|
||||
const NAME: &str = "bucket/.usage-cache.bin";
|
||||
|
||||
fn empty_bucket_checkpoint() -> DataUsageCache {
|
||||
let mut cache = DataUsageCache::default();
|
||||
cache.info.name = "bucket".into();
|
||||
cache.info.source = Some(DataUsageCacheSource::new(0, 0));
|
||||
cache.info.scan_identity = Some(crate::data_usage_define::DataUsageScanIdentity {
|
||||
version: crate::data_usage_define::DATA_USAGE_SCAN_CHECKPOINT_VERSION,
|
||||
bucket_incarnation: uuid::Uuid::from_u128(42),
|
||||
set_layout: DataUsageScanPlanDigest([7; 32]),
|
||||
publication_epoch: 0,
|
||||
tier_registry_generation: 1,
|
||||
scan_mode: HealScanMode::Normal,
|
||||
});
|
||||
cache.replace("bucket", "", DataUsageEntry::default());
|
||||
cache
|
||||
}
|
||||
|
||||
fn seeded() -> DataUsageCache {
|
||||
let mut cache = empty_bucket_checkpoint();
|
||||
for index in 0..4096 {
|
||||
cache.replace(
|
||||
&format!("bucket/prefix-{index:05}"),
|
||||
"bucket",
|
||||
DataUsageEntry {
|
||||
size: index + 1,
|
||||
objects: 1,
|
||||
versions: 1,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
}
|
||||
cache
|
||||
}
|
||||
|
||||
async fn layout(store: &MemoryConfigStore) -> HashMap<String, (usize, usize)> {
|
||||
let puts = store.put_counts.lock().await.clone();
|
||||
store
|
||||
.objects
|
||||
.lock()
|
||||
.await
|
||||
.iter()
|
||||
.map(|(k, v)| (k.clone(), (puts[k], v.len())))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn page_bytes_after_save(before: &HashMap<String, (usize, usize)>, after: &HashMap<String, (usize, usize)>) -> (usize, usize) {
|
||||
let mut existing = 0;
|
||||
let mut uploaded = 0;
|
||||
for (path, &(writes, size)) in after.iter().filter(|(path, _)| path.contains(".pages/")) {
|
||||
if let Some(&(previous_writes, _)) = before.get(path) {
|
||||
assert_eq!(writes, previous_writes, "a committed immutable page was resubmitted");
|
||||
existing += size;
|
||||
} else {
|
||||
uploaded += size;
|
||||
}
|
||||
}
|
||||
(existing, uploaded)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn compacted_checkpoint_advances_one_child_without_rewriting_300k_names() {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
let mut cache = empty_bucket_checkpoint();
|
||||
let plan = DataUsageScanPlanDigest([1; 32]);
|
||||
cache.info.scan_progress = Some(crate::DataUsageScanProgress {
|
||||
started_plan: plan,
|
||||
requested_plan: plan,
|
||||
});
|
||||
cache.info.scan_directory_source = Some(format!("{}|disk-a", uuid::Uuid::from_u128(1)));
|
||||
let [first, second] = crate::scanner_folder::DirectoryCheckpoint::wide_storage_fixture(&cache.info).await;
|
||||
let mut before: Option<HashMap<String, (usize, usize)>> = None;
|
||||
let mut prior_progress = None;
|
||||
for (objects, receipt) in [(121_034, first), (121_035, second)] {
|
||||
cache.info.scan_directory_work = vec![receipt];
|
||||
cache.replace(
|
||||
"bucket",
|
||||
"",
|
||||
DataUsageEntry {
|
||||
compacted: true,
|
||||
objects,
|
||||
size: objects * 1024,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
let revisions = DataUsageCache::read_revisions(store.clone(), NAME)
|
||||
.await
|
||||
.expect("current revisions");
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0, CancellationToken::new())
|
||||
.await
|
||||
.expect("save compacted receipt");
|
||||
let after = layout(&store).await;
|
||||
if let Some(before) = before.as_ref() {
|
||||
let (old_bytes, new_bytes) = page_bytes_after_save(before, &after);
|
||||
eprintln!("300k-name checkpoint: {new_bytes} bytes in new pages, {old_bytes} bytes in existing pages");
|
||||
assert!(old_bytes > 3_000_000, "fixture must contain the full physical listing");
|
||||
assert!(
|
||||
new_bytes > 0 && new_bytes < old_bytes / 10,
|
||||
"one more completed child wrote {new_bytes} bytes alongside {old_bytes} reused bytes"
|
||||
);
|
||||
}
|
||||
before = Some(after);
|
||||
let mut restored = DataUsageCache::default();
|
||||
restored
|
||||
.load(store.clone(), NAME)
|
||||
.await
|
||||
.expect("cold read of compacted receipt");
|
||||
assert!(restored.info.scan_directory_work[0].is_bound_to(&restored.info));
|
||||
let progress = restored.durable_scan_progress();
|
||||
if let Some(prior) = &prior_progress {
|
||||
assert!(progress.advanced_from(prior), "the new child must earn another durable work turn");
|
||||
}
|
||||
prior_progress = Some(progress);
|
||||
let total = restored.checked_flatten("bucket").expect("connected root after restore");
|
||||
assert_eq!((total.objects, total.size), (objects, objects * 1024));
|
||||
cache = restored;
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dormant_linked_tree_pages_survive_source_switch_and_cold_reencoding() {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
let mut cache = seeded();
|
||||
let plan = DataUsageScanPlanDigest([1; 32]);
|
||||
cache.info.scan_progress = Some(crate::DataUsageScanProgress {
|
||||
started_plan: plan,
|
||||
requested_plan: plan,
|
||||
});
|
||||
let endpoints = std::collections::HashSet::from(["disk-a".to_owned(), "disk-b".to_owned()]);
|
||||
let a = format!("{}|disk-a", uuid::Uuid::from_u128(1));
|
||||
let b = format!("{}|disk-b", uuid::Uuid::from_u128(2));
|
||||
cache.info.scan_directory_source = Some(a.clone());
|
||||
cache.activate_directory_source(b.clone(), &endpoints);
|
||||
cache.replace("bucket", "", DataUsageEntry::default());
|
||||
let revisions = DataUsageCache::read_revisions(store.clone(), NAME)
|
||||
.await
|
||||
.expect("before first checkpoint");
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0, CancellationToken::new())
|
||||
.await
|
||||
.expect("persist dormant linked tree");
|
||||
let previous = layout(&store).await;
|
||||
|
||||
let mut cold = DataUsageCache::default();
|
||||
cold.load(store.clone(), NAME)
|
||||
.await
|
||||
.expect("restore checkpoint in a new process");
|
||||
cold.activate_directory_source(a, &endpoints);
|
||||
assert_eq!(cold.checked_flatten("bucket").expect("first source entries").objects, 4096);
|
||||
cold.activate_directory_source(b, &endpoints);
|
||||
assert_eq!(cold.checked_flatten("bucket").expect("second source root").objects, 0);
|
||||
cold.replace(
|
||||
"bucket/new-key",
|
||||
"bucket",
|
||||
DataUsageEntry {
|
||||
objects: 1,
|
||||
size: 99,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
let revisions = DataUsageCache::read_revisions(store.clone(), NAME)
|
||||
.await
|
||||
.expect("before second checkpoint");
|
||||
cold.save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0, CancellationToken::new())
|
||||
.await
|
||||
.expect("save one new child");
|
||||
let after = layout(&store).await;
|
||||
let (prior_bytes, new_bytes) = page_bytes_after_save(&previous, &after);
|
||||
assert!(prior_bytes > 128 * 1024);
|
||||
assert!(
|
||||
new_bytes > 0 && new_bytes < prior_bytes / 8,
|
||||
"dormant tree rewritten: {new_bytes} new bytes for {prior_bytes} prior bytes"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn paged_checkpoint_writes_only_changed_pages_and_commits_root_last() {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
let cache = seeded();
|
||||
let first = DataUsageCache::read_revisions(store.clone(), NAME)
|
||||
.await
|
||||
.expect("empty revisions");
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), NAME, &first, 0, CancellationToken::new())
|
||||
.await
|
||||
.expect("initial checkpoint");
|
||||
let before = layout(&store).await;
|
||||
let encoded = cache.marshal_msg().expect("encode baseline").len();
|
||||
let main = memory_config_key(RUSTFS_META_BUCKET, &format!("{BUCKET_META_PREFIX}/{NAME}"));
|
||||
assert!(before[&main].1 < 8192, "only a compact manifest belongs in the root");
|
||||
assert!(before.keys().any(|key| key.contains(".pages/")), "large cache must be paged");
|
||||
|
||||
let mut restarted = DataUsageCache::default();
|
||||
let revisions = restarted
|
||||
.load_with_revisions(store.clone(), NAME)
|
||||
.await
|
||||
.expect("restart from durable pages");
|
||||
let prior = restarted.checked_flatten("bucket").expect("all old entries");
|
||||
restarted.replace(
|
||||
"bucket/prefix-tail",
|
||||
"bucket",
|
||||
DataUsageEntry {
|
||||
objects: 1,
|
||||
versions: 1,
|
||||
size: 4711,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
store.fail_puts_from.lock().await.insert(main.clone(), before[&main].0 + 1);
|
||||
restarted
|
||||
.save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0, CancellationToken::new())
|
||||
.await
|
||||
.expect_err("commit interrupted after writing pages");
|
||||
let interrupted = layout(&store).await;
|
||||
let mut observer = DataUsageCache::default();
|
||||
observer.load(store.clone(), NAME).await.expect("last root still readable");
|
||||
assert_eq!(observer.checked_flatten("bucket").expect("previous coverage").size, prior.size);
|
||||
store.fail_puts_from.lock().await.remove(&main);
|
||||
restarted
|
||||
.save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0, CancellationToken::new())
|
||||
.await
|
||||
.expect("retry interrupted commit");
|
||||
observer.load(store.clone(), NAME).await.expect("load committed result");
|
||||
let actual = observer.checked_flatten("bucket").expect("new root");
|
||||
assert_eq!((actual.objects, actual.size), (4097, prior.size + 4711));
|
||||
let after = layout(&store).await;
|
||||
for (key, (writes, _)) in &interrupted {
|
||||
if key.contains(".bin.pages/") {
|
||||
assert_eq!(*writes, after[key].0, "retry restaged a previously successful immutable page");
|
||||
}
|
||||
}
|
||||
let new_bytes: usize = after
|
||||
.iter()
|
||||
.filter(|(key, _)| !before.contains_key(*key))
|
||||
.map(|(_, (_, size))| *size)
|
||||
.sum();
|
||||
assert!(
|
||||
new_bytes < encoded / 3,
|
||||
"one append staged {new_bytes} new bytes for a {encoded}-byte cache"
|
||||
);
|
||||
for (key, (writes, _)) in &before {
|
||||
if key.contains(".pages/") {
|
||||
assert_eq!(*writes, after[key].0, "a committed page was resubmitted");
|
||||
}
|
||||
}
|
||||
restarted.info.next_cycle += 1;
|
||||
let stale = restarted
|
||||
.save_with_revisions_for_epoch(store.clone(), NAME, &first, 0, CancellationToken::new())
|
||||
.await;
|
||||
assert!(
|
||||
matches!(stale, Err(EcstoreError::PreconditionFailed)),
|
||||
"stale writer cannot replace the new root"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn paged_checkpoint_cold_restore_spans_fixed_duration_and_repairs_corrupt_primary() {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
let cache = seeded();
|
||||
let revisions = DataUsageCache::read_revisions(store.clone(), NAME)
|
||||
.await
|
||||
.expect("read revisions");
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0, CancellationToken::new())
|
||||
.await
|
||||
.expect("write both copies");
|
||||
store.checkpoint_page_delay_ms.store(16, Ordering::Relaxed);
|
||||
let mut loaded = DataUsageCache::default();
|
||||
let mut completed_at = None;
|
||||
for round in 1..=128 {
|
||||
if let Ok(result) = tokio::time::timeout(Duration::from_millis(26), loaded.load(store.clone(), NAME)).await {
|
||||
result.expect("resume exact committed manifest");
|
||||
completed_at = Some(round);
|
||||
break;
|
||||
}
|
||||
}
|
||||
assert!(
|
||||
completed_at.is_some_and(|round| round > 1),
|
||||
"cold recovery must advance across fixed deadlines"
|
||||
);
|
||||
store.checkpoint_page_delay_ms.store(0, Ordering::Relaxed);
|
||||
let prior = loaded.checked_flatten("bucket").expect("cold cache totals");
|
||||
assert_eq!((prior.objects, prior.size), (4096, 4096 * 4097 / 2));
|
||||
|
||||
{
|
||||
let mut objects = store.objects.lock().await;
|
||||
let (_, block) = objects
|
||||
.iter_mut()
|
||||
.find(|(key, block)| key.contains(".bin.pages/") && block.len() > 8192)
|
||||
.expect("primary data page");
|
||||
block[0] ^= 0xff;
|
||||
}
|
||||
let revisions = loaded
|
||||
.load_with_revisions(store.clone(), NAME)
|
||||
.await
|
||||
.expect("read unaffected backup");
|
||||
assert_eq!(
|
||||
revisions,
|
||||
DataUsageCache::read_revisions(store.clone(), NAME)
|
||||
.await
|
||||
.expect("re-enter admission on same heads")
|
||||
);
|
||||
loaded
|
||||
.save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0, CancellationToken::new())
|
||||
.await
|
||||
.expect("replace damaged primary generation");
|
||||
store.objects.lock().await.retain(|key, _| !key.contains(".bkp"));
|
||||
let mut primary = DataUsageCache::default();
|
||||
primary.load(store, NAME).await.expect("repaired independent primary");
|
||||
assert_eq!(primary.checked_flatten("bucket").expect("primary totals").size, prior.size);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn paged_checkpoint_bucket_admission_uses_the_scan_budget_during_cold_load() {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
let cache = seeded();
|
||||
let revisions = DataUsageCache::read_revisions(store.clone(), NAME).await.unwrap();
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0, CancellationToken::new())
|
||||
.await
|
||||
.unwrap();
|
||||
store.checkpoint_page_delay_ms.store(16, Ordering::Relaxed);
|
||||
|
||||
let owner = CancellationToken::new();
|
||||
let mut loaded = DataUsageCache::default();
|
||||
for round in 0..128 {
|
||||
let budget = ScannerCycleBudget::new_with_progress_tracking(
|
||||
&owner,
|
||||
ScannerCycleBudgetConfig {
|
||||
max_duration: Some(Duration::from_millis(45)),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
// Exercise the same admission used by local and remote workers. The
|
||||
// worker may drain after the deadline; only the scan token interrupts
|
||||
// recovery here, as it does in the daemon.
|
||||
let revisions = loaded
|
||||
.load_for_scan(store.clone(), NAME, &budget.token(), &budget)
|
||||
.await
|
||||
.unwrap();
|
||||
if revisions.is_some() {
|
||||
assert!(round > 0, "the first slice cannot finish this physical restore");
|
||||
assert!(!budget.token().is_cancelled(), "return work only to an active scan");
|
||||
let totals = loaded.checked_flatten("bucket").expect("restored full bucket");
|
||||
assert_eq!((totals.objects, totals.size), (4096, 4096 * 4097 / 2));
|
||||
return;
|
||||
}
|
||||
}
|
||||
panic!("bucket admission repeatedly consumed the entire budget restoring the same root");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn paged_checkpoint_retirement_preserves_independent_backup_and_legacy_projection() {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
let mut cache = seeded();
|
||||
cache.info.snapshot_complete = true;
|
||||
let first = DataUsageCache::read_revisions(store.clone(), NAME)
|
||||
.await
|
||||
.expect("initial revisions");
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), NAME, &first, 0, CancellationToken::new())
|
||||
.await
|
||||
.expect("persist first generation");
|
||||
let original = layout(&store).await;
|
||||
let main = memory_config_key(RUSTFS_META_BUCKET, &format!("{BUCKET_META_PREFIX}/{NAME}"));
|
||||
let backup = format!("{main}.bkp");
|
||||
let legacy =
|
||||
DataUsageCache::unmarshal(&store.objects.lock().await[&main]).expect("old reader can read compact complete root");
|
||||
let totals = legacy
|
||||
.checked_flatten_complete_scope("bucket")
|
||||
.expect("legacy compact totals");
|
||||
assert_eq!((totals.objects, totals.size), (4096, 4096 * 4097 / 2));
|
||||
let initial = DataUsageCache::read_revisions(store.clone(), NAME)
|
||||
.await
|
||||
.expect("first generation revisions");
|
||||
cache.info.next_cycle += 1;
|
||||
store
|
||||
.fail_puts_from
|
||||
.lock()
|
||||
.await
|
||||
.insert(backup.clone(), original[&backup].0 + 1);
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), NAME, &initial, 0, CancellationToken::new())
|
||||
.await
|
||||
.expect("commit primary despite failed backup");
|
||||
let updated = layout(&store).await;
|
||||
for key in original.keys().filter(|key| key.contains(".pages/")) {
|
||||
assert_eq!(
|
||||
updated.contains_key(key),
|
||||
key.contains(".bkp.pages/"),
|
||||
"unexpected reclamation scope: {key}"
|
||||
);
|
||||
}
|
||||
let mut latest = DataUsageCache::default();
|
||||
latest.load(store.clone(), NAME).await.expect("new main remains readable");
|
||||
assert_eq!(latest.info.next_cycle, cache.info.next_cycle);
|
||||
store.objects.lock().await.remove(&main);
|
||||
latest
|
||||
.load(store, NAME)
|
||||
.await
|
||||
.expect("previous independent backup remains readable");
|
||||
assert_eq!(latest.info.next_cycle + 1, cache.info.next_cycle);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn paged_checkpoint_scan_handoff_checks_the_latest_root_before_using_its_totals() {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
let mut cache = seeded();
|
||||
cache.cache.clear();
|
||||
let replace = |cache: &mut DataUsageCache, size| {
|
||||
cache.replace(
|
||||
"bucket",
|
||||
"",
|
||||
DataUsageEntry {
|
||||
objects: 1,
|
||||
size,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
};
|
||||
replace(&mut cache, 17);
|
||||
let revisions = DataUsageCache::read_revisions(store.clone(), NAME).await.unwrap();
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0, CancellationToken::new())
|
||||
.await
|
||||
.unwrap();
|
||||
let main = memory_config_key(RUSTFS_META_BUCKET, &format!("{BUCKET_META_PREFIX}/{NAME}"));
|
||||
store.delayed_gets.lock().await.insert(main, Duration::from_secs(6));
|
||||
|
||||
let parent = CancellationToken::new();
|
||||
let budget = || {
|
||||
ScannerCycleBudget::new_with_progress_tracking(
|
||||
&parent,
|
||||
ScannerCycleBudgetConfig {
|
||||
max_duration: Some(Duration::from_secs(10)),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
};
|
||||
let first = budget();
|
||||
let mut loaded = DataUsageCache::default();
|
||||
assert!(
|
||||
loaded
|
||||
.load_for_scan(store.clone(), NAME, &first.token(), &first)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert!(!first.token().is_cancelled(), "the slow load moved its result to the next admission");
|
||||
|
||||
let revisions = DataUsageCache::read_revisions(store.clone(), NAME).await.unwrap();
|
||||
replace(&mut cache, 99);
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0, CancellationToken::new())
|
||||
.await
|
||||
.unwrap();
|
||||
let next = budget();
|
||||
assert!(
|
||||
loaded
|
||||
.load_for_scan(store, NAME, &next.token(), &next)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_some()
|
||||
);
|
||||
assert_eq!(loaded.checked_flatten("bucket").expect("new root").size, 99);
|
||||
}
|
||||
@@ -0,0 +1,276 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn indefinite_partial_checkpoints_keep_unreachable_bytes_bounded() {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
let mut cache = seeded();
|
||||
let mut initial_bytes = 0;
|
||||
for round in 0..10 {
|
||||
for (name, entry) in &mut cache.cache {
|
||||
if name != "bucket" {
|
||||
entry.size += 17;
|
||||
}
|
||||
}
|
||||
// A single long-running cycle can checkpoint many times without changing cycle number.
|
||||
assert!(!cache.info.snapshot_complete);
|
||||
let revisions = DataUsageCache::read_revisions(store.clone(), NAME)
|
||||
.await
|
||||
.expect("current heads");
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0, CancellationToken::new())
|
||||
.await
|
||||
.expect("commit partial");
|
||||
let current: usize = layout(&store)
|
||||
.await
|
||||
.iter()
|
||||
.filter(|(path, _)| path.contains(".pages/"))
|
||||
.map(|(_, (_, size))| size)
|
||||
.sum();
|
||||
if round == 0 {
|
||||
initial_bytes = current;
|
||||
assert!(initial_bytes > 100_000);
|
||||
}
|
||||
assert!(
|
||||
current <= initial_bytes * 4,
|
||||
"after {} partials, {current} page bytes for {initial_bytes} initially",
|
||||
round + 1
|
||||
);
|
||||
}
|
||||
let loaded = &mut DataUsageCache::default();
|
||||
loaded.load(store, NAME).await.expect("restore bounded checkpoint");
|
||||
assert_eq!(
|
||||
loaded.checked_flatten("bucket").expect("valid restored tree").size,
|
||||
cache.checked_flatten("bucket").expect("fixture tree").size
|
||||
);
|
||||
}
|
||||
|
||||
async fn reopen_checkpoint_store(store: &MemoryConfigStore) -> Arc<MemoryConfigStore> {
|
||||
let reopened = Arc::new(MemoryConfigStore::default());
|
||||
*reopened.objects.lock().await = store.objects.lock().await.clone();
|
||||
*reopened.revisions.lock().await = store.revisions.lock().await.clone();
|
||||
*reopened.put_counts.lock().await = store.put_counts.lock().await.clone();
|
||||
reopened
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn abandoned_checkpoint_generation_is_reclaimed_after_an_inline_commit() {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
let main = memory_config_key(RUSTFS_META_BUCKET, &format!("{BUCKET_META_PREFIX}/{NAME}"));
|
||||
let revisions = DataUsageCache::read_revisions(store.clone(), NAME)
|
||||
.await
|
||||
.expect("empty store");
|
||||
let mut committed = empty_bucket_checkpoint();
|
||||
committed
|
||||
.save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0, CancellationToken::new())
|
||||
.await
|
||||
.expect("commit small checkpoint");
|
||||
|
||||
let before = layout(&store).await;
|
||||
let revisions = DataUsageCache::read_revisions(store.clone(), NAME)
|
||||
.await
|
||||
.expect("inline revision");
|
||||
store.fail_puts_from.lock().await.insert(main.clone(), before[&main].0 + 1);
|
||||
seeded()
|
||||
.save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0, CancellationToken::new())
|
||||
.await
|
||||
.expect_err("interrupt the root commit after staging the large checkpoint");
|
||||
|
||||
let staged: Vec<_> = layout(&store)
|
||||
.await
|
||||
.into_keys()
|
||||
.filter(|path| path.contains(".bin.pages/") && !before.contains_key(path))
|
||||
.collect();
|
||||
assert!(staged.len() > 1, "the interrupted commit must have written pages");
|
||||
assert_eq!(
|
||||
DataUsageCache::read_revisions(store.clone(), NAME)
|
||||
.await
|
||||
.expect("roots remain readable"),
|
||||
revisions
|
||||
);
|
||||
|
||||
// Preserve durable objects while dropping the interrupted writer's receipts.
|
||||
let store = reopen_checkpoint_store(&store).await;
|
||||
store.fail_prefix_deletes.store(true, Ordering::Relaxed);
|
||||
committed.replace(
|
||||
"bucket",
|
||||
"",
|
||||
DataUsageEntry {
|
||||
objects: 1,
|
||||
size: 17,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
committed
|
||||
.save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0, CancellationToken::new())
|
||||
.await
|
||||
.expect("publish the different inline checkpoint despite interrupted cleanup");
|
||||
let interrupted_cleanup = layout(&store).await;
|
||||
assert!(staged.iter().all(|path| interrupted_cleanup.contains_key(path)));
|
||||
|
||||
// A second restart leaves only the new root's durable cleanup record.
|
||||
let restarted = reopen_checkpoint_store(&store).await;
|
||||
let mut restored = DataUsageCache::default();
|
||||
let revisions = restored
|
||||
.load_with_revisions(restarted.clone(), NAME)
|
||||
.await
|
||||
.expect("restore the committed inline checkpoint");
|
||||
assert_eq!(restored.checked_flatten("bucket").expect("inline totals").size, 17);
|
||||
restored
|
||||
.save_with_revisions_for_epoch(restarted.clone(), NAME, &revisions, 0, CancellationToken::new())
|
||||
.await
|
||||
.expect("reclaim the abandoned generation after restart");
|
||||
let after = layout(&restarted).await;
|
||||
assert!(staged.iter().all(|path| !after.contains_key(path)), "abandoned pages leaked");
|
||||
restored.load(restarted, NAME).await.expect("load after reclaim");
|
||||
assert_eq!(restored.checked_flatten("bucket").expect("retained inline totals").size, 17);
|
||||
}
|
||||
|
||||
enum BackupDamage {
|
||||
Missing,
|
||||
Corrupt,
|
||||
UnreadableMetadata,
|
||||
}
|
||||
|
||||
async fn survives_lost_primary_after_backup_page_damage(damage: BackupDamage) {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
let cache = seeded();
|
||||
let first = DataUsageCache::read_revisions(store.clone(), NAME)
|
||||
.await
|
||||
.expect("empty heads");
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), NAME, &first, 0, CancellationToken::new())
|
||||
.await
|
||||
.expect("initial checkpoint");
|
||||
let modify_tail = |cache: &mut DataUsageCache| {
|
||||
cache.cache.get_mut("bucket/prefix-04095").expect("last leaf").size += 99;
|
||||
};
|
||||
let control = Arc::new(MemoryConfigStore::default());
|
||||
let mut target = cache.clone();
|
||||
modify_tail(&mut target);
|
||||
let empty = DataUsageCache::read_revisions(control.clone(), NAME)
|
||||
.await
|
||||
.expect("empty control store");
|
||||
target
|
||||
.save_with_revisions_for_epoch(control.clone(), NAME, &empty, 0, CancellationToken::new())
|
||||
.await
|
||||
.expect("encode target in an independent fresh bundle");
|
||||
{
|
||||
let control_objects = control.objects.lock().await;
|
||||
let mut objects = store.objects.lock().await;
|
||||
let victim = objects
|
||||
.iter()
|
||||
.find_map(|(path, data)| {
|
||||
// A fresh target bundle only contains currently reachable objects.
|
||||
(path.contains(".bkp.pages/") && data.len() > 8192 && control_objects.get(path) == Some(data))
|
||||
.then(|| path.clone())
|
||||
})
|
||||
.expect("a large unchanged leaf is reachable in both manifests");
|
||||
match damage {
|
||||
BackupDamage::Corrupt => objects.get_mut(&victim).expect("existing backup leaf")[0] ^= 1,
|
||||
BackupDamage::Missing => {
|
||||
objects.remove(&victim);
|
||||
}
|
||||
BackupDamage::UnreadableMetadata => {
|
||||
drop(objects);
|
||||
store.read_errors.lock().await.insert(victim, EcstoreError::FileCorrupt);
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut from_main = DataUsageCache::default();
|
||||
let revisions = from_main
|
||||
.load_with_revisions(store.clone(), NAME)
|
||||
.await
|
||||
.expect("primary remains healthy");
|
||||
modify_tail(&mut from_main);
|
||||
from_main
|
||||
.save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0, CancellationToken::new())
|
||||
.await
|
||||
.expect("incremental commit while primary is healthy");
|
||||
let main = memory_config_key(RUSTFS_META_BUCKET, &format!("{BUCKET_META_PREFIX}/{NAME}"));
|
||||
store.objects.lock().await.remove(&main);
|
||||
let mut recovered = DataUsageCache::default();
|
||||
recovered.load(store, NAME).await.expect("standalone cold backup recovery");
|
||||
let actual = recovered
|
||||
.checked_flatten("bucket")
|
||||
.expect("a connected backup after losing its primary");
|
||||
assert_eq!((actual.objects, actual.size), (4096, 4096 * 4097 / 2 + 99));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn incremental_save_repairs_missing_backup_leaf() {
|
||||
survives_lost_primary_after_backup_page_damage(BackupDamage::Missing).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn incremental_save_repairs_corrupt_backup_leaf() {
|
||||
survives_lost_primary_after_backup_page_damage(BackupDamage::Corrupt).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn incremental_save_rebuilds_backup_with_unreadable_page_metadata() {
|
||||
survives_lost_primary_after_backup_page_damage(BackupDamage::UnreadableMetadata).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unreclaimed_partial_generation_bounds_staging_and_can_retry() {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
let mut cache = seeded();
|
||||
cache.info.snapshot_complete = true;
|
||||
let empty = DataUsageCache::read_revisions(store.clone(), NAME)
|
||||
.await
|
||||
.expect("empty store");
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), NAME, &empty, 0, CancellationToken::new())
|
||||
.await
|
||||
.expect("committed predecessor");
|
||||
store.fail_prefix_deletes.store(true, Ordering::Relaxed);
|
||||
cache.info.snapshot_complete = false;
|
||||
cache.info.next_cycle += 1;
|
||||
for _ in 0..8 {
|
||||
for (name, entry) in &mut cache.cache {
|
||||
if name != "bucket" {
|
||||
entry.size += 17;
|
||||
}
|
||||
}
|
||||
let revisions = DataUsageCache::read_revisions(store.clone(), NAME)
|
||||
.await
|
||||
.expect("current heads");
|
||||
let before = layout(&store).await;
|
||||
if let Err(error) = cache
|
||||
.save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0, CancellationToken::new())
|
||||
.await
|
||||
{
|
||||
assert!(error.to_string().contains("reclaim"), "unexpected save failure: {error}");
|
||||
assert_eq!(before, layout(&store).await, "a blocked retirement must not stage further objects");
|
||||
assert!(
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0, CancellationToken::new())
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(
|
||||
before,
|
||||
layout(&store).await,
|
||||
"retry while cleanup is unavailable must leave both generations readable"
|
||||
);
|
||||
let mut previous = DataUsageCache::default();
|
||||
previous.load(store.clone(), NAME).await.expect("last committed checkpoint");
|
||||
assert!(previous.checked_flatten("bucket").is_some());
|
||||
store.fail_prefix_deletes.store(false, Ordering::Relaxed);
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0, CancellationToken::new())
|
||||
.await
|
||||
.expect("save after previous retirement is reclaimed");
|
||||
previous.load(store.clone(), NAME).await.expect("read latest checkpoint");
|
||||
assert_eq!(
|
||||
cache.checked_flatten("bucket").expect("fixture").size,
|
||||
previous.checked_flatten("bucket").expect("recovered root").size
|
||||
);
|
||||
return;
|
||||
}
|
||||
}
|
||||
panic!("persistent retirement failure allowed unbounded staging");
|
||||
}
|
||||
@@ -160,6 +160,7 @@ async fn issue_8297_contended_follower_clears_only_transient_cycle_recovery() {
|
||||
ScannerMaintenanceFeatures::default(),
|
||||
None,
|
||||
runtime.startup_token(),
|
||||
&mut None,
|
||||
),
|
||||
)
|
||||
.await
|
||||
|
||||
@@ -0,0 +1,275 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use super::*;
|
||||
use crate::data_usage_define::{DATA_USAGE_SCAN_CHECKPOINT_VERSION, DataUsageCacheRevisions, DataUsageScanIdentity};
|
||||
use tokio::sync::{Notify, OwnedSemaphorePermit, Semaphore};
|
||||
|
||||
const NAME: &str = "bucket/.usage-cache.bin";
|
||||
const COUNT: usize = 1536;
|
||||
|
||||
struct ReadWindow {
|
||||
pages: usize,
|
||||
device: Arc<Semaphore>,
|
||||
completed: std::sync::Mutex<Vec<String>>,
|
||||
deadline: Notify,
|
||||
}
|
||||
|
||||
tokio::task_local! {
|
||||
static READ_WINDOW: Arc<ReadWindow>;
|
||||
}
|
||||
|
||||
// Model a serial device with one page of I/O time per admission. Completing
|
||||
// a second read would exceed the deadline, so the caller drops that waiter.
|
||||
pub(super) async fn admit_page(object: &str) -> Option<OwnedSemaphorePermit> {
|
||||
let window = READ_WINDOW.try_with(Arc::clone).ok()?;
|
||||
let permit = window.device.clone().acquire_owned().await.expect("device stays open");
|
||||
{
|
||||
let mut completed = window.completed.lock().expect("device history");
|
||||
if completed.len() < window.pages {
|
||||
completed.push(object.into());
|
||||
return Some(permit);
|
||||
}
|
||||
}
|
||||
drop(permit);
|
||||
window.deadline.notify_one();
|
||||
std::future::pending().await
|
||||
}
|
||||
|
||||
async fn window(store: &Arc<MemoryConfigStore>, loaded: &mut DataUsageCache) -> (Option<DataUsageCacheRevisions>, Vec<String>) {
|
||||
let window = Arc::new(ReadWindow {
|
||||
pages: 1,
|
||||
device: Arc::new(Semaphore::new(1)),
|
||||
completed: Default::default(),
|
||||
deadline: Notify::new(),
|
||||
});
|
||||
let result = READ_WINDOW
|
||||
.scope(window.clone(), async {
|
||||
tokio::select! {
|
||||
biased;
|
||||
value = loaded.load_with_revisions(store.clone(), NAME) => Some(value.expect("checkpoint load")),
|
||||
() = window.deadline.notified() => None,
|
||||
}
|
||||
})
|
||||
.await;
|
||||
let pages = std::mem::take(&mut *window.completed.lock().expect("device history"));
|
||||
(result, pages)
|
||||
}
|
||||
|
||||
async fn fixture() -> (Arc<MemoryConfigStore>, DataUsageCache) {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
let mut cache = DataUsageCache::default();
|
||||
cache.info.name = "bucket".into();
|
||||
cache.info.source = Some(DataUsageCacheSource::new(0, 0));
|
||||
cache.info.scan_identity = Some(DataUsageScanIdentity {
|
||||
version: DATA_USAGE_SCAN_CHECKPOINT_VERSION,
|
||||
bucket_incarnation: uuid::Uuid::from_u128(9),
|
||||
set_layout: DataUsageScanPlanDigest([7; 32]),
|
||||
publication_epoch: 0,
|
||||
tier_registry_generation: 1,
|
||||
scan_mode: HealScanMode::Normal,
|
||||
});
|
||||
cache.replace("bucket", "", DataUsageEntry::default());
|
||||
for key in 0..COUNT {
|
||||
cache.replace(
|
||||
&format!("bucket/key-{key:08}"),
|
||||
"bucket",
|
||||
DataUsageEntry {
|
||||
objects: 1,
|
||||
size: key + 1,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
}
|
||||
let revisions = DataUsageCache::read_revisions(store.clone(), NAME)
|
||||
.await
|
||||
.expect("initial revisions");
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0, CancellationToken::new())
|
||||
.await
|
||||
.expect("seed both copies");
|
||||
(store, cache)
|
||||
}
|
||||
|
||||
async fn corrupt_primary(store: &MemoryConfigStore) {
|
||||
let mut objects = store.objects.lock().await;
|
||||
let mut pages = 0;
|
||||
for (_, data) in objects.iter_mut().filter(|(path, _)| path.contains(".bin.pages/")) {
|
||||
data[0] ^= 1;
|
||||
pages += 1;
|
||||
}
|
||||
assert!(pages > 4, "fixture must span more than one physical read batch");
|
||||
}
|
||||
|
||||
async fn finish(store: &Arc<MemoryConfigStore>, backup: bool) {
|
||||
finish_across_windows(store, backup, None).await;
|
||||
}
|
||||
|
||||
async fn finish_across_windows(store: &Arc<MemoryConfigStore>, backup: bool, competing: Option<&Arc<MemoryConfigStore>>) {
|
||||
let mut loaded = DataUsageCache::default();
|
||||
let mut main_reads = 0;
|
||||
let mut backup_reads = 0;
|
||||
for _ in 0..128 {
|
||||
let (result, pages) = window(store, &mut loaded).await;
|
||||
for page in pages {
|
||||
if page.contains(".bkp.pages/") {
|
||||
backup_reads += 1;
|
||||
} else {
|
||||
main_reads += 1;
|
||||
}
|
||||
}
|
||||
if result.is_some() {
|
||||
let total = loaded.checked_flatten("bucket").expect("restored bucket");
|
||||
assert_eq!(total.objects, COUNT);
|
||||
assert_eq!(total.size, COUNT * (COUNT + 1) / 2);
|
||||
if backup {
|
||||
assert_eq!(main_reads, 1, "read the corrupt primary once for this revision");
|
||||
assert!(backup_reads > 4, "the independent backup must be restored across admissions");
|
||||
} else {
|
||||
assert!(main_reads > 4);
|
||||
assert_eq!(backup_reads, 0);
|
||||
}
|
||||
return;
|
||||
}
|
||||
if let Some(other) = competing {
|
||||
tokio::time::advance(Duration::from_secs(2 * 60 * 60)).await;
|
||||
let _ = window(other, &mut DataUsageCache::default()).await;
|
||||
}
|
||||
}
|
||||
panic!("checkpoint did not converge: {main_reads} primary and {backup_reads} backup page reads");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn serial_backend_restores_one_verified_page_per_deadline() {
|
||||
let (store, _) = fixture().await;
|
||||
finish(&store, false).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn concurrent_cancellations_keep_the_longer_verified_prefix() {
|
||||
let (store, _) = fixture().await;
|
||||
let longer = Arc::new(ReadWindow {
|
||||
pages: 3,
|
||||
device: Arc::new(Semaphore::new(1)),
|
||||
completed: Default::default(),
|
||||
deadline: Notify::new(),
|
||||
});
|
||||
let mut first = DataUsageCache::default();
|
||||
let mut read = Box::pin(READ_WINDOW.scope(longer.clone(), first.load(store.clone(), NAME)));
|
||||
tokio::select! {
|
||||
result = &mut read => panic!("checkpoint unexpectedly finished within three reads: {result:?}"),
|
||||
() = longer.deadline.notified() => {}
|
||||
}
|
||||
let mut concurrent = DataUsageCache::default();
|
||||
assert!(window(&store, &mut concurrent).await.0.is_none());
|
||||
drop(read);
|
||||
|
||||
let (_, next) = window(&store, &mut concurrent).await;
|
||||
let previous = longer.completed.lock().expect("first reader history");
|
||||
assert_eq!(previous.len(), 3);
|
||||
assert!(
|
||||
!previous.contains(&next[0]),
|
||||
"a shorter concurrent cancellation displaced a verified prefix"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn corrupt_primary_yields_future_admissions_to_the_healthy_backup() {
|
||||
let (store, _) = fixture().await;
|
||||
corrupt_primary(&store).await;
|
||||
finish(&store, true).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn new_primary_revision_supersedes_an_incomplete_backup_restore() {
|
||||
let (store, _) = fixture().await;
|
||||
corrupt_primary(&store).await;
|
||||
let mut loaded = DataUsageCache::default();
|
||||
assert!(window(&store, &mut loaded).await.0.is_none());
|
||||
let (result, pages) = window(&store, &mut loaded).await;
|
||||
assert!(result.is_none());
|
||||
assert!(pages[0].contains(".bkp.pages/"), "second admission should advance backup");
|
||||
|
||||
let mut new_primary = DataUsageCache::default();
|
||||
new_primary.info.name = "bucket".into();
|
||||
new_primary.replace(
|
||||
"bucket",
|
||||
"",
|
||||
DataUsageEntry {
|
||||
objects: 1,
|
||||
size: 19,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
let root = memory_config_key(RUSTFS_META_BUCKET, &format!("{}/{NAME}", crate::BUCKET_META_PREFIX));
|
||||
let bytes = rmp_serde::to_vec_named(&new_primary).expect("replacement main root");
|
||||
store.objects.lock().await.insert(root.clone(), bytes);
|
||||
*store.revisions.lock().await.get_mut(&root).expect("primary revision") += 1;
|
||||
let (result, pages) = window(&store, &mut loaded).await;
|
||||
assert!(result.is_some(), "must prefer the replacement main head immediately");
|
||||
assert!(pages.is_empty(), "must discard work on the older backup");
|
||||
let total = loaded.checked_flatten("bucket").expect("new primary root");
|
||||
assert_eq!((total.objects, total.size), (1, 19));
|
||||
}
|
||||
|
||||
async fn inject_primary_page_error(store: &MemoryConfigStore, error: EcstoreError) {
|
||||
let errors = store
|
||||
.objects
|
||||
.lock()
|
||||
.await
|
||||
.keys()
|
||||
.filter(|key| key.contains(".bin.pages/"))
|
||||
.map(|key| (key.clone(), error.clone()))
|
||||
.collect();
|
||||
*store.read_errors.lock().await = errors;
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn a_transient_primary_page_error_does_not_schedule_the_backup() {
|
||||
let (store, _) = fixture().await;
|
||||
inject_primary_page_error(&store, EcstoreError::SlowDown).await;
|
||||
let mut loaded = DataUsageCache::default();
|
||||
assert!(window(&store, &mut loaded).await.0.is_none());
|
||||
store.read_errors.lock().await.clear();
|
||||
finish(&store, false).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn backup_phase_is_bound_to_its_storage_instance() {
|
||||
let (damaged, _) = fixture().await;
|
||||
corrupt_primary(&damaged).await;
|
||||
assert!(window(&damaged, &mut DataUsageCache::default()).await.0.is_none());
|
||||
|
||||
let (healthy, _) = fixture().await;
|
||||
let first = DataUsageCache::read_revisions(damaged.clone(), NAME)
|
||||
.await
|
||||
.expect("failed primary revision");
|
||||
let second = DataUsageCache::read_revisions(healthy.clone(), NAME)
|
||||
.await
|
||||
.expect("other storage revision");
|
||||
assert_eq!(first, second, "both stores must expose the same ETags");
|
||||
finish(&healthy, false).await;
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn idle_restore_keeps_verified_pages_until_another_bucket_needs_its_slot() {
|
||||
let (primary, _) = fixture().await;
|
||||
let (other, _) = fixture().await;
|
||||
finish_across_windows(&primary, false, Some(&other)).await;
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn idle_restore_keeps_the_failed_primary_revision_across_scheduled_rounds() {
|
||||
let (primary, _) = fixture().await;
|
||||
let (other, _) = fixture().await;
|
||||
corrupt_primary(&primary).await;
|
||||
corrupt_primary(&other).await;
|
||||
finish_across_windows(&primary, true, Some(&other)).await;
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn physical_primary_page_corruption_falls_back_to_the_healthy_copy() {
|
||||
let (store, _) = fixture().await;
|
||||
inject_primary_page_error(&store, EcstoreError::FileCorrupt).await;
|
||||
finish(&store, true).await;
|
||||
}
|
||||
@@ -526,6 +526,19 @@ fn data_usage_entry_has_progress(entry: &DataUsageEntry) -> bool {
|
||||
data_usage_root_has_progress(entry)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
tokio::task_local! {
|
||||
static TEST_FORCE_COMPACTION_WIDTH: usize;
|
||||
}
|
||||
|
||||
fn force_compaction_width() -> usize {
|
||||
#[cfg(test)]
|
||||
if let Ok(width) = TEST_FORCE_COMPACTION_WIDTH.try_with(|width| *width) {
|
||||
return width;
|
||||
}
|
||||
DATA_SCANNER_FORCE_COMPACT_AT_FOLDERS
|
||||
}
|
||||
|
||||
/// Root-fanout limit for `DataUsageCache::force_compact` at the end of a sweep.
|
||||
///
|
||||
/// An interrupted sweep still has to seal its resume frontier, so it keeps the
|
||||
@@ -847,6 +860,7 @@ pub struct FolderScanner {
|
||||
resume_failed_paths: Vec<String>,
|
||||
coverage_gap: bool,
|
||||
raw_enumeration_progress: Vec<RawEnumerationProgress>,
|
||||
directory_checkpoints: HashMap<String, HashMap<String, directory_cursor::Saved>>,
|
||||
pending_size_reconciliation_keys: HashSet<String>,
|
||||
pending_size_reconciliation_scopes: HashSet<String>,
|
||||
pending_size_reconciliation_truncated: bool,
|
||||
@@ -866,6 +880,7 @@ struct RawEnumerationProgress {
|
||||
entries_seen: u64,
|
||||
digest: Sha256,
|
||||
page_index: Option<RawEnumerationPageWriter>,
|
||||
sealed_pages: Option<(RawEnumerationPageIndex, usize)>,
|
||||
}
|
||||
|
||||
impl RawEnumerationProgress {
|
||||
@@ -881,10 +896,34 @@ impl RawEnumerationProgress {
|
||||
entries_seen: 0,
|
||||
digest,
|
||||
page_index,
|
||||
sealed_pages: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn from_sealed(parent: &str, index: &RawEnumerationPageIndex) -> Option<Self> {
|
||||
use crate::raw_page_index::RawEnumerationPageOwnerStatus;
|
||||
let count = match index.status() {
|
||||
RawEnumerationPageOwnerStatus::Building { indexed_entries, .. }
|
||||
| RawEnumerationPageOwnerStatus::Ready { indexed_entries, .. } => usize::try_from(indexed_entries).ok()?,
|
||||
RawEnumerationPageOwnerStatus::Unsupported => return None,
|
||||
};
|
||||
if count == 0 {
|
||||
return None;
|
||||
}
|
||||
Some(Self {
|
||||
parent: parent.to_owned(),
|
||||
last_entry: None,
|
||||
entries_seen: 0,
|
||||
digest: Sha256::new(),
|
||||
page_index: None,
|
||||
sealed_pages: Some((index.clone(), count)),
|
||||
})
|
||||
}
|
||||
|
||||
fn record_entry(&mut self, entry: &str) {
|
||||
if let Some((index, _)) = self.sealed_pages.take() {
|
||||
*self = Self::new(&self.parent, Some(index));
|
||||
}
|
||||
update_raw_enumeration_digest(&mut self.digest, b"entry", entry.as_bytes());
|
||||
self.last_entry = Some(entry.to_string());
|
||||
self.entries_seen = self.entries_seen.saturating_add(1);
|
||||
@@ -908,6 +947,9 @@ impl RawEnumerationProgress {
|
||||
}
|
||||
|
||||
fn page_index(&self) -> Option<RawEnumerationPageIndex> {
|
||||
if let Some((index, _)) = &self.sealed_pages {
|
||||
return Some(index.clone());
|
||||
}
|
||||
self.page_index
|
||||
.as_ref()
|
||||
.filter(|index| index.indexed_entry_count() > 0)
|
||||
@@ -919,6 +961,9 @@ impl RawEnumerationProgress {
|
||||
}
|
||||
|
||||
fn checkpointable_entry_count(&self) -> usize {
|
||||
if let Some((_, count)) = &self.sealed_pages {
|
||||
return *count;
|
||||
}
|
||||
self.page_index
|
||||
.as_ref()
|
||||
.map_or(0, RawEnumerationPageWriter::indexed_entry_count)
|
||||
@@ -991,16 +1036,26 @@ async fn wait_for_scanner_io<T>(ctx: &CancellationToken, wait: impl std::future:
|
||||
}
|
||||
}
|
||||
|
||||
struct ScannerDirectory {
|
||||
pub(crate) use directory_cursor::parked_bucket_disks;
|
||||
use directory_cursor::{DirectoryScope, ScannerDirectory};
|
||||
|
||||
struct DirectoryStream {
|
||||
reader: native_io::ReadDir,
|
||||
batch_entries: usize,
|
||||
#[cfg(test)]
|
||||
ordered: Option<tests::erasure_probe::OrderedEntries>,
|
||||
}
|
||||
|
||||
impl ScannerDirectory {
|
||||
async fn open(path: &str) -> std::io::Result<Self> {
|
||||
let reader = native_io::ReadDir::open(path).await?;
|
||||
impl DirectoryStream {
|
||||
async fn open_at(path: &str, anchor: &native_io::DirectoryAnchor) -> std::io::Result<Self> {
|
||||
Ok(Self {
|
||||
reader: native_io::ReadDir::open_at(path, anchor).await?,
|
||||
#[cfg(test)]
|
||||
ordered: None,
|
||||
})
|
||||
}
|
||||
|
||||
async fn open(path: &str, indexed: bool) -> std::io::Result<Self> {
|
||||
let reader = native_io::ReadDir::open(path, indexed).await?;
|
||||
#[cfg(test)]
|
||||
let (reader, ordered) = {
|
||||
let mut reader = reader;
|
||||
@@ -1009,7 +1064,6 @@ impl ScannerDirectory {
|
||||
};
|
||||
Ok(Self {
|
||||
reader,
|
||||
batch_entries: 0,
|
||||
#[cfg(test)]
|
||||
ordered,
|
||||
})
|
||||
@@ -1034,11 +1088,14 @@ async fn next_scanner_dir_entry(
|
||||
wait_for_scanner_io(ctx, tokio::task::yield_now()).await?;
|
||||
entries.batch_entries = 0;
|
||||
}
|
||||
let entry = wait_for_scanner_io(ctx, run_scanner_node_operation(admission, ctx, entries.next_entry())).await???;
|
||||
if entry.is_some() {
|
||||
let entry = wait_for_scanner_io(ctx, run_scanner_node_operation(admission, ctx, entries.peek_entry())).await???;
|
||||
if entry.as_ref().is_some_and(|item| !item.is_recorded()) {
|
||||
// Tokio prefetches bounded chunks. Pacing consumption also paces the
|
||||
// next refill, without charging an extra entry for EOF.
|
||||
wait_for_scanner_io(ctx, sleeper.admit_disk_io(1, 0)).await?;
|
||||
}
|
||||
if entry.is_some() {
|
||||
entries.accept_entry();
|
||||
entries.batch_entries += 1;
|
||||
}
|
||||
Ok(entry)
|
||||
@@ -1056,7 +1113,7 @@ impl FolderScanner {
|
||||
&self,
|
||||
folder: &CachedFolder,
|
||||
into: &DataUsageEntry,
|
||||
child: &CachedFolder,
|
||||
child_name: &str,
|
||||
child_hash: &DataUsageHash,
|
||||
abandoned_children: &DataUsageHashMap,
|
||||
) -> bool {
|
||||
@@ -1076,8 +1133,7 @@ impl FolderScanner {
|
||||
return false;
|
||||
}
|
||||
|
||||
let Some(entry) = child
|
||||
.name
|
||||
let Some(entry) = child_name
|
||||
.strip_prefix(folder.name.as_str())
|
||||
.and_then(|entry| entry.strip_prefix('/'))
|
||||
else {
|
||||
@@ -1086,6 +1142,21 @@ impl FolderScanner {
|
||||
!entry.is_empty() && !entry.contains('/') && !prefix_scan_scope.contains(entry)
|
||||
}
|
||||
|
||||
fn can_reuse_scoped_child(&self, folder: &CachedFolder, into: &DataUsageEntry, name: &str, hash: &DataUsageHash) -> bool {
|
||||
self.prefix_scan_scope.is_some()
|
||||
&& self
|
||||
.old_cache
|
||||
.find(&folder.name)
|
||||
.is_some_and(|parent| self.should_reuse_clean_root_child(folder, into, name, hash, &parent.children))
|
||||
}
|
||||
|
||||
fn can_restore_linked_child(&self, folder: &CachedFolder, into: &DataUsageEntry, name: &str) -> bool {
|
||||
self.can_reuse_covered_namespace(name, false)
|
||||
|| (self.prefix_scan_scope.is_some()
|
||||
&& self.old_cache.find(name).is_some()
|
||||
&& self.can_reuse_scoped_child(folder, into, name, &hash_path(name)))
|
||||
}
|
||||
|
||||
fn reuse_clean_root_child(&mut self, child: &CachedFolder, child_hash: &DataUsageHash, into: &mut DataUsageEntry) {
|
||||
self.new_cache.copy_with_children(&self.old_cache, child_hash, &child.parent);
|
||||
if let Some(updates) = &mut self.usage_updates {
|
||||
@@ -1322,7 +1393,12 @@ impl FolderScanner {
|
||||
|
||||
/// Records one raw entry. Returns whether a durable index holds it, which
|
||||
/// is what makes stopping at this entry recoverable.
|
||||
#[cfg(test)]
|
||||
fn record_raw_enumeration_entry(&mut self, parent: &str, entry: &str) -> bool {
|
||||
self.record_raw_enumeration_entry_with_stream(parent, entry, false)
|
||||
}
|
||||
|
||||
fn record_raw_enumeration_entry_with_stream(&mut self, parent: &str, entry: &str, continues_stream: bool) -> bool {
|
||||
if self.old_cache.info.scan_progress.is_none() {
|
||||
return false;
|
||||
}
|
||||
@@ -1334,8 +1410,11 @@ impl FolderScanner {
|
||||
self.raw_enumeration_progress.truncate(position + 1);
|
||||
} else {
|
||||
let page_index = self.raw_enumeration_page_index_for_parent(parent).map(clone_cache_value);
|
||||
self.raw_enumeration_progress
|
||||
.push(RawEnumerationProgress::new(parent, page_index));
|
||||
let mut progress = RawEnumerationProgress::new(parent, page_index);
|
||||
if continues_stream && let Some(writer) = &mut progress.page_index {
|
||||
writer.continue_native_stream();
|
||||
}
|
||||
self.raw_enumeration_progress.push(progress);
|
||||
}
|
||||
self.raw_enumeration_progress.last_mut().is_some_and(|progress| {
|
||||
progress.record_entry(entry);
|
||||
@@ -1402,6 +1481,17 @@ impl FolderScanner {
|
||||
.map(|(index, _)| index)
|
||||
}
|
||||
|
||||
fn saved_directories(&self) -> Vec<DirectoryCheckpoint> {
|
||||
let mut snapshots: Vec<_> = self
|
||||
.directory_checkpoints
|
||||
.values()
|
||||
.flat_map(HashMap::values)
|
||||
.filter_map(|slot| slot.lock().unwrap_or_else(|p| p.into_inner()).clone())
|
||||
.collect();
|
||||
snapshots.sort_by(|left, right| (left.parent(), left.endpoint()).cmp(&(right.parent(), right.endpoint())));
|
||||
snapshots
|
||||
}
|
||||
|
||||
/// The walk path except the directory the resume state selects.
|
||||
fn raw_enumeration_path_page_indexes(&self) -> Vec<RawEnumerationPageIndex> {
|
||||
let selected = self.selected_raw_enumeration_progress();
|
||||
@@ -1453,9 +1543,22 @@ impl FolderScanner {
|
||||
}
|
||||
|
||||
fn maybe_send_checkpoint(&mut self, folder: &CachedFolder, hash: &DataUsageHash, into: &DataUsageEntry) {
|
||||
self.maybe_send_owned_checkpoint(folder, hash, into, None);
|
||||
}
|
||||
|
||||
// A compact accumulator is checkpointable only at the boundary where
|
||||
// its owning frame has accepted a complete child. Descendant frames may
|
||||
// temporarily share the same accumulator and cannot seal it themselves.
|
||||
fn maybe_send_owned_checkpoint(
|
||||
&mut self,
|
||||
folder: &CachedFolder,
|
||||
hash: &DataUsageHash,
|
||||
into: &DataUsageEntry,
|
||||
work: Option<&directory_cursor::DirectoryWork>,
|
||||
) {
|
||||
let Some(checkpoint_tx) = self.checkpoint_tx.clone() else { return };
|
||||
let elapsed = self.last_checkpoint_at.elapsed();
|
||||
if into.compacted
|
||||
if (into.compacted && work.is_none())
|
||||
|| self.new_cache.info.scan_progress.is_none()
|
||||
|| elapsed < checkpoint_interval_for_build_time(self.last_checkpoint_build_time)
|
||||
|| (self.checkpoint_objects.saturating_sub(self.last_checkpoint_objects) < SCANNER_CHECKPOINT_OBJECT_INTERVAL
|
||||
@@ -1463,7 +1566,7 @@ impl FolderScanner {
|
||||
{
|
||||
return;
|
||||
}
|
||||
if self.new_cache.root().is_none() && self.raw_enumeration_progress.is_empty() {
|
||||
if self.new_cache.root().is_none() && self.raw_enumeration_progress.is_empty() && work.is_none() {
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -1479,6 +1582,9 @@ impl FolderScanner {
|
||||
// Invalid partial graphs must not trigger a fresh O(N) clone on every object.
|
||||
self.last_checkpoint_objects = self.checkpoint_objects;
|
||||
self.last_checkpoint_at = Instant::now();
|
||||
if let Some(work) = work {
|
||||
work.checkpoint();
|
||||
}
|
||||
let snapshot = self.checkpoint_snapshot(folder, hash, into);
|
||||
self.last_checkpoint_build_time = self.last_checkpoint_at.elapsed();
|
||||
if let Some(snapshot) = snapshot {
|
||||
@@ -1508,6 +1614,7 @@ impl FolderScanner {
|
||||
snapshot.checked_flatten_complete_scope(&snapshot.info.name)?;
|
||||
snapshot.info.last_update = Some(SystemTime::now());
|
||||
snapshot.info.snapshot_complete = false;
|
||||
snapshot.info.scan_directory_work = self.saved_directories();
|
||||
let (cursor, page_index) = self.peek_raw_enumeration_resume_state();
|
||||
snapshot.info.scan_raw_enumeration_path_page_indexes = self.raw_enumeration_path_page_indexes();
|
||||
snapshot.info.scan_raw_enumeration_cursor = cursor;
|
||||
@@ -1515,13 +1622,24 @@ impl FolderScanner {
|
||||
snapshot.seal_scan_frontier(self.coverage_frontier.as_deref()).ok()?;
|
||||
// A resumed readdir can run before covered subtrees are copied. Keep
|
||||
// its durable receipt until this snapshot can represent that coverage.
|
||||
if self.coverage_frontier.is_some() && snapshot.info.scan_coverage_receipt.is_none() {
|
||||
let compacted_work = snapshot
|
||||
.info
|
||||
.scan_directory_work
|
||||
.iter()
|
||||
.filter_map(|directory| directory.progress(&snapshot.info));
|
||||
if self
|
||||
.coverage_frontier
|
||||
.as_deref()
|
||||
.is_some_and(|frontier| !compacted_work.clone().any(|proof| proof.covers(frontier)))
|
||||
&& snapshot.info.scan_coverage_receipt.is_none()
|
||||
{
|
||||
return None;
|
||||
}
|
||||
// The receipt was just sealed from this snapshot; do not hash it twice.
|
||||
if snapshot.info.scan_raw_enumeration_cursor.is_none()
|
||||
&& snapshot.info.scan_raw_enumeration_page_index.is_none()
|
||||
&& snapshot.info.scan_coverage_receipt.is_none()
|
||||
&& snapshot.info.scan_directory_work.is_empty()
|
||||
{
|
||||
return None;
|
||||
}
|
||||
@@ -1865,7 +1983,15 @@ impl FolderScanner {
|
||||
folder: CachedFolder,
|
||||
into: &mut DataUsageEntry,
|
||||
) -> Result<(), ScannerError> {
|
||||
self.scan_folder_inner(ctx, folder, into, FolderScanMode::Namespace, false)
|
||||
if self.old_cache.info.scan_directory_source.is_none()
|
||||
&& self.old_cache.info.scan_identity.is_some()
|
||||
&& self.old_cache.info.scan_progress.is_some()
|
||||
{
|
||||
let source = wait_for_scanner_io(&ctx, directory_cursor::physical_source(&self.local_disk)).await??;
|
||||
self.old_cache.info.scan_directory_source = Some(source.clone());
|
||||
self.new_cache.info.scan_directory_source = Some(source);
|
||||
}
|
||||
Box::pin(self.scan_folder_inner(ctx, folder, into, FolderScanMode::Namespace, false))
|
||||
.await
|
||||
.map(|_| ())
|
||||
}
|
||||
@@ -1972,9 +2098,43 @@ impl FolderScanner {
|
||||
// directory that vanished during iteration instead of failing the
|
||||
// whole parent subtree. The bucket root (no parent) still propagates
|
||||
// NotFound so namespace removal is detected by the caller.
|
||||
// A small object directory normally finishes in one call. Allocate
|
||||
// listing state at the bucket root or after an interruption has
|
||||
// actually left this directory on the saved traversal path.
|
||||
let info = &self.old_cache.info;
|
||||
let previously_interrupted = info
|
||||
.scan_raw_enumeration_cursor
|
||||
.as_ref()
|
||||
.is_some_and(|cursor| cursor.parent == folder.name)
|
||||
|| info
|
||||
.scan_raw_enumeration_page_index
|
||||
.iter()
|
||||
.chain(&info.scan_raw_enumeration_path_page_indexes)
|
||||
.any(|index| index.parent() == Some(folder.name.as_str()));
|
||||
let mut directory_scope = (folder.parent.is_none()
|
||||
|| previously_interrupted
|
||||
|| self.directory_checkpoints.contains_key(&folder.name)
|
||||
|| directory_cursor::is_parked(&dir_path))
|
||||
.then(|| DirectoryScope::new(&self.local_disk, &self.old_cache, mode, self.prefix_scan_scope.clone()))
|
||||
.flatten();
|
||||
if let Some(scope) = &mut directory_scope {
|
||||
scope.bind_source(self.old_cache.info.scan_directory_source.as_deref());
|
||||
}
|
||||
let saved = directory_scope.as_ref().map(|owner| {
|
||||
self.directory_checkpoints
|
||||
.entry(folder.name.clone())
|
||||
.or_default()
|
||||
.entry(owner.endpoint().to_owned())
|
||||
.or_default()
|
||||
.clone()
|
||||
});
|
||||
let mut dir_reader = match wait_for_scanner_io(
|
||||
&ctx,
|
||||
run_scanner_node_operation(self.node_admission.as_ref(), &ctx, ScannerDirectory::open(&dir_path)),
|
||||
run_scanner_node_operation(
|
||||
self.node_admission.as_ref(),
|
||||
&ctx,
|
||||
ScannerDirectory::open(&dir_path, &folder.name, directory_scope, self.budget.clone(), saved),
|
||||
),
|
||||
)
|
||||
.await??
|
||||
{
|
||||
@@ -1997,16 +2157,48 @@ impl FolderScanner {
|
||||
if mode != FolderScanMode::Namespace {
|
||||
self.coverage_gap |= e.kind() == ErrorKind::NotADirectory && self.old_cache.info.scan_progress.is_some();
|
||||
done_folder();
|
||||
self.directory_checkpoints.remove(&folder.name);
|
||||
return Ok(FolderScanOutcome::Vanished);
|
||||
}
|
||||
None
|
||||
}
|
||||
Err(e) => return Err(ScannerError::Io(e)),
|
||||
};
|
||||
if dir_reader.as_ref().is_some_and(|reader| reader.directory_replaced) && self.resume_frontier.is_some() {
|
||||
// Names from a replaced directory cannot validate the saved
|
||||
// child prefix. Finish this walk only as an observation.
|
||||
self.coverage_gap = true;
|
||||
self.resume_frontier = None;
|
||||
}
|
||||
let mut pending_entry_progress = 0_u64;
|
||||
let mut last_entry_progress = Instant::now();
|
||||
if let Some(reader) = &mut dir_reader {
|
||||
reader.hold_abandoned(&mut abandoned_children);
|
||||
}
|
||||
let indexed = dir_reader.as_ref().is_some_and(ScannerDirectory::indexed);
|
||||
let mut raw_enumeration_complete = false;
|
||||
let raw_enumeration_committed_entries = self.raw_enumeration_committed_entry_oracle(&folder.name);
|
||||
let continues_stream = dir_reader.as_ref().is_some_and(ScannerDirectory::continues_stream);
|
||||
// A sealed work queue only emits the two classification markers;
|
||||
// it never needs the raw-name membership table again.
|
||||
let raw_enumeration_committed_entries = if dir_reader.as_ref().is_some_and(ScannerDirectory::finished_listing) {
|
||||
// A later quantum can stop in a child. Its durable proof
|
||||
// still needs every ancestor's prior immutable pages, even
|
||||
// though this cursor need not enumerate those names again.
|
||||
if let Some(progress) = self
|
||||
.old_cache
|
||||
.info
|
||||
.scan_raw_enumeration_page_index
|
||||
.iter()
|
||||
.chain(&self.old_cache.info.scan_raw_enumeration_path_page_indexes)
|
||||
.find(|index| index.parent() == Some(folder.name.as_str()))
|
||||
.and_then(|index| RawEnumerationProgress::from_sealed(&folder.name, index))
|
||||
{
|
||||
self.raw_enumeration_progress.push(progress);
|
||||
}
|
||||
HashSet::new()
|
||||
} else {
|
||||
self.raw_enumeration_committed_entry_oracle(&folder.name)
|
||||
};
|
||||
let raw_replay_floor = u64::try_from(raw_enumeration_committed_entries.len()).unwrap_or(u64::MAX);
|
||||
let mut raw_entries_observed = 0_u64;
|
||||
let mut raw_entries_charged = 0_u64;
|
||||
@@ -2017,12 +2209,11 @@ impl FolderScanner {
|
||||
loop {
|
||||
// Hold at most one metadata entry until a candidate reaches
|
||||
// EOF: a later part must not leave object accounting or ILM work.
|
||||
let (entry, replayed_metadata) = if let Some(reader) = dir_reader.as_mut() {
|
||||
let (entry, replayed_metadata) = if let Some(reader) = dir_reader.as_mut().filter(|_| !raw_enumeration_complete) {
|
||||
let entry = match next_scanner_dir_entry(&ctx, &self.sleeper, reader, self.node_admission.as_ref()).await {
|
||||
Ok(Some(entry)) => entry,
|
||||
Ok(None) => {
|
||||
raw_enumeration_complete = true;
|
||||
dir_reader = None;
|
||||
continue;
|
||||
}
|
||||
Err(ScannerError::Io(e))
|
||||
@@ -2042,7 +2233,6 @@ impl FolderScanner {
|
||||
"Scanner folder state updated"
|
||||
);
|
||||
raw_enumeration_complete = true;
|
||||
dir_reader = None;
|
||||
continue;
|
||||
}
|
||||
Err(ScannerError::Io(e)) if e.kind() == ErrorKind::NotADirectory => {
|
||||
@@ -2057,12 +2247,12 @@ impl FolderScanner {
|
||||
"Scanner folder state updated"
|
||||
);
|
||||
raw_enumeration_complete = true;
|
||||
dir_reader = None;
|
||||
continue;
|
||||
}
|
||||
Err(e) => return Err(e),
|
||||
};
|
||||
(entry, false)
|
||||
let marker = entry.is_recorded();
|
||||
(entry, marker)
|
||||
} else {
|
||||
raw_enumeration_complete = true;
|
||||
if candidate_unclassified {
|
||||
@@ -2081,6 +2271,7 @@ impl FolderScanner {
|
||||
self.budget.record_entries_visited(pending_entry_progress);
|
||||
self.finish_raw_enumeration_parent(&folder.name);
|
||||
done_folder();
|
||||
self.directory_checkpoints.remove(&folder.name);
|
||||
return Ok(FolderScanOutcome::Namespace);
|
||||
}
|
||||
candidate_unclassified = false;
|
||||
@@ -2094,7 +2285,11 @@ impl FolderScanner {
|
||||
..
|
||||
}
|
||||
) && deferred_metadata.is_none()
|
||||
&& queued_folders.is_empty();
|
||||
&& dir_reader
|
||||
.as_ref()
|
||||
.and_then(ScannerDirectory::folder_count)
|
||||
.unwrap_or(queued_folders.len())
|
||||
== 0;
|
||||
if !empty_object_child {
|
||||
if !self.budget.try_start_directory() {
|
||||
return Err(ScannerError::Other("Operation cancelled".to_string()));
|
||||
@@ -2114,21 +2309,25 @@ impl FolderScanner {
|
||||
// its old size, so charging only that tail keeps each
|
||||
// budget unit durable even if the listing order changed.
|
||||
raw_entries_observed = raw_entries_observed.saturating_add(1);
|
||||
let raw_entry_is_new =
|
||||
raw_entries_observed > raw_replay_floor && !raw_enumeration_committed_entries.contains(&file_name);
|
||||
let raw_entry_is_new = (continues_stream || raw_entries_observed > raw_replay_floor)
|
||||
&& !raw_enumeration_committed_entries.contains(&file_name);
|
||||
#[cfg(test)]
|
||||
if raw_entry_is_new {
|
||||
#[cfg(test)]
|
||||
tests::enumeration_restart::observe_raw_entry(&dir_path, &entry.file_name(), &self.budget);
|
||||
pending_entry_progress = pending_entry_progress.saturating_add(1);
|
||||
if pending_entry_progress >= SCANNER_ENTRY_PROGRESS_BATCH
|
||||
|| last_entry_progress.elapsed() >= SCANNER_ENTRY_PROGRESS_INTERVAL
|
||||
{
|
||||
self.budget.record_entries_visited(pending_entry_progress);
|
||||
pending_entry_progress = 0;
|
||||
last_entry_progress = Instant::now();
|
||||
}
|
||||
}
|
||||
let raw_entry_is_durable = self.record_raw_enumeration_entry(&folder.name, &file_name);
|
||||
// Physical discovery is progress even when a changed directory
|
||||
// must replay names retained by its logical page index. Budget
|
||||
// admission below still charges only newly durable entries.
|
||||
pending_entry_progress = pending_entry_progress.saturating_add(1);
|
||||
if pending_entry_progress >= SCANNER_ENTRY_PROGRESS_BATCH
|
||||
|| last_entry_progress.elapsed() >= SCANNER_ENTRY_PROGRESS_INTERVAL
|
||||
{
|
||||
self.budget.record_entries_visited(pending_entry_progress);
|
||||
pending_entry_progress = 0;
|
||||
last_entry_progress = Instant::now();
|
||||
}
|
||||
let raw_entry_is_durable =
|
||||
self.record_raw_enumeration_entry_with_stream(&folder.name, &file_name, continues_stream);
|
||||
// Object metadata and the entries behind it are charged to
|
||||
// the object budget; this one covers listing work.
|
||||
let object_entry = file_name == STORAGE_FORMAT_FILE || deferred_metadata.is_some();
|
||||
@@ -2142,6 +2341,9 @@ impl FolderScanner {
|
||||
}
|
||||
self.maybe_send_checkpoint(&folder, &this_hash, into);
|
||||
}
|
||||
if !replayed_metadata && dir_reader.as_ref().is_some_and(|reader| reader.enumeration_only) {
|
||||
continue;
|
||||
}
|
||||
let is_storage_format_entry = file_name == STORAGE_FORMAT_FILE;
|
||||
if candidate_unclassified && is_storage_format_entry {
|
||||
deferred_metadata = Some(entry);
|
||||
@@ -2160,12 +2362,15 @@ impl FolderScanner {
|
||||
|
||||
// Ignore entries that disappeared during traversal or hit symlink
|
||||
// loops, but propagate other walk errors.
|
||||
let mut entry_type = match wait_for_scanner_io(
|
||||
&ctx,
|
||||
run_scanner_node_operation(self.node_admission.as_ref(), &ctx, entry.file_type()),
|
||||
)
|
||||
.await??
|
||||
let file_type = if entry.is_recorded()
|
||||
&& let Some(kind) = entry.cached_type()
|
||||
{
|
||||
Ok(kind)
|
||||
} else {
|
||||
wait_for_scanner_io(&ctx, run_scanner_node_operation(self.node_admission.as_ref(), &ctx, entry.file_type()))
|
||||
.await??
|
||||
};
|
||||
let mut entry_type = match file_type {
|
||||
Ok(entry_type) => entry_type,
|
||||
Err(e) if e.kind() == ErrorKind::NotFound => {
|
||||
debug!(
|
||||
@@ -2202,6 +2407,7 @@ impl FolderScanner {
|
||||
// namespace coverage. Discard its provisional resume owner.
|
||||
self.finish_raw_enumeration_parent(&folder.name);
|
||||
done_folder();
|
||||
self.directory_checkpoints.remove(&folder.name);
|
||||
return Ok(FolderScanOutcome::ErasureData);
|
||||
}
|
||||
|
||||
@@ -2302,7 +2508,7 @@ impl FolderScanner {
|
||||
object_heal_prob_div: folder.object_heal_prob_div,
|
||||
};
|
||||
|
||||
if exists && self.should_reuse_clean_root_child(&folder, into, &this, &h, &abandoned_children) {
|
||||
if exists && self.should_reuse_clean_root_child(&folder, into, &this.name, &h, &abandoned_children) {
|
||||
abandoned_children.remove(&clone_cache_value(&h.0));
|
||||
self.reuse_clean_root_child(&this, &h, into);
|
||||
} else if exists {
|
||||
@@ -2528,7 +2734,11 @@ impl FolderScanner {
|
||||
|
||||
item.transform_meta_dir();
|
||||
|
||||
abandoned_children.remove(&path_join_buf(&[&item.bucket, &item.object_path()]));
|
||||
let resolved = path_join_buf(&[&item.bucket, &item.object_path()]);
|
||||
abandoned_children.remove(&resolved);
|
||||
if let Some(reader) = &mut dir_reader {
|
||||
reader.resolve_abandoned(&resolved);
|
||||
}
|
||||
|
||||
apply_scanner_size_summary(into, &sz);
|
||||
if !sz.size_reconciliation.is_empty() {
|
||||
@@ -2562,7 +2772,19 @@ impl FolderScanner {
|
||||
self.budget.record_entries_visited(pending_entry_progress);
|
||||
// A directory with queued children keeps its index until they end:
|
||||
// a restart replays this listing to reach the unfinished child.
|
||||
if raw_enumeration_complete && queued_folders.is_empty() {
|
||||
if indexed && dir_reader.as_ref().is_some_and(|reader| !reader.finished_listing()) {
|
||||
return Err(ScannerError::Io(std::io::Error::new(
|
||||
ErrorKind::Interrupted,
|
||||
"scanner directory changed before listing completed",
|
||||
)));
|
||||
}
|
||||
if raw_enumeration_complete
|
||||
&& dir_reader
|
||||
.as_ref()
|
||||
.and_then(ScannerDirectory::folder_count)
|
||||
.unwrap_or(queued_folders.len())
|
||||
== 0
|
||||
{
|
||||
self.finish_raw_enumeration_parent(&folder.name);
|
||||
}
|
||||
|
||||
@@ -2576,8 +2798,18 @@ impl FolderScanner {
|
||||
// the emergency 250k ceiling. Keep FORCE as the absolute backstop.
|
||||
let is_scan_root = folder.name == self.old_cache.info.name;
|
||||
let forward_sweep = self.old_cache.info.scan_progress.is_some();
|
||||
let total_folders = queued_folders.len();
|
||||
let force_compact = total_folders >= DATA_SCANNER_FORCE_COMPACT_AT_FOLDERS;
|
||||
if let Some(reader) = &dir_reader {
|
||||
reader.restore_abandoned(&mut abandoned_children);
|
||||
}
|
||||
let total_folders = dir_reader
|
||||
.as_ref()
|
||||
.and_then(ScannerDirectory::folder_count)
|
||||
.unwrap_or(queued_folders.len());
|
||||
// A completed root above the normal cache ceiling is compacted by
|
||||
// the existing completion path. A resumable sorted listing can use
|
||||
// that representation throughout processing without losing its place.
|
||||
let emergency_compact = total_folders >= force_compaction_width();
|
||||
let force_compact = emergency_compact || (indexed && total_folders > DATA_SCANNER_COMPACT_AT_CHILDREN);
|
||||
// Reused root children still have links in the new cache.
|
||||
let should_compact = force_compact
|
||||
|| (!forward_sweep
|
||||
@@ -2586,7 +2818,7 @@ impl FolderScanner {
|
||||
// Preserve the ordinary namespace walk's child isolation when the
|
||||
// root starts sharing one accumulator across its descendants.
|
||||
let isolate_compacted_children =
|
||||
isolate_compacted_namespace_errors || (is_scan_root && !into.compacted && should_compact && !force_compact);
|
||||
isolate_compacted_namespace_errors || (is_scan_root && !into.compacted && should_compact && !emergency_compact);
|
||||
|
||||
self.alert_excessive_folders(&folder.name, total_folders);
|
||||
|
||||
@@ -2603,7 +2835,7 @@ impl FolderScanner {
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_FOLDER,
|
||||
folder = %folder.name,
|
||||
entry_count = queued_folders.len(),
|
||||
entry_count = total_folders,
|
||||
state = "preemptive_compaction",
|
||||
"Scanner folder switched to compacted mode"
|
||||
);
|
||||
@@ -2632,7 +2864,7 @@ impl FolderScanner {
|
||||
.and_then(|resume_after| folder_resume_match(&folder.name, resume_after))
|
||||
.is_some_and(|resume_match| matches!(resume_match, FolderResumeMatch::Descendant));
|
||||
let scan_resume_after = checkpoint_resume_after.or(self.old_cache.info.scan_resume_after.as_deref());
|
||||
let has_queued_folders = !queued_folders.is_empty();
|
||||
let has_queued_folders = total_folders > 0;
|
||||
let resume_order = if forward_sweep {
|
||||
order_queued_folders_for_resume(&mut queued_folders, None)
|
||||
} else {
|
||||
@@ -2646,14 +2878,95 @@ impl FolderScanner {
|
||||
}
|
||||
}
|
||||
|
||||
// Scan child folders in the combined resume order.
|
||||
for queued_folder in queued_folders {
|
||||
let mut work = dir_reader.as_mut().and_then(ScannerDirectory::work);
|
||||
if let Some(work) = &mut work {
|
||||
work.restore_mode(into.compacted);
|
||||
// A saved prefix tree must actually contain the contributions
|
||||
// the parked queue has consumed. A failed cache save can leave
|
||||
// this process's classified names ahead of its durable values.
|
||||
if !into.compacted
|
||||
&& !work
|
||||
.linked
|
||||
.iter()
|
||||
.all(|name| self.can_restore_linked_child(&folder, into, name))
|
||||
{
|
||||
work.rewind();
|
||||
}
|
||||
self.coverage_gap |= work.gap;
|
||||
if into.compacted {
|
||||
into.merge(&work.compacted_total);
|
||||
} else {
|
||||
for name in &work.linked {
|
||||
let child = hash_path(name);
|
||||
self.new_cache
|
||||
.copy_with_children(&self.old_cache, &child, &Some(this_hash.clone()));
|
||||
into.add_child(&child);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut queued_folders = queued_folders.into_iter();
|
||||
let mut acknowledge_previous: Option<(String, bool)> = None;
|
||||
loop {
|
||||
if let Some(work) = &mut work
|
||||
&& let Some((name, linked)) = acknowledge_previous.take()
|
||||
{
|
||||
work.gap |= self.coverage_gap;
|
||||
work.acknowledge(&name, linked, into.compacted);
|
||||
}
|
||||
if ctx.is_cancelled() {
|
||||
return Err(ScannerError::Other("Operation cancelled".to_string()));
|
||||
}
|
||||
|
||||
let queued_folder = if let Some(work) = &mut work {
|
||||
let Some(name) = wait_for_scanner_io(
|
||||
&ctx,
|
||||
run_scanner_node_operation(self.node_admission.as_ref(), &ctx, work.next_name()),
|
||||
)
|
||||
.await???
|
||||
else {
|
||||
break;
|
||||
};
|
||||
let name = path_join_buf(&[&folder.name, &name]);
|
||||
let source = if !force_compact && self.old_cache.find(&name).is_some() {
|
||||
FolderScanSource::Existing
|
||||
} else {
|
||||
FolderScanSource::New
|
||||
};
|
||||
let h = hash_path(&name);
|
||||
if source == FolderScanSource::Existing
|
||||
&& let Some(updates) = &mut self.usage_updates
|
||||
{
|
||||
updates
|
||||
.cache
|
||||
.copy_with_children(&self.old_cache, &h, &Some(this_hash.clone()));
|
||||
}
|
||||
QueuedFolder {
|
||||
folder: CachedFolder {
|
||||
name,
|
||||
parent: Some(this_hash.clone()),
|
||||
object_heal_prob_div: folder.object_heal_prob_div,
|
||||
},
|
||||
source,
|
||||
}
|
||||
} else {
|
||||
let Some(queued) = queued_folders.next() else { break };
|
||||
queued
|
||||
};
|
||||
let mut folder_item = queued_folder.folder;
|
||||
let h = hash_path(&folder_item.name);
|
||||
if work.is_some() {
|
||||
acknowledge_previous = Some((folder_item.name.clone(), false));
|
||||
}
|
||||
if work.is_some()
|
||||
&& queued_folder.source == FolderScanSource::Existing
|
||||
&& self.can_reuse_scoped_child(&folder, into, &folder_item.name, &h)
|
||||
{
|
||||
self.reuse_clean_root_child(&folder_item, &h, into);
|
||||
if let Some((_, linked)) = &mut acknowledge_previous {
|
||||
*linked = true;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
let reuse_covered = self.can_reuse_covered_namespace(&folder_item.name, into.compacted);
|
||||
let mut child_mode = if self.is_erasure_mode && is_erasure_candidate(&folder_item.name) {
|
||||
FolderScanMode::ErasureCandidate {
|
||||
@@ -2666,6 +2979,9 @@ impl FolderScanner {
|
||||
if child_mode == FolderScanMode::Namespace && reuse_covered {
|
||||
self.new_cache.copy_with_children(&self.old_cache, &h, &folder_item.parent);
|
||||
into.add_child(&h);
|
||||
if let Some((_, linked)) = &mut acknowledge_previous {
|
||||
*linked = true;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -2744,12 +3060,16 @@ impl FolderScanner {
|
||||
// rolling back totals already committed by healthy siblings.
|
||||
let isolate_compacted_child =
|
||||
into.compacted && isolate_compacted_children && child_mode == FolderScanMode::Namespace;
|
||||
let outcome = if into.compacted && !isolate_compacted_child {
|
||||
// A parked queue owns only committed subtree totals. Run the
|
||||
// active child in its own accumulator so cancellation cannot
|
||||
// count a partial child twice at the next admission.
|
||||
let child_summary = isolate_compacted_child || (into.compacted && work.is_some());
|
||||
let outcome = if into.compacted && !child_summary {
|
||||
// Keep the shared-accumulator path free of child summaries.
|
||||
Box::pin(self.scan_folder_inner(ctx.clone(), folder_item.clone(), into, child_mode, false)).await?
|
||||
} else {
|
||||
let mut dst = DataUsageEntry {
|
||||
compacted: isolate_compacted_child,
|
||||
compacted: child_summary,
|
||||
..Default::default()
|
||||
};
|
||||
// Provisional child usage must not escape through progress
|
||||
@@ -2759,8 +3079,15 @@ impl FolderScanner {
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let result = if isolate_compacted_child {
|
||||
Box::pin(self.scan_folder_inner(ctx.clone(), folder_item.clone(), &mut dst, child_mode, true)).await
|
||||
let result = if child_summary {
|
||||
Box::pin(self.scan_folder_inner(
|
||||
ctx.clone(),
|
||||
folder_item.clone(),
|
||||
&mut dst,
|
||||
child_mode,
|
||||
isolate_compacted_child,
|
||||
))
|
||||
.await
|
||||
} else {
|
||||
self.scan_child_folder(ctx.clone(), folder_item.clone(), &mut this_hash, into, &mut dst, child_mode)
|
||||
.await
|
||||
@@ -2770,14 +3097,22 @@ impl FolderScanner {
|
||||
}
|
||||
match result {
|
||||
Ok(outcome) => {
|
||||
if isolate_compacted_child {
|
||||
if child_summary {
|
||||
into.merge(&dst);
|
||||
if let Some(work) = &mut work {
|
||||
if outcome == FolderScanOutcome::Namespace && !self.coverage_gap && dst.failed_objects == 0 {
|
||||
work.compacted_total.merge(&dst);
|
||||
work.acknowledge(&folder_item.name, false, true);
|
||||
} else if dst.failed_objects != 0 {
|
||||
self.coverage_gap = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
outcome
|
||||
}
|
||||
Err(e) => {
|
||||
if ctx.is_cancelled() || matches!(&e, ScannerError::ResourceDeferred) {
|
||||
if isolate_compacted_child {
|
||||
if child_summary {
|
||||
// Cancellation and resource deferral keep completed
|
||||
// work even though the child has not finished.
|
||||
into.merge(&dst);
|
||||
@@ -2787,7 +3122,7 @@ impl FolderScanner {
|
||||
}
|
||||
return Err(e);
|
||||
}
|
||||
if child_mode != FolderScanMode::Namespace {
|
||||
if child_mode != FolderScanMode::Namespace || (into.compacted && !isolate_compacted_child) {
|
||||
return Err(e);
|
||||
}
|
||||
warn!(
|
||||
@@ -2844,10 +3179,16 @@ impl FolderScanner {
|
||||
}
|
||||
if into.compacted {
|
||||
self.record_completed_child(&folder_item.name);
|
||||
if !was_compacted && work.is_some() {
|
||||
self.maybe_send_owned_checkpoint(&folder, &this_hash, into, work.as_deref());
|
||||
}
|
||||
self.send_update_for_entry(&this_hash, &folder.parent, into).await;
|
||||
tokio::task::yield_now().await;
|
||||
} else {
|
||||
into.add_child(&h);
|
||||
if let Some(work) = &mut work {
|
||||
work.acknowledge(&folder_item.name, true, false);
|
||||
}
|
||||
self.record_completed_child(&folder_item.name);
|
||||
tokio::task::yield_now().await;
|
||||
self.maybe_send_checkpoint(&folder, &this_hash, into);
|
||||
@@ -2856,6 +3197,14 @@ impl FolderScanner {
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(reader) = &mut dir_reader {
|
||||
wait_for_scanner_io(
|
||||
&ctx,
|
||||
run_scanner_node_operation(self.node_admission.as_ref(), &ctx, reader.verify_current()),
|
||||
)
|
||||
.await???;
|
||||
}
|
||||
|
||||
// Scan for healing
|
||||
if abandoned_children.is_empty() || !self.should_heal().await {
|
||||
debug!(
|
||||
@@ -3334,6 +3683,7 @@ impl FolderScanner {
|
||||
let scanned_objects = u64::try_from(into.objects).unwrap_or(u64::MAX);
|
||||
emit_scanner_folder_trace(&self.root, &folder.name, scanned_objects, trace_started_at, "completed");
|
||||
|
||||
self.directory_checkpoints.remove(&folder.name);
|
||||
Ok(FolderScanOutcome::Namespace)
|
||||
}
|
||||
|
||||
@@ -3367,7 +3717,7 @@ pub(crate) async fn scan_data_folder_scoped(
|
||||
budget: Arc<ScannerCycleBudget>,
|
||||
disks: Vec<Arc<Disk>>,
|
||||
local_disk: Arc<Disk>,
|
||||
cache: DataUsageCache,
|
||||
mut cache: DataUsageCache,
|
||||
updates: Option<mpsc::Sender<DataUsageEntry>>,
|
||||
scan_mode: HealScanMode,
|
||||
sleeper: DynamicSleeper,
|
||||
@@ -3381,6 +3731,19 @@ pub(crate) async fn scan_data_folder_scoped(
|
||||
return Err(ScannerError::Other("internal error: root scan attempted".to_string()));
|
||||
}
|
||||
|
||||
// URL endpoints include the storage host. A formatted disk also contributes
|
||||
// its persistent UUID, fencing replacements and clone-like device/inode IDs.
|
||||
if cache.info.scan_progress.is_some() && cache.info.scan_identity.is_some() {
|
||||
use crate::storage_api::owner::EcstoreDiskAPI;
|
||||
let endpoint = local_disk.endpoint().to_string();
|
||||
let source = wait_for_scanner_io(&ctx, directory_cursor::physical_source(&local_disk)).await??;
|
||||
let mut endpoints: HashSet<_> = disks.iter().map(|disk| disk.endpoint().to_string()).collect();
|
||||
endpoints.insert(endpoint);
|
||||
cache.activate_directory_source(source, &endpoints);
|
||||
}
|
||||
let directory_work = std::mem::take(&mut cache.info.scan_directory_work);
|
||||
cache.info.scan_directory_work = DirectoryCheckpoint::verified_work(directory_work, &cache.info);
|
||||
|
||||
// Get disk path
|
||||
let base_path = local_disk.path().to_string_lossy().to_string();
|
||||
|
||||
@@ -3437,6 +3800,16 @@ pub(crate) async fn scan_data_folder_scoped(
|
||||
Vec::new()
|
||||
};
|
||||
// Create folder scanner
|
||||
let mut directory_checkpoints = HashMap::new();
|
||||
for checkpoint in &cache.info.scan_directory_work {
|
||||
directory_checkpoints
|
||||
.entry(checkpoint.parent().to_owned())
|
||||
.or_insert_with(HashMap::new)
|
||||
.insert(
|
||||
checkpoint.endpoint().to_owned(),
|
||||
Arc::new(std::sync::Mutex::new(Some(checkpoint.clone()))),
|
||||
);
|
||||
}
|
||||
let mut scanner = FolderScanner {
|
||||
root: base_path,
|
||||
old_cache: cache,
|
||||
@@ -3477,6 +3850,7 @@ pub(crate) async fn scan_data_folder_scoped(
|
||||
resume_failed_paths,
|
||||
coverage_gap: false,
|
||||
raw_enumeration_progress: Vec::new(),
|
||||
directory_checkpoints,
|
||||
pending_size_reconciliation_keys: HashSet::new(),
|
||||
pending_size_reconciliation_scopes: HashSet::new(),
|
||||
pending_size_reconciliation_truncated: false,
|
||||
@@ -3514,6 +3888,7 @@ pub(crate) async fn scan_data_folder_scoped(
|
||||
.await
|
||||
{
|
||||
Ok(()) => {
|
||||
directory_cursor::forget_bucket(&scanner.local_disk, &scanner.old_cache.info);
|
||||
// Get the new cache and finalize it
|
||||
let coverage_gap = scanner.coverage_gap;
|
||||
scanner.reconcile_failed_objects_after_full_scan();
|
||||
@@ -3558,6 +3933,9 @@ pub(crate) async fn scan_data_folder_scoped(
|
||||
new_cache.info.scan_raw_enumeration_cursor = None;
|
||||
new_cache.info.scan_raw_enumeration_page_index = None;
|
||||
new_cache.info.scan_raw_enumeration_path_page_indexes.clear();
|
||||
new_cache.info.scan_directory_work.clear();
|
||||
new_cache.info.scan_directory_source = None;
|
||||
new_cache.info.scan_source_trees.clear();
|
||||
new_cache.info.scan_coverage_receipt = None;
|
||||
if had_scan_checkpoint {
|
||||
global_metrics().record_scanner_checkpoint_cleared();
|
||||
@@ -3606,7 +3984,9 @@ pub(crate) async fn scan_data_folder_scoped(
|
||||
scanner.carry_forward_old_children(&root_hash, &mut root);
|
||||
}
|
||||
let coverage_frontier = clone_cache_value(&scanner.coverage_frontier);
|
||||
let directories = scanner.saved_directories();
|
||||
let new_cache = scanner.as_mut_new_cache();
|
||||
new_cache.info.scan_directory_work = directories;
|
||||
if root_has_progress {
|
||||
new_cache.replace_hashed(&root_hash, &None, &root);
|
||||
} else if let Some(cache) = carry_forward_cache {
|
||||
@@ -3626,6 +4006,7 @@ pub(crate) async fn scan_data_folder_scoped(
|
||||
new_cache.info.scan_coverage_receipt = None;
|
||||
}
|
||||
if partial_cache_is_useful(&root, pending_heals_changed)
|
||||
|| !new_cache.info.scan_directory_work.is_empty()
|
||||
|| new_cache.info.scan_raw_enumeration_cursor.is_some()
|
||||
|| new_cache.info.scan_raw_enumeration_page_index.is_some()
|
||||
|| !new_cache.info.size_reconciliation.is_empty()
|
||||
@@ -3701,6 +4082,9 @@ pub(crate) async fn scan_data_folder_scoped(
|
||||
}
|
||||
}
|
||||
|
||||
mod directory_cursor;
|
||||
pub use directory_cursor::DirectoryCheckpoint;
|
||||
pub(crate) use directory_cursor::{DirectoryProgress, DirectoryReadProgress};
|
||||
mod item_actions;
|
||||
mod ledger;
|
||||
mod native_io;
|
||||
|
||||
@@ -0,0 +1,719 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
//! Cycle-local execution can borrow an interrupted physical directory stream.
|
||||
//! Only the current scanner owns admission and classification; parked streams
|
||||
//! have no task, budget or authority to update a bucket cache.
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
mod checkpoint;
|
||||
mod pages;
|
||||
pub use checkpoint::DirectoryCheckpoint;
|
||||
pub(super) use checkpoint::Saved;
|
||||
pub(crate) use checkpoint::{DirectoryProgress, DirectoryReadProgress};
|
||||
use checkpoint::{Physical, Recorder, Scope};
|
||||
use pages::NamePages;
|
||||
use std::io;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
|
||||
use std::sync::{Arc, LazyLock, Mutex, Weak};
|
||||
use std::time::{Duration, Instant, SystemTime};
|
||||
|
||||
use super::{DirectoryStream, Disk, FolderScanMode, ScannerBucketPrefixScanScope, native_io};
|
||||
use crate::data_usage_define::{DataUsageCacheInfo, DataUsageCacheSource, DataUsageScanIdentity, DataUsageScanProgress};
|
||||
use crate::scanner_budget::ScannerCycleBudget;
|
||||
|
||||
pub(super) const PARKED_DIRECTORIES_PER_DISK: usize = 32;
|
||||
|
||||
/// Only a positively identified disk can re-adopt a linked tree. A missing
|
||||
/// format on an otherwise visible local drive uses its directory generation;
|
||||
/// genuine format I/O errors propagate before any cached subtotals are used.
|
||||
pub(super) async fn physical_source(disk: &Arc<Disk>) -> Result<String, super::ScannerError> {
|
||||
use crate::storage_api::owner::EcstoreDiskAPI;
|
||||
use sha2::{Digest, Sha256};
|
||||
let endpoint = disk.endpoint().to_string();
|
||||
let generation = match disk.get_disk_id().await {
|
||||
Ok(Some(id)) if !id.is_nil() => id.to_string(),
|
||||
Ok(None) | Err(super::DiskError::UnformattedDisk) => {
|
||||
let path = disk.path();
|
||||
let stamp = DirectoryStamp::at(path.to_str().ok_or_else(|| io::Error::from(io::ErrorKind::InvalidInput))?).await?;
|
||||
// Bucket creation and scanner startup can change the volume root's
|
||||
// mtime/ctime. Birth time distinguishes inode reuse without making
|
||||
// ordinary directory content changes appear to replace the disk.
|
||||
if let Some(generation) = stamp.volume_generation() {
|
||||
let bytes = rmp_serde::to_vec_named(&(std::env::consts::OS, generation)).map_err(io::Error::other)?;
|
||||
hex_simd::encode_to_string(Sha256::digest(bytes), hex_simd::AsciiCase::Lower)
|
||||
} else {
|
||||
// The platform cannot attest the generation across processes.
|
||||
uuid::Uuid::new_v4().to_string()
|
||||
}
|
||||
}
|
||||
Ok(Some(_)) => return Err(io::Error::new(io::ErrorKind::InvalidData, "scanner disk format contains a nil ID").into()),
|
||||
Err(error) => return Err(io::Error::other(error).into()),
|
||||
};
|
||||
Ok(format!("{generation}|{endpoint}"))
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(super) struct DirectoryScope {
|
||||
disk: Weak<Disk>,
|
||||
bucket: Arc<str>,
|
||||
source: DataUsageCacheSource,
|
||||
identity: DataUsageScanIdentity,
|
||||
progress: DataUsageScanProgress,
|
||||
mode: FolderScanMode,
|
||||
prefix_scope: Option<ScannerBucketPrefixScanScope>,
|
||||
owner: (u64, u64),
|
||||
endpoint: String,
|
||||
watch: Option<[u8; 32]>,
|
||||
continuing: bool,
|
||||
}
|
||||
|
||||
impl DirectoryScope {
|
||||
pub(super) fn new(
|
||||
disk: &Arc<Disk>,
|
||||
cache: &super::DataUsageCache,
|
||||
mode: FolderScanMode,
|
||||
prefix_scope: Option<ScannerBucketPrefixScanScope>,
|
||||
) -> Option<Self> {
|
||||
use crate::storage_api::owner::EcstoreDiskAPI;
|
||||
let info = &cache.info;
|
||||
Some(Self {
|
||||
disk: Arc::downgrade(disk),
|
||||
bucket: Arc::from(info.name.as_str()),
|
||||
source: info.source?,
|
||||
identity: info.scan_identity.filter(DataUsageScanIdentity::is_valid)?,
|
||||
progress: info.scan_progress?,
|
||||
mode,
|
||||
prefix_scope,
|
||||
owner: (info.leader_epoch, info.next_cycle),
|
||||
endpoint: format!("|{}", disk.endpoint()),
|
||||
watch: info
|
||||
.scan_usage_watch
|
||||
.as_ref()
|
||||
.filter(|proof| proof.is_well_formed())
|
||||
.map(|proof| proof.digest().0),
|
||||
continuing: cache.has_scan_position(),
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn bind_source(&mut self, source: Option<&str>) {
|
||||
if let Some(source) = source {
|
||||
self.endpoint = source.to_owned();
|
||||
}
|
||||
}
|
||||
pub(super) fn endpoint(&self) -> &str {
|
||||
&self.endpoint
|
||||
}
|
||||
fn matches(&self, other: &Self) -> bool {
|
||||
self.endpoint == other.endpoint
|
||||
&& self.disk.ptr_eq(&other.disk)
|
||||
&& self.bucket == other.bucket
|
||||
&& self.source == other.source
|
||||
&& self.identity == other.identity
|
||||
&& self.progress == other.progress
|
||||
&& self.mode == other.mode
|
||||
&& self.watch == other.watch
|
||||
&& self.prefix_scope == other.prefix_scope
|
||||
&& (other.continuing || self.owner == other.owner)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(PartialEq, Eq)]
|
||||
struct DirectoryStamp {
|
||||
modified: SystemTime,
|
||||
birth: Option<SystemTime>,
|
||||
#[cfg(unix)]
|
||||
device: u64,
|
||||
#[cfg(unix)]
|
||||
inode: u64,
|
||||
#[cfg(unix)]
|
||||
changed: (i64, i64),
|
||||
#[cfg(windows)]
|
||||
identity: same_file::Handle,
|
||||
#[cfg(windows)]
|
||||
physical: Option<(u64, u64, i64)>,
|
||||
}
|
||||
|
||||
impl DirectoryStamp {
|
||||
fn volume_generation(&self) -> Option<(u64, u64, SystemTime)> {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
Some((self.device, self.inode, self.birth?))
|
||||
}
|
||||
#[cfg(windows)]
|
||||
{
|
||||
let (volume, file, _) = self.physical?;
|
||||
Some((volume, file, self.birth?))
|
||||
}
|
||||
#[cfg(not(any(unix, windows)))]
|
||||
{
|
||||
None
|
||||
}
|
||||
}
|
||||
async fn at(path: &str) -> io::Result<Self> {
|
||||
let path = PathBuf::from(path);
|
||||
crate::storage_api::ecstore_spawn_blocking_with_resource_ownership(move || {
|
||||
let meta = std::fs::symlink_metadata(&path)?;
|
||||
if !meta.is_dir() {
|
||||
return Err(io::ErrorKind::NotADirectory.into());
|
||||
}
|
||||
#[cfg(unix)]
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
#[cfg(windows)]
|
||||
let identity = same_file::Handle::from_path(&path)?;
|
||||
#[cfg(windows)]
|
||||
let physical = winapi_util::file::information(identity.as_file()).ok().and_then(|info| {
|
||||
let created = i64::try_from(info.creation_time()?).ok()?;
|
||||
(info.file_index() != 0).then_some((info.volume_serial_number(), info.file_index(), created))
|
||||
});
|
||||
Ok(Self {
|
||||
modified: meta.modified()?,
|
||||
birth: meta.created().ok(),
|
||||
#[cfg(unix)]
|
||||
device: meta.dev(),
|
||||
#[cfg(unix)]
|
||||
inode: meta.ino(),
|
||||
#[cfg(unix)]
|
||||
changed: (meta.ctime(), meta.ctime_nsec()),
|
||||
#[cfg(windows)]
|
||||
identity,
|
||||
#[cfg(windows)]
|
||||
physical,
|
||||
})
|
||||
})
|
||||
.await
|
||||
.map_err(io::Error::other)?
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub(super) struct DirectoryWork {
|
||||
names: NamePages,
|
||||
// For a prefix tree the canonical child values live in the bucket cache.
|
||||
// The final compaction threshold caps how many paths this list can hold.
|
||||
pub(super) linked: Vec<String>,
|
||||
pub(super) compacted_total: super::DataUsageEntry,
|
||||
pub(super) gap: bool,
|
||||
compacted: bool,
|
||||
recorder: Option<Recorder>,
|
||||
}
|
||||
|
||||
impl DirectoryWork {
|
||||
pub(super) fn restore_mode(&mut self, compacted: bool) {
|
||||
if self.compacted && !compacted {
|
||||
self.rewind();
|
||||
}
|
||||
}
|
||||
fn maybe_checkpoint(&mut self) {
|
||||
if let Some(recorder) = &mut self.recorder
|
||||
&& recorder.last_update.elapsed() >= Duration::from_secs(1)
|
||||
{
|
||||
recorder.last_update = Instant::now();
|
||||
self.checkpoint();
|
||||
}
|
||||
}
|
||||
pub(super) fn checkpoint(&self) {
|
||||
if let Some(recorder) = &self.recorder {
|
||||
recorder.save(self);
|
||||
}
|
||||
}
|
||||
pub(super) async fn next_name(&mut self) -> io::Result<Option<String>> {
|
||||
self.names.next().await
|
||||
}
|
||||
pub(super) fn acknowledge(&mut self, name: &str, linked: bool, compacted: bool) {
|
||||
if self.names.acknowledge() {
|
||||
self.compacted = compacted;
|
||||
if linked && !compacted {
|
||||
self.linked.push(name.to_owned());
|
||||
}
|
||||
self.maybe_checkpoint();
|
||||
}
|
||||
}
|
||||
pub(super) fn rewind(&mut self) {
|
||||
self.compacted = false;
|
||||
self.names.rewind();
|
||||
self.linked.clear();
|
||||
self.compacted_total = super::DataUsageEntry::default();
|
||||
self.gap = false;
|
||||
}
|
||||
}
|
||||
|
||||
struct Listing {
|
||||
reader: Option<DirectoryStream>,
|
||||
pending: Option<native_io::DirEntry>,
|
||||
delivery: Option<(native_io::DirEntry, bool)>,
|
||||
classified: Option<Option<std::fs::FileType>>,
|
||||
markers: [Option<native_io::DirEntry>; 2],
|
||||
abandoned: Option<super::DataUsageHashMap>,
|
||||
work: DirectoryWork,
|
||||
failed: bool,
|
||||
}
|
||||
|
||||
struct Parked {
|
||||
scope: DirectoryScope,
|
||||
parked_at: Instant,
|
||||
serial: u64,
|
||||
stamp: DirectoryStamp,
|
||||
listing: Box<Listing>,
|
||||
}
|
||||
|
||||
static PARKED: LazyLock<Mutex<HashMap<PathBuf, Parked>>> = LazyLock::new(|| Mutex::new(HashMap::new()));
|
||||
|
||||
fn expired_slots(slots: &mut HashMap<PathBuf, Parked>) -> Vec<(PathBuf, Parked)> {
|
||||
slots
|
||||
.extract_if(|_, parked| {
|
||||
parked.scope.disk.strong_count() == 0 || parked.parked_at.elapsed() > Duration::from_secs(24 * 3600)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// A routing hint only. The admitted walker revalidates the physical source,
|
||||
/// bucket instance, coverage plan and directory before using a parked stream.
|
||||
pub(crate) fn parked_bucket_disks(bucket: &str) -> Vec<Weak<Disk>> {
|
||||
let mut slots = PARKED.lock().unwrap_or_else(|p| p.into_inner());
|
||||
let expired = expired_slots(&mut slots);
|
||||
let matches = slots
|
||||
.values()
|
||||
.filter(|entry| entry.scope.bucket.as_ref() == bucket)
|
||||
.map(|entry| entry.scope.disk.clone())
|
||||
.collect();
|
||||
drop(slots);
|
||||
drop(expired);
|
||||
matches
|
||||
}
|
||||
|
||||
pub(super) fn is_parked(path: &str) -> bool {
|
||||
PARKED.lock().unwrap_or_else(|p| p.into_inner()).contains_key(Path::new(path))
|
||||
}
|
||||
|
||||
pub(super) fn forget_bucket(disk: &Arc<Disk>, info: &DataUsageCacheInfo) {
|
||||
let owner = Arc::downgrade(disk);
|
||||
let mut slots = PARKED.lock().unwrap_or_else(|p| p.into_inner());
|
||||
let removed = slots
|
||||
.extract_if(|_, parked| {
|
||||
parked.scope.disk.ptr_eq(&owner)
|
||||
&& parked.scope.bucket.as_ref() == info.name
|
||||
&& parked.scope.owner <= (info.leader_epoch, info.next_cycle)
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
drop(slots);
|
||||
drop(removed);
|
||||
}
|
||||
|
||||
// Prefer existing active work over cyclic LRU eviction. Under pressure, a
|
||||
// deleted or replaced directory must not monopolize one of the physical slots.
|
||||
async fn reclaim_retired(scope: &DirectoryScope, path: &Path) {
|
||||
static NEXT_PROBE: AtomicUsize = AtomicUsize::new(0);
|
||||
let candidate = {
|
||||
let slots = PARKED.lock().unwrap_or_else(|p| p.into_inner());
|
||||
if slots.contains_key(path) {
|
||||
return;
|
||||
}
|
||||
let disk_slots: Vec<_> = slots
|
||||
.iter()
|
||||
.filter(|(_, parked)| parked.scope.disk.ptr_eq(&scope.disk))
|
||||
.collect();
|
||||
if disk_slots.len() < PARKED_DIRECTORIES_PER_DISK {
|
||||
return;
|
||||
}
|
||||
let (path, parked) = disk_slots[NEXT_PROBE.fetch_add(1, Ordering::Relaxed) % disk_slots.len()];
|
||||
(path.clone(), parked.serial)
|
||||
};
|
||||
let Some(candidate_path) = candidate.0.to_str() else { return };
|
||||
let current = DirectoryStamp::at(candidate_path).await;
|
||||
let mut slots = PARKED.lock().unwrap_or_else(|p| p.into_inner());
|
||||
let stale = slots.get(&candidate.0).is_some_and(|parked| {
|
||||
parked.serial == candidate.1
|
||||
&& match ¤t {
|
||||
Ok(stamp) => stamp != &parked.stamp,
|
||||
Err(error) => matches!(error.kind(), io::ErrorKind::NotFound | io::ErrorKind::NotADirectory),
|
||||
}
|
||||
});
|
||||
let retired = stale.then(|| slots.remove(&candidate.0));
|
||||
drop(slots);
|
||||
drop(retired);
|
||||
}
|
||||
|
||||
fn valid_component(name: &str) -> bool {
|
||||
!name.is_empty()
|
||||
&& name.len() <= 4096
|
||||
&& name != "."
|
||||
&& name != ".."
|
||||
&& !name.as_bytes().contains(&0)
|
||||
&& std::path::Path::new(name)
|
||||
.file_name()
|
||||
.is_some_and(|part| part == std::ffi::OsStr::new(name))
|
||||
}
|
||||
|
||||
pub(super) struct ScannerDirectory {
|
||||
path: Arc<Path>,
|
||||
scope: Option<(DirectoryScope, DirectoryStamp)>,
|
||||
directory_name: Arc<str>,
|
||||
budget: Arc<ScannerCycleBudget>,
|
||||
listing: Option<Box<Listing>>,
|
||||
marker: usize,
|
||||
pub(super) enumeration_only: bool,
|
||||
resumed: bool,
|
||||
pub(super) directory_replaced: bool,
|
||||
pub(super) batch_entries: usize,
|
||||
}
|
||||
|
||||
impl ScannerDirectory {
|
||||
pub(super) async fn open(
|
||||
path: &str,
|
||||
directory_name: &str,
|
||||
scope: Option<DirectoryScope>,
|
||||
budget: Arc<ScannerCycleBudget>,
|
||||
saved: Option<Saved>,
|
||||
) -> io::Result<Self> {
|
||||
// A different physical directory must never inherit old names. Directory
|
||||
// timestamps supplement (and cannot replace) the scanner's plan fences.
|
||||
let stamp = if scope.is_some() {
|
||||
Some(DirectoryStamp::at(path).await?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if let Some(scope) = &scope {
|
||||
reclaim_retired(scope, Path::new(path)).await;
|
||||
}
|
||||
let parked = scope.as_ref().and_then(|current| {
|
||||
let mut slots = PARKED.lock().unwrap_or_else(|p| p.into_inner());
|
||||
let key = Path::new(path);
|
||||
if slots
|
||||
.get(key)
|
||||
.is_some_and(|p| p.scope.disk.ptr_eq(¤t.disk) && p.scope.owner > current.owner)
|
||||
{
|
||||
return None;
|
||||
}
|
||||
slots.remove(key)
|
||||
});
|
||||
let directory_replaced = parked.as_ref().is_some_and(|entry| Some(&entry.stamp) != stamp.as_ref());
|
||||
let parked =
|
||||
parked.filter(|entry| !directory_replaced && scope.as_ref().is_some_and(|current| entry.scope.matches(current)));
|
||||
let mut resumed = parked.is_some();
|
||||
let mut previous = saved
|
||||
.as_ref()
|
||||
.and_then(|slot| slot.lock().unwrap_or_else(|p| p.into_inner()).clone());
|
||||
let durable_scope = scope.as_ref().map(Scope::new);
|
||||
let endpoint = scope.as_ref().map(|s| s.endpoint.clone()).unwrap_or_default();
|
||||
let physical = scope
|
||||
.as_ref()
|
||||
.zip(stamp.as_ref())
|
||||
.and_then(|(scope, stamp)| Physical::of(&scope.endpoint, stamp));
|
||||
let mut replaced = directory_replaced
|
||||
|| previous
|
||||
.as_ref()
|
||||
.is_some_and(|old| old.endpoint() == endpoint && old.physical.is_some() && old.physical != physical);
|
||||
previous = previous.filter(|old| {
|
||||
durable_scope
|
||||
.as_ref()
|
||||
.is_some_and(|current| old.restore_allowed(directory_name, current, &physical))
|
||||
});
|
||||
let mut listing = if let Some(parked) = parked {
|
||||
parked.listing
|
||||
} else {
|
||||
let mut reader = None;
|
||||
if let Some(checkpoint) = &previous {
|
||||
if checkpoint.sealed {
|
||||
resumed = true;
|
||||
} else if let Some(anchor) = &checkpoint.anchor {
|
||||
match DirectoryStream::open_at(path, anchor).await {
|
||||
Ok(restored) => {
|
||||
reader = Some(restored);
|
||||
resumed = true;
|
||||
}
|
||||
Err(error)
|
||||
if matches!(
|
||||
error.kind(),
|
||||
io::ErrorKind::InvalidData
|
||||
| io::ErrorKind::Unsupported
|
||||
| io::ErrorKind::NotFound
|
||||
| io::ErrorKind::InvalidInput
|
||||
) =>
|
||||
{
|
||||
previous = None;
|
||||
replaced = true;
|
||||
}
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
}
|
||||
}
|
||||
if !resumed {
|
||||
reader = Some(DirectoryStream::open(path, scope.is_some()).await?);
|
||||
}
|
||||
let mut work = DirectoryWork::default();
|
||||
let mut markers = [None, None];
|
||||
if let Some(saved) = &previous {
|
||||
if resumed && let Some(owner) = &scope {
|
||||
tracing::debug!(
|
||||
event = super::EVENT_SCANNER_FOLDER_STATE,
|
||||
component = super::LOG_COMPONENT_SCANNER,
|
||||
subsystem = super::LOG_SUBSYSTEM_FOLDER,
|
||||
state = "directory_checkpoint_restored",
|
||||
bucket = %owner.bucket,
|
||||
directory = directory_name,
|
||||
disk = %endpoint,
|
||||
leader_epoch = owner.owner.0,
|
||||
cycle = owner.owner.1,
|
||||
restore_mode = if saved.sealed { "sealed_membership" } else { "native_position" },
|
||||
entries_read = saved.entries_read,
|
||||
completed_through = saved.aggregate.as_ref().map(|aggregate| aggregate.through.as_str()),
|
||||
"Restored directory checkpoint"
|
||||
);
|
||||
}
|
||||
work.names = NamePages::restore(&saved.pages, saved.sealed, saved.aggregate.as_ref().map(|a| a.through.as_str()));
|
||||
if let Some(aggregate) = &saved.aggregate {
|
||||
work.compacted = true;
|
||||
work.compacted_total = aggregate.value.clone();
|
||||
}
|
||||
let physical_path: Arc<Path> = Arc::from(Path::new(path));
|
||||
if let Some(part) = &saved.part {
|
||||
markers[0] = Some(native_io::DirEntry::recorded(physical_path.clone(), part.into(), None));
|
||||
}
|
||||
if saved.metadata {
|
||||
markers[1] = Some(native_io::DirEntry::recorded(physical_path, super::STORAGE_FORMAT_FILE.into(), None));
|
||||
}
|
||||
}
|
||||
Box::new(Listing {
|
||||
reader,
|
||||
pending: None,
|
||||
delivery: None,
|
||||
classified: None,
|
||||
markers,
|
||||
abandoned: None,
|
||||
work,
|
||||
failed: false,
|
||||
})
|
||||
};
|
||||
if let (Some(slot), Some(durable_scope)) = (saved, durable_scope) {
|
||||
if let Some(recorder) = &mut listing.work.recorder {
|
||||
recorder.slot = slot;
|
||||
} else {
|
||||
let recorder = Recorder::new(slot, directory_name, durable_scope, endpoint, physical, previous.as_ref());
|
||||
|
||||
listing.work.recorder = Some(recorder);
|
||||
}
|
||||
}
|
||||
Ok(Self {
|
||||
path: Arc::from(Path::new(path)),
|
||||
scope: scope.zip(stamp),
|
||||
directory_name: Arc::from(directory_name),
|
||||
budget,
|
||||
listing: Some(listing),
|
||||
marker: 0,
|
||||
enumeration_only: false,
|
||||
resumed,
|
||||
directory_replaced: replaced,
|
||||
batch_entries: 0,
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn continues_stream(&self) -> bool {
|
||||
self.resumed
|
||||
}
|
||||
pub(super) fn indexed(&self) -> bool {
|
||||
self.scope.is_some()
|
||||
}
|
||||
pub(super) async fn verify_current(&mut self) -> io::Result<()> {
|
||||
let Some((_, original)) = &self.scope else { return Ok(()) };
|
||||
let path = self.path.to_str().ok_or(io::ErrorKind::InvalidInput)?;
|
||||
let current = DirectoryStamp::at(path).await?;
|
||||
if *original == current {
|
||||
return Ok(());
|
||||
}
|
||||
if let Some(listing) = &mut self.listing {
|
||||
listing.failed = true;
|
||||
}
|
||||
Err(io::Error::new(io::ErrorKind::Interrupted, "scanner directory changed during traversal"))
|
||||
}
|
||||
|
||||
pub(super) fn folder_count(&self) -> Option<usize> {
|
||||
self.indexed()
|
||||
.then(|| self.listing.as_ref().map_or(0, |l| l.work.names.len()))
|
||||
}
|
||||
pub(super) fn finished_listing(&self) -> bool {
|
||||
self.listing.as_ref().is_some_and(|l| l.work.names.ready())
|
||||
}
|
||||
pub(super) fn work(&mut self) -> Option<&mut DirectoryWork> {
|
||||
if !self.indexed() {
|
||||
return None;
|
||||
}
|
||||
self.listing.as_mut().map(|listing| &mut listing.work)
|
||||
}
|
||||
pub(super) fn resolve_abandoned(&mut self, path: &str) {
|
||||
if let Some(abandoned) = self.listing.as_mut().and_then(|l| l.abandoned.as_mut()) {
|
||||
abandoned.remove(path);
|
||||
}
|
||||
}
|
||||
pub(super) fn hold_abandoned(&mut self, baseline: &mut super::DataUsageHashMap) {
|
||||
if self.indexed()
|
||||
&& let Some(listing) = &mut self.listing
|
||||
&& listing.abandoned.is_none()
|
||||
{
|
||||
if !baseline.is_empty() && self.resumed {
|
||||
// Cold restore carries classified members, not the previous
|
||||
// process's transient unresolved heal candidates.
|
||||
for page in listing.work.names.snapshot() {
|
||||
for name in page.names() {
|
||||
baseline.remove(&super::path_join_buf(&[&self.directory_name, name]));
|
||||
}
|
||||
}
|
||||
}
|
||||
listing.abandoned = Some(std::mem::take(baseline));
|
||||
}
|
||||
}
|
||||
pub(super) fn restore_abandoned(&self, baseline: &mut super::DataUsageHashMap) {
|
||||
if let Some(abandoned) = self.listing.as_ref().and_then(|listing| listing.abandoned.as_ref()) {
|
||||
baseline.clone_from(abandoned);
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn peek_entry(&mut self) -> io::Result<Option<native_io::DirEntry>> {
|
||||
if let Some((entry, enumeration_only)) = self.listing.as_ref().and_then(|listing| listing.delivery.as_ref()) {
|
||||
self.enumeration_only = *enumeration_only;
|
||||
return Ok(Some(entry.clone()));
|
||||
}
|
||||
let entry = self.next_entry().await?;
|
||||
if let (Some(listing), Some(entry)) = (&mut self.listing, &entry) {
|
||||
listing.delivery = Some((entry.clone(), self.enumeration_only));
|
||||
}
|
||||
Ok(entry)
|
||||
}
|
||||
|
||||
pub(super) fn accept_entry(&mut self) {
|
||||
let Some(listing) = &mut self.listing else { return };
|
||||
let Some((entry, _)) = listing.delivery.take() else { return };
|
||||
// Discovering a name can precede the I/O admission wait. Commit its
|
||||
// classification and native position together only after admission;
|
||||
// a cold retry then re-reads any name still pending on cancellation.
|
||||
let Some(kind) = listing.classified.take() else { return };
|
||||
let name = entry.file_name();
|
||||
let text = name.to_string_lossy();
|
||||
if let Some(state) = &mut listing.work.recorder {
|
||||
state.anchor = entry.anchor().cloned();
|
||||
state.entries_read = state.entries_read.saturating_add(1);
|
||||
state.valid_names &= name.to_str().is_some_and(valid_component);
|
||||
}
|
||||
if let Some(kind) = kind {
|
||||
let direct_part = kind.is_file() && super::is_erasure_part_name(&text);
|
||||
if kind.is_dir() {
|
||||
if let Some(abandoned) = &mut listing.abandoned
|
||||
&& !abandoned.is_empty()
|
||||
{
|
||||
abandoned.remove(&super::path_join_buf(&[&self.directory_name, &text]));
|
||||
}
|
||||
listing.work.names.add(text.into_owned());
|
||||
} else if direct_part {
|
||||
if let Some(state) = &mut listing.work.recorder {
|
||||
state.part.get_or_insert_with(|| text.to_string());
|
||||
}
|
||||
listing.markers[0].get_or_insert_with(|| native_io::DirEntry::recorded(self.path.clone(), name, Some(kind)));
|
||||
} else if text == super::STORAGE_FORMAT_FILE {
|
||||
if let Some(state) = &mut listing.work.recorder {
|
||||
state.metadata = true;
|
||||
}
|
||||
listing.markers[1] = Some(native_io::DirEntry::recorded(self.path.clone(), name, Some(kind)));
|
||||
}
|
||||
}
|
||||
listing.work.maybe_checkpoint();
|
||||
}
|
||||
|
||||
async fn next_entry(&mut self) -> io::Result<Option<native_io::DirEntry>> {
|
||||
let erasure_candidate = self
|
||||
.scope
|
||||
.as_ref()
|
||||
.is_some_and(|(scope, _)| scope.mode != FolderScanMode::Namespace);
|
||||
let indexed = self.indexed();
|
||||
let listing = self
|
||||
.listing
|
||||
.as_mut()
|
||||
.ok_or_else(|| io::Error::other("scanner directory already returned"))?;
|
||||
self.enumeration_only = false;
|
||||
if let Some(reader) = listing.reader.as_mut() {
|
||||
if !indexed {
|
||||
return reader.next_entry().await;
|
||||
}
|
||||
listing.work.names.flush_full().await?;
|
||||
if listing.pending.is_none() {
|
||||
listing.pending = match reader.next_entry().await {
|
||||
Ok(entry) => entry,
|
||||
Err(error) => {
|
||||
listing.failed = true;
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
}
|
||||
if let Some(entry) = &listing.pending {
|
||||
let kind = match entry.file_type().await {
|
||||
Ok(kind) => Some(kind),
|
||||
Err(error) if matches!(error.kind(), io::ErrorKind::NotFound | io::ErrorKind::TooManyLinks) => None,
|
||||
Err(error) => {
|
||||
listing.failed = true;
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
let name = entry.file_name();
|
||||
let text = name.to_string_lossy();
|
||||
self.enumeration_only = kind.is_none_or(|kind| {
|
||||
(!kind.is_symlink() || text == super::STORAGE_FORMAT_FILE)
|
||||
&& !(erasure_candidate && kind.is_file() && super::is_erasure_part_name(&text))
|
||||
});
|
||||
listing.classified = Some(kind);
|
||||
return Ok(listing.pending.take());
|
||||
}
|
||||
listing.reader = None;
|
||||
listing.work.names.finish();
|
||||
listing.work.maybe_checkpoint();
|
||||
}
|
||||
while self.marker < listing.markers.len() {
|
||||
let marker = self.marker;
|
||||
self.marker += 1;
|
||||
if let Some(entry) = &listing.markers[marker] {
|
||||
return Ok(Some(entry.clone()));
|
||||
}
|
||||
}
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ScannerDirectory {
|
||||
fn drop(&mut self) {
|
||||
let Some((scope, stamp)) = self.scope.take() else { return };
|
||||
let Some(listing) = self.listing.take() else { return };
|
||||
if listing.failed {
|
||||
if let Some(recorder) = &listing.work.recorder {
|
||||
*recorder.slot.lock().unwrap_or_else(|p| p.into_inner()) = None;
|
||||
}
|
||||
return;
|
||||
}
|
||||
// An owner cancellation can still persist a partial cache. Keep its
|
||||
// last accepted native position while releasing the live reader.
|
||||
listing.work.checkpoint();
|
||||
if self.budget.reason().is_none() {
|
||||
return;
|
||||
}
|
||||
let mut slots = PARKED.lock().unwrap_or_else(|p| p.into_inner());
|
||||
let expired = expired_slots(&mut slots);
|
||||
let key = self.path.to_path_buf();
|
||||
let replaced = slots
|
||||
.get(&key)
|
||||
.is_some_and(|p| !p.scope.disk.ptr_eq(&scope.disk) || p.scope.owner < scope.owner)
|
||||
.then(|| slots.remove(&key));
|
||||
if slots.values().filter(|p| p.scope.disk.ptr_eq(&scope.disk)).count() < PARKED_DIRECTORIES_PER_DISK
|
||||
&& let std::collections::hash_map::Entry::Vacant(slot) = slots.entry(key)
|
||||
{
|
||||
static NEXT_SERIAL: AtomicU64 = AtomicU64::new(0);
|
||||
slot.insert(Parked {
|
||||
scope,
|
||||
parked_at: Instant::now(),
|
||||
serial: NEXT_SERIAL.fetch_add(1, Ordering::Relaxed),
|
||||
stamp,
|
||||
listing,
|
||||
});
|
||||
}
|
||||
drop(slots);
|
||||
drop((expired, replaced));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,364 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::{Instant, SystemTime};
|
||||
|
||||
use super::{DirectoryScope, DirectoryStamp, DirectoryWork, FolderScanMode, native_io, pages::NamePage};
|
||||
use crate::data_usage_define::{
|
||||
DataUsageCacheInfo, DataUsageCacheSource, DataUsageEntry, DataUsageScanIdentity, DataUsageScanProgress,
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub(super) struct Scope {
|
||||
bucket: String,
|
||||
source: DataUsageCacheSource,
|
||||
identity: DataUsageScanIdentity,
|
||||
progress: DataUsageScanProgress,
|
||||
mode: u8,
|
||||
prefixes: Option<[u8; 32]>,
|
||||
watch: Option<[u8; 32]>,
|
||||
}
|
||||
|
||||
impl Scope {
|
||||
fn may_adopt(&self, current: &Self) -> bool {
|
||||
self.bucket == current.bucket
|
||||
&& self.source == current.source
|
||||
&& self.identity == current.identity
|
||||
&& self.mode == current.mode
|
||||
&& self.prefixes == current.prefixes
|
||||
&& self.progress.started_plan == current.progress.started_plan
|
||||
&& ((self.progress == current.progress && self.watch == current.watch)
|
||||
|| current.progress.started_plan != current.progress.requested_plan)
|
||||
}
|
||||
pub(super) fn new(owner: &DirectoryScope) -> Self {
|
||||
let mode = match owner.mode {
|
||||
FolderScanMode::Namespace => 0,
|
||||
FolderScanMode::ErasureCandidate {
|
||||
reuse_namespace,
|
||||
parent_has_metadata,
|
||||
} => 1 + u8::from(reuse_namespace) + 2 * u8::from(parent_has_metadata),
|
||||
};
|
||||
let prefixes = owner.prefix_scope.as_ref().map(|scope| {
|
||||
let mut entries: Vec<_> = scope.selected_top_level_entries.iter().collect();
|
||||
entries.sort_unstable();
|
||||
let mut hash = Sha256::new();
|
||||
for entry in entries {
|
||||
hash.update(u64::try_from(entry.len()).unwrap_or(u64::MAX).to_le_bytes());
|
||||
hash.update(entry.as_bytes());
|
||||
}
|
||||
hash.finalize().into()
|
||||
});
|
||||
Self {
|
||||
bucket: owner.bucket.to_string(),
|
||||
source: owner.source,
|
||||
identity: owner.identity,
|
||||
progress: owner.progress,
|
||||
mode,
|
||||
prefixes,
|
||||
watch: owner.watch,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub(crate) struct Physical {
|
||||
pub(crate) endpoint: String,
|
||||
device: u64,
|
||||
inode: u64,
|
||||
changed: (i64, i64),
|
||||
modified: SystemTime,
|
||||
}
|
||||
|
||||
impl Physical {
|
||||
#[cfg(unix)]
|
||||
pub(super) fn of(endpoint: &str, stamp: &DirectoryStamp) -> Option<Self> {
|
||||
Some(Self {
|
||||
endpoint: endpoint.to_owned(),
|
||||
device: stamp.device,
|
||||
inode: stamp.inode,
|
||||
changed: stamp.changed,
|
||||
modified: stamp.modified,
|
||||
})
|
||||
}
|
||||
#[cfg(windows)]
|
||||
pub(super) fn of(endpoint: &str, stamp: &DirectoryStamp) -> Option<Self> {
|
||||
let (device, inode, created) = stamp.physical?;
|
||||
Some(Self {
|
||||
endpoint: endpoint.to_owned(),
|
||||
device,
|
||||
inode,
|
||||
changed: (created, 0),
|
||||
modified: stamp.modified,
|
||||
})
|
||||
}
|
||||
#[cfg(not(any(unix, windows)))]
|
||||
pub(super) fn of(_endpoint: &str, _stamp: &DirectoryStamp) -> Option<Self> {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
pub(super) struct Aggregate {
|
||||
pub(super) through: String,
|
||||
pub(super) value: DataUsageEntry,
|
||||
}
|
||||
|
||||
/// Immutable directory membership and the last fully counted compacted child.
|
||||
/// The active child's provisional contribution is never part of this receipt.
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
pub struct DirectoryCheckpoint {
|
||||
version: u8,
|
||||
pub(super) parent: String,
|
||||
scope: Scope,
|
||||
pub(super) physical: Option<Physical>,
|
||||
pub(super) pages: Vec<NamePage>,
|
||||
pub(super) anchor: Option<native_io::DirectoryAnchor>,
|
||||
pub(super) entries_read: u64,
|
||||
pub(super) sealed: bool,
|
||||
pub(super) part: Option<String>,
|
||||
pub(super) metadata: bool,
|
||||
pub(super) aggregate: Option<Aggregate>,
|
||||
endpoint: String,
|
||||
digest: [u8; 32],
|
||||
}
|
||||
|
||||
/// Physical progress orders only callbacks from the same directory generation.
|
||||
/// The coordinator uses these receipts for fair scheduling, never for coverage.
|
||||
#[derive(Clone, Debug)]
|
||||
pub(crate) struct DirectoryReadProgress {
|
||||
pub(crate) parent: String,
|
||||
pub(crate) physical: Physical,
|
||||
position: (u64, bool),
|
||||
}
|
||||
impl DirectoryReadProgress {
|
||||
pub(crate) fn same_source(&self, other: &Self) -> bool {
|
||||
self.parent == other.parent && self.physical == other.physical
|
||||
}
|
||||
pub(crate) fn extends(&self, older: &Self) -> bool {
|
||||
self.same_source(older) && self.position >= older.position
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub(crate) struct DirectoryProgress {
|
||||
pub(crate) parent: String,
|
||||
pub(crate) physical: Physical,
|
||||
manifest: [u8; 32],
|
||||
through: String,
|
||||
}
|
||||
impl DirectoryProgress {
|
||||
pub(crate) fn same_source(&self, previous: &Self) -> bool {
|
||||
self.parent == previous.parent && self.physical == previous.physical && self.manifest == previous.manifest
|
||||
}
|
||||
pub(crate) fn extends(&self, previous: &Self) -> bool {
|
||||
self.same_source(previous) && self.through >= previous.through
|
||||
}
|
||||
pub(crate) fn covers(&self, path: &str) -> bool {
|
||||
path.strip_prefix(&self.parent)
|
||||
.and_then(|tail| tail.strip_prefix('/'))
|
||||
.and_then(|tail| tail.split('/').next())
|
||||
.is_some_and(|child| child <= self.through.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
impl DirectoryCheckpoint {
|
||||
/// Deserialize each replica's independent receipt, then intern equal
|
||||
/// verified immutable name pages across the bucket's active directories.
|
||||
pub(crate) fn verified_work(work: Vec<Self>, info: &DataUsageCacheInfo) -> Vec<Self> {
|
||||
let mut shared = HashMap::new();
|
||||
work.into_iter()
|
||||
.filter(|receipt| receipt.is_bound_to(info))
|
||||
.map(|mut receipt| {
|
||||
for page in &mut receipt.pages {
|
||||
let canonical = shared.entry(*page.digest()).or_insert_with(|| page.clone());
|
||||
*page = canonical.clone();
|
||||
}
|
||||
receipt
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
pub(crate) fn parent(&self) -> &str {
|
||||
&self.parent
|
||||
}
|
||||
|
||||
pub(super) fn manifest(&self) -> [u8; 32] {
|
||||
let mut hash = Sha256::new();
|
||||
for p in &self.pages {
|
||||
hash.update(p.digest());
|
||||
}
|
||||
hash.finalize().into()
|
||||
}
|
||||
fn checksum(&self) -> Option<[u8; 32]> {
|
||||
// Nested usage distributions can contain maps. Canonicalize those
|
||||
// map keys before hashing so restart hash randomization is irrelevant.
|
||||
let mut aggregate = serde_json::to_value(&self.aggregate).ok()?;
|
||||
aggregate.sort_all_objects();
|
||||
let header = rmp_serde::to_vec_named(&(
|
||||
self.version,
|
||||
&self.parent,
|
||||
&self.scope,
|
||||
&self.physical,
|
||||
&self.anchor,
|
||||
self.entries_read,
|
||||
self.sealed,
|
||||
&self.part,
|
||||
self.metadata,
|
||||
self.manifest(),
|
||||
aggregate,
|
||||
&self.endpoint,
|
||||
))
|
||||
.ok()?;
|
||||
Some(Sha256::digest(header).into())
|
||||
}
|
||||
pub(crate) fn is_bound_to(&self, info: &DataUsageCacheInfo) -> bool {
|
||||
self.version == 1
|
||||
&& self.scope.bucket == info.name
|
||||
&& self.scope.identity.is_valid()
|
||||
&& Some(self.scope.identity) == info.scan_identity
|
||||
&& Some(self.scope.source) == info.source
|
||||
&& info
|
||||
.scan_progress
|
||||
.is_some_and(|p| p.started_plan == self.scope.progress.started_plan)
|
||||
&& (self.parent == info.name
|
||||
|| self
|
||||
.parent
|
||||
.strip_prefix(&info.name)
|
||||
.and_then(|p| p.strip_prefix('/'))
|
||||
.is_some_and(|tail| tail.split('/').all(super::valid_component)))
|
||||
&& self.valid_payload()
|
||||
}
|
||||
pub(crate) fn read_progress(&self, info: &DataUsageCacheInfo) -> Option<DirectoryReadProgress> {
|
||||
let physical = self.physical.as_ref()?;
|
||||
(self.is_bound_to(info) && (self.sealed || self.anchor.is_some()) && self.entries_read > 0).then(|| {
|
||||
DirectoryReadProgress {
|
||||
parent: self.parent.clone(),
|
||||
physical: physical.clone(),
|
||||
position: (self.entries_read, self.sealed),
|
||||
}
|
||||
})
|
||||
}
|
||||
pub(crate) fn progress(&self, info: &DataUsageCacheInfo) -> Option<DirectoryProgress> {
|
||||
let done = self.aggregate.as_ref()?;
|
||||
let physical = self.physical.as_ref()?;
|
||||
(self.sealed && self.is_bound_to(info)).then(|| DirectoryProgress {
|
||||
parent: self.parent.clone(),
|
||||
physical: physical.clone(),
|
||||
manifest: self.manifest(),
|
||||
through: done.through.clone(),
|
||||
})
|
||||
}
|
||||
pub(crate) fn endpoint(&self) -> &str {
|
||||
&self.endpoint
|
||||
}
|
||||
fn valid_for(&self, parent: &str, owner: &Scope) -> bool {
|
||||
self.version == 1 && self.parent == parent && self.scope.may_adopt(owner) && self.valid_payload()
|
||||
}
|
||||
fn valid_payload(&self) -> bool {
|
||||
self.physical.as_ref().is_some_and(|source| source.endpoint == self.endpoint)
|
||||
&& (self.sealed || self.anchor.is_some())
|
||||
&& self.checksum() == Some(self.digest)
|
||||
&& self.pages.iter().all(NamePage::valid)
|
||||
&& self.part.as_deref().is_none_or(super::super::is_erasure_part_name)
|
||||
&& self.aggregate.as_ref().is_none_or(|done| {
|
||||
self.sealed
|
||||
&& super::valid_component(&done.through)
|
||||
&& done.value.children.is_empty()
|
||||
&& done.value.failed_objects == 0
|
||||
&& self.pages.iter().any(|page| page.contains(&done.through))
|
||||
})
|
||||
}
|
||||
pub(super) fn restore_allowed(&self, parent: &str, owner: &Scope, physical: &Option<Physical>) -> bool {
|
||||
self.valid_for(parent, owner)
|
||||
&& self.physical.is_some()
|
||||
&& &self.physical == physical
|
||||
&& (self.sealed || self.anchor.is_some())
|
||||
}
|
||||
}
|
||||
|
||||
pub(in crate::scanner_folder) type Saved = Arc<Mutex<Option<DirectoryCheckpoint>>>;
|
||||
|
||||
pub(super) struct Recorder {
|
||||
pub(super) slot: Saved,
|
||||
scope: Scope,
|
||||
parent: String,
|
||||
physical: Option<Physical>,
|
||||
pub(super) anchor: Option<native_io::DirectoryAnchor>,
|
||||
pub(super) entries_read: u64,
|
||||
pub(super) part: Option<String>,
|
||||
pub(super) metadata: bool,
|
||||
pub(super) valid_names: bool,
|
||||
endpoint: String,
|
||||
pub(super) last_update: Instant,
|
||||
}
|
||||
|
||||
impl Recorder {
|
||||
pub(super) fn new(
|
||||
slot: Saved,
|
||||
parent: &str,
|
||||
scope: Scope,
|
||||
endpoint: String,
|
||||
physical: Option<Physical>,
|
||||
previous: Option<&DirectoryCheckpoint>,
|
||||
) -> Self {
|
||||
Self {
|
||||
slot,
|
||||
scope,
|
||||
parent: parent.to_owned(),
|
||||
physical,
|
||||
endpoint,
|
||||
anchor: previous.and_then(|p| p.anchor.clone()),
|
||||
entries_read: previous.map_or(0, |p| p.entries_read),
|
||||
part: previous.and_then(|p| p.part.clone()),
|
||||
metadata: previous.is_some_and(|p| p.metadata),
|
||||
valid_names: true,
|
||||
last_update: Instant::now(),
|
||||
}
|
||||
}
|
||||
pub(super) fn save(&self, work: &DirectoryWork) {
|
||||
// An ordinary object directory is cheaper to reopen. Only a reader
|
||||
// which can actually resume or a closed directory can be checkpointed.
|
||||
let sealed = work.names.ready();
|
||||
let aggregate = if work.compacted && !work.gap {
|
||||
work.names.through().map(|through| Aggregate {
|
||||
through: through.to_owned(),
|
||||
value: work.compacted_total.clone(),
|
||||
})
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let snapshot = if self.valid_names
|
||||
&& self.physical.as_ref().is_some_and(|source| source.endpoint == self.endpoint)
|
||||
&& (sealed || self.anchor.is_some())
|
||||
{
|
||||
let mut checkpoint = DirectoryCheckpoint {
|
||||
version: 1,
|
||||
parent: self.parent.clone(),
|
||||
scope: self.scope.clone(),
|
||||
physical: self.physical.clone(),
|
||||
anchor: (!sealed).then(|| self.anchor.clone()).flatten(),
|
||||
entries_read: self.entries_read,
|
||||
pages: work.names.snapshot(),
|
||||
sealed,
|
||||
part: self.part.clone(),
|
||||
metadata: self.metadata,
|
||||
aggregate,
|
||||
endpoint: self.endpoint.clone(),
|
||||
digest: [0; 32],
|
||||
};
|
||||
checkpoint.checksum().map(|digest| {
|
||||
checkpoint.digest = digest;
|
||||
checkpoint
|
||||
})
|
||||
} else {
|
||||
None
|
||||
};
|
||||
*self.slot.lock().unwrap_or_else(|p| p.into_inner()) = snapshot;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod storage_fixture;
|
||||
@@ -0,0 +1,48 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use super::*;
|
||||
|
||||
impl DirectoryCheckpoint {
|
||||
pub(crate) async fn wide_storage_fixture(info: &DataUsageCacheInfo) -> [Self; 2] {
|
||||
let scope = Scope {
|
||||
bucket: info.name.clone(),
|
||||
source: info.source.expect("fixture source"),
|
||||
identity: info.scan_identity.expect("fixture identity"),
|
||||
progress: info.scan_progress.expect("fixture coverage plan"),
|
||||
mode: 0,
|
||||
prefixes: None,
|
||||
watch: None,
|
||||
};
|
||||
let slot = Saved::default();
|
||||
let endpoint = info.scan_directory_source.clone().expect("fixture physical source");
|
||||
let physical = Physical {
|
||||
endpoint: endpoint.clone(),
|
||||
device: 1,
|
||||
inode: 2,
|
||||
changed: (123, 0),
|
||||
modified: SystemTime::UNIX_EPOCH,
|
||||
};
|
||||
let mut recorder = Recorder::new(slot.clone(), &info.name, scope, endpoint, Some(physical), None);
|
||||
recorder.entries_read = 300_000;
|
||||
let mut work = DirectoryWork {
|
||||
compacted: true,
|
||||
..Default::default()
|
||||
};
|
||||
for i in 0..300_000 {
|
||||
work.names.add(format!("obj-{i:06}"));
|
||||
work.names.flush_full().await.expect("record real immutable name page");
|
||||
}
|
||||
work.names.finish();
|
||||
[121_034, 121_035].map(|objects| {
|
||||
work.names.seek_after(Some(&format!("obj-{:06}", objects - 1)));
|
||||
work.compacted_total.objects = objects;
|
||||
work.compacted_total.size = objects * 1024;
|
||||
recorder.save(&work);
|
||||
slot.lock()
|
||||
.expect("fixture slot")
|
||||
.take()
|
||||
.expect("sealed directory checkpoint")
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,290 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use std::cmp::Reverse;
|
||||
use std::collections::BinaryHeap;
|
||||
use std::io;
|
||||
use std::sync::{Arc, OnceLock};
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
const PAGE_NAMES: usize = 256;
|
||||
const PAGE_NAME_BYTES: usize = 32 * 1024;
|
||||
|
||||
/// One contiguous UTF-8 string per run. Names from the filesystem cannot
|
||||
/// contain NUL. Checkpoint clones share both the payload and its local index.
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
#[serde(transparent)]
|
||||
pub(super) struct NamePage(Arc<PackedNames>);
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct PackedNames {
|
||||
text: Box<str>,
|
||||
digest: [u8; 32],
|
||||
#[serde(skip)]
|
||||
starts: OnceLock<Box<[usize]>>,
|
||||
#[serde(skip)]
|
||||
valid: OnceLock<bool>,
|
||||
}
|
||||
|
||||
impl NamePage {
|
||||
fn hash(text: &str) -> [u8; 32] {
|
||||
Sha256::new()
|
||||
.chain_update(b"rustfs-scanner-directory-names-v1")
|
||||
.chain_update(text.as_bytes())
|
||||
.finalize()
|
||||
.into()
|
||||
}
|
||||
|
||||
fn new(mut names: Vec<String>) -> Self {
|
||||
names.sort_unstable();
|
||||
let mut text = String::with_capacity(names.iter().map(|n| n.len() + 1).sum());
|
||||
for name in names {
|
||||
text.push_str(&name);
|
||||
text.push('\0');
|
||||
}
|
||||
Self(Arc::new(PackedNames {
|
||||
digest: Self::hash(&text),
|
||||
text: text.into_boxed_str(),
|
||||
starts: OnceLock::new(),
|
||||
valid: OnceLock::new(),
|
||||
}))
|
||||
}
|
||||
|
||||
fn starts(&self) -> &[usize] {
|
||||
self.0.starts.get_or_init(|| {
|
||||
self.names()
|
||||
.scan(0, |offset, name| {
|
||||
let start = *offset;
|
||||
*offset += name.len() + 1;
|
||||
Some(start)
|
||||
})
|
||||
.collect()
|
||||
})
|
||||
}
|
||||
|
||||
fn at(&self, offset: usize) -> &str {
|
||||
self.0.text[offset..].split('\0').next().unwrap_or_default()
|
||||
}
|
||||
|
||||
fn get(&self, index: usize) -> Option<&str> {
|
||||
self.starts().get(index).map(|&offset| self.at(offset))
|
||||
}
|
||||
|
||||
fn after(&self, last: &str) -> usize {
|
||||
self.starts().partition_point(|&offset| self.at(offset) <= last)
|
||||
}
|
||||
|
||||
pub(super) fn contains(&self, name: &str) -> bool {
|
||||
self.starts().binary_search_by(|&offset| self.at(offset).cmp(name)).is_ok()
|
||||
}
|
||||
|
||||
pub(super) fn digest(&self) -> &[u8; 32] {
|
||||
&self.0.digest
|
||||
}
|
||||
|
||||
pub(super) fn names(&self) -> impl Iterator<Item = &str> {
|
||||
self.0.text.split_terminator('\0')
|
||||
}
|
||||
|
||||
pub(super) fn valid(&self) -> bool {
|
||||
*self.0.valid.get_or_init(|| {
|
||||
let text = &self.0.text;
|
||||
text.ends_with('\0')
|
||||
&& text.len() <= PAGE_NAME_BYTES + PAGE_NAMES + 4096
|
||||
&& self.names().take(PAGE_NAMES + 1).count() <= PAGE_NAMES
|
||||
&& self.names().all(super::valid_component)
|
||||
&& self.names().is_sorted()
|
||||
&& self.0.digest == Self::hash(text)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Immutable sorted runs are shared with checkpoint snapshots. The merge
|
||||
/// retains one head per run. At a cold resume each run seeks to the first
|
||||
/// name after the authenticated completed child using binary search.
|
||||
#[derive(Default)]
|
||||
pub(super) struct NamePages {
|
||||
pages: Vec<NamePage>,
|
||||
building: Vec<String>,
|
||||
name_bytes: usize,
|
||||
count: usize,
|
||||
heads: BinaryHeap<Reverse<(String, usize, usize)>>,
|
||||
ready: bool,
|
||||
pending: Option<String>,
|
||||
through: Option<String>,
|
||||
}
|
||||
|
||||
impl NamePages {
|
||||
pub(super) fn len(&self) -> usize {
|
||||
self.count
|
||||
}
|
||||
pub(super) fn ready(&self) -> bool {
|
||||
self.ready
|
||||
}
|
||||
pub(super) fn through(&self) -> Option<&str> {
|
||||
self.through.as_deref()
|
||||
}
|
||||
|
||||
pub(super) fn add(&mut self, name: String) {
|
||||
self.name_bytes = self.name_bytes.saturating_add(name.len());
|
||||
self.count = self.count.saturating_add(1);
|
||||
self.building.push(name);
|
||||
}
|
||||
|
||||
fn flush(&mut self) {
|
||||
if self.building.is_empty() {
|
||||
return;
|
||||
}
|
||||
let page = NamePage::new(std::mem::take(&mut self.building));
|
||||
if let Some(first) = page.get(0) {
|
||||
self.heads.push(Reverse((first.to_owned(), self.pages.len(), 0)));
|
||||
}
|
||||
self.pages.push(page);
|
||||
self.name_bytes = 0;
|
||||
}
|
||||
|
||||
pub(super) async fn flush_full(&mut self) -> io::Result<()> {
|
||||
if self.building.len() >= PAGE_NAMES || self.name_bytes >= PAGE_NAME_BYTES {
|
||||
self.flush();
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn finish(&mut self) {
|
||||
self.flush();
|
||||
self.ready = true;
|
||||
}
|
||||
|
||||
pub(super) fn snapshot(&self) -> Vec<NamePage> {
|
||||
let mut pages = self.pages.clone();
|
||||
if !self.building.is_empty() {
|
||||
pages.push(NamePage::new(self.building.clone()));
|
||||
}
|
||||
pages
|
||||
}
|
||||
|
||||
pub(super) fn restore(pages: &[NamePage], ready: bool, through: Option<&str>) -> Self {
|
||||
let mut names = Self {
|
||||
pages: pages.to_vec(),
|
||||
count: pages.iter().map(|p| p.starts().len()).sum(),
|
||||
ready,
|
||||
..Self::default()
|
||||
};
|
||||
if !ready
|
||||
&& names
|
||||
.pages
|
||||
.last()
|
||||
.is_some_and(|page| page.starts().len() < PAGE_NAMES && page.0.text.len() < PAGE_NAME_BYTES)
|
||||
&& let Some(page) = names.pages.pop()
|
||||
{
|
||||
names.building = page.names().map(str::to_owned).collect();
|
||||
names.name_bytes = names.building.iter().map(String::len).sum();
|
||||
}
|
||||
names.seek_after(through);
|
||||
names
|
||||
}
|
||||
|
||||
pub(super) fn seek_after(&mut self, through: Option<&str>) {
|
||||
self.pending = None;
|
||||
self.through = through.map(str::to_owned);
|
||||
self.heads.clear();
|
||||
for (page_index, page) in self.pages.iter().enumerate() {
|
||||
let start = through.map_or(0, |last| page.after(last));
|
||||
if let Some(next) = page.get(start) {
|
||||
self.heads.push(Reverse((next.to_owned(), page_index, start)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn rewind(&mut self) {
|
||||
self.seek_after(None);
|
||||
}
|
||||
|
||||
pub(super) async fn next(&mut self) -> io::Result<Option<String>> {
|
||||
if self.pending.is_some() {
|
||||
return Ok(self.pending.clone());
|
||||
}
|
||||
while let Some(Reverse((name, page, index))) = self.heads.pop() {
|
||||
if let Some(next) = self.pages[page].get(index + 1) {
|
||||
self.heads.push(Reverse((next.to_owned(), page, index + 1)));
|
||||
}
|
||||
if self.through.as_ref() == Some(&name) {
|
||||
continue;
|
||||
}
|
||||
self.pending = Some(name.clone());
|
||||
return Ok(Some(name));
|
||||
}
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
pub(super) fn acknowledge(&mut self) -> bool {
|
||||
if let Some(name) = self.pending.take() {
|
||||
self.through = Some(name);
|
||||
return true;
|
||||
}
|
||||
false
|
||||
}
|
||||
}
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn packed_membership_shares_names_and_rejects_corrupt_payloads() {
|
||||
let page = NamePage::new((0..PAGE_NAMES).rev().map(|i| format!("obj-{i:06}")).collect());
|
||||
assert!(page.valid());
|
||||
let shared = page.clone();
|
||||
assert!(Arc::ptr_eq(&page.0, &shared.0));
|
||||
let encoded = rmp_serde::to_vec_named(&page).expect("serialize packed names");
|
||||
assert!(encoded.len() < PAGE_NAMES * 12, "checkpoint stores one delimited string per page");
|
||||
let restored: NamePage = rmp_serde::from_slice(&encoded).expect("cold name page");
|
||||
assert!(restored.valid());
|
||||
assert!(restored.contains("obj-000137"));
|
||||
assert_eq!(restored.get(restored.after("obj-000137")), Some("obj-000138"));
|
||||
for text in ["", "obj", "a\0\0b\0", "b\0a\0", "../other\0"] {
|
||||
let malformed = NamePage(Arc::new(PackedNames {
|
||||
text: text.into(),
|
||||
digest: NamePage::hash(text),
|
||||
starts: OnceLock::new(),
|
||||
valid: OnceLock::new(),
|
||||
}));
|
||||
assert!(!malformed.valid(), "invalid membership accepted: {text:?}");
|
||||
}
|
||||
let mut bytes = encoded;
|
||||
let offset = bytes
|
||||
.windows(10)
|
||||
.position(|value| value == b"obj-000137")
|
||||
.expect("encoded name");
|
||||
bytes[offset] = b'x';
|
||||
assert!(!rmp_serde::from_slice::<NamePage>(&bytes).expect("decodable payload").valid());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn interleaved_pages_resume_at_each_unacknowledged_name() {
|
||||
let mut pages = NamePages::default();
|
||||
let mut expected = std::collections::BTreeSet::new();
|
||||
// Each consecutive sorted run interleaves with every other run. One
|
||||
// duplicate straddles two runs and a partial run remains in memory.
|
||||
for run in 0..5 {
|
||||
for index in (0..PAGE_NAMES).rev() {
|
||||
let name = format!("item-{index:04}-{run:02}");
|
||||
expected.insert(name.clone());
|
||||
pages.add(name);
|
||||
pages.flush_full().await.expect("append sorted run");
|
||||
}
|
||||
}
|
||||
pages.add("item-0000-02".to_owned());
|
||||
pages.finish();
|
||||
for name in &expected {
|
||||
assert_eq!(pages.next().await.expect("scheduled child").as_ref(), Some(name));
|
||||
assert_eq!(pages.next().await.expect("retry after cancellation").as_ref(), Some(name));
|
||||
assert!(pages.acknowledge());
|
||||
}
|
||||
assert_eq!(pages.next().await.expect("sealed EOF"), None);
|
||||
pages.rewind();
|
||||
assert_eq!(pages.next().await.expect("fresh coverage").as_ref(), expected.first());
|
||||
}
|
||||
}
|
||||
@@ -8,7 +8,7 @@ use std::ffi::OsString;
|
||||
use std::fs::{FileType, Metadata};
|
||||
use std::future::{Future, poll_fn};
|
||||
use std::io;
|
||||
use std::path::PathBuf;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::pin::Pin;
|
||||
use std::sync::Arc;
|
||||
use std::task::{Context, Poll, ready};
|
||||
@@ -18,31 +18,156 @@ use crate::storage_api::ecstore_spawn_blocking_with_resource_ownership as spawn_
|
||||
// Keep Tokio's existing bounded prefetch and one native job per refill.
|
||||
const DIRECTORY_CHUNK_SIZE: usize = 32;
|
||||
|
||||
#[cfg(any(target_os = "linux", target_os = "macos"))]
|
||||
mod positioned;
|
||||
|
||||
/// Identifies a verified prefix of a physical directory buffer. Only the
|
||||
/// matching directory generation can use its platform-specific kernel cookie.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub(crate) struct DirectoryAnchor {
|
||||
directory: [u8; 32],
|
||||
cookie: u64,
|
||||
entries: u32,
|
||||
prefix: [u8; 32],
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(super) struct DirEntry {
|
||||
entry: Arc<std::fs::DirEntry>,
|
||||
source: EntrySource,
|
||||
file_type: Option<FileType>,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
enum EntrySource {
|
||||
Native(Arc<std::fs::DirEntry>),
|
||||
#[cfg(any(target_os = "linux", target_os = "macos"))]
|
||||
Positioned(Arc<PositionedEntry>),
|
||||
Recorded {
|
||||
parent: Arc<std::path::Path>,
|
||||
name: OsString,
|
||||
},
|
||||
}
|
||||
|
||||
#[cfg(any(target_os = "linux", target_os = "macos"))]
|
||||
#[derive(Clone)]
|
||||
struct PositionedEntry {
|
||||
parent: Arc<Path>,
|
||||
name: OsString,
|
||||
anchor: Option<DirectoryAnchor>,
|
||||
}
|
||||
|
||||
impl DirEntry {
|
||||
pub(super) fn recorded(parent: Arc<std::path::Path>, name: OsString, file_type: Option<FileType>) -> Self {
|
||||
Self {
|
||||
source: EntrySource::Recorded { parent, name },
|
||||
file_type,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn anchor(&self) -> Option<&DirectoryAnchor> {
|
||||
match &self.source {
|
||||
#[cfg(any(target_os = "linux", target_os = "macos"))]
|
||||
EntrySource::Positioned(entry) => entry.anchor.as_ref(),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) fn without_anchor(self) -> Self {
|
||||
#[cfg(any(target_os = "linux", target_os = "macos"))]
|
||||
{
|
||||
let mut entry = self;
|
||||
if let EntrySource::Positioned(position) = &mut entry.source {
|
||||
Arc::make_mut(position).anchor = None;
|
||||
}
|
||||
entry
|
||||
}
|
||||
#[cfg(not(any(target_os = "linux", target_os = "macos")))]
|
||||
self
|
||||
}
|
||||
|
||||
pub(super) fn is_recorded(&self) -> bool {
|
||||
matches!(&self.source, EntrySource::Recorded { .. })
|
||||
}
|
||||
|
||||
pub(super) fn cached_type(&self) -> Option<FileType> {
|
||||
self.file_type
|
||||
}
|
||||
|
||||
pub(super) fn path(&self) -> PathBuf {
|
||||
self.entry.path()
|
||||
match &self.source {
|
||||
EntrySource::Native(entry) => entry.path(),
|
||||
#[cfg(any(target_os = "linux", target_os = "macos"))]
|
||||
EntrySource::Positioned(entry) => entry.parent.join(&entry.name),
|
||||
EntrySource::Recorded { parent, name } => parent.join(name),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn file_name(&self) -> OsString {
|
||||
self.entry.file_name()
|
||||
match &self.source {
|
||||
EntrySource::Native(entry) => entry.file_name(),
|
||||
#[cfg(any(target_os = "linux", target_os = "macos"))]
|
||||
EntrySource::Positioned(entry) => entry.name.clone(),
|
||||
EntrySource::Recorded { name, .. } => name.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn file_type(&self) -> io::Result<FileType> {
|
||||
if let Some(file_type) = self.file_type {
|
||||
return Ok(file_type);
|
||||
}
|
||||
let entry = self.entry.clone();
|
||||
spawn_native(move || entry.file_type()).await.map_err(io::Error::other)?
|
||||
match &self.source {
|
||||
EntrySource::Native(entry) => {
|
||||
let entry = entry.clone();
|
||||
spawn_native(move || entry.file_type()).await.map_err(io::Error::other)?
|
||||
}
|
||||
_ => {
|
||||
let path = self.path();
|
||||
spawn_native(move || std::fs::symlink_metadata(path).map(|meta| meta.file_type()))
|
||||
.await
|
||||
.map_err(io::Error::other)?
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
enum PhysicalReader {
|
||||
Standard(std::fs::ReadDir),
|
||||
#[cfg(any(target_os = "linux", target_os = "macos"))]
|
||||
Positioned(Box<positioned::Reader>),
|
||||
}
|
||||
|
||||
impl PhysicalReader {
|
||||
fn open(path: &Path, indexed: bool, anchor: Option<&DirectoryAnchor>) -> io::Result<Self> {
|
||||
#[cfg(any(target_os = "linux", target_os = "macos"))]
|
||||
if indexed {
|
||||
return positioned::Reader::open(path, anchor).map(Box::new).map(Self::Positioned);
|
||||
}
|
||||
#[cfg(not(any(target_os = "linux", target_os = "macos")))]
|
||||
let _ = indexed;
|
||||
if anchor.is_some() {
|
||||
return Err(io::ErrorKind::Unsupported.into());
|
||||
}
|
||||
std::fs::read_dir(path).map(Self::Standard)
|
||||
}
|
||||
|
||||
fn next(&mut self) -> Option<io::Result<DirEntry>> {
|
||||
match self {
|
||||
Self::Standard(reader) => reader.next().map(|entry| {
|
||||
entry.map(|entry| DirEntry {
|
||||
file_type: entry.file_type().ok(),
|
||||
source: EntrySource::Native(Arc::new(entry)),
|
||||
})
|
||||
}),
|
||||
#[cfg(any(target_os = "linux", target_os = "macos"))]
|
||||
Self::Positioned(reader) => reader.next(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct DirectoryBuffer {
|
||||
reader: std::fs::ReadDir,
|
||||
reader: PhysicalReader,
|
||||
entries: VecDeque<io::Result<DirEntry>>,
|
||||
more: bool,
|
||||
}
|
||||
@@ -55,10 +180,7 @@ impl DirectoryBuffer {
|
||||
break;
|
||||
};
|
||||
let success = entry.is_ok();
|
||||
self.entries.push_back(entry.map(|entry| DirEntry {
|
||||
file_type: entry.file_type().ok(),
|
||||
entry: Arc::new(entry),
|
||||
}));
|
||||
self.entries.push_back(entry);
|
||||
if !success {
|
||||
break;
|
||||
}
|
||||
@@ -74,14 +196,26 @@ enum DirectoryState {
|
||||
|
||||
pub(super) struct ReadDir {
|
||||
state: DirectoryState,
|
||||
#[cfg(test)]
|
||||
slow_directory: bool,
|
||||
}
|
||||
|
||||
impl ReadDir {
|
||||
pub(super) async fn open(path: &str) -> io::Result<Self> {
|
||||
pub(super) async fn open(path: &str, indexed: bool) -> io::Result<Self> {
|
||||
Self::open_directory(path, indexed, None).await
|
||||
}
|
||||
|
||||
pub(super) async fn open_at(path: &str, anchor: &DirectoryAnchor) -> io::Result<Self> {
|
||||
Self::open_directory(path, true, Some(anchor.clone())).await
|
||||
}
|
||||
|
||||
async fn open_directory(path: &str, indexed: bool, anchor: Option<DirectoryAnchor>) -> io::Result<Self> {
|
||||
#[cfg(test)]
|
||||
let slow_directory = super::tests::fair_prefix_quantum::is_slow_directory(path);
|
||||
let path = PathBuf::from(path);
|
||||
let buffer = spawn_native(move || {
|
||||
let mut buffer = DirectoryBuffer {
|
||||
reader: std::fs::read_dir(path)?,
|
||||
reader: PhysicalReader::open(&path, indexed, anchor.as_ref())?,
|
||||
entries: VecDeque::with_capacity(DIRECTORY_CHUNK_SIZE),
|
||||
more: true,
|
||||
};
|
||||
@@ -92,12 +226,18 @@ impl ReadDir {
|
||||
.map_err(io::Error::other)??;
|
||||
Ok(Self {
|
||||
state: DirectoryState::Idle(Some(buffer)),
|
||||
#[cfg(test)]
|
||||
slow_directory,
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) async fn next_entry(&mut self) -> io::Result<Option<DirEntry>> {
|
||||
// Keep a pending refill on the reader, not in the borrowed next-entry
|
||||
// future. Cancelling that future must not skip its prefetched entries.
|
||||
#[cfg(test)]
|
||||
if self.slow_directory {
|
||||
super::tests::fair_prefix_quantum::charge_physical_read().await;
|
||||
}
|
||||
poll_fn(|cx| self.poll_next_entry(cx)).await
|
||||
}
|
||||
|
||||
@@ -201,7 +341,7 @@ mod tests {
|
||||
Resources::Layered(owner.capture(&units.iter().collect::<Vec<_>>()).expect("native ownership"))
|
||||
};
|
||||
drop(units);
|
||||
let mut reader = ReadDir::open(path).await.expect("prepare reader");
|
||||
let mut reader = ReadDir::open(path, false).await.expect("prepare reader");
|
||||
let mut entry = reader.next_entry().await.expect("first entry").expect("file");
|
||||
entry.file_type = None;
|
||||
for _ in 1..DIRECTORY_CHUNK_SIZE {
|
||||
@@ -210,7 +350,7 @@ mod tests {
|
||||
let (release, blocker) = occupy_native_worker().await;
|
||||
let work = async {
|
||||
match operation {
|
||||
"open" => ReadDir::open(path).await.map(|_| ()),
|
||||
"open" => ReadDir::open(path, false).await.map(|_| ()),
|
||||
"refill" => reader.next_entry().await.map(|_| ()),
|
||||
"metadata" => metadata(entry.path().to_str().expect("entry path")).await.map(|_| ()),
|
||||
"file_type" => entry.file_type().await.map(|_| ()),
|
||||
@@ -253,7 +393,9 @@ mod tests {
|
||||
runtime().block_on(async {
|
||||
let count = DIRECTORY_CHUNK_SIZE * 3 + 7;
|
||||
let root = directory(count);
|
||||
let mut reader = ReadDir::open(root.path().to_str().expect("root path")).await.expect("open");
|
||||
let mut reader = ReadDir::open(root.path().to_str().expect("root path"), false)
|
||||
.await
|
||||
.expect("open");
|
||||
let mut names = std::collections::BTreeSet::new();
|
||||
for _ in 0..DIRECTORY_CHUNK_SIZE {
|
||||
let entry = reader.next_entry().await.expect("entry").expect("file");
|
||||
@@ -273,12 +415,71 @@ mod tests {
|
||||
});
|
||||
}
|
||||
|
||||
#[cfg(any(target_os = "linux", target_os = "macos"))]
|
||||
#[test]
|
||||
fn reopened_directory_matches_original_suffix_and_rejects_stale_or_corrupt_positions() {
|
||||
runtime().block_on(async {
|
||||
#[cfg(target_os = "linux")]
|
||||
use std::os::unix::ffi::OsStrExt;
|
||||
let root = directory(900);
|
||||
let path = root.path().to_str().expect("root path");
|
||||
#[cfg(target_os = "linux")]
|
||||
let raw_name = std::ffi::OsStr::from_bytes(b"object-\xff");
|
||||
// APFS requires filenames that are valid Unicode.
|
||||
#[cfg(target_os = "macos")]
|
||||
let raw_name = std::ffi::OsStr::new("object-目录");
|
||||
std::fs::write(root.path().join(raw_name), []).expect("platform-native directory entry");
|
||||
let mut reader = ReadDir::open(path, true).await.expect("open indexed directory");
|
||||
let mut names = Vec::new();
|
||||
let mut anchors = Vec::new();
|
||||
while let Some(entry) = reader.next_entry().await.expect("read indexed entry") {
|
||||
assert!(!entry.is_recorded());
|
||||
names.push(entry.file_name());
|
||||
anchors.push(entry.anchor().expect("native directory position").clone());
|
||||
}
|
||||
drop(reader);
|
||||
assert!(names.iter().any(|name| name == raw_name));
|
||||
assert!(anchors.iter().any(|anchor| anchor.cookie != 0), "exercise a later kernel buffer");
|
||||
for i in [0, 32, 145, 419, names.len() - 1] {
|
||||
let mut resumed = ReadDir::open_at(path, &anchors[i])
|
||||
.await
|
||||
.expect("reopen directory at verified entry");
|
||||
for expected in &names[i + 1..] {
|
||||
assert_eq!(
|
||||
&resumed
|
||||
.next_entry()
|
||||
.await
|
||||
.expect("suffix read")
|
||||
.expect("suffix entry")
|
||||
.file_name(),
|
||||
expected
|
||||
);
|
||||
}
|
||||
assert!(resumed.next_entry().await.expect("resumed EOF").is_none());
|
||||
}
|
||||
|
||||
let mut corrupt = anchors[0].clone();
|
||||
corrupt.prefix[0] ^= 1;
|
||||
assert!(matches!(ReadDir::open_at(path, &corrupt).await, Err(err) if err.kind() == io::ErrorKind::InvalidData));
|
||||
corrupt.entries = u32::MAX;
|
||||
assert!(matches!(ReadDir::open_at(path, &corrupt).await, Err(err) if err.kind() == io::ErrorKind::InvalidData));
|
||||
let other = directory(1);
|
||||
let another = other.path().to_str().expect("other directory");
|
||||
assert!(matches!(ReadDir::open_at(another, &anchors[0]).await, Err(err) if err.kind() == io::ErrorKind::InvalidData));
|
||||
// Removing the bound entry also invalidates its directory generation.
|
||||
std::fs::remove_file(root.path().join(&names[0])).expect("remove anchor entry");
|
||||
assert!(matches!(ReadDir::open_at(path, &anchors[0]).await, Err(err) if err.kind() == io::ErrorKind::InvalidData));
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn directory_prefetch_preserves_empty_and_exact_chunk_boundaries() {
|
||||
runtime().block_on(async {
|
||||
for count in [0, 1, DIRECTORY_CHUNK_SIZE, DIRECTORY_CHUNK_SIZE + 1] {
|
||||
let root = directory(count);
|
||||
let mut reader = ReadDir::open(root.path().to_str().expect("root path")).await.expect("open");
|
||||
let mut reader = ReadDir::open(root.path().to_str().expect("root path"), false)
|
||||
.await
|
||||
.expect("open");
|
||||
let mut names = std::collections::BTreeSet::new();
|
||||
while let Some(entry) = reader.next_entry().await.expect("read entry") {
|
||||
assert!(names.insert(entry.file_name()), "prefetch duplicated an entry");
|
||||
@@ -293,7 +494,9 @@ mod tests {
|
||||
fn failed_refill_is_an_error_after_partial_enumeration() {
|
||||
runtime().block_on(async {
|
||||
let root = directory(DIRECTORY_CHUNK_SIZE + 1);
|
||||
let mut reader = ReadDir::open(root.path().to_str().expect("root path")).await.expect("open");
|
||||
let mut reader = ReadDir::open(root.path().to_str().expect("root path"), false)
|
||||
.await
|
||||
.expect("open");
|
||||
for _ in 0..DIRECTORY_CHUNK_SIZE {
|
||||
assert!(reader.next_entry().await.expect("prefetched entry").is_some());
|
||||
}
|
||||
@@ -322,10 +525,10 @@ mod tests {
|
||||
let root = directory(1);
|
||||
let path = root.path().join("missing");
|
||||
let missing = path.to_str().expect("missing path");
|
||||
assert!(matches!(ReadDir::open(missing).await, Err(error) if error.kind() == io::ErrorKind::NotFound));
|
||||
assert!(matches!(ReadDir::open(missing, false).await, Err(error) if error.kind() == io::ErrorKind::NotFound));
|
||||
assert!(matches!(metadata(missing).await, Err(error) if error.kind() == io::ErrorKind::NotFound));
|
||||
let file = root.path().join("object-0");
|
||||
assert!(ReadDir::open(file.to_str().expect("file path")).await.is_err());
|
||||
assert!(ReadDir::open(file.to_str().expect("file path"), false).await.is_err());
|
||||
#[cfg(unix)]
|
||||
{
|
||||
let link = root.path().join("link");
|
||||
|
||||
@@ -0,0 +1,209 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
//! Reopen an indexed directory at the last entry committed by the walker.
|
||||
|
||||
use std::ffi::OsStr;
|
||||
use std::fs::{File, FileType, Metadata};
|
||||
use std::io;
|
||||
use std::os::unix::ffi::OsStrExt;
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
|
||||
use rustix::fs::{Dir, FileType as NativeType, Mode, OFlags, SeekFrom};
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use super::{DirEntry, DirectoryAnchor, EntrySource, PositionedEntry};
|
||||
|
||||
// Darwin's dirent does not expose Linux d_off. Use the kernel buffer's start
|
||||
// position and verify the consumed prefix of that buffer on a fresh handle.
|
||||
#[cfg(target_os = "macos")]
|
||||
const MAX_ANCHOR_PREFIX: u32 = 4096;
|
||||
#[cfg(target_os = "linux")]
|
||||
const MAX_ANCHOR_PREFIX: u32 = 1;
|
||||
|
||||
fn changed() -> io::Error {
|
||||
io::Error::new(io::ErrorKind::InvalidData, "scanner directory position no longer matches its checkpoint")
|
||||
}
|
||||
|
||||
fn resume_error(error: rustix::io::Errno) -> io::Error {
|
||||
use rustix::io::Errno;
|
||||
match error {
|
||||
Errno::SPIPE | Errno::OPNOTSUPP => io::ErrorKind::Unsupported.into(),
|
||||
Errno::INVAL | Errno::OVERFLOW | Errno::STALE => changed(),
|
||||
other => other.into(),
|
||||
}
|
||||
}
|
||||
|
||||
fn hash_record(prefix: &mut Sha256, record: &rustix::fs::DirEntry) {
|
||||
prefix.update(record.ino().to_le_bytes());
|
||||
// Names cannot contain NUL. Include the terminator to delimit records.
|
||||
prefix.update(record.file_name().to_bytes_with_nul());
|
||||
}
|
||||
|
||||
fn identity(meta: &Metadata) -> [u8; 32] {
|
||||
let mut digest = Sha256::new();
|
||||
digest.update(std::env::consts::OS.as_bytes());
|
||||
for value in [
|
||||
meta.dev().to_le_bytes(),
|
||||
meta.ino().to_le_bytes(),
|
||||
meta.ctime().to_le_bytes(),
|
||||
meta.ctime_nsec().to_le_bytes(),
|
||||
meta.mtime().to_le_bytes(),
|
||||
meta.mtime_nsec().to_le_bytes(),
|
||||
] {
|
||||
digest.update(value);
|
||||
}
|
||||
digest.finalize().into()
|
||||
}
|
||||
|
||||
pub(super) struct Reader {
|
||||
dir: Dir,
|
||||
parent: Arc<Path>,
|
||||
directory_type: FileType,
|
||||
regular_type: Option<FileType>,
|
||||
link_type: Option<FileType>,
|
||||
identity: [u8; 32],
|
||||
next_cookie: Option<u64>,
|
||||
#[cfg(target_os = "macos")]
|
||||
page: DarwinPage,
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
struct DarwinPage {
|
||||
position: File,
|
||||
start: Option<u64>,
|
||||
count: u32,
|
||||
prefix: Sha256,
|
||||
}
|
||||
|
||||
impl Reader {
|
||||
pub(super) fn open(path: &Path, resume: Option<&DirectoryAnchor>) -> io::Result<Self> {
|
||||
let fd = File::from(rustix::fs::open(
|
||||
path,
|
||||
OFlags::RDONLY | OFlags::DIRECTORY | OFlags::CLOEXEC | OFlags::NOFOLLOW,
|
||||
Mode::empty(),
|
||||
)?);
|
||||
let meta = fd.metadata()?;
|
||||
let identity = identity(&meta);
|
||||
let cookie = resume.map_or(0, |anchor| anchor.cookie);
|
||||
if let Some(anchor) = resume {
|
||||
if anchor.directory != identity || !(1..=MAX_ANCHOR_PREFIX).contains(&anchor.entries) {
|
||||
return Err(changed());
|
||||
}
|
||||
if rustix::fs::seek(&fd, SeekFrom::Start(cookie)).map_err(resume_error)? != cookie {
|
||||
return Err(changed());
|
||||
}
|
||||
}
|
||||
#[cfg(target_os = "macos")]
|
||||
let page = DarwinPage {
|
||||
position: fd.try_clone()?,
|
||||
start: Some(cookie),
|
||||
count: 0,
|
||||
prefix: Sha256::new(),
|
||||
};
|
||||
let mut reader = Self {
|
||||
dir: Dir::new(fd)?,
|
||||
parent: Arc::from(path),
|
||||
directory_type: meta.file_type(),
|
||||
regular_type: None,
|
||||
link_type: None,
|
||||
identity,
|
||||
next_cookie: Some(cookie),
|
||||
#[cfg(target_os = "macos")]
|
||||
page,
|
||||
};
|
||||
if let Some(expected) = resume {
|
||||
// A fresh libc stream can choose a different buffer size. Validate
|
||||
// exactly the saved prefix, regardless of its new buffer boundaries.
|
||||
let mut prefix = Sha256::new();
|
||||
for _ in 0..expected.entries {
|
||||
let record = reader.dir.read().transpose().map_err(resume_error)?.ok_or_else(changed)?;
|
||||
hash_record(&mut prefix, &record);
|
||||
reader.anchor(&record);
|
||||
}
|
||||
if prefix.finalize()[..] != expected.prefix {
|
||||
return Err(changed());
|
||||
}
|
||||
}
|
||||
Ok(reader)
|
||||
}
|
||||
|
||||
fn anchor(&mut self, record: &rustix::fs::DirEntry) -> Option<DirectoryAnchor> {
|
||||
#[cfg(target_os = "linux")]
|
||||
let mut single = Sha256::new();
|
||||
#[cfg(target_os = "linux")]
|
||||
let (cookie, entries, prefix) = {
|
||||
let cookie = self.next_cookie;
|
||||
self.next_cookie = u64::try_from(record.offset()).ok();
|
||||
(cookie?, 1, &mut single)
|
||||
};
|
||||
#[cfg(target_os = "macos")]
|
||||
let (cookie, entries, prefix) = {
|
||||
let after = rustix::fs::tell(&self.page.position).ok();
|
||||
if after != self.next_cookie {
|
||||
self.page.start = self.next_cookie;
|
||||
self.next_cookie = after;
|
||||
self.page.count = 0;
|
||||
self.page.prefix = Sha256::new();
|
||||
}
|
||||
self.page.count = self.page.count.saturating_add(1);
|
||||
if self.page.count > MAX_ANCHOR_PREFIX {
|
||||
return None;
|
||||
}
|
||||
(self.page.start?, self.page.count, &mut self.page.prefix)
|
||||
};
|
||||
hash_record(prefix, record);
|
||||
Some(DirectoryAnchor {
|
||||
directory: self.identity,
|
||||
cookie,
|
||||
entries,
|
||||
prefix: prefix.clone().finalize().into(),
|
||||
})
|
||||
}
|
||||
|
||||
fn file_type(&mut self, kind: NativeType, name: &OsStr) -> Option<FileType> {
|
||||
// std::fs::FileType has no public constructor. Reuse the observed type
|
||||
// only when the kernel dirent identified the same class. DT_UNKNOWN
|
||||
// and special files retain the per-entry symlink_metadata fallback.
|
||||
let cached = match kind {
|
||||
NativeType::Directory => return Some(self.directory_type),
|
||||
NativeType::RegularFile => &mut self.regular_type,
|
||||
NativeType::Symlink => &mut self.link_type,
|
||||
_ => return None,
|
||||
};
|
||||
if cached.is_none() {
|
||||
let ty = std::fs::symlink_metadata(self.parent.join(name)).ok()?.file_type();
|
||||
if (ty.is_file() && kind == NativeType::RegularFile) || (ty.is_symlink() && kind == NativeType::Symlink) {
|
||||
*cached = Some(ty);
|
||||
}
|
||||
return Some(ty);
|
||||
}
|
||||
*cached
|
||||
}
|
||||
|
||||
pub(super) fn next(&mut self) -> Option<io::Result<DirEntry>> {
|
||||
loop {
|
||||
let raw = match self.dir.read()? {
|
||||
Ok(record) => record,
|
||||
Err(error) => return Some(Err(error.into())),
|
||||
};
|
||||
let anchor = self.anchor(&raw);
|
||||
let bytes = raw.file_name().to_bytes();
|
||||
if matches!(bytes, b"." | b"..") {
|
||||
continue;
|
||||
}
|
||||
let name = OsStr::from_bytes(bytes);
|
||||
let file_type = self.file_type(raw.file_type(), name);
|
||||
return Some(Ok(DirEntry {
|
||||
source: EntrySource::Positioned(Arc::new(PositionedEntry {
|
||||
parent: self.parent.clone(),
|
||||
name: name.into(),
|
||||
anchor,
|
||||
})),
|
||||
file_type,
|
||||
}));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -30,7 +30,7 @@ use std::sync::Mutex;
|
||||
|
||||
pub(super) mod checkpoint_fixture;
|
||||
pub(super) mod enumeration_restart;
|
||||
mod fair_prefix_quantum;
|
||||
pub(super) mod fair_prefix_quantum;
|
||||
mod incremental_enumeration;
|
||||
mod listing_completion;
|
||||
mod metadata_visits;
|
||||
@@ -403,6 +403,7 @@ async fn build_test_scanner() -> (FolderScanner, std::path::PathBuf) {
|
||||
resume_failed_paths: Vec::new(),
|
||||
coverage_gap: false,
|
||||
raw_enumeration_progress: Vec::new(),
|
||||
directory_checkpoints: HashMap::new(),
|
||||
pending_size_reconciliation_keys: HashSet::new(),
|
||||
pending_size_reconciliation_scopes: HashSet::new(),
|
||||
pending_size_reconciliation_truncated: false,
|
||||
@@ -3144,6 +3145,12 @@ async fn scoped_root_scan_reuses_clean_top_level_entries_and_rescans_dirty_entri
|
||||
parent: None,
|
||||
object_heal_prob_div: 1,
|
||||
};
|
||||
// The sorted queue can reach a budget stop after linking a clean prefix
|
||||
// but before the first dirty child advances the durable sweep frontier.
|
||||
assert!(!scanner.can_reuse_covered_namespace("bucket/clean", false));
|
||||
assert!(scanner.can_restore_linked_child(&folder, &DataUsageEntry::default(), "bucket/clean"));
|
||||
assert!(!scanner.can_restore_linked_child(&folder, &DataUsageEntry::default(), "bucket/dirty"));
|
||||
assert!(!scanner.can_restore_linked_child(&folder, &DataUsageEntry::default(), "bucket/absent"));
|
||||
let mut root = DataUsageEntry::default();
|
||||
scanner
|
||||
.scan_folder(CancellationToken::new(), folder, &mut root)
|
||||
|
||||
@@ -1130,7 +1130,7 @@ async fn checkpoint_admission_log_capture_survives_another_future_dispatcher() {
|
||||
}
|
||||
}
|
||||
|
||||
async fn write_checkpoint_object(root: &std::path::Path, object: &str, versions: &[(Option<Uuid>, i64)]) {
|
||||
pub(super) async fn write_checkpoint_object(root: &std::path::Path, object: &str, versions: &[(Option<Uuid>, i64)]) {
|
||||
let mut metadata = FileMeta::new();
|
||||
for (index, (version_id, size)) in versions.iter().enumerate() {
|
||||
let mut info = FileInfo::new(object, 4, 2);
|
||||
|
||||
@@ -820,6 +820,11 @@ async fn periodic_nested_snapshot_adoption_records_checkpoint_used_on_real_scan_
|
||||
.await
|
||||
.expect("create the real cached subtree directory");
|
||||
let (mut cache, _) = bound_checkpoint();
|
||||
cache.info.scan_directory_source = Some(
|
||||
directory_cursor::physical_source(&scanner.local_disk)
|
||||
.await
|
||||
.expect("fixture disk source"),
|
||||
);
|
||||
cache.info.skip_healing = true;
|
||||
let parent = CancellationToken::new();
|
||||
let budget = ScannerCycleBudget::new_with_progress_tracking(
|
||||
@@ -856,6 +861,80 @@ async fn periodic_nested_snapshot_adoption_records_checkpoint_used_on_real_scan_
|
||||
assert_eq!(budget.reason(), None);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn periodic_compacted_checkpoint_survives_a_crash_inside_the_first_quantum() {
|
||||
use super::super::{enumeration_restart::seed_flat_bucket, fair_prefix_quantum::run_until_finished};
|
||||
let (mut scanner, root) = build_test_scanner().await;
|
||||
let _guard = TestGuard {
|
||||
temp_dir: Some(root.clone()),
|
||||
};
|
||||
// seed_flat_bucket creates its directory; retain the allocated native
|
||||
// disk and initialize this already-created bucket in the same shape.
|
||||
let template = tempfile::tempdir().expect("fixture directory");
|
||||
let cache = seed_flat_bucket(&template.path().join("initial"), 0).await;
|
||||
for index in 0..40 {
|
||||
write_checkpoint_object(&root, &format!("object-{index:04}"), &[(Some(Uuid::from_u128(index + 1)), 3)]).await;
|
||||
}
|
||||
prepare_scanner(&mut scanner, cache);
|
||||
let (tx, mut rx) = mpsc::channel(1);
|
||||
scanner.checkpoint_tx = Some(tx);
|
||||
scanner.checkpoint_objects = SCANNER_CHECKPOINT_OBJECT_INTERVAL - 1;
|
||||
scanner.last_checkpoint_at = Instant::now()
|
||||
.checked_sub(SCANNER_CHECKPOINT_MIN_INTERVAL)
|
||||
.expect("checkpoint interval");
|
||||
let ctx = CancellationToken::new();
|
||||
scanner.budget = ScannerCycleBudget::new_with_progress_tracking(&ctx, Default::default());
|
||||
let mut entry = DataUsageEntry::default();
|
||||
super::super::super::TEST_FORCE_COMPACTION_WIDTH
|
||||
.scope(16, async {
|
||||
let checkpoint = {
|
||||
let scan = scanner.scan_folder(
|
||||
ctx.clone(),
|
||||
CachedFolder {
|
||||
name: "bucket".to_owned(),
|
||||
parent: None,
|
||||
object_heal_prob_div: 1,
|
||||
},
|
||||
&mut entry,
|
||||
);
|
||||
tokio::pin!(scan);
|
||||
tokio::select! {
|
||||
checkpoint=rx.recv()=>checkpoint.expect("compacted owner emitted a periodic checkpoint"),
|
||||
result=&mut scan=>panic!("scan returned before it checkpointed a completed compacted child: {result:?}"),
|
||||
}
|
||||
};
|
||||
assert!(!ctx.is_cancelled());
|
||||
assert!(!checkpoint.info.snapshot_complete);
|
||||
assert!(
|
||||
checkpoint
|
||||
.info
|
||||
.scan_directory_work
|
||||
.iter()
|
||||
.any(|receipt| receipt.progress(&checkpoint.info).is_some())
|
||||
);
|
||||
let total = checkpoint.checked_flatten("bucket").expect("connected partial root");
|
||||
assert!((1..40).contains(&total.objects));
|
||||
assert_eq!(total.size, total.objects * 3);
|
||||
let bytes = checkpoint.marshal_msg().expect("persist running checkpoint");
|
||||
drop(scanner);
|
||||
let durable = DataUsageCache::unmarshal(&bytes).expect("reload checkpoint after crash");
|
||||
let resumed = run_until_finished(
|
||||
&root,
|
||||
durable,
|
||||
ScannerCycleBudgetConfig {
|
||||
max_objects: Some(6),
|
||||
..Default::default()
|
||||
},
|
||||
12,
|
||||
)
|
||||
.await;
|
||||
let complete = resumed.checked_flatten("bucket").expect("restarted compacted root");
|
||||
assert_eq!((complete.objects, complete.size, complete.versions), (40, 120, 40));
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn periodic_nested_snapshot_does_not_publish_shared_compacted_accumulator_under_child_path() {
|
||||
|
||||
@@ -15,6 +15,7 @@ struct Observation {
|
||||
name_bytes: u64,
|
||||
first_entry: Option<String>,
|
||||
last_entry: Option<String>,
|
||||
cancel_owner: Option<CancellationToken>,
|
||||
}
|
||||
|
||||
static OBSERVATION: Mutex<Option<Observation>> = Mutex::new(None);
|
||||
@@ -38,7 +39,11 @@ pub(in crate::scanner_folder) fn observe_raw_entry(dir: &str, name: &std::ffi::O
|
||||
observation.entries += 1;
|
||||
observation.name_bytes += u64::try_from(name.as_encoded_bytes().len()).expect("bounded entry name");
|
||||
if observation.entries >= observation.limit {
|
||||
budget.cancel_for_runtime();
|
||||
if let Some(owner) = &observation.cancel_owner {
|
||||
owner.cancel();
|
||||
} else {
|
||||
budget.cancel_for_runtime();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -63,10 +68,26 @@ pub(in crate::scanner_folder) fn install_raw_entry_budget(root: PathBuf, limit:
|
||||
name_bytes: 0,
|
||||
first_entry: None,
|
||||
last_entry: None,
|
||||
cancel_owner: None,
|
||||
});
|
||||
ObservationGuard
|
||||
}
|
||||
|
||||
pub(in crate::scanner_folder) fn install_raw_entry_owner_cancel(
|
||||
root: PathBuf,
|
||||
limit: u64,
|
||||
owner: CancellationToken,
|
||||
) -> ObservationGuard {
|
||||
let guard = install_raw_entry_budget(root, limit);
|
||||
OBSERVATION
|
||||
.lock()
|
||||
.expect("install owner cancellation")
|
||||
.as_mut()
|
||||
.expect("installed observation")
|
||||
.cancel_owner = Some(owner);
|
||||
guard
|
||||
}
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct Request {
|
||||
|
||||
@@ -74,7 +74,9 @@ pub(in crate::scanner_folder) async fn ordered_entries(
|
||||
}
|
||||
let mut entries = Vec::new();
|
||||
while let Some(entry) = reader.next_entry().await? {
|
||||
entries.push(entry);
|
||||
// This fixture reorders native entries. Their kernel positions no
|
||||
// longer describe the prefixes delivered to the walker.
|
||||
entries.push(entry.without_anchor());
|
||||
}
|
||||
// Real entries and file types, with an explicit adversarial order rather
|
||||
// than an assumption about the filesystem's enumeration order.
|
||||
@@ -533,6 +535,32 @@ async fn erasure_candidate_defers_metadata_until_part_classification_finishes()
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn erasure_candidate_cancelled_batch_keeps_partial_checkpoint_and_reclassifies_on_resume() {
|
||||
check_candidate_resume(CandidateResume::Legacy, 136).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn erasure_candidate_warm_resume_avoids_rereading_sealed_root() {
|
||||
check_candidate_resume(CandidateResume::Warm, 135).await;
|
||||
}
|
||||
|
||||
#[cfg(any(unix, windows))]
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn erasure_candidate_cold_resume_uses_the_validated_sealed_root() {
|
||||
check_candidate_resume(CandidateResume::Cold, 135).await;
|
||||
}
|
||||
|
||||
enum CandidateResume {
|
||||
Warm,
|
||||
Legacy,
|
||||
#[cfg(any(unix, windows))]
|
||||
Cold,
|
||||
}
|
||||
|
||||
async fn check_candidate_resume(kind: CandidateResume, expected_io: u64) {
|
||||
use crate::storage_api::owner::EcstoreDiskAPI;
|
||||
|
||||
let (mut scanner, root, folder) = candidate_scanner().await;
|
||||
let _guard = TestGuard {
|
||||
temp_dir: Some(root.clone()),
|
||||
@@ -596,7 +624,33 @@ async fn erasure_candidate_cancelled_batch_keeps_partial_checkpoint_and_reclassi
|
||||
.expect("validated candidate index");
|
||||
assert_eq!(index.indexed_entries().expect("index entries").len(), 64);
|
||||
let bytes = serde_json::to_vec(&snapshot).expect("serialize partial checkpoint");
|
||||
let reloaded: DataUsageCache = serde_json::from_slice(&bytes).expect("reload partial checkpoint");
|
||||
let mut reloaded: DataUsageCache = serde_json::from_slice(&bytes).expect("reload partial checkpoint");
|
||||
if !matches!(kind, CandidateResume::Warm) {
|
||||
scanner.directory_checkpoints.clear();
|
||||
if matches!(kind, CandidateResume::Legacy) {
|
||||
// A pre-directory-checkpoint writer persisted raw names but no
|
||||
// authenticated physical position or sealed membership manifest.
|
||||
reloaded.info.scan_directory_source = None;
|
||||
reloaded.info.scan_directory_work.clear();
|
||||
} else {
|
||||
for state in &reloaded.info.scan_directory_work {
|
||||
scanner
|
||||
.directory_checkpoints
|
||||
.entry(state.parent().to_owned())
|
||||
.or_default()
|
||||
.insert(state.endpoint().to_owned(), Arc::new(std::sync::Mutex::new(Some(state.clone()))));
|
||||
}
|
||||
}
|
||||
scanner.local_disk = new_disk(
|
||||
&scanner.local_disk.endpoint(),
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("reopen cold scanner disk");
|
||||
}
|
||||
scanner.old_cache = reloaded.clone();
|
||||
scanner.new_cache = reloaded;
|
||||
scanner.raw_enumeration_progress.clear();
|
||||
@@ -615,8 +669,8 @@ async fn erasure_candidate_cancelled_batch_keeps_partial_checkpoint_and_reclassi
|
||||
);
|
||||
assert_eq!(
|
||||
charged_ops(&root),
|
||||
136,
|
||||
"resume recharges actual reads; the raw index is not a filesystem seek"
|
||||
expected_io,
|
||||
"resume charges every physical read, including cold reconstruction of the root"
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -4,6 +4,25 @@
|
||||
use super::enumeration_restart::{install_raw_entry_observer, observed_raw_entries, seed_flat_bucket};
|
||||
use super::*;
|
||||
use crate::scanner_budget::ScannerCycleBudgetConfig;
|
||||
use std::cell::Cell;
|
||||
use std::path::PathBuf;
|
||||
|
||||
const PHYSICAL_READ_COST: Duration = Duration::from_secs(60);
|
||||
|
||||
tokio::task_local! {
|
||||
static SLOW_DIRECTORY: (PathBuf, Cell<usize>);
|
||||
}
|
||||
|
||||
pub(in crate::scanner_folder) fn is_slow_directory(path: &str) -> bool {
|
||||
SLOW_DIRECTORY.try_with(|(directory, _)| directory.as_path() == std::path::Path::new(path)) == Ok(true)
|
||||
}
|
||||
|
||||
/// Charge every read attempt (including replay and EOF) before entries leave
|
||||
/// the native buffer, so cancelling a simulated slow read cannot skip a name.
|
||||
pub(in crate::scanner_folder) async fn charge_physical_read() {
|
||||
SLOW_DIRECTORY.with(|(_, reads)| reads.set(reads.get() + 1));
|
||||
tokio::time::advance(PHYSICAL_READ_COST).await;
|
||||
}
|
||||
|
||||
async fn open_fixture_disk(disk_root: &std::path::Path) -> Arc<Disk> {
|
||||
let endpoint = Endpoint::try_from(disk_root.to_string_lossy().as_ref()).expect("fixture endpoint");
|
||||
@@ -18,6 +37,73 @@ async fn open_fixture_disk(disk_root: &std::path::Path) -> Arc<Disk> {
|
||||
.expect("open synthetic disk")
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
#[serial]
|
||||
async fn wide_directory_runtime_budget_reaches_eof() {
|
||||
const WIDTH: usize = 48;
|
||||
const ROUNDS: usize = 16;
|
||||
let temp = tempfile::tempdir().expect("fixture directory");
|
||||
let disk_root = temp.path().join("disk");
|
||||
let mut cache = seed_flat_bucket(&disk_root, WIDTH).await;
|
||||
let disk = open_fixture_disk(&disk_root).await;
|
||||
let mut anchor = FileInfo::new("large", 4, 2);
|
||||
anchor.volume = "bucket".to_owned();
|
||||
anchor.version_id = Some(Uuid::from_u128(1000));
|
||||
anchor.versioned = true;
|
||||
anchor.mod_time = Some(OffsetDateTime::now_utc());
|
||||
anchor.size = 4096;
|
||||
let mut metadata = FileMeta::new();
|
||||
metadata.add_version(anchor).expect("large fixture object");
|
||||
write_test_object_metadata_bytes(&disk_root, "bucket", "large", &metadata.marshal_msg().expect("encode large object")).await;
|
||||
let parent = CancellationToken::new();
|
||||
let expected = (WIDTH + 1, WIDTH + 4096);
|
||||
let scan = async {
|
||||
for prior_timeouts in 0..ROUNDS {
|
||||
let budget = ScannerCycleBudget::new_with_progress_tracking(
|
||||
&parent,
|
||||
ScannerCycleBudgetConfig {
|
||||
max_duration: Some(8 * PHYSICAL_READ_COST),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
let result = scan_data_folder(
|
||||
budget.token(),
|
||||
budget.clone(),
|
||||
vec![disk.clone()],
|
||||
disk.clone(),
|
||||
cache.clone(),
|
||||
None,
|
||||
HealScanMode::Normal,
|
||||
SCANNER_SLEEPER.clone(),
|
||||
)
|
||||
.await;
|
||||
match result {
|
||||
Ok(complete) => {
|
||||
assert!(prior_timeouts >= 2, "the wide directory must span multiple actual runtime deadlines");
|
||||
assert!(complete.info.snapshot_complete);
|
||||
let totals = complete.checked_flatten("bucket").expect("complete bucket root");
|
||||
assert_eq!((totals.objects, totals.size), expected);
|
||||
return;
|
||||
}
|
||||
Err(ScannerError::PartialCache(partial)) => {
|
||||
assert!(!partial.info.snapshot_complete);
|
||||
cache = DataUsageCache::unmarshal(&partial.marshal_msg().expect("persist partial")).expect("reload partial");
|
||||
}
|
||||
Err(ScannerError::Other(message)) if budget.token().is_cancelled() && message == "Operation cancelled" => {}
|
||||
Err(error) => panic!("unexpected scan failure: {error}"),
|
||||
}
|
||||
assert_eq!(budget.reason(), Some(crate::scanner_budget::ScannerCycleBudgetReason::Runtime));
|
||||
}
|
||||
let reads = SLOW_DIRECTORY.with(|(_, count)| count.get());
|
||||
let totals = cache.checked_flatten("bucket").expect("last partial root");
|
||||
panic!(
|
||||
"wide directory still incomplete after {ROUNDS} fixed runtime deadlines and {reads} physical reads: {} objects, {} bytes",
|
||||
totals.objects, totals.size
|
||||
);
|
||||
};
|
||||
SLOW_DIRECTORY.scope((disk.path().join("bucket"), Cell::new(0)), scan).await;
|
||||
}
|
||||
|
||||
/// A count budget is the operator's total work limit. It must also stop the
|
||||
/// first listing of a directory far wider than that budget, instead of letting
|
||||
/// one read_dir pass finish before the first object is charged.
|
||||
@@ -75,15 +161,28 @@ async fn scan_one_restart_round(
|
||||
disk_root: &std::path::Path,
|
||||
cache: DataUsageCache,
|
||||
config: ScannerCycleBudgetConfig,
|
||||
) -> RestartRound {
|
||||
scan_one_replica_round(disk_root, None, cache, config).await
|
||||
}
|
||||
|
||||
async fn scan_one_replica_round(
|
||||
disk_root: &std::path::Path,
|
||||
peer: Option<&std::path::Path>,
|
||||
cache: DataUsageCache,
|
||||
config: ScannerCycleBudgetConfig,
|
||||
) -> RestartRound {
|
||||
let disk = open_fixture_disk(disk_root).await;
|
||||
let mut disks = vec![disk.clone()];
|
||||
if let Some(peer) = peer {
|
||||
disks.push(open_fixture_disk(peer).await);
|
||||
}
|
||||
let parent = CancellationToken::new();
|
||||
let budget = ScannerCycleBudget::new_with_progress_tracking(&parent, config);
|
||||
let _observation = install_raw_entry_observer(disk.path());
|
||||
let result = scan_data_folder(
|
||||
budget.token(),
|
||||
budget.clone(),
|
||||
vec![disk.clone()],
|
||||
disks,
|
||||
disk,
|
||||
cache.clone(),
|
||||
None,
|
||||
@@ -105,7 +204,7 @@ async fn scan_one_restart_round(
|
||||
}
|
||||
}
|
||||
|
||||
async fn run_until_finished(
|
||||
pub(super) async fn run_until_finished(
|
||||
disk_root: &std::path::Path,
|
||||
mut cache: DataUsageCache,
|
||||
config: ScannerCycleBudgetConfig,
|
||||
@@ -231,3 +330,684 @@ async fn fair_prefix_quantum_budget_stop_keeps_progress_without_certifying_it()
|
||||
let expected = u64::try_from(OBJECTS).expect("fixture size");
|
||||
assert_eq!((totals.objects as u64, totals.versions as u64), (expected, expected));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn cold_compacted_directory_totals_progress_after_every_executor_restart() {
|
||||
let temp = tempfile::tempdir().expect("fixture directory");
|
||||
let root = temp.path().join("disk");
|
||||
let mut cache = seed_flat_bucket(&root, 40).await;
|
||||
let budget = ScannerCycleBudgetConfig {
|
||||
max_objects: Some(6),
|
||||
..Default::default()
|
||||
};
|
||||
super::super::TEST_FORCE_COMPACTION_WIDTH
|
||||
.scope(16, async {
|
||||
let mut partial_counts = Vec::new();
|
||||
for round in 0..16 {
|
||||
let step = scan_one_restart_round(&root, cache.clone(), budget).await;
|
||||
if !step.finished {
|
||||
assert!(
|
||||
step.outcome
|
||||
.durable_scan_progress()
|
||||
.advanced_from(&cache.durable_scan_progress()),
|
||||
"each cold compacted round must earn a coordinator service turn, round {round}"
|
||||
);
|
||||
}
|
||||
cache = step.outcome;
|
||||
let total = cache.checked_flatten("bucket").expect("connected compact bucket");
|
||||
if step.finished {
|
||||
assert!(round > 1);
|
||||
assert!(cache.info.snapshot_complete);
|
||||
assert_eq!((total.objects, total.size, total.versions), (40, 40, 40));
|
||||
return;
|
||||
}
|
||||
assert!(!cache.info.snapshot_complete);
|
||||
partial_counts.push(total.objects);
|
||||
}
|
||||
panic!("cold compacted scan did not finish in 16 rounds; objects per partial: {partial_counts:?}");
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn cold_compacted_replicas_finish_within_their_independent_object_budgets() {
|
||||
let temp = tempfile::tempdir().expect("fixture directory");
|
||||
let first = temp.path().join("a");
|
||||
let other = temp.path().join("b");
|
||||
let mut cache = seed_flat_bucket(&first, 40).await;
|
||||
tokio::fs::create_dir_all(&other).await.expect("fresh replica");
|
||||
for index in (0..40).rev() {
|
||||
let name = format!("object-{index:04}");
|
||||
let metadata = tokio::fs::read(first.join("bucket").join(&name).join("xl.meta"))
|
||||
.await
|
||||
.expect("original metadata");
|
||||
write_test_object_metadata_bytes(&other, "bucket", &name, &metadata).await;
|
||||
}
|
||||
let budget = ScannerCycleBudgetConfig {
|
||||
max_objects: Some(6),
|
||||
..Default::default()
|
||||
};
|
||||
super::super::TEST_FORCE_COMPACTION_WIDTH
|
||||
.scope(16, async {
|
||||
for round in 0..(2 * 40_usize.div_ceil(6) + 1) {
|
||||
let root = if round % 2 == 0 { &first } else { &other };
|
||||
let peer = if round % 2 == 0 { &other } else { &first };
|
||||
let step = scan_one_replica_round(root, Some(peer), cache.clone(), budget).await;
|
||||
if !step.finished {
|
||||
assert!(
|
||||
step.outcome
|
||||
.durable_scan_progress()
|
||||
.advanced_from(&cache.durable_scan_progress()),
|
||||
"cross-disk compacted work should advance durable scheduling, round {round}"
|
||||
);
|
||||
}
|
||||
cache = step.outcome;
|
||||
if step.finished {
|
||||
assert!(round > 1);
|
||||
assert!(cache.info.snapshot_complete);
|
||||
let total = cache.checked_flatten("bucket").expect("complete switched source");
|
||||
assert_eq!((total.objects, total.size), (40, 40));
|
||||
return;
|
||||
}
|
||||
}
|
||||
panic!("two cold physical replicas did not converge within two independent object work budgets");
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn cold_compacted_identical_names_do_not_mix_replica_version_sizes() {
|
||||
use super::checkpoint_fixture::write_checkpoint_object;
|
||||
let temp = tempfile::tempdir().expect("fixture directory");
|
||||
let a = temp.path().join("a");
|
||||
let b = temp.path().join("b");
|
||||
let start = seed_flat_bucket(&a, 40).await;
|
||||
seed_flat_bucket(&b, 40).await;
|
||||
write_checkpoint_object(&b, "object-0000", &[(Some(Uuid::from_u128(100)), 11), (Some(Uuid::from_u128(101)), 23)]).await;
|
||||
let config = ScannerCycleBudgetConfig {
|
||||
max_objects: Some(6),
|
||||
..Default::default()
|
||||
};
|
||||
super::super::TEST_FORCE_COMPACTION_WIDTH
|
||||
.scope(16, async {
|
||||
let first = scan_one_replica_round(&a, Some(&b), start, config).await;
|
||||
assert!(!first.finished);
|
||||
let mut cache = first.outcome;
|
||||
for _ in 0..10 {
|
||||
let round = scan_one_replica_round(&b, Some(&a), cache, config).await;
|
||||
cache = round.outcome;
|
||||
if round.finished {
|
||||
break;
|
||||
}
|
||||
}
|
||||
let complete = cache;
|
||||
assert!(complete.info.snapshot_complete);
|
||||
let counted = complete.checked_flatten("bucket").expect("complete B accounting");
|
||||
// 39 one-byte objects and both retained versions of the remaining key.
|
||||
assert_eq!((counted.objects, counted.size, counted.versions), (40, 73, 41));
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn legacy_unbound_partial_restarts_on_the_actual_physical_replica() {
|
||||
use super::checkpoint_fixture::write_checkpoint_object;
|
||||
let temp = tempfile::tempdir().expect("fixture");
|
||||
let a = temp.path().join("a");
|
||||
let b = temp.path().join("b");
|
||||
let cache = seed_flat_bucket(&a, 40).await;
|
||||
seed_flat_bucket(&b, 40).await;
|
||||
write_checkpoint_object(&b, "object-0000", &[(Some(Uuid::from_u128(200)), 12), (Some(Uuid::from_u128(201)), 31)]).await;
|
||||
let config = ScannerCycleBudgetConfig {
|
||||
max_objects: Some(6),
|
||||
..Default::default()
|
||||
};
|
||||
let first = scan_one_restart_round(&a, cache, config).await;
|
||||
let mut legacy = first.outcome;
|
||||
assert!(legacy.validated_scan_frontier().is_some());
|
||||
legacy.info.scan_directory_source = None;
|
||||
legacy.info.scan_directory_work.clear();
|
||||
legacy.info.scan_source_trees.clear();
|
||||
let upgraded = run_until_finished(&b, legacy, config, 10).await;
|
||||
let total = upgraded.checked_flatten("bucket").expect("B complete");
|
||||
assert_eq!((total.objects, total.size, total.versions), (40, 82, 41));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn cold_linked_prefixes_resume_their_own_replica_coverage() {
|
||||
use super::checkpoint_fixture::write_checkpoint_object;
|
||||
let temp = tempfile::tempdir().expect("fixture directory");
|
||||
let a = temp.path().join("a");
|
||||
let b = temp.path().join("b");
|
||||
let mut cache = seed_flat_bucket(&a, 40).await;
|
||||
seed_flat_bucket(&b, 40).await;
|
||||
write_checkpoint_object(&b, "object-0000", &[(Some(Uuid::from_u128(100)), 11), (Some(Uuid::from_u128(101)), 23)]).await;
|
||||
let config = ScannerCycleBudgetConfig {
|
||||
max_objects: Some(6),
|
||||
..Default::default()
|
||||
};
|
||||
// Six turns per disk leave one final B quantum. Each cold process must
|
||||
// recover that disk's linked tree as well as its complete child coverage.
|
||||
for round in 0..(2 * 40_usize.div_ceil(6) + 1) {
|
||||
let (root, peer) = if round < 12 && round % 2 == 0 { (&a, &b) } else { (&b, &a) };
|
||||
let previous_source = cache.info.scan_directory_source.clone();
|
||||
let before = cache.durable_scan_progress();
|
||||
let result = scan_one_replica_round(root, Some(peer), cache, config).await;
|
||||
if !result.finished {
|
||||
assert!(
|
||||
result.outcome.durable_scan_progress().advanced_from(&before),
|
||||
"linked source must earn another turn, round {round}"
|
||||
);
|
||||
}
|
||||
cache = result.outcome;
|
||||
if round == 1 {
|
||||
let previous_source = previous_source.expect("A's physical source");
|
||||
let endpoints = [
|
||||
previous_source.as_str(),
|
||||
cache.info.scan_directory_source.as_deref().expect("B's physical source"),
|
||||
]
|
||||
.map(|source| source.split_once('|').expect("disk generation and endpoint").1.to_owned())
|
||||
.into_iter()
|
||||
.collect();
|
||||
let mut only_switched = cache.clone();
|
||||
only_switched.activate_directory_source(previous_source, &endpoints);
|
||||
assert!(only_switched.validated_scan_frontier().is_some(), "A's linked coverage is retained");
|
||||
assert!(
|
||||
!only_switched
|
||||
.durable_scan_progress()
|
||||
.advanced_from(&cache.durable_scan_progress()),
|
||||
"moving a linked source between active and dormant earns no new turn"
|
||||
);
|
||||
}
|
||||
|
||||
if result.finished {
|
||||
assert_eq!(root, &b);
|
||||
assert!(cache.info.snapshot_complete);
|
||||
let root = cache.find("bucket").expect("complete linked bucket");
|
||||
assert!(!root.compacted);
|
||||
assert_eq!(root.children.len(), 40);
|
||||
let counted = cache.checked_flatten("bucket").expect("full B view");
|
||||
assert_eq!((counted.objects, counted.size, counted.versions), (40, 73, 41));
|
||||
return;
|
||||
}
|
||||
}
|
||||
panic!("two independent cold linked prefix scans did not converge");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn cold_compacted_changed_replica_does_not_import_another_disks_totals() {
|
||||
let temp = tempfile::tempdir().expect("fixture directory");
|
||||
let first = temp.path().join("a");
|
||||
let other = temp.path().join("b");
|
||||
let baseline = seed_flat_bucket(&first, 40).await;
|
||||
seed_flat_bucket(&other, 40).await;
|
||||
// Object zero was already counted on A but never existed in B's view.
|
||||
// Importing A's subtotal based only on a matching child boundary would
|
||||
// produce 40 objects on B even though its authoritative listing has 39.
|
||||
tokio::fs::remove_dir_all(other.join("bucket/object-0000"))
|
||||
.await
|
||||
.expect("different replica membership");
|
||||
let config = ScannerCycleBudgetConfig {
|
||||
max_objects: Some(6),
|
||||
..Default::default()
|
||||
};
|
||||
super::super::TEST_FORCE_COMPACTION_WIDTH
|
||||
.scope(16, async {
|
||||
let first = scan_one_restart_round(&first, baseline, config).await;
|
||||
assert!(!first.finished);
|
||||
let complete = run_until_finished(&other, first.outcome, config, 14).await;
|
||||
let root = complete.checked_flatten("bucket").expect("finished replica");
|
||||
assert_eq!((root.objects, root.versions, root.size), (39, 39, 39));
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[cfg(any(target_os = "linux", target_os = "macos"))]
|
||||
#[tokio::test(start_paused = true)]
|
||||
#[serial]
|
||||
async fn cold_wide_listing_rotates_physical_endpoints_before_eof() {
|
||||
let temp = tempfile::tempdir().expect("fixture directory");
|
||||
let first = temp.path().join("a");
|
||||
let other = temp.path().join("b");
|
||||
let mut cache = seed_flat_bucket(&first, 49).await;
|
||||
seed_flat_bucket(&other, 49).await;
|
||||
let config = ScannerCycleBudgetConfig {
|
||||
max_duration: Some(8 * PHYSICAL_READ_COST),
|
||||
..Default::default()
|
||||
};
|
||||
for round in 0..22 {
|
||||
let disk = if round % 2 == 0 { &first } else { &other };
|
||||
let peer = if round % 2 == 0 { &other } else { &first };
|
||||
let physical = disk.join("bucket").canonicalize().expect("physical directory");
|
||||
let before = cache.durable_scan_progress();
|
||||
let step = SLOW_DIRECTORY
|
||||
.scope((physical, Cell::new(0)), scan_one_replica_round(disk, Some(peer), cache, config))
|
||||
.await;
|
||||
if !step.finished {
|
||||
assert!(
|
||||
step.outcome.durable_scan_progress().advanced_from(&before),
|
||||
"each independent native resume must advance durable scheduling, round {round}"
|
||||
);
|
||||
}
|
||||
cache = step.outcome;
|
||||
if step.finished {
|
||||
assert!(round > 2);
|
||||
assert!(cache.info.snapshot_complete);
|
||||
let total = cache.checked_flatten("bucket").expect("switched cold listing");
|
||||
assert_eq!((total.objects, total.size), (49, 49));
|
||||
return;
|
||||
}
|
||||
}
|
||||
panic!("rotating before EOF repeatedly overwrote an endpoint's physical progress");
|
||||
}
|
||||
|
||||
#[cfg(any(target_os = "linux", target_os = "macos"))]
|
||||
#[tokio::test(start_paused = true)]
|
||||
#[serial]
|
||||
async fn cold_wide_listing_reaches_eof_after_every_executor_restart() {
|
||||
let temp = tempfile::tempdir().expect("fixture directory");
|
||||
let root = temp.path().join("disk");
|
||||
let mut cache = seed_flat_bucket(&root, 49).await;
|
||||
let budget = ScannerCycleBudgetConfig {
|
||||
max_duration: Some(8 * PHYSICAL_READ_COST),
|
||||
..Default::default()
|
||||
};
|
||||
SLOW_DIRECTORY
|
||||
.scope((root.join("bucket").canonicalize().expect("physical path"), Cell::new(0)), async {
|
||||
for round in 0..16 {
|
||||
let step = scan_one_restart_round(&root, cache.clone(), budget).await;
|
||||
cache = step.outcome;
|
||||
if step.finished {
|
||||
assert!(round > 1);
|
||||
assert!(cache.info.snapshot_complete);
|
||||
let total = cache.checked_flatten("bucket").expect("complete bucket");
|
||||
assert_eq!((total.objects, total.size), (49, 49));
|
||||
return;
|
||||
}
|
||||
assert!(!cache.info.snapshot_complete);
|
||||
}
|
||||
let reads = SLOW_DIRECTORY.with(|(_, reads)| reads.get());
|
||||
panic!("cold physical listing did not reach EOF after 16 rounds and {reads} directory reads");
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn wide_compacted_directory_reuses_only_finished_children() {
|
||||
const WIDTH: usize = 40;
|
||||
let temp = tempfile::tempdir().expect("fixture directory");
|
||||
let disk_root = temp.path().join("disk");
|
||||
let mut cache = seed_flat_bucket(&disk_root, WIDTH).await;
|
||||
let disk = open_fixture_disk(&disk_root).await;
|
||||
let parent = CancellationToken::new();
|
||||
super::super::TEST_FORCE_COMPACTION_WIDTH
|
||||
.scope(16, async {
|
||||
for round in 0..16 {
|
||||
let budget = ScannerCycleBudget::new_with_progress_tracking(
|
||||
&parent,
|
||||
ScannerCycleBudgetConfig {
|
||||
max_objects: Some(6),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
let result = scan_data_folder(
|
||||
budget.token(),
|
||||
budget,
|
||||
vec![disk.clone()],
|
||||
disk.clone(),
|
||||
cache.clone(),
|
||||
None,
|
||||
HealScanMode::Normal,
|
||||
SCANNER_SLEEPER.clone(),
|
||||
)
|
||||
.await;
|
||||
match result {
|
||||
Ok(completed) => {
|
||||
assert!(round > 1);
|
||||
assert!(completed.info.snapshot_complete);
|
||||
let root = completed.find("bucket").expect("compacted bucket");
|
||||
assert!(root.compacted);
|
||||
assert_eq!((root.objects, root.size), (WIDTH, WIDTH));
|
||||
return;
|
||||
}
|
||||
Err(ScannerError::PartialCache(partial)) => {
|
||||
cache = DataUsageCache::unmarshal(&partial.marshal_msg().expect("save compacted checkpoint"))
|
||||
.expect("reload partial");
|
||||
}
|
||||
Err(error) => panic!("unexpected compacted scan failure: {error}"),
|
||||
}
|
||||
}
|
||||
panic!("fixed object budget failed to finish compacted children");
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn wide_directory_replacement_invalidates_parked_entries() {
|
||||
const WIDTH: usize = 48;
|
||||
let temp = tempfile::tempdir().expect("fixture directory");
|
||||
let disk_root = temp.path().join("disk");
|
||||
let initial = seed_flat_bucket(&disk_root, WIDTH).await;
|
||||
let disk = open_fixture_disk(&disk_root).await;
|
||||
let parent = CancellationToken::new();
|
||||
let config = ScannerCycleBudgetConfig {
|
||||
max_entries: Some(3),
|
||||
..Default::default()
|
||||
};
|
||||
let budget = ScannerCycleBudget::new_with_progress_tracking(&parent, config);
|
||||
let Err(ScannerError::PartialCache(partial)) = scan_data_folder(
|
||||
budget.token(),
|
||||
budget,
|
||||
vec![disk.clone()],
|
||||
disk.clone(),
|
||||
initial,
|
||||
None,
|
||||
HealScanMode::Normal,
|
||||
SCANNER_SLEEPER.clone(),
|
||||
)
|
||||
.await
|
||||
else {
|
||||
panic!("the original listing must stop before EOF")
|
||||
};
|
||||
assert!(
|
||||
super::super::parked_bucket_disks("bucket")
|
||||
.iter()
|
||||
.any(|scope| scope.ptr_eq(&Arc::downgrade(&disk)))
|
||||
);
|
||||
let mut cache = *partial;
|
||||
// Rename pins the old inode. Reusing its open stream or its name pages
|
||||
// would visit the vanished objects and omit this replacement's metadata.
|
||||
tokio::fs::rename(disk_root.join("bucket"), temp.path().join("previous"))
|
||||
.await
|
||||
.expect("move old namespace");
|
||||
let metadata = metadata_for_object_version("bucket", "successor", Some(Uuid::from_u128(100)));
|
||||
write_test_object_metadata_bytes(&disk_root, "bucket", "successor", &metadata).await;
|
||||
for _ in 0..6 {
|
||||
let budget = ScannerCycleBudget::new_with_progress_tracking(&parent, config);
|
||||
match scan_data_folder(
|
||||
budget.token(),
|
||||
budget,
|
||||
vec![disk.clone()],
|
||||
disk.clone(),
|
||||
cache.clone(),
|
||||
None,
|
||||
HealScanMode::Normal,
|
||||
SCANNER_SLEEPER.clone(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(complete) => {
|
||||
assert!(complete.info.snapshot_complete);
|
||||
let total = complete.checked_flatten("bucket").expect("current root");
|
||||
assert_eq!((total.objects, total.size), (1, 1));
|
||||
assert!(complete.find("bucket/successor").is_some());
|
||||
assert!(
|
||||
!super::super::parked_bucket_disks("bucket")
|
||||
.iter()
|
||||
.any(|scope| scope.ptr_eq(&Arc::downgrade(&disk)))
|
||||
);
|
||||
return;
|
||||
}
|
||||
Err(ScannerError::PartialCache(partial)) => cache = *partial,
|
||||
Err(error) => panic!("replacement scan failed: {error}"),
|
||||
}
|
||||
}
|
||||
panic!("replacement directory never finished");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn wide_directory_deleted_bucket_releases_a_slot_under_pressure() {
|
||||
use super::super::directory_cursor::{DirectoryScope, PARKED_DIRECTORIES_PER_DISK, ScannerDirectory, is_parked};
|
||||
|
||||
let temp = tempfile::tempdir().expect("fixture directory");
|
||||
let disk_root = temp.path().join("disk");
|
||||
let cache = seed_flat_bucket(&disk_root, 0).await;
|
||||
let disk = open_fixture_disk(&disk_root).await;
|
||||
let scope = DirectoryScope::new(&disk, &cache, FolderScanMode::Namespace, None).expect("physical source and owner");
|
||||
let parent = CancellationToken::new();
|
||||
let budget = ScannerCycleBudget::new_with_progress_tracking(&parent, ScannerCycleBudgetConfig::default());
|
||||
budget.cancel_for_runtime();
|
||||
let retired = disk_root.join("retired");
|
||||
for index in 0..PARKED_DIRECTORIES_PER_DISK {
|
||||
let path = retired.join(index.to_string());
|
||||
tokio::fs::create_dir_all(&path).await.expect("create parked directory");
|
||||
let path = path.to_str().expect("fixture path");
|
||||
let directory = ScannerDirectory::open(path, "bucket", Some(scope.clone()), budget.clone(), None)
|
||||
.await
|
||||
.expect("open source");
|
||||
drop(directory);
|
||||
assert!(is_parked(path));
|
||||
}
|
||||
let next = disk_root.join("replacement");
|
||||
tokio::fs::create_dir(&next).await.expect("current source");
|
||||
let next = next.to_str().expect("fixture path");
|
||||
let directory = ScannerDirectory::open(next, "bucket", Some(scope.clone()), budget.clone(), None)
|
||||
.await
|
||||
.expect("open next source");
|
||||
drop(directory);
|
||||
assert!(!is_parked(next), "active owners retain their bounded slots");
|
||||
tokio::fs::remove_dir_all(retired).await.expect("retire directories");
|
||||
let directory = ScannerDirectory::open(next, "bucket", Some(scope), budget, None)
|
||||
.await
|
||||
.expect("admit after retirement");
|
||||
drop(directory);
|
||||
assert!(is_parked(next), "an obsolete source must not indefinitely exclude new work");
|
||||
super::super::directory_cursor::forget_bucket(&disk, &cache.info);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn raw_directory_replay_reports_discovery_without_recharging_known_entries() {
|
||||
let temp = tempfile::tempdir().expect("fixture directory");
|
||||
let disk_root = temp.path().join("disk");
|
||||
let mut cache = seed_flat_bucket(&disk_root, 0).await;
|
||||
let mut index =
|
||||
RawEnumerationPageWriter::new(RawEnumerationPageIndex::new("bucket", 16).expect("raw index")).expect("raw index writer");
|
||||
for n in 0..64 {
|
||||
tokio::fs::create_dir_all(disk_root.join("bucket").join(format!("object-{n:04}")))
|
||||
.await
|
||||
.expect("existing directory member");
|
||||
index.record_entry(&format!("object-{n:04}")).expect("known directory member");
|
||||
}
|
||||
cache.info.scan_raw_enumeration_page_index = Some(index.checkpoint().expect("persist logical enumeration"));
|
||||
assert_eq!(
|
||||
cache
|
||||
.validated_raw_enumeration_page_index()
|
||||
.expect("valid logical checkpoint")
|
||||
.committed_entries()
|
||||
.expect("known entries")
|
||||
.len(),
|
||||
64
|
||||
);
|
||||
let disk = open_fixture_disk(&disk_root).await;
|
||||
cache.info.scan_directory_source = Some(
|
||||
super::super::directory_cursor::physical_source(&disk)
|
||||
.await
|
||||
.expect("bind the retained logical index to its physical source"),
|
||||
);
|
||||
let parent = CancellationToken::new();
|
||||
let budget = ScannerCycleBudget::new_with_progress_tracking(
|
||||
&parent,
|
||||
ScannerCycleBudgetConfig {
|
||||
max_entries: Some(1),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
let complete = scan_data_folder(
|
||||
budget.token(),
|
||||
budget.clone(),
|
||||
vec![disk.clone()],
|
||||
disk.clone(),
|
||||
cache,
|
||||
None,
|
||||
HealScanMode::Normal,
|
||||
SCANNER_SLEEPER.clone(),
|
||||
)
|
||||
.await
|
||||
.expect("known entries do not exhaust the one-entry budget");
|
||||
assert_eq!(budget.reason(), None);
|
||||
assert_eq!(complete.checked_flatten("bucket").expect("bucket root").objects, 0);
|
||||
assert_eq!(
|
||||
budget.entries_visited(),
|
||||
64,
|
||||
"real directory discovery must keep the remote stream alive while replaying known names"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(any(target_os = "linux", target_os = "macos"))]
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn ordinary_owner_cancellation_persists_directory_position_for_cold_resume() {
|
||||
use super::super::directory_cursor::is_parked;
|
||||
use super::enumeration_restart::install_raw_entry_owner_cancel;
|
||||
|
||||
let temp = tempfile::tempdir().expect("fixture directory");
|
||||
let disk_root = temp.path().join("disk");
|
||||
let mut cache = seed_flat_bucket(&disk_root, 64).await;
|
||||
let tier = runtime_tier_registry_for_cycle(cache.info.next_cycle, cache.info.leader_epoch).await;
|
||||
cache
|
||||
.info
|
||||
.scan_identity
|
||||
.as_mut()
|
||||
.expect("scan identity")
|
||||
.tier_registry_generation = tier.generation;
|
||||
cache.info.tier_registry_generation = Some(tier.generation);
|
||||
let disk = open_fixture_disk(&disk_root).await;
|
||||
let parent = CancellationToken::new();
|
||||
let budget = ScannerCycleBudget::new_with_progress_tracking(&parent, ScannerCycleBudgetConfig::default());
|
||||
let observer = install_raw_entry_owner_cancel(disk.path(), 16, parent.clone());
|
||||
let result = scan_data_folder(
|
||||
budget.token(),
|
||||
budget.clone(),
|
||||
vec![disk.clone()],
|
||||
disk.clone(),
|
||||
cache,
|
||||
None,
|
||||
HealScanMode::Normal,
|
||||
SCANNER_SLEEPER.clone(),
|
||||
)
|
||||
.await;
|
||||
drop(observer);
|
||||
assert!(parent.is_cancelled());
|
||||
assert_eq!(budget.reason(), None, "the owner stopped independently of the cycle budget");
|
||||
let Err(ScannerError::PartialCache(partial)) = result else {
|
||||
panic!("ordinary cancellation must retain a partial cache");
|
||||
};
|
||||
assert_eq!(partial.info.scan_directory_work.len(), 1, "retain the physical directory receipt");
|
||||
let saved = serde_json::to_value(&partial.info.scan_directory_work[0]).expect("directory receipt");
|
||||
assert_eq!(saved["entries_read"], 16);
|
||||
assert!(!is_parked(disk.path().join("bucket").to_str().expect("fixture path")));
|
||||
let path = temp.path().join("checkpoint.bin");
|
||||
tokio::fs::write(&path, partial.marshal_msg().expect("serialize partial checkpoint"))
|
||||
.await
|
||||
.expect("persist partial checkpoint");
|
||||
let mut cache =
|
||||
DataUsageCache::unmarshal(&tokio::fs::read(&path).await.expect("read cold checkpoint")).expect("reload cold checkpoint");
|
||||
let identity = cache.info.scan_identity.expect("checkpoint identity");
|
||||
let source = cache.info.source.expect("checkpoint source");
|
||||
let plan = crate::DataUsageScanPlanDigest([31; 32]);
|
||||
assert_eq!(
|
||||
cache.prepare_bucket_checkpoint("bucket", 2, 1, source, plan, identity),
|
||||
crate::data_usage_define::DataUsageCachePrepareOutcome::Reused
|
||||
);
|
||||
let cold_disk = open_fixture_disk(&disk_root).await;
|
||||
let parent = CancellationToken::new();
|
||||
let budget = ScannerCycleBudget::new_with_progress_tracking(
|
||||
&parent,
|
||||
ScannerCycleBudgetConfig {
|
||||
max_entries: Some(1),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
let result = scan_data_folder(
|
||||
budget.token(),
|
||||
budget,
|
||||
vec![cold_disk.clone()],
|
||||
cold_disk.clone(),
|
||||
cache,
|
||||
None,
|
||||
HealScanMode::Normal,
|
||||
SCANNER_SLEEPER.clone(),
|
||||
)
|
||||
.await;
|
||||
let Err(ScannerError::PartialCache(partial)) = result else {
|
||||
panic!("one newly charged entry must stop the cold continuation");
|
||||
};
|
||||
let resumed = serde_json::to_value(&partial.info.scan_directory_work[0]).expect("resumed directory receipt");
|
||||
assert_eq!(
|
||||
resumed["entries_read"], 17,
|
||||
"the cold reader must continue after its saved native position"
|
||||
);
|
||||
assert_eq!(resumed["physical"], saved["physical"]);
|
||||
let mut cache = DataUsageCache::unmarshal(&partial.marshal_msg().expect("serialize resumed checkpoint"))
|
||||
.expect("reload resumed checkpoint");
|
||||
cache.prepare_bucket_checkpoint("bucket", 3, 2, source, plan, identity);
|
||||
let parent = CancellationToken::new();
|
||||
let budget = ScannerCycleBudget::new_with_progress_tracking(&parent, ScannerCycleBudgetConfig::default());
|
||||
let complete = scan_data_folder(
|
||||
budget.token(),
|
||||
budget,
|
||||
vec![cold_disk.clone()],
|
||||
cold_disk,
|
||||
cache,
|
||||
None,
|
||||
HealScanMode::Normal,
|
||||
SCANNER_SLEEPER.clone(),
|
||||
)
|
||||
.await
|
||||
.expect("cold continuation finishes the bucket");
|
||||
let total = complete.checked_flatten("bucket").expect("complete bucket root");
|
||||
assert_eq!((total.objects, total.size), (64, 64));
|
||||
assert!(complete.info.snapshot_complete);
|
||||
}
|
||||
|
||||
#[cfg(any(target_os = "linux", target_os = "macos"))]
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn changed_directory_listing_clears_its_physical_receipt() {
|
||||
use super::super::directory_cursor::{DirectoryScope, ScannerDirectory};
|
||||
|
||||
let temp = tempfile::tempdir().expect("fixture directory");
|
||||
let disk_root = temp.path().join("disk");
|
||||
let cache = seed_flat_bucket(&disk_root, 16).await;
|
||||
let disk = open_fixture_disk(&disk_root).await;
|
||||
let scope = DirectoryScope::new(&disk, &cache, FolderScanMode::Namespace, None).expect("directory scope");
|
||||
let parent = CancellationToken::new();
|
||||
let budget = ScannerCycleBudget::new_with_progress_tracking(&parent, ScannerCycleBudgetConfig::default());
|
||||
let saved = Arc::new(std::sync::Mutex::new(None));
|
||||
let path = disk_root.join("bucket");
|
||||
let mut directory =
|
||||
ScannerDirectory::open(path.to_str().expect("fixture path"), "bucket", Some(scope), budget, Some(saved.clone()))
|
||||
.await
|
||||
.expect("open physical directory");
|
||||
for _ in 0..8 {
|
||||
directory
|
||||
.peek_entry()
|
||||
.await
|
||||
.expect("read physical entry")
|
||||
.expect("directory member");
|
||||
directory.accept_entry();
|
||||
}
|
||||
directory.work().expect("indexed directory").checkpoint();
|
||||
assert!(saved.lock().expect("receipt").is_some());
|
||||
tokio::fs::rename(&path, disk_root.join("retired"))
|
||||
.await
|
||||
.expect("replace physical directory");
|
||||
tokio::fs::create_dir(&path).await.expect("create successor directory");
|
||||
let error = directory.verify_current().await.expect_err("the old listing is invalid");
|
||||
assert_eq!(error.kind(), std::io::ErrorKind::Interrupted);
|
||||
drop(directory);
|
||||
assert!(
|
||||
saved.lock().expect("receipt").is_none(),
|
||||
"an invalid listing cannot retain a resumable receipt"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1844,6 +1844,7 @@ pub(crate) struct ScannerCycleResult {
|
||||
usage_watch_proofs: Vec<ScannerUsageWatchProof>,
|
||||
usage_watch_retirements: Vec<ScannerUsageWatchProof>,
|
||||
pub(crate) status: ScannerCycleStatus,
|
||||
pub(crate) partial_stage: Option<&'static str>,
|
||||
publication_epoch: Option<u64>,
|
||||
activity_digest: Option<[u8; 32]>,
|
||||
activity_baseline: Option<Arc<crate::scanner::ScannerActivitySnapshot>>,
|
||||
@@ -1870,6 +1871,7 @@ impl ScannerCycleResult {
|
||||
usage_watch_proofs: Vec::new(),
|
||||
usage_watch_retirements: Vec::new(),
|
||||
status,
|
||||
partial_stage: None,
|
||||
publication_epoch: None,
|
||||
activity_digest: None,
|
||||
activity_baseline: None,
|
||||
@@ -1891,6 +1893,11 @@ impl ScannerCycleResult {
|
||||
}
|
||||
}
|
||||
|
||||
fn with_partial_stage(mut self, stage: Option<&'static str>) -> Self {
|
||||
self.partial_stage = stage;
|
||||
self
|
||||
}
|
||||
|
||||
pub(crate) fn with_publication_epoch(mut self, publication_epoch: Option<u64>) -> Self {
|
||||
self.publication_expectation = None;
|
||||
self.publication_epoch = publication_epoch;
|
||||
|
||||
@@ -862,6 +862,7 @@ pub(super) fn completed_data_usage_info(
|
||||
cancelled: bool,
|
||||
) -> Option<(DataUsageInfo, SystemTime)> {
|
||||
completed_usage_candidate(results, scope, tier_registry_names, bucket_plan_complete, cancelled)
|
||||
.ok()
|
||||
.map(|candidate| (candidate.data, candidate.last_update))
|
||||
}
|
||||
|
||||
@@ -880,7 +881,7 @@ pub(super) fn completed_usage_candidate(
|
||||
tier_registry_names: &[String],
|
||||
bucket_plan_complete: bool,
|
||||
cancelled: bool,
|
||||
) -> Option<ValidatedUsageCandidate> {
|
||||
) -> std::result::Result<ValidatedUsageCandidate, CandidateOutcome> {
|
||||
let candidate: std::result::Result<ValidatedUsageCandidate, CandidateOutcome> = (|| {
|
||||
if !bucket_plan_complete {
|
||||
return Err(CandidateOutcome::InventoryIncomplete);
|
||||
@@ -1004,7 +1005,7 @@ pub(super) fn completed_usage_candidate(
|
||||
Err(outcome) => *outcome,
|
||||
},
|
||||
);
|
||||
candidate.ok()
|
||||
candidate
|
||||
}
|
||||
|
||||
fn tier_accounting_proof_is_publishable(
|
||||
|
||||
@@ -15,6 +15,9 @@ use super::*;
|
||||
/// ScannerIOCache implementation for SetDisks: bucket ordering, worker fan-out, merge, and publish.
|
||||
use crate::data_usage_define::clone_cache_value;
|
||||
|
||||
mod walker_slots;
|
||||
use walker_slots::AvailableWalkers;
|
||||
|
||||
pub(super) async fn collect_completed_bucket_results(
|
||||
ctx: CancellationToken,
|
||||
mut cache: DataUsageCache,
|
||||
@@ -990,12 +993,12 @@ impl ScannerIOCache for SetDisks {
|
||||
if turn_bucket_count == 0 {
|
||||
bucket_work_complete.cancel();
|
||||
}
|
||||
for NamespaceScannerWalker {
|
||||
disk,
|
||||
mode: worker_mode,
|
||||
local_admission,
|
||||
} in walkers
|
||||
{
|
||||
let walker_count = walkers.len();
|
||||
let available_walkers = Arc::new(AvailableWalkers::new(walkers, |disk| {
|
||||
crate::storage_api::owner::EcstoreDiskAPI::endpoint(disk.as_ref()).to_string()
|
||||
}));
|
||||
for _ in 0..walker_count {
|
||||
let available_walkers = available_walkers.clone();
|
||||
let service_cohort_clone = service_cohort.clone();
|
||||
let usage_watch_cohort_clone = usage_watch_cohort.clone();
|
||||
let usage_watch_store_clone = usage_watch_store.clone();
|
||||
@@ -1027,18 +1030,18 @@ impl ScannerIOCache for SetDisks {
|
||||
let worker_retired_buckets = retired_buckets.clone();
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
let worker_incarnations = current_bucket_incarnations.clone();
|
||||
let remote_capabilities = match worker_mode {
|
||||
NamespaceScannerWorkerMode::RemoteV4(capabilities) => Some(capabilities),
|
||||
NamespaceScannerWorkerMode::Coordinator => None,
|
||||
};
|
||||
#[cfg(test)]
|
||||
let worker_index = futs.len();
|
||||
futs.push(AbortOnDropHandle::new(tokio::spawn(async move {
|
||||
#[cfg(test)]
|
||||
super::tests::observe_coordinator_bucket_worker_started(worker_index).await;
|
||||
let remote_session_id = uuid::Uuid::new_v4();
|
||||
let mut remote_session_sequence = 0_u64;
|
||||
// Returning at the top of the loop keeps the retiring disk unavailable
|
||||
// on every break, cancellation, or panic, including after requeue.
|
||||
let mut previous_walker = None;
|
||||
loop {
|
||||
if let Some(slot) = previous_walker.take() {
|
||||
available_walkers.release(slot);
|
||||
}
|
||||
// Do not prefetch a FIFO member into an independently
|
||||
// scheduled permit waiter: that can reorder admissions.
|
||||
let permit_wait_start = Instant::now();
|
||||
@@ -1051,18 +1054,39 @@ impl ScannerIOCache for SetDisks {
|
||||
break;
|
||||
}
|
||||
let permit_wait_elapsed = permit_wait_start.elapsed();
|
||||
let bucket = tokio::select! {
|
||||
let (bucket, slot) = tokio::select! {
|
||||
_ = bucket_work_complete_clone.cancelled() => break,
|
||||
_ = ctx_clone.cancelled() => break,
|
||||
bucket = async { bucket_rx_mutex_clone.lock().await.recv().await } => {
|
||||
let Some(bucket) = bucket else {
|
||||
break;
|
||||
};
|
||||
bucket
|
||||
selected = async {
|
||||
let mut receiver = bucket_rx_mutex_clone.lock().await;
|
||||
let bucket = receiver.recv().await?;
|
||||
// Select while still owning FIFO admission, so the oldest bucket
|
||||
// gets its preferred disk before later waiters can borrow it.
|
||||
let parked = crate::scanner_folder::parked_bucket_disks(&bucket.name);
|
||||
let slot = available_walkers.take_for_bucket(&bucket.name, |disk| {
|
||||
parked.iter().any(|owner| owner.ptr_eq(&Arc::downgrade(disk)))
|
||||
});
|
||||
Some((bucket, slot))
|
||||
} => match selected {
|
||||
Some(pair) => pair,
|
||||
None => break,
|
||||
}
|
||||
};
|
||||
let mut work_guard =
|
||||
BucketWorkGuard::new(remaining_bucket_work_clone.clone(), bucket_work_complete_clone.clone());
|
||||
let Some(slot) = slot else {
|
||||
// Each live queue consumer owns at most one walker. On retirement a
|
||||
// consumer and its last walker leave together.
|
||||
record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
|
||||
break;
|
||||
};
|
||||
let slot = previous_walker.insert(slot);
|
||||
let disk = slot.walker.disk.clone();
|
||||
let local_admission = &slot.walker.local_admission;
|
||||
let remote_capabilities = match slot.walker.mode {
|
||||
NamespaceScannerWorkerMode::RemoteV4(capabilities) => Some(capabilities),
|
||||
NamespaceScannerWorkerMode::Coordinator => None,
|
||||
};
|
||||
// Prefix hints are process-local. Never hand one to a
|
||||
// remote or legacy-coordinator disk path.
|
||||
let prefix_scan_scope = disk.is_local().then(|| scope_clone.prefix_scope_for(&bucket.name)).flatten();
|
||||
@@ -1182,8 +1206,8 @@ impl ScannerIOCache for SetDisks {
|
||||
let bucket_checkpoint_plan_digest = bucket_plans.coverage;
|
||||
|
||||
if let Some(capabilities) = remote_capabilities {
|
||||
let request_sequence = remote_session_sequence;
|
||||
let Some(next_sequence) = remote_session_sequence.checked_add(1) else {
|
||||
let request_sequence = slot.remote_session_sequence;
|
||||
let Some(next_sequence) = request_sequence.checked_add(1) else {
|
||||
record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
|
||||
error!(
|
||||
target: "rustfs::scanner::io",
|
||||
@@ -1196,7 +1220,7 @@ impl ScannerIOCache for SetDisks {
|
||||
);
|
||||
break;
|
||||
};
|
||||
remote_session_sequence = next_sequence;
|
||||
slot.remote_session_sequence = next_sequence;
|
||||
let remote_scan = crate::remote_scanner::scan_remote_bucket(
|
||||
&disk,
|
||||
if capabilities.supports_durable_progress || capabilities.supports_complete_observation || capabilities.supports_usage_watch {
|
||||
@@ -1210,7 +1234,7 @@ impl ScannerIOCache for SetDisks {
|
||||
next_cycle: want_cycle,
|
||||
leader_epoch,
|
||||
server_epoch: capabilities.server_epoch,
|
||||
session_id: remote_session_id,
|
||||
session_id: slot.remote_session_id,
|
||||
session_sequence: request_sequence,
|
||||
plans: bucket_plans,
|
||||
supports_durable_progress: capabilities.supports_durable_progress,
|
||||
@@ -1403,7 +1427,7 @@ impl ScannerIOCache for SetDisks {
|
||||
continue;
|
||||
}
|
||||
|
||||
let _local_admission = if let Some(share) = &local_admission {
|
||||
let _local_admission = if let Some(share) = local_admission {
|
||||
match share.acquire(&disk) {
|
||||
Ok(admission) => Some(admission),
|
||||
Err(e) => {
|
||||
@@ -1486,8 +1510,13 @@ impl ScannerIOCache for SetDisks {
|
||||
};
|
||||
|
||||
let mut cache = DataUsageCache::default();
|
||||
let mut revisions = match cache.load_with_revisions(store_clone_clone.clone(), &cache_name).await {
|
||||
Ok(revisions) => revisions,
|
||||
let mut revisions = match Box::pin(cache.load_for_scan(store_clone_clone.clone(), &cache_name, &scan_ctx, &bucket_budget)).await {
|
||||
Ok(Some(revisions)) => revisions,
|
||||
Ok(None) => {
|
||||
work_guard.mark_service_interrupted();
|
||||
record_partial_dirty_bucket(&partial_dirty_buckets_clone, &bucket.name).await;
|
||||
continue;
|
||||
}
|
||||
Err(e) => {
|
||||
record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
|
||||
error!(
|
||||
|
||||
@@ -0,0 +1,128 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use super::NamespaceScannerWalker;
|
||||
use sha2::{Digest as _, Sha256};
|
||||
use std::sync::Mutex;
|
||||
use uuid::Uuid;
|
||||
|
||||
/// All state belonging to one physical walker travels with it, even when a
|
||||
/// different queue consumer makes the next request on that endpoint.
|
||||
pub(super) struct WalkerSlot<T> {
|
||||
pub(super) walker: NamespaceScannerWalker<T>,
|
||||
pub(super) remote_session_id: Uuid,
|
||||
pub(super) remote_session_sequence: u64,
|
||||
endpoint: String,
|
||||
}
|
||||
|
||||
pub(super) struct AvailableWalkers<T>(Mutex<Vec<WalkerSlot<T>>>);
|
||||
|
||||
impl<T> AvailableWalkers<T> {
|
||||
pub(super) fn new(walkers: Vec<NamespaceScannerWalker<T>>, endpoint: impl Fn(&T) -> String) -> Self {
|
||||
Self(Mutex::new(
|
||||
walkers
|
||||
.into_iter()
|
||||
.map(|walker| WalkerSlot {
|
||||
endpoint: endpoint(&walker.disk),
|
||||
walker,
|
||||
remote_session_id: Uuid::new_v4(),
|
||||
remote_session_sequence: 0,
|
||||
})
|
||||
.collect(),
|
||||
))
|
||||
}
|
||||
|
||||
/// Rank available physical endpoints independently of random inventory
|
||||
/// order, waiter wakeups or the number of walker slots on a local disk.
|
||||
pub(super) fn take_for_bucket(&self, bucket: &str, parked: impl Fn(&T) -> bool) -> Option<WalkerSlot<T>> {
|
||||
let mut idle = self.0.lock().unwrap_or_else(|err| err.into_inner());
|
||||
let mut prefix = Sha256::new();
|
||||
prefix.update(bucket.as_bytes());
|
||||
prefix.update([0]);
|
||||
let index = idle
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(index, slot)| {
|
||||
let rank = prefix.clone().chain_update(slot.endpoint.as_bytes()).finalize();
|
||||
(parked(&slot.walker.disk), rank, slot.endpoint.as_str(), index)
|
||||
})
|
||||
.max()
|
||||
.map(|(_, _, _, index)| index)?;
|
||||
Some(idle.swap_remove(index))
|
||||
}
|
||||
|
||||
pub(super) fn release(&self, slot: WalkerSlot<T>) {
|
||||
self.0.lock().unwrap_or_else(|err| err.into_inner()).push(slot);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::scanner_io::NamespaceScannerWorkerMode;
|
||||
|
||||
fn pool(endpoints: &[&'static str]) -> AvailableWalkers<&'static str> {
|
||||
let walkers = endpoints
|
||||
.iter()
|
||||
.copied()
|
||||
.map(|disk| NamespaceScannerWalker {
|
||||
disk,
|
||||
mode: NamespaceScannerWorkerMode::Coordinator,
|
||||
local_admission: None,
|
||||
})
|
||||
.collect();
|
||||
AvailableWalkers::new(walkers, |disk| (*disk).to_owned())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bucket_resumes_on_same_free_endpoint_despite_reordering_or_borrower_changes() {
|
||||
let first = pool(&["disk-c", "disk-a", "disk-b"]);
|
||||
let reordered = pool(&["disk-b", "disk-c", "disk-a"]);
|
||||
for bucket in ["wide", "documents", "snapshots", "videos"] {
|
||||
let mut preferred = first.take_for_bucket(bucket, |_| false).expect("preferred walker");
|
||||
let another = reordered
|
||||
.take_for_bucket(bucket, |_| false)
|
||||
.expect("reordered preferred walker");
|
||||
assert_eq!(preferred.endpoint, another.endpoint);
|
||||
let next = first
|
||||
.take_for_bucket(bucket, |_| false)
|
||||
.expect("fallback when primary is busy");
|
||||
assert_ne!(preferred.endpoint, next.endpoint);
|
||||
let alternative = next.endpoint.clone();
|
||||
let id = preferred.remote_session_id;
|
||||
preferred.remote_session_sequence += 1;
|
||||
let sequence = preferred.remote_session_sequence;
|
||||
first.release(preferred);
|
||||
let resumed = first.take_for_bucket(bucket, |_| false).expect("released primary");
|
||||
assert_eq!((resumed.remote_session_id, resumed.remote_session_sequence), (id, sequence));
|
||||
first.release(resumed);
|
||||
first.release(next);
|
||||
let parked = first
|
||||
.take_for_bucket(bucket, |disk| *disk == alternative)
|
||||
.expect("parked directory");
|
||||
assert_eq!(parked.endpoint, alternative, "follow a parked stream on a previous fallback disk");
|
||||
first.release(parked);
|
||||
reordered.release(another);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retired_slot_leaves_same_endpoint_alternative_and_does_not_leak() {
|
||||
let slots = pool(&["disk-a", "disk-a"]);
|
||||
let retiring = slots.take_for_bucket("wide", |_| false).expect("first walker");
|
||||
let survivor = slots.take_for_bucket("wide", |_| false).expect("second local walker");
|
||||
assert!(slots.take_for_bucket("wide", |_| false).is_none());
|
||||
assert_ne!(retiring.remote_session_id, survivor.remote_session_id);
|
||||
let id = survivor.remote_session_id;
|
||||
drop(retiring);
|
||||
slots.release(survivor);
|
||||
assert_eq!(
|
||||
slots
|
||||
.take_for_bucket("wide", |_| false)
|
||||
.expect("surviving walker")
|
||||
.remote_session_id,
|
||||
id
|
||||
);
|
||||
assert!(slots.take_for_bucket("wide", |_| false).is_none());
|
||||
}
|
||||
}
|
||||
@@ -41,6 +41,48 @@ struct CollectorSlotSummary<'a> {
|
||||
source_difference_sample: Option<CollectorSlotSample<'a>>,
|
||||
}
|
||||
|
||||
impl CollectorSlotSummary<'_> {
|
||||
/// Select one observed blocker. A source emitted in a different slot is valid when the set of sources matches.
|
||||
fn partial_stage(
|
||||
&self,
|
||||
bucket_plan_complete: bool,
|
||||
bucket_scan_status: ScannerBucketScanStatus,
|
||||
dirty_usage_status: DirtyUsageSnapshotStatus,
|
||||
scan_scope_matches: bool,
|
||||
candidate_rejection: Option<rustfs_scanner_metrics::usage_candidate::ScannerUsageCandidateOutcome>,
|
||||
) -> &'static str {
|
||||
if !bucket_plan_complete {
|
||||
"inventory_incomplete"
|
||||
} else if self.unobserved > 0 {
|
||||
"source_result_missing"
|
||||
} else if bucket_scan_status == ScannerBucketScanStatus::Failed {
|
||||
"bucket_worker_failed"
|
||||
} else if bucket_scan_status == ScannerBucketScanStatus::NamespaceNotFound {
|
||||
"bucket_namespace_not_found"
|
||||
} else if bucket_scan_status == ScannerBucketScanStatus::Partial {
|
||||
"bucket_walk_incomplete"
|
||||
} else if dirty_usage_status == DirtyUsageSnapshotStatus::Unverified {
|
||||
"dirty_generation_unverified"
|
||||
} else if self.missing_source > 0 {
|
||||
"missing_source"
|
||||
} else if self.cycle_mismatch > 0 {
|
||||
"source_cycle_mismatch"
|
||||
} else if self.epoch_mismatch > 0 {
|
||||
"source_epoch_mismatch"
|
||||
} else if self.missing_plan > 0 || self.plan_mismatch > 0 {
|
||||
"source_plan_mismatch"
|
||||
} else if !scan_scope_matches {
|
||||
"source_membership"
|
||||
} else if self.incomplete > 0 {
|
||||
"source_snapshot_incomplete"
|
||||
} else if self.missing_last_update > 0 {
|
||||
"missing_last_update"
|
||||
} else {
|
||||
candidate_rejection.map_or("root_coverage_incomplete", |outcome| outcome.as_label())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn summarize_collector_slots<'a>(
|
||||
slots: impl IntoIterator<Item = (usize, DataUsageCacheSource, bool, &'a DataUsageCache)>,
|
||||
cycle: u64,
|
||||
@@ -813,7 +855,9 @@ where
|
||||
if all_buckets.is_empty() {
|
||||
reset_set_scan_gauges();
|
||||
if !bucket_plan_complete {
|
||||
return Ok(ScannerCycleResult::new(ScannerCycleStatus::Incomplete, None).with_publication_epoch(publication_epoch));
|
||||
return Ok(ScannerCycleResult::new(ScannerCycleStatus::Incomplete, None)
|
||||
.with_partial_stage(Some("inventory_incomplete"))
|
||||
.with_publication_epoch(publication_epoch));
|
||||
}
|
||||
let publication_blocked = store.scanner_data_usage_publication_blocked().await;
|
||||
let activity_after = probe_scanner_cycle_activity(
|
||||
@@ -850,7 +894,9 @@ where
|
||||
tier_registry_generation: Some(tier_registry_generation),
|
||||
},
|
||||
) else {
|
||||
return Ok(ScannerCycleResult::new(ScannerCycleStatus::Incomplete, None).with_publication_epoch(publication_epoch));
|
||||
return Ok(ScannerCycleResult::new(ScannerCycleStatus::Incomplete, None)
|
||||
.with_partial_stage(Some("empty_namespace_guard"))
|
||||
.with_publication_epoch(publication_epoch));
|
||||
};
|
||||
let (empty_usage, publication_expectation) = candidate.prepare(status, &pending_buckets);
|
||||
let usage_update_emitted = if should_publish_observational_snapshot(status) {
|
||||
@@ -859,7 +905,11 @@ where
|
||||
publish_usage_snapshot(&updates, status, &pending_buckets, empty_usage).await?
|
||||
};
|
||||
if !usage_update_emitted {
|
||||
return Ok(ScannerCycleResult::new(status, None).with_publication_epoch(publication_epoch));
|
||||
let partial_stage =
|
||||
(dirty_usage_status == DirtyUsageSnapshotStatus::Unverified).then_some("dirty_generation_unverified");
|
||||
return Ok(ScannerCycleResult::new(status, None)
|
||||
.with_partial_stage(partial_stage)
|
||||
.with_publication_epoch(publication_epoch));
|
||||
}
|
||||
if status == ScannerCycleStatus::Complete {
|
||||
complete_tier_registry_cycle(want_cycle, leader_epoch);
|
||||
@@ -902,7 +952,9 @@ where
|
||||
"Scanner set state update detected missing disk sets"
|
||||
);
|
||||
reset_set_scan_gauges();
|
||||
return Ok(ScannerCycleResult::new(ScannerCycleStatus::Incomplete, None).with_publication_epoch(publication_epoch));
|
||||
return Ok(ScannerCycleResult::new(ScannerCycleStatus::Incomplete, None)
|
||||
.with_partial_stage(Some("source_inventory_empty"))
|
||||
.with_publication_epoch(publication_epoch));
|
||||
}
|
||||
|
||||
let dispatch_limit = scanner_max_concurrent_set_scans(total_results);
|
||||
@@ -1278,7 +1330,7 @@ where
|
||||
remote_certification_attempted,
|
||||
);
|
||||
let all_bucket_names = all_buckets.iter().map(|bucket| bucket.name.clone()).collect::<Vec<_>>();
|
||||
let mut completed_usage = completed_usage_candidate(
|
||||
let completed_candidate = completed_usage_candidate(
|
||||
&results,
|
||||
&ScannerSnapshotScope {
|
||||
sources: &expected_sources,
|
||||
@@ -1295,6 +1347,8 @@ where
|
||||
bucket_plan_complete,
|
||||
owner_ctx.is_cancelled(),
|
||||
);
|
||||
let candidate_rejection = completed_candidate.as_ref().err().copied();
|
||||
let mut completed_usage = completed_candidate.ok();
|
||||
if let (Some(candidate), Some(baseline)) = (
|
||||
completed_usage.as_mut(),
|
||||
persisted_usage_baseline
|
||||
@@ -1325,7 +1379,11 @@ where
|
||||
dirty_usage_status,
|
||||
activity_status,
|
||||
);
|
||||
if cycle_status == ScannerCycleStatus::Incomplete && result.is_ok() && !budget_elapsed && !owner_ctx.is_cancelled() {
|
||||
let partial_stage = if cycle_status == ScannerCycleStatus::Incomplete
|
||||
&& result.is_ok()
|
||||
&& !budget_elapsed
|
||||
&& !owner_ctx.is_cancelled()
|
||||
{
|
||||
let slots =
|
||||
set_disks
|
||||
.iter()
|
||||
@@ -1336,6 +1394,35 @@ where
|
||||
(index, DataUsageCacheSource::new(set.pool_index, set.set_index), *observed, snapshot)
|
||||
});
|
||||
let summary = summarize_collector_slots(slots, want_cycle, leader_epoch, scan_plan_digest);
|
||||
let mut stage = summary.partial_stage(
|
||||
bucket_plan_complete,
|
||||
bucket_scan_status,
|
||||
dirty_usage_status,
|
||||
scan_scope_matches,
|
||||
candidate_rejection,
|
||||
);
|
||||
// A fully traversed mixed-plan observation needs another certification sweep. Only
|
||||
// attribute the stop to that fence when every uncarried bucket has fresh source evidence.
|
||||
if stage == "bucket_walk_incomplete"
|
||||
&& let Some((snapshot, _)) = &observational_usage
|
||||
&& partial_buckets.difference(&carried_partial_buckets).all(|bucket| {
|
||||
snapshot
|
||||
.usage_snapshot_bucket_observations
|
||||
.get(bucket)
|
||||
.is_some_and(|observation| {
|
||||
observation
|
||||
.sources
|
||||
.iter()
|
||||
.all(|source| source.scanner_epoch == leader_epoch && source.scanner_cycle == want_cycle)
|
||||
&& observation
|
||||
.sources
|
||||
.iter()
|
||||
.any(|source| source.started_plan != source.requested_plan)
|
||||
})
|
||||
})
|
||||
{
|
||||
stage = "mixed_plan_recertification_pending";
|
||||
}
|
||||
let sample = summary.sample;
|
||||
let source_difference = summary.source_difference_sample;
|
||||
let sample_snapshot = sample.filter(|sample| sample.observed).map(|sample| sample.snapshot);
|
||||
@@ -1345,6 +1432,7 @@ where
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_IO,
|
||||
state = "incomplete_cycle_collector_summary",
|
||||
partial_stage = stage,
|
||||
cycle = want_cycle,
|
||||
leader_epoch,
|
||||
configured_sets = set_disks.len(),
|
||||
@@ -1390,7 +1478,10 @@ where
|
||||
sample_snapshot_plan_matches = sample_snapshot.and_then(|snapshot| snapshot.info.scan_plan_digest.map(|plan| plan == scan_plan_digest)),
|
||||
"Scanner recorded collector observations for an incomplete cycle"
|
||||
);
|
||||
}
|
||||
Some(stage)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
// Certification is per bucket: the pending set is read after the
|
||||
// guardrail verdict so a bucket written during the walk is reported in
|
||||
// the same publication that certifies the others. A bucket whose walk
|
||||
@@ -1583,6 +1674,7 @@ where
|
||||
complete_tier_registry_cycle(want_cycle, leader_epoch);
|
||||
}
|
||||
Ok(ScannerCycleResult::new(cycle_status, dirty_usage_clear)
|
||||
.with_partial_stage(partial_stage)
|
||||
.with_usage_watch_proofs(usage_watch_proofs.into_values().collect())
|
||||
.with_usage_watch_retirements(usage_watch_retirements)
|
||||
.with_publication_epoch(publication_epoch)
|
||||
|
||||
@@ -127,6 +127,8 @@ async fn collector_reports_an_unobserved_slot_after_a_stale_set_snapshot() {
|
||||
let events = collector_events(&events);
|
||||
assert_eq!(events.len(), 1, "a non-budget incomplete cycle needs one collector observation");
|
||||
let fields = events[0];
|
||||
assert_eq!(fields["partial_stage"], "source_result_missing");
|
||||
assert_eq!(result.partial_stage, Some("source_result_missing"));
|
||||
assert_eq!(fields["cycle"], 7);
|
||||
assert_eq!(fields["leader_epoch"], LEADER_EPOCH);
|
||||
assert_eq!(fields["expected_sources"], 1);
|
||||
@@ -171,6 +173,7 @@ async fn collector_reports_received_incomplete_cache_only_for_non_budget_partial
|
||||
let events = collector_events(&events);
|
||||
assert_eq!(events.len(), 1, "a non-budget incomplete cycle needs one collector observation");
|
||||
let fields = events[0];
|
||||
assert_eq!(fields["partial_stage"], second.partial_stage.expect("non-budget incomplete stage"));
|
||||
assert_eq!(fields["observed_slots"], 1);
|
||||
assert_eq!(fields["unobserved_slots"], 0);
|
||||
assert_eq!(fields["sample_observed"], true);
|
||||
@@ -187,6 +190,7 @@ async fn collector_reports_received_incomplete_cache_only_for_non_budget_partial
|
||||
|
||||
let ((third, snapshot, _), events) = capture_cycle(run_cycle(&store, 3, None, false)).await;
|
||||
assert_eq!(third.status, ScannerCycleStatus::Complete);
|
||||
assert_eq!(third.partial_stage, None);
|
||||
assert!(snapshot.is_some_and(|snapshot| snapshot.usage_snapshot_complete));
|
||||
assert!(
|
||||
collector_events(&events).is_empty(),
|
||||
|
||||
@@ -82,7 +82,9 @@ fn completed_outcome(
|
||||
},
|
||||
};
|
||||
assert_one_outcome(Kind::Completed, outcome, || {
|
||||
completed_usage_candidate(caches, &scope, &[], plan_complete, owner_cancelled)
|
||||
let candidate = completed_usage_candidate(caches, &scope, &[], plan_complete, owner_cancelled);
|
||||
assert_eq!(candidate.as_ref().err().copied().unwrap_or(Outcome::Accepted), outcome);
|
||||
candidate.ok()
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -1,57 +0,0 @@
|
||||
# SSE Crate Instructions
|
||||
|
||||
Applies to `crates/sse/`.
|
||||
|
||||
## Boundary
|
||||
|
||||
- This crate sits between the S3 layer and the storage engine and depends on neither. Its `rustfs-*` dependencies are pinned by `scripts/check_architecture_migration_rules.sh` to `rustfs-config`, `rustfs-kms`, `rustfs-policy`, `rustfs-s3-types`, and `rustfs-utils`; see [crate boundaries](../../docs/architecture/crate-boundaries.md).
|
||||
- Anything the crate needs from the running server arrives through `SseHost` (`src/host.rs`), passed as the first argument of every entry point that consults the server. Do not add a global that stands in for the host: a caller must not be able to reach key selection without naming the server it speaks for.
|
||||
- Translation to and from storage-engine types (`ObjectEncryptionResolver`, `WriteEncryption`, `MultipartCompletionAuthorization`) and request types belongs in `rustfs/src/storage/sse.rs`, not here.
|
||||
- Report failures as `SseError`, classified with the S3 error code the caller answers with. A failure that did not originate from the request goes through `SseError::internal`, which keeps its detail out of the response.
|
||||
|
||||
## Layout
|
||||
|
||||
- `src/startup.rs` settles the state that is fixed for the lifetime of the process and reports it. Nothing that follows the KMS service state belongs there: the service follows configuration and can be started, restarted, or stopped from the admin API, so every choice that depends on it is made per request.
|
||||
- `src/core/` holds the entry points (`write.rs`, `read.rs`, `rewrap.rs`) and the request principal (`request.rs`). An entry point only orders the steps; the decisions live in the two layers below it.
|
||||
- `src/core/s3/` is what S3 defines: the SSE headers of a request and a response, the scheme a write or a stored object resolves to, the SSE-C checks, and the authorization of a KMS key.
|
||||
- `src/core/keying/` is how RustFS forms, seals, and records an object's key. `direct.rs` and `object.rs` are the two modes, and each owns both directions of its stored record: the code that writes a key's metadata sits next to the code that reads it back. `mod.rs` is the only place that says which mode a build writes; `stored.rs` holds the order in which a read recognizes the modes and picks the provider that unwraps the data key; `metadata.rs` records the scheme and lists every stored key name.
|
||||
- `src/kms/` holds the data-key providers: they wrap and unwrap a data key and decide nothing about schemes. The write and rewrap entry points, `core/keying/stored.rs`, and the startup check of the local master key call them; `core/s3/` never does.
|
||||
|
||||
Rules that keep the layers apart:
|
||||
|
||||
- `core/keying/` never imports from `core/s3/`. `core/s3/` reads stored metadata only through `core/keying/metadata.rs`. Anything else that needs both layers is joined in an entry point.
|
||||
- `#[cfg(feature = "rio-v2")]` appears only under `core/keying/` and `kms/`. An entry point or `core/s3/` that needs a build-dependent answer asks `core/keying/` for it.
|
||||
- A new stored format, including a MinIO-compatible one, is added under `core/keying/` (and `kms/` when it brings its own wrapping) without changes under `core/s3/`: a mode file with its record and read functions, one entry in the recognition order of `stored.rs`, and its key names in the lists of `metadata.rs`.
|
||||
- A read decides from what the object recorded. The scheme a request advertises and whether a KMS is running now are write policy and runtime state; neither says how an existing object was keyed.
|
||||
|
||||
## Security
|
||||
|
||||
- Never log plaintext customer keys, decrypted key material, data-key envelopes, nonces, or SSE request payloads, including through error strings and `Debug` output.
|
||||
- KMS audit tags may carry only the fields documented on `KmsRequestAuditState`.
|
||||
- The `test-util` feature exposes authorization overrides and DEK-provider injection. It is a dev-dependency feature only; never enable it from a normal dependency.
|
||||
|
||||
## Change Coordination
|
||||
|
||||
When changing key resolution, metadata, or error classification, verify compatibility with:
|
||||
|
||||
- `rustfs/src/storage/sse.rs` (the host implementation and the engine-boundary adapters)
|
||||
- `crates/kms/`
|
||||
- `crates/utils/src/http/object_encryption_keys.rs` (the stored-metadata key vocabulary)
|
||||
- `rustfs/src/kms_rekey.rs` and `rustfs/src/admin/handlers/kms*.rs`
|
||||
|
||||
## Testing
|
||||
|
||||
Run both builds; the `rio-v2` feature swaps the object-key scheme and compiles different code paths:
|
||||
|
||||
```bash
|
||||
cargo nextest run -p rustfs-sse --no-tests fail
|
||||
cargo nextest run -p rustfs-sse --features rio-v2 --no-tests fail
|
||||
```
|
||||
|
||||
The adapter tests live with the adapter:
|
||||
|
||||
```bash
|
||||
cargo nextest run -p rustfs --no-tests fail -E 'test(/^storage::sse::/)'
|
||||
```
|
||||
|
||||
A change that should leave client-visible behavior or the stored form untouched (a refactor, a move between layers) is checked against a real server: run `scripts/test/sse_blackbox/` on the binary before and after the change, compare the two runs, and cross-read their data directories. Its README has the commands.
|
||||
@@ -1,70 +0,0 @@
|
||||
# Copyright 2024 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
[package]
|
||||
name = "rustfs-sse"
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
rust-version.workspace = true
|
||||
version.workspace = true
|
||||
homepage.workspace = true
|
||||
description = "S3 server-side encryption (SSE-S3, SSE-KMS, SSE-C) key resolution and metadata for RustFS."
|
||||
keywords = ["sse", "encryption", "s3", "kms", "rustfs"]
|
||||
categories = ["cryptography"]
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[features]
|
||||
default = []
|
||||
# MinIO-compatible sealed object keys (DARE). Mirrors the `rio-v2` feature of
|
||||
# the server binary, which enables both together.
|
||||
rio-v2 = ["dep:chacha20poly1305", "dep:hmac", "dep:sha2"]
|
||||
# Test seams for dependents: per-principal authorization overrides, DEK
|
||||
# provider injection and metadata fixtures. Never enable in a release build.
|
||||
test-util = []
|
||||
|
||||
[dependencies]
|
||||
# The only RustFS crates this one may reach. It sits between the S3 layer and
|
||||
# the storage engine and must not depend on either: what it needs from the
|
||||
# running server arrives through `SseHost`.
|
||||
rustfs-config = { workspace = true }
|
||||
rustfs-kms = { workspace = true }
|
||||
rustfs-policy = { workspace = true }
|
||||
rustfs-s3-types = { workspace = true }
|
||||
rustfs-utils = { workspace = true, features = ["http", "path"] }
|
||||
|
||||
aes-gcm = { workspace = true, features = ["rand_core", "zeroize"] }
|
||||
async-trait = { workspace = true }
|
||||
base64-simd = { workspace = true }
|
||||
chacha20poly1305 = { workspace = true, optional = true }
|
||||
hmac = { workspace = true, optional = true }
|
||||
http = { workspace = true }
|
||||
md-5 = { workspace = true }
|
||||
rand = { workspace = true }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
serde_json = { workspace = true }
|
||||
sha2 = { workspace = true, optional = true }
|
||||
tracing = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
# The fake AWS KMS endpoint the backend-selection tests talk to.
|
||||
axum = { workspace = true }
|
||||
temp-env = { workspace = true, features = ["async_closure"] }
|
||||
tempfile = { workspace = true }
|
||||
tokio = { workspace = true, features = ["macros", "net", "rt-multi-thread", "sync", "time"] }
|
||||
|
||||
[lib]
|
||||
doctest = false
|
||||
@@ -1,292 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Per-request summary of data-plane KMS use, attached to the request's audit entry.
|
||||
|
||||
use crate::core::request::SseKmsPrincipal;
|
||||
use crate::error::SseError;
|
||||
use base64_simd::STANDARD as BASE64_STANDARD;
|
||||
use rustfs_kms::is_data_key_envelope;
|
||||
use rustfs_s3_types::S3ErrorCode;
|
||||
use rustfs_utils::http::object_encryption_keys::{INTERNAL_ENCRYPTION_KEY_HEADER, SSEType, normalize_managed_metadata};
|
||||
use serde_json::Value;
|
||||
use std::collections::{BTreeSet, HashMap};
|
||||
use std::sync::{Arc, LazyLock, Mutex, Weak};
|
||||
|
||||
/// Tag keys carrying the KMS summary on an S3 audit entry.
|
||||
///
|
||||
/// Audit consumers key off these strings, so they are a wire contract: append
|
||||
/// new keys rather than renaming existing ones.
|
||||
const KMS_AUDIT_TAG_SSE_TYPE: &str = "sseType";
|
||||
|
||||
const KMS_AUDIT_TAG_KEY_ID: &str = "kmsKeyId";
|
||||
|
||||
const KMS_AUDIT_TAG_KEY_VERSION: &str = "kmsKeyVersion";
|
||||
|
||||
const KMS_AUDIT_TAG_OUTCOME: &str = "kmsOutcome";
|
||||
|
||||
const KMS_AUDIT_TAG_ERROR_CLASS: &str = "kmsErrorClass";
|
||||
|
||||
/// Separator for the rare request that touched more than one key or scheme.
|
||||
const KMS_AUDIT_VALUE_SEPARATOR: &str = ",";
|
||||
|
||||
/// What the data path did with KMS while serving one S3 request.
|
||||
///
|
||||
/// This rides on the request's existing S3 audit entry instead of becoming its
|
||||
/// own event: the entry already carries the principal, request ID and API, so
|
||||
/// attaching the KMS outcome costs no extra event volume and needs no second
|
||||
/// correlation key.
|
||||
///
|
||||
/// # Redaction
|
||||
///
|
||||
/// Only the fields below may ever be recorded. The S3 audit entry fans out to
|
||||
/// every configured target, several of which sit outside the KMS trust
|
||||
/// boundary, so nothing derived from a data key — plaintext, ciphertext,
|
||||
/// envelope, nonce — nor any caller-supplied encryption-context value belongs
|
||||
/// here. Key identifiers and scheme names are configuration, not secrets.
|
||||
#[derive(Debug, Default)]
|
||||
struct KmsRequestAuditState {
|
||||
sse_types: BTreeSet<&'static str>,
|
||||
key_ids: BTreeSet<String>,
|
||||
key_versions: BTreeSet<u32>,
|
||||
/// Set by the first failure. A request that failed any KMS interaction is
|
||||
/// reported as failed even if others succeeded: a partially completed
|
||||
/// envelope operation is not a success for an audit reader.
|
||||
error_class: Option<&'static str>,
|
||||
recorded: bool,
|
||||
}
|
||||
|
||||
/// Shared accumulator for one request's KMS summary.
|
||||
///
|
||||
/// Written by the managed-SSE code paths through the request's principal, read
|
||||
/// once through [`KmsRequestAuditScope::audit_tags`] by whoever owns the
|
||||
/// request's audit entry.
|
||||
#[derive(Debug, Default)]
|
||||
pub(crate) struct KmsRequestAudit(Mutex<KmsRequestAuditState>);
|
||||
|
||||
impl KmsRequestAudit {
|
||||
pub(crate) fn record(
|
||||
&self,
|
||||
sse_type: SSEType,
|
||||
key_id: Option<&str>,
|
||||
key_version: Option<u32>,
|
||||
error_class: Option<&'static str>,
|
||||
) {
|
||||
// A poisoned lock costs the audit entry its KMS tags; it must never cost
|
||||
// the request its result.
|
||||
let Ok(mut state) = self.0.lock() else {
|
||||
return;
|
||||
};
|
||||
|
||||
state.recorded = true;
|
||||
state.sse_types.insert(sse_type.audit_label());
|
||||
if let Some(key_id) = key_id.filter(|value| !value.is_empty()) {
|
||||
state.key_ids.insert(key_id.to_string());
|
||||
}
|
||||
if let Some(key_version) = key_version {
|
||||
state.key_versions.insert(key_version);
|
||||
}
|
||||
if state.error_class.is_none() {
|
||||
state.error_class = error_class;
|
||||
}
|
||||
}
|
||||
|
||||
/// Render the summary as audit tags, or nothing when the request performed
|
||||
/// no KMS work at all (SSE-C and unencrypted objects never reach KMS).
|
||||
pub fn audit_tags(&self) -> Vec<(&'static str, Value)> {
|
||||
let Ok(state) = self.0.lock() else {
|
||||
return Vec::new();
|
||||
};
|
||||
if !state.recorded {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
let mut tags = Vec::with_capacity(5);
|
||||
tags.push((KMS_AUDIT_TAG_SSE_TYPE, join_audit_values(state.sse_types.iter().copied())));
|
||||
if !state.key_ids.is_empty() {
|
||||
tags.push((KMS_AUDIT_TAG_KEY_ID, join_audit_values(state.key_ids.iter().map(String::as_str))));
|
||||
}
|
||||
if !state.key_versions.is_empty() {
|
||||
tags.push((
|
||||
KMS_AUDIT_TAG_KEY_VERSION,
|
||||
join_audit_values(state.key_versions.iter().map(u32::to_string)),
|
||||
));
|
||||
}
|
||||
let outcome = if state.error_class.is_some() { "failure" } else { "success" };
|
||||
tags.push((KMS_AUDIT_TAG_OUTCOME, Value::String(outcome.to_string())));
|
||||
if let Some(error_class) = state.error_class {
|
||||
tags.push((KMS_AUDIT_TAG_ERROR_CLASS, Value::String(error_class.to_string())));
|
||||
}
|
||||
|
||||
tags
|
||||
}
|
||||
}
|
||||
|
||||
/// Render a set of values as one tag value.
|
||||
///
|
||||
/// Always a string, including for the single-value case: a consumer that would
|
||||
/// have to branch on the JSON type of a tag is a consumer that will get it
|
||||
/// wrong for the request that happens to touch two keys.
|
||||
fn join_audit_values(values: impl Iterator<Item = impl AsRef<str>>) -> Value {
|
||||
let joined = values.map(|value| value.as_ref().to_string()).collect::<Vec<_>>();
|
||||
Value::String(joined.join(KMS_AUDIT_VALUE_SEPARATOR))
|
||||
}
|
||||
|
||||
/// Slots of the requests currently being audited, keyed by canonical request ID.
|
||||
///
|
||||
/// Only a weak reference is held: the owning [`KmsRequestAuditScope`] decides the
|
||||
/// lifetime, so a scope that is somehow not dropped cannot keep a slot alive.
|
||||
static KMS_REQUEST_AUDITS: LazyLock<Mutex<HashMap<String, Weak<KmsRequestAudit>>>> = LazyLock::new(|| Mutex::new(HashMap::new()));
|
||||
|
||||
/// Registration of one request's KMS audit slot, dropped with the request's
|
||||
/// audit entry.
|
||||
pub struct KmsRequestAuditScope {
|
||||
request_id: String,
|
||||
audit: Arc<KmsRequestAudit>,
|
||||
}
|
||||
|
||||
impl KmsRequestAuditScope {
|
||||
/// Open a slot for `request_id`, so managed-SSE operations served under it can
|
||||
/// be summarised onto its audit entry.
|
||||
pub fn register(request_id: &str) -> Self {
|
||||
let audit = Arc::new(KmsRequestAudit::default());
|
||||
if let Ok(mut registry) = KMS_REQUEST_AUDITS.lock() {
|
||||
registry.insert(request_id.to_string(), Arc::downgrade(&audit));
|
||||
}
|
||||
|
||||
Self {
|
||||
request_id: request_id.to_string(),
|
||||
audit,
|
||||
}
|
||||
}
|
||||
|
||||
/// Tags describing the KMS work recorded under this request.
|
||||
pub fn audit_tags(&self) -> Vec<(&'static str, Value)> {
|
||||
self.audit.audit_tags()
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for KmsRequestAuditScope {
|
||||
fn drop(&mut self) {
|
||||
if let Ok(mut registry) = KMS_REQUEST_AUDITS.lock() {
|
||||
registry.remove(&self.request_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolve the audit slot opened for `request_id`, if the request is being audited.
|
||||
pub(crate) fn kms_request_audit(request_id: &str) -> Option<Arc<KmsRequestAudit>> {
|
||||
KMS_REQUEST_AUDITS.lock().ok()?.get(request_id)?.upgrade()
|
||||
}
|
||||
|
||||
/// Failure classes for data-path failures that never reached a KMS backend.
|
||||
///
|
||||
/// Backend failures carry the classification defined by the KMS audit contract;
|
||||
/// everything else is classified at the S3 boundary it surfaced through.
|
||||
pub(crate) fn kms_data_plane_error_class(error: &SseError) -> &'static str {
|
||||
if let Some(failure) = error
|
||||
.source
|
||||
.as_ref()
|
||||
.and_then(|source| source.downcast_ref::<KmsDataPlaneFailure>())
|
||||
{
|
||||
return failure.class;
|
||||
}
|
||||
|
||||
match error.code {
|
||||
S3ErrorCode::AccessDenied => "access_denied",
|
||||
S3ErrorCode::InvalidArgument | S3ErrorCode::InvalidRequest => "invalid_argument",
|
||||
_ => "sse_internal",
|
||||
}
|
||||
}
|
||||
|
||||
/// Carries a KMS failure class across the conversion to an S3-facing error.
|
||||
///
|
||||
/// The class is decided while the `KmsError` is still typed; re-deriving it from
|
||||
/// a rendered message downstream would be guesswork.
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct KmsDataPlaneFailure {
|
||||
pub(crate) class: &'static str,
|
||||
pub(crate) source: Option<Box<dyn std::error::Error + Send + Sync>>,
|
||||
}
|
||||
|
||||
impl std::fmt::Display for KmsDataPlaneFailure {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match &self.source {
|
||||
Some(source) => source.fmt(formatter),
|
||||
None => formatter.write_str(self.class),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for KmsDataPlaneFailure {
|
||||
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
|
||||
self.source
|
||||
.as_ref()
|
||||
.map(|source| source.as_ref() as &(dyn std::error::Error + 'static))
|
||||
}
|
||||
}
|
||||
|
||||
/// Record the outcome of one managed-SSE operation on the request's audit entry.
|
||||
///
|
||||
/// A `None` principal marks an internal caller — replication, lifecycle, heal —
|
||||
/// which has no S3 audit entry to attach to. `key_version` is a closure for
|
||||
/// exactly that caller: extracting the version means base64-decoding and
|
||||
/// parsing the stored envelope, work that must not run on the internal hot
|
||||
/// paths that discard it.
|
||||
pub(crate) fn record_managed_kms_outcome(
|
||||
principal: Option<&SseKmsPrincipal>,
|
||||
sse_type: SSEType,
|
||||
key_id: Option<&str>,
|
||||
key_version: impl FnOnce() -> Option<u32>,
|
||||
result: Result<(), &SseError>,
|
||||
) {
|
||||
let Some(audit) = principal.and_then(|principal| principal.request_audit.as_ref()) else {
|
||||
return;
|
||||
};
|
||||
|
||||
audit.record(sse_type, key_id, key_version(), result.err().map(kms_data_plane_error_class));
|
||||
}
|
||||
|
||||
/// Master-key version recorded in a managed-SSE data-key envelope, if the
|
||||
/// wrapping backend recorded one.
|
||||
///
|
||||
/// `None` is the honest answer for every other shape: Transit and AWS keep
|
||||
/// the version inside their own opaque ciphertext, Local records no version,
|
||||
/// and pre-versioning envelopes never carried the field. The single field is
|
||||
/// read through `serde_json::Value` rather than a full `DataKeyEnvelope`
|
||||
/// parse, so the audit path cannot double-count the envelope's unknown-field
|
||||
/// observability and touches nothing else in the envelope.
|
||||
pub(crate) fn envelope_master_key_version(envelope_bytes: &[u8]) -> Option<u32> {
|
||||
if !is_data_key_envelope(envelope_bytes) {
|
||||
return None;
|
||||
}
|
||||
u32::try_from(
|
||||
serde_json::from_slice::<Value>(envelope_bytes)
|
||||
.ok()?
|
||||
.get("master_key_version")?
|
||||
.as_u64()?,
|
||||
)
|
||||
.ok()
|
||||
}
|
||||
|
||||
/// Master-key version of the envelope stored on an object, for the audit
|
||||
/// summary of a read against that object.
|
||||
pub(crate) fn stored_envelope_master_key_version(metadata: &HashMap<String, String>) -> Option<u32> {
|
||||
// No context recoder: the recode only ever inserts the context key, which
|
||||
// this lookup never reads, so the normalized result is identical without it.
|
||||
let encoded = normalize_managed_metadata(metadata, None);
|
||||
let encoded = encoded.get(INTERNAL_ENCRYPTION_KEY_HEADER)?;
|
||||
let envelope = BASE64_STANDARD.decode_to_vec(encoded).ok()?;
|
||||
envelope_master_key_version(&envelope)
|
||||
}
|
||||
@@ -1,67 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! The encryption context a data key is bound to.
|
||||
|
||||
use crate::error::SseError;
|
||||
use base64_simd::STANDARD as BASE64_STANDARD;
|
||||
use rustfs_kms::types::ObjectEncryptionContext;
|
||||
use rustfs_utils::http::object_encryption_keys::MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER;
|
||||
use rustfs_utils::path::path_join_buf;
|
||||
use std::collections::HashMap;
|
||||
|
||||
pub(crate) fn canonical_kms_bucket_path(bucket: &str, key: &str) -> String {
|
||||
path_join_buf(&[bucket, key])
|
||||
}
|
||||
|
||||
pub(crate) fn build_kms_request_context(
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
provided_context: Option<&HashMap<String, String>>,
|
||||
) -> HashMap<String, String> {
|
||||
let mut context = provided_context.cloned().unwrap_or_default();
|
||||
context
|
||||
.entry(bucket.to_string())
|
||||
.or_insert_with(|| canonical_kms_bucket_path(bucket, key));
|
||||
context
|
||||
}
|
||||
|
||||
pub(crate) fn build_object_encryption_context(
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
provided_context: Option<&HashMap<String, String>>,
|
||||
) -> ObjectEncryptionContext {
|
||||
let mut context = ObjectEncryptionContext::new(bucket.to_string(), key.to_string());
|
||||
for (ctx_key, ctx_value) in build_kms_request_context(bucket, key, provided_context) {
|
||||
context = context.with_encryption_context(ctx_key, ctx_value);
|
||||
}
|
||||
context
|
||||
}
|
||||
|
||||
pub(crate) fn encode_minio_kms_context(context: &HashMap<String, String>) -> Result<String, SseError> {
|
||||
let encoded = serde_json::to_vec(context).map_err(|e| SseError::internal(format!("Failed to serialize KMS context: {e}")))?;
|
||||
Ok(BASE64_STANDARD.encode_to_string(encoded))
|
||||
}
|
||||
|
||||
pub(crate) fn decode_minio_kms_context(metadata: &HashMap<String, String>) -> Result<Option<HashMap<String, String>>, SseError> {
|
||||
let Some(encoded) = metadata.get(MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER) else {
|
||||
return Ok(None);
|
||||
};
|
||||
let decoded = BASE64_STANDARD
|
||||
.decode_to_vec(encoded)
|
||||
.map_err(|e| SseError::internal(format!("Failed to decode MinIO KMS context: {e}")))?;
|
||||
serde_json::from_slice(&decoded)
|
||||
.map(Some)
|
||||
.map_err(|e| SseError::internal(format!("Failed to parse MinIO KMS context: {e}")))
|
||||
}
|
||||
@@ -1,212 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Direct keying: the data key encrypts the object itself, under a stored base nonce.
|
||||
|
||||
#[cfg(not(feature = "rio-v2"))]
|
||||
use crate::core::keying::NewObjectKey;
|
||||
#[cfg(not(feature = "rio-v2"))]
|
||||
use crate::core::keying::material::EncryptionKeyKind;
|
||||
use crate::core::keying::{StoredDataKey, StoredKeyMode};
|
||||
use crate::error::SseError;
|
||||
use crate::host::SseHost;
|
||||
use base64_simd::STANDARD as BASE64_STANDARD;
|
||||
use md5::{Digest as Md5Digest, Md5};
|
||||
#[cfg(not(feature = "rio-v2"))]
|
||||
use rand::Rng;
|
||||
#[cfg(not(feature = "rio-v2"))]
|
||||
use rustfs_kms::DataKey;
|
||||
#[cfg(not(feature = "rio-v2"))]
|
||||
use rustfs_utils::http::object_encryption_keys::SSEType;
|
||||
use rustfs_utils::http::object_encryption_keys::{
|
||||
INTERNAL_ENCRYPTION_ALGORITHM_HEADER, INTERNAL_ENCRYPTION_IV_HEADER, INTERNAL_ENCRYPTION_KEY_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_IV_HEADER,
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
|
||||
/// SSE-C: record the base nonce the object is encrypted under.
|
||||
///
|
||||
/// The nonce is random, so decrypt reads it back instead of recomputing a
|
||||
/// deterministic value. It is written under both the RustFS and the MinIO key per
|
||||
/// repo convention. This covers a single PUT and a multipart session alike:
|
||||
/// CreateMultipartUpload builds the session metadata through the same call.
|
||||
pub(crate) fn record_customer_key_nonce(metadata: &mut HashMap<String, String>, base_nonce: [u8; 12]) {
|
||||
metadata.insert(INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode_to_string(base_nonce));
|
||||
metadata.insert(
|
||||
MINIO_INTERNAL_ENCRYPTION_IV_HEADER.to_string(),
|
||||
BASE64_STANDARD.encode_to_string(base_nonce),
|
||||
);
|
||||
}
|
||||
|
||||
/// Managed SSE: record the wrapped data key, the nonce issued with it, and the cipher.
|
||||
pub(crate) fn record_wrapped_data_key(
|
||||
metadata: &mut HashMap<String, String>,
|
||||
wrapped_data_key: &[u8],
|
||||
base_nonce: [u8; 12],
|
||||
algorithm: &str,
|
||||
) {
|
||||
metadata.insert(
|
||||
INTERNAL_ENCRYPTION_KEY_HEADER.to_string(),
|
||||
BASE64_STANDARD.encode_to_string(wrapped_data_key),
|
||||
);
|
||||
metadata.insert(INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode_to_string(base_nonce));
|
||||
metadata.insert(INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(), algorithm.to_string());
|
||||
}
|
||||
|
||||
/// Managed SSE: read back what [`record_wrapped_data_key`] stored.
|
||||
///
|
||||
/// `normalized_metadata` is the object's metadata with the managed keys normalized.
|
||||
pub(crate) async fn read_wrapped_data_key(
|
||||
host: &dyn SseHost,
|
||||
normalized_metadata: &HashMap<String, String>,
|
||||
) -> Result<StoredDataKey, SseError> {
|
||||
let (wrapped, iv, algorithm) = if let Some(service) = host.encryption_service().await {
|
||||
// Production mode: use service for metadata parsing
|
||||
let parsed = service
|
||||
.headers_to_metadata(normalized_metadata)
|
||||
.map_err(|e| SseError::internal(format!("Failed to parse encryption metadata: {e}")))?;
|
||||
|
||||
(parsed.encrypted_data_key, parsed.iv, parsed.algorithm)
|
||||
} else {
|
||||
// Test mode: parse metadata manually
|
||||
let encrypted_key_b64 = normalized_metadata
|
||||
.get(INTERNAL_ENCRYPTION_KEY_HEADER)
|
||||
.ok_or_else(|| SseError::internal("Missing encrypted key in metadata"))?;
|
||||
let encrypted_data_key = BASE64_STANDARD
|
||||
.decode_to_vec(encrypted_key_b64)
|
||||
.map_err(|e| SseError::internal(format!("Failed to decode encrypted key: {e}")))?;
|
||||
|
||||
let iv_b64 = normalized_metadata
|
||||
.get(INTERNAL_ENCRYPTION_IV_HEADER)
|
||||
.ok_or_else(|| SseError::internal("Missing IV in metadata"))?;
|
||||
let iv = BASE64_STANDARD
|
||||
.decode_to_vec(iv_b64)
|
||||
.map_err(|e| SseError::internal(format!("Failed to decode IV: {e}")))?;
|
||||
|
||||
let algorithm = normalized_metadata
|
||||
.get(INTERNAL_ENCRYPTION_ALGORITHM_HEADER)
|
||||
.cloned()
|
||||
.unwrap_or_else(|| "AES256".to_string());
|
||||
|
||||
(encrypted_data_key, iv, algorithm)
|
||||
};
|
||||
|
||||
let base_nonce = <[u8; 12]>::try_from(iv.as_slice())
|
||||
.map_err(|_| SseError::internal("Invalid encryption nonce length; expected 12 bytes"))?;
|
||||
|
||||
Ok(StoredDataKey {
|
||||
wrapped,
|
||||
algorithm,
|
||||
mode: StoredKeyMode::Direct { base_nonce },
|
||||
})
|
||||
}
|
||||
|
||||
/// Resolve the SSE-C Direct base nonce for decryption.
|
||||
///
|
||||
/// New objects persist a random nonce at encryption time (see
|
||||
/// [`record_customer_key_nonce`]) under `INTERNAL_ENCRYPTION_IV_HEADER` (preferred) or,
|
||||
/// for MinIO interop, `MINIO_INTERNAL_ENCRYPTION_IV_HEADER`. Legacy objects written before
|
||||
/// random nonces were persisted carry no stored IV; for those we fall back to the
|
||||
/// deterministic `generate_ssec_nonce(bucket, key)` value they were originally encrypted
|
||||
/// with, so previously stored objects still decrypt correctly.
|
||||
pub(crate) fn read_stored_ssec_nonce(metadata: &HashMap<String, String>, bucket: &str, key: &str) -> [u8; 12] {
|
||||
metadata
|
||||
.get(INTERNAL_ENCRYPTION_IV_HEADER)
|
||||
.or_else(|| metadata.get(MINIO_INTERNAL_ENCRYPTION_IV_HEADER))
|
||||
.and_then(|encoded| BASE64_STANDARD.decode_to_vec(encoded).ok())
|
||||
.and_then(|bytes| <[u8; 12]>::try_from(bytes.as_slice()).ok())
|
||||
.unwrap_or_else(|| generate_ssec_nonce(bucket, key))
|
||||
}
|
||||
|
||||
/// Generate deterministic nonce for SSE-C encryption
|
||||
///
|
||||
/// The nonce is derived from the bucket and key to ensure:
|
||||
/// 1. Same object always gets the same nonce (required for SSE-C)
|
||||
/// 2. Different objects get different nonces
|
||||
pub(crate) fn generate_ssec_nonce(bucket: &str, key: &str) -> [u8; 12] {
|
||||
let nonce_source = format!("{bucket}-{key}");
|
||||
let nonce_hash: [u8; 16] = Md5::digest(nonce_source.as_bytes()).into();
|
||||
let mut nonce = [0u8; 12];
|
||||
nonce.copy_from_slice(&nonce_hash[..12]);
|
||||
nonce
|
||||
}
|
||||
|
||||
/// Whether a multipart session needs the customer key when it is created.
|
||||
///
|
||||
/// Direct mode records only a nonce for the session; the customer key is checked
|
||||
/// per part, against what the session recorded.
|
||||
#[cfg(not(feature = "rio-v2"))]
|
||||
pub(crate) const SESSION_KEY_NEEDS_CUSTOMER_KEY: bool = false;
|
||||
|
||||
#[cfg(not(feature = "rio-v2"))]
|
||||
fn random_base_nonce() -> [u8; 12] {
|
||||
let mut base_nonce = [0u8; 12];
|
||||
rand::rng().fill_bytes(&mut base_nonce);
|
||||
base_nonce
|
||||
}
|
||||
|
||||
/// SSE-C object: the customer key encrypts the object itself.
|
||||
///
|
||||
/// Uses a fresh random nonce per encryption. A deterministic nonce derived from
|
||||
/// (bucket, key) would repeat whenever the same object is overwritten under the same
|
||||
/// SSE-C key, reusing an identical (key, nonce) pair and catastrophically breaking
|
||||
/// AES-256-GCM. The nonce is persisted (see [`record_customer_key_nonce`]) and read
|
||||
/// back on decrypt (see [`read_stored_ssec_nonce`]).
|
||||
#[cfg(not(feature = "rio-v2"))]
|
||||
pub(crate) fn new_key_under_customer_key(customer_key: [u8; 32], _bucket: &str, _key: &str) -> Result<NewObjectKey, SseError> {
|
||||
Ok(NewObjectKey {
|
||||
key_bytes: customer_key,
|
||||
base_nonce: random_base_nonce(),
|
||||
key_kind: EncryptionKeyKind::Direct,
|
||||
sealed_key: None,
|
||||
})
|
||||
}
|
||||
|
||||
/// SSE-C multipart session: a nonce every part of the upload shares.
|
||||
///
|
||||
/// Generates a real random nonce for the session and persists it (see
|
||||
/// [`record_customer_key_nonce`]). Every part of this upload reads the same
|
||||
/// persisted nonce back, so all parts share one nonce without reintroducing the
|
||||
/// deterministic (bucket, key) reuse hazard. `key_bytes` stays a placeholder: it is
|
||||
/// neither used for encryption nor persisted; the real customer key is validated per
|
||||
/// part on upload.
|
||||
#[cfg(not(feature = "rio-v2"))]
|
||||
pub(crate) fn new_session_key_under_customer_key(
|
||||
_customer_key: Option<[u8; 32]>,
|
||||
_bucket: &str,
|
||||
_key: &str,
|
||||
) -> Result<NewObjectKey, SseError> {
|
||||
Ok(NewObjectKey {
|
||||
key_bytes: [0; 32],
|
||||
base_nonce: random_base_nonce(),
|
||||
key_kind: EncryptionKeyKind::Direct,
|
||||
sealed_key: None,
|
||||
})
|
||||
}
|
||||
|
||||
/// Managed SSE: the data key encrypts the object itself, under the nonce issued with it.
|
||||
#[cfg(not(feature = "rio-v2"))]
|
||||
pub(crate) fn new_key_under_data_key(
|
||||
data_key: &DataKey,
|
||||
_sse_type: SSEType,
|
||||
_bucket: &str,
|
||||
_key: &str,
|
||||
) -> Result<NewObjectKey, SseError> {
|
||||
Ok(NewObjectKey {
|
||||
key_bytes: data_key.plaintext_key,
|
||||
base_nonce: data_key.nonce,
|
||||
key_kind: EncryptionKeyKind::Direct,
|
||||
sealed_key: None,
|
||||
})
|
||||
}
|
||||
@@ -1,115 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! The key material an encrypted write or read runs on.
|
||||
|
||||
#[cfg(feature = "rio-v2")]
|
||||
use crate::core::keying::object::{SEALED_KEY_IV_SIZE, SEALED_KEY_SIZE};
|
||||
use rustfs_utils::http::object_encryption_keys::SSEType;
|
||||
use std::collections::HashMap;
|
||||
|
||||
/// Encryption material returned by [`sse_encryption`] / [`sse_prepare_encryption`].
|
||||
pub struct EncryptionMaterial {
|
||||
#[allow(unused)]
|
||||
pub sse_type: SSEType,
|
||||
pub server_side_encryption: String,
|
||||
pub kms_key_id: Option<String>,
|
||||
|
||||
#[allow(unused)]
|
||||
pub algorithm: String,
|
||||
|
||||
/// Encryption key bytes
|
||||
pub key_bytes: [u8; 32],
|
||||
/// Base nonce/IV used by rio to derive block/part nonces.
|
||||
pub base_nonce: [u8; 12],
|
||||
/// Encrypted DEK for managed SSE. Absent for SSE-C.
|
||||
pub encrypted_data_key: Option<Vec<u8>>,
|
||||
/// SSE-C key MD5 if customer-managed encryption is in use.
|
||||
pub customer_key_md5: Option<String>,
|
||||
/// Original plaintext size when it should be persisted alongside metadata.
|
||||
pub original_size: Option<i64>,
|
||||
/// Indicates whether `key_bytes` is a direct stream key or a MinIO-style object key.
|
||||
pub key_kind: EncryptionKeyKind,
|
||||
/// Original client-provided SSE-KMS context, stored in MinIO-compatible metadata.
|
||||
pub managed_kms_context: Option<HashMap<String, String>>,
|
||||
/// MinIO-compatible sealed object-key metadata for managed SSE object-key mode.
|
||||
#[cfg_attr(not(feature = "rio-v2"), allow(dead_code))]
|
||||
pub managed_sealed_key: Option<ManagedSealedKey>,
|
||||
}
|
||||
|
||||
/// Decryption material returned by [`sse_decryption`].
|
||||
pub struct DecryptionMaterial {
|
||||
#[allow(unused)]
|
||||
pub sse_type: SSEType,
|
||||
pub server_side_encryption: String,
|
||||
pub kms_key_id: Option<String>,
|
||||
#[allow(unused)]
|
||||
pub algorithm: String,
|
||||
pub customer_key_md5: Option<String>, // if use SSE-C, check key md5
|
||||
|
||||
/// Decryption key bytes
|
||||
pub key_bytes: [u8; 32],
|
||||
/// Base nonce/IV used by rio to derive block/part nonces.
|
||||
pub base_nonce: [u8; 12],
|
||||
/// Indicates whether `key_bytes` is a direct stream key or a MinIO-style object key.
|
||||
pub key_kind: EncryptionKeyKind,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for EncryptionMaterial {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("EncryptionMaterial")
|
||||
.field("sse_type", &self.sse_type)
|
||||
.field("server_side_encryption", &self.server_side_encryption)
|
||||
.field("original_size", &self.original_size)
|
||||
.finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for DecryptionMaterial {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("DecryptionMaterial")
|
||||
.field("sse_type", &self.sse_type)
|
||||
.field("server_side_encryption", &self.server_side_encryption)
|
||||
.finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum EncryptionKeyKind {
|
||||
Direct,
|
||||
Object,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ManagedSealedKey {
|
||||
#[cfg(feature = "rio-v2")]
|
||||
pub iv: [u8; SEALED_KEY_IV_SIZE],
|
||||
#[cfg(feature = "rio-v2")]
|
||||
pub sealed_key: [u8; SEALED_KEY_SIZE],
|
||||
}
|
||||
|
||||
impl EncryptionMaterial {
|
||||
/// The KMS key id a write response may advertise.
|
||||
///
|
||||
/// `kms_key_id` is always set for managed SSE because SSE-S3 also wraps
|
||||
/// its data key under the service default key, but that key is an
|
||||
/// internal detail of an `AES256` object: only an `aws:kms` object names
|
||||
/// a key the caller can act on, and `x-amz-server-side-encryption-aws-kms-key-id`
|
||||
/// is defined only for that scheme.
|
||||
pub fn response_kms_key_id(&self) -> Option<String> {
|
||||
matches!(self.sse_type, SSEType::SseKms)
|
||||
.then(|| self.kms_key_id.clone())
|
||||
.flatten()
|
||||
}
|
||||
}
|
||||
@@ -1,270 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Encryption metadata as it is stored with the object.
|
||||
|
||||
use crate::core::keying::context::encode_minio_kms_context;
|
||||
use crate::core::keying::direct::{record_customer_key_nonce, record_wrapped_data_key};
|
||||
use crate::core::keying::material::{EncryptionKeyKind, EncryptionMaterial};
|
||||
#[cfg(feature = "rio-v2")]
|
||||
use crate::core::keying::object::{record_data_key_in_minio_slots, record_sealed_customer_key};
|
||||
use crate::error::SseError;
|
||||
use base64_simd::STANDARD as BASE64_STANDARD;
|
||||
use rustfs_utils::http::object_encryption_keys::{
|
||||
INTERNAL_ENCRYPTION_ALGORITHM_HEADER, INTERNAL_ENCRYPTION_CONTEXT_HEADER, INTERNAL_ENCRYPTION_IV_HEADER,
|
||||
INTERNAL_ENCRYPTION_KEY_HEADER, INTERNAL_ENCRYPTION_KEY_ID_HEADER, INTERNAL_ENCRYPTION_ORIGINAL_SIZE_HEADER,
|
||||
INTERNAL_ENCRYPTION_TAG_HEADER, MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER, MINIO_INTERNAL_ENCRYPTION_IV_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER, MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER, MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_MULTIPART_HEADER, MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER, SSEC_ORIGINAL_SIZE_HEADER, SSEType,
|
||||
};
|
||||
use std::borrow::Cow;
|
||||
use std::collections::HashMap;
|
||||
|
||||
/// Two spellings of one encryption metadata key carry different values.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct ConflictingEncryptionMetadata {
|
||||
/// The canonical key whose case-variants disagree.
|
||||
pub key: &'static str,
|
||||
}
|
||||
|
||||
impl std::fmt::Display for ConflictingEncryptionMetadata {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(formatter, "conflicting object encryption metadata for {}", self.key)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for ConflictingEncryptionMetadata {}
|
||||
|
||||
/// Fold case-variant spellings of the encryption metadata keys onto their canonical names.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// When two spellings of the same key hold different values: picking either
|
||||
/// would decrypt with material the object may not have been written under.
|
||||
pub fn normalize_encryption_metadata_case(
|
||||
metadata: &HashMap<String, String>,
|
||||
) -> Result<Cow<'_, HashMap<String, String>>, ConflictingEncryptionMetadata> {
|
||||
const CANONICAL_KEYS: &[&str] = &[
|
||||
"x-amz-server-side-encryption",
|
||||
"x-amz-server-side-encryption-aws-kms-key-id",
|
||||
"x-amz-server-side-encryption-customer-algorithm",
|
||||
"x-amz-server-side-encryption-customer-key-md5",
|
||||
SSEC_ORIGINAL_SIZE_HEADER,
|
||||
INTERNAL_ENCRYPTION_KEY_ID_HEADER,
|
||||
INTERNAL_ENCRYPTION_KEY_HEADER,
|
||||
INTERNAL_ENCRYPTION_ALGORITHM_HEADER,
|
||||
INTERNAL_ENCRYPTION_IV_HEADER,
|
||||
INTERNAL_ENCRYPTION_CONTEXT_HEADER,
|
||||
INTERNAL_ENCRYPTION_TAG_HEADER,
|
||||
INTERNAL_ENCRYPTION_ORIGINAL_SIZE_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_MULTIPART_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_IV_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER,
|
||||
];
|
||||
|
||||
let needs_normalization = metadata.keys().any(|key| {
|
||||
CANONICAL_KEYS
|
||||
.iter()
|
||||
.any(|canonical| key != canonical && key.eq_ignore_ascii_case(canonical))
|
||||
});
|
||||
if !needs_normalization {
|
||||
return Ok(Cow::Borrowed(metadata));
|
||||
}
|
||||
|
||||
let mut normalized = metadata.clone();
|
||||
for canonical in CANONICAL_KEYS {
|
||||
let mut matching_values = metadata
|
||||
.iter()
|
||||
.filter_map(|(key, value)| key.eq_ignore_ascii_case(canonical).then_some(value));
|
||||
let Some(value) = matching_values.next() else {
|
||||
continue;
|
||||
};
|
||||
if matching_values.any(|candidate| candidate != value) {
|
||||
return Err(ConflictingEncryptionMetadata { key: canonical });
|
||||
}
|
||||
if !normalized.contains_key(*canonical) {
|
||||
normalized.insert((*canonical).to_string(), value.clone());
|
||||
}
|
||||
}
|
||||
Ok(Cow::Owned(normalized))
|
||||
}
|
||||
|
||||
/// Turn the material of an encrypted write into the metadata stored with the object.
|
||||
///
|
||||
/// The scheme is recorded here. How the key itself is recorded belongs to the mode
|
||||
/// that formed it, in `direct.rs` and `object.rs`, next to the code that reads it back.
|
||||
pub fn encryption_material_to_metadata(material: &EncryptionMaterial) -> Result<HashMap<String, String>, SseError> {
|
||||
let mut metadata = HashMap::new();
|
||||
|
||||
match material.sse_type {
|
||||
SSEType::SseC => {
|
||||
record_ssec_scheme(&mut metadata, material);
|
||||
|
||||
// A `rio-v2` build seals an object key instead, so a Direct key there is a
|
||||
// multipart session created without its customer key.
|
||||
if material.key_kind == EncryptionKeyKind::Direct {
|
||||
record_customer_key_nonce(&mut metadata, material.base_nonce);
|
||||
}
|
||||
#[cfg(feature = "rio-v2")]
|
||||
if let Some(sealed) = &material.managed_sealed_key {
|
||||
record_sealed_customer_key(&mut metadata, sealed);
|
||||
}
|
||||
}
|
||||
SSEType::SseS3 | SSEType::SseKms => {
|
||||
let wrapped_data_key = material
|
||||
.encrypted_data_key
|
||||
.as_deref()
|
||||
.ok_or_else(|| SseError::internal("managed SSE materials must carry an encrypted data key"))?;
|
||||
record_managed_scheme(&mut metadata, material);
|
||||
|
||||
if material.key_kind == EncryptionKeyKind::Direct {
|
||||
record_wrapped_data_key(&mut metadata, wrapped_data_key, material.base_nonce, material.algorithm.as_str());
|
||||
}
|
||||
#[cfg(feature = "rio-v2")]
|
||||
record_data_key_in_minio_slots(&mut metadata, material, wrapped_data_key);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(metadata)
|
||||
}
|
||||
|
||||
/// SSE-C: the scheme, the fingerprint of the customer key, and the plaintext size.
|
||||
fn record_ssec_scheme(metadata: &mut HashMap<String, String>, material: &EncryptionMaterial) {
|
||||
metadata.insert(
|
||||
"x-amz-server-side-encryption".to_string(),
|
||||
material.server_side_encryption.as_str().to_string(),
|
||||
);
|
||||
metadata.insert(
|
||||
"x-amz-server-side-encryption-customer-algorithm".to_string(),
|
||||
material.algorithm.as_str().to_string(),
|
||||
);
|
||||
if let Some(customer_key_md5) = &material.customer_key_md5 {
|
||||
metadata.insert("x-amz-server-side-encryption-customer-key-md5".to_string(), customer_key_md5.to_string());
|
||||
}
|
||||
if let Some(original_size) = material.original_size {
|
||||
metadata.insert(SSEC_ORIGINAL_SIZE_HEADER.to_string(), original_size.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
/// Managed SSE: the scheme, the key the data key is wrapped under, the plaintext
|
||||
/// size, and the encryption context of the request.
|
||||
fn record_managed_scheme(metadata: &mut HashMap<String, String>, material: &EncryptionMaterial) {
|
||||
metadata.insert(
|
||||
"x-amz-server-side-encryption".to_string(),
|
||||
material.server_side_encryption.as_str().to_string(),
|
||||
);
|
||||
|
||||
let internal_key_id = material.kms_key_id.clone().unwrap_or_else(|| "default".to_string());
|
||||
metadata.insert(INTERNAL_ENCRYPTION_KEY_ID_HEADER.to_string(), internal_key_id.clone());
|
||||
|
||||
if matches!(material.sse_type, SSEType::SseKms) {
|
||||
metadata.insert("x-amz-server-side-encryption-aws-kms-key-id".to_string(), internal_key_id);
|
||||
}
|
||||
|
||||
if let Some(original_size) = material.original_size {
|
||||
metadata.insert(INTERNAL_ENCRYPTION_ORIGINAL_SIZE_HEADER.to_string(), original_size.to_string());
|
||||
}
|
||||
|
||||
if let Some(kms_context) = &material.managed_kms_context
|
||||
&& !kms_context.is_empty()
|
||||
{
|
||||
if let Ok(serialized) = serde_json::to_string(kms_context) {
|
||||
metadata.insert(INTERNAL_ENCRYPTION_CONTEXT_HEADER.to_string(), serialized);
|
||||
}
|
||||
if matches!(material.sse_type, SSEType::SseKms)
|
||||
&& let Ok(encoded) = encode_minio_kms_context(kms_context)
|
||||
{
|
||||
metadata.insert(MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER.to_string(), encoded);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Strip source encryption metadata before constructing metadata for a copy destination.
|
||||
///
|
||||
/// Encryption metadata describes the physical source representation and must never be
|
||||
/// inherited by a plaintext destination or by a destination using a different key.
|
||||
pub fn strip_managed_encryption_metadata(metadata: &mut HashMap<String, String>) {
|
||||
const KEYS: [&str; 21] = [
|
||||
"x-amz-server-side-encryption",
|
||||
"x-amz-server-side-encryption-aws-kms-key-id",
|
||||
"x-amz-server-side-encryption-customer-algorithm",
|
||||
"x-amz-server-side-encryption-customer-key-md5",
|
||||
SSEC_ORIGINAL_SIZE_HEADER,
|
||||
INTERNAL_ENCRYPTION_KEY_ID_HEADER,
|
||||
INTERNAL_ENCRYPTION_ALGORITHM_HEADER,
|
||||
INTERNAL_ENCRYPTION_IV_HEADER,
|
||||
INTERNAL_ENCRYPTION_TAG_HEADER,
|
||||
INTERNAL_ENCRYPTION_KEY_HEADER,
|
||||
INTERNAL_ENCRYPTION_CONTEXT_HEADER,
|
||||
INTERNAL_ENCRYPTION_ORIGINAL_SIZE_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_MULTIPART_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_IV_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER,
|
||||
];
|
||||
|
||||
for key in KEYS.iter() {
|
||||
metadata.remove(*key);
|
||||
}
|
||||
rustfs_utils::http::metadata_compat::remove_str(metadata, rustfs_utils::http::SUFFIX_OPAQUE_ENCRYPTED_ETAG);
|
||||
}
|
||||
|
||||
pub fn mark_encrypted_multipart_metadata(metadata: &mut HashMap<String, String>) {
|
||||
metadata.insert(MINIO_INTERNAL_ENCRYPTION_MULTIPART_HEADER.to_string(), String::new());
|
||||
}
|
||||
|
||||
/// True when the stored metadata marks the object as SSE-C protected.
|
||||
///
|
||||
/// RustFS persists the public `x-amz-server-side-encryption-customer-algorithm`
|
||||
/// key on SSE-C writes. MinIO persists only its internal sealed-key slot and
|
||||
/// synthesizes the public header onto responses (backlog#1638), so the slot is
|
||||
/// the only durable evidence on a MinIO-written object. That shape is readable
|
||||
/// solely through the rio-v2 sealed-object-key path, so it participates in
|
||||
/// detection only in builds that can actually read it — default builds keep
|
||||
/// the resolver's fail-closed diagnosis instead of demanding customer keys
|
||||
/// they could not use.
|
||||
pub fn stored_ssec_metadata(metadata: &HashMap<String, String>) -> bool {
|
||||
metadata.contains_key("x-amz-server-side-encryption-customer-algorithm") || minio_sealed_ssec_metadata(metadata)
|
||||
}
|
||||
|
||||
/// True when the object carries MinIO's SSE-C sealed-key slot — the shape that
|
||||
/// stores no customer-key MD5 and proves the key through the AEAD unseal
|
||||
/// instead.
|
||||
pub(crate) fn minio_sealed_ssec_metadata(metadata: &HashMap<String, String>) -> bool {
|
||||
cfg!(feature = "rio-v2") && metadata.contains_key(MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER)
|
||||
}
|
||||
|
||||
/// Recodes a stored MinIO KMS context value (base64-wrapped JSON) into the
|
||||
/// plain-JSON form RustFS stores under [`INTERNAL_ENCRYPTION_CONTEXT_HEADER`].
|
||||
///
|
||||
/// Injected into the shared [`normalize_managed_metadata`] because the shared
|
||||
/// crate carries no JSON codec; any decode failure returns `None`, which skips
|
||||
/// the context mapping exactly like the historical inline `if let Ok` chain.
|
||||
pub(crate) fn recode_minio_kms_context(value: &str) -> Option<String> {
|
||||
let decoded = BASE64_STANDARD.decode_to_vec(value).ok()?;
|
||||
let context = serde_json::from_slice::<HashMap<String, String>>(&decoded).ok()?;
|
||||
serde_json::to_string(&context).ok()
|
||||
}
|
||||
@@ -1,67 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! The internal keying modes: how an object's key is formed and how it is stored.
|
||||
|
||||
pub(crate) mod context;
|
||||
pub(crate) mod direct;
|
||||
pub(crate) mod material;
|
||||
pub(crate) mod metadata;
|
||||
#[cfg(feature = "rio-v2")]
|
||||
pub(crate) mod object;
|
||||
pub(crate) mod stored;
|
||||
|
||||
use crate::core::keying::material::{EncryptionKeyKind, ManagedSealedKey};
|
||||
|
||||
/// The key a new object is encrypted with, in the mode this build writes.
|
||||
pub(crate) struct NewObjectKey {
|
||||
/// Key handed to the stream cipher.
|
||||
pub(crate) key_bytes: [u8; 32],
|
||||
/// Base nonce the stream cipher derives its block nonces from.
|
||||
pub(crate) base_nonce: [u8; 12],
|
||||
/// The mode `key_bytes` belongs to.
|
||||
pub(crate) key_kind: EncryptionKeyKind,
|
||||
/// The sealed form to store, in a mode that seals a separate object key.
|
||||
pub(crate) sealed_key: Option<ManagedSealedKey>,
|
||||
}
|
||||
|
||||
/// A managed-SSE data key as a stored object recorded it, still wrapped.
|
||||
pub(crate) struct StoredDataKey {
|
||||
/// The data key, wrapped by the provider that issued it.
|
||||
pub(crate) wrapped: Vec<u8>,
|
||||
/// The cipher the object recorded.
|
||||
pub(crate) algorithm: String,
|
||||
/// What the data key is to the object once it is unwrapped.
|
||||
pub(crate) mode: StoredKeyMode,
|
||||
}
|
||||
|
||||
/// The mode a stored key was recorded in.
|
||||
pub(crate) enum StoredKeyMode {
|
||||
/// The data key is the object's key, under the nonce stored beside it.
|
||||
Direct { base_nonce: [u8; 12] },
|
||||
/// The data key unseals the object's key.
|
||||
#[cfg(feature = "rio-v2")]
|
||||
Object { sealed_key: ManagedSealedKey },
|
||||
}
|
||||
|
||||
// The mode new objects are written in. Every build reads every mode it can
|
||||
// recognize; writing picks exactly one, and this is the only place that says which.
|
||||
#[cfg(not(feature = "rio-v2"))]
|
||||
pub(crate) use direct::{
|
||||
SESSION_KEY_NEEDS_CUSTOMER_KEY, new_key_under_customer_key, new_key_under_data_key, new_session_key_under_customer_key,
|
||||
};
|
||||
#[cfg(feature = "rio-v2")]
|
||||
pub(crate) use object::{
|
||||
SESSION_KEY_NEEDS_CUSTOMER_KEY, new_key_under_customer_key, new_key_under_data_key, new_session_key_under_customer_key,
|
||||
};
|
||||
@@ -1,436 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Object keying: a derived object key, sealed in the MinIO-compatible format.
|
||||
|
||||
use crate::core::keying::material::{EncryptionKeyKind, EncryptionMaterial, ManagedSealedKey};
|
||||
use crate::core::keying::{NewObjectKey, StoredDataKey, StoredKeyMode};
|
||||
use crate::error::SseError;
|
||||
use aes_gcm::aead::{Aead, KeyInit, Payload};
|
||||
use aes_gcm::{Aes256Gcm, Nonce};
|
||||
use base64_simd::STANDARD as BASE64_STANDARD;
|
||||
use chacha20poly1305::ChaCha20Poly1305;
|
||||
use hmac::{Hmac, Mac};
|
||||
use rand::RngExt;
|
||||
use rustfs_kms::DataKey;
|
||||
use rustfs_utils::http::object_encryption_keys::{
|
||||
INTERNAL_ENCRYPTION_ALGORITHM_HEADER, INTERNAL_ENCRYPTION_IV_HEADER, INTERNAL_ENCRYPTION_KEY_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER, MINIO_INTERNAL_ENCRYPTION_IV_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER, MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER, MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER, SSEType,
|
||||
};
|
||||
use sha2::Sha256;
|
||||
use std::collections::HashMap;
|
||||
|
||||
pub(crate) const MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM: &str = "DAREv2-HMAC-SHA256";
|
||||
|
||||
const DARE_VERSION_20: u8 = 0x20;
|
||||
|
||||
pub(crate) const DARE_CIPHER_AES_256_GCM: u8 = 0x00;
|
||||
|
||||
pub(crate) const DARE_CIPHER_CHACHA20_POLY1305: u8 = 0x01;
|
||||
|
||||
const DARE_HEADER_SIZE: usize = 16;
|
||||
|
||||
const DARE_TAG_SIZE: usize = 16;
|
||||
|
||||
pub(crate) const SEALED_KEY_IV_SIZE: usize = 32;
|
||||
|
||||
pub(crate) const SEALED_KEY_SIZE: usize = DARE_HEADER_SIZE + 32 + DARE_TAG_SIZE;
|
||||
|
||||
const OBJECT_KEY_DERIVATION_CONTEXT: &[u8] = b"object-encryption-key generation";
|
||||
|
||||
pub(crate) type HmacSha256 = Hmac<Sha256>;
|
||||
|
||||
fn managed_sse_domain(sse_type: SSEType) -> &'static str {
|
||||
match sse_type {
|
||||
SSEType::SseS3 => "SSE-S3",
|
||||
SSEType::SseKms => "SSE-KMS",
|
||||
SSEType::SseC => "SSE-C",
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn is_supported_sealed_object_key_cipher(cipher: u8) -> bool {
|
||||
matches!(cipher, DARE_CIPHER_AES_256_GCM | DARE_CIPHER_CHACHA20_POLY1305)
|
||||
}
|
||||
|
||||
fn decrypt_sealed_object_key_payload(sealing_key: [u8; 32], header: &[u8], sealed_key: &[u8]) -> Result<Vec<u8>, SseError> {
|
||||
let nonce = &header[4..16];
|
||||
let ciphertext = &sealed_key[DARE_HEADER_SIZE..];
|
||||
let aad = &header[..4];
|
||||
match header[1] {
|
||||
DARE_CIPHER_AES_256_GCM => {
|
||||
let cipher = Aes256Gcm::new_from_slice(&sealing_key)
|
||||
.map_err(|err| SseError::internal(format!("Invalid AES-GCM sealing key: {err}")))?;
|
||||
let nonce = Nonce::try_from(nonce).map_err(|_| SseError::internal("Invalid sealed object-key package nonce"))?;
|
||||
cipher.decrypt(&nonce, Payload { msg: ciphertext, aad })
|
||||
}
|
||||
DARE_CIPHER_CHACHA20_POLY1305 => {
|
||||
let cipher = ChaCha20Poly1305::new_from_slice(&sealing_key)
|
||||
.map_err(|err| SseError::internal(format!("Invalid ChaCha20-Poly1305 sealing key: {err}")))?;
|
||||
let nonce = chacha20poly1305::Nonce::try_from(nonce)
|
||||
.map_err(|_| SseError::internal("Invalid sealed object-key package nonce"))?;
|
||||
cipher.decrypt(&nonce, Payload { msg: ciphertext, aad })
|
||||
}
|
||||
_ => return Err(SseError::internal("Unsupported sealed object-key DARE header")),
|
||||
}
|
||||
.map_err(|err| SseError::internal(format!("Failed to unseal object key: {err}")))
|
||||
}
|
||||
|
||||
fn canonical_sse_path(bucket: &str, object: &str) -> String {
|
||||
let bucket = bucket.trim_matches('/');
|
||||
let object = object.trim_matches('/');
|
||||
if object.is_empty() {
|
||||
bucket.to_string()
|
||||
} else if bucket.is_empty() {
|
||||
object.to_string()
|
||||
} else {
|
||||
format!("{bucket}/{object}")
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn derive_object_key(external_key: [u8; 32]) -> Result<[u8; 32], SseError> {
|
||||
let mut random = [0u8; 32];
|
||||
rand::rng().fill(&mut random);
|
||||
|
||||
let mut mac = HmacSha256::new_from_slice(&external_key)
|
||||
.map_err(|err| SseError::internal(format!("Invalid HMAC key for object-key derivation: {err}")))?;
|
||||
mac.update(OBJECT_KEY_DERIVATION_CONTEXT);
|
||||
mac.update(&random);
|
||||
|
||||
let mut object_key = [0u8; 32];
|
||||
object_key.copy_from_slice(mac.finalize().into_bytes().as_slice());
|
||||
Ok(object_key)
|
||||
}
|
||||
|
||||
fn derive_sealing_key(
|
||||
external_key: [u8; 32],
|
||||
iv: [u8; SEALED_KEY_IV_SIZE],
|
||||
domain: &str,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
) -> Result<[u8; 32], SseError> {
|
||||
let mut mac = HmacSha256::new_from_slice(&external_key)
|
||||
.map_err(|err| SseError::internal(format!("Invalid HMAC key for sealing-key derivation: {err}")))?;
|
||||
mac.update(&iv);
|
||||
mac.update(domain.as_bytes());
|
||||
mac.update(MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM.as_bytes());
|
||||
mac.update(canonical_sse_path(bucket, object).as_bytes());
|
||||
|
||||
let mut sealing_key = [0u8; 32];
|
||||
sealing_key.copy_from_slice(mac.finalize().into_bytes().as_slice());
|
||||
Ok(sealing_key)
|
||||
}
|
||||
|
||||
pub(crate) fn seal_object_key(
|
||||
object_key: [u8; 32],
|
||||
external_key: [u8; 32],
|
||||
sse_type: SSEType,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
) -> Result<ManagedSealedKey, SseError> {
|
||||
let mut iv = [0u8; SEALED_KEY_IV_SIZE];
|
||||
rand::rng().fill(&mut iv);
|
||||
let sealing_key = derive_sealing_key(external_key, iv, managed_sse_domain(sse_type), bucket, object)?;
|
||||
|
||||
let mut header = [0u8; DARE_HEADER_SIZE];
|
||||
header[0] = DARE_VERSION_20;
|
||||
header[1] = DARE_CIPHER_AES_256_GCM;
|
||||
header[2..4].copy_from_slice(&(31u16).to_le_bytes());
|
||||
let mut stream_nonce = [0u8; 12];
|
||||
rand::rng().fill(&mut stream_nonce);
|
||||
stream_nonce[0] |= 0x80;
|
||||
header[4..16].copy_from_slice(&stream_nonce);
|
||||
|
||||
let cipher =
|
||||
Aes256Gcm::new_from_slice(&sealing_key).map_err(|err| SseError::internal(format!("Invalid sealing key: {err}")))?;
|
||||
let nonce =
|
||||
Nonce::try_from(stream_nonce.as_slice()).map_err(|_| SseError::internal("Invalid sealed object-key stream nonce"))?;
|
||||
let ciphertext = cipher
|
||||
.encrypt(
|
||||
&nonce,
|
||||
aes_gcm::aead::Payload {
|
||||
msg: &object_key,
|
||||
aad: &header[..4],
|
||||
},
|
||||
)
|
||||
.map_err(|err| SseError::internal(format!("Failed to seal object key: {err}")))?;
|
||||
|
||||
let mut sealed_key = [0u8; SEALED_KEY_SIZE];
|
||||
sealed_key[..DARE_HEADER_SIZE].copy_from_slice(&header);
|
||||
sealed_key[DARE_HEADER_SIZE..].copy_from_slice(&ciphertext);
|
||||
|
||||
Ok(ManagedSealedKey { iv, sealed_key })
|
||||
}
|
||||
|
||||
pub(crate) fn unseal_object_key(
|
||||
sealed: &ManagedSealedKey,
|
||||
external_key: [u8; 32],
|
||||
sse_type: SSEType,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
) -> Result<[u8; 32], SseError> {
|
||||
let header = &sealed.sealed_key[..DARE_HEADER_SIZE];
|
||||
if header[0] != DARE_VERSION_20 || !is_supported_sealed_object_key_cipher(header[1]) {
|
||||
return Err(SseError::internal("Unsupported sealed object-key DARE header"));
|
||||
}
|
||||
if u16::from_le_bytes([header[2], header[3]]) != 31 || header[4] & 0x80 == 0 {
|
||||
return Err(SseError::internal("Invalid sealed object-key payload header"));
|
||||
}
|
||||
|
||||
let sealing_key = derive_sealing_key(external_key, sealed.iv, managed_sse_domain(sse_type), bucket, object)?;
|
||||
let plaintext = decrypt_sealed_object_key_payload(sealing_key, header, &sealed.sealed_key)?;
|
||||
|
||||
let object_key: [u8; 32] = plaintext
|
||||
.as_slice()
|
||||
.try_into()
|
||||
.map_err(|_| SseError::internal("Sealed object key must decrypt to 32 bytes"))?;
|
||||
Ok(object_key)
|
||||
}
|
||||
|
||||
fn try_decode_minio_sealed_key(bytes: &str) -> Result<Option<[u8; SEALED_KEY_SIZE]>, SseError> {
|
||||
let decoded = BASE64_STANDARD
|
||||
.decode_to_vec(bytes)
|
||||
.map_err(|e| SseError::internal(format!("Failed to decode sealed object key: {e}")))?;
|
||||
match decoded.as_slice().try_into() {
|
||||
Ok(sealed_key) => Ok(Some(sealed_key)),
|
||||
Err(_) => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
fn try_decode_minio_sealing_iv(bytes: &str) -> Result<Option<[u8; SEALED_KEY_IV_SIZE]>, SseError> {
|
||||
let decoded = BASE64_STANDARD
|
||||
.decode_to_vec(bytes)
|
||||
.map_err(|e| SseError::internal(format!("Failed to decode sealing IV: {e}")))?;
|
||||
match decoded.as_slice().try_into() {
|
||||
Ok(iv) => Ok(Some(iv)),
|
||||
Err(_) => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn is_legacy_rustfs_managed_metadata(metadata: &HashMap<String, String>) -> bool {
|
||||
metadata.contains_key(INTERNAL_ENCRYPTION_KEY_HEADER)
|
||||
&& metadata.contains_key(INTERNAL_ENCRYPTION_IV_HEADER)
|
||||
&& !metadata.contains_key(MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER)
|
||||
&& !metadata.contains_key(MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER)
|
||||
}
|
||||
|
||||
/// Infer the managed SSE scheme from the MinIO sealed-key slot that is present.
|
||||
///
|
||||
/// Returns `None` when no managed MinIO slot is present, which keeps callers on
|
||||
/// their fail-closed path. SSE-C is not a managed scheme and is handled by the
|
||||
/// SSE-C read path, so its slot is not considered here.
|
||||
pub(crate) fn infer_minio_managed_sse_type(metadata: &HashMap<String, String>) -> Option<SSEType> {
|
||||
if metadata.contains_key(MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER) {
|
||||
Some(SSEType::SseS3)
|
||||
} else if metadata.contains_key(MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER) {
|
||||
Some(SSEType::SseKms)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn parse_minio_managed_sealed_key(
|
||||
metadata: &HashMap<String, String>,
|
||||
sse_type: SSEType,
|
||||
) -> Result<Option<ManagedSealedKey>, SseError> {
|
||||
let algorithm = match metadata.get(MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER) {
|
||||
Some(algorithm) => algorithm,
|
||||
None => return Ok(None),
|
||||
};
|
||||
if algorithm != MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let iv = match metadata.get(MINIO_INTERNAL_ENCRYPTION_IV_HEADER) {
|
||||
Some(iv) => match try_decode_minio_sealing_iv(iv)? {
|
||||
Some(iv) => iv,
|
||||
None => return Ok(None),
|
||||
},
|
||||
None => return Ok(None),
|
||||
};
|
||||
|
||||
let header_key = match sse_type {
|
||||
SSEType::SseS3 => MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER,
|
||||
SSEType::SseKms => MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER,
|
||||
SSEType::SseC => MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER,
|
||||
};
|
||||
let sealed_key = match metadata.get(header_key) {
|
||||
Some(value) => match try_decode_minio_sealed_key(value)? {
|
||||
Some(sealed_key) => sealed_key,
|
||||
None => return Ok(None),
|
||||
},
|
||||
None => return Ok(None),
|
||||
};
|
||||
|
||||
Ok(Some(ManagedSealedKey { iv, sealed_key }))
|
||||
}
|
||||
|
||||
/// SSE-C: record the object key sealed under the customer key.
|
||||
pub(crate) fn record_sealed_customer_key(metadata: &mut HashMap<String, String>, sealed: &ManagedSealedKey) {
|
||||
metadata.insert(
|
||||
MINIO_INTERNAL_ENCRYPTION_IV_HEADER.to_string(),
|
||||
BASE64_STANDARD.encode_to_string(sealed.iv),
|
||||
);
|
||||
metadata.insert(
|
||||
MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(),
|
||||
MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM.to_string(),
|
||||
);
|
||||
metadata.insert(
|
||||
MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER.to_string(),
|
||||
BASE64_STANDARD.encode_to_string(sealed.sealed_key),
|
||||
);
|
||||
}
|
||||
|
||||
/// Managed SSE: record the wrapped data key in MinIO's slots.
|
||||
///
|
||||
/// With a sealed object key the slots hold what MinIO itself writes. Without one
|
||||
/// the key was formed in Direct mode, and the slots mirror its wrapped data key and
|
||||
/// nonce: [`parse_minio_managed_sealed_key`] turns that shape down by its length, so
|
||||
/// the object is read back in Direct mode.
|
||||
pub(crate) fn record_data_key_in_minio_slots(
|
||||
metadata: &mut HashMap<String, String>,
|
||||
material: &EncryptionMaterial,
|
||||
wrapped_data_key: &[u8],
|
||||
) {
|
||||
metadata.insert(
|
||||
MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(),
|
||||
MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM.to_string(),
|
||||
);
|
||||
if let Some(kms_key_id) = &material.kms_key_id {
|
||||
metadata.insert(MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER.to_string(), kms_key_id.to_string());
|
||||
}
|
||||
|
||||
let encoded_data_key = BASE64_STANDARD.encode_to_string(wrapped_data_key);
|
||||
match &material.managed_sealed_key {
|
||||
Some(sealed) => {
|
||||
metadata.insert(
|
||||
MINIO_INTERNAL_ENCRYPTION_IV_HEADER.to_string(),
|
||||
BASE64_STANDARD.encode_to_string(sealed.iv),
|
||||
);
|
||||
match material.sse_type {
|
||||
SSEType::SseS3 => {
|
||||
metadata.insert(
|
||||
MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER.to_string(),
|
||||
BASE64_STANDARD.encode_to_string(sealed.sealed_key),
|
||||
);
|
||||
}
|
||||
SSEType::SseKms => {
|
||||
metadata.insert(
|
||||
MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER.to_string(),
|
||||
BASE64_STANDARD.encode_to_string(sealed.sealed_key),
|
||||
);
|
||||
}
|
||||
SSEType::SseC => {}
|
||||
}
|
||||
metadata.insert(MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER.to_string(), encoded_data_key);
|
||||
}
|
||||
None => {
|
||||
metadata.insert(
|
||||
MINIO_INTERNAL_ENCRYPTION_IV_HEADER.to_string(),
|
||||
BASE64_STANDARD.encode_to_string(material.base_nonce),
|
||||
);
|
||||
match material.sse_type {
|
||||
SSEType::SseS3 => {
|
||||
metadata.insert(MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER.to_string(), encoded_data_key);
|
||||
}
|
||||
SSEType::SseKms => {
|
||||
metadata.insert(MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER.to_string(), encoded_data_key.clone());
|
||||
metadata.insert(MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER.to_string(), encoded_data_key);
|
||||
}
|
||||
SSEType::SseC => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Managed SSE: read back the wrapped data key that seals `sealed_key`.
|
||||
///
|
||||
/// `normalized_metadata` is `metadata` with the managed keys normalized.
|
||||
pub(crate) fn read_sealed_data_key(
|
||||
metadata: &HashMap<String, String>,
|
||||
normalized_metadata: &HashMap<String, String>,
|
||||
sealed_key: ManagedSealedKey,
|
||||
) -> Result<StoredDataKey, SseError> {
|
||||
let encrypted_key_b64 = normalized_metadata
|
||||
.get(INTERNAL_ENCRYPTION_KEY_HEADER)
|
||||
.or_else(|| metadata.get(MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER))
|
||||
.ok_or_else(|| SseError::internal("Missing encrypted key in metadata"))?;
|
||||
let wrapped = BASE64_STANDARD
|
||||
.decode_to_vec(encrypted_key_b64)
|
||||
.map_err(|e| SseError::internal(format!("Failed to decode encrypted key: {e}")))?;
|
||||
|
||||
Ok(StoredDataKey {
|
||||
wrapped,
|
||||
algorithm: normalized_metadata
|
||||
.get(INTERNAL_ENCRYPTION_ALGORITHM_HEADER)
|
||||
.cloned()
|
||||
.unwrap_or_else(|| "AES256".to_string()),
|
||||
mode: StoredKeyMode::Object { sealed_key },
|
||||
})
|
||||
}
|
||||
|
||||
/// Whether a multipart session needs the customer key when it is created.
|
||||
///
|
||||
/// Object mode seals the session's object key under the customer key, so the key
|
||||
/// has to be present, and valid, at that point.
|
||||
pub(crate) const SESSION_KEY_NEEDS_CUSTOMER_KEY: bool = true;
|
||||
|
||||
/// Derive a fresh object key from `external_key` and seal it under the same key.
|
||||
fn new_sealed_object_key(external_key: [u8; 32], sse_type: SSEType, bucket: &str, key: &str) -> Result<NewObjectKey, SseError> {
|
||||
let object_key = derive_object_key(external_key)?;
|
||||
let sealed_key = seal_object_key(object_key, external_key, sse_type, bucket, key)?;
|
||||
Ok(NewObjectKey {
|
||||
key_bytes: object_key,
|
||||
base_nonce: [0; 12],
|
||||
key_kind: EncryptionKeyKind::Object,
|
||||
sealed_key: Some(sealed_key),
|
||||
})
|
||||
}
|
||||
|
||||
/// SSE-C object: an object key sealed under the customer key.
|
||||
pub(crate) fn new_key_under_customer_key(customer_key: [u8; 32], bucket: &str, key: &str) -> Result<NewObjectKey, SseError> {
|
||||
new_sealed_object_key(customer_key, SSEType::SseC, bucket, key)
|
||||
}
|
||||
|
||||
/// SSE-C multipart session: an object key sealed under the customer key.
|
||||
///
|
||||
/// A session created without its customer key has nothing to seal and records a
|
||||
/// placeholder instead.
|
||||
pub(crate) fn new_session_key_under_customer_key(
|
||||
customer_key: Option<[u8; 32]>,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
) -> Result<NewObjectKey, SseError> {
|
||||
match customer_key {
|
||||
Some(customer_key) => new_sealed_object_key(customer_key, SSEType::SseC, bucket, key),
|
||||
None => Ok(NewObjectKey {
|
||||
key_bytes: [0; 32],
|
||||
base_nonce: [0; 12],
|
||||
key_kind: EncryptionKeyKind::Direct,
|
||||
sealed_key: None,
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
/// Managed SSE: an object key sealed under the data key.
|
||||
pub(crate) fn new_key_under_data_key(
|
||||
data_key: &DataKey,
|
||||
sse_type: SSEType,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
) -> Result<NewObjectKey, SseError> {
|
||||
new_sealed_object_key(data_key.plaintext_key, sse_type, bucket, key)
|
||||
}
|
||||
@@ -1,253 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Reading: recognize how a stored object was keyed, and recover its key.
|
||||
//!
|
||||
//! What an object recorded is the only source of truth here. The scheme a request
|
||||
//! advertises and whether a KMS is running now are write policy and runtime state;
|
||||
//! neither says how an object written earlier was keyed.
|
||||
|
||||
use crate::core::keying::context::{build_object_encryption_context, decode_minio_kms_context};
|
||||
use crate::core::keying::direct::{read_stored_ssec_nonce, read_wrapped_data_key};
|
||||
use crate::core::keying::material::EncryptionKeyKind;
|
||||
use crate::core::keying::metadata::recode_minio_kms_context;
|
||||
#[cfg(feature = "rio-v2")]
|
||||
use crate::core::keying::object::{
|
||||
infer_minio_managed_sse_type, is_legacy_rustfs_managed_metadata, parse_minio_managed_sealed_key, read_sealed_data_key,
|
||||
unseal_object_key,
|
||||
};
|
||||
use crate::core::keying::{StoredDataKey, StoredKeyMode};
|
||||
use crate::error::SseError;
|
||||
use crate::host::SseHost;
|
||||
use crate::kms::local::is_local_sse_dek;
|
||||
#[cfg(feature = "rio-v2")]
|
||||
use crate::kms::local::is_rustfs_managed_data_key;
|
||||
#[cfg(feature = "rio-v2")]
|
||||
use crate::kms::provider::decrypt_unenveloped_sealed_dek;
|
||||
use crate::kms::provider::{SseDekProvider, bare_data_key_kms_provider, get_local_sse_dek_provider, kms_sse_dek_provider};
|
||||
use rustfs_kms::is_data_key_envelope;
|
||||
use rustfs_utils::http::headers::AMZ_ENCRYPTION_KMS;
|
||||
use rustfs_utils::http::object_encryption_keys::{
|
||||
INTERNAL_ENCRYPTION_KEY_ID_HEADER, SSEType, contains_managed_encryption_metadata, normalize_managed_metadata,
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
/// The key a stored object is decrypted with, recovered from what the object recorded.
|
||||
pub(crate) struct OpenedObjectKey {
|
||||
/// Key handed to the stream cipher.
|
||||
pub(crate) key_bytes: [u8; 32],
|
||||
/// Base nonce the stream cipher derives its block nonces from.
|
||||
pub(crate) base_nonce: [u8; 12],
|
||||
/// The mode the object was written in.
|
||||
pub(crate) key_kind: EncryptionKeyKind,
|
||||
}
|
||||
|
||||
/// The managed scheme a stored object was written under, or `None` when this build
|
||||
/// cannot establish it.
|
||||
fn stored_managed_sse_type(metadata: &HashMap<String, String>) -> Option<SSEType> {
|
||||
match metadata.get("x-amz-server-side-encryption").map(String::as_str) {
|
||||
Some(AMZ_ENCRYPTION_KMS) => Some(SSEType::SseKms),
|
||||
Some(_) => Some(SSEType::SseS3),
|
||||
// MinIO never persists the public scheme header: `crypto.S3.CreateMetadata`
|
||||
// writes only the `X-Minio-Internal-*` family and the public header is
|
||||
// synthesized onto the response by `DecryptObjectInfo`. Requiring it here
|
||||
// is what made every MinIO-encrypted object unreadable (backlog#1638).
|
||||
//
|
||||
// Inferring from the sealed-key slot is self-consistent by construction:
|
||||
// the slot decides which header the unseal reads AND which domain string
|
||||
// the sealing key is derived under, so a scheme that disagrees with the
|
||||
// slot cannot silently derive a wrong key — it finds no key at all.
|
||||
// Inferring from the KMS key id would NOT be safe: MinIO writes
|
||||
// `-S3-Kms-Key-Id` on SSE-S3 objects too.
|
||||
//
|
||||
// An object whose scheme cannot be established stays fail-closed, and
|
||||
// deliberately not through an error raised here: the read plan independently
|
||||
// classifies the object as encrypted from its markers and refuses to serve
|
||||
// it without material, so it never degrades into a plaintext read.
|
||||
#[cfg(feature = "rio-v2")]
|
||||
None => infer_minio_managed_sse_type(metadata),
|
||||
// Without the rio-v2 reader there is no MinIO-format read path to serve
|
||||
// such an object with, so it stays on the fail-closed branch.
|
||||
#[cfg(not(feature = "rio-v2"))]
|
||||
None => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// SSE-C: recover the object's key from the customer key.
|
||||
///
|
||||
/// A sealed object key recorded in MinIO's slot is unsealed under the customer key;
|
||||
/// otherwise the customer key is the object's key, under the nonce the object recorded.
|
||||
pub(crate) fn open_under_customer_key(
|
||||
metadata: &HashMap<String, String>,
|
||||
customer_key: [u8; 32],
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
) -> Result<OpenedObjectKey, SseError> {
|
||||
#[cfg(feature = "rio-v2")]
|
||||
if let Some(sealed_key) = parse_minio_managed_sealed_key(metadata, SSEType::SseC)? {
|
||||
return Ok(OpenedObjectKey {
|
||||
key_bytes: unseal_object_key(&sealed_key, customer_key, SSEType::SseC, bucket, key)?,
|
||||
base_nonce: [0; 12],
|
||||
key_kind: EncryptionKeyKind::Object,
|
||||
});
|
||||
}
|
||||
|
||||
Ok(OpenedObjectKey {
|
||||
key_bytes: customer_key,
|
||||
base_nonce: read_stored_ssec_nonce(metadata, bucket, key),
|
||||
key_kind: EncryptionKeyKind::Direct,
|
||||
})
|
||||
}
|
||||
|
||||
/// A stored managed-SSE object, as recognized from its metadata alone.
|
||||
pub(crate) struct StoredManagedKey<'a> {
|
||||
/// The object's metadata as stored.
|
||||
metadata: &'a HashMap<String, String>,
|
||||
/// The same metadata with the managed keys normalized.
|
||||
normalized_metadata: HashMap<String, String>,
|
||||
/// The scheme the object was written under.
|
||||
pub(crate) sse_type: SSEType,
|
||||
/// The KMS key id the object recorded, if it recorded one.
|
||||
pub(crate) recorded_kms_key_id: Option<String>,
|
||||
}
|
||||
|
||||
/// Recognize a stored managed-SSE object.
|
||||
///
|
||||
/// `None` means the object carries no managed-SSE metadata, or carries some whose
|
||||
/// scheme this build cannot establish.
|
||||
pub(crate) fn recognize_stored_managed_key(metadata: &HashMap<String, String>) -> Option<StoredManagedKey<'_>> {
|
||||
if !contains_managed_encryption_metadata(metadata) {
|
||||
return None;
|
||||
}
|
||||
let sse_type = stored_managed_sse_type(metadata)?;
|
||||
|
||||
let normalized_metadata = normalize_managed_metadata(metadata, Some(recode_minio_kms_context));
|
||||
let recorded_kms_key_id = normalized_metadata
|
||||
.get(INTERNAL_ENCRYPTION_KEY_ID_HEADER)
|
||||
.or_else(|| metadata.get("x-amz-server-side-encryption-aws-kms-key-id"))
|
||||
.cloned();
|
||||
|
||||
Some(StoredManagedKey {
|
||||
metadata,
|
||||
normalized_metadata,
|
||||
sse_type,
|
||||
recorded_kms_key_id,
|
||||
})
|
||||
}
|
||||
|
||||
/// Managed SSE: unwrap the stored data key and recover the object's key from it.
|
||||
///
|
||||
/// `kms_key_id` is the key the read was authorized against. Returns the object's key
|
||||
/// together with the algorithm the object recorded.
|
||||
pub(crate) async fn open_under_stored_data_key(
|
||||
host: &dyn SseHost,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
stored: &StoredManagedKey<'_>,
|
||||
kms_key_id: &str,
|
||||
) -> Result<(OpenedObjectKey, String), SseError> {
|
||||
let record = read_stored_data_key(host, stored).await?;
|
||||
let data_key = unwrap_stored_data_key(host, bucket, key, stored, &record, kms_key_id).await?;
|
||||
|
||||
let opened = match record.mode {
|
||||
StoredKeyMode::Direct { base_nonce } => OpenedObjectKey {
|
||||
key_bytes: data_key,
|
||||
base_nonce,
|
||||
key_kind: EncryptionKeyKind::Direct,
|
||||
},
|
||||
#[cfg(feature = "rio-v2")]
|
||||
StoredKeyMode::Object { sealed_key } => OpenedObjectKey {
|
||||
key_bytes: unseal_object_key(&sealed_key, data_key, stored.sse_type, bucket, key)?,
|
||||
base_nonce: [0u8; 12],
|
||||
key_kind: EncryptionKeyKind::Object,
|
||||
},
|
||||
};
|
||||
|
||||
Ok((opened, record.algorithm))
|
||||
}
|
||||
|
||||
/// Read the stored data key in the mode the object recorded.
|
||||
///
|
||||
/// This is the recognition order: a mode listed earlier claims the object first, and
|
||||
/// Direct takes whatever no other mode recognizes. A mode this build cannot read is
|
||||
/// not listed, so its objects fail in the Direct reader instead of being misread.
|
||||
async fn read_stored_data_key(host: &dyn SseHost, stored: &StoredManagedKey<'_>) -> Result<StoredDataKey, SseError> {
|
||||
#[cfg(feature = "rio-v2")]
|
||||
if let Some(sealed_key) = parse_minio_managed_sealed_key(stored.metadata, stored.sse_type)? {
|
||||
return read_sealed_data_key(stored.metadata, &stored.normalized_metadata, sealed_key);
|
||||
}
|
||||
|
||||
read_wrapped_data_key(host, &stored.normalized_metadata).await
|
||||
}
|
||||
|
||||
/// Unwrap the stored data key with the provider its own bytes identify.
|
||||
async fn unwrap_stored_data_key(
|
||||
host: &dyn SseHost,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
stored: &StoredManagedKey<'_>,
|
||||
record: &StoredDataKey,
|
||||
kms_key_id: &str,
|
||||
) -> Result<[u8; 32], SseError> {
|
||||
let kms_context = if matches!(stored.sse_type, SSEType::SseKms) {
|
||||
decode_minio_kms_context(stored.metadata)?
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let object_context = build_object_encryption_context(bucket, key, kms_context.as_ref());
|
||||
let wrapped = record.wrapped.as_slice();
|
||||
|
||||
// Persisted wrapping format is the read-side source of truth. The
|
||||
// advertised SSE scheme and current KMS availability are write policy
|
||||
// and runtime state, neither of which identifies the historical provider.
|
||||
//
|
||||
// The one shape that cannot name its provider is a backend-native blob
|
||||
// stored without an envelope. It is offered to the KMS only when the
|
||||
// local provider could not have written those bytes and the active
|
||||
// backend is one whose released versions persisted them that way.
|
||||
let bare_kms = if is_data_key_envelope(wrapped) || is_local_sse_dek(wrapped) {
|
||||
None
|
||||
} else {
|
||||
bare_data_key_kms_provider(host).await
|
||||
};
|
||||
|
||||
// A MinIO sealed key alone does not mean MinIO wrote the object: RustFS's own
|
||||
// writer fills MinIO's metadata slots too, while still storing a RustFS
|
||||
// envelope in them, so neither the slot nor the header name distinguishes the
|
||||
// two. The data key's own shape does. RustFS envelopes are strictly-parsed
|
||||
// JSON; MinIO's builtin-KMS ciphertext is opaque bytes that match neither, so
|
||||
// recognizing RustFS positively — and treating only the remainder as MinIO —
|
||||
// keeps a RustFS envelope from ever reaching MinIO's decoder.
|
||||
#[cfg(feature = "rio-v2")]
|
||||
if matches!(record.mode, StoredKeyMode::Object { .. }) && !is_rustfs_managed_data_key(wrapped) {
|
||||
return decrypt_unenveloped_sealed_dek(bare_kms, get_local_sse_dek_provider(), wrapped, kms_key_id, &object_context)
|
||||
.await;
|
||||
}
|
||||
|
||||
let provider: Arc<dyn SseDekProvider> = if is_data_key_envelope(wrapped) {
|
||||
kms_sse_dek_provider(host).await?
|
||||
} else if let Some(kms_provider) = bare_kms {
|
||||
kms_provider
|
||||
} else {
|
||||
get_local_sse_dek_provider()?
|
||||
};
|
||||
|
||||
#[cfg(feature = "rio-v2")]
|
||||
if is_legacy_rustfs_managed_metadata(&stored.normalized_metadata) {
|
||||
return provider.decrypt_legacy_sse_dek(wrapped, kms_key_id, &object_context).await;
|
||||
}
|
||||
|
||||
provider.decrypt_sse_dek(wrapped, kms_key_id, &object_context).await
|
||||
}
|
||||
@@ -1,23 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! SSE key resolution: the S3-defined schemes, the internal keying modes, and the entry points that join them.
|
||||
|
||||
pub(crate) mod audit;
|
||||
pub(crate) mod keying;
|
||||
pub(crate) mod read;
|
||||
pub(crate) mod request;
|
||||
pub(crate) mod rewrap;
|
||||
pub(crate) mod s3;
|
||||
pub(crate) mod write;
|
||||
@@ -1,303 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Read entry points: resolve the decryption material, or the response headers, for a stored object.
|
||||
|
||||
use crate::core::audit::{record_managed_kms_outcome, stored_envelope_master_key_version};
|
||||
use crate::core::keying::material::DecryptionMaterial;
|
||||
use crate::core::keying::metadata::stored_ssec_metadata;
|
||||
use crate::core::keying::stored::{
|
||||
StoredManagedKey, open_under_customer_key, open_under_stored_data_key, recognize_stored_managed_key,
|
||||
};
|
||||
use crate::core::request::SseKmsPrincipal;
|
||||
use crate::core::s3::authorization::authorize_sse_kms_key;
|
||||
use crate::core::s3::headers::{SseReadResponseHeaders, managed_sse_public_header};
|
||||
use crate::core::s3::scheme::resolve_stored_kms_key_id;
|
||||
use crate::core::s3::ssec::{SsecParams, require_matching_ssec_key, stored_ssec_algorithm, validate_ssec_params};
|
||||
use crate::error::SseError;
|
||||
use crate::host::SseHost;
|
||||
use rustfs_policy::policy::action::KmsAction;
|
||||
use rustfs_utils::http::headers::AMZ_ENCRYPTION_AES;
|
||||
use rustfs_utils::http::object_encryption_keys::{SSEType, contains_managed_encryption_metadata, stored_managed_encryption_key};
|
||||
use std::collections::HashMap;
|
||||
|
||||
impl std::fmt::Debug for DecryptionRequest<'_> {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("DecryptionRequest").finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
|
||||
/// Request parameters for unified decryption
|
||||
pub struct DecryptionRequest<'a> {
|
||||
/// Bucket name
|
||||
pub bucket: &'a str,
|
||||
/// Object key
|
||||
pub key: &'a str,
|
||||
/// Object metadata containing encryption headers
|
||||
pub metadata: &'a HashMap<String, String>,
|
||||
/// SSE-C key (Base64-encoded) - required if object was encrypted with SSE-C
|
||||
pub sse_customer_key: Option<&'a String>,
|
||||
/// SSE-C key MD5 (Base64-encoded) - required if object was encrypted with SSE-C
|
||||
pub sse_customer_key_md5: Option<&'a String>,
|
||||
/// Caller the SSE-KMS key usage is authorized as. `None` marks an internal caller.
|
||||
pub principal: Option<&'a SseKmsPrincipal>,
|
||||
}
|
||||
|
||||
/// **Core API**: Apply decryption based on stored metadata
|
||||
///
|
||||
/// This function automatically detects the encryption type from metadata:
|
||||
/// - SSE-C if customer key is provided
|
||||
/// - SSE-S3/SSE-KMS if managed encryption metadata is found
|
||||
/// - None if object is not encrypted
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `request` - Decryption request with metadata and optional customer key
|
||||
///
|
||||
/// # Returns
|
||||
/// * `Ok(Some(material))` - Decryption should be applied with the returned material
|
||||
/// * `Ok(None)` - Object is not encrypted
|
||||
/// * `Err` - Decryption configuration error or key mismatch
|
||||
///
|
||||
/// # Example
|
||||
/// ```rust,ignore
|
||||
/// let request = DecryptionRequest {
|
||||
/// bucket: &bucket,
|
||||
/// key: &key,
|
||||
/// metadata: &metadata,
|
||||
/// sse_customer_key: sse_customer_key.as_deref(),
|
||||
/// sse_customer_key_md5: sse_customer_key_md5.as_deref(),
|
||||
/// };
|
||||
///
|
||||
/// if let Some(material) = sse_decryption(host, request).await? {
|
||||
/// content_size = material.original_size.unwrap_or(actual_size);
|
||||
/// }
|
||||
/// ```
|
||||
pub async fn sse_decryption(host: &dyn SseHost, request: DecryptionRequest<'_>) -> Result<Option<DecryptionMaterial>, SseError> {
|
||||
// Check for SSE-C encryption
|
||||
if stored_ssec_metadata(request.metadata) {
|
||||
let (key, key_md5) = require_matching_ssec_key(request.metadata, request.sse_customer_key, request.sse_customer_key_md5)?;
|
||||
|
||||
let mut material = apply_ssec_decryption_material(request.bucket, request.key, request.metadata, key, key_md5).await?;
|
||||
material.customer_key_md5 = Some(key_md5.clone());
|
||||
return Ok(Some(material));
|
||||
}
|
||||
|
||||
// Check for managed SSE encryption
|
||||
if contains_managed_encryption_metadata(request.metadata) {
|
||||
return apply_managed_decryption_material(host, request.bucket, request.key, request.metadata, request.principal).await;
|
||||
}
|
||||
|
||||
// No encryption detected
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
/// Read-side response classification for an object whose payload the object
|
||||
/// layer's encryption resolver already decrypted.
|
||||
///
|
||||
/// [`sse_decryption`] both unwraps the data key and derives the response
|
||||
/// headers. The GET path only needs the latter — its stream comes out of the
|
||||
/// object layer decrypted (see [`SseObjectEncryptionResolver`]) — so calling
|
||||
/// [`sse_decryption`] there performed a second KMS `Decrypt` per request whose
|
||||
/// key bytes were discarded. This function reproduces that call's header,
|
||||
/// validation, authorization and audit behavior from stored metadata alone.
|
||||
///
|
||||
/// Contract, mirrored from [`sse_decryption`] item by item:
|
||||
/// - SSE-C: missing request key/MD5 and stored/request MD5 mismatch fail with
|
||||
/// the same errors and precedence, and the provided key is fully validated.
|
||||
/// - Managed SSE: per-key `kms:Decrypt` authorization runs ahead of every other
|
||||
/// failure mode, and the request's KMS audit summary records the same
|
||||
/// scheme/key/version/outcome fields the unwrap-based path recorded. The
|
||||
/// success outcome is honest because a failed unwrap aborts the read in the
|
||||
/// object layer before response classification is ever reached.
|
||||
/// - Objects without SSE metadata — and managed metadata whose scheme cannot
|
||||
/// be established in this build — classify as `None`, exactly where
|
||||
/// [`sse_decryption`] returned `None`.
|
||||
pub async fn classify_sse_read_response(
|
||||
host: &dyn SseHost,
|
||||
request: DecryptionRequest<'_>,
|
||||
) -> Result<Option<SseReadResponseHeaders>, SseError> {
|
||||
if stored_ssec_metadata(request.metadata) {
|
||||
let (key, key_md5) = require_matching_ssec_key(request.metadata, request.sse_customer_key, request.sse_customer_key_md5)?;
|
||||
|
||||
let algorithm = stored_ssec_algorithm(request.metadata);
|
||||
validate_ssec_params(SsecParams {
|
||||
algorithm: algorithm.to_string(),
|
||||
key: key.to_string(),
|
||||
key_md5: key_md5.to_string(),
|
||||
})?;
|
||||
|
||||
return Ok(Some(SseReadResponseHeaders {
|
||||
server_side_encryption: String::from(AMZ_ENCRYPTION_AES),
|
||||
sse_customer_algorithm: Some(String::from(algorithm)),
|
||||
sse_customer_key_md5: Some(key_md5.clone()),
|
||||
ssekms_key_id: None,
|
||||
}));
|
||||
}
|
||||
|
||||
// Same scheme and key resolution as the unwrap path, so authorization and the
|
||||
// response header name the same key.
|
||||
let Some((stored, kms_key_id)) = resolve_stored_managed_key(host, request.metadata).await else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
// Ahead of every other failure mode, so a denied caller learns nothing
|
||||
// about the key beyond "not yours".
|
||||
if let Err(error) =
|
||||
authorize_sse_kms_key(host, request.principal, stored.sse_type, KmsAction::DecryptAction, &kms_key_id).await
|
||||
{
|
||||
if let Some((sse_type, key_id)) = stored_managed_encryption_key(request.metadata) {
|
||||
record_managed_kms_outcome(
|
||||
request.principal,
|
||||
sse_type,
|
||||
Some(&key_id),
|
||||
|| stored_envelope_master_key_version(request.metadata),
|
||||
Err(&error),
|
||||
);
|
||||
}
|
||||
return Err(error);
|
||||
}
|
||||
|
||||
record_managed_kms_outcome(
|
||||
request.principal,
|
||||
stored.sse_type,
|
||||
Some(&kms_key_id),
|
||||
|| stored_envelope_master_key_version(request.metadata),
|
||||
Ok(()),
|
||||
);
|
||||
|
||||
Ok(Some(SseReadResponseHeaders {
|
||||
server_side_encryption: managed_sse_public_header(stored.sse_type).to_string(),
|
||||
sse_customer_algorithm: None,
|
||||
sse_customer_key_md5: None,
|
||||
// The key id was needed above to authorize the read, but an AES256
|
||||
// object's wrapping key is internal: only aws:kms objects advertise it.
|
||||
ssekms_key_id: matches!(stored.sse_type, SSEType::SseKms).then_some(kms_key_id),
|
||||
}))
|
||||
}
|
||||
|
||||
/// Recognize a stored managed-SSE object and name the key it is read under.
|
||||
///
|
||||
/// One resolution for every read: authorization, the response headers and the
|
||||
/// unwrap all name the key this returns. `None` means the object carries no
|
||||
/// managed-SSE metadata, or carries some whose scheme this build cannot establish.
|
||||
async fn resolve_stored_managed_key<'m>(
|
||||
host: &dyn SseHost,
|
||||
metadata: &'m HashMap<String, String>,
|
||||
) -> Option<(StoredManagedKey<'m>, String)> {
|
||||
let stored = recognize_stored_managed_key(metadata)?;
|
||||
let kms_key_id = resolve_stored_kms_key_id(host, stored.recorded_kms_key_id.clone()).await;
|
||||
Some((stored, kms_key_id))
|
||||
}
|
||||
|
||||
pub(crate) async fn apply_ssec_decryption_material(
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
metadata: &HashMap<String, String>,
|
||||
sse_key: &str,
|
||||
sse_key_md5: &str,
|
||||
) -> Result<DecryptionMaterial, SseError> {
|
||||
// Validate provided key
|
||||
let algorithm = stored_ssec_algorithm(metadata);
|
||||
|
||||
let params = SsecParams {
|
||||
algorithm: algorithm.to_string(),
|
||||
key: sse_key.to_string(),
|
||||
key_md5: sse_key_md5.to_string(),
|
||||
};
|
||||
|
||||
let validated = validate_ssec_params(params)?;
|
||||
let opened = open_under_customer_key(metadata, validated.key_bytes, bucket, key)?;
|
||||
|
||||
Ok(DecryptionMaterial {
|
||||
sse_type: SSEType::SseC,
|
||||
server_side_encryption: String::from(AMZ_ENCRYPTION_AES), // const
|
||||
kms_key_id: None,
|
||||
algorithm: String::from(algorithm),
|
||||
|
||||
customer_key_md5: None,
|
||||
key_bytes: opened.key_bytes,
|
||||
base_nonce: opened.base_nonce,
|
||||
key_kind: opened.key_kind,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) async fn apply_managed_decryption_material(
|
||||
host: &dyn SseHost,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
metadata: &HashMap<String, String>,
|
||||
principal: Option<&SseKmsPrincipal>,
|
||||
) -> Result<Option<DecryptionMaterial>, SseError> {
|
||||
let result = apply_managed_decryption_material_inner(host, bucket, key, metadata, principal).await;
|
||||
|
||||
match &result {
|
||||
// `None` means the object carries no managed-SSE metadata — SSE-C and
|
||||
// plaintext objects never reach KMS and must not appear in the summary.
|
||||
Ok(None) => {}
|
||||
Ok(Some(material)) => record_managed_kms_outcome(
|
||||
principal,
|
||||
material.sse_type,
|
||||
material.kms_key_id.as_deref(),
|
||||
|| stored_envelope_master_key_version(metadata),
|
||||
Ok(()),
|
||||
),
|
||||
Err(error) => {
|
||||
if let Some((sse_type, key_id)) = stored_managed_encryption_key(metadata) {
|
||||
record_managed_kms_outcome(
|
||||
principal,
|
||||
sse_type,
|
||||
Some(&key_id),
|
||||
|| stored_envelope_master_key_version(metadata),
|
||||
Err(error),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
result
|
||||
}
|
||||
|
||||
async fn apply_managed_decryption_material_inner(
|
||||
host: &dyn SseHost,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
metadata: &HashMap<String, String>,
|
||||
principal: Option<&SseKmsPrincipal>,
|
||||
) -> Result<Option<DecryptionMaterial>, SseError> {
|
||||
let Some((stored, kms_key_id)) = resolve_stored_managed_key(host, metadata).await else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
// Ahead of every other failure mode below, so a denied caller learns nothing about the
|
||||
// key beyond "not yours" — not whether it is disabled, pending deletion, or unreadable.
|
||||
authorize_sse_kms_key(host, principal, stored.sse_type, KmsAction::DecryptAction, &kms_key_id).await?;
|
||||
|
||||
let (opened, algorithm) = open_under_stored_data_key(host, bucket, key, &stored, &kms_key_id).await?;
|
||||
|
||||
Ok(Some(DecryptionMaterial {
|
||||
sse_type: stored.sse_type,
|
||||
// Synthesized from the resolved scheme rather than read back from
|
||||
// metadata: a MinIO-written object has no stored scheme header, which is
|
||||
// exactly why the scheme had to be inferred. MinIO synthesizes the same
|
||||
// header onto its own responses.
|
||||
server_side_encryption: managed_sse_public_header(stored.sse_type).to_string(),
|
||||
kms_key_id: Some(kms_key_id),
|
||||
algorithm,
|
||||
customer_key_md5: None,
|
||||
|
||||
key_bytes: opened.key_bytes,
|
||||
base_nonce: opened.base_nonce,
|
||||
key_kind: opened.key_kind,
|
||||
}))
|
||||
}
|
||||
@@ -1,148 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! The caller an SSE operation is authorized as.
|
||||
|
||||
use crate::core::audit::{KmsRequestAudit, kms_request_audit};
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
use crate::host::KmsKeyAuthorizer;
|
||||
use serde_json::Value;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
/// Identity a data-path KMS key operation is attributed to.
|
||||
///
|
||||
/// Only requests that crossed the S3 authorization boundary carry one. Internal
|
||||
/// callers — replication, lifecycle transition, heal, scanner — enter the object
|
||||
/// layer directly and run with server credentials, so they never build a principal
|
||||
/// and are exempt by construction; `None` therefore means "system principal".
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct SseKmsPrincipal {
|
||||
pub(crate) account: String,
|
||||
groups: Option<Vec<String>>,
|
||||
is_owner: bool,
|
||||
claims: HashMap<String, Value>,
|
||||
conditions: HashMap<String, Vec<String>>,
|
||||
/// Audit slot of the S3 request this principal was built for, when the request
|
||||
/// is being audited. The principal is the only request-derived value that reaches
|
||||
/// the managed-SSE code paths, so it doubles as the carrier for the slot.
|
||||
pub(crate) request_audit: Option<Arc<KmsRequestAudit>>,
|
||||
/// Test-only decision overrides. Carried per principal rather than in a global slot so
|
||||
/// concurrent tests cannot observe each other's injection.
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub(crate) test_hooks: Option<TestAuthorizationHooks>,
|
||||
}
|
||||
|
||||
impl SseKmsPrincipal {
|
||||
/// Build the principal of a request that crossed the S3 authorization boundary.
|
||||
///
|
||||
/// Anonymous requests use an empty account, matching the bucket-policy authorization
|
||||
/// path. This keeps every request that crossed the S3 boundary distinct from trusted
|
||||
/// internal callers, which are represented by `None`.
|
||||
pub fn new(
|
||||
account: String,
|
||||
groups: Option<Vec<String>>,
|
||||
is_owner: bool,
|
||||
claims: HashMap<String, Value>,
|
||||
conditions: HashMap<String, Vec<String>>,
|
||||
) -> Self {
|
||||
Self {
|
||||
account,
|
||||
groups,
|
||||
is_owner,
|
||||
claims,
|
||||
conditions,
|
||||
request_audit: None,
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
test_hooks: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// The access key the request authenticated with; empty for an anonymous request.
|
||||
pub fn account(&self) -> &str {
|
||||
&self.account
|
||||
}
|
||||
|
||||
pub fn groups(&self) -> &Option<Vec<String>> {
|
||||
&self.groups
|
||||
}
|
||||
|
||||
pub fn is_owner(&self) -> bool {
|
||||
self.is_owner
|
||||
}
|
||||
|
||||
pub fn claims(&self) -> &HashMap<String, Value> {
|
||||
&self.claims
|
||||
}
|
||||
|
||||
/// Policy condition values of the request that do not depend on the resource.
|
||||
pub fn conditions(&self) -> &HashMap<String, Vec<String>> {
|
||||
&self.conditions
|
||||
}
|
||||
|
||||
/// Attach the audit slot opened for `request_id`, if that request is being audited
|
||||
/// (see [`KmsRequestAuditScope::register`]).
|
||||
#[must_use]
|
||||
pub fn with_request_id(mut self, request_id: &str) -> Self {
|
||||
self.request_audit = kms_request_audit(request_id);
|
||||
self
|
||||
}
|
||||
|
||||
/// Override the enforcement switch and the authorizer for this principal only.
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
#[must_use]
|
||||
pub fn with_test_hooks(mut self, hooks: Option<TestAuthorizationHooks>) -> Self {
|
||||
self.test_hooks = hooks;
|
||||
self
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn for_test(account: &str, enforced: bool, authorizer: Arc<dyn KmsKeyAuthorizer>) -> Self {
|
||||
Self {
|
||||
account: account.to_string(),
|
||||
groups: None,
|
||||
is_owner: false,
|
||||
claims: HashMap::new(),
|
||||
conditions: HashMap::new(),
|
||||
request_audit: None,
|
||||
test_hooks: Some(TestAuthorizationHooks { enforced, authorizer }),
|
||||
}
|
||||
}
|
||||
|
||||
/// Bind this principal to `audit` so managed-SSE operations performed for it are
|
||||
/// summarised onto the request's S3 audit entry.
|
||||
#[cfg(test)]
|
||||
pub(crate) fn with_request_audit(mut self, audit: Arc<KmsRequestAudit>) -> Self {
|
||||
self.request_audit = Some(audit);
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
/// Per-principal override of SSE-KMS key authorization, for tests.
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
#[derive(Clone)]
|
||||
pub struct TestAuthorizationHooks {
|
||||
pub enforced: bool,
|
||||
pub authorizer: Arc<dyn KmsKeyAuthorizer>,
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
impl std::fmt::Debug for TestAuthorizationHooks {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
formatter
|
||||
.debug_struct("TestAuthorizationHooks")
|
||||
.field("enforced", &self.enforced)
|
||||
.finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
@@ -1,144 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Re-wrap a stored data key onto its master key's current version.
|
||||
|
||||
use crate::core::keying::context::{build_object_encryption_context, decode_minio_kms_context};
|
||||
use crate::core::keying::metadata::recode_minio_kms_context;
|
||||
use crate::error::SseError;
|
||||
use crate::host::SseHost;
|
||||
use crate::kms::provider::kms_sse_dek_provider;
|
||||
use base64_simd::STANDARD as BASE64_STANDARD;
|
||||
use rustfs_kms::is_data_key_envelope;
|
||||
use rustfs_utils::http::headers::AMZ_ENCRYPTION_KMS;
|
||||
use rustfs_utils::http::object_encryption_keys::{
|
||||
INTERNAL_ENCRYPTION_KEY_HEADER, MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER, MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER, SSEType,
|
||||
contains_managed_encryption_metadata, normalize_managed_metadata,
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
|
||||
/// Outcome of a single object's DEK rewrap attempt.
|
||||
// Consumed by the bulk rekey sweep in the follow-up PR; tests exercise it now.
|
||||
#[allow(dead_code)]
|
||||
#[derive(Debug)]
|
||||
pub enum ObjectDekRewrapOutcome {
|
||||
/// The object carries no KMS-wrapped RustFS data-key envelope this build
|
||||
/// can rewrap: plaintext, SSE-C, or a MinIO-sealed data key.
|
||||
NotApplicable,
|
||||
/// The envelope is already on the current master key version and format;
|
||||
/// the caller must persist nothing, so a sweep re-run converges.
|
||||
AlreadyCurrent,
|
||||
/// The envelope was rewrapped. `metadata` holds the overrides to merge
|
||||
/// into the object's user-defined metadata — every stored copy of the old
|
||||
/// envelope, replaced — via `ObjectLayer::put_object_metadata`.
|
||||
Rewrapped { metadata: HashMap<String, String> },
|
||||
}
|
||||
|
||||
/// Rewrap the KMS-wrapped data key in an object's stored metadata onto its
|
||||
/// master key's current version (and current envelope format), without
|
||||
/// touching the object's data or its plaintext DEK.
|
||||
///
|
||||
/// Mirrors the managed decrypt path byte for byte where it matters: the
|
||||
/// envelope is located through the same normalized-header resolution, and the
|
||||
/// encryption context is rebuilt with the same helpers, so an envelope the
|
||||
/// read path can open is exactly an envelope this can rewrap. The KMS backend
|
||||
/// owns the format and the no-op decision.
|
||||
///
|
||||
/// The returned overrides replace every stored copy of the old envelope
|
||||
/// (RustFS's internal header and the MinIO-compatible slots that the writer
|
||||
/// fills with the same bytes). A copy left behind would win a read-path
|
||||
/// fallback and resurrect the old wrapping, so finding no replaceable copy is
|
||||
/// an error, never a silent success.
|
||||
pub async fn rewrap_object_encryption_metadata(
|
||||
host: &dyn SseHost,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
metadata: &HashMap<String, String>,
|
||||
) -> Result<ObjectDekRewrapOutcome, SseError> {
|
||||
if !contains_managed_encryption_metadata(metadata) {
|
||||
return Ok(ObjectDekRewrapOutcome::NotApplicable);
|
||||
}
|
||||
|
||||
let encryption_type = match metadata.get("x-amz-server-side-encryption").map(String::as_str) {
|
||||
Some(AMZ_ENCRYPTION_KMS) => SSEType::SseKms,
|
||||
Some(_) => SSEType::SseS3,
|
||||
// MinIO-written objects synthesize the public header on read; their
|
||||
// sealed data keys are not RustFS envelopes and fall out below.
|
||||
None => SSEType::SseS3,
|
||||
};
|
||||
|
||||
let normalized_metadata = normalize_managed_metadata(metadata, Some(recode_minio_kms_context));
|
||||
let Some(envelope_b64) = normalized_metadata
|
||||
.get(INTERNAL_ENCRYPTION_KEY_HEADER)
|
||||
.or_else(|| metadata.get(MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER))
|
||||
else {
|
||||
return Ok(ObjectDekRewrapOutcome::NotApplicable);
|
||||
};
|
||||
let encrypted_data_key = BASE64_STANDARD
|
||||
.decode_to_vec(envelope_b64)
|
||||
.map_err(|e| SseError::internal(format!("Failed to decode encrypted key: {e}")))?;
|
||||
// Only RustFS envelopes are rewrappable here; MinIO's builtin-KMS
|
||||
// ciphertext is opaque bytes owned by a different root of trust.
|
||||
if !is_data_key_envelope(&encrypted_data_key) {
|
||||
return Ok(ObjectDekRewrapOutcome::NotApplicable);
|
||||
}
|
||||
|
||||
let kms_context = if matches!(encryption_type, SSEType::SseKms) {
|
||||
decode_minio_kms_context(metadata)?
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let object_context = build_object_encryption_context(bucket, key, kms_context.as_ref());
|
||||
|
||||
// The same provider selection as the managed decrypt path.
|
||||
let provider = kms_sse_dek_provider(host).await?;
|
||||
let response = provider.rewrap_sse_dek(&encrypted_data_key, &object_context).await?;
|
||||
if !response.rewrapped {
|
||||
return Ok(ObjectDekRewrapOutcome::AlreadyCurrent);
|
||||
}
|
||||
|
||||
// Replace every stored copy of the old envelope, keyed by value and
|
||||
// matched case-insensitively: metadata key casing drifts through the
|
||||
// storage layer, and an override inserted under a differently-cased name
|
||||
// would sit beside the old copy instead of replacing it. The MinIO
|
||||
// sealed-key slots can instead hold a sealed *object* key (rio-v2 writer);
|
||||
// those bytes differ from the envelope and are untouched — the data key
|
||||
// they seal is unchanged by a rewrap.
|
||||
const REWRAP_ENVELOPE_HEADERS: [&str; 4] = [
|
||||
INTERNAL_ENCRYPTION_KEY_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER,
|
||||
];
|
||||
let old_envelope_b64 = envelope_b64.clone();
|
||||
let new_envelope_b64 = BASE64_STANDARD.encode_to_string(&response.ciphertext);
|
||||
let mut overrides = HashMap::new();
|
||||
for (stored_name, stored_value) in metadata {
|
||||
let is_envelope_slot = REWRAP_ENVELOPE_HEADERS
|
||||
.iter()
|
||||
.any(|header| stored_name.eq_ignore_ascii_case(header));
|
||||
if is_envelope_slot && *stored_value == old_envelope_b64 {
|
||||
overrides.insert(stored_name.clone(), new_envelope_b64.clone());
|
||||
}
|
||||
}
|
||||
if overrides.is_empty() {
|
||||
return Err(SseError::internal(
|
||||
"rewrapped a data-key envelope but found no stored metadata copy to replace; refusing a write that would \
|
||||
leave the old wrapping live",
|
||||
));
|
||||
}
|
||||
|
||||
Ok(ObjectDekRewrapOutcome::Rewrapped { metadata: overrides })
|
||||
}
|
||||
@@ -1,177 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Per-key KMS authorization on the SSE-KMS data path.
|
||||
|
||||
use crate::core::audit::{record_managed_kms_outcome, stored_envelope_master_key_version};
|
||||
use crate::core::request::SseKmsPrincipal;
|
||||
use crate::error::SseError;
|
||||
use crate::host::SseHost;
|
||||
use crate::{LOG_COMPONENT_STORAGE, LOG_SUBSYSTEM_SSE};
|
||||
use rustfs_config::ENV_RUSTFS_KMS_ENFORCE_SSE_KEY_POLICY;
|
||||
use rustfs_policy::policy::action::KmsAction;
|
||||
use rustfs_s3_types::S3ErrorCode;
|
||||
use rustfs_utils::http::object_encryption_keys::{SSEType, stored_managed_encryption_key};
|
||||
use std::collections::HashMap;
|
||||
use tracing::debug;
|
||||
|
||||
/// Ask the authorizer that decides for `principal`: the host's, unless a test overrode it.
|
||||
async fn is_kms_key_allowed(
|
||||
host: &dyn SseHost,
|
||||
principal: &SseKmsPrincipal,
|
||||
action: KmsAction,
|
||||
key_id: &str,
|
||||
) -> Result<bool, SseError> {
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
if let Some(hooks) = principal.test_hooks.as_ref() {
|
||||
return hooks.authorizer.is_allowed(principal, action, key_id).await;
|
||||
}
|
||||
|
||||
host.kms_key_authorizer().is_allowed(principal, action, key_id).await
|
||||
}
|
||||
|
||||
/// Whether per-key KMS authorization is enforced on the SSE-KMS data path.
|
||||
fn sse_kms_key_policy_enforced(principal: Option<&SseKmsPrincipal>) -> bool {
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
if let Some(hooks) = principal.and_then(|candidate| candidate.test_hooks.as_ref()) {
|
||||
return hooks.enforced;
|
||||
}
|
||||
#[cfg(not(any(test, feature = "test-util")))]
|
||||
let _ = principal;
|
||||
|
||||
crate::startup::key_policy_enforced()
|
||||
}
|
||||
|
||||
/// The principal whose KMS key permissions this operation must satisfy, if any.
|
||||
///
|
||||
/// Scoping is limited to SSE-KMS, matching AWS: SSE-S3 wraps its data key with a
|
||||
/// server-owned key the caller never names, and SSE-C never reaches KMS at all.
|
||||
pub(crate) fn kms_authorization_subject(
|
||||
enforced: bool,
|
||||
principal: Option<&SseKmsPrincipal>,
|
||||
sse_type: SSEType,
|
||||
) -> Option<&SseKmsPrincipal> {
|
||||
if !enforced || !matches!(sse_type, SSEType::SseKms) {
|
||||
return None;
|
||||
}
|
||||
|
||||
principal
|
||||
}
|
||||
|
||||
/// Authorize `action` against `key_id` for the caller behind this SSE operation.
|
||||
///
|
||||
/// Runs before the key is used, so a denied request cannot distinguish an unauthorized
|
||||
/// key from a disabled or pending-deletion one.
|
||||
pub(crate) async fn authorize_sse_kms_key(
|
||||
host: &dyn SseHost,
|
||||
principal: Option<&SseKmsPrincipal>,
|
||||
sse_type: SSEType,
|
||||
action: KmsAction,
|
||||
key_id: &str,
|
||||
) -> Result<(), SseError> {
|
||||
let Some(principal) = kms_authorization_subject(sse_kms_key_policy_enforced(principal), principal, sse_type) else {
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
if is_kms_key_allowed(host, principal, action, key_id).await? {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
debug!(
|
||||
component = LOG_COMPONENT_STORAGE,
|
||||
subsystem = LOG_SUBSYSTEM_SSE,
|
||||
event = "sse_kms_key_authorization_denied",
|
||||
account = %principal.account,
|
||||
action = ?action,
|
||||
"Principal is not authorized for the KMS key resolved for this request"
|
||||
);
|
||||
|
||||
// One warn per process, not per request: anonymous denials are driven by
|
||||
// unauthenticated traffic, so a per-request warn would let anyone flood the
|
||||
// log. Per-request detail stays on the audit entry and the debug event above.
|
||||
if principal.account.is_empty() {
|
||||
static ANONYMOUS_DENIAL_WARNED: std::sync::Once = std::sync::Once::new();
|
||||
ANONYMOUS_DENIAL_WARNED.call_once(|| {
|
||||
tracing::warn!(
|
||||
component = LOG_COMPONENT_STORAGE,
|
||||
subsystem = LOG_SUBSYSTEM_SSE,
|
||||
event = "sse_kms_anonymous_key_authorization_denied",
|
||||
action = ?action,
|
||||
"Anonymous requests are being denied by SSE-KMS per-key authorization: anonymous \
|
||||
callers hold no kms grants, so a public bucket serving SSE-KMS objects is \
|
||||
incompatible with {ENV_RUSTFS_KMS_ENFORCE_SSE_KEY_POLICY}=true. Reported once per \
|
||||
process; per-request denials are on audit entries and at debug level."
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
Err(SseError {
|
||||
code: S3ErrorCode::AccessDenied,
|
||||
message: "Access Denied".to_string(),
|
||||
source: None,
|
||||
})
|
||||
}
|
||||
|
||||
/// Authorize `kms:Decrypt` for an object whose stored encryption material is unwrapped
|
||||
/// outside [`sse_decryption`].
|
||||
///
|
||||
/// The copy-source read resolves its material inside the object layer, which has no
|
||||
/// request identity, so the S3 layer has to run the check itself.
|
||||
pub async fn authorize_sse_kms_object_read(
|
||||
host: &dyn SseHost,
|
||||
principal: Option<&SseKmsPrincipal>,
|
||||
metadata: &HashMap<String, String>,
|
||||
) -> Result<(), SseError> {
|
||||
let Some((sse_type, key_id)) = stored_managed_encryption_key(metadata) else {
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
let result = authorize_sse_kms_key(host, principal, sse_type, KmsAction::DecryptAction, &key_id).await;
|
||||
// Only a denial is recorded here: an allowed read goes on to unwrap the key,
|
||||
// and that operation reports its own outcome.
|
||||
if let Err(error) = &result {
|
||||
record_managed_kms_outcome(
|
||||
principal,
|
||||
sse_type,
|
||||
Some(&key_id),
|
||||
|| stored_envelope_master_key_version(metadata),
|
||||
Err(error),
|
||||
);
|
||||
}
|
||||
|
||||
result
|
||||
}
|
||||
|
||||
/// Authorize the current request to write under the upload's stored KMS key.
|
||||
/// Upload IDs do not confer the creator's KMS permissions on another caller.
|
||||
pub async fn authorize_sse_kms_multipart_write(
|
||||
host: &dyn SseHost,
|
||||
principal: Option<&SseKmsPrincipal>,
|
||||
metadata: &HashMap<String, String>,
|
||||
) -> Result<(), SseError> {
|
||||
let Some((sse_type, key_id)) = stored_managed_encryption_key(metadata) else {
|
||||
return Ok(());
|
||||
};
|
||||
let result = authorize_sse_kms_key(host, principal, sse_type, KmsAction::GenerateDataKeyAction, &key_id).await;
|
||||
if let Err(error) = &result {
|
||||
record_managed_kms_outcome(
|
||||
principal,
|
||||
sse_type,
|
||||
Some(&key_id),
|
||||
|| stored_envelope_master_key_version(metadata),
|
||||
Err(error),
|
||||
);
|
||||
}
|
||||
result
|
||||
}
|
||||
@@ -1,454 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! SSE request headers in, SSE response headers out.
|
||||
|
||||
use crate::core::keying::metadata::{normalize_encryption_metadata_case, recode_minio_kms_context, stored_ssec_metadata};
|
||||
use crate::core::s3::ssec::ssec_invalid_request;
|
||||
use crate::error::SseError;
|
||||
use base64_simd::STANDARD as BASE64_STANDARD;
|
||||
use http::{HeaderMap, HeaderName, HeaderValue};
|
||||
use rustfs_s3_types::S3ErrorCode;
|
||||
use rustfs_utils::http::headers::{
|
||||
AMZ_ENCRYPTION_AES, AMZ_ENCRYPTION_KMS, AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM,
|
||||
AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY, AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5, AMZ_SERVER_SIDE_ENCRYPTION_KMS_CONTEXT,
|
||||
SSEC_ALGORITHM_HEADER, SSEC_KEY_MD5_HEADER,
|
||||
};
|
||||
use rustfs_utils::http::object_encryption_keys::{
|
||||
INTERNAL_ENCRYPTION_CONTEXT_HEADER, INTERNAL_ENCRYPTION_KEY_ID_HEADER, MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER, MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER, MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER, SSEType,
|
||||
normalize_managed_metadata,
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
|
||||
// The same header names s3s publishes. Owned so this module does not import s3s.
|
||||
pub(crate) const X_AMZ_SERVER_SIDE_ENCRYPTION: HeaderName = HeaderName::from_static("x-amz-server-side-encryption");
|
||||
|
||||
pub(crate) const X_AMZ_SERVER_SIDE_ENCRYPTION_AWS_KMS_KEY_ID: HeaderName =
|
||||
HeaderName::from_static("x-amz-server-side-encryption-aws-kms-key-id");
|
||||
|
||||
pub(crate) const X_AMZ_SERVER_SIDE_ENCRYPTION_CONTEXT: HeaderName =
|
||||
HeaderName::from_static("x-amz-server-side-encryption-context");
|
||||
|
||||
pub(crate) const X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM: HeaderName =
|
||||
HeaderName::from_static("x-amz-server-side-encryption-customer-algorithm");
|
||||
|
||||
pub(crate) const X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5: HeaderName =
|
||||
HeaderName::from_static("x-amz-server-side-encryption-customer-key-md5");
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn sse_invalid_argument(message: &str) -> SseError {
|
||||
SseError {
|
||||
code: S3ErrorCode::InvalidArgument,
|
||||
message: message.to_string(),
|
||||
source: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// SSE-C parameters extracted from headers (algorithm, key, key MD5).
|
||||
pub type SsecParamsFromHeaders = (Option<String>, Option<String>, Option<String>);
|
||||
|
||||
/// Extract SSE-C parameters from request headers.
|
||||
/// Used as fallback when the S3 layer does not populate them in the input struct.
|
||||
///
|
||||
/// Returns an error if an SSE-C header is present but cannot be parsed as valid UTF-8,
|
||||
/// ensuring malformed headers do not bypass validation.
|
||||
pub fn extract_ssec_params_from_headers(headers: &HeaderMap) -> Result<SsecParamsFromHeaders, SseError> {
|
||||
let algorithm = match headers.get("x-amz-server-side-encryption-customer-algorithm") {
|
||||
None => Ok(None),
|
||||
Some(v) => v
|
||||
.to_str()
|
||||
.map(|s| Some(s.to_string()))
|
||||
.map_err(|_| sse_invalid_argument("The x-amz-server-side-encryption-customer-algorithm header must be valid UTF-8.")),
|
||||
}?;
|
||||
|
||||
let key = match headers.get("x-amz-server-side-encryption-customer-key") {
|
||||
None => Ok(None),
|
||||
Some(v) => v
|
||||
.to_str()
|
||||
.map(|s| Some(s.to_string()))
|
||||
.map_err(|_| sse_invalid_argument("The x-amz-server-side-encryption-customer-key header must be valid UTF-8.")),
|
||||
}?;
|
||||
|
||||
let key_md5 = match headers.get("x-amz-server-side-encryption-customer-key-md5") {
|
||||
None => Ok(None),
|
||||
Some(v) => v
|
||||
.to_str()
|
||||
.map(|s| Some(s.to_string()))
|
||||
.map_err(|_| sse_invalid_argument("The x-amz-server-side-encryption-customer-key-md5 header must be valid UTF-8.")),
|
||||
}?;
|
||||
|
||||
Ok((algorithm, key, key_md5))
|
||||
}
|
||||
|
||||
/// Extract x-amz-server-side-encryption from request headers.
|
||||
/// Used as fallback when the S3 layer does not populate it in the input struct.
|
||||
///
|
||||
/// Returns an error if the header is present but cannot be parsed as valid UTF-8,
|
||||
/// ensuring malformed headers do not bypass validation.
|
||||
pub fn extract_server_side_encryption_from_headers(headers: &HeaderMap) -> Result<Option<String>, SseError> {
|
||||
match headers.get("x-amz-server-side-encryption") {
|
||||
None => Ok(None),
|
||||
Some(v) => v
|
||||
.to_str()
|
||||
.map(|s| Some(s.to_string()))
|
||||
.map_err(|_| sse_invalid_argument("The x-amz-server-side-encryption header must be valid UTF-8.")),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn extract_ssekms_context_from_headers(headers: &HeaderMap) -> Result<Option<HashMap<String, String>>, SseError> {
|
||||
let Some(v) = headers.get(AMZ_SERVER_SIDE_ENCRYPTION_KMS_CONTEXT) else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
let value = v
|
||||
.to_str()
|
||||
.map_err(|_| sse_invalid_argument("The x-amz-server-side-encryption-context header must be valid UTF-8."))?;
|
||||
let decoded = BASE64_STANDARD.decode_to_vec(value).map_err(|_| {
|
||||
sse_invalid_argument("The x-amz-server-side-encryption-context header must be valid base64-encoded JSON.")
|
||||
})?;
|
||||
|
||||
serde_json::from_slice(&decoded).map(Some).map_err(|_| {
|
||||
sse_invalid_argument("The x-amz-server-side-encryption-context header must be a base64-encoded JSON object.")
|
||||
})
|
||||
}
|
||||
|
||||
/// Rejects a client encryption context that would replace the location binding.
|
||||
///
|
||||
/// The request context binds the data key to `{bucket: bucket/key}` only when the
|
||||
/// client supplied no entry under that name, and only the client context is
|
||||
/// persisted. A client entry keyed by the bucket name therefore travels with the
|
||||
/// object and replaces the binding on every read, so the envelope would no longer
|
||||
/// be tied to the object's location. Only the write side rejects it: objects stored
|
||||
/// with such an entry stay readable because the read side rebuilds the context
|
||||
/// unchanged.
|
||||
pub(crate) fn reject_kms_context_location_override(
|
||||
bucket: &str,
|
||||
ssekms_context: Option<&HashMap<String, String>>,
|
||||
) -> Result<(), SseError> {
|
||||
if ssekms_context.is_some_and(|context| context.contains_key(bucket)) {
|
||||
return Err(sse_invalid_argument(&format!(
|
||||
"The x-amz-server-side-encryption-context header contains the key \"{bucket}\", which equals the bucket name; \
|
||||
that key is reserved for binding the data key to the object's location."
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn validate_sse_headers_for_write(
|
||||
server_side_encryption: Option<&str>,
|
||||
ssekms_key_id: Option<&String>,
|
||||
ssekms_context: Option<&HashMap<String, String>>,
|
||||
sse_customer_algorithm: Option<&String>,
|
||||
sse_customer_key: Option<&String>,
|
||||
sse_customer_key_md5: Option<&String>,
|
||||
require_sse_customer_key: bool,
|
||||
) -> Result<(), SseError> {
|
||||
if let Some(sse) = server_side_encryption {
|
||||
let s = sse;
|
||||
if s != AMZ_ENCRYPTION_AES && s != AMZ_ENCRYPTION_KMS {
|
||||
return Err(sse_invalid_argument(
|
||||
"The SSE algorithm specified is not supported. The valid values are AES256 or aws:kms.",
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
let has_ssec_headers = sse_customer_algorithm.is_some() || sse_customer_key.is_some() || sse_customer_key_md5.is_some();
|
||||
let has_managed_headers = server_side_encryption.is_some() || ssekms_key_id.is_some() || ssekms_context.is_some();
|
||||
|
||||
if has_ssec_headers {
|
||||
if has_managed_headers {
|
||||
return Err(sse_invalid_argument(
|
||||
"The SSE-C and managed server-side encryption headers cannot be used together.",
|
||||
));
|
||||
}
|
||||
|
||||
let has_valid_ssec_headers = if require_sse_customer_key {
|
||||
matches!(
|
||||
(sse_customer_algorithm, sse_customer_key, sse_customer_key_md5),
|
||||
(Some(_), Some(_), Some(_))
|
||||
)
|
||||
} else {
|
||||
matches!((sse_customer_algorithm, sse_customer_key_md5), (Some(_), Some(_)))
|
||||
};
|
||||
|
||||
if !has_valid_ssec_headers {
|
||||
let message = if require_sse_customer_key {
|
||||
"Missing SSE-C parameters. Algorithm, customer key and customer key MD5 are all required."
|
||||
} else {
|
||||
"Missing SSE-C parameters. Algorithm and customer key MD5 are required."
|
||||
};
|
||||
|
||||
return Err(ssec_invalid_request(message));
|
||||
}
|
||||
}
|
||||
|
||||
if ssekms_key_id.is_some() && server_side_encryption.is_none_or(|sse| sse != AMZ_ENCRYPTION_KMS) {
|
||||
return Err(sse_invalid_argument(
|
||||
"The SSE-KMS key ID header can only be used when x-amz-server-side-encryption is set to aws:kms.",
|
||||
));
|
||||
}
|
||||
|
||||
if ssekms_context.is_some() && server_side_encryption.is_none_or(|sse| sse != AMZ_ENCRYPTION_KMS) {
|
||||
return Err(sse_invalid_argument(
|
||||
"The SSE-KMS context header can only be used when x-amz-server-side-encryption is set to aws:kms.",
|
||||
));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn validate_sse_headers_for_read(metadata: &HashMap<String, String>, headers: &HeaderMap) -> Result<(), SseError> {
|
||||
let has_req_ssec = headers.contains_key("x-amz-server-side-encryption-customer-algorithm")
|
||||
|| headers.contains_key("x-amz-server-side-encryption-customer-key")
|
||||
|| headers.contains_key("x-amz-server-side-encryption-customer-key-md5");
|
||||
|
||||
let has_req_sse = headers.contains_key("x-amz-server-side-encryption")
|
||||
|| headers.contains_key("x-amz-server-side-encryption-aws-kms-key-id")
|
||||
|| headers.contains_key("x-amz-server-side-encryption-context");
|
||||
|
||||
let is_object_ssec = stored_ssec_metadata(metadata);
|
||||
let is_object_sse = metadata.contains_key("x-amz-server-side-encryption");
|
||||
|
||||
if is_object_ssec {
|
||||
if has_req_sse {
|
||||
return Err(sse_invalid_argument(
|
||||
"Server-side encryption headers cannot be used with an object encrypted using SSE-C.",
|
||||
));
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if is_object_sse && has_req_ssec {
|
||||
return Err(sse_invalid_argument(
|
||||
"SSE-C headers cannot be used with an object encrypted using server-side managed encryption.",
|
||||
));
|
||||
}
|
||||
|
||||
if has_req_ssec {
|
||||
return Err(ssec_invalid_request(
|
||||
"The object was stored without SSE-C. The correct SSE-C parameters must not be provided.",
|
||||
));
|
||||
}
|
||||
|
||||
if has_req_sse {
|
||||
return Err(sse_invalid_argument(
|
||||
"The object is not encrypted with server-side encryption. Do not provide server-side encryption headers.",
|
||||
));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The public `x-amz-server-side-encryption` value a managed scheme reports.
|
||||
pub(crate) fn managed_sse_public_header(sse_type: SSEType) -> &'static str {
|
||||
match sse_type {
|
||||
SSEType::SseKms => AMZ_ENCRYPTION_KMS,
|
||||
// SSE-C never reaches the managed path; reporting AES256 keeps this
|
||||
// total without inventing a third public value.
|
||||
SSEType::SseS3 | SSEType::SseC => AMZ_ENCRYPTION_AES,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn build_ssec_read_headers(algorithm: Option<&String>, key: Option<&String>, key_md5: Option<&String>) -> HeaderMap {
|
||||
let mut headers = HeaderMap::new();
|
||||
|
||||
if let Some(algorithm) = algorithm
|
||||
&& let Ok(mut value) = HeaderValue::from_str(algorithm.as_str())
|
||||
{
|
||||
value.set_sensitive(true);
|
||||
headers.insert(AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM, value);
|
||||
}
|
||||
|
||||
if let Some(key) = key
|
||||
&& let Ok(mut value) = HeaderValue::from_str(key.as_str())
|
||||
{
|
||||
value.set_sensitive(true);
|
||||
headers.insert(AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY, value);
|
||||
}
|
||||
|
||||
if let Some(key_md5) = key_md5
|
||||
&& let Ok(mut value) = HeaderValue::from_str(key_md5.as_str())
|
||||
{
|
||||
value.set_sensitive(true);
|
||||
headers.insert(AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5, value);
|
||||
}
|
||||
|
||||
headers
|
||||
}
|
||||
|
||||
/// Response headers derived for a decrypted read without unwrapping the data key.
|
||||
#[derive(Debug)]
|
||||
pub struct SseReadResponseHeaders {
|
||||
pub server_side_encryption: String,
|
||||
pub sse_customer_algorithm: Option<String>,
|
||||
pub sse_customer_key_md5: Option<String>,
|
||||
pub ssekms_key_id: Option<String>,
|
||||
}
|
||||
|
||||
fn invalid_sse_response_metadata() -> SseError {
|
||||
SseError {
|
||||
code: S3ErrorCode::InternalError,
|
||||
message: "Persisted object encryption metadata is invalid.".to_string(),
|
||||
source: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn insert_sse_response_header(headers: &mut HeaderMap, name: HeaderName, value: &str) -> Result<(), SseError> {
|
||||
let value = HeaderValue::from_str(value).map_err(|_| invalid_sse_response_metadata())?;
|
||||
headers.insert(name, value);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Project response headers after the caller has completed its existing read
|
||||
/// validation and KMS authorization path.
|
||||
pub fn project_sse_read_response_headers(
|
||||
metadata: &HashMap<String, String>,
|
||||
sse_customer_algorithm: Option<&String>,
|
||||
sse_customer_key_md5: Option<&String>,
|
||||
) -> Result<HeaderMap, SseError> {
|
||||
if metadata.keys().any(|key| {
|
||||
key.eq_ignore_ascii_case(AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY)
|
||||
|| key.eq_ignore_ascii_case(AMZ_SERVER_SIDE_ENCRYPTION_KMS_CONTEXT)
|
||||
}) {
|
||||
return Err(invalid_sse_response_metadata());
|
||||
}
|
||||
|
||||
let metadata = normalize_encryption_metadata_case(metadata).map_err(|_| invalid_sse_response_metadata())?;
|
||||
let public_mode = metadata.get("x-amz-server-side-encryption").map(String::as_str);
|
||||
let customer_algorithm = metadata.get(SSEC_ALGORITHM_HEADER).map(String::as_str);
|
||||
|
||||
let public_mode = match public_mode {
|
||||
Some(AMZ_ENCRYPTION_AES) => Some(SSEType::SseS3),
|
||||
Some(AMZ_ENCRYPTION_KMS) => Some(SSEType::SseKms),
|
||||
Some(_) => return Err(invalid_sse_response_metadata()),
|
||||
None => None,
|
||||
};
|
||||
if customer_algorithm.is_some_and(|algorithm| algorithm != AMZ_ENCRYPTION_AES) {
|
||||
return Err(invalid_sse_response_metadata());
|
||||
}
|
||||
|
||||
let mode = if customer_algorithm.is_some() {
|
||||
if public_mode == Some(SSEType::SseKms) {
|
||||
return Err(invalid_sse_response_metadata());
|
||||
}
|
||||
Some(SSEType::SseC)
|
||||
} else {
|
||||
public_mode
|
||||
};
|
||||
for marker_mode in [
|
||||
metadata
|
||||
.contains_key(MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER)
|
||||
.then_some(SSEType::SseC),
|
||||
metadata
|
||||
.contains_key(MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER)
|
||||
.then_some(SSEType::SseS3),
|
||||
metadata
|
||||
.contains_key(MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER)
|
||||
.then_some(SSEType::SseKms),
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
{
|
||||
if mode != Some(marker_mode) {
|
||||
return Err(invalid_sse_response_metadata());
|
||||
}
|
||||
}
|
||||
let Some(mode) = mode else {
|
||||
if metadata
|
||||
.keys()
|
||||
.any(|key| rustfs_utils::http::is_object_encryption_marker(key))
|
||||
{
|
||||
return Err(invalid_sse_response_metadata());
|
||||
}
|
||||
return Ok(HeaderMap::new());
|
||||
};
|
||||
|
||||
let mut key_id = None;
|
||||
for value in [
|
||||
metadata.get("x-amz-server-side-encryption-aws-kms-key-id"),
|
||||
metadata.get(INTERNAL_ENCRYPTION_KEY_ID_HEADER),
|
||||
metadata.get(MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER),
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
{
|
||||
if key_id.is_some_and(|current| current != value) {
|
||||
return Err(invalid_sse_response_metadata());
|
||||
}
|
||||
key_id = Some(value);
|
||||
}
|
||||
|
||||
let normalized_managed = normalize_managed_metadata(metadata.as_ref(), Some(recode_minio_kms_context));
|
||||
let mut response_headers = HeaderMap::with_capacity(3);
|
||||
match mode {
|
||||
SSEType::SseS3 => {
|
||||
if metadata.contains_key("x-amz-server-side-encryption-aws-kms-key-id") || metadata.contains_key(SSEC_KEY_MD5_HEADER)
|
||||
{
|
||||
return Err(invalid_sse_response_metadata());
|
||||
}
|
||||
response_headers.insert(X_AMZ_SERVER_SIDE_ENCRYPTION, HeaderValue::from_static(AMZ_ENCRYPTION_AES));
|
||||
}
|
||||
SSEType::SseKms => {
|
||||
if metadata.contains_key(SSEC_KEY_MD5_HEADER) {
|
||||
return Err(invalid_sse_response_metadata());
|
||||
}
|
||||
let key_id = key_id
|
||||
.map(String::as_str)
|
||||
.filter(|value| !value.is_empty())
|
||||
.ok_or_else(invalid_sse_response_metadata)?;
|
||||
response_headers.insert(X_AMZ_SERVER_SIDE_ENCRYPTION, HeaderValue::from_static(AMZ_ENCRYPTION_KMS));
|
||||
insert_sse_response_header(&mut response_headers, X_AMZ_SERVER_SIDE_ENCRYPTION_AWS_KMS_KEY_ID, key_id)?;
|
||||
|
||||
if metadata.contains_key(MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER)
|
||||
&& !normalized_managed.contains_key(INTERNAL_ENCRYPTION_CONTEXT_HEADER)
|
||||
{
|
||||
return Err(invalid_sse_response_metadata());
|
||||
}
|
||||
if let Some(context) = normalized_managed.get(INTERNAL_ENCRYPTION_CONTEXT_HEADER) {
|
||||
serde_json::from_str::<HashMap<String, String>>(context).map_err(|_| invalid_sse_response_metadata())?;
|
||||
let encoded = BASE64_STANDARD.encode_to_string(context);
|
||||
insert_sse_response_header(&mut response_headers, X_AMZ_SERVER_SIDE_ENCRYPTION_CONTEXT, &encoded)?;
|
||||
}
|
||||
}
|
||||
SSEType::SseC => {
|
||||
if key_id.is_some() || normalized_managed.contains_key(INTERNAL_ENCRYPTION_CONTEXT_HEADER) {
|
||||
return Err(invalid_sse_response_metadata());
|
||||
}
|
||||
let algorithm = sse_customer_algorithm
|
||||
.map(|value| value.as_ref())
|
||||
.filter(|value| *value == AMZ_ENCRYPTION_AES)
|
||||
.ok_or_else(|| {
|
||||
ssec_invalid_request(
|
||||
"The object was stored using a form of Server Side Encryption. The correct parameters must be provided to retrieve the object.",
|
||||
)
|
||||
})?;
|
||||
let key_md5 = sse_customer_key_md5
|
||||
.map(|value| value.as_ref())
|
||||
.ok_or_else(|| ssec_invalid_request("Missing SSE-C customer key MD5."))?;
|
||||
let stored_md5 = metadata.get(SSEC_KEY_MD5_HEADER).ok_or_else(invalid_sse_response_metadata)?;
|
||||
if stored_md5 != key_md5 {
|
||||
return Err(ssec_invalid_request(
|
||||
"The provided encryption parameters did not match the ones used originally to encrypt the object.",
|
||||
));
|
||||
}
|
||||
insert_sse_response_header(&mut response_headers, X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM, algorithm)?;
|
||||
insert_sse_response_header(&mut response_headers, X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5, key_md5)?;
|
||||
}
|
||||
}
|
||||
Ok(response_headers)
|
||||
}
|
||||
@@ -1,20 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! The schemes S3 defines: SSE-S3, SSE-KMS and SSE-C, as a request names them and a response reports them.
|
||||
|
||||
pub(crate) mod authorization;
|
||||
pub(crate) mod headers;
|
||||
pub(crate) mod scheme;
|
||||
pub(crate) mod ssec;
|
||||
@@ -1,288 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Which SSE scheme and which key a request resolves to: the request first, then the bucket default.
|
||||
|
||||
use crate::core::s3::headers::validate_sse_headers_for_write;
|
||||
use crate::error::SseError;
|
||||
use crate::host::SseHost;
|
||||
use crate::{LOG_COMPONENT_STORAGE, LOG_SUBSYSTEM_SSE};
|
||||
use rustfs_utils::http::headers::{AMZ_ENCRYPTION_AES, AMZ_ENCRYPTION_KMS};
|
||||
use rustfs_utils::http::object_encryption_keys::SSEType;
|
||||
use std::collections::HashMap;
|
||||
use tracing::debug;
|
||||
|
||||
pub const DEFAULT_SSE_ALGORITHM: &str = "AES256";
|
||||
|
||||
pub(crate) const SUPPORT_SSE_ALGORITHMS: &[&str] = &[DEFAULT_SSE_ALGORITHM];
|
||||
|
||||
/// SSE configuration resolved from request and bucket defaults
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct SseConfiguration {
|
||||
/// Effective server-side encryption algorithm (after considering bucket defaults)
|
||||
pub effective_sse: String,
|
||||
/// Effective KMS key ID (after considering bucket defaults)
|
||||
pub effective_kms_key_id: Option<String>,
|
||||
}
|
||||
|
||||
/// Managed SSE resolved from a bucket default encryption rule on a write path.
|
||||
///
|
||||
/// The single mapping shared by every writer: the resolver in this crate, and
|
||||
/// the server's PUT, COPY and extract paths. Unknown algorithms fall back to AES256 rather
|
||||
/// than to `None`. Resolving `None` instead lets a same-name copy under a
|
||||
/// malformed bucket default pass the `copy_changes_encryption` guard and take
|
||||
/// the metadata-only shortcut while this layer still encrypts: fresh DEK
|
||||
/// metadata is committed beside the untouched plaintext blocks and the object
|
||||
/// becomes unreadable. PutBucketEncryption refuses unknown algorithms, so this
|
||||
/// is reachable only through a configuration stored before that check or
|
||||
/// through hand-edited bucket metadata (backlog#1826).
|
||||
pub fn bucket_default_write_sse(sse_algorithm: &str) -> String {
|
||||
match sse_algorithm {
|
||||
"AES256" => String::from(AMZ_ENCRYPTION_AES),
|
||||
"aws:kms" => String::from(AMZ_ENCRYPTION_KMS),
|
||||
_ => String::from(AMZ_ENCRYPTION_AES),
|
||||
}
|
||||
}
|
||||
|
||||
/// Prepare SSE configuration by resolving request parameters with bucket defaults
|
||||
///
|
||||
/// This function:
|
||||
/// 1. Queries bucket default encryption configuration
|
||||
/// 2. Resolves effective encryption (request overrides bucket default)
|
||||
/// 3. Prepares metadata headers for managed SSE
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `bucket` - Bucket name
|
||||
/// * `server_side_encryption` - SSE algorithm from request (SSE-S3 or SSE-KMS)
|
||||
/// * `ssekms_key_id` - KMS key ID from request
|
||||
/// * `sse_customer_algorithm` - SSE-C algorithm from request
|
||||
///
|
||||
/// # Returns
|
||||
/// `SseConfiguration` with resolved encryption parameters and metadata headers
|
||||
pub(crate) async fn prepare_sse_configuration(
|
||||
host: &dyn SseHost,
|
||||
bucket: &str,
|
||||
server_side_encryption: Option<String>,
|
||||
ssekms_key_id: Option<String>,
|
||||
) -> Result<Option<SseConfiguration>, SseError> {
|
||||
if let Some(server_side_encryption) = server_side_encryption.clone()
|
||||
&& server_side_encryption.as_str() == AMZ_ENCRYPTION_AES
|
||||
{
|
||||
return Ok(Some(SseConfiguration {
|
||||
effective_sse: server_side_encryption,
|
||||
effective_kms_key_id: None,
|
||||
}));
|
||||
}
|
||||
|
||||
if let Some(server_side_encryption) = server_side_encryption.clone()
|
||||
&& let Some(ssekms_key_id) = ssekms_key_id
|
||||
{
|
||||
return Ok(Some(SseConfiguration {
|
||||
effective_sse: server_side_encryption,
|
||||
effective_kms_key_id: Some(ssekms_key_id),
|
||||
}));
|
||||
}
|
||||
|
||||
// Get bucket default encryption configuration.
|
||||
let bucket_encryption = host.bucket_encryption(bucket).await;
|
||||
debug!(
|
||||
component = LOG_COMPONENT_STORAGE,
|
||||
subsystem = LOG_SUBSYSTEM_SSE,
|
||||
event = "bucket_sse_config_lookup",
|
||||
bucket = %bucket,
|
||||
found = matches!(bucket_encryption, Ok(Some(_))),
|
||||
"Bucket SSE configuration lookup completed"
|
||||
);
|
||||
|
||||
let Some(bucket_encryption) = bucket_encryption? else {
|
||||
// The bucket has no SSE config. If the user explicitly requested
|
||||
// aws:kms, we must honor that — return the explicit SSE header so
|
||||
// downstream logic can try (and fail if KMS is unavailable).
|
||||
return Ok(server_side_encryption.map(|sse| SseConfiguration {
|
||||
effective_sse: sse,
|
||||
effective_kms_key_id: ssekms_key_id,
|
||||
}));
|
||||
};
|
||||
|
||||
let default_rule = bucket_encryption.default_rule.as_ref();
|
||||
let effective_sse = server_side_encryption.clone().or_else(|| {
|
||||
default_rule.map(|sse| {
|
||||
debug!(
|
||||
component = LOG_COMPONENT_STORAGE,
|
||||
subsystem = LOG_SUBSYSTEM_SSE,
|
||||
event = "bucket_sse_default_applied",
|
||||
bucket = %bucket,
|
||||
algorithm = sse.sse_algorithm.as_str(),
|
||||
has_kms_key_id = sse.kms_master_key_id.is_some(),
|
||||
"Bucket SSE default resolved"
|
||||
);
|
||||
bucket_default_write_sse(&sse.sse_algorithm)
|
||||
})
|
||||
});
|
||||
let Some(effective_sse) = effective_sse else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
debug!(
|
||||
component = LOG_COMPONENT_STORAGE,
|
||||
subsystem = LOG_SUBSYSTEM_SSE,
|
||||
event = "effective_sse_resolved",
|
||||
bucket = %bucket,
|
||||
requested = ?server_side_encryption,
|
||||
effective = ?effective_sse,
|
||||
"Resolved effective SSE configuration"
|
||||
);
|
||||
|
||||
let effective_kms_key_id = resolve_effective_kms_key_id(Some(&effective_sse), ssekms_key_id, || {
|
||||
default_rule.and_then(|sse| sse.kms_master_key_id.clone())
|
||||
});
|
||||
|
||||
Ok(Some(SseConfiguration {
|
||||
effective_sse,
|
||||
effective_kms_key_id,
|
||||
}))
|
||||
}
|
||||
|
||||
pub(crate) fn resolve_effective_kms_key_id<F>(
|
||||
effective_sse: Option<&str>,
|
||||
requested_kms_key_id: Option<String>,
|
||||
bucket_default_kms_key_id: F,
|
||||
) -> Option<String>
|
||||
where
|
||||
F: FnOnce() -> Option<String>,
|
||||
{
|
||||
if effective_sse.is_none_or(|sse| sse != AMZ_ENCRYPTION_KMS) {
|
||||
return requested_kms_key_id;
|
||||
}
|
||||
|
||||
requested_kms_key_id.or_else(bucket_default_kms_key_id)
|
||||
}
|
||||
|
||||
/// The SSE headers of a write request, as the S3 layer received them.
|
||||
pub(crate) struct WriteSchemeHeaders<'a> {
|
||||
pub(crate) server_side_encryption: Option<String>,
|
||||
pub(crate) ssekms_key_id: Option<String>,
|
||||
pub(crate) ssekms_context: Option<&'a HashMap<String, String>>,
|
||||
pub(crate) sse_customer_algorithm: Option<String>,
|
||||
pub(crate) sse_customer_key: Option<String>,
|
||||
pub(crate) sse_customer_key_md5: Option<String>,
|
||||
}
|
||||
|
||||
/// The scheme a write resolved to, in S3's terms.
|
||||
#[derive(Clone)]
|
||||
pub(crate) enum WriteScheme {
|
||||
/// `AES256`: the server chooses the wrapping key and the caller names none.
|
||||
SseS3,
|
||||
/// `aws:kms`, under the key the request or the bucket default named, if either did.
|
||||
SseKms { key_id: Option<String> },
|
||||
/// A key the client supplies; only its MD5 is ever stored.
|
||||
SseC {
|
||||
algorithm: String,
|
||||
key: String,
|
||||
key_md5: String,
|
||||
},
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for WriteScheme {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.write_str(match self {
|
||||
Self::SseS3 => "SseS3",
|
||||
Self::SseKms { .. } => "SseKms(..)",
|
||||
Self::SseC { .. } => "SseC(..)",
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Decide the scheme of a write: validate the headers, then the request first and the
|
||||
/// bucket default second.
|
||||
///
|
||||
/// Every write goes through here, an object and a multipart session alike, so the two
|
||||
/// cannot resolve the same headers differently.
|
||||
pub(crate) async fn resolve_write_scheme(
|
||||
host: &dyn SseHost,
|
||||
bucket: &str,
|
||||
headers: WriteSchemeHeaders<'_>,
|
||||
) -> Result<Option<WriteScheme>, SseError> {
|
||||
validate_sse_headers_for_write(
|
||||
headers.server_side_encryption.as_deref(),
|
||||
headers.ssekms_key_id.as_ref(),
|
||||
headers.ssekms_context,
|
||||
headers.sse_customer_algorithm.as_ref(),
|
||||
headers.sse_customer_key.as_ref(),
|
||||
headers.sse_customer_key_md5.as_ref(),
|
||||
true,
|
||||
)?;
|
||||
|
||||
// Validation admits SSE-C headers only as a complete set, and never beside the
|
||||
// managed ones, so their presence alone decides.
|
||||
if let (Some(algorithm), Some(key), Some(key_md5)) =
|
||||
(headers.sse_customer_algorithm, headers.sse_customer_key, headers.sse_customer_key_md5)
|
||||
{
|
||||
return Ok(Some(WriteScheme::SseC { algorithm, key, key_md5 }));
|
||||
}
|
||||
|
||||
let Some(config) = prepare_sse_configuration(host, bucket, headers.server_side_encryption, headers.ssekms_key_id).await?
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
Ok(match config.effective_sse.as_str() {
|
||||
AMZ_ENCRYPTION_AES => {
|
||||
// Validation admits a key id only beside `aws:kms`, and the bucket default's
|
||||
// key is consulted only for `aws:kms`.
|
||||
debug_assert!(config.effective_kms_key_id.is_none(), "SSE-S3 never names a KMS key");
|
||||
Some(WriteScheme::SseS3)
|
||||
}
|
||||
AMZ_ENCRYPTION_KMS => Some(WriteScheme::SseKms {
|
||||
key_id: config.effective_kms_key_id,
|
||||
}),
|
||||
_ => None,
|
||||
})
|
||||
}
|
||||
|
||||
/// Scheme a managed-SSE header names, defaulting to SSE-S3 the same way the
|
||||
/// encryption and decryption paths do.
|
||||
pub(crate) fn managed_sse_type(server_side_encryption: &str) -> SSEType {
|
||||
match server_side_encryption {
|
||||
AMZ_ENCRYPTION_KMS => SSEType::SseKms,
|
||||
_ => SSEType::SseS3,
|
||||
}
|
||||
}
|
||||
|
||||
/// Key id an SSE-KMS read authorizes and reports when the stored metadata
|
||||
/// carries none (legacy envelopes and MinIO-written objects predate the
|
||||
/// explicit key-id headers).
|
||||
///
|
||||
/// The KMS service's configured default key is the key such envelopes were
|
||||
/// actually wrapped under, so per-key authorization must target it — the bare
|
||||
/// literal `default` would silently escape any policy narrowed to real key
|
||||
/// ids. The literal remains only as the last resort when no service is
|
||||
/// running, matching what the legacy write path recorded implicitly.
|
||||
pub(crate) async fn resolve_stored_kms_key_id(host: &dyn SseHost, stored: Option<String>) -> String {
|
||||
if let Some(key_id) = stored {
|
||||
return key_id;
|
||||
}
|
||||
if let Some(service) = host.encryption_service().await
|
||||
&& let Some(key_id) = service.get_default_key_id()
|
||||
{
|
||||
return key_id.clone();
|
||||
}
|
||||
"default".to_string()
|
||||
}
|
||||
|
||||
/// Check if the server_side_encryption is a managed SSE type (SSE-S3 or SSE-KMS)
|
||||
#[inline]
|
||||
pub(crate) fn is_managed_sse(server_side_encryption: &str) -> bool {
|
||||
matches!(server_side_encryption, "AES256" | "aws:kms")
|
||||
}
|
||||
@@ -1,217 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! SSE-C: validating the customer-provided key and matching it against the stored object.
|
||||
|
||||
use crate::core::keying::metadata::{minio_sealed_ssec_metadata, stored_ssec_metadata};
|
||||
use crate::core::s3::scheme::{DEFAULT_SSE_ALGORITHM, SUPPORT_SSE_ALGORITHMS};
|
||||
use crate::error::SseError;
|
||||
use base64_simd::STANDARD as BASE64_STANDARD;
|
||||
use md5::{Digest as Md5Digest, Md5};
|
||||
use rustfs_s3_types::S3ErrorCode;
|
||||
use std::collections::HashMap;
|
||||
use tracing::error;
|
||||
|
||||
fn md5_bytes(input: impl AsRef<[u8]>) -> [u8; 16] {
|
||||
let mut hasher = Md5::new();
|
||||
hasher.update(input.as_ref());
|
||||
hasher.finalize().into()
|
||||
}
|
||||
|
||||
pub(crate) fn md5_base64(input: impl AsRef<[u8]>) -> String {
|
||||
BASE64_STANDARD.encode_to_string(md5_bytes(input))
|
||||
}
|
||||
|
||||
/// Validated SSE-C parameters
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct ValidatedSsecParams {
|
||||
/// Encryption algorithm (always "AES256" for SSE-C)
|
||||
pub algorithm: String,
|
||||
/// Decoded encryption key bytes (32 bytes for AES-256)
|
||||
pub key_bytes: [u8; 32],
|
||||
/// Base64-encoded MD5 of the key
|
||||
pub key_md5: String,
|
||||
}
|
||||
|
||||
/// SSE-C parameters from client request
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct SsecParams {
|
||||
/// Encryption algorithm
|
||||
pub algorithm: String,
|
||||
/// Base64-encoded encryption key
|
||||
pub key: String,
|
||||
/// Base64-encoded MD5 of the key
|
||||
pub key_md5: String,
|
||||
}
|
||||
|
||||
/// Validate SSE-C parameters from client request
|
||||
///
|
||||
/// Validates:
|
||||
/// 1. Algorithm is "AES256"
|
||||
/// 2. Key is valid Base64 and exactly 32 bytes
|
||||
/// 3. MD5 hash matches the key
|
||||
///
|
||||
/// # Returns
|
||||
/// `ValidatedSsecParams` with decoded key bytes
|
||||
pub(crate) fn validate_ssec_params(params: SsecParams) -> Result<ValidatedSsecParams, SseError> {
|
||||
if !SUPPORT_SSE_ALGORITHMS.contains(¶ms.algorithm.as_str()) {
|
||||
return Err(ssec_invalid_request(&format!(
|
||||
"Unsupported SSE-C algorithm. Only {DEFAULT_SSE_ALGORITHM} is supported."
|
||||
)));
|
||||
}
|
||||
|
||||
let key_bytes = BASE64_STANDARD.decode_to_vec(¶ms.key).map_err(|e| {
|
||||
error!("Failed to decode SSE-C key: {}", e);
|
||||
ssec_invalid_request("Invalid SSE-C key: not valid Base64.")
|
||||
})?;
|
||||
|
||||
if key_bytes.len() != 32 {
|
||||
return Err(ssec_invalid_request(&format!(
|
||||
"SSE-C key must be 32 bytes (256 bits), got {} bytes.",
|
||||
key_bytes.len()
|
||||
)));
|
||||
}
|
||||
|
||||
let computed_md5 = md5_base64(&key_bytes);
|
||||
if computed_md5 != params.key_md5 {
|
||||
return Err(ssec_invalid_request(
|
||||
"The calculated MD5 hash of the key did not match the hash that was provided.",
|
||||
));
|
||||
}
|
||||
|
||||
let key_array: [u8; 32] = key_bytes
|
||||
.try_into()
|
||||
.map_err(|_| ssec_invalid_request("SSE-C key must be exactly 32 bytes."))?;
|
||||
|
||||
Ok(ValidatedSsecParams {
|
||||
algorithm: params.algorithm,
|
||||
key_bytes: key_array,
|
||||
key_md5: params.key_md5,
|
||||
})
|
||||
}
|
||||
|
||||
/// Verify SSE-C key matches the stored metadata.
|
||||
///
|
||||
/// Used during GetObject/HeadObject to ensure the client provided the correct key.
|
||||
/// Returns 400 InvalidRequest on mismatch, consistent with AWS S3 behavior.
|
||||
pub(crate) fn verify_ssec_key_match(
|
||||
provided_md5: &str,
|
||||
stored_md5: Option<&String>,
|
||||
minio_sealed_ssec: bool,
|
||||
) -> Result<(), SseError> {
|
||||
match stored_md5 {
|
||||
Some(stored) if stored == provided_md5 => Ok(()),
|
||||
Some(_) => Err(ssec_invalid_request(
|
||||
"The provided encryption parameters did not match the ones used originally to encrypt the object.",
|
||||
)),
|
||||
// MinIO never persists the customer-key MD5, so this early check has
|
||||
// nothing to compare against on a migrated object. The key itself is
|
||||
// still proven: a wrong key fails the AEAD unseal of the sealed object
|
||||
// key. The check exists only for the friendlier early error, so its
|
||||
// absence is tolerated for exactly that shape and no other.
|
||||
None if minio_sealed_ssec => Ok(()),
|
||||
None => Err(ssec_invalid_request("Object has no stored SSE-C key metadata.")),
|
||||
}
|
||||
}
|
||||
|
||||
/// Algorithm an SSE-C object recorded; objects that recorded none were written as `AES256`.
|
||||
pub(crate) fn stored_ssec_algorithm(metadata: &HashMap<String, String>) -> &str {
|
||||
metadata
|
||||
.get("x-amz-server-side-encryption-customer-algorithm")
|
||||
.map(|s| s.as_str())
|
||||
.unwrap_or("AES256")
|
||||
}
|
||||
|
||||
/// The customer key and MD5 of a read of an SSE-C object, once both are present and
|
||||
/// the MD5 matches what the object recorded.
|
||||
///
|
||||
/// The key itself is not validated here; the caller does that next, so a request
|
||||
/// that names the wrong key learns of the mismatch before anything else.
|
||||
pub(crate) fn require_matching_ssec_key<'r>(
|
||||
metadata: &HashMap<String, String>,
|
||||
sse_customer_key: Option<&'r String>,
|
||||
sse_customer_key_md5: Option<&'r String>,
|
||||
) -> Result<(&'r String, &'r String), SseError> {
|
||||
let (key, key_md5) = match (sse_customer_key, sse_customer_key_md5) {
|
||||
(Some(k), Some(md5)) => (k, md5),
|
||||
_ => {
|
||||
return Err(ssec_invalid_request(
|
||||
"The object was stored using a form of Server Side Encryption. \
|
||||
The correct parameters must be provided to retrieve the object.",
|
||||
));
|
||||
}
|
||||
};
|
||||
|
||||
// Verify that the provided key MD5 matches the stored MD5 for security
|
||||
let stored_md5 = metadata.get("x-amz-server-side-encryption-customer-key-md5");
|
||||
verify_ssec_key_match(key_md5, stored_md5, minio_sealed_ssec_metadata(metadata))?;
|
||||
|
||||
Ok((key, key_md5))
|
||||
}
|
||||
|
||||
/// Validate that the SSE-C headers required for reading an SSE-C encrypted object
|
||||
/// are present in the request. This is used by HeadObject which does not decrypt
|
||||
/// the data but still must verify the caller holds the correct key.
|
||||
///
|
||||
/// Performs full validation: decodes the customer key, recomputes its MD5,
|
||||
/// verifies the client-provided MD5 header matches the key, then compares
|
||||
/// the computed MD5 against the stored metadata. This prevents a client from
|
||||
/// bypassing validation by guessing/obtaining only the stored MD5 without
|
||||
/// possessing the actual encryption key.
|
||||
///
|
||||
/// Returns `Ok(())` if either the object is not SSE-C encrypted, or valid SSE-C
|
||||
/// headers are provided and the key matches. Returns 400 InvalidRequest otherwise.
|
||||
pub fn validate_ssec_for_read(
|
||||
metadata: &HashMap<String, String>,
|
||||
sse_customer_key: Option<&String>,
|
||||
sse_customer_key_md5: Option<&String>,
|
||||
) -> Result<(), SseError> {
|
||||
if !stored_ssec_metadata(metadata) {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let (key, key_md5) = match (sse_customer_key, sse_customer_key_md5) {
|
||||
(Some(k), Some(md5)) => (k, md5),
|
||||
_ => {
|
||||
return Err(ssec_invalid_request(
|
||||
"The object was stored using a form of Server Side Encryption. \
|
||||
The correct parameters must be provided to retrieve the object.",
|
||||
));
|
||||
}
|
||||
};
|
||||
|
||||
// Full param validation: decode key, verify 32 bytes, recompute MD5
|
||||
// from actual key bytes and compare to the client-provided MD5 header.
|
||||
let algorithm = metadata
|
||||
.get("x-amz-server-side-encryption-customer-algorithm")
|
||||
.cloned()
|
||||
.unwrap_or_else(|| DEFAULT_SSE_ALGORITHM.to_string());
|
||||
let validated = validate_ssec_params(SsecParams {
|
||||
algorithm,
|
||||
key: key.to_string(),
|
||||
key_md5: key_md5.clone(),
|
||||
})?;
|
||||
|
||||
let stored_md5 = metadata.get("x-amz-server-side-encryption-customer-key-md5");
|
||||
verify_ssec_key_match(&validated.key_md5, stored_md5, minio_sealed_ssec_metadata(metadata))
|
||||
}
|
||||
|
||||
/// Build an `SseError` with `InvalidRequest` (HTTP 400) for SSE-C related errors.
|
||||
pub fn ssec_invalid_request(message: &str) -> SseError {
|
||||
SseError {
|
||||
code: S3ErrorCode::InvalidRequest,
|
||||
message: message.to_string(),
|
||||
source: None,
|
||||
}
|
||||
}
|
||||
@@ -1,550 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Write entry points: resolve the encryption material for a new object or multipart upload.
|
||||
|
||||
use crate::core::audit::{envelope_master_key_version, record_managed_kms_outcome};
|
||||
use crate::core::keying::context::build_object_encryption_context;
|
||||
use crate::core::keying::material::EncryptionMaterial;
|
||||
use crate::core::keying::{
|
||||
SESSION_KEY_NEEDS_CUSTOMER_KEY, new_key_under_customer_key, new_key_under_data_key, new_session_key_under_customer_key,
|
||||
};
|
||||
use crate::core::request::SseKmsPrincipal;
|
||||
use crate::core::s3::authorization::authorize_sse_kms_key;
|
||||
use crate::core::s3::headers::reject_kms_context_location_override;
|
||||
use crate::core::s3::scheme::{
|
||||
DEFAULT_SSE_ALGORITHM, WriteScheme, WriteSchemeHeaders, is_managed_sse, managed_sse_type, resolve_write_scheme,
|
||||
};
|
||||
use crate::core::s3::ssec::{SsecParams, ssec_invalid_request, validate_ssec_params, verify_ssec_key_match};
|
||||
use crate::error::SseError;
|
||||
use crate::host::SseHost;
|
||||
use crate::kms::provider::get_sse_dek_provider;
|
||||
use base64_simd::STANDARD as BASE64_STANDARD;
|
||||
use rustfs_kms::KmsUnavailableError;
|
||||
use rustfs_policy::policy::action::KmsAction;
|
||||
use rustfs_s3_types::S3ErrorCode;
|
||||
use rustfs_utils::http::headers::{AMZ_ENCRYPTION_AES, AMZ_ENCRYPTION_KMS};
|
||||
use rustfs_utils::http::object_encryption_keys::SSEType;
|
||||
use std::collections::HashMap;
|
||||
|
||||
/// Request parameters for unified encryption
|
||||
#[derive(Clone)]
|
||||
pub struct EncryptionRequest<'a> {
|
||||
/// Bucket name
|
||||
pub bucket: &'a str,
|
||||
/// Object key
|
||||
pub key: &'a str,
|
||||
/// Server-side encryption algorithm (SSE-S3 or SSE-KMS)
|
||||
pub server_side_encryption: Option<String>,
|
||||
/// KMS key ID (for SSE-KMS)
|
||||
pub ssekms_key_id: Option<String>,
|
||||
/// Optional client-provided KMS context for SSE-KMS.
|
||||
pub ssekms_context: Option<HashMap<String, String>>,
|
||||
/// SSE-C algorithm (customer-provided key)
|
||||
pub sse_customer_algorithm: Option<String>,
|
||||
/// SSE-C key (Base64-encoded)
|
||||
pub sse_customer_key: Option<String>,
|
||||
/// SSE-C key MD5 (Base64-encoded)
|
||||
pub sse_customer_key_md5: Option<String>,
|
||||
/// Content size (for metadata)
|
||||
pub content_size: i64,
|
||||
/// Caller the SSE-KMS key usage is authorized as. `None` marks an internal caller.
|
||||
pub principal: Option<&'a SseKmsPrincipal>,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for EncryptionRequest<'_> {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("EncryptionRequest")
|
||||
.field("server_side_encryption", &self.server_side_encryption)
|
||||
.field("content_size", &self.content_size)
|
||||
.finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
|
||||
impl EncryptionRequest<'_> {
|
||||
pub fn validate_complete_multipart_ssec(&self, user_defined: &HashMap<String, String>) -> Result<(), SseError> {
|
||||
let request_uses_ssec =
|
||||
self.sse_customer_algorithm.is_some() || self.sse_customer_key.is_some() || self.sse_customer_key_md5.is_some();
|
||||
if !request_uses_ssec {
|
||||
let stored_algorithm = user_defined.get("x-amz-server-side-encryption-customer-algorithm");
|
||||
let stored_key_md5 = user_defined.get("x-amz-server-side-encryption-customer-key-md5");
|
||||
return match (stored_algorithm, stored_key_md5) {
|
||||
(None, None) => Ok(()),
|
||||
(Some(algorithm), Some(key_md5))
|
||||
if algorithm == DEFAULT_SSE_ALGORITHM
|
||||
&& BASE64_STANDARD
|
||||
.decode_to_vec(key_md5)
|
||||
.is_ok_and(|decoded| decoded.len() == 16) =>
|
||||
{
|
||||
if rustfs_utils::http::contains_key_str(user_defined, rustfs_utils::http::SUFFIX_PLAINTEXT_CHECKSUM) {
|
||||
return Err(ssec_invalid_request(
|
||||
"SSE-C multipart uploads with additional checksums require the customer key for completion.",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
_ => Err(ssec_invalid_request("The multipart upload contains invalid SSE-C metadata.")),
|
||||
};
|
||||
}
|
||||
self.validate_multipart_ssec(user_defined)
|
||||
}
|
||||
|
||||
pub fn validate_multipart_ssec(&self, user_defined: &HashMap<String, String>) -> Result<(), SseError> {
|
||||
let stored_algorithm = user_defined.get("x-amz-server-side-encryption-customer-algorithm");
|
||||
let stored_key_md5 = user_defined.get("x-amz-server-side-encryption-customer-key-md5");
|
||||
let session_uses_ssec = stored_algorithm.is_some() || stored_key_md5.is_some();
|
||||
let request_uses_ssec =
|
||||
self.sse_customer_algorithm.is_some() || self.sse_customer_key.is_some() || self.sse_customer_key_md5.is_some();
|
||||
|
||||
if !session_uses_ssec {
|
||||
return if request_uses_ssec {
|
||||
Err(ssec_invalid_request(
|
||||
"SSE-C parameters cannot be used for a multipart upload that was not initiated with SSE-C.",
|
||||
))
|
||||
} else {
|
||||
Ok(())
|
||||
};
|
||||
}
|
||||
|
||||
let (Some(algorithm), Some(key), Some(key_md5)) = (
|
||||
self.sse_customer_algorithm.as_ref(),
|
||||
self.sse_customer_key.as_ref(),
|
||||
self.sse_customer_key_md5.as_ref(),
|
||||
) else {
|
||||
return Err(ssec_invalid_request(
|
||||
"Missing SSE-C parameters. Algorithm, customer key and customer key MD5 are all required.",
|
||||
));
|
||||
};
|
||||
|
||||
let validated = validate_ssec_params(SsecParams {
|
||||
algorithm: algorithm.to_string(),
|
||||
key: key.to_string(),
|
||||
key_md5: key_md5.to_string(),
|
||||
})?;
|
||||
if stored_algorithm.map(String::as_str) != Some(validated.algorithm.as_str()) {
|
||||
return Err(ssec_invalid_request(
|
||||
"The provided encryption parameters did not match the multipart upload.",
|
||||
));
|
||||
}
|
||||
// Multipart session metadata is written by our own CreateMultipartUpload,
|
||||
// which always records the MD5 — the MinIO no-stored-MD5 shape cannot occur.
|
||||
verify_ssec_key_match(&validated.key_md5, stored_key_md5, false)
|
||||
}
|
||||
}
|
||||
|
||||
/// **Core API**: Apply encryption based on request parameters
|
||||
///
|
||||
/// This function automatically routes to the appropriate encryption method:
|
||||
/// - SSE-C if customer key is provided
|
||||
/// - SSE-S3/SSE-KMS if server-side encryption is requested
|
||||
/// - None if no encryption is requested
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `request` - Encryption request with all possible encryption parameters
|
||||
///
|
||||
/// # Returns
|
||||
/// * `Ok(Some(material))` - Encryption should be applied with the returned material
|
||||
/// * `Ok(None)` - No encryption requested
|
||||
/// * `Err` - Encryption configuration error
|
||||
///
|
||||
/// # Example
|
||||
/// ```rust,ignore
|
||||
/// let request = EncryptionRequest {
|
||||
/// bucket: &bucket,
|
||||
/// key: &key,
|
||||
/// server_side_encryption: effective_sse.as_ref(),
|
||||
/// ssekms_key_id: effective_kms_key_id.as_deref(),
|
||||
/// sse_customer_algorithm: sse_customer_algorithm.as_ref(),
|
||||
/// sse_customer_key: sse_customer_key.as_deref(),
|
||||
/// sse_customer_key_md5: sse_customer_key_md5.as_deref(),
|
||||
/// content_size: actual_size,
|
||||
/// part_number: None,
|
||||
/// };
|
||||
///
|
||||
/// if let Some(material) = sse_encryption(host, request).await? {
|
||||
/// reader = material.wrap_reader(reader);
|
||||
/// metadata.extend(material.metadata);
|
||||
/// }
|
||||
/// ```
|
||||
pub async fn sse_encryption(host: &dyn SseHost, request: EncryptionRequest<'_>) -> Result<Option<EncryptionMaterial>, SseError> {
|
||||
let scheme = resolve_write_scheme(
|
||||
host,
|
||||
request.bucket,
|
||||
WriteSchemeHeaders {
|
||||
server_side_encryption: request.server_side_encryption,
|
||||
ssekms_key_id: request.ssekms_key_id,
|
||||
ssekms_context: request.ssekms_context.as_ref(),
|
||||
sse_customer_algorithm: request.sse_customer_algorithm,
|
||||
sse_customer_key: request.sse_customer_key,
|
||||
sse_customer_key_md5: request.sse_customer_key_md5,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
|
||||
let (server_side_encryption, kms_key_id) = match scheme {
|
||||
None => return Ok(None),
|
||||
Some(WriteScheme::SseC { algorithm, key, key_md5 }) => {
|
||||
return apply_ssec_encryption_material(request.bucket, request.key, algorithm, key, key_md5, request.content_size)
|
||||
.await
|
||||
.map(Some);
|
||||
}
|
||||
Some(WriteScheme::SseS3) => (String::from(AMZ_ENCRYPTION_AES), None),
|
||||
Some(WriteScheme::SseKms { key_id }) => (String::from(AMZ_ENCRYPTION_KMS), key_id),
|
||||
};
|
||||
|
||||
apply_managed_encryption_material(
|
||||
host,
|
||||
ManagedEncryptionRequest {
|
||||
bucket: request.bucket,
|
||||
key: request.key,
|
||||
server_side_encryption,
|
||||
kms_key_id,
|
||||
ssekms_context: request.ssekms_context,
|
||||
content_size: request.content_size,
|
||||
principal: request.principal,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map(Some)
|
||||
}
|
||||
|
||||
/// **Core API**: Apply encryption based on request parameters
|
||||
///
|
||||
/// sse_prepare_encryption, support SSE-C, SSE-S3, SSE-KMS
|
||||
pub struct PrepareEncryptionRequest<'a> {
|
||||
/// Bucket name
|
||||
pub bucket: &'a str,
|
||||
/// Object key
|
||||
pub key: &'a str,
|
||||
/// Server-side encryption algorithm (SSE-S3 or SSE-KMS)
|
||||
pub server_side_encryption: Option<String>,
|
||||
/// KMS key ID (for SSE-KMS)
|
||||
pub ssekms_key_id: Option<String>,
|
||||
/// Optional client-provided KMS context for SSE-KMS.
|
||||
pub ssekms_context: Option<HashMap<String, String>>,
|
||||
/// SSE-C algorithm (customer-provided key)
|
||||
pub sse_customer_algorithm: Option<String>,
|
||||
/// SSE-C key (Base64-encoded)
|
||||
pub sse_customer_key: Option<String>,
|
||||
/// SSE-C key MD5 (Base64-encoded)
|
||||
pub sse_customer_key_md5: Option<String>,
|
||||
/// Caller the SSE-KMS key usage is authorized as. `None` marks an internal caller.
|
||||
///
|
||||
/// This is the only point at which a multipart upload evaluates KMS key permissions:
|
||||
/// it is where the session data key is generated. `UploadPart`, `UploadPartCopy` and
|
||||
/// `CompleteMultipartUpload` reuse that envelope and are not re-authorized.
|
||||
pub principal: Option<&'a SseKmsPrincipal>,
|
||||
}
|
||||
|
||||
pub async fn sse_prepare_encryption(
|
||||
host: &dyn SseHost,
|
||||
request: PrepareEncryptionRequest<'_>,
|
||||
) -> Result<Option<EncryptionMaterial>, SseError> {
|
||||
let scheme = resolve_write_scheme(
|
||||
host,
|
||||
request.bucket,
|
||||
WriteSchemeHeaders {
|
||||
server_side_encryption: request.server_side_encryption,
|
||||
ssekms_key_id: request.ssekms_key_id,
|
||||
ssekms_context: request.ssekms_context.as_ref(),
|
||||
sse_customer_algorithm: request.sse_customer_algorithm,
|
||||
sse_customer_key: request.sse_customer_key,
|
||||
sse_customer_key_md5: request.sse_customer_key_md5,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
|
||||
let (server_side_encryption, kms_key_id) = match scheme {
|
||||
None => return Ok(None),
|
||||
Some(WriteScheme::SseC { algorithm, key, key_md5 }) => {
|
||||
return apply_ssec_prepare_encryption_material(request.bucket, request.key, algorithm, Some(key), key_md5)
|
||||
.await
|
||||
.map(Some);
|
||||
}
|
||||
Some(WriteScheme::SseS3) => (String::from(AMZ_ENCRYPTION_AES), None),
|
||||
Some(WriteScheme::SseKms { key_id }) => (String::from(AMZ_ENCRYPTION_KMS), key_id),
|
||||
};
|
||||
|
||||
// A session records no plaintext size: the parts decide it.
|
||||
apply_managed_encryption_material(
|
||||
host,
|
||||
ManagedEncryptionRequest {
|
||||
bucket: request.bucket,
|
||||
key: request.key,
|
||||
server_side_encryption,
|
||||
kms_key_id,
|
||||
ssekms_context: request.ssekms_context,
|
||||
content_size: 0,
|
||||
principal: request.principal,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map(Some)
|
||||
}
|
||||
|
||||
async fn apply_ssec_prepare_encryption_material(
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
algorithm: String,
|
||||
sse_key: Option<String>,
|
||||
sse_key_md5: String,
|
||||
) -> Result<EncryptionMaterial, SseError> {
|
||||
// The customer key is validated here only in a mode that seals the session's
|
||||
// key under it. Otherwise it is validated per part, against what the session
|
||||
// records.
|
||||
let customer_key = match sse_key {
|
||||
Some(sse_key) if SESSION_KEY_NEEDS_CUSTOMER_KEY => Some(
|
||||
validate_ssec_params(SsecParams {
|
||||
algorithm: algorithm.clone(),
|
||||
key: sse_key,
|
||||
key_md5: sse_key_md5.clone(),
|
||||
})?
|
||||
.key_bytes,
|
||||
),
|
||||
_ => None,
|
||||
};
|
||||
let object_key = new_session_key_under_customer_key(customer_key, bucket, key)?;
|
||||
|
||||
Ok(EncryptionMaterial {
|
||||
sse_type: SSEType::SseC,
|
||||
server_side_encryption: String::from(AMZ_ENCRYPTION_AES),
|
||||
kms_key_id: None,
|
||||
algorithm,
|
||||
key_bytes: object_key.key_bytes,
|
||||
base_nonce: object_key.base_nonce,
|
||||
encrypted_data_key: None,
|
||||
customer_key_md5: Some(sse_key_md5),
|
||||
original_size: None,
|
||||
key_kind: object_key.key_kind,
|
||||
managed_kms_context: None,
|
||||
managed_sealed_key: object_key.sealed_key,
|
||||
})
|
||||
}
|
||||
|
||||
async fn apply_ssec_encryption_material(
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
algorithm: String,
|
||||
sse_key: String,
|
||||
sse_key_md5: String,
|
||||
content_size: i64,
|
||||
) -> Result<EncryptionMaterial, SseError> {
|
||||
let params = SsecParams {
|
||||
algorithm,
|
||||
key: sse_key,
|
||||
key_md5: sse_key_md5,
|
||||
};
|
||||
|
||||
let validated = validate_ssec_params(params)?;
|
||||
let object_key = new_key_under_customer_key(validated.key_bytes, bucket, key)?;
|
||||
|
||||
// Build metadata
|
||||
Ok(EncryptionMaterial {
|
||||
sse_type: SSEType::SseC,
|
||||
server_side_encryption: String::from(AMZ_ENCRYPTION_AES),
|
||||
kms_key_id: None,
|
||||
algorithm: validated.algorithm,
|
||||
key_bytes: object_key.key_bytes,
|
||||
base_nonce: object_key.base_nonce,
|
||||
encrypted_data_key: None,
|
||||
customer_key_md5: Some(validated.key_md5),
|
||||
original_size: Some(content_size),
|
||||
key_kind: object_key.key_kind,
|
||||
managed_kms_context: None,
|
||||
managed_sealed_key: object_key.sealed_key,
|
||||
})
|
||||
}
|
||||
|
||||
/// A managed-SSE (SSE-S3 / SSE-KMS) write, once its effective scheme is known.
|
||||
pub(crate) struct ManagedEncryptionRequest<'a> {
|
||||
pub(crate) bucket: &'a str,
|
||||
pub(crate) key: &'a str,
|
||||
pub(crate) server_side_encryption: String,
|
||||
pub(crate) kms_key_id: Option<String>,
|
||||
pub(crate) ssekms_context: Option<HashMap<String, String>>,
|
||||
pub(crate) content_size: i64,
|
||||
pub(crate) principal: Option<&'a SseKmsPrincipal>,
|
||||
}
|
||||
|
||||
pub(crate) async fn apply_managed_encryption_material(
|
||||
host: &dyn SseHost,
|
||||
request: ManagedEncryptionRequest<'_>,
|
||||
) -> Result<EncryptionMaterial, SseError> {
|
||||
// Request validation, not a KMS operation: reject before the KMS is called
|
||||
// and before an audit outcome is recorded.
|
||||
reject_kms_context_location_override(request.bucket, request.ssekms_context.as_ref())?;
|
||||
|
||||
let principal = request.principal;
|
||||
let requested_sse_type = managed_sse_type(request.server_side_encryption.as_str());
|
||||
let requested_key_id = request.kms_key_id.clone();
|
||||
let result = apply_managed_encryption_material_inner(host, request).await;
|
||||
|
||||
match &result {
|
||||
// The resolved key is only known on success: it may come from the request,
|
||||
// the bucket default or the KMS service default. On failure the audit entry
|
||||
// records what the caller asked for, which is what a reader needs to see.
|
||||
Ok(material) => record_managed_kms_outcome(
|
||||
principal,
|
||||
material.sse_type,
|
||||
material.kms_key_id.as_deref(),
|
||||
|| material.encrypted_data_key.as_deref().and_then(envelope_master_key_version),
|
||||
Ok(()),
|
||||
),
|
||||
Err(error) => record_managed_kms_outcome(principal, requested_sse_type, requested_key_id.as_deref(), || None, Err(error)),
|
||||
}
|
||||
|
||||
result
|
||||
}
|
||||
|
||||
pub(crate) async fn apply_managed_encryption_material_inner(
|
||||
host: &dyn SseHost,
|
||||
request: ManagedEncryptionRequest<'_>,
|
||||
) -> Result<EncryptionMaterial, SseError> {
|
||||
let ManagedEncryptionRequest {
|
||||
bucket,
|
||||
key,
|
||||
server_side_encryption,
|
||||
kms_key_id,
|
||||
ssekms_context,
|
||||
content_size,
|
||||
principal,
|
||||
} = request;
|
||||
if !is_managed_sse(&server_side_encryption) {
|
||||
return Err(SseError::internal(format!(
|
||||
"Unsupported server-side encryption: {}",
|
||||
server_side_encryption.as_str()
|
||||
)));
|
||||
}
|
||||
|
||||
let encryption_type = match server_side_encryption.as_str() {
|
||||
"AES256" => SSEType::SseS3,
|
||||
"aws:kms" => SSEType::SseKms,
|
||||
_ => SSEType::SseS3,
|
||||
};
|
||||
|
||||
// Determine KMS key ID to use for internal key wrapping.
|
||||
let mut kms_key_candidate = kms_key_id.clone();
|
||||
if kms_key_candidate.is_none() {
|
||||
// Try to get default key from KMS service (if available)
|
||||
if let Some(service) = host.encryption_service().await {
|
||||
kms_key_candidate = service.get_default_key_id().cloned();
|
||||
tracing::debug!(
|
||||
default_key_id = ?kms_key_candidate,
|
||||
"SSE-S3: KMS service available, default_key_id from config"
|
||||
);
|
||||
} else {
|
||||
tracing::debug!("SSE-S3: KMS encryption service not available");
|
||||
}
|
||||
}
|
||||
|
||||
let kms_key_to_use = match (encryption_type, kms_key_candidate.clone()) {
|
||||
(SSEType::SseS3, Some(kms_key_id)) => kms_key_id,
|
||||
(SSEType::SseS3, None) => {
|
||||
tracing::debug!("SSE-S3: no KMS key configured, falling back to \"default\" key ID");
|
||||
"default".to_string()
|
||||
}
|
||||
(SSEType::SseKms, Some(kms_key_id)) => kms_key_id,
|
||||
(SSEType::SseKms, None) => {
|
||||
// Neither the request nor the bucket default named a key and no
|
||||
// service default filled in. Without a service this is the same
|
||||
// outage/misconfiguration the provider check below reports, so it
|
||||
// must carry the same 503/400 split rather than an untyped
|
||||
// internal error; with a running service that has no default key
|
||||
// the caller simply has to name one.
|
||||
if host.encryption_service().await.is_none() {
|
||||
return Err(sse_kms_unavailable_error(kms_configured_but_unavailable(host).await));
|
||||
}
|
||||
return Err(SseError {
|
||||
code: S3ErrorCode::InvalidRequest,
|
||||
message: "SSE-KMS requires a KMS key id: the request named none and the KMS service has no default key"
|
||||
.to_string(),
|
||||
source: None,
|
||||
});
|
||||
}
|
||||
_ => unreachable!("managed SSE branch only supports SSE-S3 or SSE-KMS"),
|
||||
};
|
||||
|
||||
// The key is fully resolved here (request header, then bucket default, then the KMS
|
||||
// service default), so this is the first point at which the caller can be held to the
|
||||
// key it will actually be encrypted under.
|
||||
authorize_sse_kms_key(host, principal, encryption_type, KmsAction::GenerateDataKeyAction, &kms_key_to_use).await?;
|
||||
|
||||
// A node-local master key is the explicit SSE-S3 fallback. Letting it serve
|
||||
// an SSE-KMS request would persist an `aws:kms` marker and a KMS key id
|
||||
// that never wrapped the data key, so refuse rather than downgrade.
|
||||
//
|
||||
// The refusal sits after the authorization gate so an unauthorized caller
|
||||
// still sees AccessDenied whatever the KMS runtime state is, and it asks
|
||||
// the resolved provider rather than a parallel availability signal,
|
||||
// because the provider is what actually wraps the DEK.
|
||||
let provider = match get_sse_dek_provider(host).await {
|
||||
Ok(provider) => {
|
||||
if matches!(encryption_type, SSEType::SseKms) && provider.wraps_dek_with_local_master_key() {
|
||||
return Err(sse_kms_unavailable_error(kms_configured_but_unavailable(host).await));
|
||||
}
|
||||
provider
|
||||
}
|
||||
// With no master key set the local fallback fails with an SSE-S3-worded
|
||||
// configuration error. An SSE-KMS request never asked for that provider,
|
||||
// so it gets the SSE-KMS refusal instead of a message naming the wrong
|
||||
// scheme.
|
||||
Err(err) => {
|
||||
if matches!(encryption_type, SSEType::SseKms) && host.encryption_service().await.is_none() {
|
||||
return Err(sse_kms_unavailable_error(kms_configured_but_unavailable(host).await));
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
};
|
||||
let object_context = build_object_encryption_context(bucket, key, ssekms_context.as_ref());
|
||||
let (data_key, encrypted_data_key) = provider.generate_sse_dek(&object_context, &kms_key_to_use).await?;
|
||||
|
||||
let algorithm = server_side_encryption.as_str().to_string();
|
||||
let object_key = new_key_under_data_key(&data_key, encryption_type, bucket, key)?;
|
||||
|
||||
Ok(EncryptionMaterial {
|
||||
sse_type: encryption_type,
|
||||
server_side_encryption,
|
||||
kms_key_id: Some(kms_key_to_use),
|
||||
algorithm,
|
||||
key_bytes: object_key.key_bytes,
|
||||
base_nonce: object_key.base_nonce,
|
||||
encrypted_data_key: Some(encrypted_data_key),
|
||||
customer_key_md5: None,
|
||||
original_size: Some(content_size),
|
||||
key_kind: object_key.key_kind,
|
||||
managed_kms_context: matches!(encryption_type, SSEType::SseKms).then_some(ssekms_context.unwrap_or_default()),
|
||||
managed_sealed_key: object_key.sealed_key,
|
||||
})
|
||||
}
|
||||
|
||||
/// Whether this node has a KMS configured that is not currently serving, which
|
||||
/// separates a transient outage (retryable, 503) from a deployment that never
|
||||
/// configured KMS at all (a client-side configuration error, 400).
|
||||
async fn kms_configured_but_unavailable(host: &dyn SseHost) -> bool {
|
||||
match host.kms_service_manager() {
|
||||
Some(manager) => !matches!(manager.get_status().await, rustfs_kms::KmsServiceStatus::NotConfigured),
|
||||
None => false,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn sse_kms_unavailable_error(configured_but_unavailable: bool) -> SseError {
|
||||
if configured_but_unavailable {
|
||||
return SseError::internal(KmsUnavailableError);
|
||||
}
|
||||
SseError {
|
||||
code: S3ErrorCode::InvalidRequest,
|
||||
message: "SSE-KMS requires a configured and running KMS service".to_string(),
|
||||
source: None,
|
||||
}
|
||||
}
|
||||
@@ -1,176 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! The error every SSE operation reports, already classified for the S3 data path.
|
||||
|
||||
use rustfs_kms::{KmsError, KmsUnavailableError};
|
||||
use rustfs_s3_types::S3ErrorCode;
|
||||
|
||||
type BoxError = Box<dyn std::error::Error + Send + Sync>;
|
||||
|
||||
/// S3 error code a KMS key that does not exist is reported under.
|
||||
pub const KMS_KEY_NOT_FOUND_ERROR_CODE: &str = "KMS.NotFoundException";
|
||||
|
||||
pub(crate) const INTERNAL_ERROR_MESSAGE: &str = "We encountered an internal error, please try again.";
|
||||
const SERVICE_UNAVAILABLE_MESSAGE: &str = "The service is unavailable. Please retry.";
|
||||
const ACCESS_DENIED_MESSAGE: &str = "Access Denied.";
|
||||
|
||||
/// A failed SSE operation, carrying the S3 error code and message the caller answers with.
|
||||
///
|
||||
/// `source` keeps the underlying failure for logs and for classification by
|
||||
/// callers; it is never part of the response.
|
||||
#[derive(Debug)]
|
||||
pub struct SseError {
|
||||
pub code: S3ErrorCode,
|
||||
pub message: String,
|
||||
pub source: Option<BoxError>,
|
||||
}
|
||||
|
||||
impl SseError {
|
||||
/// Classify a failure that did not originate from the request itself.
|
||||
///
|
||||
/// A [`KmsUnavailableError`] is a retryable `ServiceUnavailable`, a
|
||||
/// [`KmsError`] follows [`data_plane_kms_error`], and everything else is an
|
||||
/// `InternalError` whose detail stays in `source` and out of the response.
|
||||
pub fn internal(error: impl Into<BoxError>) -> Self {
|
||||
let source = error.into();
|
||||
let (code, message) = if source.downcast_ref::<KmsUnavailableError>().is_some() {
|
||||
(S3ErrorCode::ServiceUnavailable, SERVICE_UNAVAILABLE_MESSAGE.to_string())
|
||||
} else if let Some(classified) = source.downcast_ref::<KmsError>().and_then(data_plane_kms_error) {
|
||||
classified
|
||||
} else {
|
||||
(S3ErrorCode::InternalError, INTERNAL_ERROR_MESSAGE.to_string())
|
||||
};
|
||||
Self {
|
||||
code,
|
||||
message,
|
||||
source: Some(source),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Display for SseError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.write_str(&self.message)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for SseError {
|
||||
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
|
||||
self.source.as_deref().map(|source| source as _)
|
||||
}
|
||||
}
|
||||
|
||||
/// Map a KMS failure that surfaced on the S3 data path to its S3 error code.
|
||||
///
|
||||
/// The contract deliberately differs from the admin key-lifecycle handlers:
|
||||
/// there a key id is the resource being addressed, so a missing key is `404`.
|
||||
/// Here the key id arrives inside a request header or a bucket default, so a
|
||||
/// key that is missing, disabled, or otherwise unusable is configuration the
|
||||
/// caller has to correct — AWS answers `400`, and reporting `500` instead both
|
||||
/// misfiles the failure as a server fault and makes SDKs back off and retry a
|
||||
/// request that cannot succeed.
|
||||
///
|
||||
/// `None` keeps the caller's fallthrough, which is the `500` that integrity
|
||||
/// faults — damaged, unreadable, or unknown-format key material — must keep.
|
||||
///
|
||||
/// Messages either name what the caller asked for or stay generic; detail that
|
||||
/// belongs to the deployment rather than the request stays in `source`, which
|
||||
/// the caller attaches and the logs retain.
|
||||
pub fn data_plane_kms_error(error: &KmsError) -> Option<(S3ErrorCode, String)> {
|
||||
use KmsError as Kms;
|
||||
|
||||
match error {
|
||||
Kms::KeyNotFound { key_id } => Some((
|
||||
S3ErrorCode::Custom(KMS_KEY_NOT_FOUND_ERROR_CODE.into()),
|
||||
format!("KMS key not found: {key_id}"),
|
||||
)),
|
||||
Kms::KeyVersionNotFound { key_id, version } => Some((
|
||||
S3ErrorCode::Custom(KMS_KEY_NOT_FOUND_ERROR_CODE.into()),
|
||||
format!("KMS key version {version} not found for key {key_id}"),
|
||||
)),
|
||||
// The key exists but its state forbids the operation (disabled,
|
||||
// pending deletion). AWS treats an invalid key state as a request
|
||||
// error, not a server fault.
|
||||
Kms::InvalidOperation { .. } => Some((S3ErrorCode::InvalidRequest, error.to_string())),
|
||||
// A policy decision needs a human, so it must be distinguishable from
|
||||
// the transient classes an SDK retries.
|
||||
Kms::AccessDenied { .. } => Some((S3ErrorCode::AccessDenied, ACCESS_DENIED_MESSAGE.to_string())),
|
||||
// Request-side faults: what was asked for cannot be served as asked.
|
||||
Kms::ContextMismatch { .. }
|
||||
| Kms::InvalidKey { .. }
|
||||
| Kms::ValidationError { .. }
|
||||
| Kms::UnsupportedAlgorithm { .. }
|
||||
| Kms::InvalidKeySize { .. } => Some((S3ErrorCode::InvalidRequest, error.to_string())),
|
||||
// A deployment whose KMS configuration cannot serve the request (for
|
||||
// example no key named and no default). Actionable, but the detail
|
||||
// describes the deployment, so it stays out of the response body.
|
||||
Kms::ConfigurationError { .. } => {
|
||||
Some((S3ErrorCode::InvalidRequest, "The KMS configuration cannot serve this request".to_string()))
|
||||
}
|
||||
// Transient: worth retrying, and must be counted against availability
|
||||
// rather than against the caller. `IoError` belongs here because it is
|
||||
// how a backend reports that its key store itself was unreachable —
|
||||
// rustfs/rustfs#7470 separated that from a missing key precisely so the
|
||||
// two stop looking alike, and leaving it on the 500 fallthrough would
|
||||
// erase that distinction again at the S3 boundary.
|
||||
Kms::BackendError { .. }
|
||||
| Kms::IoError { .. }
|
||||
| Kms::OperationTimedOut { .. }
|
||||
| Kms::OperationCancelled { .. }
|
||||
| Kms::CredentialsUnavailable { .. }
|
||||
| Kms::CacheError { .. } => Some((S3ErrorCode::ServiceUnavailable, SERVICE_UNAVAILABLE_MESSAGE.to_string())),
|
||||
// A permanent gap in the configured backend, never a missing resource.
|
||||
Kms::UnsupportedCapability { .. } => Some((S3ErrorCode::NotImplemented, error.to_string())),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn internal_keeps_request_free_failures_generic() {
|
||||
let error = SseError::internal("Failed to decode IV: bad base64");
|
||||
|
||||
assert_eq!(error.code, S3ErrorCode::InternalError);
|
||||
assert_eq!(error.message, INTERNAL_ERROR_MESSAGE);
|
||||
assert!(
|
||||
error
|
||||
.source
|
||||
.as_ref()
|
||||
.is_some_and(|source| source.to_string().contains("Failed to decode IV"))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn internal_reports_an_unavailable_kms_as_retryable() {
|
||||
let error = SseError::internal(KmsUnavailableError);
|
||||
|
||||
assert_eq!(error.code, S3ErrorCode::ServiceUnavailable);
|
||||
assert_eq!(error.message, SERVICE_UNAVAILABLE_MESSAGE);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn internal_classifies_kms_errors_for_the_data_path() {
|
||||
let missing = SseError::internal(KmsError::key_not_found("absent-key"));
|
||||
assert_eq!(missing.code, S3ErrorCode::Custom(KMS_KEY_NOT_FOUND_ERROR_CODE.into()));
|
||||
assert_eq!(missing.message, "KMS key not found: absent-key");
|
||||
|
||||
let damaged = SseError::internal(KmsError::cryptographic_error("decrypt", "authentication failed"));
|
||||
assert_eq!(damaged.code, S3ErrorCode::InternalError);
|
||||
assert_eq!(damaged.message, INTERNAL_ERROR_MESSAGE);
|
||||
}
|
||||
}
|
||||
@@ -1,85 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! What SSE needs from the server it runs in.
|
||||
//!
|
||||
//! This crate reaches the running server only through [`SseHost`]: it holds no
|
||||
//! handle to the storage engine, to IAM, or to the bucket metadata system. Every
|
||||
//! entry point that consults the server takes the host as its first argument, so
|
||||
//! a caller cannot reach key selection without saying which server it speaks for.
|
||||
|
||||
use crate::{SseError, SseKmsPrincipal};
|
||||
use async_trait::async_trait;
|
||||
use rustfs_kms::KmsServiceManager;
|
||||
use rustfs_kms::service::ObjectEncryptionService;
|
||||
use rustfs_policy::policy::action::KmsAction;
|
||||
use std::sync::Arc;
|
||||
|
||||
/// The server capabilities SSE key resolution depends on.
|
||||
#[async_trait]
|
||||
pub trait SseHost: Send + Sync {
|
||||
/// The KMS object-encryption service currently serving, if one is.
|
||||
///
|
||||
/// Resolved on every call: the KMS backend can be reconfigured while the
|
||||
/// server runs, so a service handle must not outlive the operation it serves.
|
||||
async fn encryption_service(&self) -> Option<Arc<ObjectEncryptionService>>;
|
||||
|
||||
/// The KMS service manager, when the server has one.
|
||||
///
|
||||
/// Only its status is read, to tell a configured KMS that is not serving
|
||||
/// (retryable) from a deployment that never configured one (a request error).
|
||||
fn kms_service_manager(&self) -> Option<Arc<KmsServiceManager>>;
|
||||
|
||||
/// The bucket's default-encryption configuration.
|
||||
///
|
||||
/// `Ok(None)` means the bucket has no encryption configuration.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Any failure to read the configuration, already classified for the S3
|
||||
/// response. A write must not proceed as if the bucket had no default when
|
||||
/// the configuration could not be read.
|
||||
async fn bucket_encryption(&self, bucket: &str) -> Result<Option<BucketEncryption>, SseError>;
|
||||
|
||||
/// The authorizer deciding per-key KMS permissions on the data path.
|
||||
fn kms_key_authorizer(&self) -> &dyn KmsKeyAuthorizer;
|
||||
}
|
||||
|
||||
/// A bucket's stored encryption configuration, reduced to what a write consults.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct BucketEncryption {
|
||||
/// The default-encryption rule of the first configuration rule, if it has one.
|
||||
pub default_rule: Option<BucketDefaultEncryption>,
|
||||
}
|
||||
|
||||
/// The `ApplyServerSideEncryptionByDefault` element of a bucket encryption rule.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct BucketDefaultEncryption {
|
||||
/// The stored algorithm, exactly as persisted (`AES256`, `aws:kms`, or an unknown value).
|
||||
pub sse_algorithm: String,
|
||||
pub kms_master_key_id: Option<String>,
|
||||
}
|
||||
|
||||
/// Decides whether a principal may use a specific KMS key on the data path.
|
||||
///
|
||||
/// Behind a trait so tests can pin the allow/deny contract of the SSE call sites
|
||||
/// without a running IAM subsystem.
|
||||
#[async_trait]
|
||||
pub trait KmsKeyAuthorizer: Send + Sync {
|
||||
/// # Errors
|
||||
///
|
||||
/// When the decision itself could not be made, for example because the
|
||||
/// policy subsystem is not ready. A denial is `Ok(false)`.
|
||||
async fn is_allowed(&self, principal: &SseKmsPrincipal, action: KmsAction, key_id: &str) -> Result<bool, SseError>;
|
||||
}
|
||||
@@ -1,180 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! The provider that wraps and unwraps data keys through the KMS service.
|
||||
|
||||
use crate::core::audit::KmsDataPlaneFailure;
|
||||
use crate::error::SseError;
|
||||
use crate::host::SseHost;
|
||||
use crate::kms::provider::SseDekProvider;
|
||||
use async_trait::async_trait;
|
||||
use rustfs_kms::types::ObjectEncryptionContext;
|
||||
use rustfs_kms::{DataKey, KmsUnavailableError};
|
||||
use rustfs_s3_types::S3ErrorCode;
|
||||
use std::sync::Arc;
|
||||
|
||||
/// Production KMS-backed DEK provider
|
||||
/// Resolves the latest ObjectEncryptionService on each call.
|
||||
pub(crate) struct KmsSseDekProvider<'a> {
|
||||
service_source: KmsServiceSource<'a>,
|
||||
}
|
||||
|
||||
/// Where a [`KmsSseDekProvider`] finds the service that is serving right now.
|
||||
enum KmsServiceSource<'a> {
|
||||
Host(&'a dyn SseHost),
|
||||
#[cfg(test)]
|
||||
Manager(Arc<rustfs_kms::KmsServiceManager>),
|
||||
}
|
||||
|
||||
pub(crate) fn kms_operation_error(error: rustfs_kms::KmsError) -> SseError {
|
||||
let class = rustfs_kms::audit::error_class(&error);
|
||||
let mut api_error = SseError::internal(error);
|
||||
// Wrap rather than replace the source so the original chain stays intact for
|
||||
// logging; the wrapper only adds the class the audit attachment needs.
|
||||
let source = api_error.source.take();
|
||||
api_error.source = Some(Box::new(KmsDataPlaneFailure { class, source }));
|
||||
api_error
|
||||
}
|
||||
|
||||
/// Classification for a failed data-key unwrap.
|
||||
///
|
||||
/// An AEAD failure stays `500` (the envelope is an integrity fault, not a
|
||||
/// request a retry or a different header can fix), but the generic internal
|
||||
/// error text hides the one diagnosis an operator needs: the configured
|
||||
/// backend holds different key material under this key id than the one that
|
||||
/// wrapped the object, typically after re-creating a key of the same name or
|
||||
/// switching backends.
|
||||
pub(crate) fn kms_unwrap_error(error: rustfs_kms::KmsError) -> SseError {
|
||||
let unwrap_rejected = matches!(error, rustfs_kms::KmsError::CryptographicError { .. });
|
||||
let mut api_error = kms_operation_error(error);
|
||||
if unwrap_rejected && api_error.code == S3ErrorCode::InternalError {
|
||||
api_error.message = "The object's data key envelope could not be unwrapped by the configured KMS backend: the \
|
||||
key material under this key id differs from the one that wrapped it, or the envelope is damaged"
|
||||
.to_string();
|
||||
}
|
||||
api_error
|
||||
}
|
||||
|
||||
impl<'a> KmsSseDekProvider<'a> {
|
||||
/// Create a new KMS-backed provider
|
||||
pub async fn new(host: &'a dyn SseHost) -> Result<Self, SseError> {
|
||||
let provider = Self {
|
||||
service_source: KmsServiceSource::Host(host),
|
||||
};
|
||||
provider
|
||||
.current_service()
|
||||
.await
|
||||
.ok_or_else(|| SseError::internal(KmsUnavailableError))?;
|
||||
Ok(provider)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn new_with_service_manager(service_manager: Arc<rustfs_kms::KmsServiceManager>) -> Result<Self, SseError> {
|
||||
let provider = Self {
|
||||
service_source: KmsServiceSource::Manager(service_manager),
|
||||
};
|
||||
provider
|
||||
.current_service()
|
||||
.await
|
||||
.ok_or_else(|| SseError::internal(KmsUnavailableError))?;
|
||||
Ok(provider)
|
||||
}
|
||||
|
||||
async fn current_service(&self) -> Option<Arc<rustfs_kms::service::ObjectEncryptionService>> {
|
||||
match &self.service_source {
|
||||
KmsServiceSource::Host(host) => host.encryption_service().await,
|
||||
#[cfg(test)]
|
||||
KmsServiceSource::Manager(service_manager) => service_manager.get_encryption_service().await,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl SseDekProvider for KmsSseDekProvider<'_> {
|
||||
async fn generate_sse_dek(
|
||||
&self,
|
||||
context: &ObjectEncryptionContext,
|
||||
kms_key_id: &str,
|
||||
) -> Result<(DataKey, Vec<u8>), SseError> {
|
||||
let kms_key_option = Some(kms_key_id.to_string());
|
||||
let service = self
|
||||
.current_service()
|
||||
.await
|
||||
.ok_or_else(|| SseError::internal(KmsUnavailableError))?;
|
||||
let (data_key, encrypted_data_key) = service
|
||||
.create_data_key(&kms_key_option, context)
|
||||
.await
|
||||
.map_err(kms_operation_error)?;
|
||||
|
||||
Ok((data_key, encrypted_data_key))
|
||||
}
|
||||
|
||||
async fn reads_bare_data_key_ciphertext(&self) -> bool {
|
||||
self.current_service()
|
||||
.await
|
||||
.is_some_and(|service| service.reads_bare_data_key_ciphertext())
|
||||
}
|
||||
|
||||
async fn rewrap_sse_dek(
|
||||
&self,
|
||||
encrypted_dek: &[u8],
|
||||
context: &ObjectEncryptionContext,
|
||||
) -> Result<rustfs_kms::types::RewrapDataKeyResponse, SseError> {
|
||||
let service = self
|
||||
.current_service()
|
||||
.await
|
||||
.ok_or_else(|| SseError::internal(KmsUnavailableError))?;
|
||||
service
|
||||
.rewrap_data_key(encrypted_dek, context)
|
||||
.await
|
||||
.map_err(kms_operation_error)
|
||||
}
|
||||
|
||||
async fn decrypt_sse_dek(
|
||||
&self,
|
||||
encrypted_dek: &[u8],
|
||||
_kms_key_id: &str,
|
||||
context: &ObjectEncryptionContext,
|
||||
) -> Result<[u8; 32], SseError> {
|
||||
let service = self
|
||||
.current_service()
|
||||
.await
|
||||
.ok_or_else(|| SseError::internal(KmsUnavailableError))?;
|
||||
let data_key = service
|
||||
.decrypt_data_key(encrypted_dek, context)
|
||||
.await
|
||||
.map_err(kms_unwrap_error)?;
|
||||
|
||||
Ok(data_key.plaintext_key)
|
||||
}
|
||||
|
||||
#[cfg(feature = "rio-v2")]
|
||||
async fn decrypt_legacy_sse_dek(
|
||||
&self,
|
||||
encrypted_dek: &[u8],
|
||||
_kms_key_id: &str,
|
||||
_context: &ObjectEncryptionContext,
|
||||
) -> Result<[u8; 32], SseError> {
|
||||
let service = self
|
||||
.current_service()
|
||||
.await
|
||||
.ok_or_else(|| SseError::internal(KmsUnavailableError))?;
|
||||
let data_key = service
|
||||
.decrypt_legacy_data_key(encrypted_dek)
|
||||
.await
|
||||
.map_err(kms_unwrap_error)?;
|
||||
|
||||
Ok(data_key.plaintext_key)
|
||||
}
|
||||
}
|
||||
@@ -1,323 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! The provider that wraps data keys with a node-local master key when no KMS is configured.
|
||||
|
||||
use crate::error::SseError;
|
||||
#[cfg(feature = "rio-v2")]
|
||||
use crate::kms::minio::{decrypt_minio_kms_data_key, minio_kms_associated_data};
|
||||
use crate::kms::provider::SseDekProvider;
|
||||
use aes_gcm::aead::{Aead, KeyInit};
|
||||
use aes_gcm::{Aes256Gcm, Key, Nonce};
|
||||
use async_trait::async_trait;
|
||||
use base64_simd::STANDARD as BASE64_STANDARD;
|
||||
use rand::Rng;
|
||||
use rustfs_kms::DataKey;
|
||||
use rustfs_kms::types::ObjectEncryptionContext;
|
||||
use rustfs_s3_types::S3ErrorCode;
|
||||
use rustfs_utils::get_env_opt_str;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Local SSE DEK provider for deployments without a KMS.
|
||||
///
|
||||
/// Uses `RUSTFS_SSE_S3_MASTER_KEY` (base64-encoded 32-byte key) to wrap
|
||||
/// data-encryption keys with AES-256-GCM. This is the production fallback
|
||||
/// when no KMS service is configured.
|
||||
///
|
||||
/// # Environment Variable
|
||||
///
|
||||
/// ```text
|
||||
/// RUSTFS_SSE_S3_MASTER_KEY=<base64_encoded_32_byte_key>
|
||||
/// ```
|
||||
pub struct LocalSseDekProvider {
|
||||
master_key: [u8; 32],
|
||||
}
|
||||
|
||||
pub(crate) const LOCAL_SSE_DEK_FORMAT_VERSION: u8 = 1;
|
||||
|
||||
/// Environment variable that carries the local SSE-S3 master key.
|
||||
pub(crate) const ENV_SSE_S3_MASTER_KEY: &str = "RUSTFS_SSE_S3_MASTER_KEY";
|
||||
|
||||
/// The configured local master key, still encoded; `None` when unset or blank.
|
||||
fn local_master_key_env_value() -> Option<String> {
|
||||
get_env_opt_str(ENV_SSE_S3_MASTER_KEY).filter(|value| !value.trim().is_empty())
|
||||
}
|
||||
|
||||
/// Whether the operator configured a local SSE-S3 master key at all.
|
||||
///
|
||||
/// Says nothing about whether the value is usable; loading it decides that.
|
||||
pub(crate) fn local_master_key_configured() -> bool {
|
||||
local_master_key_env_value().is_some()
|
||||
}
|
||||
|
||||
#[cfg(feature = "rio-v2")]
|
||||
/// Returns true when a managed-SSE data key is one RustFS itself wrote.
|
||||
///
|
||||
/// Both RustFS envelope shapes are strict JSON — the KMS envelope
|
||||
/// ([`rustfs_kms::is_data_key_envelope`]) and the local provider's
|
||||
/// [`LocalSseDekEnvelope`], whose `deny_unknown_fields` keeps it from accepting
|
||||
/// anything else. Recognition is deliberately positive: an unrecognized payload
|
||||
/// is left to MinIO's decoder rather than guessed at, and neither decoder is
|
||||
/// ever handed the other's format.
|
||||
pub(crate) fn is_rustfs_managed_data_key(encrypted_dek: &[u8]) -> bool {
|
||||
if rustfs_kms::is_data_key_envelope(encrypted_dek) {
|
||||
return true;
|
||||
}
|
||||
std::str::from_utf8(encrypted_dek)
|
||||
.ok()
|
||||
.is_some_and(|text| serde_json::from_str::<LocalSseDekEnvelope<'_>>(text).is_ok())
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub(crate) struct LocalSseDekEnvelope<'a> {
|
||||
pub(crate) version: u8,
|
||||
pub(crate) nonce: &'a str,
|
||||
ciphertext: &'a str,
|
||||
}
|
||||
|
||||
/// Whether [`LocalSseDekProvider`] could have written these bytes.
|
||||
///
|
||||
/// Covers both shapes [`LocalSseDekProvider::decrypt_dek`] accepts — anything
|
||||
/// presented as a JSON object, so a damaged local envelope keeps its own
|
||||
/// diagnosis, and the deprecated `nonce:ciphertext` base64 pair. Everything
|
||||
/// else is bytes the local provider can only reject.
|
||||
pub(crate) fn is_local_sse_dek(encrypted_dek: &[u8]) -> bool {
|
||||
let Ok(text) = std::str::from_utf8(encrypted_dek) else {
|
||||
return false;
|
||||
};
|
||||
if text.trim_start().starts_with('{') {
|
||||
return true;
|
||||
}
|
||||
let is_base64 = |part: &str| BASE64_STANDARD.decode_to_vec(part).is_ok();
|
||||
text.split_once(':')
|
||||
.is_some_and(|(nonce, ciphertext)| is_base64(nonce) && is_base64(ciphertext))
|
||||
}
|
||||
|
||||
/// Test-only alias so existing test code that references `TestSseDekProvider`
|
||||
/// continues to compile without changes.
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
#[allow(non_camel_case_types)]
|
||||
pub type TestSseDekProvider = LocalSseDekProvider;
|
||||
|
||||
/// Parse the base64-encoded 32-byte master key from `__RUSTFS_SSE_SIMPLE_CMK`.
|
||||
///
|
||||
/// Returns an error (never crashes) for a missing, non-base64, wrong-length, or
|
||||
/// all-zero key so callers on the request path can fail the request instead of
|
||||
/// taking the whole server down (backlog#806).
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub(crate) fn parse_simple_sse_cmk(cmk_value: &str) -> Result<[u8; 32], SseError> {
|
||||
let trimmed = cmk_value.trim();
|
||||
if trimmed.is_empty() {
|
||||
return Err(SseError::internal(
|
||||
"SSE simple mode requires __RUSTFS_SSE_SIMPLE_CMK to be set to a base64-encoded 32-byte key",
|
||||
));
|
||||
}
|
||||
let decoded = BASE64_STANDARD
|
||||
.decode_to_vec(trimmed)
|
||||
.map_err(|e| SseError::internal(format!("__RUSTFS_SSE_SIMPLE_CMK must be valid base64: {e}")))?;
|
||||
let master_key: [u8; 32] = decoded.try_into().map_err(|v: Vec<u8>| {
|
||||
SseError::internal(format!("__RUSTFS_SSE_SIMPLE_CMK must decode to exactly 32 bytes, got {} bytes", v.len()))
|
||||
})?;
|
||||
if master_key == [0u8; 32] {
|
||||
return Err(SseError::internal("__RUSTFS_SSE_SIMPLE_CMK must not be an all-zero key"));
|
||||
}
|
||||
Ok(master_key)
|
||||
}
|
||||
|
||||
impl LocalSseDekProvider {
|
||||
/// Create a LocalSseDekProvider with a predefined key (for testing)
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub fn new_with_key(master_key: [u8; 32]) -> Self {
|
||||
Self { master_key }
|
||||
}
|
||||
|
||||
/// Create a TestSseDekProvider from `__RUSTFS_SSE_SIMPLE_CMK` (test-only).
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub fn new() -> Result<Self, SseError> {
|
||||
let cmk_value = std::env::var("__RUSTFS_SSE_SIMPLE_CMK").unwrap_or_default();
|
||||
// A missing/invalid key must surface as a request error, never crash the
|
||||
// whole server: `LocalSseDekProvider::new` is reached from the SSE request
|
||||
// path (get_sse_dek_provider), so `process::exit(1)` here turned a bad
|
||||
// `__RUSTFS_SSE_SIMPLE_CMK` into a process crash-loop DoS (backlog#806).
|
||||
let master_key = parse_simple_sse_cmk(&cmk_value)?;
|
||||
tracing::info!("Successfully loaded SSE master key (32 bytes) from __RUSTFS_SSE_SIMPLE_CMK");
|
||||
Ok(Self { master_key })
|
||||
}
|
||||
|
||||
/// Create a local SSE DEK provider for SSE-S3 when KMS is not configured.
|
||||
/// Requires RUSTFS_SSE_S3_MASTER_KEY to be a valid base64-encoded 32-byte key.
|
||||
///
|
||||
/// The failures here are server configuration problems, not internal
|
||||
/// faults: surface them as `InvalidRequest` (HTTP 400) so a managed-SSE
|
||||
/// request against an unconfigured server does not report 500 (rustfs#4844).
|
||||
pub fn new_from_env() -> Result<Self, SseError> {
|
||||
fn sse_not_configured(message: impl Into<String>) -> SseError {
|
||||
SseError {
|
||||
code: S3ErrorCode::InvalidRequest,
|
||||
message: message.into(),
|
||||
source: None,
|
||||
}
|
||||
}
|
||||
|
||||
let Some(raw_value) = local_master_key_env_value() else {
|
||||
return Err(sse_not_configured(
|
||||
"SSE-S3 requires RUSTFS_SSE_S3_MASTER_KEY to be set to a base64-encoded 32-byte key when KMS is not configured",
|
||||
));
|
||||
};
|
||||
|
||||
let decoded = BASE64_STANDARD.decode_to_vec(raw_value.trim()).map_err(|err| {
|
||||
sse_not_configured(format!(
|
||||
"RUSTFS_SSE_S3_MASTER_KEY must be valid base64 for SSE-S3 when KMS is not configured: {err}"
|
||||
))
|
||||
})?;
|
||||
let master_key: [u8; 32] = decoded.try_into().map_err(|_| {
|
||||
sse_not_configured("RUSTFS_SSE_S3_MASTER_KEY must decode to exactly 32 bytes for SSE-S3 when KMS is not configured")
|
||||
})?;
|
||||
|
||||
Ok(Self { master_key })
|
||||
}
|
||||
|
||||
// Simple encryption of DEK
|
||||
pub fn encrypt_dek(dek: [u8; 32], cmk_value: [u8; 32]) -> Result<String, SseError> {
|
||||
// Use AES-256-GCM to encrypt DEK
|
||||
let key = Key::<Aes256Gcm>::from(cmk_value);
|
||||
|
||||
let cipher = Aes256Gcm::new(&key);
|
||||
let mut nonce_bytes = [0u8; 12];
|
||||
rand::rng().fill_bytes(&mut nonce_bytes);
|
||||
let nonce = Nonce::from(nonce_bytes);
|
||||
let ciphertext = cipher
|
||||
.encrypt(&nonce, dek.as_slice())
|
||||
.map_err(|_| SseError::internal("Failed to encrypt DEK"))?;
|
||||
|
||||
let nonce = BASE64_STANDARD.encode_to_string(nonce);
|
||||
let ciphertext = BASE64_STANDARD.encode_to_string(ciphertext);
|
||||
serde_json::to_string(&LocalSseDekEnvelope {
|
||||
version: LOCAL_SSE_DEK_FORMAT_VERSION,
|
||||
nonce: &nonce,
|
||||
ciphertext: &ciphertext,
|
||||
})
|
||||
.map_err(|e| SseError::internal(format!("Failed to serialize encrypted DEK: {e}")))
|
||||
}
|
||||
|
||||
// Simple decryption of DEK
|
||||
pub fn decrypt_dek(encrypted_dek: &str, cmk_value: [u8; 32]) -> Result<[u8; 32], SseError> {
|
||||
let envelope = serde_json::from_str::<LocalSseDekEnvelope<'_>>(encrypted_dek);
|
||||
let (nonce, ciphertext) = match envelope {
|
||||
Ok(envelope) => {
|
||||
if envelope.version != LOCAL_SSE_DEK_FORMAT_VERSION {
|
||||
return Err(SseError::internal(format!(
|
||||
"Unsupported encrypted DEK format version: {}",
|
||||
envelope.version
|
||||
)));
|
||||
}
|
||||
(envelope.nonce, envelope.ciphertext)
|
||||
}
|
||||
Err(json_error) if encrypted_dek.trim_start().starts_with('{') => {
|
||||
return Err(SseError::internal(format!("Invalid encrypted DEK JSON format: {json_error}")));
|
||||
}
|
||||
Err(_) => {
|
||||
// DEPRECATED: read-only compatibility for persisted colon-delimited DEKs.
|
||||
// RUSTFS_COMPAT_TODO(sse-local-dek-json-v1): Remove after all supported upgrades have rewritten legacy DEKs.
|
||||
let Some((nonce, ciphertext)) = encrypted_dek.split_once(':') else {
|
||||
return Err(SseError::internal("Invalid encrypted DEK format"));
|
||||
};
|
||||
if ciphertext.contains(':') {
|
||||
return Err(SseError::internal("Invalid encrypted DEK format"));
|
||||
}
|
||||
(nonce, ciphertext)
|
||||
}
|
||||
};
|
||||
let nonce_vec = BASE64_STANDARD
|
||||
.decode_to_vec(nonce)
|
||||
.map_err(|_| SseError::internal("Invalid nonce format"))?;
|
||||
let ciphertext = BASE64_STANDARD
|
||||
.decode_to_vec(ciphertext)
|
||||
.map_err(|_| SseError::internal("Invalid ciphertext format"))?;
|
||||
|
||||
let key = Key::<Aes256Gcm>::from(cmk_value);
|
||||
let cipher = Aes256Gcm::new(&key);
|
||||
|
||||
let nonce_array: [u8; 12] = nonce_vec.try_into().map_err(|_| SseError::internal("Invalid nonce length"))?;
|
||||
let nonce = Nonce::from(nonce_array);
|
||||
|
||||
let plaintext = cipher
|
||||
.decrypt(&nonce, ciphertext.as_slice())
|
||||
.map_err(|e| SseError::internal(format!("Failed to decrypt DEK: {e}")))?;
|
||||
|
||||
let dek: [u8; 32] = plaintext
|
||||
.try_into()
|
||||
.map_err(|_| SseError::internal("Decrypted DEK has invalid length"))?;
|
||||
|
||||
Ok(dek)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl SseDekProvider for LocalSseDekProvider {
|
||||
fn wraps_dek_with_local_master_key(&self) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
async fn generate_sse_dek(
|
||||
&self,
|
||||
_context: &ObjectEncryptionContext,
|
||||
_kms_key_id: &str,
|
||||
) -> Result<(DataKey, Vec<u8>), SseError> {
|
||||
// Generate a 32-byte array as data key
|
||||
let mut dek = [0u8; 32];
|
||||
rand::rng().fill_bytes(&mut dek);
|
||||
|
||||
// Generate a 12-byte array as IV
|
||||
let mut nonce = [0u8; 12];
|
||||
rand::rng().fill_bytes(&mut nonce);
|
||||
|
||||
// Encrypt data key with master key
|
||||
let encrypted_dek = Self::encrypt_dek(dek, self.master_key)?;
|
||||
|
||||
// Return data key and IV
|
||||
Ok((
|
||||
DataKey {
|
||||
plaintext_key: dek,
|
||||
nonce,
|
||||
},
|
||||
encrypted_dek.into_bytes(),
|
||||
))
|
||||
}
|
||||
|
||||
async fn decrypt_sse_dek(
|
||||
&self,
|
||||
encrypted_dek: &[u8],
|
||||
_kms_key_id: &str,
|
||||
_context: &ObjectEncryptionContext,
|
||||
) -> Result<[u8; 32], SseError> {
|
||||
// Decrypt data key with master key
|
||||
let encrypted_dek_str =
|
||||
std::str::from_utf8(encrypted_dek).map_err(|_| SseError::internal("Invalid UTF-8 in encrypted DEK"))?;
|
||||
let dek = Self::decrypt_dek(encrypted_dek_str, self.master_key)?;
|
||||
Ok(dek)
|
||||
}
|
||||
|
||||
#[cfg(feature = "rio-v2")]
|
||||
async fn decrypt_minio_sse_dek(
|
||||
&self,
|
||||
encrypted_dek: &[u8],
|
||||
_kms_key_id: &str,
|
||||
context: &ObjectEncryptionContext,
|
||||
) -> Result<[u8; 32], SseError> {
|
||||
let aad = minio_kms_associated_data(context)?;
|
||||
decrypt_minio_kms_data_key(encrypted_dek, &self.master_key, &aad)
|
||||
}
|
||||
}
|
||||
@@ -1,151 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Reading data keys sealed by MinIO's builtin KMS.
|
||||
|
||||
use crate::core::keying::context::canonical_kms_bucket_path;
|
||||
use crate::core::keying::object::HmacSha256;
|
||||
use crate::error::SseError;
|
||||
use base64_simd::STANDARD as BASE64_STANDARD;
|
||||
use hmac::Mac;
|
||||
use rustfs_kms::types::ObjectEncryptionContext;
|
||||
use serde::Deserialize;
|
||||
|
||||
/// Associated data MinIO binds when sealing a data key.
|
||||
///
|
||||
/// MinIO passes the object's encryption context as the AEAD's associated data,
|
||||
/// serialized as canonical JSON with sorted keys — the same canonicalization
|
||||
/// [`rustfs_kms::context_aad`] performs, which is why the context RustFS
|
||||
/// already rebuilds for the read can be reused verbatim. For SSE-S3 that
|
||||
/// context is `{bucket: "bucket/object"}`; for SSE-KMS it is whatever the
|
||||
/// request supplied, recovered from the stored MinIO context header.
|
||||
pub(crate) fn minio_kms_associated_data(context: &ObjectEncryptionContext) -> Result<Vec<u8>, SseError> {
|
||||
let mut ctx = context.encryption_context.clone();
|
||||
ctx.entry(context.bucket.clone())
|
||||
.or_insert_with(|| canonical_kms_bucket_path(&context.bucket, &context.object_key));
|
||||
rustfs_kms::context_aad(&ctx).map_err(|e| SseError::internal(format!("Failed to canonicalize MinIO KMS context: {e}")))
|
||||
}
|
||||
|
||||
/// MinIO's builtin-KMS ciphertext in its JSON encoding.
|
||||
///
|
||||
/// Deliberately its own type rather than a relaxation of
|
||||
/// [`LocalSseDekEnvelope`]: widening that envelope's `deny_unknown_fields`
|
||||
/// to admit this shape would also admit malformed RustFS envelopes, which
|
||||
/// backlog#1567 requires to keep failing closed.
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct MinioKmsCiphertextJson {
|
||||
aead: String,
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "present in MinIO's encoding; the key is identified by metadata instead"
|
||||
)]
|
||||
#[serde(default)]
|
||||
id: String,
|
||||
iv: String,
|
||||
nonce: String,
|
||||
bytes: String,
|
||||
}
|
||||
|
||||
/// Bytes of trailing randomness every MinIO builtin-KMS ciphertext carries:
|
||||
/// a 16-byte IV followed by a 12-byte nonce, *after* the sealed bytes.
|
||||
const MINIO_KMS_RANDOM_LEN: usize = 28;
|
||||
|
||||
const MINIO_KMS_IV_LEN: usize = 16;
|
||||
|
||||
const MINIO_KMS_AEAD_AES_GCM: &str = "AES-256-GCM-HMAC-SHA-256";
|
||||
|
||||
const MINIO_KMS_AEAD_CHACHA20: &str = "ChaCha20Poly1305";
|
||||
|
||||
/// Unwrap a data key sealed by MinIO's builtin (static-secret) KMS.
|
||||
///
|
||||
/// The wire format is `sealed_bytes || iv[16] || nonce[12]` — the randomness
|
||||
/// trails the ciphertext rather than leading it, and MinIO's own decoder
|
||||
/// normalizes its legacy JSON encoding into exactly that byte order before
|
||||
/// opening it (`internal/kms/secret-key.go`, `parseCiphertext`). A raw
|
||||
/// (non-JSON) ciphertext is AES-256-GCM by definition there; the JSON form
|
||||
/// names its algorithm.
|
||||
///
|
||||
/// The sealing key is derived per ciphertext rather than being the master key:
|
||||
/// `HMAC-SHA256(master, iv)` for AES-256-GCM, `HChaCha20(master, iv)` for
|
||||
/// ChaCha20-Poly1305. The encryption context is bound as associated data.
|
||||
pub(crate) fn decrypt_minio_kms_data_key(encrypted_dek: &[u8], master_key: &[u8; 32], aad: &[u8]) -> Result<[u8; 32], SseError> {
|
||||
let (body, algorithm) = match std::str::from_utf8(encrypted_dek) {
|
||||
// MinIO only treats a payload as JSON when it both starts and ends like
|
||||
// an object, and falls back to the raw layout when it does not parse —
|
||||
// mirrored here so a ciphertext that merely looks like JSON is not
|
||||
// rejected outright.
|
||||
Ok(text)
|
||||
if text.starts_with('{')
|
||||
&& text.ends_with('}')
|
||||
&& let Ok(json) = serde_json::from_str::<MinioKmsCiphertextJson>(text) =>
|
||||
{
|
||||
let decode = |what: &str, value: &str| -> Result<Vec<u8>, SseError> {
|
||||
BASE64_STANDARD
|
||||
.decode_to_vec(value)
|
||||
.map_err(|e| SseError::internal(format!("Invalid MinIO KMS {what}: {e}")))
|
||||
};
|
||||
let mut body = decode("ciphertext", &json.bytes)?;
|
||||
body.extend_from_slice(&decode("iv", &json.iv)?);
|
||||
body.extend_from_slice(&decode("nonce", &json.nonce)?);
|
||||
(body, json.aead)
|
||||
}
|
||||
_ => (encrypted_dek.to_vec(), MINIO_KMS_AEAD_AES_GCM.to_string()),
|
||||
};
|
||||
|
||||
if body.len() <= MINIO_KMS_RANDOM_LEN {
|
||||
return Err(SseError::internal("MinIO KMS ciphertext is too short to carry its IV and nonce"));
|
||||
}
|
||||
let (sealed, random) = body.split_at(body.len() - MINIO_KMS_RANDOM_LEN);
|
||||
let (iv, nonce) = random.split_at(MINIO_KMS_IV_LEN);
|
||||
|
||||
let plaintext = match algorithm.as_str() {
|
||||
MINIO_KMS_AEAD_AES_GCM => {
|
||||
use aes_gcm::{Aes256Gcm, KeyInit, aead::Aead};
|
||||
let mut mac = HmacSha256::new_from_slice(master_key)
|
||||
.map_err(|_| SseError::internal("MinIO KMS sealing key derivation failed"))?;
|
||||
mac.update(iv);
|
||||
let sealing_key: [u8; 32] = mac.finalize().into_bytes().into();
|
||||
let cipher = Aes256Gcm::new_from_slice(&sealing_key)
|
||||
.map_err(|_| SseError::internal("MinIO KMS sealing key is not a valid AES-256 key"))?;
|
||||
let nonce = aes_gcm::Nonce::try_from(nonce).map_err(|_| SseError::internal("MinIO KMS nonce is not 12 bytes"))?;
|
||||
cipher.decrypt(&nonce, aes_gcm::aead::Payload { msg: sealed, aad })
|
||||
}
|
||||
MINIO_KMS_AEAD_CHACHA20 => {
|
||||
use chacha20poly1305::{KeyInit, XChaCha20Poly1305, aead::Aead};
|
||||
// MinIO derives this branch's key with HChaCha20 over the 16-byte
|
||||
// IV, which is exactly XChaCha20-Poly1305's own construction, so the
|
||||
// extended-nonce cipher does the derivation rather than hand-rolling it.
|
||||
let mut extended = Vec::with_capacity(MINIO_KMS_IV_LEN + nonce.len());
|
||||
extended.extend_from_slice(iv);
|
||||
extended.extend_from_slice(nonce);
|
||||
let cipher = XChaCha20Poly1305::new_from_slice(master_key)
|
||||
.map_err(|_| SseError::internal("MinIO KMS master key is not a valid ChaCha20 key"))?;
|
||||
let nonce = chacha20poly1305::XNonce::try_from(extended.as_slice())
|
||||
.map_err(|_| SseError::internal("MinIO KMS extended nonce is not 24 bytes"))?;
|
||||
cipher.decrypt(&nonce, chacha20poly1305::aead::Payload { msg: sealed, aad })
|
||||
}
|
||||
other => {
|
||||
return Err(SseError::internal(format!("Unsupported MinIO KMS AEAD algorithm: {other}")));
|
||||
}
|
||||
}
|
||||
// An AEAD failure here is authentication, not a decode slip: a wrong master
|
||||
// key, a tampered ciphertext, and an encryption context that does not match
|
||||
// what sealed it all land here and must all fail closed.
|
||||
.map_err(|_| SseError::internal("MinIO KMS data key failed authentication"))?;
|
||||
|
||||
plaintext
|
||||
.try_into()
|
||||
.map_err(|value: Vec<u8>| SseError::internal(format!("MinIO KMS data key must be 32 bytes, got {}", value.len())))
|
||||
}
|
||||
@@ -1,21 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Data-key providers: where a data key is wrapped and unwrapped.
|
||||
|
||||
pub(crate) mod kms_backed;
|
||||
pub(crate) mod local;
|
||||
#[cfg(feature = "rio-v2")]
|
||||
pub(crate) mod minio;
|
||||
pub(crate) mod provider;
|
||||
@@ -1,288 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! The data-key provider interface and the per-request choice between the KMS and the local master key.
|
||||
|
||||
use crate::error::SseError;
|
||||
use crate::host::SseHost;
|
||||
use crate::kms::kms_backed::KmsSseDekProvider;
|
||||
use crate::kms::local::LocalSseDekProvider;
|
||||
use async_trait::async_trait;
|
||||
use rustfs_kms::DataKey;
|
||||
use rustfs_kms::types::ObjectEncryptionContext;
|
||||
use std::sync::{Arc, LazyLock, RwLock};
|
||||
use tracing::debug;
|
||||
|
||||
/// Trait for SSE data encryption key management
|
||||
/// Abstracts the source of encryption keys (KMS, test provider, etc.)
|
||||
#[async_trait]
|
||||
pub trait SseDekProvider: Send + Sync {
|
||||
/// Generate an SSE data encryption key
|
||||
async fn generate_sse_dek(&self, context: &ObjectEncryptionContext, kms_key_id: &str)
|
||||
-> Result<(DataKey, Vec<u8>), SseError>;
|
||||
|
||||
/// Decrypt an SSE data encryption key (returns only plaintext key, nonce should be read from metadata)
|
||||
async fn decrypt_sse_dek(
|
||||
&self,
|
||||
encrypted_dek: &[u8],
|
||||
kms_key_id: &str,
|
||||
context: &ObjectEncryptionContext,
|
||||
) -> Result<[u8; 32], SseError>;
|
||||
|
||||
/// Whether this provider may own a stored data key that is not a RustFS
|
||||
/// envelope — see `KmsBackend::reads_bare_data_key_ciphertext`.
|
||||
async fn reads_bare_data_key_ciphertext(&self) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
/// Re-wrap a KMS-wrapped DEK envelope onto its master key's current
|
||||
/// version without exposing the plaintext DEK to the caller.
|
||||
///
|
||||
/// Defaults to refusing: only the KMS-backed provider can rewrap, and a
|
||||
/// provider that cannot must say so rather than hand back the input as if
|
||||
/// it had been re-protected.
|
||||
async fn rewrap_sse_dek(
|
||||
&self,
|
||||
_encrypted_dek: &[u8],
|
||||
_context: &ObjectEncryptionContext,
|
||||
) -> Result<rustfs_kms::types::RewrapDataKeyResponse, SseError> {
|
||||
Err(SseError::internal("This DEK provider cannot rewrap KMS-wrapped data keys"))
|
||||
}
|
||||
|
||||
/// Whether this provider wraps data keys with a node-local master key
|
||||
/// instead of a KMS service.
|
||||
///
|
||||
/// SSE-KMS must never be served by such a provider: the stored object would
|
||||
/// claim `aws:kms` and name a KMS key id that never wrapped anything.
|
||||
/// Defaults to false so only the local fallback has to declare itself.
|
||||
fn wraps_dek_with_local_master_key(&self) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
/// Decrypt a DEK from positively identified legacy managed metadata.
|
||||
#[cfg(feature = "rio-v2")]
|
||||
async fn decrypt_legacy_sse_dek(
|
||||
&self,
|
||||
encrypted_dek: &[u8],
|
||||
kms_key_id: &str,
|
||||
context: &ObjectEncryptionContext,
|
||||
) -> Result<[u8; 32], SseError> {
|
||||
self.decrypt_sse_dek(encrypted_dek, kms_key_id, context).await
|
||||
}
|
||||
|
||||
/// Unwrap a data key that MinIO's builtin KMS sealed.
|
||||
///
|
||||
/// A separate entry point rather than a shape sniff inside
|
||||
/// [`Self::decrypt_sse_dek`]: the caller already knows the object carries a
|
||||
/// MinIO sealed key, and MinIO's raw ciphertext is unstructured bytes that
|
||||
/// no parser can reliably tell apart from anything else. Routing on the
|
||||
/// caller's knowledge keeps a RustFS envelope from ever reaching MinIO's
|
||||
/// decoder, and vice versa.
|
||||
///
|
||||
/// Defaults to refusing: only a provider holding the MinIO master secret
|
||||
/// can serve these, and a provider that cannot must fail rather than fall
|
||||
/// back to a decoder that would misread the bytes.
|
||||
#[cfg(feature = "rio-v2")]
|
||||
async fn decrypt_minio_sse_dek(
|
||||
&self,
|
||||
_encrypted_dek: &[u8],
|
||||
_kms_key_id: &str,
|
||||
_context: &ObjectEncryptionContext,
|
||||
) -> Result<[u8; 32], SseError> {
|
||||
Err(SseError::internal(
|
||||
"This KMS provider cannot unwrap a data key sealed by MinIO's builtin KMS",
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
/// Unwrap the data key of an object that carries a sealed object key but no
|
||||
/// RustFS envelope.
|
||||
///
|
||||
/// Two writers produce that shape, and nothing in the bytes tells them apart:
|
||||
/// MinIO, whose data key is sealed under the static master key, and a RustFS
|
||||
/// `rio-v2` build on a backend that persisted its native ciphertext bare.
|
||||
/// MinIO's sealing is authenticated, so opening it under the local master key
|
||||
/// is a proof rather than a guess: the bare blob is offered to the KMS only
|
||||
/// once that has failed, or when no local master key exists to try. Without a
|
||||
/// backend that reads bare ciphertext the MinIO route is the only one, exactly
|
||||
/// as before.
|
||||
#[cfg(feature = "rio-v2")]
|
||||
pub(crate) async fn decrypt_unenveloped_sealed_dek(
|
||||
bare_kms: Option<Arc<dyn SseDekProvider + '_>>,
|
||||
local: Result<Arc<dyn SseDekProvider>, SseError>,
|
||||
encrypted_dek: &[u8],
|
||||
kms_key_id: &str,
|
||||
context: &ObjectEncryptionContext,
|
||||
) -> Result<[u8; 32], SseError> {
|
||||
let Some(kms) = bare_kms else {
|
||||
return local?.decrypt_minio_sse_dek(encrypted_dek, kms_key_id, context).await;
|
||||
};
|
||||
if let Ok(local) = local
|
||||
&& let Ok(data_key) = local.decrypt_minio_sse_dek(encrypted_dek, kms_key_id, context).await
|
||||
{
|
||||
return Ok(data_key);
|
||||
}
|
||||
kms.decrypt_sse_dek(encrypted_dek, kms_key_id, context).await
|
||||
}
|
||||
|
||||
/// The provider that unwraps KMS-wrapped data keys.
|
||||
///
|
||||
/// A provider registered through `set_sse_dek_provider_for_test` stands in for
|
||||
/// the fresh [`KmsSseDekProvider`], which cannot resolve a KMS service from a
|
||||
/// host that has none. Only `GLOBAL_KMS_DEK_PROVIDER` is consulted, never
|
||||
/// `GLOBAL_SSE_DEK_PROVIDER`, so a local provider cached by an earlier
|
||||
/// `get_local_sse_dek_provider` call cannot be selected to unwrap a KMS envelope.
|
||||
pub(crate) async fn kms_sse_dek_provider<'a>(host: &'a dyn SseHost) -> Result<Arc<dyn SseDekProvider + 'a>, SseError> {
|
||||
if let Some(cached) = GLOBAL_KMS_DEK_PROVIDER.read().ok().and_then(|guard| guard.as_ref().cloned()) {
|
||||
return Ok(cached);
|
||||
}
|
||||
Ok(Arc::new(KmsSseDekProvider::new(host).await?))
|
||||
}
|
||||
|
||||
/// The KMS provider, when its backend may own a stored data key that is not a
|
||||
/// RustFS envelope. `None` without a running KMS, which leaves such a key to
|
||||
/// the local provider exactly as before.
|
||||
pub(crate) async fn bare_data_key_kms_provider<'a>(host: &'a dyn SseHost) -> Option<Arc<dyn SseDekProvider + 'a>> {
|
||||
let provider = kms_sse_dek_provider(host).await.ok()?;
|
||||
provider.reads_bare_data_key_ciphertext().await.then_some(provider)
|
||||
}
|
||||
|
||||
/// Global SSE DEK provider cache for local / test providers.
|
||||
///
|
||||
/// Populated by `get_local_sse_dek_provider` and (for backward-compat in tests)
|
||||
/// `set_sse_dek_provider_for_test`. Read by `get_local_sse_dek_provider` only —
|
||||
/// the KMS-envelope decrypt path uses `GLOBAL_KMS_DEK_PROVIDER` so that a cached
|
||||
/// local provider can never be selected for a KMS-wrapped data-key.
|
||||
pub(crate) static GLOBAL_SSE_DEK_PROVIDER: LazyLock<RwLock<Option<Arc<dyn SseDekProvider>>>> =
|
||||
LazyLock::new(|| RwLock::new(None));
|
||||
|
||||
/// Global KMS DEK provider cache for test-injected KMS providers.
|
||||
///
|
||||
/// Populated **only** by `set_sse_dek_provider_for_test` (test-only).
|
||||
/// Read **only** by the KMS-envelope branch of `apply_managed_decryption_material`.
|
||||
/// Separation from `GLOBAL_SSE_DEK_PROVIDER` prevents a previously-cached local
|
||||
/// provider from being selected to unwrap a KMS data-key envelope.
|
||||
static GLOBAL_KMS_DEK_PROVIDER: LazyLock<RwLock<Option<Arc<dyn SseDekProvider>>>> = LazyLock::new(|| RwLock::new(None));
|
||||
|
||||
/// Get or initialize the global SSE DEK provider
|
||||
///
|
||||
/// Factory function that automatically selects the appropriate provider:
|
||||
/// - If `__RUSTFS_SSE_SIMPLE_CMK` environment variable exists: use SimpleSseDekProvider (test mode)
|
||||
/// - Otherwise: use KmsSseDekProvider (production mode with real KMS)
|
||||
///
|
||||
/// # Returns
|
||||
/// Arc to the global SSE DEK provider instance
|
||||
///
|
||||
/// # Example
|
||||
/// ```rust,ignore
|
||||
/// let provider = get_sse_dek_provider(host).await?;
|
||||
/// let (data_key, encrypted_dek) = provider
|
||||
/// .generate_sse_dek("bucket", "key", "kms-key-id")
|
||||
/// .await?;
|
||||
/// ```
|
||||
pub(crate) async fn get_sse_dek_provider<'a>(host: &'a dyn SseHost) -> Result<Arc<dyn SseDekProvider + 'a>, SseError> {
|
||||
if host.encryption_service().await.is_some() {
|
||||
debug!("Using KmsSseDekProvider (KMS configured)");
|
||||
return Ok(Arc::new(KmsSseDekProvider::new(host).await?));
|
||||
}
|
||||
|
||||
get_local_sse_dek_provider()
|
||||
}
|
||||
|
||||
/// Get the local data-key provider, loading and caching it on first use.
|
||||
///
|
||||
/// Startup calls this to check the local master key ahead of traffic; a request
|
||||
/// that arrives without that step loads it the same way and gets the same error.
|
||||
pub(crate) fn get_local_sse_dek_provider() -> Result<Arc<dyn SseDekProvider>, SseError> {
|
||||
// An explicitly injected test provider overrides both the cache and the
|
||||
// environment-based selection below.
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
if let Some(injected) = test_injected_sse_dek_provider() {
|
||||
return Ok(injected);
|
||||
}
|
||||
|
||||
// Check if already initialized
|
||||
if let Some(provider) = GLOBAL_SSE_DEK_PROVIDER
|
||||
.read()
|
||||
.map_err(|_| SseError::internal("Failed to read global SSE DEK provider cache"))?
|
||||
.as_ref()
|
||||
.cloned()
|
||||
{
|
||||
return Ok(provider);
|
||||
}
|
||||
|
||||
// In test mode, prefer the simple CMK provider when the env var is set.
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
{
|
||||
if std::env::var("__RUSTFS_SSE_SIMPLE_CMK").is_ok() {
|
||||
debug!("Using LocalSseDekProvider (test mode) based on __RUSTFS_SSE_SIMPLE_CMK");
|
||||
let provider: Arc<dyn SseDekProvider> = Arc::new(LocalSseDekProvider::new()?);
|
||||
let mut slot = GLOBAL_SSE_DEK_PROVIDER
|
||||
.write()
|
||||
.map_err(|_| SseError::internal("Failed to update global SSE DEK provider cache"))?;
|
||||
if let Some(existing) = slot.as_ref() {
|
||||
return Ok(existing.clone());
|
||||
}
|
||||
*slot = Some(provider.clone());
|
||||
return Ok(provider);
|
||||
}
|
||||
}
|
||||
|
||||
// Production fallback: local SSE-S3 provider (no KMS configured).
|
||||
debug!("Using local SSE-S3 provider (KMS not configured)");
|
||||
let provider: Arc<dyn SseDekProvider> = Arc::new(LocalSseDekProvider::new_from_env()?);
|
||||
|
||||
let mut slot = GLOBAL_SSE_DEK_PROVIDER
|
||||
.write()
|
||||
.map_err(|_| SseError::internal("Failed to update global SSE DEK provider cache"))?;
|
||||
if let Some(existing) = slot.as_ref() {
|
||||
return Ok(existing.clone());
|
||||
}
|
||||
*slot = Some(provider.clone());
|
||||
|
||||
Ok(provider)
|
||||
}
|
||||
|
||||
/// Reset both global SSE DEK provider caches (for testing only)
|
||||
///
|
||||
/// Clears GLOBAL_SSE_DEK_PROVIDER (local/test providers) and
|
||||
/// GLOBAL_KMS_DEK_PROVIDER (test-injected KMS providers).
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub fn reset_sse_dek_provider() {
|
||||
if let Ok(mut slot) = GLOBAL_SSE_DEK_PROVIDER.write() {
|
||||
*slot = None;
|
||||
}
|
||||
if let Ok(mut slot) = GLOBAL_KMS_DEK_PROVIDER.write() {
|
||||
*slot = None;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub fn set_sse_dek_provider_for_test(provider: Arc<dyn SseDekProvider>) {
|
||||
if let Ok(mut slot) = GLOBAL_KMS_DEK_PROVIDER.write() {
|
||||
*slot = Some(provider.clone());
|
||||
}
|
||||
if let Ok(mut slot) = GLOBAL_SSE_DEK_PROVIDER.write() {
|
||||
*slot = Some(provider);
|
||||
}
|
||||
}
|
||||
|
||||
/// Provider explicitly injected via `set_sse_dek_provider_for_test`, if any.
|
||||
///
|
||||
/// Reads `GLOBAL_KMS_DEK_PROVIDER` because that slot is populated *only* by the
|
||||
/// test setter, so a hit here always means an explicit test injection.
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
fn test_injected_sse_dek_provider() -> Option<Arc<dyn SseDekProvider>> {
|
||||
GLOBAL_KMS_DEK_PROVIDER.read().ok().and_then(|guard| guard.as_ref().cloned())
|
||||
}
|
||||
@@ -1,110 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! S3 server-side encryption: which key encrypts an object, and what is stored to find it again.
|
||||
//!
|
||||
//! Covers the three S3 schemes:
|
||||
//! - SSE-S3 (`AES256`): a data key wrapped by a key the server owns
|
||||
//! - SSE-KMS (`aws:kms`): a data key wrapped by a KMS key the caller names
|
||||
//! - SSE-C: a key the client supplies on every request and the server never stores
|
||||
//!
|
||||
//! ## Where this crate sits
|
||||
//!
|
||||
//! Between the S3 layer and the storage engine, depending on neither. It decides
|
||||
//! the effective scheme of a write, obtains and unwraps data keys, authorizes the
|
||||
//! use of a KMS key, and encodes and reads the encryption metadata of an object.
|
||||
//! It does not transform object bytes: it hands back key material
|
||||
//! ([`EncryptionMaterial`], [`DecryptionMaterial`]) and the caller applies it.
|
||||
//!
|
||||
//! What it needs from the running server — the KMS service, a bucket's default
|
||||
//! encryption, a key-permission decision — arrives through [`SseHost`], passed
|
||||
//! to every entry point that consults the server. Failures are reported as
|
||||
//! [`SseError`], already classified with the S3 error code the caller answers with.
|
||||
//!
|
||||
//! ## Entry points
|
||||
//!
|
||||
//! - [`sse_encryption`] and [`sse_prepare_encryption`] resolve the material of a write;
|
||||
//! [`encryption_material_to_metadata`] turns it into the metadata to store.
|
||||
//! - [`sse_decryption`] resolves the material of a read from stored metadata;
|
||||
//! [`classify_sse_read_response`] derives the response headers without unwrapping a key.
|
||||
//! - [`rewrap_object_encryption_metadata`] moves a stored data key onto the current
|
||||
//! version of its KMS key.
|
||||
//! - [`startup::init`] settles the environment-driven configuration once per process
|
||||
//! and reports it; calling it is optional.
|
||||
//!
|
||||
//! ```rust,ignore
|
||||
//! let request = EncryptionRequest {
|
||||
//! bucket: &bucket,
|
||||
//! key: &key,
|
||||
//! server_side_encryption: effective_sse,
|
||||
//! ssekms_key_id: effective_kms_key_id,
|
||||
//! ssekms_context: None,
|
||||
//! sse_customer_algorithm: None,
|
||||
//! sse_customer_key: None,
|
||||
//! sse_customer_key_md5: None,
|
||||
//! content_size: actual_size,
|
||||
//! principal: principal.as_ref(),
|
||||
//! };
|
||||
//!
|
||||
//! if let Some(material) = sse_encryption(host, request).await? {
|
||||
//! metadata.extend(encryption_material_to_metadata(&material)?);
|
||||
//! }
|
||||
//! ```
|
||||
|
||||
mod core;
|
||||
mod error;
|
||||
mod host;
|
||||
mod kms;
|
||||
pub mod startup;
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub mod testing;
|
||||
#[cfg(test)]
|
||||
#[allow(unused_imports)]
|
||||
mod tests;
|
||||
|
||||
pub use crate::core::audit::KmsRequestAuditScope;
|
||||
pub use crate::core::keying::material::{DecryptionMaterial, EncryptionKeyKind, EncryptionMaterial, ManagedSealedKey};
|
||||
pub use crate::core::keying::metadata::{
|
||||
ConflictingEncryptionMetadata, encryption_material_to_metadata, mark_encrypted_multipart_metadata,
|
||||
normalize_encryption_metadata_case, stored_ssec_metadata, strip_managed_encryption_metadata,
|
||||
};
|
||||
pub use crate::core::read::{DecryptionRequest, classify_sse_read_response, sse_decryption};
|
||||
pub use crate::core::request::SseKmsPrincipal;
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub use crate::core::request::TestAuthorizationHooks;
|
||||
pub use crate::core::rewrap::{ObjectDekRewrapOutcome, rewrap_object_encryption_metadata};
|
||||
pub use crate::core::s3::authorization::{authorize_sse_kms_multipart_write, authorize_sse_kms_object_read};
|
||||
pub use crate::core::s3::headers::{
|
||||
SseReadResponseHeaders, SsecParamsFromHeaders, build_ssec_read_headers, extract_server_side_encryption_from_headers,
|
||||
extract_ssec_params_from_headers, extract_ssekms_context_from_headers, project_sse_read_response_headers,
|
||||
validate_sse_headers_for_read, validate_sse_headers_for_write,
|
||||
};
|
||||
pub use crate::core::s3::scheme::{DEFAULT_SSE_ALGORITHM, bucket_default_write_sse};
|
||||
pub use crate::core::s3::ssec::{ssec_invalid_request, validate_ssec_for_read};
|
||||
pub use crate::core::write::{EncryptionRequest, PrepareEncryptionRequest, sse_encryption, sse_prepare_encryption};
|
||||
pub use crate::error::{KMS_KEY_NOT_FOUND_ERROR_CODE, SseError, data_plane_kms_error};
|
||||
pub use crate::host::{BucketDefaultEncryption, BucketEncryption, KmsKeyAuthorizer, SseHost};
|
||||
pub use crate::kms::local::LocalSseDekProvider;
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub use crate::kms::local::TestSseDekProvider;
|
||||
pub use crate::kms::provider::SseDekProvider;
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub use crate::kms::provider::{reset_sse_dek_provider, set_sse_dek_provider_for_test};
|
||||
// The managed-SSE classifier lives in the shared encryption-keys module so the
|
||||
// scanner can reuse it; re-exported here so SSE callers find it beside the rest.
|
||||
pub use rustfs_utils::http::object_encryption_keys::SSEType;
|
||||
pub use rustfs_utils::http::object_encryption_keys::contains_managed_encryption_metadata;
|
||||
|
||||
pub(crate) const LOG_COMPONENT_STORAGE: &str = "storage";
|
||||
pub(crate) const LOG_SUBSYSTEM_SSE: &str = "sse";
|
||||
@@ -1,250 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! The crate's process-startup step.
|
||||
//!
|
||||
//! [`init`] settles the two things that cannot change without a restart, both
|
||||
//! read from the environment, and reports them once:
|
||||
//!
|
||||
//! - whether SSE-KMS requests are authorized against the key they name;
|
||||
//! - whether the local SSE-S3 master key is usable.
|
||||
//!
|
||||
//! Nothing that follows the KMS service is decided here. The KMS can be
|
||||
//! configured, started, restarted, and stopped while the server runs, so the
|
||||
//! choice between the KMS and the local master key is made per request, and no
|
||||
//! KMS state is cached by this crate.
|
||||
//!
|
||||
//! Calling [`init`] is optional. A process that never calls it reads the same
|
||||
//! switch and loads the same master key on first use, with the same errors; it
|
||||
//! only goes without the startup report.
|
||||
|
||||
use crate::error::SseError;
|
||||
use crate::kms::local::{ENV_SSE_S3_MASTER_KEY, local_master_key_configured};
|
||||
use crate::kms::provider::get_local_sse_dek_provider;
|
||||
use crate::{LOG_COMPONENT_STORAGE, LOG_SUBSYSTEM_SSE};
|
||||
use rustfs_config::{DEFAULT_KMS_ENFORCE_SSE_KEY_POLICY, ENV_RUSTFS_KMS_ENFORCE_SSE_KEY_POLICY};
|
||||
use rustfs_utils::get_env_bool;
|
||||
use std::sync::{LazyLock, OnceLock};
|
||||
|
||||
const EVENT_SSE_KMS_KEY_POLICY_MODE: &str = "sse_kms_key_policy_mode";
|
||||
const EVENT_SSE_LOCAL_MASTER_KEY: &str = "sse_local_master_key";
|
||||
|
||||
/// State of the local SSE-S3 master key at startup.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum LocalMasterKeyStatus {
|
||||
/// Configured and usable; SSE-S3 works without a KMS.
|
||||
Loaded,
|
||||
/// Not configured; SSE-S3 requires a running KMS.
|
||||
NotSet,
|
||||
/// Configured but unusable; SSE-S3 requests that need it are rejected.
|
||||
Invalid,
|
||||
}
|
||||
|
||||
impl LocalMasterKeyStatus {
|
||||
fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Loaded => "loaded",
|
||||
Self::NotSet => "not_set",
|
||||
Self::Invalid => "invalid",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// What [`init`] settled for this process.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct StartupReport {
|
||||
/// Whether SSE-KMS requests are authorized against the KMS key they name.
|
||||
pub key_policy_enforced: bool,
|
||||
/// State of the local SSE-S3 master key.
|
||||
pub local_master_key: LocalMasterKeyStatus,
|
||||
}
|
||||
|
||||
/// Settle the process-wide SSE configuration and report it.
|
||||
///
|
||||
/// Idempotent: the first call does the work and logs; later calls, such as a
|
||||
/// second embedded server in the same process, return the same report silently.
|
||||
/// An unusable master key does not fail startup, because a server whose KMS is
|
||||
/// running never needs it.
|
||||
pub fn init() -> StartupReport {
|
||||
static REPORT: OnceLock<StartupReport> = OnceLock::new();
|
||||
*REPORT.get_or_init(|| {
|
||||
let key_policy_enforced = key_policy_enforced();
|
||||
log_key_policy_mode(key_policy_enforced);
|
||||
|
||||
let local_master_key = load_local_master_key();
|
||||
log_local_master_key(&local_master_key);
|
||||
|
||||
StartupReport {
|
||||
key_policy_enforced,
|
||||
local_master_key: local_master_key.unwrap_or(LocalMasterKeyStatus::Invalid),
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// Whether per-key KMS authorization is enforced on the SSE-KMS data path.
|
||||
///
|
||||
/// Frozen on first read, by [`init`] or by the first request, so one process
|
||||
/// never answers the same question two ways.
|
||||
pub(crate) fn key_policy_enforced() -> bool {
|
||||
static ENFORCED: LazyLock<bool> =
|
||||
LazyLock::new(|| get_env_bool(ENV_RUSTFS_KMS_ENFORCE_SSE_KEY_POLICY, DEFAULT_KMS_ENFORCE_SSE_KEY_POLICY));
|
||||
*ENFORCED
|
||||
}
|
||||
|
||||
/// Load the local master key through the loader the request path uses.
|
||||
///
|
||||
/// An absent key is a state, not a failure: only a configured key is loaded, so
|
||||
/// the error case is exactly "configured but unusable".
|
||||
fn load_local_master_key() -> Result<LocalMasterKeyStatus, SseError> {
|
||||
if !local_master_key_configured() {
|
||||
return Ok(LocalMasterKeyStatus::NotSet);
|
||||
}
|
||||
get_local_sse_dek_provider().map(|_| LocalMasterKeyStatus::Loaded)
|
||||
}
|
||||
|
||||
/// Report the configured SSE-KMS authorization mode.
|
||||
///
|
||||
/// The disabled case warns rather than logs: while enforcement is off, any identity
|
||||
/// allowed to write an object can encrypt it under any key, and operators should hear
|
||||
/// about that even though disabled is the long-term default.
|
||||
fn log_key_policy_mode(enforced: bool) {
|
||||
if enforced {
|
||||
tracing::info!(
|
||||
event = EVENT_SSE_KMS_KEY_POLICY_MODE,
|
||||
component = LOG_COMPONENT_STORAGE,
|
||||
subsystem = LOG_SUBSYSTEM_SSE,
|
||||
enforced = true,
|
||||
"SSE-KMS requests are authorized against the resolved KMS key (kms:GenerateDataKey / kms:Decrypt)"
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
tracing::warn!(
|
||||
event = EVENT_SSE_KMS_KEY_POLICY_MODE,
|
||||
component = LOG_COMPONENT_STORAGE,
|
||||
subsystem = LOG_SUBSYSTEM_SSE,
|
||||
enforced = false,
|
||||
"SSE-KMS requests are not authorized against the KMS key they name; any identity allowed to \
|
||||
write an object may encrypt it under any key, and any identity allowed to read it may have it \
|
||||
decrypted. Grant kms:GenerateDataKey and kms:Decrypt on the keys your workloads use, then set \
|
||||
{ENV_RUSTFS_KMS_ENFORCE_SSE_KEY_POLICY}=true."
|
||||
);
|
||||
}
|
||||
|
||||
/// Report the local master key's state.
|
||||
///
|
||||
/// The failure text is the one a request is answered with. It describes the
|
||||
/// configuration problem and never carries the configured value.
|
||||
fn log_local_master_key(outcome: &Result<LocalMasterKeyStatus, SseError>) {
|
||||
match outcome {
|
||||
Ok(status @ LocalMasterKeyStatus::Loaded) => tracing::info!(
|
||||
event = EVENT_SSE_LOCAL_MASTER_KEY,
|
||||
component = LOG_COMPONENT_STORAGE,
|
||||
subsystem = LOG_SUBSYSTEM_SSE,
|
||||
state = status.as_str(),
|
||||
"{ENV_SSE_S3_MASTER_KEY} is loaded; SSE-S3 uses it whenever no KMS is running"
|
||||
),
|
||||
Ok(status) => tracing::info!(
|
||||
event = EVENT_SSE_LOCAL_MASTER_KEY,
|
||||
component = LOG_COMPONENT_STORAGE,
|
||||
subsystem = LOG_SUBSYSTEM_SSE,
|
||||
state = status.as_str(),
|
||||
"{ENV_SSE_S3_MASTER_KEY} is not set; SSE-S3 requires a running KMS"
|
||||
),
|
||||
Err(error) => tracing::error!(
|
||||
event = EVENT_SSE_LOCAL_MASTER_KEY,
|
||||
component = LOG_COMPONENT_STORAGE,
|
||||
subsystem = LOG_SUBSYSTEM_SSE,
|
||||
state = LocalMasterKeyStatus::Invalid.as_str(),
|
||||
reason = %error.message,
|
||||
"{ENV_SSE_S3_MASTER_KEY} is set but unusable; SSE-S3 requests are rejected while no KMS is running"
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::kms::provider::reset_sse_dek_provider;
|
||||
use crate::tests::lock_sse_test_state;
|
||||
use base64_simd::STANDARD as BASE64_STANDARD;
|
||||
use rustfs_s3_types::S3ErrorCode;
|
||||
use temp_env::with_vars;
|
||||
|
||||
/// Run `check` with the local master key set to `value` and nothing cached or injected.
|
||||
async fn with_master_key<T>(value: Option<&str>, check: impl FnOnce() -> T) -> T {
|
||||
let _guard = lock_sse_test_state().await;
|
||||
reset_sse_dek_provider();
|
||||
let outcome = with_vars([(ENV_SSE_S3_MASTER_KEY, value), ("__RUSTFS_SSE_SIMPLE_CMK", None)], check);
|
||||
reset_sse_dek_provider();
|
||||
outcome
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn an_absent_master_key_is_reported_as_not_set() {
|
||||
let status = with_master_key(None, load_local_master_key)
|
||||
.await
|
||||
.expect("an absent key is a state, not a failure");
|
||||
assert_eq!(status, LocalMasterKeyStatus::NotSet);
|
||||
|
||||
let blank = with_master_key(Some(" "), load_local_master_key)
|
||||
.await
|
||||
.expect("a blank value counts as absent");
|
||||
assert_eq!(blank, LocalMasterKeyStatus::NotSet);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_valid_master_key_is_loaded_and_cached_for_requests() {
|
||||
let encoded = BASE64_STANDARD.encode_to_string([7u8; 32]);
|
||||
let cached = with_master_key(Some(&encoded), || {
|
||||
let status = load_local_master_key().expect("a 32-byte key should load");
|
||||
assert_eq!(status, LocalMasterKeyStatus::Loaded);
|
||||
// The request path must find the provider startup loaded, not load a second one.
|
||||
crate::kms::provider::GLOBAL_SSE_DEK_PROVIDER
|
||||
.read()
|
||||
.expect("provider cache lock")
|
||||
.is_some()
|
||||
})
|
||||
.await;
|
||||
assert!(cached, "startup should leave the provider cached for the request path");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn an_unusable_master_key_fails_the_way_a_request_would() {
|
||||
let short = BASE64_STANDARD.encode_to_string([7u8; 16]);
|
||||
for unusable in ["not base64 at all!", short.as_str()] {
|
||||
let error = with_master_key(Some(unusable), load_local_master_key)
|
||||
.await
|
||||
.expect_err("an unusable key must not load");
|
||||
assert_eq!(error.code, S3ErrorCode::InvalidRequest);
|
||||
assert!(
|
||||
!error.message.contains(unusable),
|
||||
"the failure text must not carry the configured value: {}",
|
||||
error.message
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn init_reports_once_and_repeats_the_same_report() {
|
||||
let encoded = BASE64_STANDARD.encode_to_string([9u8; 32]);
|
||||
let (first, second) = with_master_key(Some(&encoded), || (init(), init())).await;
|
||||
|
||||
assert_eq!(first, second, "a second call must return the first report");
|
||||
assert_eq!(first.key_policy_enforced, key_policy_enforced());
|
||||
// The report is settled once per process, so this holds whether this test
|
||||
// or another caller of `init` ran first.
|
||||
assert_eq!(init(), first);
|
||||
}
|
||||
}
|
||||
@@ -1,70 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Fixtures for tests of SSE call sites, in this crate and in its dependents.
|
||||
|
||||
use crate::{KmsKeyAuthorizer, SseError, SseKmsPrincipal};
|
||||
use async_trait::async_trait;
|
||||
use base64_simd::STANDARD as BASE64_STANDARD;
|
||||
use rustfs_policy::policy::action::KmsAction;
|
||||
use rustfs_utils::http::headers::AMZ_ENCRYPTION_KMS;
|
||||
use rustfs_utils::http::object_encryption_keys::{
|
||||
INTERNAL_ENCRYPTION_ALGORITHM_HEADER, INTERNAL_ENCRYPTION_IV_HEADER, INTERNAL_ENCRYPTION_KEY_HEADER,
|
||||
INTERNAL_ENCRYPTION_KEY_ID_HEADER,
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
use std::sync::{Arc, Mutex, PoisonError};
|
||||
|
||||
/// An authorizer that answers every request the same way and records what it was asked.
|
||||
pub struct RecordingKmsKeyAuthorizer {
|
||||
allowed: bool,
|
||||
calls: Mutex<Vec<(KmsAction, String)>>,
|
||||
}
|
||||
|
||||
impl RecordingKmsKeyAuthorizer {
|
||||
pub fn new(allowed: bool) -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
allowed,
|
||||
calls: Mutex::new(Vec::new()),
|
||||
})
|
||||
}
|
||||
|
||||
/// The `(action, key id)` pairs this authorizer was asked about, in order.
|
||||
pub fn calls(&self) -> Vec<(KmsAction, String)> {
|
||||
self.calls.lock().unwrap_or_else(PoisonError::into_inner).clone()
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl KmsKeyAuthorizer for RecordingKmsKeyAuthorizer {
|
||||
async fn is_allowed(&self, _principal: &SseKmsPrincipal, action: KmsAction, key_id: &str) -> Result<bool, SseError> {
|
||||
self.calls
|
||||
.lock()
|
||||
.unwrap_or_else(PoisonError::into_inner)
|
||||
.push((action, key_id.to_string()));
|
||||
Ok(self.allowed)
|
||||
}
|
||||
}
|
||||
|
||||
/// Stored metadata of an SSE-KMS object wrapped under the key `finance-key`.
|
||||
pub fn sse_kms_object_metadata() -> HashMap<String, String> {
|
||||
HashMap::from([
|
||||
("x-amz-server-side-encryption".to_string(), AMZ_ENCRYPTION_KMS.to_string()),
|
||||
("x-amz-server-side-encryption-aws-kms-key-id".to_string(), "finance-key".to_string()),
|
||||
(INTERNAL_ENCRYPTION_KEY_ID_HEADER.to_string(), "finance-key".to_string()),
|
||||
(INTERNAL_ENCRYPTION_KEY_HEADER.to_string(), BASE64_STANDARD.encode_to_string([7u8; 48])),
|
||||
(INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode_to_string([9u8; 12])),
|
||||
(INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(), "aws:kms".to_string()),
|
||||
])
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -30,11 +30,9 @@ Contract crates stay below implementation crates. Forbidden edges:
|
||||
| `security-governance -> rustfs` | Governance contracts stay below the binary crate |
|
||||
| `extension-schema -> rustfs` | The extension schema is consumed by the binary, never the reverse |
|
||||
| `extension-schema -> ecstore` | The extension schema must not reach storage internals |
|
||||
| `sse -> ecstore`, `sse -> iam`, `sse -> rustfs` | SSE key resolution reaches the running server only through its `SseHost` port |
|
||||
|
||||
- `rustfs-storage-api` exposes storage-facing replication status/state contracts only through `crates/storage-api/src/replication.rs`, so its temporary dependency on `rustfs-filemeta` wire types stays centralized and no `rustfs-replication` / `rustfs-storage-api` cycle appears.
|
||||
- Leaf crates (`config`, `credentials`, `crypto`, `io-metrics`, `madmin`) may not depend on other `rustfs-*` crates, in either TOML spelling, except the adjudicated edges pinned in the guard's leaf allowlist: `io-metrics -> rustfs-s3-ops` (pure contract crates sharing the `S3Operation` vocabulary) and `madmin -> rustfs-signer` (the SigV4-signed admin SDK client). A new leaf exception must be a pure contract dependency (types and enums only, no I/O, no globals, no non-contract internal dependencies) and land together with its allowlist entry.
|
||||
- `rustfs-sse` sits between the S3 layer and the storage engine and depends on neither. Its internal dependencies are pinned in the same guard to `rustfs-config`, `rustfs-kms`, `rustfs-policy`, `rustfs-s3-types`, and `rustfs-utils`; the KMS service, a bucket's default encryption, and key-permission decisions arrive through `rustfs_sse::SseHost`, which `rustfs/src/storage/sse.rs` implements. That file also owns every translation across the engine boundary (`ObjectEncryptionResolver`, `WriteEncryption`, `MultipartCompletionAuthorization`), so neither side names the other's types.
|
||||
- Compile-time source reads follow the same direction: `include_str!` / `include!` of a `.rs` file must not resolve outside the including crate (`scripts/check_layer_dependencies.sh`). Shared source-text expectations belong in a contract surface such as `rustfs_protos::compat_manifest` (`crates/protos/src/compat_manifest.rs`) and are asserted by each owning crate.
|
||||
|
||||
## ECStore Access Boundary
|
||||
|
||||
@@ -47,7 +47,7 @@ RustFS-side statics that matter architecturally because other modules are tempte
|
||||
| `KEYSTONE_AUTH`, `KEYSTONE_MAPPER`, `KEYSTONE_CONFIG` | `rustfs/src/auth_keystone.rs` | Keystone provider, mapper, and config stay private to the Keystone owner. |
|
||||
| `DEADLOCK_DETECTOR` | `rustfs/src/storage/deadlock_detector.rs` | Detector lifecycle stays private to the storage deadlock detector. |
|
||||
| `CONCURRENCY_MANAGER` | `rustfs/src/storage/concurrency/manager.rs` | Storage concurrency scheduler state stays inside the concurrency owner. |
|
||||
| `GLOBAL_KMS_DEK_PROVIDER`, `GLOBAL_SSE_DEK_PROVIDER` | `crates/sse/src/kms/provider.rs` | DEK provider caches stay private to the SSE owner. |
|
||||
| `GLOBAL_KMS_DEK_PROVIDER`, `GLOBAL_SSE_DEK_PROVIDER` | `rustfs/src/storage/sse.rs` | DEK provider caches stay private to the SSE owner. |
|
||||
| `ECSTORE_EVENT_DISPATCH_HOOK` | `rustfs/src/server/event.rs` | Event bridge registration goes through the storage facade. |
|
||||
| `AUDIT_MODULE_ENABLED`, `NOTIFY_MODULE_ENABLED` | `rustfs/src/module_switches.rs` | Module toggles are read through module-switch helpers; `MODULE_SWITCH_RMW_LOCK` (`rustfs/src/server/module_switch.rs`) serializes persisted updates. |
|
||||
| `RUNTIME_CONFIG_RELOAD_MUTEX` | `rustfs/src/admin/service/config.rs` | Serializes dynamic config reload fanout. |
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# KMS Bulk Rekey Job Contract
|
||||
|
||||
**Use this when:** changing the bulk envelope re-wrap sweep (`rustfs/src/kms_rekey.rs`), its admin endpoints (`rustfs/src/admin/handlers/kms_rekey.rs`), the re-wrap primitive, or anything that decides which objects a rekey may touch.
|
||||
**Source of truth:** `rustfs/src/kms_rekey.rs`, `rustfs/src/admin/handlers/kms_rekey.rs`, `rewrap_object_encryption_metadata` in `crates/sse/src/core/rewrap.rs`, `KmsManager::rewrap_data_key` / `KmsManager::describe_data_key_wrapping` in `crates/kms/src/manager.rs`, `put_object_metadata` in `crates/ecstore/src/set_disk/ops/object.rs`.
|
||||
**Source of truth:** `rustfs/src/kms_rekey.rs`, `rustfs/src/admin/handlers/kms_rekey.rs`, `rewrap_object_encryption_metadata` in `rustfs/src/storage/sse.rs`, `KmsManager::rewrap_data_key` / `KmsManager::describe_data_key_wrapping` in `crates/kms/src/manager.rs`, `put_object_metadata` in `crates/ecstore/src/set_disk/ops/object.rs`.
|
||||
|
||||
The bulk rekey job re-wraps stored data-key envelopes under the current key-encryption key (KEK) without rewriting object bodies. This document is the acceptance bar; where the shipped v1 sweep deliberately narrows it, [Implementation Status](#implementation-status-v1-sweep) records the deviation.
|
||||
|
||||
@@ -52,7 +52,7 @@ Idempotency comes from object metadata itself: the envelope's state is the targe
|
||||
|
||||
### Reading the wrapping KEK version
|
||||
|
||||
There is no key-version metadata key. Object metadata carries the key id (`x-rustfs-encryption-key-id`) and the sealed blob under `x-rustfs-encryption-key`; `DecryptResponse` in `crates/kms/src/types.rs` does not report a version either. The version is recoverable because the sealed blob is structured: for every backend that builds one, the ciphertext is the JSON of `DataKeyEnvelope` (`crates/kms/src/encryption/dek.rs`), and the read path already discriminates on it via `is_data_key_envelope` in `crates/sse/src/core/read.rs`.
|
||||
There is no key-version metadata key. Object metadata carries the key id (`x-rustfs-encryption-key-id`) and the sealed blob under `x-rustfs-encryption-key`; `DecryptResponse` in `crates/kms/src/types.rs` does not report a version either. The version is recoverable because the sealed blob is structured: for every backend that builds one, the ciphertext is the JSON of `DataKeyEnvelope` (`crates/kms/src/encryption/dek.rs`), and the read path already discriminates on it via `is_data_key_envelope` in `rustfs/src/storage/sse.rs`.
|
||||
|
||||
| Backend | Rotates | Where the wrapping version lives | Recoverable by a scan |
|
||||
|---|---|---|---|
|
||||
@@ -95,7 +95,7 @@ Replication destinations are unresolved: propagation depends on `rustfs/backlog#
|
||||
|
||||
Three properties of `put_object_metadata` (`crates/ecstore/src/set_disk/ops/object.rs`) constrain the re-wrap write:
|
||||
|
||||
- **The merge is additive; it cannot remove keys.** Overwriting a key that keeps its name is safe; a re-wrap that changes *which* keys describe the envelope leaves the old keys behind. This is a live hazard: `parse_minio_managed_sealed_key` in `crates/sse/src/core/keying/object.rs` selects the MinIO decrypt branch on the mere presence of the MinIO seal-algorithm header, so a RustFS-native envelope written onto MinIO-compatible headers without neutralizing them steers reads down the stale branch. Any envelope-shape change must neutralize superseded keys in the same write.
|
||||
- **The merge is additive; it cannot remove keys.** Overwriting a key that keeps its name is safe; a re-wrap that changes *which* keys describe the envelope leaves the old keys behind. This is a live hazard: `parse_minio_managed_sealed_key` in `rustfs/src/storage/sse.rs` selects the MinIO decrypt branch on the mere presence of the MinIO seal-algorithm header, so a RustFS-native envelope written onto MinIO-compatible headers without neutralizing them steers reads down the stale branch. Any envelope-shape change must neutralize superseded keys in the same write.
|
||||
- **Object-lock retention is enforced before the merge.** `check_object_lock_retention_update` (`crates/ecstore/src/set_disk/mod.rs`) runs first; rekey inherits its decision and must not acquire a bypass.
|
||||
- **`mod_time` is preserved unless the caller sets it.** The re-wrap path leaves it unset so age-based lifecycle rules are not perturbed.
|
||||
|
||||
|
||||
@@ -1,13 +1,13 @@
|
||||
# MinIO On-Disk Format Interoperability
|
||||
|
||||
**Use this when:** deciding whether a MinIO drive set, bucket-metadata blob, or SSE object can be read or imported by a given RustFS build, or before touching any constant or codec listed under Version Anchors.
|
||||
**Source of truth:** `crates/filemeta/src/filemeta.rs`, `crates/filemeta/src/filemeta/codec.rs`, `crates/ecstore/src/bucket/metadata.rs`, `crates/ecstore/src/bucket/migration.rs`, `crates/sse/src/core/`, `crates/sse/src/kms/`, `rustfs/Cargo.toml` `[features]`, `.github/workflows/ci.yml`, `.github/workflows/minio-interop.yml`.
|
||||
**Source of truth:** `crates/filemeta/src/filemeta.rs`, `crates/filemeta/src/filemeta/codec.rs`, `crates/ecstore/src/bucket/metadata.rs`, `crates/ecstore/src/bucket/migration.rs`, `rustfs/src/storage/sse.rs`, `rustfs/Cargo.toml` `[features]`, `.github/workflows/ci.yml`, `.github/workflows/minio-interop.yml`.
|
||||
|
||||
This is an interop contract, not a plan. Migration is one-way (MinIO to RustFS). Erasure-coding internals are owned by [erasure-coding.md](erasure-coding.md); this document owns the interop claim, the fixture evidence, and the out-of-scope list.
|
||||
|
||||
## Scope Matrix By Build Variant
|
||||
|
||||
Build variants are the `rustfs` crate features in `rustfs/Cargo.toml`: `default`, `full`, and `rio-v2` (which enables `rustfs-ecstore/rio-v2` and `rustfs-sse/rio-v2`, and pulls in `crates/rio-v2`). `rio-v2` is absent from both `default` and `full`.
|
||||
Build variants are the `rustfs` crate features in `rustfs/Cargo.toml`: `default`, `full`, and `rio-v2` (which enables `rustfs-ecstore/rio-v2` and pulls in `crates/rio-v2`). `rio-v2` is absent from both `default` and `full`.
|
||||
|
||||
| MinIO artifact | `default` / `full` build | `rio-v2` build | Notes |
|
||||
|---|:--:|:--:|---|
|
||||
@@ -81,7 +81,7 @@ The `xl.meta` around a MinIO SSE object parses in every build, so such objects l
|
||||
|
||||
### Seams
|
||||
|
||||
The cryptography (DARE v2 stream format, object-key derivation, sealing) was never the gap. Three metadata seams above it rejected MinIO objects; all are closed in `rio-v2` builds. Symbols are under `crates/sse/src/core/` unless noted.
|
||||
The cryptography (DARE v2 stream format, object-key derivation, sealing) was never the gap. Three metadata seams above it rejected MinIO objects; all are closed in `rio-v2` builds. Symbols are in `rustfs/src/storage/sse.rs` unless noted.
|
||||
|
||||
| Seam | Resolution |
|
||||
|---|---|
|
||||
@@ -113,10 +113,10 @@ Inventory the source first: bucket default encryption means objects can be encry
|
||||
| Fact | Value |
|
||||
|---|---|
|
||||
| Shipping status | Ships in no default build: absent from `default` and `full` in `rustfs/Cargo.toml`; released binaries and container images never include it. Enable with `--features rio-v2`. |
|
||||
| Pull-request coverage | `test-and-lint-rio-v2` in `.github/workflows/ci.yml`: clippy plus `cargo nextest` for `rustfs`, `rustfs-ecstore`, and `rustfs-sse` with `--features rio-v2`. This is the cfg-seam guard; it keeps the feature compiling and its unit suite green on every pull request. |
|
||||
| Pull-request coverage | `test-and-lint-rio-v2` in `.github/workflows/ci.yml`: clippy plus `cargo nextest` for `rustfs` and `rustfs-ecstore` with `--features rio-v2`. This is the cfg-seam guard; it keeps the feature compiling and its unit suite green on every pull request. |
|
||||
| Full-suite lane | `build-rustfs-debug-binary-rio-v2` and `e2e-tests-rio-v2` in `ci.yml` run only on the nightly schedule and manual dispatch (`cache-warm.yml` keeps the `ci-feat-rio` cache warm so the scheduled build fits its timeout). |
|
||||
| Interop evidence | `.github/workflows/minio-interop.yml` (nightly plus manual) regenerates real MinIO backend trees via `crates/rio-v2/tests/minio_fixture_lab/` and runs the `#[ignore]` reader tests in `rustfs/src/storage/minio_generated_read_test.rs` with `--features rio-v2`. Its freshness is tracked in `.github/scheduled-validations.json`. |
|
||||
| Promote-or-delete condition | The variant stays dormant until one of two things happens. **Promote**: a release commits to shipping MinIO SSE migration as a supported capability; then `rio-v2` joins `default`/`full`, the scheduled lanes run on every pull request, and this section is rewritten. **Delete**: no release commits to it and the scheduled lanes are not kept green; then the feature flag, `crates/rio-v2`, the cfg seams under `crates/sse/src/`, the three `ci.yml` jobs, the `cache-warm.yml` warm step, `minio-interop.yml`, and its `scheduled-validations.json` entry are removed in one change. Either outcome must update `ARCHITECTURE.md` and the `ci.yml` job comments that cite this section. |
|
||||
| Promote-or-delete condition | The variant stays dormant until one of two things happens. **Promote**: a release commits to shipping MinIO SSE migration as a supported capability; then `rio-v2` joins `default`/`full`, the scheduled lanes run on every pull request, and this section is rewritten. **Delete**: no release commits to it and the scheduled lanes are not kept green; then the feature flag, `crates/rio-v2`, the cfg seams in `rustfs/src/storage/sse.rs`, the three `ci.yml` jobs, the `cache-warm.yml` warm step, `minio-interop.yml`, and its `scheduled-validations.json` entry are removed in one change. Either outcome must update `ARCHITECTURE.md` and the `ci.yml` job comments that cite this section. |
|
||||
|
||||
## Fixture Evidence
|
||||
|
||||
|
||||
@@ -15,7 +15,7 @@ An unchanged interval requires matching owner, process session, watch token, buc
|
||||
|
||||
Watch renewal keeps the original starting revision. A caller may extend its conservative local monotonic deadline after a successful renewal, but it must not replace the original proof with the renewed revision. A persisted checkpoint stores identity and revision, not a usable `Instant`; after restart, the old producer token must be checked again against the complete current owner set.
|
||||
|
||||
One keeper belongs to each leader-local cohort, with weak store and cohort references. It revisits retained watches every sixty seconds, with four concurrent queries and a five-second total limit per keepalive query. At the 256-watch limit one pass is bounded by 380 seconds, below the ten-minute producer lease. Retained checkpoints and finished sources awaiting root publication and ACK keep their original proof alive. A subsequent cycle may restart a cancelled keeper for the same instance, but cannot reset the original token, revision, or expired deadline.
|
||||
One keeper belongs to each service cohort, with weak store and cohort references. After a cycle deadline the scanner supervisor retains its cohort only when the worker has stopped, cycle state has been persisted, and the epoch has been fenced. A successful successor claim on the same storage instance can resume the cohort only from the exact fenced revision and immediately following epoch. An intervening claim, lock contention, or recovery drops the continuation. The cohort preserves its bounded watch renewals and scheduling order across the retry interval. The new leader needs its own fences for publication and dirty acknowledgements. The keeper revisits retained watches every sixty seconds, with four concurrent queries and a five-second total limit per keepalive query. At the 256-watch limit one pass is bounded by 380 seconds, below the ten-minute producer lease. Retained checkpoints and finished sources awaiting root publication and ACK keep their original proof alive. A subsequent cycle may restart a cancelled keeper for the same instance, but cannot reset the original token, revision, or expired deadline.
|
||||
|
||||
## Consumer and persistence order
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# AWS KMS backend: setup, verification, and operations
|
||||
|
||||
**Use this when:** connecting RustFS to AWS KMS for the first time, verifying an existing AWS-backed deployment, upgrading one from 1.0.4 or 1.0.5, or deciding how to run and protect the AWS keys RustFS depends on.
|
||||
**Source of truth:** `crates/kms/src/backends/aws.rs` (backend behavior and its deviations), `crates/kms/src/config.rs` (`ENV_KMS_AWS_*`), `crates/kms/src/policy.rs` (retry, deadline and circuit breaker), `crates/sse/src/core/` (SSE data path). 中文版: [kms-aws-backend-guide.zh-CN.md](kms-aws-backend-guide.zh-CN.md).
|
||||
**Source of truth:** `crates/kms/src/backends/aws.rs` (backend behavior and its deviations), `crates/kms/src/config.rs` (`ENV_KMS_AWS_*`), `crates/kms/src/policy.rs` (retry, deadline and circuit breaker), `rustfs/src/storage/sse.rs` (SSE data path). 中文版: [kms-aws-backend-guide.zh-CN.md](kms-aws-backend-guide.zh-CN.md).
|
||||
|
||||
The statements about behavior on this page were observed against the real AWS KMS service (region `eu-north-1`) and, for failures AWS cannot be asked to produce on demand, against a loopback stand-in that speaks the AWS KMS wire protocol. Where a statement is guidance that was not exercised, it says so. For the contract-level differences between AWS and the other backends, see [KMS backend security properties](kms-backend-security.md#aws-kms-deviations-from-the-shared-backend-contract).
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# AWS KMS 后端:接入、验证与运维
|
||||
|
||||
**适用场景:** 首次把 RustFS 接入 AWS KMS、验证已有的 AWS 后端部署、从 1.0.4 或 1.0.5 升级,或者规划如何运维并保护 RustFS 所依赖的 AWS 密钥。
|
||||
**事实来源:** `crates/kms/src/backends/aws.rs`(后端行为及其差异)、`crates/kms/src/config.rs`(`ENV_KMS_AWS_*`)、`crates/kms/src/policy.rs`(重试、超时与熔断)、`crates/sse/src/core/`(SSE 数据面)。English: [kms-aws-backend-guide.md](kms-aws-backend-guide.md)。
|
||||
**事实来源:** `crates/kms/src/backends/aws.rs`(后端行为及其差异)、`crates/kms/src/config.rs`(`ENV_KMS_AWS_*`)、`crates/kms/src/policy.rs`(重试、超时与熔断)、`rustfs/src/storage/sse.rs`(SSE 数据面)。English: [kms-aws-backend-guide.md](kms-aws-backend-guide.md)。
|
||||
|
||||
本页关于行为的描述,均来自对真实 AWS KMS 服务(区域 `eu-north-1`)的实测;对于无法让 AWS 按需产生的故障,则使用一个实现了 AWS KMS 线上协议的本机替身来实测。属于未经实测的建议之处,文中会明确标注。AWS 与其它后端在契约层面的差异,见 [KMS backend security properties](kms-backend-security.md#aws-kms-deviations-from-the-shared-backend-contract)。
|
||||
|
||||
|
||||
@@ -82,7 +82,13 @@ The startup usage preflight is read-only and has a 30-second deadline. If it can
|
||||
|
||||
Cold-cache startup and startup without active replication avoid an extra bucket listing. Maintenance discovery keeps looking for replication after finding lifecycle rules in an earlier bucket. Once replication is found, both scheduling decisions are settled and remaining buckets need no startup inspection. Reapplying an unchanged resolved configuration does not reconfigure per-disk budgets or emit another configuration wakeup.
|
||||
|
||||
The `cycle_max_duration`, `cycle_max_objects`, and `cycle_max_directories` controls bound a cycle's work; they do not guarantee that every budget can complete a sweep in bounded chunks. A partial scan preserves useful checkpoints and coverage, but reopening ancestors and revalidating directory entries costs work again. A budget below the minimum replay or traversal cost can repeatedly stop at the same directory. Cross-bucket fairness also needs to be distinguished from progress within one bucket. Use pacing controls to limit background I/O, and choose cycle budgets large enough for the namespace to advance.
|
||||
The `cycle_max_duration`, `cycle_max_objects` and `cycle_max_directories` settings also apply to resumed walks. For a stable wide directory, a checkpoint records which child names have been discovered, which children have finished and the usage accumulated from those children. The next executor can restore that work after a cycle deadline or process restart. Objects from a child that was interrupted before completion are accounted for when that child finishes.
|
||||
|
||||
On Linux and macOS, the scanner can reopen a partially enumerated physical directory and verify its last saved position against the directory generation and actual entries. Each physical replica has its own position. A replacement or changed directory requires enumeration again. When an executor moves to another disk, it advances that disk's own checkpoint. Returning to the first disk resumes its saved subtotal. This keeps sizes and version counts tied to the replica whose object metadata was read. Other platforms can reuse a live directory handle and already completed work on an identifiable disk. An interrupted native directory read may need to start again after a process restart.
|
||||
|
||||
After an upgrade, an older partial traversal without a recorded physical disk starts a fresh walk. This preserves the existing published usage snapshot when the partial checkpoint is discarded. To verify a handoff, enable debug logging for `rustfs_scanner::scanner_folder::directory_cursor` and look for `state="directory_checkpoint_restored"`. The event identifies the bucket, directory, physical disk, leader epoch and cycle. `restore_mode="native_position"` records how many entries had already been read. `restore_mode="sealed_membership"` records reuse of a fully enumerated directory and includes `completed_through` when the checkpoint contains a compacted subtotal.
|
||||
|
||||
The bucket usage API can return a carried value or a newer observation while a walk is incomplete. Check `usage_value_state` and `usage_certification` to identify the last certified value. If writes overlap a resumed walk, reaching the end can produce an observation that still needs a fresh sweep before certification. Progress requires enough budget for individual I/O operations and a successful checkpoint save.
|
||||
|
||||
Budget cancellation alone does not publish complete usage or acknowledge dirty buckets. A partial cycle must retain that uncertainty; a growing directories-scanned counter is not proof of new coverage or published totals. Inspect checkpoint/frontier changes and publication status across attempts before changing budgets. A budget cancellation does not persist a disabled scheduler flag and does not justify deleting scanner state.
|
||||
|
||||
@@ -230,7 +236,7 @@ Only `xl.meta` files enter object metadata accounting. Other payload files consu
|
||||
|
||||
UUID candidates under a directory without `xl.meta` are classified during their normal directory traversal. Each directory open and entry is charged once; traversal yields every 64 entries and continues under the budget until a regular `part.N` is found or EOF is reached. A full batch is not evidence of absence. Candidate metadata accounting is deferred until classification finishes. A data directory retains the existing missing-parent-metadata and non-destructive heal path; a namespace continues from the same listing, without a second probe/open/entry pass. Data directories do not consume namespace-directory quota or a preset folder delay; a candidate proven to be a namespace consumes that quota and delay once.
|
||||
|
||||
Candidate entries use the normal raw-enumeration progress and checkpoint path. Cancellation or an iteration error leaves classification incomplete, never an EOF or successful coverage result. A child already absent or replaced by a non-directory at open is handled as vanished, without stale namespace totals or a missing-metadata heal request. A replacement retains the existing coverage gap; mid-iteration errors never become EOF. A clean namespace covered by a restored frontier reuses its prior totals only after current classification reaches EOF; metadata is not read or charged to object quota again. A later data candidate prevents completed namespace siblings from certifying the unresolved parent. Existing raw-page indexes are membership receipts, not portable filesystem seek offsets: a later cycle or process restart reopens and revalidates the directory, with actual reads charged again. This does not guarantee that a directory larger than the configured cycle budget can finish within that budget.
|
||||
Candidate entries use the normal raw-enumeration progress and checkpoint path. Cancellation or an iteration error leaves classification incomplete, never an EOF or successful coverage result. A child already absent or replaced by a non-directory at open is handled as vanished, without stale namespace totals or a missing-metadata heal request. A replacement retains the existing coverage gap; mid-iteration errors never become EOF. A clean namespace covered by a restored frontier reuses its prior totals only after current classification reaches EOF; metadata is not read or charged to object quota again. A later data candidate prevents completed namespace siblings from certifying the unresolved parent. The disk worker parks the physical enumeration between turns and checks directory identity and modification state before reuse. A cold process reopens the directory and pays for its physical reads again. Raw-page indexes identify visited names; the scanner still requires EOF, complete child accounting and a valid publication fence before certifying totals.
|
||||
|
||||
- Media is detected once per disk path from the backing block device's `rotational` flag on Linux, through partitions, `dm-*`, and software RAID; any spindle in the stack counts as rotational. Undetectable media, including non-Linux hosts and overlay filesystems without a bind-mounted volume, uses the rotational defaults because a spindle is the case that needs protection.
|
||||
- `auto` resolves to `100` operations/s and `16 MiB/s` on rotational or unknown media and to `4000` operations/s and `256 MiB/s` on flash. The rotational values leave roughly half of a 7200 rpm spindle's random reads to foreground traffic; the flash values are a few percent of a SATA SSD. The `fastest` speed preset resolves `auto` to no cap; explicit numbers always apply.
|
||||
@@ -313,6 +319,7 @@ metrics.maintenance_control.primary_control
|
||||
metrics.source_work
|
||||
metrics.replication_repair
|
||||
metrics.scan_checkpoint
|
||||
metrics.last_cycle_partial_stage
|
||||
metrics.cycle_timeout_total
|
||||
metrics.cycle_last_progress_age
|
||||
metrics.leader_lease_without_progress
|
||||
@@ -335,6 +342,8 @@ catch_up_estimate.discovered_expiry_items
|
||||
catch_up_estimate.discovered_transition_items
|
||||
```
|
||||
|
||||
For a partial cycle, `metrics.last_cycle_partial_stage` records the blocking check for that cycle. For example, `entry_budget` counts raw directory enumeration, `source_result_missing` means an expected set supplied no result, and `activity_baseline_unverified` means the activity proof could not certify publication. A `mixed_plan_recertification_pending` stage means every outstanding bucket completed traversal and supplied a fresh cross-plan observation that still requires certification. Candidate rejections retain the existing candidate outcome labels such as `invalid_root` and `tier_proof`. The field clears on the next non-partial cycle outcome. Inspect `metrics.usage_publication` for the outcome of writing the root snapshot. When a cycle ends with incomplete coverage or an unverifiable observation, the terminal error event includes the cycle, leader epoch and blocking stage. Budget stops and saved checkpoints remain visible in scanner status and warning logs. A supervisor deadline that interrupts the worker before a terminal result increments `metrics.cycle_timeout_total`.
|
||||
|
||||
### Deep scan cadence state
|
||||
|
||||
`bitrot` reports how the scanner decided the scan mode of the running or most recent cycle:
|
||||
@@ -361,6 +370,8 @@ A resumed walk retains its previous durable coverage while initial enumeration h
|
||||
|
||||
Shared compacted accumulators defer periodic emission until the owning subtree completes; replay inside that compacted subtree remains possible after an interruption. When bucket checkpoint preparation returns `Reset` for existing resume state, it logs `checkpoint_reset` with the validation reason. The `scan_checkpoint_used` counter includes validated coverage adoption by the forward sweep.
|
||||
|
||||
Fenced per-bucket checkpoints keep small payloads inline. Large checkpoints stage immutable, checksum-addressed pages and then conditionally replace a small manifest. The primary and backup use separate page objects. Interruption before the manifest write leaves the prior checkpoint readable. A staging marker identifies a new page bundle before any pages are written. If a later commit uses an inline checkpoint or a different bundle, that commit records the abandoned bundle for fenced cleanup, which a later save can retry after a restart. Cold recovery uses background disk admission, verifies the referenced pages, and can carry verified restore progress across local time slices. A reader that predates the page format can rebuild a partial checkpoint and can read compact complete totals when the compatibility projection fits the manifest.
|
||||
|
||||
A confirmed periodic save does not prove restart resumability. Adoption still validates bucket/set identity and, for non-empty caches, connected cache scope. Skipping completed subtrees additionally requires a validated coverage frontier; raw enumeration state alone does not prove completed subtree coverage.
|
||||
|
||||
### Cycle recovery on followers
|
||||
@@ -936,7 +947,7 @@ For a mostly idle single-node, single-disk deployment with sustained CPU usage w
|
||||
6. If `primary_pressure` is `throttle_pause` and pause ratios are low, raise `scanner.delay` first.
|
||||
7. If individual sleeps are too short, raise `scanner.max_wait`.
|
||||
8. If each scan cycle finishes but starts too often, raise `scanner.cycle`.
|
||||
9. To bound one cycle, set `scanner.cycle_max_duration`, `scanner.cycle_max_objects`, or `scanner.cycle_max_directories`; confirm that checkpoints advance under that budget. These controls do not guarantee resumable bounded chunks for every directory shape; see [Startup and cycle budgets](#startup-and-cycle-budgets).
|
||||
9. To bound one cycle, set `scanner.cycle_max_duration`, `scanner.cycle_max_objects`, or `scanner.cycle_max_directories`; confirm that checkpoints advance under that budget. For the conditions that permit a wide directory to advance across turns and process restarts, see [Startup and cycle budgets](#startup-and-cycle-budgets).
|
||||
10. Recheck `pacing_pressure`, `maintenance_control`, source work, and lifecycle transition status after one or more scanner cycles.
|
||||
|
||||
Do not rely only on a longer cycle interval if lifecycle, replication, heal, or bitrot work must keep moving; use source work and transition status to confirm that background maintenance still progresses.
|
||||
|
||||
@@ -64,7 +64,7 @@ A coverage receipt binds the completed traversal frontier to its scope, starting
|
||||
|
||||
The Normal-to-Deep regression holds the mutation plan fixed, changes metadata in a previously visited prefix, and verifies that the real Deep disk-scan entry point reads that prefix again. It also checks that a complete Normal cache cannot satisfy the Deep `Current` path. The fixture disables heal side effects; it proves traversal re-entry, not actual bitrot detection or repair. Additional tests cover map round-trips, stale identities, existing-but-uncovered cursors, coverage gaps, and per-instance metadata ownership.
|
||||
|
||||
This fixture bounds object processing after directory enumeration. It does not prove fixed-budget enumeration of arbitrarily wide directories or real process-restart convergence. Those gates require a storage-owned resumable enumeration capability, including its initial construction cost; a readdir offset, an in-memory iterator or a last-name filter is not that capability.
|
||||
The independent `wide_directory_runtime_budget_reaches_eof` regression charges virtual time for every physical directory read, including replay and EOF, and checks exact object and byte totals across capped rounds. The `paged_checkpoint` tests exercise immutable page reuse, interrupted root commits, stale writers and cold restoration. The process-level regression in [the E2E crate](../../crates/e2e_test/src/scanner_wide_directory_test.rs) seeds objects through S3, restarts the server after a partial save, and checks the certified admin usage against a paginated S3 listing using the same scanner budget on both boots.
|
||||
|
||||
For every saved partial cache, the fixture also passes its progress through the production authenticated remote terminal-frame writer and stream consumer. A remote partial result must remain partial even when its progress reports visited objects. This covers the return-frame contract; it does not execute the remote RPC server, distributed locks, EC quorum persistence, mixed-version peers, process crashes, or fsync durability. The file backend models revision preconditions and persistence errors, not a concurrent object store.
|
||||
|
||||
|
||||
+1
-3
@@ -80,7 +80,7 @@ e2e-test-hooks = ["rustfs-ecstore/e2e-test-hooks", "rustfs-scanner/e2e-test-hook
|
||||
connect-e2e-short-credentials = []
|
||||
# Builds the dedicated rustfs-cli-e2e target with a build-time public enrollment root.
|
||||
offline-enrollment-e2e-root = []
|
||||
rio-v2 = ["rustfs-ecstore/rio-v2", "rustfs-sse/rio-v2"]
|
||||
rio-v2 = ["rustfs-ecstore/rio-v2"]
|
||||
# Linux-only cuObject S3-over-RDMA GET into client (GPU) memory; needs libcuobjserver.
|
||||
cuobject = ["dep:rustfs-cuobject", "rustfs-cuobject/server"]
|
||||
# NIXL/UCX transport for S3-over-RDMA; replaces cuobject when both are enabled.
|
||||
@@ -281,7 +281,6 @@ rustfs-rio = { workspace = true }
|
||||
rustfs-s3-client = { workspace = true }
|
||||
rustfs-s3-types = { workspace = true, features = ["compat-s3s"] }
|
||||
rustfs-s3-ops = { workspace = true }
|
||||
rustfs-sse = { workspace = true }
|
||||
rustfs-security-governance = { workspace = true }
|
||||
rustfs-data-usage = { workspace = true }
|
||||
rustfs-s3select-api = { workspace = true }
|
||||
@@ -425,7 +424,6 @@ h2.workspace = true
|
||||
rustfs-targets = { workspace = true, features = ["test-support"] }
|
||||
uuid = { workspace = true, features = ["v4", "v5", "fast-rng", "macro-diagnostics"] }
|
||||
rustfs-iam = { workspace = true, features = ["test-util"] }
|
||||
rustfs-sse = { workspace = true, features = ["test-util"] }
|
||||
serial_test = { workspace = true }
|
||||
tempfile = { workspace = true }
|
||||
aws-config = { workspace = true }
|
||||
|
||||
@@ -62,7 +62,7 @@ use super::storage_api::multipart_usecase::s3_api::multipart::{
|
||||
};
|
||||
use super::storage_api::multipart_usecase::set_disk::is_valid_storage_class;
|
||||
use super::storage_api::multipart_usecase::sse::{
|
||||
DecryptionRequest, EncryptionKeyKind, EncryptionRequest, PrepareEncryptionRequest, SseKmsPrincipal, SseKmsPrincipalExt,
|
||||
DecryptionRequest, EncryptionKeyKind, EncryptionRequest, PrepareEncryptionRequest, SseKmsPrincipal,
|
||||
apply_bucket_default_lock_retention, authorize_sse_kms_multipart_write, authorize_sse_kms_object_read,
|
||||
build_ssec_read_headers, encryption_material_to_metadata, extract_server_side_encryption_from_headers,
|
||||
extract_ssec_params_from_headers, extract_ssekms_context_from_headers, get_buffer_size_opt_in,
|
||||
|
||||
@@ -1357,7 +1357,58 @@ mod tests {
|
||||
part.e_tag = Some(ETag::Strong("guessed-etag".to_string()));
|
||||
part.checksum_sha256 = Some("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=".to_string());
|
||||
}
|
||||
let result = usecase.complete_for_s3s_request(build_request(input, Method::POST)).await;
|
||||
use rustfs_gateway_types::compat::s3s_5761ddfe::generated::ops::complete_multipart_upload;
|
||||
|
||||
let mut s3s_input = input;
|
||||
let has_upload = s3s_input.multipart_upload.is_some();
|
||||
s3s_input.multipart_upload.get_or_insert_with(Default::default);
|
||||
let tags: Vec<Option<String>> = s3s_input
|
||||
.multipart_upload
|
||||
.iter_mut()
|
||||
.flat_map(|upload| upload.parts.iter_mut().flatten())
|
||||
.map(|part| {
|
||||
part.part_number.get_or_insert(0);
|
||||
part.e_tag.take().map(|tag| tag.value().to_owned())
|
||||
})
|
||||
.collect();
|
||||
for checksum in [
|
||||
&mut s3s_input.checksum_crc32,
|
||||
&mut s3s_input.checksum_crc32c,
|
||||
&mut s3s_input.checksum_crc64nvme,
|
||||
&mut s3s_input.checksum_md5,
|
||||
&mut s3s_input.checksum_sha1,
|
||||
&mut s3s_input.checksum_sha256,
|
||||
&mut s3s_input.checksum_sha512,
|
||||
&mut s3s_input.checksum_xxhash128,
|
||||
&mut s3s_input.checksum_xxhash3,
|
||||
&mut s3s_input.checksum_xxhash64,
|
||||
] {
|
||||
*checksum = None;
|
||||
}
|
||||
let result = match complete_multipart_upload::input_from_s3s(s3s_input) {
|
||||
Err(error) => Err(crate::app::dto_values::seam_error(error)),
|
||||
Ok(mut owned) => {
|
||||
for (part, tag) in owned.multipart_upload.parts.iter_mut().zip(tags) {
|
||||
part.e_tag = tag.map(rustfs_gateway_types::ETag::from_stored);
|
||||
}
|
||||
let taken = std::mem::take(&mut owned.multipart_upload);
|
||||
let parts = has_upload.then_some(taken);
|
||||
match parts {
|
||||
Some(parts) => {
|
||||
owned.multipart_upload = parts;
|
||||
Box::pin(usecase.execute_complete_multipart_upload(owned_envelope(Method::POST), owned)).await
|
||||
}
|
||||
None => {
|
||||
Box::pin(usecase.execute_complete_multipart_upload_parts(
|
||||
owned_envelope(Method::POST),
|
||||
owned,
|
||||
None,
|
||||
))
|
||||
.await
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
if keyed {
|
||||
let response = result.expect("verified key completes and replays");
|
||||
assert!(response.checksum_sha256.is_some(), "verified key retains checksum response");
|
||||
|
||||
@@ -382,8 +382,7 @@ impl DefaultMultipartUsecase {
|
||||
content_size: actual_size,
|
||||
principal: None,
|
||||
}
|
||||
.validate_multipart_ssec(&fi.user_defined)
|
||||
.map_err(ApiError::from)?;
|
||||
.validate_multipart_ssec(&fi.user_defined)?;
|
||||
}
|
||||
let (requested_sse, requested_kms_key_id) = if has_ssec {
|
||||
let ssec_material = sse_decryption(DecryptionRequest {
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user