mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-22 16:02:46 +00:00
Compare commits
73 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| e6435510f8 | |||
| 5696c6f759 | |||
| 7c3732799e | |||
| adad36dc3b | |||
| 83afcf52f7 | |||
| bb044df449 | |||
| 3565ae1ddf | |||
| cce3b0873d | |||
| 6bb25182c9 | |||
| ceb8f49aae | |||
| 33aae31cb7 | |||
| f8924deb68 | |||
| 9e8c426e6f | |||
| 9815b67c51 | |||
| 496a34a6a7 | |||
| 7c9ebadffe | |||
| e376e7d774 | |||
| d211df88d5 | |||
| 21caf8bf34 | |||
| d3eaaa21a6 | |||
| ad457a2d1c | |||
| fa3ea5b4e5 | |||
| f3396952b6 | |||
| 8736b49d08 | |||
| eec468e488 | |||
| f2af45ee68 | |||
| 25b55febc5 | |||
| c2e50b3b10 | |||
| 6fde592da1 | |||
| 98259d7c46 | |||
| b5a35fc2c7 | |||
| 11a7572bba | |||
| 4c4dc84671 | |||
| 52986eefd5 | |||
| 7a5ffcfee7 | |||
| 334d615a67 | |||
| 0c813e1a9e | |||
| 0de0b85374 | |||
| 78f78ce5ee | |||
| 82ad3f0007 | |||
| 82f094f4d5 | |||
| e1df1d569b | |||
| 19487a6e34 | |||
| 6177fc280f | |||
| 14db51bba7 | |||
| 353de54fcc | |||
| 6a4f93bbaf | |||
| 74fab85ba0 | |||
| 44db2be5f9 | |||
| 3e41233589 | |||
| f36a771389 | |||
| 0447e68073 | |||
| 94dbca3840 | |||
| d2939fb43b | |||
| 9f7d1aa706 | |||
| 98675ddcbd | |||
| 357e78c523 | |||
| 411caa97d7 | |||
| 317e895096 | |||
| 7c55e265dd | |||
| f858a65f41 | |||
| f96c324734 | |||
| 7e5b3e4137 | |||
| c749351780 | |||
| 2381096448 | |||
| 99a1803773 | |||
| 5132be9c01 | |||
| 13399d98fa | |||
| ee3880ee84 | |||
| af48bb8539 | |||
| 850d7a2beb | |||
| 1acf30910f | |||
| 26757d5d5e |
@@ -57,6 +57,7 @@ rustfs-scanner 1392 20
|
||||
rustfs-scanner-metrics 90 0
|
||||
rustfs-security-governance 20 0
|
||||
rustfs-signer 34 0
|
||||
rustfs-sse 145 0
|
||||
rustfs-storage-api 34 0
|
||||
rustfs-targets 612 26
|
||||
rustfs-test-utils 1 0
|
||||
|
||||
@@ -231,7 +231,7 @@ jobs:
|
||||
# --all-targets links the same test binaries as the reader lane.
|
||||
CARGO_BUILD_JOBS: "2"
|
||||
run: |
|
||||
cargo build -p rustfs -p rustfs-ecstore --all-targets --features rio-v2
|
||||
cargo build -p rustfs -p rustfs-ecstore -p rustfs-sse --all-targets --features rio-v2
|
||||
cargo build -p rustfs --bins --features rio-v2,e2e-test-hooks
|
||||
|
||||
# Readers: the swift and sftp legs of test-and-lint-protocols. Built in
|
||||
|
||||
@@ -563,7 +563,7 @@ jobs:
|
||||
run: chmod go-w "$(realpath "$HOME")"
|
||||
|
||||
- name: Run rio-v2 clippy lints
|
||||
run: cargo clippy -p rustfs -p rustfs-ecstore --all-targets --features rio-v2 -- -D warnings
|
||||
run: cargo clippy -p rustfs -p rustfs-ecstore -p rustfs-sse --all-targets --features rio-v2 -- -D warnings
|
||||
|
||||
- name: Run rio-v2 feature tests
|
||||
shell: bash
|
||||
@@ -580,7 +580,7 @@ jobs:
|
||||
rm -f target/nextest/ci/junit.xml
|
||||
./scripts/ci/resource_sampler.sh start nextest-rio-v2
|
||||
trap './scripts/ci/resource_sampler.sh stop' EXIT
|
||||
cargo nextest run --profile ci -p rustfs -p rustfs-ecstore --features rio-v2 \
|
||||
cargo nextest run --profile ci -p rustfs -p rustfs-ecstore -p rustfs-sse --features rio-v2 \
|
||||
2>&1 | tee artifacts/workspace/nextest.log
|
||||
cargo test -p rustfs --doc --features rio-v2
|
||||
|
||||
|
||||
+1
-1
@@ -94,7 +94,7 @@ refactors.
|
||||
|--------|--------------------------|----------------|
|
||||
| Foundation | `checksums`, `common`, `config`, `data-usage`, `heal-contracts`, `scanner-metrics`, `utils` | Shared configuration, data-usage models, heal domain contracts, scanner telemetry types, utilities, and checksums. |
|
||||
| I/O and storage | `concurrency`, `ecstore`, `filemeta`, `heal`, `io-core`, `io-metrics`, `lifecycle`, `lock`, `object-capacity`, `object-data-cache`, `replication`, `rio`, `rio-v2`, `s3-client`, `scanner`, `storage-api` | Erasure-coded object storage, metadata, recovery, lifecycle, replication, locking, cache, I/O pipelines, and the engine-side S3 client for remote tier/transition targets. |
|
||||
| Security and identity | `credentials`, `crypto`, `iam`, `keystone`, `kms`, `policy`, `security-governance`, `signer`, `tls-runtime`, `trusted-proxies` | Credentials, authentication, authorization, encryption, key management, TLS, and security contracts. |
|
||||
| Security and identity | `credentials`, `crypto`, `iam`, `keystone`, `kms`, `policy`, `security-governance`, `signer`, `sse`, `tls-runtime`, `trusted-proxies` | Credentials, authentication, authorization, encryption, key management, server-side encryption key resolution, TLS, and security contracts. |
|
||||
| Protocols and contracts | `extension-schema`, `madmin`, `protos`, `protocols`, `s3-ops`, `s3-types`, `s3select-api`, `s3select-query` | Admin, inter-node, S3, S3 Select, and optional protocol contracts. |
|
||||
| Operations and integration | `audit`, `notify`, `obs`, `targets`, `zip` | Auditing, observability, event delivery, notification targets, and archive support. |
|
||||
| Test support | `e2e_test`, `test-utils` | End-to-end validation and shared test bootstrap utilities. |
|
||||
|
||||
@@ -116,6 +116,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
- Four-layer regression-prevention tests guard against silent feature deletion: compile-time module assertion, module-presence unit test, cross-module `Protocol` enum assertion, end-to-end SSH banner test against the running binary
|
||||
|
||||
### Changed
|
||||
- **Startup reports the state of `RUSTFS_SSE_S3_MASTER_KEY`**: the server now checks the local SSE-S3 master key when it starts and logs one `sse_local_master_key` event with `state` `loaded`, `not_set`, or `invalid`. An unusable value used to surface only as a `400 InvalidRequest` on the first SSE-S3 request served without a KMS; it is now logged as an error at startup with the reason. Startup still succeeds and request behavior is unchanged. The line "Using RUSTFS_SSE_S3_MASTER_KEY for SSE-S3 (KMS not configured)", logged when the key was first used, is replaced by this event.
|
||||
- **Scanner bucket walkers are no longer tied to the disk count**: a new scanner key `walkers_per_disk` (`RUSTFS_SCANNER_WALKERS_PER_DISK`, `1` through `16`, default `1`) spawns that many bucket walkers per local disk of an erasure set, so a single-disk deployment can walk several buckets at once instead of one bucket, one object at a time. The walkers of one disk share its scanner admission, so a peer still sees one scan session per physical disk, and remote disks keep one walker. The walker count is what `max_concurrent_disk_scans` caps and the disk count stays the floor; the `disk_concurrency_budget` log line now reports `walkers_per_disk`, `walkers`, and the `reason` for the effective limit. The default keeps today's I/O profile because a local disk cannot yet report whether it is rotational; the design intends a higher default on flash once it can (`docs/architecture/scanner-per-bucket-convergence.md`). As groundwork for pacing, the raw-enumeration page writer inserts into its building page in place and commits pages without copying them, and a regression test pins the cost of recording 20,000 entries in one prefix so the quadratic bookkeeping removed by [#8114](https://github.com/rustfs/rustfs/pull/8114) (rustfs#8113) cannot return. No persisted format, protocol or admin API changes beyond the additional config key, which older releases preserve and ignore.
|
||||
- **Encryption and KMS work merged since `1.0.0-rc.5`** (entries were missing from this section):
|
||||
- **Persisted KMS configuration secrets** are sealed field-by-field with `RUSTFS_KMS_CONFIG_SECRET`. **When the variable is unset the secrets are persisted in cleartext and the server only warns** (`persisted KMS configuration carries cleartext secrets`); it never refuses the write. Set it, identically, on every node, and re-save the configuration to seal an existing one.
|
||||
|
||||
Generated
+212
-187
File diff suppressed because it is too large
Load Diff
+24
-19
@@ -60,6 +60,7 @@ members = [
|
||||
"crates/security-governance", # Security governance contracts
|
||||
"crates/extension-schema", # Extension schema contracts
|
||||
"crates/signer", # client signer
|
||||
"crates/sse", # S3 server-side encryption key resolution and metadata
|
||||
"crates/storage-api", # Storage API contracts
|
||||
"crates/test-utils", # Shared test bootstrap helpers (dev-dependency only)
|
||||
"crates/targets", # Target-specific configurations and utilities
|
||||
@@ -140,6 +141,7 @@ rustfs-scanner = { path = "crates/scanner", version = "1.0.5" }
|
||||
rustfs-security-governance = { path = "crates/security-governance", version = "1.0.5" }
|
||||
rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.5" }
|
||||
rustfs-signer = { path = "crates/signer", version = "1.0.5" }
|
||||
rustfs-sse = { path = "crates/sse", version = "1.0.5" }
|
||||
rustfs-storage-api = { path = "crates/storage-api", version = "1.0.5" }
|
||||
rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.5" }
|
||||
rustfs-targets = { path = "crates/targets", version = "1.0.5" }
|
||||
@@ -175,9 +177,9 @@ futures = "0.3.34"
|
||||
futures-core = "0.3.34"
|
||||
futures-lite = "2.6.1"
|
||||
futures-util = "0.3.34"
|
||||
pollster = "1.0.2"
|
||||
pollster = "1.0.3"
|
||||
pulsar = { default-features = false, version = "=6.9.0" }
|
||||
lapin = { default-features = false, version = "4.12.3" }
|
||||
lapin = { default-features = false, version = "4.13.0" }
|
||||
hyper = { version = "1.12.0" }
|
||||
h2 = { version = "0.4.20" }
|
||||
hyper-rustls = { default-features = false, version = "0.27.10" }
|
||||
@@ -216,7 +218,7 @@ csv-core = "0.1.13"
|
||||
bytesize = "2.7.0"
|
||||
byteorder = "1.5.0"
|
||||
flatbuffers = "25.12.19"
|
||||
form_urlencoded = "1.2.2"
|
||||
form_urlencoded = "1.2.3"
|
||||
prost = "0.14.6"
|
||||
quick-xml = "0.42.0"
|
||||
rmp = { version = "0.8.15" }
|
||||
@@ -258,7 +260,7 @@ proptest = "1"
|
||||
# Time and Date
|
||||
chrono = { version = "0.4.45" }
|
||||
humantime = "2.4.0"
|
||||
jiff = { version = "1.0.1" }
|
||||
jiff = { version = "1.0.2" }
|
||||
time = { version = "0.3.55" }
|
||||
|
||||
# Database
|
||||
@@ -278,15 +280,15 @@ atomic_enum = "0.3.0"
|
||||
aws-config = { version = "1.13.3" }
|
||||
aws-credential-types = { version = "1.3.1" }
|
||||
aws-sdk-kms = { default-features = false, version = "1.133.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.160.0" }
|
||||
aws-sdk-sts = { default-features = false, version = "1.127.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.161.0" }
|
||||
aws-sdk-sts = { default-features = false, version = "1.128.0" }
|
||||
aws-smithy-async = { version = "1.3.1" }
|
||||
aws-smithy-http-client = { default-features = false, version = "1.5.3" }
|
||||
aws-smithy-runtime-api = { version = "1.22.0" }
|
||||
aws-smithy-types = { version = "1.9.4" }
|
||||
base64-simd = "0.8.0"
|
||||
brotli = "9.1.3"
|
||||
clap = { version = "4.6.8" }
|
||||
clap = { version = "4.6.9" }
|
||||
const-str = { version = "1.1.0" }
|
||||
convert_case = "0.12.0"
|
||||
criterion = { version = "0.8" }
|
||||
@@ -298,12 +300,12 @@ datafusion = { default-features = false, version = "55.3.0" }
|
||||
derive_builder = "0.20.2"
|
||||
ed25519-dalek = "3.0.0"
|
||||
enumset = "1.1.15"
|
||||
faster-hex = "1.0.0"
|
||||
faster-hex = "2.0.0"
|
||||
flate2 = "1.1.11"
|
||||
glob = "0.3.4"
|
||||
google-cloud-storage = "1.22.0"
|
||||
google-cloud-auth = "1.19.0"
|
||||
hashbrown = { version = "0.17.1" }
|
||||
hashbrown = { version = "0.17.2" }
|
||||
# Base32 for RFC 6238 TOTP shared secrets (RFC 4648 unpadded, the alphabet
|
||||
# every authenticator app expects). Already in the graph transitively.
|
||||
data-encoding = "2.11.1"
|
||||
@@ -337,7 +339,7 @@ percent-encoding = "2.3.2"
|
||||
# CLI needs its own QR encoder. No default features: the image/render backends
|
||||
# pull in an image stack this only needs SVG and text output from.
|
||||
qrcode-rs = { version = "2.2.0", default-features = false, features = ["std", "svg"] }
|
||||
pin-project-lite = "0.2.17"
|
||||
pin-project-lite = "0.2.18"
|
||||
pretty_assertions = "1.4.1"
|
||||
rand = { version = "0.10.4" }
|
||||
ratelimit = "2.0.0"
|
||||
@@ -360,7 +362,7 @@ legacy-s3s = { package = "s3s", git = "https://github.com/s3s-project/s3s", rev
|
||||
serial_test = "4.0.1"
|
||||
shadow-rs = { default-features = false, version = "2.0.0" }
|
||||
siphasher = "1.0.4"
|
||||
smallvec = { version = "1.16.5" }
|
||||
smallvec = { version = "1.16.6" }
|
||||
compact_str = "0.10.0"
|
||||
snap = "1.1.2"
|
||||
starshard = { version = "2.4.1" }
|
||||
@@ -378,23 +380,23 @@ tracing-error = "0.2.1"
|
||||
tracing-opentelemetry = { version = "0.34.0" }
|
||||
tracing-subscriber = { version = "0.3.23" }
|
||||
transform-stream = "0.3.1"
|
||||
url = "2.5.8"
|
||||
url = "2.5.9"
|
||||
urlencoding = "2.1.3"
|
||||
uuid = { version = "1.29.3" }
|
||||
vaultrs = { version = "0.8.0" }
|
||||
tar = "0.4.46"
|
||||
walkdir = "2.5.0"
|
||||
walkdir = "2.5.1"
|
||||
winapi-util = "0.1.11"
|
||||
windows = { version = "0.62.2" }
|
||||
windows-sys = "0.61.2"
|
||||
xxhash-rust = { version = "0.8.21" }
|
||||
zip = "9.7.0"
|
||||
zip = "9.8.1"
|
||||
zstd = "0.14.4"
|
||||
|
||||
# Observability and Metrics
|
||||
metrics = "0.24.6"
|
||||
metrics-util = "0.20"
|
||||
dial9 = { version = "0.5.8", default-features = false }
|
||||
dial9 = { version = "0.5.9", default-features = false }
|
||||
opentelemetry = { version = "0.33.1" }
|
||||
opentelemetry-appender-tracing = { version = "0.33.1" }
|
||||
opentelemetry-otlp = { version = "0.33.1" }
|
||||
@@ -405,12 +407,12 @@ opentelemetry-stdout = { version = "0.33.1" }
|
||||
pyroscope = { version = "2.1.5" }
|
||||
|
||||
# FTP and SFTP
|
||||
libunftp = { version = "0.25.0" }
|
||||
libunftp = { version = "0.25.1" }
|
||||
unftp-core = "0.1.1"
|
||||
suppaftp = { version = "13.0.0" }
|
||||
rcgen = { version = "0.14.10", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
|
||||
russh = { version = "0.65.2" }
|
||||
russh-sftp = "3.0.1"
|
||||
russh = { version = "0.65.4" }
|
||||
russh-sftp = "4.0.0"
|
||||
|
||||
# WebDAV
|
||||
dav-server = "0.12.0"
|
||||
@@ -423,7 +425,10 @@ hotpath = { version = "0.32.1", default-features = false }
|
||||
insta = { version = "1.49" }
|
||||
|
||||
[workspace.metadata.cargo-shear]
|
||||
ignored = ["hotpath", "rustfs"]
|
||||
# The `#[path]` test modules under `rustfs/src/server/http/tests/` are linked by
|
||||
# rustc (including the `http.rs` + `http/` sibling-directory rule), but cargo-shear
|
||||
# resolves those nested `#[path]` attributes against the wrong base directory.
|
||||
ignored-paths = ["rustfs/src/server/http/tests/**/*.rs"]
|
||||
|
||||
[profile.dev]
|
||||
# Full debuginfo roughly doubles compile+link time and produces multi-GB
|
||||
|
||||
@@ -15,12 +15,11 @@ workspace = true
|
||||
|
||||
[features]
|
||||
default = []
|
||||
hotpath = ["hotpath/hotpath", "hotpath/tokio", "rustfs-io-core/hotpath"]
|
||||
hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc", "rustfs-io-core/hotpath-alloc"]
|
||||
hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu", "rustfs-io-core/hotpath-cpu"]
|
||||
hotpath = ["rustfs-io-core/hotpath"]
|
||||
hotpath-alloc = ["hotpath", "rustfs-io-core/hotpath-alloc"]
|
||||
hotpath-cpu = ["hotpath", "rustfs-io-core/hotpath-cpu"]
|
||||
|
||||
[dependencies]
|
||||
hotpath.workspace = true
|
||||
# Internal crates
|
||||
rustfs-io-core = { workspace = true }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
|
||||
@@ -129,6 +129,12 @@ when restoring `/rustfs/console` for a custom OEM build.
|
||||
- `RUSTFS_SCANNER_MAX_CONCURRENT_SET_SCANS` (canonical)
|
||||
- `RUSTFS_SCANNER_MAX_CONCURRENT_DISK_SCANS` (canonical)
|
||||
- `RUSTFS_SCANNER_WALKERS_PER_DISK` (canonical)
|
||||
- `RUSTFS_SCANNER_READ_ONLY_HANDOFF` (canonical, default `false`)
|
||||
|
||||
Read-only restart continuity:
|
||||
|
||||
- `RUSTFS_SCANNER_READ_ONLY_HANDOFF=true` enables reuse of unpublished scanner progress across verified read-only process restarts. Unset, false or invalid values disable both identity restoration and receipt creation. Existing receipts are still durably consumed before recovery starts, even when the feature is disabled.
|
||||
- Deploy receipt-aware readers to every possible writer before enabling this flag. Before rolling back to an older binary, disable it on every node, start each node with all its drives present, wait for readiness, and shut down cleanly. Do not start the older binary until every node completes this preparation. Direct rollback after enabling receipt writes is unsupported: older startup/recovery paths can write without durably invalidating the receipt. See [the receipt and rollback contract](../../docs/architecture/scanner-usage-publication.md#read-only-restart-continuity).
|
||||
|
||||
Scanner cycle budget controls:
|
||||
|
||||
|
||||
@@ -21,6 +21,11 @@ pub const APP_NAME: &str = "RustFS";
|
||||
/// Cargo package version for this workspace member.
|
||||
pub const VERSION: &str = env!("CARGO_PKG_VERSION");
|
||||
|
||||
/// Opt in to scanner checkpoint continuity across verified read-only restarts.
|
||||
/// Defaults to false. Enable only after all possible writers support receipt
|
||||
/// consumption; disable and restart before rolling back to an older writer.
|
||||
pub const ENV_SCANNER_READ_ONLY_HANDOFF: &str = "RUSTFS_SCANNER_READ_ONLY_HANDOFF";
|
||||
|
||||
/// Default configuration logger level
|
||||
/// Default value: error
|
||||
/// Environment variable: RUSTFS_OBS_LOGGER_LEVEL
|
||||
|
||||
@@ -1758,6 +1758,129 @@ mod upgrade_write_readiness_tests {
|
||||
}
|
||||
}
|
||||
|
||||
async fn stop_scanner_handoff_server(env: &mut RustFSTestEnvironment) -> TestResult {
|
||||
let process = env.process.as_mut().ok_or("handoff server is not running")?;
|
||||
signal_process(process.id(), "TERM")?;
|
||||
tokio::time::timeout(Duration::from_secs(90), async {
|
||||
loop {
|
||||
if let Some(status) = process.try_wait()? {
|
||||
assert!(status.success(), "handoff shutdown failed: {status}");
|
||||
return Ok::<(), BoxError>(());
|
||||
}
|
||||
sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
})
|
||||
.await??;
|
||||
env.process.take();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn scanner_handoff_identity(env: &RustFSTestEnvironment) -> Result<String, BoxError> {
|
||||
let receipt: Value = serde_json::from_slice(&std::fs::read(Path::new(&env.temp_dir).join(".rustfs.sys/unclean-shutdown"))?)?;
|
||||
assert_eq!(receipt["schema"], 1);
|
||||
assert_eq!(receipt["owner"], env.address);
|
||||
Ok(receipt["checkpoint_instance_id"]
|
||||
.as_str()
|
||||
.ok_or("handoff receipt omitted checkpoint identity")?
|
||||
.to_owned())
|
||||
}
|
||||
|
||||
async fn assert_scanner_handoff_consumed(marker: &Path) -> TestResult {
|
||||
let marker = tokio::fs::read_to_string(marker).await?;
|
||||
// Heal may replace the startup marker with its ordinary timestamp before
|
||||
// readiness. Neither form carries the prior checkpoint identity.
|
||||
assert!(
|
||||
marker == "running" || marker.parse::<u64>().is_ok(),
|
||||
"readiness must expose an ordinary unclean marker, never a handoff receipt"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_read_only_handoff_requires_explicit_opt_in() -> TestResult {
|
||||
const HANDOFF: &str = "RUSTFS_SCANNER_READ_ONLY_HANDOFF";
|
||||
assert!(std::env::var_os(HANDOFF).is_none(), "this scenario tests the unset default");
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
let marker = Path::new(&env.temp_dir).join(".rustfs.sys/unclean-shutdown");
|
||||
env.start_rustfs_server_without_cleanup(vec![]).await?;
|
||||
stop_scanner_handoff_server(&mut env).await?;
|
||||
assert!(!marker.exists(), "the default must not create a receipt");
|
||||
|
||||
env.start_rustfs_server_without_cleanup_with_env(&[(HANDOFF, "true")]).await?;
|
||||
stop_scanner_handoff_server(&mut env).await?;
|
||||
let previous = scanner_handoff_identity(&env)?;
|
||||
env.start_rustfs_server_without_cleanup_with_env(&[(HANDOFF, "true")]).await?;
|
||||
assert_scanner_handoff_consumed(&marker).await?;
|
||||
stop_scanner_handoff_server(&mut env).await?;
|
||||
assert_eq!(scanner_handoff_identity(&env)?, previous, "opted-in read-only boots retain identity");
|
||||
|
||||
for disabled in ["invalid", "false"] {
|
||||
let before_disabled = scanner_handoff_identity(&env)?;
|
||||
env.start_rustfs_server_without_cleanup_with_env(&[(HANDOFF, disabled)])
|
||||
.await?;
|
||||
assert_scanner_handoff_consumed(&marker).await?;
|
||||
stop_scanner_handoff_server(&mut env).await?;
|
||||
assert!(!marker.exists(), "disabled shutdown must not recreate the receipt");
|
||||
env.start_rustfs_server_without_cleanup_with_env(&[(HANDOFF, "true")]).await?;
|
||||
stop_scanner_handoff_server(&mut env).await?;
|
||||
assert_ne!(scanner_handoff_identity(&env)?, before_disabled, "opt-out breaks the prior lineage");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// Exercise the receipt protocol in the existing pinned-release rollback lane.
|
||||
async fn assert_scanner_handoff_prepares_legacy_rollback(previous_binary: &Path) -> TestResult {
|
||||
const HANDOFF: &str = "RUSTFS_SCANNER_READ_ONLY_HANDOFF";
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
let marker = Path::new(&env.temp_dir).join(".rustfs.sys/unclean-shutdown");
|
||||
env.start_rustfs_server_without_cleanup_with_env(&[(HANDOFF, "true")]).await?;
|
||||
stop_scanner_handoff_server(&mut env).await?;
|
||||
let previous = scanner_handoff_identity(&env)?;
|
||||
|
||||
// Complete the current reader's rollback preparation before any old writer
|
||||
// opens storage, including an old writer with all background services off.
|
||||
env.start_rustfs_server_without_cleanup_with_env(&[(HANDOFF, "false")])
|
||||
.await?;
|
||||
assert_scanner_handoff_consumed(&marker).await?;
|
||||
stop_scanner_handoff_server(&mut env).await?;
|
||||
assert!(!marker.exists());
|
||||
env.start_rustfs_server_from_binary_without_cleanup(
|
||||
previous_binary,
|
||||
vec![],
|
||||
&[
|
||||
("RUSTFS_SCANNER_ENABLED", "false"),
|
||||
("RUSTFS_HEAL_ENABLED", "false"),
|
||||
("RUSTFS_HEAL_AUTO_HEAL_ENABLE", "false"),
|
||||
("RUSTFS_HEAL_MRF_ENABLE", "false"),
|
||||
],
|
||||
)
|
||||
.await?;
|
||||
let client = env.create_s3_client();
|
||||
env.create_test_bucket("handoff-rollback").await?;
|
||||
client
|
||||
.put_object()
|
||||
.bucket("handoff-rollback")
|
||||
.key("legacy-write")
|
||||
.body(ByteStream::from_static(b"legacy writer survived rollback preparation"))
|
||||
.send()
|
||||
.await?;
|
||||
env.stop_server();
|
||||
assert!(!marker.exists(), "the legacy crash cannot resurrect a consumed receipt");
|
||||
|
||||
env.start_rustfs_server_without_cleanup_with_env(&[(HANDOFF, "true")]).await?;
|
||||
assert_eq!(
|
||||
read_object(&client, "handoff-rollback", "legacy-write", None).await?.1,
|
||||
b"legacy writer survived rollback preparation"
|
||||
);
|
||||
stop_scanner_handoff_server(&mut env).await?;
|
||||
assert_ne!(
|
||||
scanner_handoff_identity(&env)?,
|
||||
previous,
|
||||
"a legacy write must not retain the old lineage"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Pins the published old writer's limitation and the supported recovery
|
||||
/// procedure. This is not a promise that mixed-version ODM is supported.
|
||||
/// Replace the loss assertion when ODM gains independent persistence;
|
||||
@@ -3462,6 +3585,7 @@ async fn rollback_to_previous_release_reads_current_bucket_metadata() -> TestRes
|
||||
init_logging();
|
||||
let previous_binary = source_binary()?;
|
||||
let evidence_context = g09_evidence_context(&previous_binary).await?;
|
||||
assert_scanner_handoff_prepares_legacy_rollback(&previous_binary).await?;
|
||||
|
||||
let replication_target = FakeS3Target::start().await?;
|
||||
replication_target.create_bucket(ROLLBACK_REPLICA_BUCKET);
|
||||
|
||||
@@ -183,7 +183,6 @@ rmp.workspace = true
|
||||
rmp-serde.workspace = true
|
||||
tokio-util = { workspace = true, features = ["io", "compat", "rt"] }
|
||||
tokio-stream = { workspace = true, features = ["sync"] }
|
||||
hmac = { workspace = true }
|
||||
sha2 = { workspace = true }
|
||||
hex-simd = { workspace = true }
|
||||
tempfile.workspace = true
|
||||
|
||||
@@ -503,7 +503,7 @@ pub mod disk {
|
||||
PoolMetaNativeInspection, PoolMetaNativeObject, PoolMetaNativeRecoveryRequest, PoolMetaNativeRecoveryResponse,
|
||||
QUOTA_MUTATION_FENCE_METADATA_SUFFIX, RUSTFS_META_BUCKET, RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_TMP_BUCKET,
|
||||
ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, STORAGE_FORMAT_FILE, SnapshotLeaseToken,
|
||||
UpdateMetadataOpts, VolumeInfo, WalkDirOptions, is_quota_mutation_fence_path, new_disk,
|
||||
UNCLEAN_SHUTDOWN_MARKER_PATH, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, is_quota_mutation_fence_path, new_disk,
|
||||
validate_batch_read_version_item_count, validate_format_control_path,
|
||||
};
|
||||
pub use crate::disk::{
|
||||
@@ -769,8 +769,8 @@ pub mod storage {
|
||||
pub use crate::store::external_rpc_disk;
|
||||
pub use crate::store::{
|
||||
BootstrapLocalTarget, ECStore, SCANNER_PUBLICATION_LEASE_TTL_MS, ScannerDataMovementPauseStatus, all_local_disk,
|
||||
all_local_disk_path, find_local_disk_by_ref, find_local_disk_by_ref_for_rpc, init_local_disks,
|
||||
init_local_disks_with_instance_ctx, init_lock_clients, prewarm_local_disk_id_map,
|
||||
all_local_disk_path, find_local_disk_by_ref, find_local_disk_by_ref_for_rpc, find_local_disk_by_ref_for_scanner_rpc,
|
||||
init_local_disks, init_local_disks_with_instance_ctx, init_lock_clients, prewarm_local_disk_id_map,
|
||||
prewarm_local_disk_id_map_with_instance_ctx,
|
||||
};
|
||||
pub use crate::store::{HealObjectAbsenceProof, HealObjectStorageResult};
|
||||
|
||||
@@ -5082,7 +5082,14 @@ mod tests {
|
||||
let cert = generate_simple_self_signed(vec!["localhost".to_string()]).expect("certificate should generate");
|
||||
let (port, server) = spawn_https_server(&cert, 2);
|
||||
let endpoint = format!("https://localhost:{port}");
|
||||
let strict = target_client_for_test("arn:strict", endpoint.clone(), s3_client_for_test(port, None));
|
||||
let strict_client = s3_client_for_test(port, None);
|
||||
// A failed trust check must consume just one of the two fixture connections.
|
||||
let strict_config = strict_client
|
||||
.config()
|
||||
.to_builder()
|
||||
.retry_config(aws_sdk_s3::config::retry::RetryConfig::disabled())
|
||||
.build();
|
||||
let strict = target_client_for_test("arn:strict", endpoint.clone(), S3Client::from_conf(strict_config));
|
||||
let insecure = target_client_for_test(
|
||||
"arn:insecure",
|
||||
endpoint,
|
||||
|
||||
@@ -21,7 +21,7 @@ use crate::cluster::rpc::{set_tonic_canonical_body_digest, set_tonic_mutation_bo
|
||||
use crate::error::{Error, Result};
|
||||
use crate::storage_api_contracts::internode::{
|
||||
SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION, SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SCANNER_ACTIVITY_PROTOCOL_VERSION,
|
||||
SCANNER_ACTIVITY_V6_PROTOCOL_VERSION, SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES,
|
||||
SCANNER_ACTIVITY_V6_PROTOCOL_VERSION, SCANNER_ACTIVITY_V7_PROTOCOL_VERSION, SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES,
|
||||
SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION, SCANNER_DIRTY_USAGE_SNAPSHOT_RPC_MAX_MESSAGE_SIZE,
|
||||
};
|
||||
use crate::{
|
||||
@@ -192,6 +192,7 @@ fn validate_signal_service_protocol(sig: u64, sub_sys: &str, protocol_version: u
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct ScannerPeerActivity {
|
||||
pub instance_id: String,
|
||||
pub checkpoint_instance_id: Option<String>,
|
||||
pub namespace_generation: u64,
|
||||
pub maintenance_generation: u64,
|
||||
pub protocol_version: u32,
|
||||
@@ -354,6 +355,9 @@ fn decode_scanner_activity_with_verifier(
|
||||
challenge: &[u8; 16],
|
||||
verify_proof: impl FnOnce(&[u8], &[u8]) -> Result<()>,
|
||||
) -> Result<ScannerPeerActivity> {
|
||||
if response.protocol_version != SCANNER_ACTIVITY_PROTOCOL_VERSION && response.checkpoint_instance_id.is_some() {
|
||||
return Err(Error::other("legacy scanner activity peer returned unauthenticated checkpoint identity"));
|
||||
}
|
||||
let instance_id = &response.instance_id;
|
||||
if !scanner_instance_id_is_valid(instance_id) {
|
||||
return Err(Error::other("peer returned an invalid scanner activity instance ID"));
|
||||
@@ -423,7 +427,7 @@ fn decode_scanner_activity_with_verifier(
|
||||
None,
|
||||
)
|
||||
}
|
||||
SCANNER_ACTIVITY_PROTOCOL_VERSION => {
|
||||
SCANNER_ACTIVITY_V7_PROTOCOL_VERSION | SCANNER_ACTIVITY_PROTOCOL_VERSION => {
|
||||
if response.dirty_usage_pending && response.dirty_usage_generation == 0 {
|
||||
return Err(Error::other("scanner activity peer returned pending dirty usage without a generation"));
|
||||
}
|
||||
@@ -436,8 +440,20 @@ fn decode_scanner_activity_with_verifier(
|
||||
if movement_generation == u64::MAX {
|
||||
return Err(Error::other("scanner activity peer exhausted its movement generation"));
|
||||
}
|
||||
let canonical = rustfs_protos::canonical_scanner_activity_v7_response_body(challenge, &response)
|
||||
.map_err(|_| Error::other("scanner activity peer response is too large to authenticate"))?;
|
||||
let canonical = if response.protocol_version == SCANNER_ACTIVITY_PROTOCOL_VERSION {
|
||||
let instance = response
|
||||
.checkpoint_instance_id
|
||||
.as_deref()
|
||||
.filter(|instance| scanner_instance_id_is_valid(instance))
|
||||
.ok_or_else(|| Error::other("scanner activity peer omitted its checkpoint identity"))?;
|
||||
if instance.bytes().all(|byte| byte == b'0') {
|
||||
return Err(Error::other("scanner activity peer returned an empty checkpoint identity"));
|
||||
}
|
||||
rustfs_protos::canonical_scanner_activity_v8_response_body(challenge, &response)
|
||||
} else {
|
||||
rustfs_protos::canonical_scanner_activity_v7_response_body(challenge, &response)
|
||||
}
|
||||
.map_err(|_| Error::other("scanner activity peer response is too large to authenticate"))?;
|
||||
verify_proof(&canonical, &response.response_proof)?;
|
||||
(
|
||||
Some(
|
||||
@@ -458,6 +474,7 @@ fn decode_scanner_activity_with_verifier(
|
||||
};
|
||||
Ok(ScannerPeerActivity {
|
||||
instance_id: response.instance_id,
|
||||
checkpoint_instance_id: response.checkpoint_instance_id,
|
||||
namespace_generation: response.namespace_generation,
|
||||
maintenance_generation: response.maintenance_generation,
|
||||
protocol_version: response.protocol_version,
|
||||
@@ -2452,11 +2469,15 @@ impl PeerRestClient {
|
||||
.scanner_activity_request_with_protocol(String::new(), 0, SCANNER_ACTIVITY_PROTOCOL_VERSION)
|
||||
.await;
|
||||
if result.as_ref().err().is_some_and(scanner_activity_protocol_unsupported) {
|
||||
// A v6 peer cannot parse the v7 marker. Its authenticated
|
||||
// response is still decoded as untrusted terminal state, so the
|
||||
// scanner will defer publication until every peer is v7.
|
||||
self.scanner_activity_request_with_protocol(String::new(), 0, SCANNER_ACTIVITY_V6_PROTOCOL_VERSION)
|
||||
.await
|
||||
let previous = self
|
||||
.scanner_activity_request_with_protocol(String::new(), 0, SCANNER_ACTIVITY_V7_PROTOCOL_VERSION)
|
||||
.await;
|
||||
if previous.as_ref().err().is_some_and(scanner_activity_protocol_unsupported) {
|
||||
self.scanner_activity_request_with_protocol(String::new(), 0, SCANNER_ACTIVITY_V6_PROTOCOL_VERSION)
|
||||
.await
|
||||
} else {
|
||||
previous
|
||||
}
|
||||
} else {
|
||||
result
|
||||
}
|
||||
@@ -2723,8 +2744,15 @@ impl PeerRestClient {
|
||||
.scanner_activity_request_with_protocol(instance_id.clone(), generation, SCANNER_ACTIVITY_PROTOCOL_VERSION)
|
||||
.await;
|
||||
if result.as_ref().err().is_some_and(scanner_activity_protocol_unsupported) {
|
||||
self.scanner_activity_request_with_protocol(instance_id, generation, SCANNER_ACTIVITY_V6_PROTOCOL_VERSION)
|
||||
.await
|
||||
let previous = self
|
||||
.scanner_activity_request_with_protocol(instance_id.clone(), generation, SCANNER_ACTIVITY_V7_PROTOCOL_VERSION)
|
||||
.await;
|
||||
if previous.as_ref().err().is_some_and(scanner_activity_protocol_unsupported) {
|
||||
self.scanner_activity_request_with_protocol(instance_id, generation, SCANNER_ACTIVITY_V6_PROTOCOL_VERSION)
|
||||
.await
|
||||
} else {
|
||||
previous
|
||||
}
|
||||
} else {
|
||||
result
|
||||
}
|
||||
@@ -3680,6 +3708,7 @@ mod tests {
|
||||
#[test]
|
||||
fn scanner_scoped_dirty_usage_ack_reconciliation_requires_same_clean_instance() {
|
||||
let activity = |instance_id: &str, generation, pending| ScannerPeerActivity {
|
||||
checkpoint_instance_id: Some("1123456789abcdef0123456789abcdef".to_owned()),
|
||||
instance_id: instance_id.to_string(),
|
||||
namespace_generation: 1,
|
||||
maintenance_generation: 1,
|
||||
@@ -3961,9 +3990,44 @@ mod tests {
|
||||
.await;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_checkpoint_identity_requires_v8_proof_and_preserves_rpc_epoch() {
|
||||
let response = ScannerActivityResponse {
|
||||
protocol_version: SCANNER_ACTIVITY_PROTOCOL_VERSION,
|
||||
instance_id: "1".repeat(32),
|
||||
checkpoint_instance_id: Some("2".repeat(32)),
|
||||
topology_digest: vec![7; 32].into(),
|
||||
movement_generation: Some(0),
|
||||
publication_blocked: Some(false),
|
||||
response_proof: b"proof".to_vec().into(),
|
||||
..Default::default()
|
||||
};
|
||||
let decoded = decode_test_scanner_activity(response.clone()).expect("authenticated v8");
|
||||
assert_eq!(decoded.instance_id, "1".repeat(32));
|
||||
assert_eq!(decoded.checkpoint_instance_id, Some("2".repeat(32)));
|
||||
for checkpoint_instance_id in [None, Some(String::new()), Some("0".repeat(32)), Some("A".repeat(32))] {
|
||||
let malformed = ScannerActivityResponse {
|
||||
checkpoint_instance_id,
|
||||
..response.clone()
|
||||
};
|
||||
assert!(decode_test_scanner_activity(malformed).is_err(), "invalid continuity must fail closed");
|
||||
}
|
||||
let v7 = ScannerActivityResponse {
|
||||
protocol_version: SCANNER_ACTIVITY_V7_PROTOCOL_VERSION,
|
||||
..response
|
||||
};
|
||||
assert!(decode_test_scanner_activity(v7.clone()).is_err(), "legacy proof does not bind continuity");
|
||||
let v7 = ScannerActivityResponse {
|
||||
checkpoint_instance_id: None,
|
||||
..v7
|
||||
};
|
||||
assert!(decode_test_scanner_activity(v7).is_ok(), "v7 without continuity remains supported");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_activity_requires_restart_safe_peer_identity() {
|
||||
let legacy = decode_test_scanner_activity(ScannerActivityResponse {
|
||||
checkpoint_instance_id: None,
|
||||
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
|
||||
namespace_generation: 7,
|
||||
maintenance_generation: 3,
|
||||
@@ -3980,6 +4044,7 @@ mod tests {
|
||||
assert_eq!(
|
||||
legacy,
|
||||
ScannerPeerActivity {
|
||||
checkpoint_instance_id: None,
|
||||
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
|
||||
namespace_generation: 7,
|
||||
maintenance_generation: 3,
|
||||
@@ -3994,6 +4059,7 @@ mod tests {
|
||||
);
|
||||
|
||||
let previous = decode_test_scanner_activity(ScannerActivityResponse {
|
||||
checkpoint_instance_id: None,
|
||||
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
|
||||
namespace_generation: 7,
|
||||
maintenance_generation: 3,
|
||||
@@ -4010,6 +4076,7 @@ mod tests {
|
||||
assert_eq!(
|
||||
previous,
|
||||
ScannerPeerActivity {
|
||||
checkpoint_instance_id: None,
|
||||
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
|
||||
namespace_generation: 7,
|
||||
maintenance_generation: 3,
|
||||
@@ -4024,6 +4091,7 @@ mod tests {
|
||||
);
|
||||
|
||||
let v6 = decode_test_scanner_activity(ScannerActivityResponse {
|
||||
checkpoint_instance_id: None,
|
||||
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
|
||||
namespace_generation: 7,
|
||||
maintenance_generation: 3,
|
||||
@@ -4042,6 +4110,7 @@ mod tests {
|
||||
assert_eq!(v6.dirty_usage_generation, Some(11));
|
||||
|
||||
let malformed_topology = ScannerActivityResponse {
|
||||
checkpoint_instance_id: Some("1123456789abcdef0123456789abcdef".to_owned()),
|
||||
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
|
||||
namespace_generation: 7,
|
||||
maintenance_generation: 3,
|
||||
@@ -4062,6 +4131,7 @@ mod tests {
|
||||
);
|
||||
|
||||
let missing_instance = ScannerActivityResponse {
|
||||
checkpoint_instance_id: Some("1123456789abcdef0123456789abcdef".to_owned()),
|
||||
instance_id: String::new(),
|
||||
namespace_generation: 7,
|
||||
maintenance_generation: 3,
|
||||
@@ -4082,6 +4152,7 @@ mod tests {
|
||||
);
|
||||
|
||||
let malformed_instance = ScannerActivityResponse {
|
||||
checkpoint_instance_id: Some("1123456789abcdef0123456789abcdef".to_owned()),
|
||||
instance_id: "ABCDEF0123456789ABCDEF0123456789".to_string(),
|
||||
namespace_generation: 7,
|
||||
maintenance_generation: 3,
|
||||
@@ -4102,6 +4173,7 @@ mod tests {
|
||||
);
|
||||
|
||||
let activity = decode_test_scanner_activity(ScannerActivityResponse {
|
||||
checkpoint_instance_id: Some("1123456789abcdef0123456789abcdef".to_owned()),
|
||||
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
|
||||
namespace_generation: 7,
|
||||
maintenance_generation: 3,
|
||||
@@ -4118,6 +4190,7 @@ mod tests {
|
||||
assert_eq!(
|
||||
activity,
|
||||
ScannerPeerActivity {
|
||||
checkpoint_instance_id: Some("1123456789abcdef0123456789abcdef".to_owned()),
|
||||
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
|
||||
namespace_generation: 7,
|
||||
maintenance_generation: 3,
|
||||
@@ -4132,6 +4205,7 @@ mod tests {
|
||||
);
|
||||
|
||||
let missing_movement_generation = ScannerActivityResponse {
|
||||
checkpoint_instance_id: Some("1123456789abcdef0123456789abcdef".to_owned()),
|
||||
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
|
||||
namespace_generation: 7,
|
||||
maintenance_generation: 3,
|
||||
@@ -4152,6 +4226,7 @@ mod tests {
|
||||
);
|
||||
|
||||
let pending_without_generation = ScannerActivityResponse {
|
||||
checkpoint_instance_id: Some("1123456789abcdef0123456789abcdef".to_owned()),
|
||||
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
|
||||
namespace_generation: 7,
|
||||
maintenance_generation: 3,
|
||||
@@ -4172,6 +4247,7 @@ mod tests {
|
||||
);
|
||||
|
||||
let previous_with_dirty_usage = ScannerActivityResponse {
|
||||
checkpoint_instance_id: None,
|
||||
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
|
||||
namespace_generation: 7,
|
||||
maintenance_generation: 3,
|
||||
@@ -4192,6 +4268,7 @@ mod tests {
|
||||
);
|
||||
|
||||
let legacy_with_topology = ScannerActivityResponse {
|
||||
checkpoint_instance_id: None,
|
||||
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
|
||||
namespace_generation: 7,
|
||||
maintenance_generation: 3,
|
||||
@@ -4212,6 +4289,7 @@ mod tests {
|
||||
);
|
||||
|
||||
let unsupported_protocol = ScannerActivityResponse {
|
||||
checkpoint_instance_id: None,
|
||||
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
|
||||
namespace_generation: 7,
|
||||
maintenance_generation: 3,
|
||||
@@ -4232,6 +4310,7 @@ mod tests {
|
||||
);
|
||||
|
||||
let missing_proof = ScannerActivityResponse {
|
||||
checkpoint_instance_id: Some("1123456789abcdef0123456789abcdef".to_owned()),
|
||||
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
|
||||
namespace_generation: 7,
|
||||
maintenance_generation: 3,
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
use crate::disk::cleanup_runtime;
|
||||
use crate::disk::io_admission::{
|
||||
AdmissionReader, BoxedChunkReader, DiskIoAdmission, DiskIoClass, DiskIoContext, current_disk_io_context,
|
||||
current_resource_unit, with_disk_io_context, with_disk_io_permit, with_optional_resource_unit,
|
||||
current_resource_unit, with_detached_disk_io_context, with_disk_io_context, with_disk_io_permit, with_optional_resource_unit,
|
||||
};
|
||||
use crate::disk::{
|
||||
CheckPartsResp, DataDirDeleteStatus, DeleteOptions, DiskAPI, DiskError, DiskInfo, DiskInfoOptions, DiskLocation, Endpoint,
|
||||
@@ -479,10 +479,13 @@ where
|
||||
if external_guard.is_none() {
|
||||
return operation().await;
|
||||
}
|
||||
tokio::spawn(with_optional_resource_unit(current_resource_unit(), async move {
|
||||
let _external_guard = external_guard;
|
||||
operation().await
|
||||
}))
|
||||
tokio::spawn(with_optional_resource_unit(
|
||||
current_resource_unit(),
|
||||
with_detached_disk_io_context(async move {
|
||||
let _external_guard = external_guard;
|
||||
operation().await
|
||||
}),
|
||||
))
|
||||
.await
|
||||
.map_err(|_| Error::other("owned mutation task failed"))?
|
||||
}
|
||||
@@ -1828,6 +1831,7 @@ impl LocalDiskWrapper {
|
||||
current_disk_io_context().unwrap_or_else(|| DiskIoContext {
|
||||
class: DiskIoClass::Foreground,
|
||||
cancellation: CancellationToken::new(),
|
||||
admission_deadline: None,
|
||||
admission: None,
|
||||
permit: None,
|
||||
resource_unit: current_resource_unit(),
|
||||
@@ -1870,10 +1874,10 @@ impl LocalDiskWrapper {
|
||||
|
||||
async fn acquire_disk_io_permit(&self) -> Result<Option<crate::disk::io_admission::DiskIoPermit>> {
|
||||
if let Some(context) = current_disk_io_context() {
|
||||
if let Some(permit) = context.permit
|
||||
if let Some(permit) = context.permit.as_ref()
|
||||
&& permit.is_for(&self.io_admission)
|
||||
{
|
||||
return Ok(Some(permit));
|
||||
return Ok(Some(permit.clone()));
|
||||
}
|
||||
if context
|
||||
.admission
|
||||
@@ -1883,9 +1887,8 @@ impl LocalDiskWrapper {
|
||||
return Ok(None);
|
||||
}
|
||||
if context.class == DiskIoClass::Background {
|
||||
return self
|
||||
.io_admission
|
||||
.background(&context.cancellation)
|
||||
return context
|
||||
.background_permit(&self.io_admission)
|
||||
.await
|
||||
.map(Some)
|
||||
.map_err(DiskError::Io);
|
||||
@@ -4409,6 +4412,29 @@ mod tests {
|
||||
assert!(matches!(wrapper.read_all(volume, &body_path).await, Err(DiskError::FileNotFound)));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn owned_mutation_retains_background_class_without_waiter_cancellation() {
|
||||
let cancellation = CancellationToken::new();
|
||||
let caller = cancellation.clone();
|
||||
let guard: Arc<dyn Send + Sync> = Arc::new(());
|
||||
crate::disk::with_background_disk_io(
|
||||
cancellation,
|
||||
run_owned_mutation(Some(guard), move || async move {
|
||||
caller.cancel();
|
||||
let context = current_disk_io_context().expect("publication owner retains background class");
|
||||
assert_eq!(context.class, DiskIoClass::Background);
|
||||
assert!(!context.cancellation.is_cancelled(), "waiter cancellation must not interrupt publication");
|
||||
assert!(
|
||||
context.admission_deadline.is_some(),
|
||||
"queued publication must retain a bounded admission lifetime"
|
||||
);
|
||||
Ok(())
|
||||
}),
|
||||
)
|
||||
.await
|
||||
.expect("owned publication completes");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn owned_mutation_keeps_publication_guard_after_waiter_cancellation() {
|
||||
use rustfs_concurrency::adaptive::AdaptiveAdmission;
|
||||
|
||||
@@ -161,11 +161,24 @@ pub(crate) enum DiskIoClass {
|
||||
pub(crate) struct DiskIoContext {
|
||||
pub(crate) class: DiskIoClass,
|
||||
pub(crate) cancellation: CancellationToken,
|
||||
pub(crate) admission_deadline: Option<Instant>,
|
||||
pub(crate) admission: Option<Arc<DiskIoAdmission>>,
|
||||
pub(crate) permit: Option<DiskIoPermit>,
|
||||
pub(crate) resource_unit: Option<ResourceIoUnit>,
|
||||
}
|
||||
|
||||
impl DiskIoContext {
|
||||
pub(crate) async fn background_permit(&self, admission: &Arc<DiskIoAdmission>) -> io::Result<DiskIoPermit> {
|
||||
let acquire = admission.background_with_resource(&self.cancellation, self.resource_unit.clone());
|
||||
match self.admission_deadline {
|
||||
Some(deadline) => tokio::time::timeout_at(deadline, acquire)
|
||||
.await
|
||||
.map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "owned background disk admission timed out"))?,
|
||||
None => acquire.await,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Mark disk I/O issued by `future` as background work for local-disk admission.
|
||||
#[doc(hidden)]
|
||||
pub async fn with_background_disk_io<F: Future>(cancellation: CancellationToken, future: F) -> F::Output {
|
||||
@@ -173,6 +186,7 @@ pub async fn with_background_disk_io<F: Future>(cancellation: CancellationToken,
|
||||
DiskIoContext {
|
||||
class: DiskIoClass::Background,
|
||||
cancellation,
|
||||
admission_deadline: None,
|
||||
admission: None,
|
||||
permit: None,
|
||||
resource_unit: current_resource_unit(),
|
||||
@@ -245,6 +259,7 @@ pub(crate) async fn with_disk_io_permit<F: Future>(permit: Option<DiskIoPermit>,
|
||||
let mut context = current_disk_io_context().unwrap_or_else(|| DiskIoContext {
|
||||
class: DiskIoClass::Foreground,
|
||||
cancellation: CancellationToken::new(),
|
||||
admission_deadline: None,
|
||||
admission: None,
|
||||
permit: None,
|
||||
resource_unit: None,
|
||||
@@ -279,6 +294,34 @@ pub(crate) fn with_optional_disk_io_context<F: Future>(
|
||||
}
|
||||
}
|
||||
|
||||
/// An owned commit outlives its waiter. Retain an outer I/O class and physical
|
||||
/// ownership, but give admission its own cancellation lifetime. Leave per-disk
|
||||
/// scopes to the caller's existing ownership handoff: inheriting their class
|
||||
/// without a lease could wait behind the parent awaiting this commit.
|
||||
/// The drive timeout bounds only queued admission; an admitted mutation still
|
||||
/// drains to completion. Nested owners retain the original admission deadline.
|
||||
pub(crate) fn with_detached_disk_io_context<F: Future>(future: F) -> impl Future<Output = F::Output> {
|
||||
let context = current_disk_io_context()
|
||||
.filter(|context| context.admission.is_none() && context.permit.is_none())
|
||||
.map(|mut context| {
|
||||
context.cancellation = CancellationToken::new();
|
||||
let timeout = super::disk_store::get_max_timeout_duration();
|
||||
if context.class == DiskIoClass::Background
|
||||
&& !timeout.is_zero()
|
||||
&& let Some(deadline) = Instant::now().checked_add(timeout)
|
||||
{
|
||||
context.admission_deadline = Some(context.admission_deadline.map_or(deadline, |old| old.min(deadline)));
|
||||
}
|
||||
context
|
||||
});
|
||||
async move {
|
||||
match context {
|
||||
Some(context) => with_disk_io_context(context, future).await,
|
||||
None => future.await,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn current_disk_io_context() -> Option<DiskIoContext> {
|
||||
DISK_IO_CONTEXT
|
||||
.try_with(|context| {
|
||||
@@ -297,17 +340,17 @@ pub(crate) async fn acquire_context_disk_io_permit() -> io::Result<Option<DiskIo
|
||||
let Some(context) = current_disk_io_context() else {
|
||||
return Ok(None);
|
||||
};
|
||||
let Some(admission) = context.admission else {
|
||||
let Some(admission) = context.admission.as_ref() else {
|
||||
return Ok(None);
|
||||
};
|
||||
if let Some(permit) = context.permit
|
||||
&& permit.is_for(&admission)
|
||||
if let Some(permit) = context.permit.as_ref()
|
||||
&& permit.is_for(admission)
|
||||
{
|
||||
return Ok(Some(permit));
|
||||
return Ok(Some(permit.clone()));
|
||||
}
|
||||
match context.class {
|
||||
DiskIoClass::Foreground => Ok(Some(admission.foreground())),
|
||||
DiskIoClass::Background => Ok(Some(admission.background(&context.cancellation).await?)),
|
||||
DiskIoClass::Background => Ok(Some(context.background_permit(admission).await?)),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -381,6 +424,7 @@ impl DiskIoAdmission {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn background(self: &Arc<Self>, cancellation: &CancellationToken) -> std::io::Result<DiskIoPermit> {
|
||||
self.background_with_resource(cancellation, current_resource_unit()).await
|
||||
}
|
||||
@@ -663,11 +707,8 @@ impl<R> AdmissionReader<R> {
|
||||
let mut wait = self.wait.lock().unwrap_or_else(|error| error.into_inner());
|
||||
if wait.is_none() {
|
||||
let admission = self.admission.clone();
|
||||
let cancellation = self.context.cancellation.clone();
|
||||
let resource_unit = self.context.resource_unit.clone();
|
||||
*wait = Some(Box::pin(
|
||||
async move { admission.background_with_resource(&cancellation, resource_unit).await },
|
||||
));
|
||||
let context = self.context.clone();
|
||||
*wait = Some(Box::pin(async move { context.background_permit(&admission).await }));
|
||||
}
|
||||
let Some(wait_future) = wait.as_mut() else {
|
||||
return Poll::Ready(Err(io::Error::other("background disk I/O admission waiter is missing")));
|
||||
@@ -909,6 +950,7 @@ mod tests {
|
||||
let existing = DiskIoContext {
|
||||
class: DiskIoClass::Foreground,
|
||||
cancellation: cancellation.clone(),
|
||||
admission_deadline: None,
|
||||
admission: None,
|
||||
permit: None,
|
||||
resource_unit: None,
|
||||
@@ -954,6 +996,94 @@ mod tests {
|
||||
assert_eq!(gate.snapshot().active, 0);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn detached_commit_keeps_background_admission_after_waiter_cancellation() {
|
||||
let admission = DiskIoAdmission::new("detached-commit".to_string());
|
||||
let foreground = admission.foreground();
|
||||
let cancellation = CancellationToken::new();
|
||||
let worker_admission = admission.clone();
|
||||
with_background_disk_io(cancellation.clone(), async {
|
||||
let worker = tokio::spawn(with_detached_disk_io_context(async move {
|
||||
let context = current_disk_io_context().expect("owned commit retains its class");
|
||||
assert_eq!(context.class, DiskIoClass::Background);
|
||||
context.background_permit(&worker_admission).await
|
||||
}));
|
||||
wait_for_background_waiters(&admission, 1).await;
|
||||
cancellation.cancel();
|
||||
tokio::task::yield_now().await;
|
||||
assert!(!worker.is_finished(), "caller cancellation must not abandon a queued commit");
|
||||
drop(foreground);
|
||||
let permit = tokio::time::timeout(Duration::from_secs(1), worker)
|
||||
.await
|
||||
.expect("commit admission should finish after foreground work")
|
||||
.expect("owned worker joins")
|
||||
.expect("owned commit survives waiter cancellation");
|
||||
assert_eq!(admission.background_admissions_for_tests(), 1);
|
||||
drop(permit);
|
||||
})
|
||||
.await;
|
||||
assert!(!admission.background_active_for_tests());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn detached_commit_does_not_inherit_a_parent_disk_lease() {
|
||||
let admission = DiskIoAdmission::new("parent-lease".to_string());
|
||||
let cancellation = CancellationToken::new();
|
||||
let permit = admission.background(&cancellation).await.expect("parent disk admission");
|
||||
with_background_disk_io(
|
||||
cancellation,
|
||||
with_disk_io_permit(Some(permit), async {
|
||||
tokio::spawn(with_detached_disk_io_context(async {
|
||||
assert!(current_disk_io_context().is_none(), "do not queue behind the parent's disk lease");
|
||||
}))
|
||||
.await
|
||||
.expect("resource-only commit joins while parent keeps its disk lease");
|
||||
assert!(admission.background_active_for_tests());
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
assert!(!admission.background_active_for_tests());
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn detached_commit_bounds_queued_admission_without_extending_an_existing_deadline() {
|
||||
let admission = DiskIoAdmission::new("stalled-background-disk");
|
||||
let held = admission
|
||||
.background(&CancellationToken::new())
|
||||
.await
|
||||
.expect("existing native I/O");
|
||||
let deadline = Instant::now() + Duration::from_secs(2);
|
||||
let worker_admission = admission.clone();
|
||||
let error = with_background_disk_io(CancellationToken::new(), async {
|
||||
let mut context = current_disk_io_context().expect("background scope");
|
||||
context.admission_deadline = Some(deadline);
|
||||
with_disk_io_context(context, async {
|
||||
let worker = tokio::spawn(with_detached_disk_io_context(async move {
|
||||
let context = current_disk_io_context().expect("owned scope");
|
||||
assert_eq!(
|
||||
context.admission_deadline,
|
||||
Some(deadline),
|
||||
"nested ownership must not reset admission time"
|
||||
);
|
||||
context.background_permit(&worker_admission).await
|
||||
}));
|
||||
wait_for_background_waiters(&admission, 1).await;
|
||||
tokio::time::advance(Duration::from_secs(2)).await;
|
||||
worker
|
||||
.await
|
||||
.expect("owned admission worker")
|
||||
.expect_err("stalled disk must not retain queued commit")
|
||||
})
|
||||
.await
|
||||
})
|
||||
.await;
|
||||
assert_eq!(error.kind(), io::ErrorKind::TimedOut);
|
||||
assert_eq!(admission.background_waiters.load(Ordering::Acquire), 0);
|
||||
assert!(admission.background_active_for_tests(), "timeout must not interrupt admitted native I/O");
|
||||
drop(held);
|
||||
assert!(!admission.background_active_for_tests());
|
||||
}
|
||||
|
||||
struct BlockingWorker(tokio::task::JoinHandle<()>);
|
||||
|
||||
async fn wait_for_background_waiters(admission: &DiskIoAdmission, expected: usize) {
|
||||
@@ -1072,6 +1202,7 @@ mod tests {
|
||||
DiskIoContext {
|
||||
class: DiskIoClass::Foreground,
|
||||
cancellation: CancellationToken::new(),
|
||||
admission_deadline: None,
|
||||
admission: None,
|
||||
permit: None,
|
||||
resource_unit: None,
|
||||
|
||||
@@ -79,12 +79,21 @@ impl BackgroundMemoryBudget {
|
||||
pub(crate) fn reserve(self: &Arc<Self>, bytes: usize) -> io::Result<Reservation> {
|
||||
let bytes = u64::try_from(bytes).map_err(|_| io::Error::from(io::ErrorKind::OutOfMemory))?;
|
||||
let mut state = self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
let used = state
|
||||
.used
|
||||
.checked_add(bytes)
|
||||
.ok_or_else(|| io::Error::new(io::ErrorKind::WouldBlock, BackgroundMemoryExhausted))?;
|
||||
if bytes != 0 && (Instant::now() >= state.valid_until || used > state.limit) {
|
||||
return Err(io::Error::new(io::ErrorKind::WouldBlock, BackgroundMemoryExhausted));
|
||||
let expired = Instant::now() >= state.valid_until;
|
||||
let exhausted = || {
|
||||
io::Error::new(
|
||||
io::ErrorKind::WouldBlock,
|
||||
BackgroundMemoryExhausted {
|
||||
requested: bytes,
|
||||
used: state.used,
|
||||
limit: state.limit,
|
||||
expired,
|
||||
},
|
||||
)
|
||||
};
|
||||
let used = state.used.checked_add(bytes).ok_or_else(exhausted)?;
|
||||
if bytes != 0 && (expired || used > state.limit) {
|
||||
return Err(exhausted());
|
||||
}
|
||||
state.used = used;
|
||||
Ok(Reservation {
|
||||
@@ -95,11 +104,20 @@ impl BackgroundMemoryBudget {
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct BackgroundMemoryExhausted;
|
||||
struct BackgroundMemoryExhausted {
|
||||
requested: u64,
|
||||
used: u64,
|
||||
limit: u64,
|
||||
expired: bool,
|
||||
}
|
||||
|
||||
impl std::fmt::Display for BackgroundMemoryExhausted {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
formatter.write_str("background buffer memory budget exhausted")
|
||||
write!(
|
||||
formatter,
|
||||
"background buffer memory budget exhausted (requested={}, used={}, limit={}, expired={})",
|
||||
self.requested, self.used, self.limit, self.expired,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -321,6 +339,31 @@ impl io::Write for BackgroundEncodeBuffer {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn rejected_reservation_reports_capacity_and_expiration_without_charging() {
|
||||
let owner = BackgroundMemoryBudget::new();
|
||||
owner.set_sampled_limit(64, Duration::from_secs(3));
|
||||
let held = owner.reserve(32).expect("initial reservation fits");
|
||||
let error = owner.reserve(33).err().expect("one byte over remaining capacity");
|
||||
assert!(is_memory_pressure(&error));
|
||||
assert_eq!(
|
||||
error.to_string(),
|
||||
"background buffer memory budget exhausted (requested=33, used=32, limit=64, expired=false)"
|
||||
);
|
||||
assert_eq!(owner.snapshot().used, 32);
|
||||
|
||||
tokio::time::advance(Duration::from_secs(3)).await;
|
||||
let error = owner.reserve(1).err().expect("expired lease rejects a fitting request");
|
||||
assert!(is_memory_pressure(&error));
|
||||
assert_eq!(
|
||||
error.to_string(),
|
||||
"background buffer memory budget exhausted (requested=1, used=32, limit=64, expired=true)"
|
||||
);
|
||||
assert_eq!(owner.snapshot().used, 32);
|
||||
drop(held);
|
||||
assert_eq!(owner.snapshot().used, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn encoded_buffer_growth_retains_both_allocations_and_the_last_slice() {
|
||||
use std::io::Write;
|
||||
|
||||
@@ -140,6 +140,28 @@ pub(crate) fn admit(binding: &PhysicalDiskBinding) -> io::Result<()> {
|
||||
.unwrap_or(Ok(()))
|
||||
}
|
||||
|
||||
/// Wait only at the namespace RPC entry, before storage locks or native I/O.
|
||||
/// The caller owns the deadline; dropping this future releases its queue turn.
|
||||
pub(crate) async fn admit_scanner_preflight(binding: &PhysicalDiskBinding) -> io::Result<()> {
|
||||
let node = REMOTE_RESOURCES
|
||||
.try_with(|context| context.admitted.borrow().is_none().then(|| context.node.clone()))
|
||||
.ok()
|
||||
.flatten();
|
||||
let Some(node) = node else {
|
||||
return admit(binding);
|
||||
};
|
||||
let deferred = || io::Error::from(io::ErrorKind::WouldBlock);
|
||||
let plan = binding.plan().ok_or_else(deferred)?;
|
||||
let permit = plan.acquire(&node).await.ok_or_else(deferred)?;
|
||||
REMOTE_RESOURCES.with(|context| {
|
||||
let mut admitted = context.admitted.borrow_mut();
|
||||
if admitted.is_none() {
|
||||
*admitted = Some(permit);
|
||||
}
|
||||
});
|
||||
admit(binding)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -162,6 +184,76 @@ mod tests {
|
||||
node
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn scanner_preflight_cancellation_and_timeout_release_queued_capacity() {
|
||||
let node = node();
|
||||
let gate = AdaptiveAdmission::new(NonZeroU16::MIN, 1);
|
||||
let binding = PhysicalDiskBinding::default();
|
||||
binding.set_test_capacity(vec![gate.clone()]).expect("device plan");
|
||||
for deadline in [false, true] {
|
||||
let held = gate.try_acquire().expect("concurrent disk operation");
|
||||
with_remote_background_io(Some(RemoteBackgroundIo::new(node.clone())), async {
|
||||
let mut wait = Box::pin(admit_scanner_preflight(&binding));
|
||||
assert!(futures_util::poll!(&mut wait).is_pending());
|
||||
assert_eq!(node.snapshot().active, 0, "queued disk work cannot reserve node capacity");
|
||||
if deadline {
|
||||
assert!(tokio::time::timeout(std::time::Duration::from_secs(1), wait).await.is_err());
|
||||
} else {
|
||||
drop(wait);
|
||||
}
|
||||
drop(held);
|
||||
admit(&binding).expect("cancelled queue turn cannot block the immediate path");
|
||||
})
|
||||
.await;
|
||||
assert_eq!([node.snapshot().active, gate.snapshot().active], [0, 0]);
|
||||
assert_eq!([node.completed(), gate.completed()], [Some(0), Some(0)]);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_preflight_rejects_a_plan_replaced_while_waiting() {
|
||||
let node = node();
|
||||
let gate = AdaptiveAdmission::new(NonZeroU16::MIN, 1);
|
||||
let binding = PhysicalDiskBinding::default();
|
||||
binding.set_test_capacity(vec![gate.clone()]).expect("old plan");
|
||||
let held = gate.try_acquire().expect("busy disk");
|
||||
with_remote_background_io(Some(RemoteBackgroundIo::new(node.clone())), async {
|
||||
let wait = admit_scanner_preflight(&binding);
|
||||
tokio::pin!(wait);
|
||||
assert!(futures_util::poll!(&mut wait).is_pending());
|
||||
binding
|
||||
.set_test_capacity(vec![AdaptiveAdmission::new(NonZeroU16::MIN, 1)])
|
||||
.expect("new plan");
|
||||
drop(held);
|
||||
assert_eq!(wait.await.expect_err("old plan is invalid").kind(), io::ErrorKind::WouldBlock);
|
||||
assert!(resources().is_none());
|
||||
})
|
||||
.await;
|
||||
assert_eq!([node.snapshot().active, gate.snapshot().active], [0, 0]);
|
||||
assert_eq!(node.completed(), Some(0));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_preflight_never_waits_for_additional_devices_under_existing_admission() {
|
||||
let node = node();
|
||||
let first = PhysicalDiskBinding::default();
|
||||
let gate = AdaptiveAdmission::new(NonZeroU16::MIN, 1);
|
||||
first.set_test_capacity(vec![gate.clone()]).expect("first plan");
|
||||
let other = PhysicalDiskBinding::default();
|
||||
other
|
||||
.set_test_capacity(vec![AdaptiveAdmission::new(NonZeroU16::MIN, 0)])
|
||||
.expect("closed plan");
|
||||
with_remote_background_io(Some(RemoteBackgroundIo::new(node.clone())), async {
|
||||
admit(&first).expect("existing request admission");
|
||||
let other = admit_scanner_preflight(&other);
|
||||
tokio::pin!(other);
|
||||
assert!(matches!(futures_util::poll!(other), std::task::Poll::Ready(Err(error)) if error.kind() == io::ErrorKind::WouldBlock));
|
||||
assert_eq!([node.snapshot().active, gate.snapshot().active], [1, 1]);
|
||||
})
|
||||
.await;
|
||||
assert_eq!([node.snapshot().active, gate.snapshot().active], [0, 0]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn receiver_reuses_one_receipt_across_phases_and_releases_only_native_subsets() {
|
||||
let node = node();
|
||||
|
||||
@@ -3692,10 +3692,17 @@ pub(crate) fn has_delete_version_test_failpoints() -> bool {
|
||||
#[cfg(not(test))]
|
||||
pub(crate) fn has_delete_version_test_failpoints() -> bool {
|
||||
#[cfg(all(feature = "test-util", not(windows)))]
|
||||
if os::prepared_publication_test_hooks::has_any_hooks() {
|
||||
return true;
|
||||
let prepared_hooks = os::prepared_publication_test_hooks::has_any_hooks();
|
||||
#[cfg(not(all(feature = "test-util", not(windows))))]
|
||||
let prepared_hooks = false;
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
{
|
||||
prepared_hooks || std::env::var_os("RUSTFS_DELETE_RECOVERY_BARRIER_DIR").is_some()
|
||||
}
|
||||
#[cfg(not(feature = "e2e-test-hooks"))]
|
||||
{
|
||||
prepared_hooks
|
||||
}
|
||||
cfg!(feature = "e2e-test-hooks") && std::env::var_os("RUSTFS_DELETE_RECOVERY_BARRIER_DIR").is_some()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -12481,9 +12488,13 @@ impl LocalDisk {
|
||||
// Files written here (format.json, ...) stay where they land — no
|
||||
// rename follows — so the new directory entry must be fsynced too.
|
||||
// System-critical volumes are pinned to strict by effective_durability;
|
||||
// only the legacy full-off switch (historical semantics) skips this.
|
||||
let durability = effective_durability(volume);
|
||||
let sync = durability.metadata_sync_mode();
|
||||
// A consumed shutdown receipt must never survive a later writer crash,
|
||||
// including when data-shard syncing is disabled.
|
||||
let sync = if volume == RUSTFS_META_BUCKET && path == super::UNCLEAN_SHUTDOWN_MARKER_PATH {
|
||||
SyncMode::FileAndDir
|
||||
} else {
|
||||
effective_durability(volume).metadata_sync_mode()
|
||||
};
|
||||
self.write_all_private(volume, path, data, sync, &volume_dir).await?;
|
||||
|
||||
Ok(())
|
||||
@@ -22552,6 +22563,33 @@ mod test {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[allow(clippy::await_holding_lock)]
|
||||
async fn shutdown_receipt_is_durable_with_legacy_sync_disabled() {
|
||||
let _mode = durability_mode_override::set(DurabilityMode::LegacyOff);
|
||||
let dir = tempfile::tempdir().expect("shutdown marker disk");
|
||||
let endpoint = Endpoint::try_from(dir.path().to_str().expect("disk path")).expect("local endpoint");
|
||||
let disk = LocalDisk::new(&endpoint, false).await.expect("local disk");
|
||||
ensure_test_volume(&disk, RUSTFS_META_BUCKET).await;
|
||||
disk.write_all(
|
||||
RUSTFS_META_BUCKET,
|
||||
crate::disk::UNCLEAN_SHUTDOWN_MARKER_PATH,
|
||||
Bytes::from_static(b"running"),
|
||||
)
|
||||
.await
|
||||
.expect("consume shutdown receipt");
|
||||
let parent = disk
|
||||
.get_object_path(RUSTFS_META_BUCKET, crate::disk::UNCLEAN_SHUTDOWN_MARKER_PATH)
|
||||
.expect("marker path")
|
||||
.parent()
|
||||
.expect("marker parent")
|
||||
.to_path_buf();
|
||||
assert!(
|
||||
os::fsync_dir_recorder::was_fsynced(&parent),
|
||||
"a stale clean receipt must not reappear after a crash"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[allow(clippy::await_holding_lock)]
|
||||
async fn test_write_all_public_still_fsyncs_parent_dir() {
|
||||
|
||||
@@ -122,17 +122,27 @@ fn validate_certificate_shape(
|
||||
}
|
||||
|
||||
impl LocalDisk {
|
||||
// Give readonly inspection its own future layout while retaining one
|
||||
// selected Box per request and constructing it outside caller poll frames.
|
||||
// Separate inspection, abort and recovery layouts, and construct the
|
||||
// selected boxed future outside caller poll frames.
|
||||
#[inline(never)]
|
||||
pub(super) fn recover_multipart_source_native(
|
||||
&self,
|
||||
request: MultipartSourceRequest,
|
||||
) -> impl std::future::Future<Output = Result<MultipartSourceResponse>> + Send + '_ {
|
||||
if matches!(&request.operation, MultipartSourceOperation::Inspect { .. }) {
|
||||
futures::future::Either::Left(Box::pin(self.inspect_multipart_source_native(request)))
|
||||
} else {
|
||||
futures::future::Either::Right(Box::pin(self.recover_multipart_source_native_inner(request)))
|
||||
match &request.operation {
|
||||
MultipartSourceOperation::Inspect { .. } => {
|
||||
futures::future::Either::Left(Box::pin(self.inspect_multipart_source_native(request)))
|
||||
}
|
||||
MultipartSourceOperation::UpgradeLegacy { .. }
|
||||
| MultipartSourceOperation::PrepareAbort { .. }
|
||||
| MultipartSourceOperation::CommitAbort { .. }
|
||||
| MultipartSourceOperation::DeleteAborted { .. }
|
||||
| MultipartSourceOperation::ConfirmCleanupTarget { .. } => futures::future::Either::Right(
|
||||
futures::future::Either::Left(Box::pin(self.abort_multipart_source_native(request))),
|
||||
),
|
||||
_ => futures::future::Either::Right(futures::future::Either::Right(Box::pin(
|
||||
self.recover_multipart_source_native_inner(request),
|
||||
))),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -180,20 +190,24 @@ impl LocalDisk {
|
||||
}))
|
||||
}
|
||||
|
||||
async fn abort_multipart_source_native(&self, request: MultipartSourceRequest) -> Result<MultipartSourceResponse> {
|
||||
request.validate()?;
|
||||
let store = self.open_multipart_source_store(&request.path, None).await?;
|
||||
self.recover_multipart_abort_owned(store, request.operation).await
|
||||
}
|
||||
|
||||
async fn recover_multipart_source_native_inner(&self, request: MultipartSourceRequest) -> Result<MultipartSourceResponse> {
|
||||
request.validate()?;
|
||||
// Source RW ownership precedes source namespace, target metadata, and
|
||||
// target publication ownership. Every filesystem tail retains it.
|
||||
let store = self.open_multipart_source_store(&request.path, None).await?;
|
||||
match request.operation {
|
||||
operation @ (MultipartSourceOperation::UpgradeLegacy { .. }
|
||||
MultipartSourceOperation::Inspect { .. }
|
||||
| MultipartSourceOperation::UpgradeLegacy { .. }
|
||||
| MultipartSourceOperation::PrepareAbort { .. }
|
||||
| MultipartSourceOperation::CommitAbort { .. }
|
||||
| MultipartSourceOperation::DeleteAborted { .. }
|
||||
| MultipartSourceOperation::ConfirmCleanupTarget { .. }) => {
|
||||
self.recover_multipart_abort_owned(store, operation).await
|
||||
}
|
||||
MultipartSourceOperation::Inspect { .. } => Err(DiskError::UnsupportedDisk),
|
||||
| MultipartSourceOperation::ConfirmCleanupTarget { .. } => Err(DiskError::UnsupportedDisk),
|
||||
MultipartSourceOperation::Claim { expected, round } => {
|
||||
let loaded = store.load().await?;
|
||||
require_expected(&loaded, &expected)?;
|
||||
|
||||
@@ -316,11 +316,18 @@ pub(super) fn put_active_multipart_source_head(source: impl AsRef<Path>, raw: By
|
||||
}
|
||||
|
||||
fn stamp_matches_active_disk(source: &Path, expected_head_stamp: FileStamp) -> bool {
|
||||
FileStamp::from_path(&source.join(STORAGE_FORMAT_FILE)) == Some(expected_head_stamp)
|
||||
&& !(cfg!(test)
|
||||
&& (source.join(STATE).exists()
|
||||
|| source.join(crate::disk::MULTIPART_RECOVERY_DIR).exists()
|
||||
|| source.join(".mpu-completions").exists()))
|
||||
let stamp_matches = FileStamp::from_path(&source.join(STORAGE_FORMAT_FILE)) == Some(expected_head_stamp);
|
||||
#[cfg(test)]
|
||||
{
|
||||
stamp_matches
|
||||
&& !source.join(STATE).exists()
|
||||
&& !source.join(crate::disk::MULTIPART_RECOVERY_DIR).exists()
|
||||
&& !source.join(".mpu-completions").exists()
|
||||
}
|
||||
#[cfg(not(test))]
|
||||
{
|
||||
stamp_matches
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn get_active_multipart_source_head(source: &Path) -> Option<(Bytes, FileInfo)> {
|
||||
@@ -600,11 +607,13 @@ impl SourceStore {
|
||||
let disk_id = self.disk_id;
|
||||
let physical_slot = self.physical_slot;
|
||||
let disk_ids = self.disk_ids.clone();
|
||||
// Keep the loaded state out of each blocking-queue and oneshot poll frame.
|
||||
os::run_blocking_namespace_operation(self.lease.clone(), move || {
|
||||
Ok(load_source(&source, &base, &root, &path, disk_id, physical_slot, &disk_ids))
|
||||
Ok(load_source(&source, &base, &root, &path, disk_id, physical_slot, &disk_ids).map(Box::new))
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)?
|
||||
.map(|loaded| *loaded)
|
||||
}
|
||||
pub async fn persist(&self, state: &SourceState) -> Result<()> {
|
||||
self.persist_with_source_sync(state, false).await.map(|_| ())
|
||||
|
||||
@@ -45,7 +45,7 @@ pub use local::pool_meta_native::{PoolMetaNativeCasCondition, PoolMetaNativeFile
|
||||
pub(crate) use io_admission::{DiskIoClass, test_resource_units};
|
||||
pub(crate) use io_admission::{
|
||||
PhysicalHealBinding, acquire_context_disk_io_permit, current_disk_io_context, current_resource_unit, has_resource_io_scope,
|
||||
with_optional_disk_io_context, with_optional_resource_unit,
|
||||
with_detached_disk_io_context, with_optional_disk_io_context, with_optional_resource_unit,
|
||||
};
|
||||
|
||||
pub const RUSTFS_META_BUCKET: &str = ".rustfs.sys";
|
||||
@@ -77,6 +77,8 @@ pub fn validate_format_control_path(path: &str) -> Result<()> {
|
||||
/// Per-disk marker present while an erasure-set heal is rebuilding this disk.
|
||||
/// `LocalDisk::disk_info` reports `healing = true` while the file exists.
|
||||
pub const HEALING_MARKER_PATH: &str = "healing.bin";
|
||||
/// Boot-time crash fence and clean scanner continuity receipt.
|
||||
pub const UNCLEAN_SHUTDOWN_MARKER_PATH: &str = "unclean-shutdown";
|
||||
pub const STORAGE_FORMAT_FILE: &str = "xl.meta";
|
||||
pub const STORAGE_FORMAT_FILE_BACKUP: &str = "xl.meta.bkp";
|
||||
/// Original upload authority after it stops admitting active part mutations.
|
||||
|
||||
@@ -417,11 +417,15 @@ pub(crate) mod prepared_publication_test_hooks {
|
||||
}
|
||||
|
||||
pub(crate) fn has_any_hooks() -> bool {
|
||||
let before_publication = !BEFORE_PUBLICATION.lock().is_empty();
|
||||
#[cfg(test)]
|
||||
if !RENAME_DESTINATIONS.lock().is_empty() || !HARDLINK_DESTINATIONS.lock().is_empty() {
|
||||
return true;
|
||||
{
|
||||
before_publication || !RENAME_DESTINATIONS.lock().is_empty() || !HARDLINK_DESTINATIONS.lock().is_empty()
|
||||
}
|
||||
#[cfg(not(test))]
|
||||
{
|
||||
before_publication
|
||||
}
|
||||
!BEFORE_PUBLICATION.lock().is_empty()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -1045,6 +1045,13 @@ impl InstanceContext {
|
||||
self.local_disk_map.clone()
|
||||
}
|
||||
|
||||
/// Snapshot the boot-time disk registry before a store is published.
|
||||
/// Missing local drives remain explicit so startup receipts cannot treat
|
||||
/// an available subset as the complete instance inventory.
|
||||
pub async fn local_disk_inventory(&self) -> Vec<Option<DiskStore>> {
|
||||
self.local_disk_map.read().await.values().cloned().collect()
|
||||
}
|
||||
|
||||
/// Handle to this instance's local disk-id→endpoint map.
|
||||
pub(crate) fn local_disk_id_map(&self) -> Arc<RwLock<HashMap<Uuid, String>>> {
|
||||
self.local_disk_id_map.clone()
|
||||
|
||||
@@ -5958,6 +5958,7 @@ mod tests {
|
||||
#[test]
|
||||
fn scanner_dirty_usage_acknowledgement_preserves_newer_pending_work() {
|
||||
let activity = |dirty_usage_pending| ScannerPeerActivity {
|
||||
checkpoint_instance_id: None,
|
||||
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
|
||||
namespace_generation: 1,
|
||||
maintenance_generation: 1,
|
||||
|
||||
@@ -4638,10 +4638,11 @@ impl SetDisks {
|
||||
write_quorum,
|
||||
};
|
||||
let resource_unit = resource_unit.clone();
|
||||
tasks.spawn(crate::disk::with_optional_resource_unit(resource_unit, async move {
|
||||
let producer = crate::disk::with_optional_resource_unit(resource_unit, async move {
|
||||
let outcome = Self::run_unary_rename_slot(context).await;
|
||||
let _ = outcome_tx.send(outcome).await;
|
||||
}));
|
||||
});
|
||||
tasks.spawn(crate::disk::with_detached_disk_io_context(producer));
|
||||
}
|
||||
for group in fanout_plan.batch_groups {
|
||||
let outcome_tx = outcome_tx.clone();
|
||||
@@ -4652,7 +4653,7 @@ impl SetDisks {
|
||||
let group_disks = disks.clone();
|
||||
let group_tokens = fence_tokens.clone();
|
||||
let resource_unit = resource_unit.clone();
|
||||
tasks.spawn(crate::disk::with_optional_resource_unit(resource_unit, async move {
|
||||
let producer = crate::disk::with_optional_resource_unit(resource_unit, async move {
|
||||
let prepared = join_all(group.slots.into_iter().map(|slot| {
|
||||
let context = RenameSlotContext {
|
||||
slot,
|
||||
@@ -4815,7 +4816,8 @@ impl SetDisks {
|
||||
}
|
||||
}
|
||||
}
|
||||
}));
|
||||
});
|
||||
tasks.spawn(crate::disk::with_detached_disk_io_context(producer));
|
||||
}
|
||||
drop(outcome_tx);
|
||||
RenameOutcomeProducers {
|
||||
@@ -4870,7 +4872,7 @@ impl SetDisks {
|
||||
|
||||
let coordinator_failure_receipt = rollback_receipt.clone();
|
||||
let early_ack_spawn_to_quorum_wait_started = rustfs_io_metrics::put_stage_timer();
|
||||
let tail_drain = tokio::spawn({
|
||||
let tail_drain = tokio::spawn(crate::disk::with_detached_disk_io_context({
|
||||
let fanout_src_bucket = src_bucket.clone();
|
||||
let fanout_src_object = src_object.clone();
|
||||
let fanout_dst_bucket = dst_bucket.clone();
|
||||
@@ -5124,7 +5126,7 @@ impl SetDisks {
|
||||
all_online_local_succeeded,
|
||||
})
|
||||
}
|
||||
});
|
||||
}));
|
||||
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_EARLY_ACK_SPAWN_TO_QUORUM_WAIT,
|
||||
@@ -5305,6 +5307,7 @@ impl SetDisks {
|
||||
rollback_fault_injection::after_fanout(&fanout_dst_object);
|
||||
(results, fanout_file_infos)
|
||||
};
|
||||
let fanout_future = crate::disk::with_detached_disk_io_context(fanout_future);
|
||||
let fanout = if protected_multipart {
|
||||
tokio::spawn(crate::disk::with_optional_resource_unit(
|
||||
crate::disk::current_resource_unit(),
|
||||
|
||||
@@ -6601,7 +6601,7 @@ impl SetDisks {
|
||||
let diagnostic_context = crate::cluster::rpc::runtime_sources::ReadVersionDiagnosticContext::capture_entry(
|
||||
rustfs_io_metrics::internode_metrics::ReadVersionEntrySource::PutCommitOwner,
|
||||
);
|
||||
let result = tokio::spawn(async move {
|
||||
let result = tokio::spawn(crate::disk::with_detached_disk_io_context(async move {
|
||||
let commit = Box::pin(commit(commit_submit_to_closure_enter_started, Some(child_token)));
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
return crate::cluster::rpc::runtime_sources::with_read_version_diagnostic_context(
|
||||
@@ -6612,7 +6612,7 @@ impl SetDisks {
|
||||
.await;
|
||||
#[cfg(not(feature = "diagnostic-read-version-caller"))]
|
||||
commit.await
|
||||
})
|
||||
}))
|
||||
.await
|
||||
.map_err(|err| Error::other(format!("put_object commit task failed: {err}")))?;
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
@@ -25096,6 +25096,64 @@ mod put_object_tmp_cleanup_tests {
|
||||
wait_for_tmp_workspace_to_drain(&dirs, "cancelled full-tail commit should release staging ownership").await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn background_put_retains_disk_admission_across_owned_commit() {
|
||||
use crate::disk::with_background_disk_io;
|
||||
|
||||
let (_dirs, disks, set) = hermetic_set_disks(4).await;
|
||||
let object = "background-cache-commit";
|
||||
let barriers: Vec<_> = (0..disks.len())
|
||||
.map(|index| rename_fanout_barrier::arm(object, index, rename_fanout_barrier::PHASE_RENAME))
|
||||
.collect();
|
||||
let writer = Arc::clone(&set);
|
||||
let put = tokio::spawn(async move {
|
||||
with_background_disk_io(CancellationToken::new(), async {
|
||||
let mut reader = PutObjReader::from_vec(b"scanner checkpoint".to_vec());
|
||||
writer
|
||||
.put_object(
|
||||
RUSTFS_META_BUCKET,
|
||||
object,
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
})
|
||||
.await
|
||||
});
|
||||
for barrier in &barriers {
|
||||
tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
|
||||
.await
|
||||
.expect("every disk reaches the commit boundary");
|
||||
}
|
||||
let local_disks: Vec<_> = disks
|
||||
.iter()
|
||||
.map(|disk| match disk.as_ref() {
|
||||
disk::Disk::Local(local) => local,
|
||||
disk::Disk::Remote(_) => panic!("hermetic fixture must use local disks"),
|
||||
})
|
||||
.collect();
|
||||
let before: Vec<_> = local_disks
|
||||
.iter()
|
||||
.map(|disk| disk.background_io_admissions_for_tests())
|
||||
.collect();
|
||||
for barrier in barriers {
|
||||
barrier.release();
|
||||
}
|
||||
put.await.expect("owned put task").expect("background checkpoint commits");
|
||||
for ((disk, local), before) in disks.iter().zip(local_disks).zip(before) {
|
||||
assert!(
|
||||
local.background_io_admissions_for_tests() > before,
|
||||
"the detached rename must retain background admission on every disk"
|
||||
);
|
||||
disk.read_version("", RUSTFS_META_BUCKET, object, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("committed checkpoint remains readable");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(capacity_dirty_scope)]
|
||||
async fn borrowed_write_context_rejects_wrong_namespace_and_cache_flag() {
|
||||
|
||||
@@ -33,9 +33,10 @@ pub(crate) mod internode {
|
||||
PUT_FILE_AUTH_TRAILER_MAC_LEN, PUT_FILE_AUTH_TRAILER_MAGIC, PUT_FILE_AUTH_V1, PUT_FILE_CAPABILITY_CHALLENGE_QUERY,
|
||||
PUT_FILE_CAPABILITY_QUERY, PUT_FILE_CAPABILITY_VERSION, PUT_FILE_NONCE_QUERY, PUT_FILE_SERVER_EPOCH_QUERY,
|
||||
PutFileCapabilityResponse, SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION, SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION,
|
||||
SCANNER_ACTIVITY_PROTOCOL_VERSION, SCANNER_ACTIVITY_V6_PROTOCOL_VERSION, SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES,
|
||||
SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION, SCANNER_DIRTY_USAGE_SNAPSHOT_RPC_MAX_MESSAGE_SIZE,
|
||||
WALK_DIR_BODY_SHA256_QUERY, WALK_DIR_STREAM_COMPLETION_QUERY, WALK_DIR_STREAM_COMPLETION_V1,
|
||||
SCANNER_ACTIVITY_PROTOCOL_VERSION, SCANNER_ACTIVITY_V6_PROTOCOL_VERSION, SCANNER_ACTIVITY_V7_PROTOCOL_VERSION,
|
||||
SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES, SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION,
|
||||
SCANNER_DIRTY_USAGE_SNAPSHOT_RPC_MAX_MESSAGE_SIZE, WALK_DIR_BODY_SHA256_QUERY, WALK_DIR_STREAM_COMPLETION_QUERY,
|
||||
WALK_DIR_STREAM_COMPLETION_V1,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -469,8 +469,8 @@ pub use peer::external_rpc_disk;
|
||||
use peer::init_local_peer;
|
||||
pub use peer::{
|
||||
BootstrapLocalTarget, all_local_disk, all_local_disk_path, find_local_disk_by_ref, find_local_disk_by_ref_for_rpc,
|
||||
get_disk_infos, init_local_disks, init_local_disks_with_instance_ctx, init_lock_clients, prewarm_local_disk_id_map,
|
||||
prewarm_local_disk_id_map_with_instance_ctx,
|
||||
find_local_disk_by_ref_for_scanner_rpc, get_disk_infos, init_local_disks, init_local_disks_with_instance_ctx,
|
||||
init_lock_clients, prewarm_local_disk_id_map, prewarm_local_disk_id_map_with_instance_ctx,
|
||||
};
|
||||
|
||||
pub struct ECStore {
|
||||
|
||||
@@ -261,6 +261,7 @@ impl ECStore {
|
||||
}
|
||||
|
||||
/// Execute on this instance's active local disk through the physical owner.
|
||||
/// A verified scanner publication uses the receiver's background disk admission.
|
||||
pub async fn rename_local_data(
|
||||
&self,
|
||||
disk_ref: &str,
|
||||
@@ -278,7 +279,7 @@ impl ECStore {
|
||||
} else {
|
||||
None
|
||||
};
|
||||
rename_local_data_with_ctx(
|
||||
let rename = rename_local_data_with_ctx(
|
||||
&self.ctx,
|
||||
disk_ref,
|
||||
source,
|
||||
@@ -289,8 +290,12 @@ impl ECStore {
|
||||
external_guard,
|
||||
namespace_owner: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
);
|
||||
if scanner_token.is_some() {
|
||||
crate::disk::with_background_disk_io(tokio_util::sync::CancellationToken::new(), rename).await
|
||||
} else {
|
||||
rename.await
|
||||
}
|
||||
}
|
||||
|
||||
/// Recovery stays bound to this admitted instance even after RPC cancellation.
|
||||
@@ -338,10 +343,24 @@ async fn write_local_format_control_with_ctx(
|
||||
}
|
||||
|
||||
async fn local_disk_candidate(ctx: &Arc<InstanceContext>, disk_ref: &str) -> DiskResult<(DiskStore, Option<Uuid>)> {
|
||||
local_disk_candidate_with_admission(ctx, disk_ref, RpcDiskAdmission::Immediate).await
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
enum RpcDiskAdmission {
|
||||
Immediate,
|
||||
ScannerPreflight,
|
||||
}
|
||||
|
||||
async fn local_disk_candidate_with_admission(
|
||||
ctx: &Arc<InstanceContext>,
|
||||
disk_ref: &str,
|
||||
admission: RpcDiskAdmission,
|
||||
) -> DiskResult<(DiskStore, Option<Uuid>)> {
|
||||
let map = ctx.local_disk_map();
|
||||
let path_disk = map.read().await.get(disk_ref).and_then(Option::as_ref).cloned();
|
||||
if let Some(disk) = path_disk {
|
||||
admit_rpc_disk(&disk)?;
|
||||
admit_rpc_disk(&disk, admission).await?;
|
||||
return Ok((disk, None));
|
||||
}
|
||||
let disk_id = Uuid::parse_str(disk_ref).map_err(|_| DiskError::DiskNotFound)?;
|
||||
@@ -351,7 +370,7 @@ async fn local_disk_candidate(ctx: &Arc<InstanceContext>, disk_ref: &str) -> Dis
|
||||
if let Some(disk) = cached_disk
|
||||
&& matches!(disk.as_ref(), Disk::Local(_))
|
||||
{
|
||||
admit_rpc_disk(&disk)?;
|
||||
admit_rpc_disk(&disk, admission).await?;
|
||||
if disk.get_disk_id().await? == Some(disk_id) {
|
||||
return Ok((disk, None));
|
||||
}
|
||||
@@ -370,13 +389,24 @@ async fn local_disk_candidate(ctx: &Arc<InstanceContext>, disk_ref: &str) -> Dis
|
||||
Err(DiskError::DiskNotFound)
|
||||
}
|
||||
|
||||
fn admit_rpc_disk(disk: &DiskStore) -> DiskResult<()> {
|
||||
async fn local_scanner_disk_candidate(ctx: &Arc<InstanceContext>, disk_ref: &str) -> DiskResult<(DiskStore, Option<Uuid>)> {
|
||||
let (disk, id) = local_disk_candidate_with_admission(ctx, disk_ref, RpcDiskAdmission::ScannerPreflight).await?;
|
||||
// A queued preflight must not resume on a replaced or quarantined attachment.
|
||||
admit_local_disk(ctx, &disk, id, false, None).await?;
|
||||
Ok((disk, id))
|
||||
}
|
||||
|
||||
async fn admit_rpc_disk(disk: &DiskStore, admission: RpcDiskAdmission) -> DiskResult<()> {
|
||||
use crate::disk::remote_io_admission;
|
||||
if remote_io_admission::active() {
|
||||
let binding = disk
|
||||
.physical_admission()
|
||||
.ok_or_else(|| DiskError::Io(std::io::Error::from(std::io::ErrorKind::WouldBlock)))?;
|
||||
remote_io_admission::admit(&binding).map_err(DiskError::Io)?;
|
||||
match admission {
|
||||
RpcDiskAdmission::Immediate => remote_io_admission::admit(&binding),
|
||||
RpcDiskAdmission::ScannerPreflight => remote_io_admission::admit_scanner_preflight(&binding).await,
|
||||
}
|
||||
.map_err(DiskError::Io)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -567,6 +597,17 @@ pub async fn find_local_disk_by_ref_for_rpc(disk_ref: &str) -> DiskResult<Option
|
||||
Ok(Some(disk))
|
||||
}
|
||||
|
||||
/// Resolve an authenticated namespace scan before its storage locks, waiting for
|
||||
/// initial background capacity. The caller must bound the wait with a deadline.
|
||||
/// Cancellation returns queued capacity; unknown or replaced targets are rejected.
|
||||
pub async fn find_local_disk_by_ref_for_scanner_rpc(disk_ref: &str) -> DiskResult<Option<DiskStore>> {
|
||||
if !crate::disk::remote_io_admission::active() {
|
||||
return Ok(find_local_disk_by_ref(disk_ref).await);
|
||||
}
|
||||
let (disk, _) = local_scanner_disk_candidate(&crate::runtime::global::current_ctx(), disk_ref).await?;
|
||||
Ok(Some(disk))
|
||||
}
|
||||
|
||||
/// Resolve the target from the verified listener's instance and acquire any background admission.
|
||||
#[cfg(feature = "internode-transport")]
|
||||
pub async fn external_rpc_disk(ctx: &Arc<InstanceContext>, disk_ref: &str) -> DiskResult<DiskStore> {
|
||||
@@ -1041,6 +1082,123 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn namespace_target_lookup_waits_for_busy_preflight_capacity() {
|
||||
use crate::disk::{RemoteBackgroundIo, with_remote_background_io};
|
||||
use rustfs_concurrency::adaptive::{
|
||||
AdaptiveAdmission, FeedbackController, FeedbackObservation, NodeAdmission, ResourcePressure,
|
||||
};
|
||||
use std::num::NonZeroU16;
|
||||
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
let root = tempfile::tempdir().expect("scanner target directory");
|
||||
let disk = target_disk(&ctx, root.path(), Uuid::new_v4()).await;
|
||||
let gate = AdaptiveAdmission::new(NonZeroU16::MIN, 1);
|
||||
disk.physical_admission()
|
||||
.expect("local binding")
|
||||
.set_test_capacity(vec![gate.clone()])
|
||||
.expect("physical device");
|
||||
let node = NodeAdmission::new(NonZeroU16::MIN);
|
||||
node.update(
|
||||
&mut FeedbackController::new(),
|
||||
Some(FeedbackObservation {
|
||||
generation: 1,
|
||||
max_concurrency: 1,
|
||||
pressure: ResourcePressure::Unknown,
|
||||
completed_units: None,
|
||||
}),
|
||||
);
|
||||
let held = gate.try_acquire().expect("concurrent scanner operation");
|
||||
let path = disk.endpoint().to_string();
|
||||
with_remote_background_io(Some(RemoteBackgroundIo::new(node.clone())), async {
|
||||
let lookup = local_scanner_disk_candidate(&ctx, &path);
|
||||
tokio::pin!(lookup);
|
||||
assert!(
|
||||
futures_util::poll!(lookup.as_mut()).is_pending(),
|
||||
"a namespace preflight must wait for busy capacity before storage locks, not fail the bucket"
|
||||
);
|
||||
assert_eq!(node.snapshot().active, 0, "waiting for the disk must not reserve node capacity");
|
||||
drop(held);
|
||||
let (selected, _) = tokio::time::timeout(std::time::Duration::from_secs(5), lookup)
|
||||
.await
|
||||
.expect("capacity release wakes namespace preflight")
|
||||
.expect("namespace target admission");
|
||||
assert!(Arc::ptr_eq(&selected, &disk));
|
||||
assert_eq!([node.snapshot().active, gate.snapshot().active], [1, 1]);
|
||||
})
|
||||
.await;
|
||||
assert_eq!([node.snapshot().active, gate.snapshot().active], [0, 0]);
|
||||
assert_eq!([node.completed(), gate.completed()], [Some(0), Some(0)]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn namespace_target_wait_rejects_quarantined_or_replaced_disk() {
|
||||
use crate::disk::{RemoteBackgroundIo, with_remote_background_io};
|
||||
use rustfs_concurrency::adaptive::{
|
||||
AdaptiveAdmission, FeedbackController, FeedbackObservation, NodeAdmission, ResourcePressure,
|
||||
};
|
||||
use std::num::NonZeroU16;
|
||||
|
||||
for replace in [false, true] {
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
let root = tempfile::tempdir().expect("target directory");
|
||||
let id = Uuid::new_v4();
|
||||
let disk = target_disk(&ctx, root.path(), id).await;
|
||||
assert!(record_local_disk_id_if_active(&ctx, &disk, id).await);
|
||||
let replacement = if replace {
|
||||
let replacement = target_disk(&ctx, root.path(), id).await;
|
||||
ctx.local_disk_map()
|
||||
.write()
|
||||
.await
|
||||
.insert(disk.endpoint().to_string(), Some(disk.clone()));
|
||||
Some(replacement)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let gate = AdaptiveAdmission::new(NonZeroU16::MIN, 1);
|
||||
disk.physical_admission()
|
||||
.expect("local binding")
|
||||
.set_test_capacity(vec![gate.clone()])
|
||||
.expect("physical device");
|
||||
let node = NodeAdmission::new(NonZeroU16::MIN);
|
||||
node.update(
|
||||
&mut FeedbackController::new(),
|
||||
Some(FeedbackObservation {
|
||||
generation: 1,
|
||||
max_concurrency: 1,
|
||||
pressure: ResourcePressure::Unknown,
|
||||
completed_units: None,
|
||||
}),
|
||||
);
|
||||
let held = gate.try_acquire().expect("busy disk");
|
||||
// Both lookup forms must revalidate the registry after queueing.
|
||||
let disk_ref = if replace {
|
||||
id.to_string()
|
||||
} else {
|
||||
disk.endpoint().to_string()
|
||||
};
|
||||
with_remote_background_io(Some(RemoteBackgroundIo::new(node.clone())), async {
|
||||
let lookup = local_scanner_disk_candidate(&ctx, &disk_ref);
|
||||
tokio::pin!(lookup);
|
||||
assert!(futures_util::poll!(&mut lookup).is_pending());
|
||||
if let Some(replacement) = replacement {
|
||||
assert!(!Arc::ptr_eq(&replacement, &disk));
|
||||
ctx.local_disk_map()
|
||||
.write()
|
||||
.await
|
||||
.insert(disk.endpoint().to_string(), Some(replacement));
|
||||
} else {
|
||||
ctx.local_disk_map().write().await.insert(disk.endpoint().to_string(), None);
|
||||
}
|
||||
drop(held);
|
||||
assert!(matches!(lookup.await, Err(DiskError::DiskNotFound)));
|
||||
})
|
||||
.await;
|
||||
assert_eq!([node.snapshot().active, gate.snapshot().active], [0, 0]);
|
||||
assert_eq!([node.completed(), gate.completed()], [Some(0), Some(0)]);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn target_uuid_cache_cannot_admit_a_different_format_at_the_same_path() {
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
@@ -1384,6 +1542,10 @@ mod tests {
|
||||
.await
|
||||
.expect("bounded physical entry");
|
||||
let at_entry = (ctx.namespace_commits_pending(), ctx.namespace_commit_generation());
|
||||
let Disk::Local(wrapper) = disk.as_ref() else {
|
||||
panic!("fixture must use a local disk");
|
||||
};
|
||||
assert!(!wrapper.background_io_active_for_tests(), "an ordinary rename stays foreground");
|
||||
assert!(!rename.is_finished(), "caller must still await the paused physical rename");
|
||||
rename.abort();
|
||||
let cancelled = tokio::time::timeout(Duration::from_secs(5), &mut rename)
|
||||
@@ -1502,7 +1664,18 @@ mod tests {
|
||||
})
|
||||
.await
|
||||
.expect("bounded physical entry");
|
||||
let Disk::Local(wrapper) = disk.as_ref() else {
|
||||
panic!("fixture must use a local disk");
|
||||
};
|
||||
assert!(
|
||||
wrapper.background_io_active_for_tests(),
|
||||
"a verified scanner publication must retain background admission on the receiving disk"
|
||||
);
|
||||
drop(rename);
|
||||
assert!(
|
||||
wrapper.background_io_active_for_tests(),
|
||||
"native publication keeps admission after RPC cancellation"
|
||||
);
|
||||
assert!(
|
||||
!store.scanner_data_usage_publication_blocked().await,
|
||||
"a pending namespace commit is not a local publication barrier"
|
||||
|
||||
@@ -165,22 +165,26 @@ impl HealManager {
|
||||
let mut recovery_disks = Vec::new();
|
||||
for disk in &local_disks {
|
||||
let endpoint = disk.endpoint();
|
||||
match disk
|
||||
.read_all(super::super::RUSTFS_META_BUCKET, super::super::UNCLEAN_SHUTDOWN_MARKER_PATH)
|
||||
.await
|
||||
{
|
||||
Ok(_) => unclean = true,
|
||||
Err(DiskError::FileNotFound) | Err(DiskError::VolumeNotFound) => {}
|
||||
Err(err) => {
|
||||
debug!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_UNCLEAN_SHUTDOWN,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
endpoint = %endpoint,
|
||||
error = ?err,
|
||||
"Unclean-shutdown marker check failed"
|
||||
);
|
||||
if let Some(previous) = super::super::shutdown_marker::previous_shutdown_was_unclean(&endpoint.to_string()) {
|
||||
unclean |= previous;
|
||||
} else {
|
||||
match disk
|
||||
.read_all(super::super::RUSTFS_META_BUCKET, super::super::UNCLEAN_SHUTDOWN_MARKER_PATH)
|
||||
.await
|
||||
{
|
||||
Ok(_) => unclean = true,
|
||||
Err(DiskError::FileNotFound) | Err(DiskError::VolumeNotFound) => {}
|
||||
Err(err) => {
|
||||
debug!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_UNCLEAN_SHUTDOWN,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
endpoint = %endpoint,
|
||||
error = ?err,
|
||||
"Unclean-shutdown marker check failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -23,6 +23,7 @@ pub mod progress;
|
||||
mod replacement_execution;
|
||||
pub(crate) mod replacement_readiness;
|
||||
pub mod resume;
|
||||
mod shutdown_marker;
|
||||
pub mod storage;
|
||||
pub(crate) mod storage_api;
|
||||
pub mod task;
|
||||
@@ -38,6 +39,7 @@ use storage_api::owner::{
|
||||
pub use erasure_healer::ErasureSetHealer;
|
||||
pub use manager::{HealAdmissionTelemetry, HealManager, HealOperationsSnapshot, HealPriorityCounts, HealSourceCounts};
|
||||
pub use resume::{CheckpointManager, ResumeCheckpoint, ResumeManager, ResumeState, ResumeUtils};
|
||||
pub use shutdown_marker::{consume_read_only_scanner_handoff, save_read_only_scanner_handoff};
|
||||
pub use task::{HealOptions, HealPriority, HealRequest, HealTask, HealType};
|
||||
|
||||
pub(crate) const DATA_USAGE_CACHE_NAME: &str = ECSTORE_DATA_USAGE_CACHE_NAME;
|
||||
@@ -49,7 +51,7 @@ pub(crate) const POOL_META_NAME: &str = ECSTORE_POOL_META_NAME;
|
||||
/// [`clear_unclean_shutdown_markers`] on graceful shutdown. Finding it at
|
||||
/// startup means the previous run crashed or lost power, so the heal manager
|
||||
/// proactively re-verifies all local erasure sets.
|
||||
pub(crate) const UNCLEAN_SHUTDOWN_MARKER_PATH: &str = "unclean-shutdown";
|
||||
pub(crate) const UNCLEAN_SHUTDOWN_MARKER_PATH: &str = storage_api::ECSTORE_UNCLEAN_SHUTDOWN_MARKER_PATH;
|
||||
|
||||
/// Remove the unclean-shutdown markers from all local disks. Call at the end of
|
||||
/// a graceful shutdown, after the data plane has stopped accepting writes.
|
||||
|
||||
@@ -0,0 +1,214 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
//! A clean, read-only handoff reuses the existing shutdown marker. Receipt-aware
|
||||
//! readers must consume it before legacy rollback; legacy startup alone cannot
|
||||
//! guarantee that checkpoint continuity expires before a write.
|
||||
|
||||
use super::{DiskError, DiskStore, EcstoreDiskAPI, RUSTFS_META_BUCKET, UNCLEAN_SHUTDOWN_MARKER_PATH, local_disk_map_read};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::HashMap;
|
||||
use std::sync::{LazyLock, Mutex};
|
||||
use uuid::Uuid;
|
||||
|
||||
static PREVIOUS_SHUTDOWN: LazyLock<Mutex<HashMap<String, bool>>> = LazyLock::new(|| Mutex::new(HashMap::new()));
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct ReadOnlyHandoff {
|
||||
schema: u32,
|
||||
owner: String,
|
||||
disk_id: Uuid,
|
||||
inventory: [u8; 32],
|
||||
checkpoint_instance_id: String,
|
||||
}
|
||||
|
||||
fn valid_instance(instance: &str) -> bool {
|
||||
instance.len() == 32
|
||||
&& instance
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
|
||||
&& instance.bytes().any(|byte| byte != b'0')
|
||||
}
|
||||
|
||||
fn decode(bytes: &[u8], owner: &str, disk_id: Option<Uuid>, inventory: Option<[u8; 32]>) -> Option<ReadOnlyHandoff> {
|
||||
if bytes.len() > 2048 || owner.is_empty() || disk_id.is_none_or(|id| id.is_nil()) {
|
||||
return None;
|
||||
}
|
||||
let record: ReadOnlyHandoff = serde_json::from_slice(bytes).ok()?;
|
||||
(record.schema == 1
|
||||
&& record.owner == owner
|
||||
&& Some(record.disk_id) == disk_id
|
||||
&& Some(record.inventory) == inventory
|
||||
&& valid_instance(&record.checkpoint_instance_id))
|
||||
.then_some(record)
|
||||
}
|
||||
|
||||
async fn local_inventory(disks: &[Option<DiskStore>]) -> Vec<(DiskStore, Option<Uuid>)> {
|
||||
let mut inventory = Vec::with_capacity(disks.len());
|
||||
for disk in disks.iter().flatten() {
|
||||
let id = disk.get_disk_id().await.ok().flatten();
|
||||
inventory.push((disk.clone(), id));
|
||||
}
|
||||
inventory
|
||||
}
|
||||
|
||||
fn inventory_digest(disks: &[(DiskStore, Option<Uuid>)]) -> Option<[u8; 32]> {
|
||||
if disks.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let mut members = disks
|
||||
.iter()
|
||||
.map(|(disk, id)| (disk.endpoint().to_string(), *id))
|
||||
.collect::<Vec<_>>();
|
||||
members.sort_unstable_by(|left, right| left.0.cmp(&right.0));
|
||||
if members.windows(2).any(|pair| pair[0].0 == pair[1].0) {
|
||||
return None;
|
||||
}
|
||||
let mut hash = Sha256::new();
|
||||
hash.update(b"scanner-read-only-local-disks-v1");
|
||||
for (endpoint, id) in members {
|
||||
let id = id.filter(|id| !id.is_nil())?;
|
||||
hash.update(u64::try_from(endpoint.len()).ok()?.to_be_bytes());
|
||||
hash.update(endpoint.as_bytes());
|
||||
hash.update(id.as_bytes());
|
||||
}
|
||||
Some(hash.finalize().into())
|
||||
}
|
||||
|
||||
pub(super) fn previous_shutdown_was_unclean(endpoint: &str) -> Option<bool> {
|
||||
PREVIOUS_SHUTDOWN
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.get(endpoint)
|
||||
.copied()
|
||||
}
|
||||
|
||||
/// Consume every local receipt before starting mutation producers or Scanner.
|
||||
/// Continuity requires matching receipts from the complete local disk inventory.
|
||||
/// The fresh unclean marker must be durable before returning a reusable identity.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns a storage error if a valid handoff cannot be consumed. Starting a
|
||||
/// writer in that case could leave an older receipt authorizing stale coverage.
|
||||
pub async fn consume_read_only_scanner_handoff(owner: &str, registry: &[Option<DiskStore>]) -> crate::Result<Option<String>> {
|
||||
let disks = local_inventory(registry).await;
|
||||
let inventory = if registry.iter().any(Option::is_none) {
|
||||
None
|
||||
} else {
|
||||
inventory_digest(&disks)
|
||||
};
|
||||
let mut candidate = None;
|
||||
let mut complete = inventory.is_some() && !owner.is_empty();
|
||||
let mut previous = HashMap::new();
|
||||
for (disk, disk_id) in disks {
|
||||
let (record, unclean, absent) = match disk.read_all(RUSTFS_META_BUCKET, UNCLEAN_SHUTDOWN_MARKER_PATH).await {
|
||||
Ok(bytes) => {
|
||||
let record = decode(&bytes, owner, disk_id, inventory);
|
||||
let unclean = record.is_none();
|
||||
(record, unclean, false)
|
||||
}
|
||||
Err(DiskError::FileNotFound | DiskError::VolumeNotFound) => (None, false, true),
|
||||
Err(_) => (None, true, false),
|
||||
};
|
||||
previous.insert(disk.endpoint().to_string(), unclean);
|
||||
if let Some(record) = &record {
|
||||
if candidate
|
||||
.as_ref()
|
||||
.is_some_and(|instance| instance != &record.checkpoint_instance_id)
|
||||
{
|
||||
complete = false;
|
||||
}
|
||||
candidate = Some(record.checkpoint_instance_id.clone());
|
||||
} else {
|
||||
complete = false;
|
||||
}
|
||||
// This system-volume write uses the disk's metadata durability path.
|
||||
// A crash after consumption leaves no clean receipt to reuse.
|
||||
if let Err(error) = disk
|
||||
.write_all(RUSTFS_META_BUCKET, UNCLEAN_SHUTDOWN_MARKER_PATH, bytes::Bytes::from_static(b"running"))
|
||||
.await
|
||||
{
|
||||
if !absent {
|
||||
return Err(error.into());
|
||||
}
|
||||
complete = false;
|
||||
}
|
||||
}
|
||||
*PREVIOUS_SHUTDOWN.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = previous;
|
||||
Ok(complete.then_some(candidate).flatten())
|
||||
}
|
||||
|
||||
/// Publish a handoff only after frontends, background producers and detached
|
||||
/// mutations have drained, and only if the caller observed no namespace or
|
||||
/// maintenance mutation during this process. No dirty hint is cleared here.
|
||||
///
|
||||
/// # Errors
|
||||
/// Fails closed on a missing disk identity, invalid owner, or persistence error.
|
||||
/// A partially written inventory cannot authorize continuity on the next start.
|
||||
pub async fn save_read_only_scanner_handoff(owner: &str, instance: &str) -> crate::Result<()> {
|
||||
if owner.is_empty() || !valid_instance(instance) {
|
||||
return Err(crate::Error::InvalidCheckpoint("invalid scanner handoff identity".to_owned()));
|
||||
}
|
||||
let registry = local_disk_map_read().await.values().cloned().collect::<Vec<_>>();
|
||||
if registry.iter().any(Option::is_none) {
|
||||
return Err(crate::Error::InvalidCheckpoint(
|
||||
"scanner handoff has an unavailable local disk".to_owned(),
|
||||
));
|
||||
}
|
||||
let disks = local_inventory(®istry).await;
|
||||
let inventory = inventory_digest(&disks)
|
||||
.ok_or_else(|| crate::Error::InvalidCheckpoint("scanner handoff has no complete disk inventory".to_owned()))?;
|
||||
for (disk, disk_id) in disks {
|
||||
let disk_id = disk_id
|
||||
.filter(|id| !id.is_nil())
|
||||
.ok_or_else(|| crate::Error::InvalidCheckpoint("scanner handoff has no disk identity".to_owned()))?;
|
||||
let record = ReadOnlyHandoff {
|
||||
schema: 1,
|
||||
owner: owner.to_owned(),
|
||||
disk_id,
|
||||
inventory,
|
||||
checkpoint_instance_id: instance.to_owned(),
|
||||
};
|
||||
let bytes = serde_json::to_vec(&record).map_err(|error| crate::Error::Serialization(error.to_string()))?;
|
||||
disk.write_all(RUSTFS_META_BUCKET, UNCLEAN_SHUTDOWN_MARKER_PATH, bytes.into())
|
||||
.await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn read_only_handoff_requires_exact_owner_disk_and_supported_state() {
|
||||
let id = Uuid::new_v4();
|
||||
let record = ReadOnlyHandoff {
|
||||
schema: 1,
|
||||
owner: "node-a".to_owned(),
|
||||
disk_id: id,
|
||||
inventory: [7; 32],
|
||||
checkpoint_instance_id: "1".repeat(32),
|
||||
};
|
||||
let bytes = serde_json::to_vec(&record).expect("valid handoff");
|
||||
assert!(decode(&bytes, "node-a", Some(id), Some([7; 32])).is_some());
|
||||
assert!(decode(&bytes, "node-a", Some(id), Some([8; 32])).is_none());
|
||||
for (owner, disk) in [("node-b", Some(id)), ("node-a", Some(Uuid::new_v4())), ("node-a", None)] {
|
||||
assert!(decode(&bytes, owner, disk, Some([7; 32])).is_none());
|
||||
}
|
||||
for bytes in [b"running".as_slice(), b"1730000000", b"{}", b"{\"schema\":1}"] {
|
||||
assert!(decode(bytes, "node-a", Some(id), Some([7; 32])).is_none());
|
||||
}
|
||||
for field in ["schema", "checkpoint_instance_id", "unknown"] {
|
||||
let mut value = serde_json::to_value(&record).expect("handoff JSON");
|
||||
value[field] = if field == "schema" {
|
||||
serde_json::json!(2)
|
||||
} else {
|
||||
serde_json::json!("0".repeat(32))
|
||||
};
|
||||
assert!(decode(&serde_json::to_vec(&value).expect("invalid handoff"), "node-a", Some(id), Some([7; 32])).is_none());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -25,6 +25,7 @@ pub(crate) use rustfs_ecstore::api::disk::{
|
||||
DiskStore as EcstoreDiskStore, HEAL_REPLACEMENT_RECOVERY_DIR as ECSTORE_HEAL_REPLACEMENT_RECOVERY_DIR,
|
||||
HEALING_MARKER_PATH as ECSTORE_HEALING_MARKER_PATH, RUSTFS_META_BUCKET as ECSTORE_RUSTFS_META_BUCKET,
|
||||
ReplacementExecutionLease as EcstoreReplacementExecutionLease,
|
||||
UNCLEAN_SHUTDOWN_MARKER_PATH as ECSTORE_UNCLEAN_SHUTDOWN_MARKER_PATH,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::disk::{
|
||||
|
||||
@@ -29,12 +29,11 @@ workspace = true
|
||||
|
||||
[features]
|
||||
default = []
|
||||
hotpath = ["hotpath/hotpath", "hotpath/tokio", "rustfs-io-metrics/hotpath"]
|
||||
hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc", "rustfs-io-metrics/hotpath-alloc"]
|
||||
hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu", "rustfs-io-metrics/hotpath-cpu"]
|
||||
hotpath = ["rustfs-io-metrics/hotpath"]
|
||||
hotpath-alloc = ["hotpath", "rustfs-io-metrics/hotpath-alloc"]
|
||||
hotpath-cpu = ["hotpath", "rustfs-io-metrics/hotpath-cpu"]
|
||||
|
||||
[dependencies]
|
||||
hotpath.workspace = true
|
||||
bytes = { workspace = true, features = ["serde"] }
|
||||
thiserror = { workspace = true }
|
||||
tokio = { workspace = true, features = ["io-util", "fs", "sync", "rt-multi-thread"] }
|
||||
|
||||
@@ -3351,6 +3351,9 @@ pub struct AllocatorMemoryObservation {
|
||||
pub reserved_bytes: Option<u64>,
|
||||
pub committed_bytes: Option<u64>,
|
||||
pub page_committed_bytes: Option<u64>,
|
||||
/// Allocator-reported block bytes, including size-class rounding and delayed statistics updates.
|
||||
/// This is not live requested bytes or resident memory.
|
||||
pub allocated_bytes: Option<u64>,
|
||||
pub malloc_requested_bytes: Option<u64>,
|
||||
pub malloc_requested_peak_bytes: Option<u64>,
|
||||
pub malloc_requested_total_bytes: Option<u64>,
|
||||
@@ -3385,6 +3388,11 @@ pub fn record_allocator_memory_observation(backend: &'static str, observation: A
|
||||
.set(if observation.reserved_bytes.is_some() { 1.0 } else { 0.0 });
|
||||
gauge!("rustfs_memory_allocator_committed_bytes_available", "backend" => backend)
|
||||
.set(if observation.committed_bytes.is_some() { 1.0 } else { 0.0 });
|
||||
gauge!("rustfs_memory_allocator_allocated_bytes_available", "backend" => backend)
|
||||
.set(if observation.allocated_bytes.is_some() { 1.0 } else { 0.0 });
|
||||
if let Some(allocated_bytes) = observation.allocated_bytes {
|
||||
gauge!("rustfs_memory_allocator_allocated_bytes", "backend" => backend).set(allocated_bytes as f64);
|
||||
}
|
||||
if let Some(reserved_bytes) = observation.reserved_bytes {
|
||||
gauge!("rustfs_memory_allocator_reserved_bytes", "backend" => backend).set(reserved_bytes as f64);
|
||||
}
|
||||
@@ -3859,6 +3867,41 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn allocator_block_bytes_preserve_decreases_missing_and_zero() {
|
||||
let _guard = METRICS_FLAG_LOCK.lock().unwrap_or_else(|e| e.into_inner());
|
||||
let sequence = [
|
||||
(None, None, 0.0),
|
||||
(Some(128), Some(128.0), 1.0),
|
||||
(Some(64), Some(64.0), 1.0),
|
||||
(None, Some(64.0), 0.0),
|
||||
(Some(0), Some(0.0), 1.0),
|
||||
];
|
||||
for (index, (_, expected, available)) in sequence.into_iter().enumerate() {
|
||||
// Debugging snapshots consume values; replay each prefix before its only snapshot.
|
||||
let recorder = DebuggingRecorder::new();
|
||||
let snapshotter = recorder.snapshotter();
|
||||
metrics::with_local_recorder(&recorder, || {
|
||||
set_metrics_enabled(true);
|
||||
for (current, ..) in sequence.iter().take(index + 1) {
|
||||
record_allocator_memory_observation(
|
||||
"mimalloc",
|
||||
AllocatorMemoryObservation {
|
||||
allocated_bytes: *current,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
}
|
||||
set_metrics_enabled(false);
|
||||
});
|
||||
let rows = snapshotter.snapshot().into_vec();
|
||||
assert_eq!(gauge_value(&rows, "rustfs_memory_allocator_allocated_bytes_available"), Some(available));
|
||||
assert_eq!(gauge_value(&rows, "rustfs_memory_allocator_allocated_bytes"), expected);
|
||||
assert_eq!(gauge_value(&rows, "rustfs_memory_allocator_malloc_requested_bytes_available"), Some(0.0));
|
||||
assert_eq!(gauge_value(&rows, "rustfs_memory_allocator_malloc_requested_bytes"), None);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn allocator_current_availability_tracks_missing_zero_and_recovery() {
|
||||
let _guard = METRICS_FLAG_LOCK.lock().unwrap_or_else(|e| e.into_inner());
|
||||
|
||||
@@ -6,7 +6,8 @@ Applies to `crates/kms/`.
|
||||
|
||||
When changing key-management behavior, verify compatibility with:
|
||||
|
||||
- `rustfs/src/storage/sse.rs`
|
||||
- `crates/sse/` (SSE key resolution; consumes this crate on the data path)
|
||||
- `rustfs/src/storage/sse.rs` (binds `crates/sse/` to the running server)
|
||||
- `rustfs/src/admin/handlers/kms.rs`
|
||||
- `rustfs/src/admin/handlers/kms_dynamic.rs`
|
||||
- `rustfs/src/admin/handlers/kms_keys.rs`
|
||||
|
||||
@@ -410,7 +410,10 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
|
||||
}
|
||||
};
|
||||
let handle = self.allocate_handle(HandleState::Dir(cursor))?;
|
||||
Ok(Handle { id, handle })
|
||||
Ok(Handle {
|
||||
id,
|
||||
handle: handle.into(),
|
||||
})
|
||||
}
|
||||
|
||||
/// READDIR body shared with the Handler trait wrapper. Removes the
|
||||
@@ -666,8 +669,11 @@ mod tests {
|
||||
assert_eq!(auth.continuation_token, call.continuation_token);
|
||||
}
|
||||
assert_eq!(authorized[2].continuation_token.as_ref().map(|value| value.as_str()), Some("next-page"));
|
||||
let denied =
|
||||
with_test_auth_override(|_, _, prefix| prefix != Some("private/"), driver.readdir_inner(2, handle.handle)).await;
|
||||
let denied = with_test_auth_override(
|
||||
|_, _, prefix| prefix != Some("private/"),
|
||||
driver.readdir_inner(2, String::from_utf8(handle.handle.into_bytes()).expect("utf8 handle")),
|
||||
)
|
||||
.await;
|
||||
assert!(denied.is_err(), "policy change must deny the next READDIR");
|
||||
assert_eq!(backend.list_objects_calls().len(), 2, "denied READDIR cannot reach storage");
|
||||
}
|
||||
@@ -716,7 +722,7 @@ mod tests {
|
||||
};
|
||||
let handle_id = driver.allocate_handle(HandleState::Dir(cursor)).expect("allocate");
|
||||
|
||||
let readdir_fut = driver.readdir(1, handle_id.clone());
|
||||
let readdir_fut = driver.readdir(1, handle_id.clone().into());
|
||||
|
||||
with_test_auth_override(|_, _, _| true, async {
|
||||
tokio::select! {
|
||||
@@ -758,7 +764,7 @@ mod tests {
|
||||
// token, no skipped entries).
|
||||
backend.clear_stall_list_objects_v2();
|
||||
backend.queue_list_objects_v2_ok_empty();
|
||||
let result = with_test_auth_override(|_, _, _| true, driver.readdir(2, handle_id)).await;
|
||||
let result = with_test_auth_override(|_, _, _| true, driver.readdir(2, handle_id.into())).await;
|
||||
// Empty page returns Eof per readdir's empty-Name-to-Eof translation.
|
||||
let err = result.expect_err("re-issued READDIR against an empty listing must return Eof, not Ok");
|
||||
assert!(
|
||||
@@ -785,7 +791,7 @@ mod tests {
|
||||
let handle_id = driver.allocate_handle(HandleState::Dir(cursor)).expect("allocate");
|
||||
|
||||
let (result, captured) =
|
||||
capture_tracing_at(Level::ERROR, with_test_auth_override(|_, _, _| true, driver.readdir(7, handle_id))).await;
|
||||
capture_tracing_at(Level::ERROR, with_test_auth_override(|_, _, _| true, driver.readdir(7, handle_id.into()))).await;
|
||||
let err = result.expect_err("exhausted cursor must return Eof");
|
||||
assert!(matches!(StatusCode::from(err), StatusCode::Eof));
|
||||
assert!(
|
||||
@@ -818,7 +824,7 @@ mod tests {
|
||||
let handle_id = driver.allocate_handle(HandleState::Dir(cursor)).expect("allocate");
|
||||
|
||||
let (result, captured) =
|
||||
capture_tracing_at(Level::ERROR, with_test_auth_override(|_, _, _| true, driver.readdir(8, handle_id))).await;
|
||||
capture_tracing_at(Level::ERROR, with_test_auth_override(|_, _, _| true, driver.readdir(8, handle_id.into()))).await;
|
||||
let err = result.expect_err("backend error must propagate as Err");
|
||||
assert!(!matches!(StatusCode::from(err), StatusCode::Eof), "backend error must not be Eof");
|
||||
assert!(
|
||||
|
||||
@@ -33,7 +33,9 @@ use super::write::{build_write_tombstone, fstat_reported_size, rejects_excl_or_t
|
||||
use crate::common::client::s3::StorageBackend;
|
||||
use crate::common::gateway::{AuthorizationError, S3Action};
|
||||
use crate::common::session::SessionContext;
|
||||
use russh_sftp::protocol::{Attrs, Data, File, FileAttributes, Handle, Name, OpenFlags, Packet, Status, StatusCode, Version};
|
||||
use russh_sftp::protocol::{
|
||||
Attrs, Data, File, FileAttributes, Handle, Name, OpenFlags, Packet, SftpHandle, Status, StatusCode, Version,
|
||||
};
|
||||
use rustfs_credentials::Credentials;
|
||||
use rustfs_gateway_types::dto::{AbortMultipartUploadInput, CopyObjectInput, ListObjectsV2Input};
|
||||
use rustfs_gateway_types::{BucketName, ObjectKey, UploadIdClaim};
|
||||
@@ -208,6 +210,12 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
|
||||
}
|
||||
}
|
||||
|
||||
/// russh-sftp 4 hands the Handler opaque `SftpHandle` bytes; the table
|
||||
/// keys remain the UTF-8 UUID strings this driver mints.
|
||||
fn handle_table_key(handle: SftpHandle) -> Result<String, SftpError> {
|
||||
String::from_utf8(handle.into_bytes()).map_err(|_| SftpError::code(StatusCode::Failure))
|
||||
}
|
||||
|
||||
/// Generate a fresh UUID v4 handle, insert the given state into the
|
||||
/// per-session handle table, and return the handle string. Enforces
|
||||
/// self.handles_per_session before any UUID generation. Cap-exceeded
|
||||
@@ -494,8 +502,9 @@ impl<S: StorageBackend + Send + Sync + 'static> russh_sftp::server::Handler for
|
||||
/// Streaming returns (next_part_number - 1) * part_size + buffer
|
||||
/// length. Failed returns the size recorded at the most recent
|
||||
/// successful write. An unknown handle returns Failure.
|
||||
#[tracing::instrument(level = "debug", skip(self), fields(id, handle = %handle), err(Debug))]
|
||||
async fn fstat(&mut self, id: u32, handle: String) -> Result<Attrs, Self::Error> {
|
||||
#[tracing::instrument(level = "debug", skip(self), fields(id, handle = ?handle), err(Debug))]
|
||||
async fn fstat(&mut self, id: u32, handle: SftpHandle) -> Result<Attrs, Self::Error> {
|
||||
let handle = Self::handle_table_key(handle)?;
|
||||
self.session_diag.stamp();
|
||||
let part_size = self.part_size;
|
||||
let result = self.with_handle_ref(&handle, |state| match state {
|
||||
@@ -541,8 +550,9 @@ impl<S: StorageBackend + Send + Sync + 'static> russh_sftp::server::Handler for
|
||||
/// exhaustion, so it is normal control flow on this handler. The
|
||||
/// instrument attribute therefore omits err(Debug) and non-Eof
|
||||
/// failures are surfaced via the explicit error log below.
|
||||
#[tracing::instrument(level = "debug", skip(self), fields(id, handle = %handle))]
|
||||
async fn readdir(&mut self, id: u32, handle: String) -> Result<Name, Self::Error> {
|
||||
#[tracing::instrument(level = "debug", skip(self), fields(id, handle = ?handle))]
|
||||
async fn readdir(&mut self, id: u32, handle: SftpHandle) -> Result<Name, Self::Error> {
|
||||
let handle = Self::handle_table_key(handle)?;
|
||||
self.session_diag.stamp();
|
||||
let result = self.readdir_inner(id, handle.clone()).await;
|
||||
self.session_diag.stamp();
|
||||
@@ -617,8 +627,9 @@ impl<S: StorageBackend + Send + Sync + 'static> russh_sftp::server::Handler for
|
||||
/// read-past-end-of-file, so it is normal control flow on this
|
||||
/// handler. The instrument attribute therefore omits err(Debug) and
|
||||
/// non-Eof failures are surfaced via the explicit error log below.
|
||||
#[tracing::instrument(level = "debug", skip(self), fields(id, handle = %handle, offset, len))]
|
||||
async fn read(&mut self, id: u32, handle: String, offset: u64, len: u32) -> Result<Data, Self::Error> {
|
||||
#[tracing::instrument(level = "debug", skip(self), fields(id, handle = ?handle, offset, len))]
|
||||
async fn read(&mut self, id: u32, handle: SftpHandle, offset: u64, len: u32) -> Result<Data, Self::Error> {
|
||||
let handle = Self::handle_table_key(handle)?;
|
||||
self.session_diag.stamp();
|
||||
let result = self.read_inner(id, handle.clone(), offset, len).await;
|
||||
self.session_diag.stamp();
|
||||
@@ -650,8 +661,9 @@ impl<S: StorageBackend + Send + Sync + 'static> russh_sftp::server::Handler for
|
||||
///
|
||||
/// A missing handle is treated as Ok to tolerate clients that
|
||||
/// double-close on session teardown.
|
||||
#[tracing::instrument(level = "info", skip(self), fields(id, handle = %handle), err(Debug))]
|
||||
async fn close(&mut self, id: u32, handle: String) -> Result<Status, Self::Error> {
|
||||
#[tracing::instrument(level = "info", skip(self), fields(id, handle = ?handle), err(Debug))]
|
||||
async fn close(&mut self, id: u32, handle: SftpHandle) -> Result<Status, Self::Error> {
|
||||
let handle = Self::handle_table_key(handle)?;
|
||||
self.session_diag.stamp();
|
||||
let removed = self.handles.remove(&handle);
|
||||
let Some(HandleState::Write {
|
||||
@@ -755,8 +767,9 @@ impl<S: StorageBackend + Send + Sync + 'static> russh_sftp::server::Handler for
|
||||
/// to Streaming by issuing CreateMultipartUpload. A Failed handle
|
||||
/// rejects every subsequent write with the same status that caused
|
||||
/// the failure.
|
||||
#[tracing::instrument(level = "debug", skip(self, data), fields(id, handle = %handle, offset, len = data.len()), err(Debug))]
|
||||
async fn write(&mut self, id: u32, handle: String, offset: u64, data: Vec<u8>) -> Result<Status, Self::Error> {
|
||||
#[tracing::instrument(level = "debug", skip(self, data), fields(id, handle = ?handle, offset, len = data.len()), err(Debug))]
|
||||
async fn write(&mut self, id: u32, handle: SftpHandle, offset: u64, data: Vec<u8>) -> Result<Status, Self::Error> {
|
||||
let handle = Self::handle_table_key(handle)?;
|
||||
self.session_diag.stamp();
|
||||
self.enforce_server_readonly()?;
|
||||
|
||||
@@ -983,8 +996,8 @@ impl<S: StorageBackend + Send + Sync + 'static> russh_sftp::server::Handler for
|
||||
/// SSH_FXP_FSETSTAT, SFTP Internet Draft section 6.6. Same rationale
|
||||
/// as setstat: S3 cannot honour POSIX attributes, and clients use
|
||||
/// FSETSTAT during transfers to stamp the in-flight handle.
|
||||
#[tracing::instrument(level = "debug", skip(self, _attrs), fields(id, handle = %_handle), err(Debug))]
|
||||
async fn fsetstat(&mut self, id: u32, _handle: String, _attrs: FileAttributes) -> Result<Status, Self::Error> {
|
||||
#[tracing::instrument(level = "debug", skip(self, _attrs), fields(id, handle = ?_handle), err(Debug))]
|
||||
async fn fsetstat(&mut self, id: u32, _handle: SftpHandle, _attrs: FileAttributes) -> Result<Status, Self::Error> {
|
||||
self.enforce_server_readonly()?;
|
||||
Ok(ok_status(id))
|
||||
}
|
||||
@@ -1254,7 +1267,7 @@ mod tests {
|
||||
let handle_id = driver
|
||||
.allocate_handle(file_handle("bucket", "k", 1234, attrs.clone()))
|
||||
.expect("allocate");
|
||||
let out = driver.fstat(4, handle_id).await.expect("fstat on File must succeed");
|
||||
let out = driver.fstat(4, handle_id.into()).await.expect("fstat on File must succeed");
|
||||
assert_eq!(out.attrs.size, Some(1234));
|
||||
assert_eq!(out.attrs.mtime, Some(1_700_000_000));
|
||||
}
|
||||
@@ -1267,7 +1280,7 @@ mod tests {
|
||||
part_buffer: vec![0u8; 4096],
|
||||
};
|
||||
let handle_id = driver.allocate_handle(write_handle("bucket", "k", phase)).expect("allocate");
|
||||
let out = driver.fstat(5, handle_id).await.expect("fstat on Write must succeed");
|
||||
let out = driver.fstat(5, handle_id.into()).await.expect("fstat on Write must succeed");
|
||||
assert_eq!(
|
||||
out.attrs.size,
|
||||
Some(4096),
|
||||
@@ -1283,7 +1296,7 @@ mod tests {
|
||||
.allocate_handle(file_handle("bucket", "k", 0, FileAttributes::default()))
|
||||
.expect("allocate");
|
||||
let status = driver
|
||||
.fsetstat(6, handle_id, FileAttributes::default())
|
||||
.fsetstat(6, handle_id.into(), FileAttributes::default())
|
||||
.await
|
||||
.expect("fsetstat must succeed on any attrs");
|
||||
assert!(matches!(status.status_code, StatusCode::Ok));
|
||||
@@ -1417,7 +1430,7 @@ mod tests {
|
||||
let handle_id = driver
|
||||
.allocate_handle(file_handle("bucket", "k", 0, FileAttributes::default()))
|
||||
.expect("allocate");
|
||||
let result = driver.fsetstat(10, handle_id, FileAttributes::default()).await;
|
||||
let result = driver.fsetstat(10, handle_id.into(), FileAttributes::default()).await;
|
||||
match result {
|
||||
Err(err) => assert!(matches!(err.0, StatusCode::PermissionDenied)),
|
||||
Ok(_) => panic!("fsetstat must error in read-only mode"),
|
||||
|
||||
@@ -69,7 +69,10 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
|
||||
attrs,
|
||||
read_cache,
|
||||
})?;
|
||||
Ok(Handle { id, handle })
|
||||
Ok(Handle {
|
||||
id,
|
||||
handle: handle.into(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Body of the SSH_FXP_READ handler. Returns up to len bytes starting
|
||||
@@ -256,7 +259,7 @@ mod tests {
|
||||
.allocate_handle(file_handle("bucket", "secret.txt", 4096, FileAttributes::default()))
|
||||
.expect("allocate read handle");
|
||||
driver.try_populate_read_cache(&handle, 0, vec![42; 4096]);
|
||||
let error = with_test_auth_override(|_, _, _| true, driver.read(1, handle, 0, 1024))
|
||||
let error = with_test_auth_override(|_, _, _| true, driver.read(1, handle.into(), 0, 1024))
|
||||
.await
|
||||
.expect_err("backend policy denial must prevent cached content disclosure");
|
||||
assert!(matches!(StatusCode::from(error), StatusCode::PermissionDenied));
|
||||
@@ -270,7 +273,7 @@ mod tests {
|
||||
.allocate_handle(file_handle("bucket", "k", 100, FileAttributes::default()))
|
||||
.expect("allocate");
|
||||
let err = driver
|
||||
.read(1, handle_id, 0, 0)
|
||||
.read(1, handle_id.into(), 0, 0)
|
||||
.await
|
||||
.expect_err("len=0 must return BadMessage");
|
||||
assert!(matches!(StatusCode::from(err), StatusCode::BadMessage));
|
||||
@@ -284,7 +287,7 @@ mod tests {
|
||||
.allocate_handle(file_handle("bucket", "k", 10, FileAttributes::default()))
|
||||
.expect("allocate");
|
||||
let err = driver
|
||||
.read(2, handle_id, 10, 4)
|
||||
.read(2, handle_id.into(), 10, 4)
|
||||
.await
|
||||
.expect_err("offset==size must return Eof");
|
||||
assert!(matches!(StatusCode::from(err), StatusCode::Eof));
|
||||
@@ -299,7 +302,7 @@ mod tests {
|
||||
.allocate_handle(file_handle("bucket", "k", 5, FileAttributes::default()))
|
||||
.expect("allocate");
|
||||
|
||||
let data = with_test_auth_override(|_, _, _| true, driver.read(3, handle_id, 0, 1024))
|
||||
let data = with_test_auth_override(|_, _, _| true, driver.read(3, handle_id.into(), 0, 1024))
|
||||
.await
|
||||
.expect("read must succeed");
|
||||
assert_eq!(data.data, b"hello".to_vec());
|
||||
@@ -317,7 +320,7 @@ mod tests {
|
||||
.allocate_handle(file_handle("bucket", "k", 10, FileAttributes::default()))
|
||||
.expect("allocate");
|
||||
|
||||
let (result, captured) = capture_tracing_at(Level::ERROR, async { driver.read(11, handle_id, 10, 4).await }).await;
|
||||
let (result, captured) = capture_tracing_at(Level::ERROR, async { driver.read(11, handle_id.into(), 10, 4).await }).await;
|
||||
let err = result.expect_err("offset==size must return Eof");
|
||||
assert!(matches!(StatusCode::from(err), StatusCode::Eof));
|
||||
assert!(
|
||||
@@ -342,8 +345,11 @@ mod tests {
|
||||
.allocate_handle(file_handle("bucket", "k", 1024, FileAttributes::default()))
|
||||
.expect("allocate");
|
||||
|
||||
let (result, captured) =
|
||||
capture_tracing_at(Level::ERROR, with_test_auth_override(|_, _, _| true, driver.read(12, handle_id, 0, 256))).await;
|
||||
let (result, captured) = capture_tracing_at(
|
||||
Level::ERROR,
|
||||
with_test_auth_override(|_, _, _| true, driver.read(12, handle_id.into(), 0, 256)),
|
||||
)
|
||||
.await;
|
||||
let err = result.expect_err("backend error must propagate as Err");
|
||||
assert!(!matches!(StatusCode::from(err), StatusCode::Eof), "backend error must not be Eof");
|
||||
assert!(
|
||||
@@ -379,7 +385,7 @@ mod tests {
|
||||
let start = Instant::now();
|
||||
let outcome = tokio::time::timeout(
|
||||
Duration::from_secs(10),
|
||||
with_test_auth_override(|_, _, _| true, driver.read(14, handle_id, 0, 4096)),
|
||||
with_test_auth_override(|_, _, _| true, driver.read(14, handle_id.into(), 0, 4096)),
|
||||
)
|
||||
.await;
|
||||
let elapsed = start.elapsed();
|
||||
@@ -425,7 +431,7 @@ mod tests {
|
||||
let mut assembled: Vec<u8> = Vec::with_capacity(object_size as usize);
|
||||
let mut reads: u32 = 0;
|
||||
while offset < object_size {
|
||||
let data = with_test_auth_override(|_, _, _| true, driver.read(20 + reads, handle_id.clone(), offset, chunk))
|
||||
let data = with_test_auth_override(|_, _, _| true, driver.read(20 + reads, handle_id.clone().into(), offset, chunk))
|
||||
.await
|
||||
.expect("read inside the cached window must succeed without a second backend call");
|
||||
assert!(!data.data.is_empty(), "non-empty hit");
|
||||
@@ -461,21 +467,21 @@ mod tests {
|
||||
.expect("allocate");
|
||||
|
||||
// First read fetches from the backend and populates window 1.
|
||||
let r1 = with_test_auth_override(|_, _, _| true, driver.read(30, handle_id.clone(), 0, 1024))
|
||||
let r1 = with_test_auth_override(|_, _, _| true, driver.read(30, handle_id.clone().into(), 0, 1024))
|
||||
.await
|
||||
.expect("first read must succeed");
|
||||
assert!(r1.data.iter().all(|b| *b == 0xAA), "first read must come from window 1");
|
||||
|
||||
// Second read inside the cached chunk is served from the
|
||||
// buffer. No second backend call yet.
|
||||
let r2 = with_test_auth_override(|_, _, _| true, driver.read(31, handle_id.clone(), 1024, 1024))
|
||||
let r2 = with_test_auth_override(|_, _, _| true, driver.read(31, handle_id.clone().into(), 1024, 1024))
|
||||
.await
|
||||
.expect("mid-window read must succeed from cache");
|
||||
assert!(r2.data.iter().all(|b| *b == 0xAA), "mid-window read still in window 1");
|
||||
|
||||
// Reading at offset == window falls outside the cached chunk
|
||||
// and triggers the second backend call.
|
||||
let r3 = with_test_auth_override(|_, _, _| true, driver.read(32, handle_id.clone(), window, 1024))
|
||||
let r3 = with_test_auth_override(|_, _, _| true, driver.read(32, handle_id.clone().into(), window, 1024))
|
||||
.await
|
||||
.expect("read at offset=window must succeed via second backend call");
|
||||
assert!(r3.data.iter().all(|b| *b == 0xBB), "read at window boundary must come from window 2");
|
||||
@@ -483,7 +489,7 @@ mod tests {
|
||||
// A read inside the second cached chunk is served from the
|
||||
// buffer. The queue is empty by now, so any third backend
|
||||
// call would fail.
|
||||
let r4 = with_test_auth_override(|_, _, _| true, driver.read(33, handle_id, window + 1024, 1024))
|
||||
let r4 = with_test_auth_override(|_, _, _| true, driver.read(33, handle_id.into(), window + 1024, 1024))
|
||||
.await
|
||||
.expect("mid-window-2 read must succeed from cache");
|
||||
assert!(r4.data.iter().all(|b| *b == 0xBB), "mid-window-2 read still in window 2");
|
||||
@@ -511,13 +517,13 @@ mod tests {
|
||||
.expect("allocate");
|
||||
|
||||
// Populate window 1 with a full read.
|
||||
let _ = with_test_auth_override(|_, _, _| true, driver.read(40, handle_id.clone(), 0, window as u32))
|
||||
let _ = with_test_auth_override(|_, _, _| true, driver.read(40, handle_id.clone().into(), 0, window as u32))
|
||||
.await
|
||||
.expect("populate window 1");
|
||||
|
||||
// Ask for 256 bytes starting 64 bytes before window end. Only
|
||||
// 64 bytes are in the window. The driver must return 64.
|
||||
let edge = with_test_auth_override(|_, _, _| true, driver.read(41, handle_id, window - 64, 256))
|
||||
let edge = with_test_auth_override(|_, _, _| true, driver.read(41, handle_id.into(), window - 64, 256))
|
||||
.await
|
||||
.expect("partial-hit read must succeed");
|
||||
assert_eq!(edge.data.len(), 64, "partial hit must return only the in-window portion");
|
||||
@@ -552,9 +558,10 @@ mod tests {
|
||||
|
||||
for i in 0..read_count {
|
||||
let offset = (chunk_size as u64) * (i as u64);
|
||||
let data = with_test_auth_override(|_, _, _| true, driver.read(50 + i, handle_id.clone(), offset, chunk_size as u32))
|
||||
.await
|
||||
.expect("each read must succeed via the backend");
|
||||
let data =
|
||||
with_test_auth_override(|_, _, _| true, driver.read(50 + i, handle_id.clone().into(), offset, chunk_size as u32))
|
||||
.await
|
||||
.expect("each read must succeed via the backend");
|
||||
assert_eq!(data.data.len(), chunk_size, "read must return full requested length");
|
||||
let expected_byte = (i + 1) as u8;
|
||||
assert!(
|
||||
|
||||
@@ -336,7 +336,10 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
|
||||
open_attrs,
|
||||
phase: WritePhase::Buffering { part_buffer: Vec::new() },
|
||||
})?;
|
||||
Ok(Handle { id, handle })
|
||||
Ok(Handle {
|
||||
id,
|
||||
handle: handle.into(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Upload the buffered bytes to S3 with a single PutObject call.
|
||||
@@ -2105,7 +2108,7 @@ mod tests {
|
||||
)
|
||||
.await
|
||||
.expect("WRITE | CREATE | TRUNCATE on a missing file must allocate a handle");
|
||||
assert!(!handle.handle.is_empty());
|
||||
assert!(!handle.handle.as_bytes().is_empty());
|
||||
assert!(backend.head_object_calls().is_empty(), "the non-EXCL accept path must not HEAD");
|
||||
}
|
||||
|
||||
@@ -2130,7 +2133,7 @@ mod tests {
|
||||
)
|
||||
.await
|
||||
.expect("WRITE | CREATE | TRUNCATE on an existing file must allocate a handle");
|
||||
assert!(!handle.handle.is_empty());
|
||||
assert!(!handle.handle.as_bytes().is_empty());
|
||||
assert!(backend.head_object_calls().is_empty(), "the non-EXCL accept path must not HEAD");
|
||||
}
|
||||
|
||||
@@ -2177,7 +2180,7 @@ mod tests {
|
||||
)
|
||||
.await
|
||||
.expect("EXCL on a missing key must allow creation");
|
||||
assert!(!handle.handle.is_empty());
|
||||
assert!(!handle.handle.as_bytes().is_empty());
|
||||
assert_eq!(backend.head_object_calls().len(), 1, "EXCL must HEAD exactly once");
|
||||
}
|
||||
|
||||
|
||||
@@ -38,8 +38,6 @@ default = []
|
||||
# event. Requires `RUSTFLAGS="--cfg hyper_unstable_tracing"`.
|
||||
diagnostic-internode-http2-ping = ["dep:hyper", "rustfs-io-metrics/diagnostic-internode-http2-ping"]
|
||||
hotpath = [
|
||||
"hotpath/hotpath",
|
||||
"hotpath/tokio",
|
||||
"rustfs-common/hotpath",
|
||||
"rustfs-config/hotpath",
|
||||
"rustfs-io-metrics/hotpath",
|
||||
@@ -48,7 +46,6 @@ hotpath = [
|
||||
]
|
||||
hotpath-alloc = [
|
||||
"hotpath",
|
||||
"hotpath/hotpath-alloc",
|
||||
"rustfs-common/hotpath-alloc",
|
||||
"rustfs-config/hotpath-alloc",
|
||||
"rustfs-io-metrics/hotpath-alloc",
|
||||
@@ -57,7 +54,6 @@ hotpath-alloc = [
|
||||
]
|
||||
hotpath-cpu = [
|
||||
"hotpath",
|
||||
"hotpath/hotpath-cpu",
|
||||
"rustfs-common/hotpath-cpu",
|
||||
"rustfs-config/hotpath-cpu",
|
||||
"rustfs-io-metrics/hotpath-cpu",
|
||||
@@ -66,7 +62,6 @@ hotpath-cpu = [
|
||||
]
|
||||
|
||||
[dependencies]
|
||||
hotpath.workspace = true
|
||||
rustfs-common.workspace = true
|
||||
rustfs-heal-contracts.workspace = true
|
||||
rustfs-io-metrics.workspace = true
|
||||
|
||||
@@ -0,0 +1,300 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::{INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT, dial_channel, proto_gen::node_service};
|
||||
use prost::Message;
|
||||
use std::{
|
||||
collections::HashSet,
|
||||
io,
|
||||
sync::{
|
||||
Arc,
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
},
|
||||
time::Duration,
|
||||
};
|
||||
use tokio::{
|
||||
io::{AsyncReadExt, AsyncWriteExt},
|
||||
net::{TcpListener, TcpStream},
|
||||
sync::{mpsc, oneshot},
|
||||
task::JoinSet,
|
||||
};
|
||||
|
||||
const BATCHES: usize = 80;
|
||||
const HEADERS: u8 = 1;
|
||||
const RST_STREAM: u8 = 3;
|
||||
const SETTINGS: u8 = 4;
|
||||
const PING: u8 = 6;
|
||||
const GOAWAY: u8 = 7;
|
||||
const WINDOW_UPDATE: u8 = 8;
|
||||
const END_HEADERS: u8 = 4;
|
||||
const END_STREAM: u8 = 1;
|
||||
|
||||
struct Frame {
|
||||
kind: u8,
|
||||
flags: u8,
|
||||
stream: u32,
|
||||
payload: Vec<u8>,
|
||||
}
|
||||
|
||||
async fn read_frame(socket: &mut TcpStream) -> io::Result<Frame> {
|
||||
let mut header = [0; 9];
|
||||
socket.read_exact(&mut header).await?;
|
||||
let length = usize::from(header[0]) * 65536 + usize::from(header[1]) * 256 + usize::from(header[2]);
|
||||
if length > 16384 {
|
||||
return Err(io::Error::other("test peer received an oversized frame"));
|
||||
}
|
||||
let mut payload = vec![0; length];
|
||||
socket.read_exact(&mut payload).await?;
|
||||
Ok(Frame {
|
||||
kind: header[3],
|
||||
flags: header[4],
|
||||
stream: u32::from_be_bytes(header[5..9].try_into().expect("four stream ID bytes")) & 0x7fff_ffff,
|
||||
payload,
|
||||
})
|
||||
}
|
||||
|
||||
async fn write_frame(socket: &mut TcpStream, kind: u8, flags: u8, stream: u32, payload: &[u8]) -> io::Result<()> {
|
||||
let length = u32::try_from(payload.len()).expect("bounded test frame length").to_be_bytes();
|
||||
assert_eq!(length[0], 0, "HTTP/2 frame length fits 24 bits");
|
||||
let mut header = [0; 9];
|
||||
header[..3].copy_from_slice(&length[1..]);
|
||||
header[3] = kind;
|
||||
header[4] = flags;
|
||||
header[5..].copy_from_slice(&stream.to_be_bytes());
|
||||
socket.write_all(&header).await?;
|
||||
socket.write_all(payload).await
|
||||
}
|
||||
|
||||
async fn write_probe_response(socket: &mut TcpStream, stream: u32) -> io::Result<()> {
|
||||
// HPACK's static :status=200 and literal content-type=application/grpc.
|
||||
write_frame(socket, HEADERS, END_HEADERS, stream, b"\x88\x0f\x10\x10application/grpc").await?;
|
||||
let response = node_service::ReadVersionResponse {
|
||||
success: true,
|
||||
..Default::default()
|
||||
}
|
||||
.encode_to_vec();
|
||||
let mut body = vec![0];
|
||||
body.extend_from_slice(&u32::try_from(response.len()).expect("small probe response").to_be_bytes());
|
||||
body.extend(response);
|
||||
write_frame(socket, 0, 0, stream, &body).await?;
|
||||
// A literal grpc-status=0 trailer ends the live probe RPC.
|
||||
write_frame(socket, HEADERS, END_HEADERS | END_STREAM, stream, b"\x00\x0bgrpc-status\x010").await
|
||||
}
|
||||
|
||||
async fn accept_peer(listener: TcpListener) -> io::Result<TcpStream> {
|
||||
let (mut socket, _) = listener.accept().await?;
|
||||
socket.set_nodelay(true)?;
|
||||
let mut preface = [0; 24];
|
||||
socket.read_exact(&mut preface).await?;
|
||||
assert_eq!(&preface, b"PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n");
|
||||
write_frame(&mut socket, SETTINGS, 0, 0, &[]).await?;
|
||||
Ok(socket)
|
||||
}
|
||||
|
||||
async fn delayed_response_peer(listener: TcpListener, ready: mpsc::Sender<()>, error_resets: Arc<AtomicUsize>) -> io::Result<()> {
|
||||
let mut socket = accept_peer(listener).await?;
|
||||
let mut cancelled = Vec::with_capacity(INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT);
|
||||
let mut resets = HashSet::new();
|
||||
let mut probe = None;
|
||||
loop {
|
||||
let frame = read_frame(&mut socket).await?;
|
||||
match frame.kind {
|
||||
SETTINGS | PING if frame.flags & 1 == 0 => {
|
||||
let payload = if frame.kind == PING { frame.payload.as_slice() } else { &[] };
|
||||
write_frame(&mut socket, frame.kind, 1, 0, payload).await?;
|
||||
}
|
||||
0 if !frame.payload.is_empty() => {
|
||||
let credit = u32::try_from(frame.payload.len()).expect("bounded DATA frame").to_be_bytes();
|
||||
write_frame(&mut socket, WINDOW_UPDATE, 0, 0, &credit).await?;
|
||||
}
|
||||
HEADERS if cancelled.len() < INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT => {
|
||||
cancelled.push(frame.stream);
|
||||
if cancelled.len() == INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT {
|
||||
ready.send(()).await.map_err(|_| io::Error::other("test driver stopped"))?;
|
||||
}
|
||||
}
|
||||
HEADERS => {
|
||||
assert!(probe.replace(frame.stream).is_none(), "one live probe per batch");
|
||||
}
|
||||
RST_STREAM => {
|
||||
let reason = u32::from_be_bytes(frame.payload.as_slice().try_into().expect("RST_STREAM error code"));
|
||||
if reason == 5 {
|
||||
error_resets.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
if cancelled.contains(&frame.stream) {
|
||||
resets.insert(frame.stream);
|
||||
}
|
||||
}
|
||||
GOAWAY => return Err(io::Error::other(format!("client GOAWAY: {:?}", frame.payload))),
|
||||
_ => {}
|
||||
}
|
||||
if let Some(stream) = probe
|
||||
&& resets.len() == INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT
|
||||
{
|
||||
// A raw peer controls bytes already in flight at cancellation. A normal
|
||||
// server would consume RST_STREAM and suppress these delayed responses.
|
||||
for cancelled_stream in &cancelled {
|
||||
write_frame(&mut socket, HEADERS, END_HEADERS | END_STREAM, *cancelled_stream, b"\x88").await?;
|
||||
}
|
||||
write_probe_response(&mut socket, stream).await?;
|
||||
cancelled.clear();
|
||||
resets.clear();
|
||||
probe = None;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn assert_cancelled_responses_are_ignored(bulk: bool) {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind isolated peer");
|
||||
let address = format!("http://{}", listener.local_addr().expect("peer address"));
|
||||
let (ready_tx, mut ready_rx) = mpsc::channel(1);
|
||||
let error_resets = Arc::new(AtomicUsize::new(0));
|
||||
let peer = tokio::spawn(delayed_response_peer(listener, ready_tx, error_resets.clone()));
|
||||
let cache_key = if bulk {
|
||||
super::bulk_cache_key(&address, 0)
|
||||
} else {
|
||||
address.clone()
|
||||
};
|
||||
let result = tokio::time::timeout(Duration::from_secs(30), async {
|
||||
let (channel, _) = dial_channel(&address, &cache_key, Duration::from_secs(10))
|
||||
.await
|
||||
.expect("dial real internode channel");
|
||||
let mut client = node_service::node_service_client::NodeServiceClient::new(channel);
|
||||
for batch in 0..BATCHES {
|
||||
let mut requests = JoinSet::new();
|
||||
for _ in 0..INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT {
|
||||
let mut pending_client = client.clone();
|
||||
requests.spawn(async move { pending_client.read_version(node_service::ReadVersionRequest::default()).await });
|
||||
}
|
||||
ready_rx.recv().await.expect("peer observed every pending request");
|
||||
requests.abort_all();
|
||||
while let Some(result) = requests.join_next().await {
|
||||
assert!(result.expect_err("pending RPC task was cancelled").is_cancelled());
|
||||
}
|
||||
let response = client
|
||||
.read_version(node_service::ReadVersionRequest::default())
|
||||
.await
|
||||
.map_err(|error| format!("live probe after cancellation batch {batch}: {error}"))?;
|
||||
assert!(response.into_inner().success);
|
||||
}
|
||||
Ok::<_, String>(())
|
||||
})
|
||||
.await;
|
||||
peer.abort();
|
||||
let peer_result = peer.await;
|
||||
assert!(matches!(result, Ok(Ok(()))), "shared channel failed: {result:?}; peer: {peer_result:?}");
|
||||
assert_eq!(
|
||||
error_resets.load(Ordering::Relaxed),
|
||||
0,
|
||||
"late cancelled responses must not spend the error budget"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn cancelled_metadata_responses_keep_the_shared_channel_usable() {
|
||||
assert_cancelled_responses_are_ignored(false).await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn cancelled_metadata_responses_keep_the_bulk_channel_usable() {
|
||||
temp_env::async_with_vars(
|
||||
[(rustfs_config::ENV_INTERNODE_BULK_CONCURRENCY_LIMIT, Some("64"))],
|
||||
assert_cancelled_responses_are_ignored(true),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
async fn unsolicited_response_peer(listener: TcpListener, attack: oneshot::Receiver<()>) -> io::Result<(usize, Vec<u8>)> {
|
||||
let mut socket = accept_peer(listener).await?;
|
||||
let mut completed = Vec::with_capacity(1025);
|
||||
while completed.len() < 1025 {
|
||||
let frame = read_frame(&mut socket).await?;
|
||||
match frame.kind {
|
||||
HEADERS => {
|
||||
write_probe_response(&mut socket, frame.stream).await?;
|
||||
completed.push(frame.stream);
|
||||
}
|
||||
SETTINGS if frame.flags & 1 == 0 => write_frame(&mut socket, SETTINGS, 1, 0, &[]).await?,
|
||||
PING if frame.flags & 1 == 0 => write_frame(&mut socket, PING, 1, 0, &frame.payload).await?,
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
attack
|
||||
.await
|
||||
.map_err(|_| io::Error::other("test driver stopped before attack"))?;
|
||||
// These streams completed normally, so duplicate responses are protocol errors,
|
||||
// not late responses to a cancellation that the grace period should absorb.
|
||||
let mut resets = 0;
|
||||
for stream in completed {
|
||||
write_frame(&mut socket, HEADERS, END_HEADERS | END_STREAM, stream, b"\x88").await?;
|
||||
// Drain each reset before provoking the next one: GOAWAY may discard
|
||||
// queued frames, so a burst's wire count is not the internal error count.
|
||||
loop {
|
||||
let frame = read_frame(&mut socket).await?;
|
||||
match frame.kind {
|
||||
RST_STREAM => {
|
||||
assert_eq!(frame.stream, stream);
|
||||
assert_eq!(frame.payload, 5_u32.to_be_bytes());
|
||||
resets += 1;
|
||||
break;
|
||||
}
|
||||
GOAWAY => return Ok((resets, frame.payload)),
|
||||
PING if frame.flags & 1 == 0 => write_frame(&mut socket, PING, 1, 0, &frame.payload).await?,
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(io::Error::other("peer accepted more than 1024 protocol errors"))
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn cancelled_stream_retention_preserves_the_protocol_error_limit() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind isolated peer");
|
||||
let address = format!("http://{}", listener.local_addr().expect("peer address"));
|
||||
let (attack_tx, attack_rx) = oneshot::channel();
|
||||
let mut peer = tokio::spawn(unsolicited_response_peer(listener, attack_rx));
|
||||
let result = tokio::time::timeout(Duration::from_secs(30), async {
|
||||
let (channel, _) = dial_channel(&address, &address, Duration::from_secs(10))
|
||||
.await
|
||||
.expect("dial real internode channel");
|
||||
let mut client = node_service::node_service_client::NodeServiceClient::new(channel);
|
||||
for _ in 0..1025 {
|
||||
assert!(
|
||||
client
|
||||
.read_version(node_service::ReadVersionRequest::default())
|
||||
.await
|
||||
.expect("initial live probe")
|
||||
.into_inner()
|
||||
.success
|
||||
);
|
||||
}
|
||||
attack_tx
|
||||
.send(())
|
||||
.expect("release duplicate responses after the RPC completes");
|
||||
(&mut peer).await
|
||||
})
|
||||
.await;
|
||||
if result.is_err() {
|
||||
peer.abort();
|
||||
let _ = peer.await;
|
||||
}
|
||||
let (resets, payload) = result
|
||||
.expect("error budget is bounded")
|
||||
.expect("peer task")
|
||||
.expect("peer exchange");
|
||||
assert_eq!(resets, 1024, "the existing local protocol-error reset limit remains enabled");
|
||||
assert!(payload.len() >= 8);
|
||||
assert_eq!(u32::from_be_bytes(payload[4..8].try_into().expect("GOAWAY reason")), 11);
|
||||
assert_eq!(&payload[8..], b"too_many_internal_resets");
|
||||
}
|
||||
@@ -1588,6 +1588,10 @@ pub struct ScannerActivityResponse {
|
||||
pub movement_generation: ::core::option::Option<u64>,
|
||||
#[prost(bool, optional, tag = "11")]
|
||||
pub publication_blocked: ::core::option::Option<bool>,
|
||||
/// v8 binds checkpoint reuse to a verified clean, read-only handoff.
|
||||
/// The process instance above remains fresh for RPC and ACK fencing.
|
||||
#[prost(string, optional, tag = "12")]
|
||||
pub checkpoint_instance_id: ::core::option::Option<::prost::alloc::string::String>,
|
||||
}
|
||||
#[allow(clippy::all)]
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
|
||||
@@ -12,6 +12,8 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#[cfg(test)]
|
||||
mod channel_reset_tests;
|
||||
pub mod compat_manifest;
|
||||
// SAFETY: `generated` is prost/tonic-generated protocol code. The allowance is
|
||||
// scoped to that module so generated internals do not relax lints elsewhere.
|
||||
@@ -707,6 +709,21 @@ pub fn canonical_scanner_activity_v7_response_body(
|
||||
pub mod scoped_dirty_usage;
|
||||
pub mod usage_watch;
|
||||
|
||||
/// Binds checkpoint continuity independently of the fresh RPC process ID.
|
||||
pub fn canonical_scanner_activity_v8_response_body(
|
||||
challenge: &[u8],
|
||||
response: &proto_gen::node_service::ScannerActivityResponse,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
let mut body = canonical_scanner_activity_v7_response_body(challenge, response)?;
|
||||
body.extend_from_slice(b"rustfs-scanner-checkpoint-continuity-v1\0");
|
||||
body.push(u8::from(response.checkpoint_instance_id.is_some()));
|
||||
if let Some(instance) = &response.checkpoint_instance_id {
|
||||
body.extend_from_slice(&u64::try_from(instance.len())?.to_be_bytes());
|
||||
body.extend_from_slice(instance.as_bytes());
|
||||
}
|
||||
Ok(body)
|
||||
}
|
||||
|
||||
pub fn canonical_scanner_dirty_usage_snapshot_request_body(
|
||||
request: &proto_gen::node_service::ScannerDirtyUsageSnapshotRequest,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
@@ -2498,9 +2515,9 @@ mod scanner_activity_tests {
|
||||
use super::{
|
||||
canonical_scanner_activity_request_body, canonical_scanner_activity_response_body,
|
||||
canonical_scanner_activity_v4_response_body, canonical_scanner_activity_v7_response_body,
|
||||
canonical_scanner_dirty_usage_snapshot_request_body, canonical_scanner_dirty_usage_snapshot_response_body,
|
||||
canonical_scanner_publication_lease_release_request_body, canonical_scanner_publication_lease_request_body,
|
||||
canonical_scanner_publication_lease_response_body,
|
||||
canonical_scanner_activity_v8_response_body, canonical_scanner_dirty_usage_snapshot_request_body,
|
||||
canonical_scanner_dirty_usage_snapshot_response_body, canonical_scanner_publication_lease_release_request_body,
|
||||
canonical_scanner_publication_lease_request_body, canonical_scanner_publication_lease_response_body,
|
||||
proto_gen::node_service::{
|
||||
ScannerActivityRequest, ScannerActivityResponse, ScannerDirtyUsageBucket, ScannerDirtyUsageSnapshotRequest,
|
||||
ScannerDirtyUsageSnapshotResponse, ScannerPublicationLeaseRequest, ScannerPublicationLeaseResponse,
|
||||
@@ -2652,6 +2669,7 @@ mod scanner_activity_tests {
|
||||
#[test]
|
||||
fn canonical_scanner_activity_response_binds_challenge_and_every_status_field() {
|
||||
let response = ScannerActivityResponse {
|
||||
checkpoint_instance_id: None,
|
||||
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
|
||||
namespace_generation: 7,
|
||||
maintenance_generation: 3,
|
||||
@@ -2734,9 +2752,36 @@ mod scanner_activity_tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_activity_v8_authenticates_checkpoint_continuity() {
|
||||
let response = ScannerActivityResponse {
|
||||
protocol_version: 8,
|
||||
instance_id: "1".repeat(32),
|
||||
checkpoint_instance_id: Some("2".repeat(32)),
|
||||
..Default::default()
|
||||
};
|
||||
let baseline = canonical_scanner_activity_v8_response_body(&[1; 16], &response).expect("v8 body");
|
||||
for checkpoint_instance_id in [None, Some("3".repeat(32))] {
|
||||
let changed = ScannerActivityResponse {
|
||||
checkpoint_instance_id,
|
||||
..response.clone()
|
||||
};
|
||||
assert_ne!(
|
||||
baseline,
|
||||
canonical_scanner_activity_v8_response_body(&[1; 16], &changed).expect("changed body")
|
||||
);
|
||||
assert_eq!(
|
||||
canonical_scanner_activity_v7_response_body(&[1; 16], &response).expect("v7 body"),
|
||||
canonical_scanner_activity_v7_response_body(&[1; 16], &changed).expect("v7 compatibility"),
|
||||
"v7 does not authenticate continuity and its decoder must reject this field"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_activity_v4_response_canonicalization_ignores_v5_fields() {
|
||||
let response = ScannerActivityResponse {
|
||||
checkpoint_instance_id: None,
|
||||
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
|
||||
namespace_generation: 7,
|
||||
maintenance_generation: 3,
|
||||
@@ -3733,9 +3778,9 @@ fn complete_pending_rebuild_on_cache_hit(dial_addr: &str, cache_key: &str) -> bo
|
||||
pub const INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT: usize = 64;
|
||||
|
||||
// Quorum reads cancel surplus metadata RPCs in bursts. Keep bounded reset state
|
||||
// for their in-flight replies without spending the connection's protocol-error
|
||||
// budget (h2 remembers only 50 resets by default).
|
||||
const INTERNODE_RPC_MAX_CONCURRENT_RESET_STREAMS: usize = 4096;
|
||||
// for their in-flight replies through the HTTP/2 grace period without spending
|
||||
// the connection's protocol-error budget (h2 remembers only 50 resets by default).
|
||||
const INTERNODE_RPC_MAX_CONCURRENT_RESET_STREAMS: usize = 8192;
|
||||
|
||||
/// Whether control/bulk channel isolation is enabled (env-gated, default off for safe rollout).
|
||||
fn channel_isolation_enabled() -> bool {
|
||||
|
||||
@@ -1011,6 +1011,9 @@ message ScannerActivityResponse {
|
||||
// response shape while newer readers fail closed when they are absent.
|
||||
optional uint64 movement_generation = 10;
|
||||
optional bool publication_blocked = 11;
|
||||
// v8 binds checkpoint reuse to a verified clean, read-only handoff.
|
||||
// The process instance above remains fresh for RPC and ACK fencing.
|
||||
optional string checkpoint_instance_id = 12;
|
||||
}
|
||||
|
||||
message ScannerDirtyUsageBucket {
|
||||
|
||||
@@ -31,29 +31,24 @@ workspace = true
|
||||
[features]
|
||||
default = []
|
||||
hotpath = [
|
||||
"hotpath/hotpath",
|
||||
"hotpath/tokio",
|
||||
"rustfs-rio/hotpath",
|
||||
"rustfs-utils/hotpath",
|
||||
"rustfs-filemeta/hotpath",
|
||||
]
|
||||
hotpath-alloc = [
|
||||
"hotpath",
|
||||
"hotpath/hotpath-alloc",
|
||||
"rustfs-rio/hotpath-alloc",
|
||||
"rustfs-utils/hotpath-alloc",
|
||||
"rustfs-filemeta/hotpath-alloc",
|
||||
]
|
||||
hotpath-cpu = [
|
||||
"hotpath",
|
||||
"hotpath/hotpath-cpu",
|
||||
"rustfs-rio/hotpath-cpu",
|
||||
"rustfs-utils/hotpath-cpu",
|
||||
"rustfs-filemeta/hotpath-cpu",
|
||||
]
|
||||
|
||||
[dependencies]
|
||||
hotpath.workspace = true
|
||||
aes-gcm = { workspace = true, features = ["rand_core"] }
|
||||
bytes = { workspace = true, features = ["serde"] }
|
||||
chacha20poly1305.workspace = true
|
||||
|
||||
@@ -32,6 +32,7 @@ workspace = true
|
||||
internode-transport = []
|
||||
default = []
|
||||
hotpath = [
|
||||
"dep:hotpath",
|
||||
"hotpath/hotpath",
|
||||
"hotpath/tokio",
|
||||
"hotpath/futures",
|
||||
@@ -59,7 +60,8 @@ hotpath-cpu = [
|
||||
]
|
||||
|
||||
[dependencies]
|
||||
hotpath.workspace = true
|
||||
# Feature-gated in src (measure_block!); optional so default builds do not warn.
|
||||
hotpath = { workspace = true, optional = true }
|
||||
arc-swap.workspace = true
|
||||
tokio = { workspace = true, features = ["io-util", "macros", "net", "rt-multi-thread", "sync", "time"] }
|
||||
rand = { workspace = true, features = ["serde"] }
|
||||
@@ -79,7 +81,6 @@ rustfs-io-metrics.workspace = true
|
||||
rustfs-tls-runtime.workspace = true
|
||||
rustfs-utils = { workspace = true, features = ["io", "hash", "compress"] }
|
||||
serde_json = { workspace = true, features = ["raw_value"] }
|
||||
md-5 = { workspace = true }
|
||||
hmac.workspace = true
|
||||
minlz.workspace = true
|
||||
tracing.workspace = true
|
||||
@@ -91,6 +92,7 @@ xxhash-rust = { workspace = true, features = ["xxh64", "xxh3"] }
|
||||
hex-simd.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
md-5 = { workspace = true }
|
||||
faster-hex.workspace = true
|
||||
rcgen.workspace = true
|
||||
tokio-rustls.workspace = true
|
||||
|
||||
@@ -58,7 +58,7 @@ impl From<Region> for legacy_s3s::region::Region {
|
||||
fn from(region: Region) -> Self {
|
||||
match Self::new(region.into_boxed_str()) {
|
||||
Ok(region) => region,
|
||||
Err(invalid) => unreachable!("rustfs_s3_types::Region and s3s::region::Region share one grammar: {invalid}"),
|
||||
Err(invalid) => unreachable!("rustfs_s3_types::Region and the legacy region type share one grammar: {invalid}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -67,7 +67,7 @@ impl From<legacy_s3s::region::Region> for Region {
|
||||
fn from(region: legacy_s3s::region::Region) -> Self {
|
||||
match Self::new(region.into_boxed_str()) {
|
||||
Ok(region) => region,
|
||||
Err(invalid) => unreachable!("s3s::region::Region and rustfs_s3_types::Region share one grammar: {invalid}"),
|
||||
Err(invalid) => unreachable!("the legacy region type and rustfs_s3_types::Region share one grammar: {invalid}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -133,6 +133,16 @@ impl From<S3Error> for legacy_s3s::S3Error {
|
||||
}
|
||||
}
|
||||
|
||||
impl S3Error {
|
||||
/// The pinned legacy serializer's XML response. Call sites stay on this type.
|
||||
/// DELETE BY T4.2 with the rest of this file.
|
||||
pub fn to_http_response(self) -> Result<http::Response<crate::Body>, Self> {
|
||||
let response = legacy_s3s::S3Error::from(self).to_http_response().map_err(Self::from)?;
|
||||
let (parts, body) = response.into_parts();
|
||||
Ok(http::Response::from_parts(parts, crate::Body::from(body)))
|
||||
}
|
||||
}
|
||||
|
||||
impl From<legacy_s3s::S3Error> for S3Error {
|
||||
fn from(error: legacy_s3s::S3Error) -> Self {
|
||||
let code: S3ErrorCode = error.code().clone().into();
|
||||
|
||||
@@ -933,8 +933,8 @@ pub struct DataUsageCacheInfo {
|
||||
#[serde(default)]
|
||||
pub scan_plan_digest: Option<DataUsageScanPlanDigest>,
|
||||
/// Full activity and inventory scope of a set scan; only a complete
|
||||
/// snapshot proves coverage. Bucket caches bind this scope into their
|
||||
/// opaque scan plan digest instead.
|
||||
/// snapshot proves coverage. A completed bucket stores its checkpoint
|
||||
/// coverage plan here until the authoritative aggregate includes it.
|
||||
#[serde(default)]
|
||||
pub scan_coverage_digest: Option<DataUsageScanPlanDigest>,
|
||||
#[serde(default)]
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
use super::*;
|
||||
use bytes::Bytes;
|
||||
use std::sync::Arc;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
#[derive(Debug)]
|
||||
struct CachePublicationAdmissionUnavailable;
|
||||
@@ -625,14 +626,22 @@ impl DataUsageCache {
|
||||
self.save_inner(store, name, Some(revisions), None).await
|
||||
}
|
||||
|
||||
/// All epoch-fenced cache writes use background disk admission. Incremental
|
||||
/// checkpoints pass their scan token; finalization uses a fresh token so a
|
||||
/// spent scan budget can still persist progress within the save timeout.
|
||||
pub(crate) async fn save_with_revisions_for_epoch<S: ScannerObjectIO + ScannerConfigObjectDelete>(
|
||||
&self,
|
||||
store: Arc<S>,
|
||||
name: &str,
|
||||
revisions: &DataUsageCacheRevisions,
|
||||
expected_epoch: u64,
|
||||
cancellation: CancellationToken,
|
||||
) -> StorageResult<()> {
|
||||
self.save_inner(store, name, Some(revisions), Some(expected_epoch)).await
|
||||
crate::storage_api::ecstore_with_background_disk_io(
|
||||
cancellation,
|
||||
self.save_inner(store, name, Some(revisions), Some(expected_epoch)),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn save_inner<S: ScannerObjectIO + ScannerConfigObjectDelete>(
|
||||
|
||||
@@ -5,6 +5,7 @@ use super::*;
|
||||
use std::hint::black_box;
|
||||
use std::sync::atomic::AtomicU64;
|
||||
use std::time::Instant as WallInstant;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
const MAX_WIRE_BYTES: u64 = 32 * 1024 * 1024;
|
||||
const CACHE_NAME: &str = "bucket/cache-cost.bin";
|
||||
@@ -230,7 +231,7 @@ async fn profile_case(objects: usize, scenario: &str, samples: usize) {
|
||||
.await
|
||||
.expect("initial revisions");
|
||||
baseline
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &initial, 0)
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &initial, 0, CancellationToken::new())
|
||||
.await
|
||||
.expect("baseline save");
|
||||
|
||||
@@ -287,7 +288,7 @@ async fn profile_case(objects: usize, scenario: &str, samples: usize) {
|
||||
store.reset_counts();
|
||||
let started = WallInstant::now();
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0)
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0, CancellationToken::new())
|
||||
.await
|
||||
.expect("measured save");
|
||||
save_ns.push(elapsed_ns(started));
|
||||
@@ -309,7 +310,7 @@ async fn profile_case(objects: usize, scenario: &str, samples: usize) {
|
||||
conflicting.info.next_cycle += 1;
|
||||
assert!(matches!(
|
||||
conflicting
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &initial, 0)
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &initial, 0, CancellationToken::new())
|
||||
.await,
|
||||
Err(Error::PreconditionFailed)
|
||||
));
|
||||
|
||||
@@ -114,8 +114,10 @@ pub use scanner_io::{
|
||||
probe_scoped_dirty_usage, record_dirty_usage_bucket, record_dirty_usage_bucket_from_producer,
|
||||
record_dirty_usage_bucket_from_producers, record_dirty_usage_object, record_dirty_usage_object_from_producer,
|
||||
record_scanner_maintenance_change, record_status_class_mutation, replay_durable_dirty_usage_producer_record,
|
||||
scanner_activity_epoch, scanner_dirty_usage_bucket_generation, scanner_dirty_usage_snapshot, scanner_dirty_usage_state,
|
||||
scanner_maintenance_generation, set_scanner_dirty_usage_clear_observer, set_scanner_dirty_usage_mutation_observer,
|
||||
restore_scanner_checkpoint_instance_id, scanner_activity_epoch, scanner_checkpoint_instance_id,
|
||||
scanner_dirty_usage_bucket_generation, scanner_dirty_usage_snapshot, scanner_dirty_usage_state,
|
||||
scanner_maintenance_generation, scanner_process_was_read_only, set_scanner_dirty_usage_clear_observer,
|
||||
set_scanner_dirty_usage_mutation_observer,
|
||||
};
|
||||
pub use segment_invalidation::SegmentInvalidationProducerIdentity;
|
||||
pub use sleeper::{
|
||||
@@ -125,9 +127,9 @@ pub use sleeper::{
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
pub use storage_api::ScannerReplicationConfig as ReplicationConfig;
|
||||
pub use storage_api::scan::{
|
||||
SCANNER_ACTIVITY_PROTOCOL_VERSION, SCANNER_ACTIVITY_V6_PROTOCOL_VERSION, SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES,
|
||||
SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION, SCANNER_DIRTY_USAGE_SNAPSHOT_RPC_MAX_MESSAGE_SIZE,
|
||||
SCANNER_SCOPED_DIRTY_USAGE_ACK_MAX_ENTRIES,
|
||||
SCANNER_ACTIVITY_PROTOCOL_VERSION, SCANNER_ACTIVITY_V6_PROTOCOL_VERSION, SCANNER_ACTIVITY_V7_PROTOCOL_VERSION,
|
||||
SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES, SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION,
|
||||
SCANNER_DIRTY_USAGE_SNAPSHOT_RPC_MAX_MESSAGE_SIZE, SCANNER_SCOPED_DIRTY_USAGE_ACK_MAX_ENTRIES,
|
||||
};
|
||||
pub use workload_admission::{set_scanner_node_admission, set_scanner_workload_admission_snapshot_provider};
|
||||
|
||||
|
||||
@@ -595,7 +595,13 @@ pub(crate) async fn write_budget_drain_fixture_frames<W: AsyncWrite + Unpin>(
|
||||
cache.info.scan_plan_digest = Some(request.scan_plan_digest);
|
||||
cache.info.last_update = Some(std::time::SystemTime::now());
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), &cache_name, &revisions, identity.publication_epoch)
|
||||
.save_with_revisions_for_epoch(
|
||||
store.clone(),
|
||||
&cache_name,
|
||||
&revisions,
|
||||
identity.publication_epoch,
|
||||
CancellationToken::new(),
|
||||
)
|
||||
.await
|
||||
.expect("durable current-request root CAS");
|
||||
let mut persisted = DataUsageCache::default();
|
||||
@@ -848,6 +854,7 @@ async fn classify_remote_usage_watch_outcome(
|
||||
// The folder completes under its coverage plan; finished cache
|
||||
// reuse and the response remain bound to the requested reuse plan.
|
||||
cache.info.scan_plan_digest = Some(scan_plan_digest);
|
||||
cache.info.scan_coverage_digest = checkpoint_plan_digest;
|
||||
(cache, None)
|
||||
}
|
||||
}
|
||||
@@ -982,6 +989,27 @@ async fn scan_and_persist_local_bucket_with_store(
|
||||
let tier_registry_generation = crate::runtime_tier_registry_for_cycle(next_cycle, leader_epoch)
|
||||
.await
|
||||
.generation;
|
||||
// Only a leader handoff needs this authoritative floor; ordinary cycles
|
||||
// avoid an extra config read. An unavailable/legacy floor forces a walk.
|
||||
let completed_checkpoint_cycle_floor = if cache.info.snapshot_complete
|
||||
&& cache.info.leader_epoch < leader_epoch
|
||||
&& cache.info.scan_plan_digest == Some(scan_plan_digest)
|
||||
&& checkpoint_plan_digest.is_some()
|
||||
&& cache.info.scan_coverage_digest == checkpoint_plan_digest
|
||||
&& cache.info.next_cycle < next_cycle
|
||||
{
|
||||
let baseline = read_config(store.clone(), crate::DATA_USAGE_OBJ_NAME_PATH.as_str())
|
||||
.await
|
||||
.ok();
|
||||
let sources = store
|
||||
.all_set_disks()
|
||||
.iter()
|
||||
.map(|set| DataUsageCacheSource::new(set.pool_index, set.set_index))
|
||||
.collect();
|
||||
crate::scanner_io::completed_checkpoint_cycle_floor(baseline.as_deref(), &sources, next_cycle, leader_epoch)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let current_identity = crate::scanner_io::scanner_bucket_checkpoint_identity(
|
||||
&set,
|
||||
&bucket,
|
||||
@@ -1012,6 +1040,7 @@ async fn scan_and_persist_local_bucket_with_store(
|
||||
tier_registry_generation: Some(tier_registry_generation),
|
||||
checkpoint_identity: current_identity,
|
||||
checkpoint_plan_digest,
|
||||
completed_checkpoint_cycle_floor,
|
||||
},
|
||||
);
|
||||
match scan_state {
|
||||
@@ -1264,7 +1293,7 @@ async fn scan_and_persist_local_bucket_with_store(
|
||||
// the scan. A movement transition that starts and ends during the scan
|
||||
// therefore cannot admit the stale cache under the new epoch.
|
||||
let save_result = cache
|
||||
.save_with_revisions_for_epoch(set.clone(), &cache_name, &revisions, expected_publication_epoch)
|
||||
.save_with_revisions_for_epoch(set.clone(), &cache_name, &revisions, expected_publication_epoch, CancellationToken::new())
|
||||
.await;
|
||||
done_save();
|
||||
if let Err(err) = save_result {
|
||||
|
||||
@@ -591,6 +591,7 @@ async fn remote_worker_rechecks_real_producers_before_persisting_complete_usage(
|
||||
tier_registry_generation: Some(tier_generation),
|
||||
checkpoint_identity: Some(identity),
|
||||
checkpoint_plan_digest: Some(plan),
|
||||
completed_checkpoint_cycle_floor: None,
|
||||
},
|
||||
);
|
||||
crate::scanner_io::bind_usage_watch_to_checkpoint(&mut cache, Some(&proof), plan);
|
||||
|
||||
@@ -4451,9 +4451,10 @@ pub use activity::scanner_topology_digest;
|
||||
pub(crate) use activity::{
|
||||
LOCAL_SCANNER_ACTIVITY_NODE, ScannerActivityGuardrailVeto, ScannerActivitySnapshot, ScannerDirtyUsageAcknowledgement,
|
||||
ScannerDirtyUsageAcknowledgementKind, probe_scanner_activity, scanner_activity_allows_usage_publication,
|
||||
scanner_activity_coverage_digest, scanner_activity_dirty_usage_state_for_host, scanner_activity_guardrail_veto,
|
||||
scanner_activity_publication_lease_targets, scanner_activity_snapshot_digest, scanner_activity_structural_digest,
|
||||
scanner_activity_usage_watch_structure_digest, scanner_dirty_usage_acknowledgements, scanner_peer_dirty_usage_moved,
|
||||
scanner_activity_bucket_reuse_digest, scanner_activity_coverage_digest, scanner_activity_dirty_usage_state_for_host,
|
||||
scanner_activity_guardrail_veto, scanner_activity_publication_lease_targets, scanner_activity_snapshot_digest,
|
||||
scanner_activity_structural_digest, scanner_activity_usage_watch_structure_digest, scanner_dirty_usage_acknowledgements,
|
||||
scanner_peer_dirty_usage_moved,
|
||||
};
|
||||
pub(crate) use activity::{ScannerCycleOutcome, scanner_cycle_outcome_with_pending_maintenance};
|
||||
pub use backlog::{
|
||||
|
||||
@@ -535,6 +535,7 @@ pub(crate) const LOCAL_SCANNER_ACTIVITY_NODE: &str = "<local>";
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub(crate) struct ScannerNodeActivity {
|
||||
pub(super) instance_id: String,
|
||||
pub(super) checkpoint_instance_id: Option<String>,
|
||||
pub(super) namespace_generation: u64,
|
||||
pub(super) maintenance_generation: u64,
|
||||
pub(super) protocol_version: u32,
|
||||
@@ -551,6 +552,7 @@ impl ScannerNodeActivity {
|
||||
pub(crate) fn e2e_observation(&self) -> serde_json::Value {
|
||||
serde_json::json!({
|
||||
"instance_id": self.instance_id,
|
||||
"checkpoint_instance_id": self.checkpoint_instance_id,
|
||||
"namespace_generation": self.namespace_generation,
|
||||
"maintenance_generation": self.maintenance_generation,
|
||||
"protocol_version": self.protocol_version,
|
||||
@@ -575,6 +577,7 @@ pub(crate) fn scanner_node_activity_for_tests(
|
||||
) -> ScannerNodeActivity {
|
||||
ScannerNodeActivity {
|
||||
instance_id: instance_id.to_string(),
|
||||
checkpoint_instance_id: None,
|
||||
namespace_generation,
|
||||
maintenance_generation: 0,
|
||||
protocol_version: SCANNER_ACTIVITY_PROTOCOL_VERSION,
|
||||
@@ -1095,11 +1098,32 @@ where
|
||||
}
|
||||
|
||||
pub(crate) fn scanner_activity_snapshot_digest(snapshot: &ScannerActivitySnapshot) -> [u8; 32] {
|
||||
hash_scanner_activity_members(snapshot.iter().map(|(host, activity)| (host.as_str(), activity)), false)
|
||||
}
|
||||
|
||||
/// Completed bucket reuse retains all mutation and maintenance fences while
|
||||
/// allowing a verified read-only handoff to replace only the RPC process epoch.
|
||||
pub(crate) fn scanner_activity_bucket_reuse_digest(snapshot: &ScannerActivitySnapshot, local_node: &str) -> [u8; 32] {
|
||||
hash_scanner_activity_members(scanner_activity_members(snapshot, local_node).into_iter(), true)
|
||||
}
|
||||
|
||||
fn hash_scanner_activity_members<'a>(
|
||||
members: impl ExactSizeIterator<Item = (&'a str, &'a ScannerNodeActivity)>,
|
||||
checkpoint_continuity: bool,
|
||||
) -> [u8; 32] {
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(u64::try_from(snapshot.len()).unwrap_or(u64::MAX).to_be_bytes());
|
||||
for (host, activity) in snapshot {
|
||||
if checkpoint_continuity {
|
||||
hasher.update(b"scanner-bucket-reuse-v1");
|
||||
}
|
||||
hasher.update(u64::try_from(members.len()).unwrap_or(u64::MAX).to_be_bytes());
|
||||
for (host, activity) in members {
|
||||
let host = host.as_bytes();
|
||||
let instance_id = activity.instance_id.as_bytes();
|
||||
let instance_id = if checkpoint_continuity {
|
||||
activity.checkpoint_instance_id.as_deref().unwrap_or(&activity.instance_id)
|
||||
} else {
|
||||
&activity.instance_id
|
||||
}
|
||||
.as_bytes();
|
||||
hasher.update(u64::try_from(host.len()).unwrap_or(u64::MAX).to_be_bytes());
|
||||
hasher.update(host);
|
||||
hasher.update(u64::try_from(instance_id.len()).unwrap_or(u64::MAX).to_be_bytes());
|
||||
@@ -1118,11 +1142,11 @@ pub(crate) fn scanner_activity_snapshot_digest(snapshot: &ScannerActivitySnapsho
|
||||
|
||||
/// The reuse identity: hash the activity inputs that make an existing
|
||||
/// scanner result unsafe to reuse without walking again. Besides the
|
||||
/// coverage identity below it binds each peer's `instance_id` and
|
||||
/// `maintenance_generation`: a restarted node may have lost the dirty hints
|
||||
/// of its last journal window and a changed lifecycle or replication rule
|
||||
/// needs a maintenance pass, so both invalidate the authoritative baseline
|
||||
/// and the set caches and force the next cycle to walk every bucket. With
|
||||
/// coverage identity below it binds each peer's checkpoint continuity and
|
||||
/// `maintenance_generation`. A read-only, clean shutdown may preserve that
|
||||
/// continuity; a writer or crash starts fresh because its last dirty-journal
|
||||
/// window may be missing. A changed lifecycle or replication rule also needs
|
||||
/// a maintenance pass. With
|
||||
/// the bucket list it is the plan under which set caches and the
|
||||
/// scoped-scan baseline are reused (`scanner_bucket_plan_digest`).
|
||||
///
|
||||
@@ -1131,12 +1155,17 @@ pub(crate) fn scanner_activity_snapshot_digest(snapshot: &ScannerActivitySnapsho
|
||||
/// `publication_blocked` is omitted from every identity: a movement pause
|
||||
/// vetoes the cycle it moves in (`scanner_activity_guardrail_veto`) and
|
||||
/// describes nothing about the heap.
|
||||
pub(crate) fn scanner_activity_structural_digest(snapshot: &ScannerActivitySnapshot) -> [u8; 32] {
|
||||
pub(crate) fn scanner_activity_structural_digest(snapshot: &ScannerActivitySnapshot, local_node: &str) -> [u8; 32] {
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(b"scanner-reuse-members-v2");
|
||||
hasher.update(u64::try_from(snapshot.len()).unwrap_or(u64::MAX).to_be_bytes());
|
||||
for (host, activity) in snapshot {
|
||||
for (host, activity) in scanner_activity_members(snapshot, local_node) {
|
||||
let host = host.as_bytes();
|
||||
let instance_id = activity.instance_id.as_bytes();
|
||||
let instance_id = activity
|
||||
.checkpoint_instance_id
|
||||
.as_deref()
|
||||
.unwrap_or(&activity.instance_id)
|
||||
.as_bytes();
|
||||
hasher.update(u64::try_from(host.len()).unwrap_or(u64::MAX).to_be_bytes());
|
||||
hasher.update(host);
|
||||
hasher.update(u64::try_from(instance_id.len()).unwrap_or(u64::MAX).to_be_bytes());
|
||||
@@ -1150,33 +1179,64 @@ pub(crate) fn scanner_activity_structural_digest(snapshot: &ScannerActivitySnaps
|
||||
hasher.finalize().into()
|
||||
}
|
||||
|
||||
fn scanner_activity_members<'a>(
|
||||
snapshot: &'a ScannerActivitySnapshot,
|
||||
local_node: &'a str,
|
||||
) -> Vec<(&'a str, &'a ScannerNodeActivity)> {
|
||||
// The local marker moves to another member when leadership changes.
|
||||
// Hash the actual member identities in canonical order instead.
|
||||
let mut nodes = snapshot
|
||||
.iter()
|
||||
.map(|(host, activity)| {
|
||||
(
|
||||
if host == LOCAL_SCANNER_ACTIVITY_NODE {
|
||||
local_node
|
||||
} else {
|
||||
host.as_str()
|
||||
},
|
||||
activity,
|
||||
)
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
nodes.sort_unstable_by_key(|(host, _)| *host);
|
||||
nodes
|
||||
}
|
||||
|
||||
/// The coverage identity: hash the activity inputs under which a bucket
|
||||
/// sweep's traversal stays one traversal, so a checkpoint resumed in a later
|
||||
/// cycle is not mixed coverage. It binds the peer set, the protocol, the
|
||||
/// topology and the data-movement state. A coordinator restart, a
|
||||
/// maintenance publication or a publication pause changes none of the
|
||||
/// namespace a sweep walks, so `instance_id`, `maintenance_generation` and
|
||||
/// `publication_blocked` are left out; hashing them made every resumed sweep
|
||||
/// restart as mixed coverage after such an event (rustfs/rustfs#8494). The
|
||||
/// local node's namespace and dirty generations are left out too: the
|
||||
/// namespace a sweep walks, so the RPC `instance_id`, `maintenance_generation`
|
||||
/// and `publication_blocked` are left out. Protocol v8 instead binds the
|
||||
/// checkpoint identity inherited only from a durable read-only handoff;
|
||||
/// a lost write hint must invalidate partial as well as completed coverage. The
|
||||
/// standalone node's namespace and dirty generations are left out too: the
|
||||
/// bucket's own dirty generation joins the checkpoint plan in
|
||||
/// `scanner_bucket_cache_digest`, which is what certification compares. A
|
||||
/// remote peer's generations stay in, because its dirty buckets are still
|
||||
/// acknowledged per cycle rather than per bucket, and a write there between
|
||||
/// the two halves of a sweep would otherwise be acknowledged without having
|
||||
/// been walked.
|
||||
pub(crate) fn scanner_activity_coverage_digest(snapshot: &ScannerActivitySnapshot) -> [u8; 32] {
|
||||
/// `scanner_bucket_cache_digest`, which is what certification compares.
|
||||
/// Distributed plans retain every member's generations, including the
|
||||
/// coordinator's, so a leader rotation cannot remove a mutation witness.
|
||||
pub(crate) fn scanner_activity_coverage_digest(snapshot: &ScannerActivitySnapshot, local_node: &str) -> [u8; 32] {
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(b"scanner-checkpoint-coverage-v2");
|
||||
hasher.update(u64::try_from(snapshot.len()).unwrap_or(u64::MAX).to_be_bytes());
|
||||
for (host, activity) in snapshot {
|
||||
for (host, activity) in scanner_activity_members(snapshot, local_node) {
|
||||
let host_bytes = host.as_bytes();
|
||||
hasher.update(u64::try_from(host_bytes.len()).unwrap_or(u64::MAX).to_be_bytes());
|
||||
hasher.update(host_bytes);
|
||||
if let Some(instance) = &activity.checkpoint_instance_id {
|
||||
hasher.update(instance.as_bytes());
|
||||
}
|
||||
hasher.update(activity.protocol_version.to_be_bytes());
|
||||
hasher.update(activity.topology_digest);
|
||||
hasher.update([u8::from(activity.data_movement_active)]);
|
||||
hasher.update(activity.movement_generation.to_be_bytes());
|
||||
if host.as_str() != LOCAL_SCANNER_ACTIVITY_NODE {
|
||||
if host == LOCAL_SCANNER_ACTIVITY_NODE {
|
||||
// Standalone writes are fenced by the bucket's own generation.
|
||||
hasher.update(0_u64.to_be_bytes());
|
||||
hasher.update(0_u64.to_be_bytes());
|
||||
hasher.update([0]);
|
||||
} else {
|
||||
hasher.update(activity.namespace_generation.to_be_bytes());
|
||||
hasher.update(activity.dirty_usage_generation.to_be_bytes());
|
||||
hasher.update([u8::from(activity.dirty_usage_pending)]);
|
||||
@@ -1314,8 +1374,10 @@ pub(crate) fn scanner_peer_dirty_usage_moved(
|
||||
pub(crate) fn scanner_activity_allows_usage_publication(snapshot: &ScannerActivitySnapshot) -> bool {
|
||||
!snapshot.is_empty()
|
||||
&& snapshot.values().all(|activity| {
|
||||
activity.protocol_version == SCANNER_ACTIVITY_PROTOCOL_VERSION
|
||||
&& activity.movement_generation != u64::MAX
|
||||
matches!(
|
||||
activity.protocol_version,
|
||||
crate::SCANNER_ACTIVITY_V7_PROTOCOL_VERSION | SCANNER_ACTIVITY_PROTOCOL_VERSION
|
||||
) && activity.movement_generation != u64::MAX
|
||||
&& !activity.data_movement_active
|
||||
&& !activity.publication_blocked
|
||||
})
|
||||
@@ -1422,6 +1484,7 @@ where
|
||||
LOCAL_SCANNER_ACTIVITY_NODE.to_string(),
|
||||
ScannerNodeActivity {
|
||||
instance_id: local_instance_id,
|
||||
checkpoint_instance_id: Some(crate::scanner_checkpoint_instance_id().to_owned()),
|
||||
namespace_generation,
|
||||
maintenance_generation,
|
||||
protocol_version: SCANNER_ACTIVITY_PROTOCOL_VERSION,
|
||||
@@ -1471,7 +1534,7 @@ where
|
||||
SCANNER_ACTIVITY_V6_PROTOCOL_VERSION
|
||||
));
|
||||
}
|
||||
SCANNER_ACTIVITY_PROTOCOL_VERSION => (
|
||||
crate::SCANNER_ACTIVITY_V7_PROTOCOL_VERSION | SCANNER_ACTIVITY_PROTOCOL_VERSION => (
|
||||
activity
|
||||
.topology_digest
|
||||
.ok_or_else(|| format!("scanner activity peer {host} omitted its storage topology"))?,
|
||||
@@ -1510,6 +1573,7 @@ where
|
||||
host.clone(),
|
||||
ScannerNodeActivity {
|
||||
instance_id: activity.instance_id,
|
||||
checkpoint_instance_id: activity.checkpoint_instance_id,
|
||||
namespace_generation: activity.namespace_generation,
|
||||
maintenance_generation: activity.maintenance_generation,
|
||||
protocol_version: activity.protocol_version,
|
||||
|
||||
+101
-116
@@ -899,7 +899,7 @@ async fn checkpoint_fixture_runtime_deadline_save_reload_resume() {
|
||||
.await
|
||||
.expect("initial cache revisions");
|
||||
checkpoint
|
||||
.save_with_revisions_for_epoch(store.clone(), cache_name, &cache_revisions, 0)
|
||||
.save_with_revisions_for_epoch(store.clone(), cache_name, &cache_revisions, 0, CancellationToken::new())
|
||||
.await
|
||||
.expect("publish the current checkpoint into the fenced store");
|
||||
let cache_path = rustfs_utils::path::path_join_buf(&[crate::BUCKET_META_PREFIX, cache_name]);
|
||||
@@ -1635,94 +1635,6 @@ async fn run_data_scanner_cycle_publishes_activity_for_owner_lifetime() {
|
||||
global_metrics().set_cycle(None).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn single_drive_cycle_persists_usage_and_advances_with_healing_marker() {
|
||||
use crate::scanner_io::{clear_dirty_usage_buckets_for_tests, tests::wait_for_namespace_commit_tails};
|
||||
use crate::storage_api::scan::{HEALING_MARKER_PATH, MakeBucketOptions};
|
||||
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
let temp_dir = tempfile::tempdir().expect("scanner cycle test directory should be created");
|
||||
let store = setup_scanner_cycle_store_at_path_with_layout(temp_dir.path(), true, 1, 1, 1).await;
|
||||
let bucket = "marked-drive-bucket";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("fixture bucket should be created");
|
||||
let mut reader = PutObjReader::from_vec(vec![7; 16]);
|
||||
store
|
||||
.put_object(bucket, "object", &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("fixture object should persist");
|
||||
wait_for_namespace_commit_tails(store.as_ref()).await;
|
||||
|
||||
let marker = temp_dir
|
||||
.path()
|
||||
.join("pool0/disk0")
|
||||
.join(RUSTFS_META_BUCKET)
|
||||
.join(HEALING_MARKER_PATH);
|
||||
let marker_contents = b"leftover marker";
|
||||
tokio::fs::write(&marker, marker_contents)
|
||||
.await
|
||||
.expect("fixture healing marker should persist");
|
||||
let set = &store.pools[0].disk_set[0];
|
||||
tokio::time::timeout(Duration::from_secs(10), async {
|
||||
while !set.get_online_disks_with_healing(true).await.1 {
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("disk info should observe the marker after its cache expires");
|
||||
|
||||
let ctx = CancellationToken::new();
|
||||
let mut cycle_info = CurrentCycle {
|
||||
next: 1,
|
||||
..Default::default()
|
||||
};
|
||||
let mut revision = DataUsageCacheRevision::Missing;
|
||||
let leader_epoch = 11;
|
||||
let outcome = tokio::time::timeout(
|
||||
Duration::from_secs(60),
|
||||
run_data_scanner_cycle(&ctx, &store, &mut cycle_info, &mut revision, leader_epoch),
|
||||
)
|
||||
.await
|
||||
.expect("scanner cycle should finish");
|
||||
assert!(
|
||||
matches!(
|
||||
outcome,
|
||||
ScannerCycleOutcome::Completed | ScannerCycleOutcome::CompletedWithPendingMaintenance
|
||||
),
|
||||
"unexpected cycle outcome: {outcome:?}",
|
||||
);
|
||||
|
||||
let mut set_snapshot = DataUsageCache::default();
|
||||
set_snapshot
|
||||
.load_with_revisions(Arc::clone(set), DATA_USAGE_CACHE_NAME)
|
||||
.await
|
||||
.expect("set usage snapshot should be readable");
|
||||
assert!(set_snapshot.info.snapshot_complete);
|
||||
assert_eq!(set_snapshot.info.next_cycle, 1);
|
||||
|
||||
let usage = read_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
|
||||
.await
|
||||
.expect("global usage snapshot should be durable");
|
||||
let usage: DataUsageInfo = serde_json::from_slice(&usage).expect("global usage snapshot should decode");
|
||||
assert!(usage.usage_snapshot_complete);
|
||||
assert!(usage.last_update.is_some());
|
||||
assert_eq!(usage.buckets_usage[bucket].objects_count, 1);
|
||||
|
||||
let persisted_cycle = read_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
|
||||
.await
|
||||
.expect("scanner cycle cursor should be durable");
|
||||
let (persisted_cycle, persisted_epoch) =
|
||||
decode_scanner_cycle_state(&persisted_cycle).expect("scanner cycle cursor should decode");
|
||||
assert_eq!(persisted_epoch, leader_epoch);
|
||||
assert_eq!(persisted_cycle.next, 2);
|
||||
assert_eq!(tokio::fs::read(marker).await.expect("healing marker should remain"), marker_contents);
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
global_metrics().set_cycle(None).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn coordinator_publishes_authority_during_pending_put() {
|
||||
@@ -10445,6 +10357,7 @@ async fn recovery_wake_interrupts_normal_cycle_wait() {
|
||||
|
||||
fn scanner_node_activity(epoch: &str, namespace_generation: u64, maintenance_generation: u64) -> ScannerNodeActivity {
|
||||
ScannerNodeActivity {
|
||||
checkpoint_instance_id: None,
|
||||
instance_id: epoch.to_string(),
|
||||
namespace_generation,
|
||||
maintenance_generation,
|
||||
@@ -10465,7 +10378,10 @@ fn scoped_scan_remote_dirty_coverage_invalidates_local_bucket_current() {
|
||||
let remote = after.get_mut("remote").expect("remote activity should exist");
|
||||
remote.dirty_usage_generation += 1;
|
||||
remote.dirty_usage_pending = true;
|
||||
assert_eq!(scanner_activity_structural_digest(&before), scanner_activity_structural_digest(&after));
|
||||
assert_eq!(
|
||||
scanner_activity_structural_digest(&before, crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE),
|
||||
scanner_activity_structural_digest(&after, crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE)
|
||||
);
|
||||
let old_plan = crate::scanner_io::checkpoint_fixture_bucket_digest(
|
||||
DataUsageScanPlanDigest(scanner_activity_snapshot_digest(&before)),
|
||||
None,
|
||||
@@ -10507,6 +10423,7 @@ fn scoped_scan_remote_dirty_coverage_invalidates_local_bucket_current() {
|
||||
tier_registry_generation: None,
|
||||
checkpoint_identity: None,
|
||||
checkpoint_plan_digest: None,
|
||||
completed_checkpoint_cycle_floor: None,
|
||||
},
|
||||
),
|
||||
crate::scanner_io::DataUsageCacheScanState::Prepared {
|
||||
@@ -10696,6 +10613,14 @@ fn scanner_activity_snapshot_fences_data_movement() {
|
||||
assert!(scanner_activity_allows_usage_publication(&idle));
|
||||
assert!(!scanner_activity_allows_usage_publication(&moving));
|
||||
assert_ne!(scanner_activity_snapshot_digest(&idle), scanner_activity_snapshot_digest(&moving));
|
||||
let mut v7 = idle;
|
||||
v7.get_mut("node-2").expect("node").protocol_version = crate::SCANNER_ACTIVITY_V7_PROTOCOL_VERSION;
|
||||
assert!(
|
||||
scanner_activity_allows_usage_publication(&v7),
|
||||
"v7 still authenticates terminal movement state"
|
||||
);
|
||||
v7.get_mut("node-2").expect("node").protocol_version = crate::SCANNER_ACTIVITY_V6_PROTOCOL_VERSION;
|
||||
assert!(!scanner_activity_allows_usage_publication(&v7));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -10724,8 +10649,8 @@ fn scanner_activity_structural_digest_ignores_regular_bucket_writes() {
|
||||
|
||||
assert_ne!(scanner_activity_snapshot_digest(&baseline), scanner_activity_snapshot_digest(&written));
|
||||
assert_eq!(
|
||||
scanner_activity_structural_digest(&baseline),
|
||||
scanner_activity_structural_digest(&written),
|
||||
scanner_activity_structural_digest(&baseline, crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE),
|
||||
scanner_activity_structural_digest(&written, crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE),
|
||||
"bucket writes are refreshed through the dirty-bucket scope rather than invalidating every cache"
|
||||
);
|
||||
}
|
||||
@@ -10747,18 +10672,24 @@ fn scanner_activity_structural_digest_fences_restart_and_maintenance_but_not_pub
|
||||
// maintenance rule needs a maintenance pass, so both invalidate every
|
||||
// reusable result (the baseline and the set caches).
|
||||
assert_ne!(
|
||||
scanner_activity_structural_digest(&baseline),
|
||||
scanner_activity_structural_digest(&restarted)
|
||||
scanner_activity_structural_digest(&baseline, crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE),
|
||||
scanner_activity_structural_digest(&restarted, crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE)
|
||||
);
|
||||
assert_ne!(
|
||||
scanner_activity_structural_digest(&baseline),
|
||||
scanner_activity_structural_digest(&maintained)
|
||||
scanner_activity_structural_digest(&baseline, crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE),
|
||||
scanner_activity_structural_digest(&maintained, crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE)
|
||||
);
|
||||
// A publication pause describes nothing about the heap: it vetoes the
|
||||
// cycle it moves in and is part of no identity.
|
||||
assert_eq!(scanner_activity_structural_digest(&baseline), scanner_activity_structural_digest(&paused));
|
||||
assert_eq!(
|
||||
scanner_activity_structural_digest(&baseline, crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE),
|
||||
scanner_activity_structural_digest(&paused, crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE)
|
||||
);
|
||||
assert_eq!(scanner_activity_snapshot_digest(&baseline), scanner_activity_snapshot_digest(&paused));
|
||||
assert_eq!(scanner_activity_coverage_digest(&baseline), scanner_activity_coverage_digest(&paused));
|
||||
assert_eq!(
|
||||
scanner_activity_coverage_digest(&baseline, crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE),
|
||||
scanner_activity_coverage_digest(&paused, crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE)
|
||||
);
|
||||
assert_eq!(
|
||||
scanner_activity_guardrail_veto(&baseline, &paused).map(|veto| veto.field),
|
||||
Some("publication_blocked")
|
||||
@@ -10787,8 +10718,8 @@ fn scanner_activity_coverage_digest_survives_restart_and_maintenance_but_fences_
|
||||
let mut changed = baseline.clone();
|
||||
mutate(changed.get_mut(host).expect("host should be present"));
|
||||
assert_eq!(
|
||||
scanner_activity_coverage_digest(&baseline),
|
||||
scanner_activity_coverage_digest(&changed),
|
||||
scanner_activity_coverage_digest(&baseline, crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE),
|
||||
scanner_activity_coverage_digest(&changed, crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE),
|
||||
"{field} on {host} must not change the coverage identity"
|
||||
);
|
||||
assert_eq!(
|
||||
@@ -10807,8 +10738,8 @@ fn scanner_activity_coverage_digest_survives_restart_and_maintenance_but_fences_
|
||||
let mut changed = baseline.clone();
|
||||
mutate(changed.get_mut(LOCAL_SCANNER_ACTIVITY_NODE).expect("local activity"));
|
||||
assert_eq!(
|
||||
scanner_activity_coverage_digest(&baseline),
|
||||
scanner_activity_coverage_digest(&changed),
|
||||
scanner_activity_coverage_digest(&baseline, crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE),
|
||||
scanner_activity_coverage_digest(&changed, crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE),
|
||||
"local {field} is certified per bucket through the bucket's own dirty generation"
|
||||
);
|
||||
// A peer's generations stay in: its dirty buckets are acknowledged
|
||||
@@ -10817,8 +10748,8 @@ fn scanner_activity_coverage_digest_survives_restart_and_maintenance_but_fences_
|
||||
let mut peer_changed = baseline.clone();
|
||||
mutate(peer_changed.get_mut("node-2").expect("peer activity"));
|
||||
assert_ne!(
|
||||
scanner_activity_coverage_digest(&baseline),
|
||||
scanner_activity_coverage_digest(&peer_changed),
|
||||
scanner_activity_coverage_digest(&baseline, crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE),
|
||||
scanner_activity_coverage_digest(&peer_changed, crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE),
|
||||
"remote {field} must change the coverage identity"
|
||||
);
|
||||
}
|
||||
@@ -10833,20 +10764,63 @@ fn scanner_activity_coverage_digest_survives_restart_and_maintenance_but_fences_
|
||||
let mut changed = baseline.clone();
|
||||
mutate(changed.get_mut(host).expect("host should be present"));
|
||||
assert_ne!(
|
||||
scanner_activity_coverage_digest(&baseline),
|
||||
scanner_activity_coverage_digest(&changed),
|
||||
scanner_activity_coverage_digest(&baseline, crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE),
|
||||
scanner_activity_coverage_digest(&changed, crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE),
|
||||
"{field} on {host} must change the coverage identity"
|
||||
);
|
||||
assert_ne!(
|
||||
scanner_activity_structural_digest(&baseline),
|
||||
scanner_activity_structural_digest(&changed),
|
||||
scanner_activity_structural_digest(&baseline, crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE),
|
||||
scanner_activity_structural_digest(&changed, crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE),
|
||||
"{field} on {host} must change the reuse identity"
|
||||
);
|
||||
}
|
||||
}
|
||||
let mut peer_lost = baseline.clone();
|
||||
peer_lost.remove("node-2");
|
||||
assert_ne!(scanner_activity_coverage_digest(&baseline), scanner_activity_coverage_digest(&peer_lost));
|
||||
assert_ne!(
|
||||
scanner_activity_coverage_digest(&baseline, crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE),
|
||||
scanner_activity_coverage_digest(&peer_lost, crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_checkpoint_coverage_survives_clean_leader_rotation() {
|
||||
let first = BTreeMap::from([
|
||||
(LOCAL_SCANNER_ACTIVITY_NODE.to_string(), scanner_node_activity("leader-a", 0, 0)),
|
||||
("node-b".to_string(), scanner_node_activity("peer-b", 0, 0)),
|
||||
("node-c".to_string(), scanner_node_activity("peer-c", 0, 0)),
|
||||
]);
|
||||
let second = BTreeMap::from([
|
||||
(LOCAL_SCANNER_ACTIVITY_NODE.to_string(), scanner_node_activity("leader-b", 0, 0)),
|
||||
("node-a".to_string(), scanner_node_activity("peer-a", 0, 0)),
|
||||
("node-c".to_string(), scanner_node_activity("peer-c", 0, 0)),
|
||||
]);
|
||||
assert_eq!(
|
||||
scanner_activity_coverage_digest(&first, "node-a"),
|
||||
scanner_activity_coverage_digest(&second, "node-b"),
|
||||
"a quiet namespace must retain checkpoint coverage when a different node becomes leader"
|
||||
);
|
||||
assert_ne!(
|
||||
scanner_activity_structural_digest(&first, "node-a"),
|
||||
scanner_activity_structural_digest(&second, "node-b"),
|
||||
"restarted processes must invalidate completed results even with reset mutation counters"
|
||||
);
|
||||
let mut rotated = first.clone();
|
||||
let previous_leader = rotated.remove(LOCAL_SCANNER_ACTIVITY_NODE).expect("previous leader");
|
||||
let next_leader = rotated.remove("node-b").expect("next leader");
|
||||
rotated.insert("node-a".to_string(), previous_leader);
|
||||
rotated.insert(LOCAL_SCANNER_ACTIVITY_NODE.to_string(), next_leader);
|
||||
assert_eq!(
|
||||
scanner_activity_structural_digest(&first, "node-a"),
|
||||
scanner_activity_structural_digest(&rotated, "node-b"),
|
||||
"a leadership change alone must retain completed coverage"
|
||||
);
|
||||
rotated.get_mut("node-a").expect("previous leader").namespace_generation += 1;
|
||||
assert_ne!(
|
||||
scanner_activity_coverage_digest(&first, "node-a"),
|
||||
scanner_activity_coverage_digest(&rotated, "node-b"),
|
||||
"a write on the previous coordinator must invalidate its completed coverage"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -10878,10 +10852,18 @@ fn bucket_checkpoint_plan_survives_a_coordinator_restart() {
|
||||
// restarted coordinator requests, or every resumed sweep restarts as
|
||||
// mixed coverage and the bucket can never certify (rustfs/rustfs#8494).
|
||||
// The reuse plan does change: finished results are walked again.
|
||||
let coverage_plan =
|
||||
|snapshot| crate::scanner_io::scanner_bucket_plan_digest(&buckets, scanner_activity_coverage_digest(snapshot));
|
||||
let reuse_plan =
|
||||
|snapshot| crate::scanner_io::scanner_bucket_plan_digest(&buckets, scanner_activity_structural_digest(snapshot));
|
||||
let coverage_plan = |snapshot| {
|
||||
crate::scanner_io::scanner_bucket_plan_digest(
|
||||
&buckets,
|
||||
scanner_activity_coverage_digest(snapshot, crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE),
|
||||
)
|
||||
};
|
||||
let reuse_plan = |snapshot| {
|
||||
crate::scanner_io::scanner_bucket_plan_digest(
|
||||
&buckets,
|
||||
scanner_activity_structural_digest(snapshot, crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE),
|
||||
)
|
||||
};
|
||||
assert_eq!(coverage_plan(&before), coverage_plan(&restarted));
|
||||
assert_ne!(reuse_plan(&before), reuse_plan(&restarted));
|
||||
|
||||
@@ -10889,7 +10871,10 @@ fn bucket_checkpoint_plan_survives_a_coordinator_restart() {
|
||||
extra_bucket.push(bucket_info("gamma"));
|
||||
assert_ne!(
|
||||
coverage_plan(&before),
|
||||
crate::scanner_io::scanner_bucket_plan_digest(&extra_bucket, scanner_activity_coverage_digest(&before)),
|
||||
crate::scanner_io::scanner_bucket_plan_digest(
|
||||
&extra_bucket,
|
||||
scanner_activity_coverage_digest(&before, crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE)
|
||||
),
|
||||
"the bucket list stays part of the plan"
|
||||
);
|
||||
}
|
||||
@@ -11085,8 +11070,8 @@ fn generation_movement_shades_buckets_without_a_guardrail_veto() {
|
||||
assert_eq!(scanner_activity_guardrail_veto(&before, &after), None, "{field} on {host} must not veto");
|
||||
assert!(SCANNER_ACTIVITY_SHADE_FIELDS.contains(field));
|
||||
assert_eq!(
|
||||
scanner_activity_structural_digest(&before),
|
||||
scanner_activity_structural_digest(&after),
|
||||
scanner_activity_structural_digest(&before, crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE),
|
||||
scanner_activity_structural_digest(&after, crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE),
|
||||
"{field} must stay outside the structural digest"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -395,7 +395,7 @@ async fn assert_coordinator_partial_progress(resume_mixed_plan: bool, fail_cycle
|
||||
progress.started_plan.0[0] ^= 1;
|
||||
let epoch = checkpoint.info.scan_identity.expect("checkpoint identity").publication_epoch;
|
||||
checkpoint
|
||||
.save_with_revisions_for_epoch(set.clone(), &cache_name, &revisions, epoch)
|
||||
.save_with_revisions_for_epoch(set.clone(), &cache_name, &revisions, epoch, CancellationToken::new())
|
||||
.await
|
||||
.expect("persist a prefix from a different process plan");
|
||||
|
||||
|
||||
@@ -932,7 +932,7 @@ async fn usage_durable_root_ack_requires_fresh_contributions_from_every_source()
|
||||
progress.started_plan.0[0] ^= 1;
|
||||
let epoch = cache.info.scan_identity.expect("checkpoint identity").publication_epoch;
|
||||
cache
|
||||
.save_with_revisions_for_epoch(set, &cache_name, &revisions, epoch)
|
||||
.save_with_revisions_for_epoch(set, &cache_name, &revisions, epoch, CancellationToken::new())
|
||||
.await
|
||||
.expect("persist a resumed interval mismatch on one source");
|
||||
mixed_source = Some(index);
|
||||
|
||||
@@ -623,7 +623,7 @@ async fn checkpoint_fixture_existing_uncovered_cursor_cannot_skip_to_complete()
|
||||
cache.info.scan_coverage_receipt.as_mut().expect("receipt").through = "bucket/z-stale".into();
|
||||
}
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0)
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0, CancellationToken::new())
|
||||
.await
|
||||
.expect("persist corrupted existing cursor");
|
||||
let mut loaded = store.strict_load().await;
|
||||
@@ -663,7 +663,7 @@ async fn checkpoint_fixture_existing_uncovered_cursor_cannot_skip_to_complete()
|
||||
assert_eq!(cache.checked_flatten("bucket/b-pending").expect("revisited prefix").size, 7);
|
||||
assert!(!cache.info.snapshot_complete);
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0)
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0, CancellationToken::new())
|
||||
.await
|
||||
.expect("persist verified partial");
|
||||
assert!(!store.strict_load().await.info.snapshot_complete);
|
||||
@@ -827,7 +827,7 @@ async fn check_complete_sampling_resumption(resume_mode: HealScanMode) {
|
||||
.await
|
||||
.expect("fixture revision");
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0)
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0, CancellationToken::new())
|
||||
.await
|
||||
.expect("persist sampling partial");
|
||||
if resume_mode == HealScanMode::Deep {
|
||||
@@ -876,7 +876,7 @@ async fn check_complete_sampling_resumption(resume_mode: HealScanMode) {
|
||||
assert_eq!(cache.find("bucket/prefix/0000").expect("Deep revisits earlier prefix").size, 7);
|
||||
}
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0)
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0, CancellationToken::new())
|
||||
.await
|
||||
.expect("save bounded recovery");
|
||||
if complete {
|
||||
@@ -964,6 +964,7 @@ async fn run_hot_bucket_certification(advance_epoch: bool, logs: &CapturedLogs)
|
||||
tier_registry_generation: Some(identity.tier_registry_generation),
|
||||
checkpoint_identity: Some(identity),
|
||||
checkpoint_plan_digest: None,
|
||||
completed_checkpoint_cycle_floor: None,
|
||||
},
|
||||
);
|
||||
cache.info.skip_healing = true;
|
||||
@@ -995,7 +996,7 @@ async fn run_hot_bucket_certification(advance_epoch: bool, logs: &CapturedLogs)
|
||||
};
|
||||
assert_eq!(budget.budget_elapsed(), max_objects.is_some(), "round {round} budget");
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0)
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0, CancellationToken::new())
|
||||
.await
|
||||
.expect("save hot bucket sweep");
|
||||
let saved = store.strict_load().await;
|
||||
@@ -1209,7 +1210,7 @@ pub(super) async fn scan_budgeted_metadata_repair_tail() -> DataUsageCache {
|
||||
assert!(!cache.info.skip_healing);
|
||||
assert!(!cache.info.snapshot_complete);
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0)
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0, CancellationToken::new())
|
||||
.await
|
||||
.expect("persist scanner repair obligation with CAS");
|
||||
let saved = store.strict_load().await;
|
||||
@@ -1282,6 +1283,7 @@ where
|
||||
tier_registry_generation: None,
|
||||
checkpoint_identity: Some(identity),
|
||||
checkpoint_plan_digest: None,
|
||||
completed_checkpoint_cycle_floor: None,
|
||||
},
|
||||
);
|
||||
let prepared = retained(&cache);
|
||||
@@ -1318,7 +1320,7 @@ where
|
||||
assert_eq!(budget.reason(), Some(crate::scanner_budget::ScannerCycleBudgetReason::Objects));
|
||||
let scanned = retained(&cache);
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0)
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0, CancellationToken::new())
|
||||
.await
|
||||
.expect("persist partial checkpoint");
|
||||
let mut loaded = DataUsageCache::default();
|
||||
@@ -1357,14 +1359,14 @@ where
|
||||
.await
|
||||
.expect("read durable checkpoint bytes");
|
||||
let epoch_error = loaded
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 1)
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 1, CancellationToken::new())
|
||||
.await
|
||||
.expect_err("stale publication epoch must reject persistence");
|
||||
assert!(epoch_error.to_string().contains(crate::SCANNER_PUBLICATION_EPOCH_CHANGED));
|
||||
store.reject_save.store(true, Ordering::SeqCst);
|
||||
loaded.info.next_cycle += 1;
|
||||
loaded
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0)
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0, CancellationToken::new())
|
||||
.await
|
||||
.expect_err("injected save failure must not report durable progress");
|
||||
assert_eq!(
|
||||
@@ -1422,6 +1424,7 @@ where
|
||||
tier_registry_generation: Some(identity.tier_registry_generation),
|
||||
checkpoint_identity: Some(identity),
|
||||
checkpoint_plan_digest: None,
|
||||
completed_checkpoint_cycle_floor: None,
|
||||
},
|
||||
);
|
||||
cache.info.skip_healing = true;
|
||||
@@ -1452,7 +1455,7 @@ where
|
||||
ScannerDiskScanOutcome::NamespaceNotFound(_) => panic!("fixture namespace exists"),
|
||||
};
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0)
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0, CancellationToken::new())
|
||||
.await
|
||||
.expect("save each bounded sweep");
|
||||
let saved = store.strict_load().await;
|
||||
|
||||
@@ -134,7 +134,7 @@ async fn assert_completed_compaction_resumes(mode: HealScanMode) {
|
||||
.await
|
||||
.expect("initial fixture revisions");
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0)
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0, CancellationToken::new())
|
||||
.await
|
||||
.expect("persist compacted partial cache through the production codec and CAS path");
|
||||
let mut reloaded = store.strict_load().await;
|
||||
|
||||
@@ -42,7 +42,7 @@ async fn save_reload(store: &Arc<FixtureStore>, cache: &DataUsageCache) -> DataU
|
||||
.await
|
||||
.expect("fixture save revisions");
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0)
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0, CancellationToken::new())
|
||||
.await
|
||||
.expect("save compacted checkpoint through real codec and revision checks");
|
||||
let loaded = store.strict_load().await;
|
||||
|
||||
@@ -150,7 +150,7 @@ async fn nested_periodic_snapshot(
|
||||
.await
|
||||
.expect("initial periodic fixture revisions");
|
||||
snapshot
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0)
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0, CancellationToken::new())
|
||||
.await
|
||||
.expect("persist real mid-walk periodic snapshot");
|
||||
let mut loaded = DataUsageCache::default();
|
||||
@@ -223,7 +223,7 @@ async fn periodic_mixed_tail_requires_terminal_persistence_before_fresh_certific
|
||||
.await
|
||||
.expect("initial periodic fixture revisions");
|
||||
periodic
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0)
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0, CancellationToken::new())
|
||||
.await
|
||||
.expect("seed durable periodic snapshot");
|
||||
|
||||
@@ -281,7 +281,7 @@ async fn periodic_mixed_tail_requires_terminal_persistence_before_fresh_certific
|
||||
|
||||
store.reject_save.store(round == 0, Ordering::SeqCst);
|
||||
let saved = terminal
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0)
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0, CancellationToken::new())
|
||||
.await;
|
||||
if round == 0 {
|
||||
assert!(saved.is_err(), "injected terminal persistence failure must be observable");
|
||||
@@ -369,7 +369,7 @@ async fn periodic_nested_snapshot_keeps_durable_frontier_across_early_repeated_h
|
||||
.await
|
||||
.expect("initial repeated-handoff revisions");
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0)
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0, CancellationToken::new())
|
||||
.await
|
||||
.expect("save the inherited valid subtree frontier");
|
||||
let (mut scanner, unused_root) = build_test_scanner().await;
|
||||
@@ -442,7 +442,7 @@ async fn periodic_nested_snapshot_keeps_durable_frontier_across_early_repeated_h
|
||||
"inherited completed object and both newly read objects must survive"
|
||||
);
|
||||
candidate
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0)
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0, CancellationToken::new())
|
||||
.await
|
||||
.expect("persist later coverage without losing the inherited prefix");
|
||||
let mut latest = DataUsageCache::default();
|
||||
@@ -958,7 +958,7 @@ async fn periodic_failed_subtree_resumes_healthy_descendants_and_compacted_debt(
|
||||
.await
|
||||
.expect("fresh revisions");
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0)
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0, CancellationToken::new())
|
||||
.await
|
||||
.expect("persist failure-bearing coverage");
|
||||
let mut adopted = DataUsageCache::default();
|
||||
@@ -1220,7 +1220,7 @@ async fn checkpoint_debt_restart_round(request: &CheckpointDebtRequest) -> serde
|
||||
let expected = checkpoint_debt_observation(&cache, state.identity, frontier, failed_objects);
|
||||
assert_eq!(cache.info.leader_epoch, 8 + u64::from(request.round));
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0)
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0, CancellationToken::new())
|
||||
.await
|
||||
.expect("persist combined healthy coverage and failure debt with CAS");
|
||||
let saved = store.strict_load().await;
|
||||
|
||||
@@ -303,7 +303,7 @@ async fn walk_and_save(observe: bool) -> (Vec<String>, serde_json::Value) {
|
||||
.expect("read empty fixture revisions");
|
||||
scanner
|
||||
.new_cache
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0)
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0, CancellationToken::new())
|
||||
.await
|
||||
.expect("save actual walker output through the cache codec and revision gate");
|
||||
let loaded = store.strict_load().await;
|
||||
|
||||
@@ -931,6 +931,7 @@ pub struct ScannerBucketScanPlan {
|
||||
/// coordinator restart or a maintenance publication does not turn a
|
||||
/// resumed sweep into mixed coverage.
|
||||
checkpoint_coverage_digest: DataUsageScanPlanDigest,
|
||||
completed_checkpoint_cycle_floor: Option<u64>,
|
||||
requires_full_scan: bool,
|
||||
service_cohort: Option<Arc<StdMutex<ScannerServiceCohort>>>,
|
||||
// Cache work must invalidate on namespace completion even when its scoped baseline remains reusable.
|
||||
@@ -2145,8 +2146,8 @@ use guards::*;
|
||||
|
||||
pub(crate) use cache::{
|
||||
DataUsageCacheReuseOptions, DataUsageCacheScanState, ScannerCheckpointPersistContext, ScannerCheckpointPersistResult,
|
||||
acquire_scanner_cache_locks, current_cache_root_or_prepare_with_generation, log_saved_partial_bucket_cache,
|
||||
persist_scanner_checkpoint,
|
||||
acquire_scanner_cache_locks, completed_checkpoint_cycle_floor, current_cache_root_or_prepare_with_generation,
|
||||
log_saved_partial_bucket_cache, persist_scanner_checkpoint,
|
||||
};
|
||||
pub use dirty_usage::{
|
||||
ScannerDirtyUsageAckError, ScannerDirtyUsageBucket, ScannerDirtyUsageClearObserver, ScannerDirtyUsageMutationObserver,
|
||||
@@ -2156,8 +2157,10 @@ pub use dirty_usage::{
|
||||
probe_scoped_dirty_usage, record_dirty_usage_bucket, record_dirty_usage_bucket_from_producer,
|
||||
record_dirty_usage_bucket_from_producers, record_dirty_usage_object, record_dirty_usage_object_from_producer,
|
||||
record_scanner_maintenance_change, record_status_class_mutation, replay_durable_dirty_usage_producer_record,
|
||||
scanner_activity_epoch, scanner_dirty_usage_bucket_generation, scanner_dirty_usage_snapshot, scanner_dirty_usage_state,
|
||||
scanner_maintenance_generation, set_scanner_dirty_usage_clear_observer, set_scanner_dirty_usage_mutation_observer,
|
||||
restore_scanner_checkpoint_instance_id, scanner_activity_epoch, scanner_checkpoint_instance_id,
|
||||
scanner_dirty_usage_bucket_generation, scanner_dirty_usage_snapshot, scanner_dirty_usage_state,
|
||||
scanner_maintenance_generation, scanner_process_was_read_only, set_scanner_dirty_usage_clear_observer,
|
||||
set_scanner_dirty_usage_mutation_observer,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use dirty_usage::{clear_dirty_usage_buckets_for_tests, dirty_usage_buckets_for_tests};
|
||||
|
||||
@@ -97,11 +97,15 @@ where
|
||||
return ScannerCheckpointPersistResult::FenceChanged;
|
||||
}
|
||||
|
||||
let checkpoint_save = crate::storage_api::ecstore_with_background_disk_io(
|
||||
context.ctx.clone(),
|
||||
checkpoint.save_with_revisions_for_epoch(store.clone(), cache_name, revisions, context.expected_publication_epoch),
|
||||
)
|
||||
.await;
|
||||
let checkpoint_save = checkpoint
|
||||
.save_with_revisions_for_epoch(
|
||||
store.clone(),
|
||||
cache_name,
|
||||
revisions,
|
||||
context.expected_publication_epoch,
|
||||
context.ctx.clone(),
|
||||
)
|
||||
.await;
|
||||
if let Err(error) = checkpoint_save {
|
||||
record_checkpoint_save_failure(cache_name, checkpoint, &error);
|
||||
return if matches!(error, StorageError::PreconditionFailed) {
|
||||
@@ -359,6 +363,39 @@ pub(crate) struct DataUsageCacheReuseOptions {
|
||||
/// (`scanner_activity_coverage_digest`); `None` resumes under the reuse
|
||||
/// plan itself for requests from older remote coordinators.
|
||||
pub(crate) checkpoint_plan_digest: Option<DataUsageScanPlanDigest>,
|
||||
/// The last authoritative aggregate's cycle. Later completed bucket
|
||||
/// sweeps are still checkpoints of an unpublished aggregate.
|
||||
pub(crate) completed_checkpoint_cycle_floor: Option<u64>,
|
||||
}
|
||||
|
||||
pub(crate) fn completed_checkpoint_cycle_floor(
|
||||
data: Option<&[u8]>,
|
||||
expected_sources: &HashSet<DataUsageCacheSource>,
|
||||
next_cycle: u64,
|
||||
leader_epoch: u64,
|
||||
) -> Option<u64> {
|
||||
let usage: DataUsageInfo = serde_json::from_slice(data?).ok()?;
|
||||
if usage.scanner_epoch.is_some_and(|epoch| epoch > leader_epoch) {
|
||||
return None;
|
||||
}
|
||||
if crate::scanner::data_usage_info_is_bootstrap_pending(&usage) {
|
||||
return Some(0);
|
||||
}
|
||||
let cycle = usage.scanner_cycle.filter(|cycle| *cycle < next_cycle)?;
|
||||
let plan = DataUsageScanPlanDigest(usage.usage_snapshot_set_states.first()?.scan_plan_digest?);
|
||||
complete_scanner_cache_snapshot_plan_digest(
|
||||
&usage,
|
||||
ScannerCacheBaselineProof {
|
||||
authoritative_data: None,
|
||||
observed_candidate_data: None,
|
||||
expected_sources,
|
||||
leader_epoch: usage.scanner_epoch?,
|
||||
want_cycle: cycle,
|
||||
scan_plan_digest: plan,
|
||||
},
|
||||
None,
|
||||
)?;
|
||||
Some(cycle)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -383,6 +420,7 @@ pub(crate) fn current_cache_root_or_prepare(
|
||||
tier_registry_generation: None,
|
||||
checkpoint_identity: None,
|
||||
checkpoint_plan_digest: None,
|
||||
completed_checkpoint_cycle_floor: None,
|
||||
},
|
||||
)
|
||||
}
|
||||
@@ -397,6 +435,38 @@ pub(crate) fn current_cache_root_or_prepare_with_generation(
|
||||
options: DataUsageCacheReuseOptions,
|
||||
) -> DataUsageCacheScanState {
|
||||
let checkpoint_plan_digest = options.checkpoint_plan_digest.unwrap_or(scan_plan_digest);
|
||||
// A new leader must not discard finished progress while other buckets
|
||||
// still need budget. Adopt only completed, unpublished coverage; after
|
||||
// the aggregate is published the cycle floor forces the usual refresh.
|
||||
// Keep the saved cycle unchanged so reuse cannot extend this authority.
|
||||
// The reuse plan still binds every process incarnation: a restart may
|
||||
// lose unflushed dirty hints and must invalidate completed coverage.
|
||||
let unpublished_completion = options.completed_checkpoint_cycle_floor.is_some_and(|floor| {
|
||||
cache.info.next_cycle > floor && cache.info.next_cycle < next_cycle && cache.info.leader_epoch < leader_epoch
|
||||
}) && options.checkpoint_identity.is_some_and(|identity| {
|
||||
identity.is_valid()
|
||||
&& cache.info.scan_identity == Some(identity)
|
||||
&& cache.info.tier_registry_generation == Some(identity.tier_registry_generation)
|
||||
&& options.tier_registry_generation == Some(identity.tier_registry_generation)
|
||||
}) && name != DATA_USAGE_ROOT
|
||||
&& cache.info.name == name
|
||||
&& cache.info.source == Some(source)
|
||||
&& cache.info.cache_key_format == DATA_USAGE_CACHE_KEY_FORMAT
|
||||
&& cache.info.snapshot_complete
|
||||
&& cache.info.last_update.is_some()
|
||||
&& cache.info.scan_progress.is_none()
|
||||
&& !cache.has_scan_position()
|
||||
&& cache.info.scan_plan_digest == Some(scan_plan_digest)
|
||||
&& options.checkpoint_plan_digest.is_some()
|
||||
&& cache.info.scan_coverage_digest == Some(checkpoint_plan_digest)
|
||||
&& !cache
|
||||
.info
|
||||
.replication
|
||||
.as_ref()
|
||||
.is_some_and(|config| config.has_active_rules("", true));
|
||||
if unpublished_completion && let Ok(root) = cache_root_entry_info(cache) {
|
||||
return DataUsageCacheScanState::Current(Box::new(root));
|
||||
}
|
||||
if cache.info.next_cycle <= next_cycle
|
||||
&& cache.info.leader_epoch <= leader_epoch
|
||||
&& cache.info.scan_identity != options.checkpoint_identity
|
||||
@@ -1479,7 +1549,13 @@ pub(super) async fn persist_scanner_service_ledger(
|
||||
ledger.leader_epoch = leader_epoch;
|
||||
persisted.info.scan_bucket_service = Some(ledger);
|
||||
match persisted
|
||||
.save_with_revisions_for_epoch(store, DATA_USAGE_CACHE_NAME, &revisions, expected_publication_epoch)
|
||||
.save_with_revisions_for_epoch(
|
||||
store,
|
||||
DATA_USAGE_CACHE_NAME,
|
||||
&revisions,
|
||||
expected_publication_epoch,
|
||||
CancellationToken::new(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(()) => true,
|
||||
@@ -1662,7 +1738,13 @@ pub(super) async fn persist_and_publish_cache_snapshot(
|
||||
|
||||
let done_save = Metrics::time(Metric::SaveUsage);
|
||||
if let Err(e) = cache_snapshot
|
||||
.save_with_revisions_for_epoch(store.clone(), DATA_USAGE_CACHE_NAME, &revisions, expected_publication_epoch)
|
||||
.save_with_revisions_for_epoch(
|
||||
store.clone(),
|
||||
DATA_USAGE_CACHE_NAME,
|
||||
&revisions,
|
||||
expected_publication_epoch,
|
||||
CancellationToken::new(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
error!(
|
||||
|
||||
@@ -40,6 +40,8 @@ static DIRTY_USAGE_MUTATION_OBSERVER: LazyLock<StdRwLock<Option<ScannerDirtyUsag
|
||||
pub(super) static DIRTY_USAGE_PRODUCER_COVERAGE: AtomicU64 = AtomicU64::new(0);
|
||||
pub(super) static DIRTY_USAGE_BUCKET_NOTIFY: LazyLock<Notify> = LazyLock::new(Notify::new);
|
||||
pub(super) static SCANNER_ACTIVITY_EPOCH: LazyLock<String> = LazyLock::new(|| format!("{:032x}", rand::random::<u128>()));
|
||||
static SCANNER_CHECKPOINT_INSTANCE: std::sync::OnceLock<String> = std::sync::OnceLock::new();
|
||||
static SCANNER_PROCESS_MUTATED: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false);
|
||||
pub(super) static SCANNER_MAINTENANCE_GENERATION: AtomicU64 = AtomicU64::new(0);
|
||||
pub(super) static SCANNER_MAINTENANCE_NOTIFY: LazyLock<Notify> = LazyLock::new(Notify::new);
|
||||
const SCANNER_DURABLE_DIRTY_USAGE_REPLAY_SCHEMA: u16 = 1;
|
||||
@@ -152,6 +154,7 @@ fn notify_dirty_usage_mutation(
|
||||
if bucket.is_empty() {
|
||||
return;
|
||||
}
|
||||
SCANNER_PROCESS_MUTATED.store(true, Ordering::Release);
|
||||
let observer = DIRTY_USAGE_MUTATION_OBSERVER
|
||||
.read()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
@@ -1372,6 +1375,30 @@ pub fn scanner_activity_epoch() -> &'static str {
|
||||
SCANNER_ACTIVITY_EPOCH.as_str()
|
||||
}
|
||||
|
||||
/// Identity used only for checkpoint continuity, never for RPC or dirty ACKs.
|
||||
/// Without a consumed clean-shutdown receipt it is the fresh process identity.
|
||||
pub fn scanner_checkpoint_instance_id() -> &'static str {
|
||||
SCANNER_CHECKPOINT_INSTANCE.get_or_init(|| scanner_activity_epoch().to_owned())
|
||||
}
|
||||
|
||||
/// Dirty replay is not a new mutation. Live invalidations prevent handing
|
||||
/// checkpoint authority to another process even after their dirty ACKs drain.
|
||||
pub fn scanner_process_was_read_only() -> bool {
|
||||
!SCANNER_PROCESS_MUTATED.load(Ordering::Acquire) && scanner_maintenance_generation() == 0
|
||||
}
|
||||
|
||||
/// Restore continuity before starting any scanner or mutation producer.
|
||||
/// The caller must first consume matching, durable, read-only shutdown receipts
|
||||
/// on every local disk. Returns false for invalid or already initialized state.
|
||||
pub fn restore_scanner_checkpoint_instance_id(instance: &str) -> bool {
|
||||
instance.len() == 32
|
||||
&& instance
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
|
||||
&& instance.bytes().any(|byte| byte != b'0')
|
||||
&& SCANNER_CHECKPOINT_INSTANCE.set(instance.to_owned()).is_ok()
|
||||
}
|
||||
|
||||
pub fn scanner_dirty_usage_state() -> ScannerDirtyUsageState {
|
||||
let dirty_buckets = dirty_usage_buckets();
|
||||
ScannerDirtyUsageState {
|
||||
|
||||
@@ -389,6 +389,7 @@ impl ScannerIOCache for SetDisks {
|
||||
digest: scan_plan_digest,
|
||||
bucket_coverage_digest,
|
||||
checkpoint_coverage_digest,
|
||||
completed_checkpoint_cycle_floor,
|
||||
requires_full_scan,
|
||||
service_cohort,
|
||||
execution_digest,
|
||||
@@ -620,10 +621,7 @@ impl ScannerIOCache for SetDisks {
|
||||
.ok_or_else(|| StorageError::other("failed to persist empty scanner set scope"));
|
||||
}
|
||||
|
||||
let single_drive = self.set_drive_count == 1;
|
||||
// A single-drive set has no alternate scan source. A heal marker
|
||||
// still disables scanner heals, but must not exclude its only usage walk.
|
||||
let (disks, healing) = self.get_online_disks_with_healing(single_drive).await;
|
||||
let (disks, healing) = self.get_online_disks_with_healing(false).await;
|
||||
#[cfg(test)]
|
||||
let disks = super::tests::usage_remote_drain::walker_disks(&self, &buckets, disks);
|
||||
if disks.is_empty() {
|
||||
@@ -760,18 +758,7 @@ impl ScannerIOCache for SetDisks {
|
||||
let _ = updates.send(incomplete_scope).await;
|
||||
return Ok(ScannerSourceTurnReceipt::default());
|
||||
}
|
||||
// Coordinator workers need the full inventory only for cross-drive
|
||||
// healing. Remote workers own their disk inventory on the peer.
|
||||
let heal_disk_inventory = if !single_drive
|
||||
&& !healing
|
||||
&& workers
|
||||
.iter()
|
||||
.any(|(_, mode)| *mode == NamespaceScannerWorkerMode::Coordinator)
|
||||
{
|
||||
Some(Arc::new(scanner_set_disk_inventory(self.as_ref()).await))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let set_disk_inventory = Arc::new(scanner_set_disk_inventory(self.as_ref()).await);
|
||||
if unsupported_remote_disks > 0 {
|
||||
debug!(
|
||||
target: "rustfs::scanner::io",
|
||||
@@ -1024,6 +1011,7 @@ impl ScannerIOCache for SetDisks {
|
||||
let store_clone_clone = self.clone();
|
||||
let bucket_result_tx_clone = bucket_result_tx.clone();
|
||||
let bucket_observations_clone = bucket_observations.clone();
|
||||
let set_disk_inventory_clone = set_disk_inventory.clone();
|
||||
let disk_scan_semaphore_clone = disk_scan_semaphore.clone();
|
||||
let queued_disk_bucket_scans_clone = queued_disk_bucket_scans.clone();
|
||||
let active_disk_bucket_scans_clone = active_disk_bucket_scans.clone();
|
||||
@@ -1039,9 +1027,9 @@ impl ScannerIOCache for SetDisks {
|
||||
let worker_retired_buckets = retired_buckets.clone();
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
let worker_incarnations = current_bucket_incarnations.clone();
|
||||
let (remote_capabilities, heal_disk_inventory) = match worker_mode {
|
||||
NamespaceScannerWorkerMode::Coordinator => (None, heal_disk_inventory.clone()),
|
||||
NamespaceScannerWorkerMode::RemoteV4(capabilities) => (Some(capabilities), None),
|
||||
let remote_capabilities = match worker_mode {
|
||||
NamespaceScannerWorkerMode::RemoteV4(capabilities) => Some(capabilities),
|
||||
NamespaceScannerWorkerMode::Coordinator => None,
|
||||
};
|
||||
#[cfg(test)]
|
||||
let worker_index = futs.len();
|
||||
@@ -1528,6 +1516,7 @@ impl ScannerIOCache for SetDisks {
|
||||
tier_registry_generation: Some(tier_registry_generation),
|
||||
checkpoint_identity,
|
||||
checkpoint_plan_digest: Some(bucket_checkpoint_plan_digest),
|
||||
completed_checkpoint_cycle_floor,
|
||||
},
|
||||
);
|
||||
let outcome = match scan_state {
|
||||
@@ -1684,7 +1673,7 @@ impl ScannerIOCache for SetDisks {
|
||||
let scan = crate::workload_admission::with_scanner_node_requests(node_tx, disk.clone().nsscanner_disk(
|
||||
scan_ctx.clone(),
|
||||
bucket_budget.clone(),
|
||||
heal_disk_inventory.as_deref().cloned().unwrap_or_default(),
|
||||
set_disk_inventory_clone.as_ref().clone(),
|
||||
cache,
|
||||
None,
|
||||
ScannerDiskScanOptions {
|
||||
@@ -1887,10 +1876,11 @@ impl ScannerIOCache for SetDisks {
|
||||
} else {
|
||||
// The sweep finished under its coverage plan; the
|
||||
// complete cache is current under the reuse plan,
|
||||
// which a restart or a maintenance publication
|
||||
// changes so the bucket is walked again next time.
|
||||
// while its coverage plan remains available until
|
||||
// the coordinated aggregate is published.
|
||||
if completed_cache.info.snapshot_complete {
|
||||
completed_cache.info.scan_plan_digest = Some(bucket_scan_plan_digest);
|
||||
completed_cache.info.scan_coverage_digest = Some(bucket_checkpoint_plan_digest);
|
||||
}
|
||||
(completed_cache, false)
|
||||
}
|
||||
@@ -1942,6 +1932,7 @@ impl ScannerIOCache for SetDisks {
|
||||
cache_name.as_str(),
|
||||
&revisions,
|
||||
expected_publication_epoch_clone,
|
||||
CancellationToken::new(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -2055,6 +2046,7 @@ impl ScannerIOCache for SetDisks {
|
||||
&cache_name,
|
||||
&revisions,
|
||||
expected_publication_epoch_clone,
|
||||
CancellationToken::new(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
|
||||
@@ -619,6 +619,14 @@ where
|
||||
};
|
||||
|
||||
let distributed = store.setup_is_dist_erasure().await;
|
||||
let local_node = if distributed {
|
||||
store
|
||||
.scanner_local_node_name()
|
||||
.filter(|name| !name.is_empty())
|
||||
.ok_or_else(|| Error::other("scanner local member identity is unavailable"))?
|
||||
} else {
|
||||
crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE.to_string()
|
||||
};
|
||||
#[cfg(test)]
|
||||
let distributed = distributed || peer_observations.is_some();
|
||||
let activity_before = match scanner_activity_preflight(
|
||||
@@ -723,15 +731,23 @@ where
|
||||
let activity_before = remote_start
|
||||
.as_ref()
|
||||
.map_or(activity_before, ScannerRemoteDirtyUsageWindow::activity_for_walk);
|
||||
let structural_scan_plan_digest =
|
||||
scanner_bucket_plan_digest(&all_buckets, crate::scanner::scanner_activity_structural_digest(&activity_before));
|
||||
let structural_scan_plan_digest = scanner_bucket_plan_digest(
|
||||
&all_buckets,
|
||||
crate::scanner::scanner_activity_structural_digest(&activity_before, &local_node),
|
||||
);
|
||||
let scan_plan_digest = scanner_bucket_work_digest(structural_scan_plan_digest, scan_mode, requires_full_scan);
|
||||
let activity_digest = crate::scanner::scanner_activity_snapshot_digest(&activity_before);
|
||||
let bucket_coverage_digest = scanner_bucket_plan_digest(&all_buckets, activity_digest);
|
||||
// A bucket checkpoint must not be invalidated by another bucket joining
|
||||
// the inventory. The set plan above still binds the complete inventory.
|
||||
let checkpoint_coverage_digest = DataUsageScanPlanDigest(crate::scanner::scanner_activity_coverage_digest(&activity_before));
|
||||
let execution_digest = scanner_bucket_work_digest(bucket_coverage_digest, scan_mode, requires_full_scan);
|
||||
let bucket_coverage_digest = scanner_bucket_plan_digest(
|
||||
&all_buckets,
|
||||
crate::scanner::scanner_activity_bucket_reuse_digest(&activity_before, &local_node),
|
||||
);
|
||||
// A bucket checkpoint must survive unrelated inventory changes.
|
||||
let checkpoint_coverage_digest =
|
||||
DataUsageScanPlanDigest(crate::scanner::scanner_activity_coverage_digest(&activity_before, &local_node));
|
||||
let completed_checkpoint_cycle_floor =
|
||||
completed_checkpoint_cycle_floor(persisted_usage_baseline.as_deref(), &expected_sources, want_cycle, leader_epoch);
|
||||
let execution_digest =
|
||||
scanner_bucket_work_digest(scanner_bucket_plan_digest(&all_buckets, activity_digest), scan_mode, requires_full_scan);
|
||||
let dirty_usage_snapshot = Arc::new(snapshot_dirty_usage_buckets_after_listing(&all_buckets, dirty_usage_cutoff));
|
||||
let baseline_proof = ScannerCacheBaselineProof {
|
||||
authoritative_data: persisted_usage_baseline.as_ref(),
|
||||
@@ -1014,6 +1030,7 @@ where
|
||||
digest: structural_scan_plan_digest,
|
||||
bucket_coverage_digest,
|
||||
checkpoint_coverage_digest,
|
||||
completed_checkpoint_cycle_floor,
|
||||
requires_full_scan,
|
||||
service_cohort: service_cohort.clone(),
|
||||
execution_digest,
|
||||
|
||||
@@ -635,6 +635,7 @@ mod tests {
|
||||
tier_registry_generation: None,
|
||||
checkpoint_identity: None,
|
||||
checkpoint_plan_digest: None,
|
||||
completed_checkpoint_cycle_floor: None,
|
||||
};
|
||||
assert!(matches!(
|
||||
current_cache_root_or_prepare_with_generation(&mut cache.clone(), "cold-bucket", source, 7, 11, old_plan, options()),
|
||||
|
||||
@@ -827,6 +827,7 @@ fn current_cache_root_with_new_tier_generation_resets_old_cache() {
|
||||
tier_registry_generation: Some(2),
|
||||
checkpoint_identity: None,
|
||||
checkpoint_plan_digest: None,
|
||||
completed_checkpoint_cycle_floor: None,
|
||||
},
|
||||
);
|
||||
|
||||
|
||||
@@ -547,11 +547,102 @@ async fn setup_two_pool_scanner_store() -> (tempfile::TempDir, Arc<ECStore>) {
|
||||
pub(crate) async fn setup_local_scanner_store(pool_count: usize, drives_per_set: usize) -> (tempfile::TempDir, Arc<ECStore>) {
|
||||
init_ecstore_config_for_scanner_tests();
|
||||
let temp_dir = tempfile::tempdir().expect("multi-pool scanner test directory should be created");
|
||||
let store = open_local_scanner_store(temp_dir.path(), pool_count, drives_per_set, false).await;
|
||||
(temp_dir, store)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn epoch_fenced_cache_save_uses_background_admission_on_every_disk() {
|
||||
#[derive(Clone)]
|
||||
struct AdmissionRecorder {
|
||||
root: String,
|
||||
counts: Arc<std::sync::Mutex<HashMap<String, Arc<AtomicU64>>>>,
|
||||
}
|
||||
impl metrics::Recorder for AdmissionRecorder {
|
||||
fn describe_counter(&self, _: metrics::KeyName, _: Option<metrics::Unit>, _: metrics::SharedString) {}
|
||||
fn describe_gauge(&self, _: metrics::KeyName, _: Option<metrics::Unit>, _: metrics::SharedString) {}
|
||||
fn describe_histogram(&self, _: metrics::KeyName, _: Option<metrics::Unit>, _: metrics::SharedString) {}
|
||||
fn register_counter(&self, key: &metrics::Key, _: &metrics::Metadata<'_>) -> metrics::Counter {
|
||||
if key.name() != "rustfs_disk_io_admission_total"
|
||||
|| !key
|
||||
.labels()
|
||||
.any(|label| label.key() == "class" && label.value() == "background")
|
||||
{
|
||||
return metrics::Counter::noop();
|
||||
}
|
||||
let Some(disk) = key
|
||||
.labels()
|
||||
.find(|label| label.key() == "disk" && label.value().starts_with(&self.root))
|
||||
else {
|
||||
return metrics::Counter::noop();
|
||||
};
|
||||
metrics::Counter::from_arc(
|
||||
self.counts
|
||||
.lock()
|
||||
.expect("admission counters")
|
||||
.entry(disk.value().to_string())
|
||||
.or_default()
|
||||
.clone(),
|
||||
)
|
||||
}
|
||||
fn register_gauge(&self, _: &metrics::Key, _: &metrics::Metadata<'_>) -> metrics::Gauge {
|
||||
metrics::Gauge::noop()
|
||||
}
|
||||
fn register_histogram(&self, _: &metrics::Key, _: &metrics::Metadata<'_>) -> metrics::Histogram {
|
||||
metrics::Histogram::noop()
|
||||
}
|
||||
}
|
||||
let (root, store) = setup_local_scanner_store(1, 4).await;
|
||||
let set = store.pools[0].disk_set[0].clone();
|
||||
let epoch = scanner_publication_epoch(set.clone()).await.expect("publication epoch");
|
||||
let cache_name = "background-admission/.usage-cache.bin";
|
||||
let mut cache = DataUsageCache::default();
|
||||
cache.info.name = "background-admission".to_string();
|
||||
let recorder = AdmissionRecorder {
|
||||
root: root.path().to_str().expect("fixture path").to_string(),
|
||||
counts: Arc::default(),
|
||||
};
|
||||
let counts = recorder.counts.clone();
|
||||
// nextest runs this test in its own process; the recorder also observes the
|
||||
// detached commit tasks and filters out every disk outside this fixture.
|
||||
metrics::set_global_recorder(recorder).expect("native save recorder");
|
||||
cache.save(set.clone(), cache_name).await.expect("unclassified control save");
|
||||
assert!(counts.lock().expect("control counters").is_empty());
|
||||
let revisions = DataUsageCache::read_revisions(set.clone(), cache_name)
|
||||
.await
|
||||
.expect("cache revisions");
|
||||
cache.info.next_cycle = 7;
|
||||
cache
|
||||
.save_with_revisions_for_epoch(set.clone(), cache_name, &revisions, epoch, CancellationToken::new())
|
||||
.await
|
||||
.expect("background primary and backup save");
|
||||
{
|
||||
let counts = counts.lock().expect("background counters");
|
||||
assert_eq!(counts.len(), 4, "every local disk must receive background cache I/O");
|
||||
assert!(counts.values().all(|counter| counter.load(Ordering::Relaxed) > 0));
|
||||
}
|
||||
for name in [cache_name.to_string(), format!("{cache_name}.bkp")] {
|
||||
let body = crate::read_config(set.clone(), &path_join_buf(&[crate::BUCKET_META_PREFIX, &name]))
|
||||
.await
|
||||
.expect("persisted cache copy");
|
||||
let saved: DataUsageCache = rmp_serde::from_slice(&body).expect("decode persisted cache");
|
||||
assert_eq!(saved.info.name, cache.info.name);
|
||||
assert_eq!(saved.info.next_cycle, 7);
|
||||
}
|
||||
}
|
||||
|
||||
async fn open_local_scanner_store(
|
||||
root: &std::path::Path,
|
||||
pool_count: usize,
|
||||
drives_per_set: usize,
|
||||
process_boot: bool,
|
||||
) -> Arc<ECStore> {
|
||||
init_ecstore_config_for_scanner_tests();
|
||||
let mut pools = Vec::new();
|
||||
for pool_index in 0..pool_count {
|
||||
let mut endpoints = Vec::new();
|
||||
for disk_index in 0..drives_per_set {
|
||||
let disk_path = temp_dir.path().join(format!("pool{pool_index}-disk{disk_index}"));
|
||||
let disk_path = root.join(format!("pool{pool_index}-disk{disk_index}"));
|
||||
tokio::fs::create_dir_all(&disk_path)
|
||||
.await
|
||||
.expect("multi-pool scanner test disk should be created");
|
||||
@@ -573,6 +664,16 @@ pub(crate) async fn setup_local_scanner_store(pool_count: usize, drives_per_set:
|
||||
}
|
||||
|
||||
let endpoint_pools = EndpointServerPools::from(pools);
|
||||
if process_boot {
|
||||
crate::storage_api::ecstore_init_local_disks(endpoint_pools.clone())
|
||||
.await
|
||||
.expect("process disk registry");
|
||||
let store = ECStore::new("127.0.0.1:0".parse().expect("test address"), endpoint_pools, CancellationToken::new())
|
||||
.await
|
||||
.expect("process scanner store");
|
||||
init_bucket_metadata_sys_for_scanner_tests(store.clone()).await;
|
||||
return store;
|
||||
}
|
||||
let instance_ctx = Arc::new(InstanceContext::new());
|
||||
instance_ctx.set_endpoints(endpoint_pools.clone());
|
||||
init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone())
|
||||
@@ -601,7 +702,7 @@ pub(crate) async fn setup_local_scanner_store(pool_count: usize, drives_per_set:
|
||||
.expect("scanner fixture must expose its live metadata owner");
|
||||
assert!(Arc::ptr_eq(&owner, &store), "scanner watch metadata owner must be this fixture");
|
||||
|
||||
(temp_dir, store)
|
||||
store
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -2494,6 +2595,7 @@ async fn set_snapshot_reuse_requires_execution_identity_and_fences_stale_writers
|
||||
digest: DataUsageScanPlanDigest([6; 32]),
|
||||
bucket_coverage_digest: DataUsageScanPlanDigest([6; 32]),
|
||||
checkpoint_coverage_digest: DataUsageScanPlanDigest([6; 32]),
|
||||
completed_checkpoint_cycle_floor: None,
|
||||
requires_full_scan: false,
|
||||
execution_digest: empty_execution,
|
||||
leader_epoch: 11,
|
||||
|
||||
@@ -14,6 +14,143 @@ use super::*;
|
||||
const HOT_OBJECTS: [&str; 6] = ["p0/a", "p1/b", "p2/c", "p3/d", "p4/e", "p5/f"];
|
||||
const LEADER_EPOCH: u64 = 11;
|
||||
|
||||
#[test]
|
||||
fn completed_checkpoint_reuse_requires_unpublished_unchanged_coverage() {
|
||||
let source = DataUsageCacheSource::new(0, 0);
|
||||
let plan = DataUsageScanPlanDigest([1; 32]);
|
||||
let coverage = DataUsageScanPlanDigest([2; 32]);
|
||||
let identity = crate::DataUsageScanIdentity {
|
||||
version: 1,
|
||||
bucket_incarnation: Uuid::new_v4(),
|
||||
set_layout: DataUsageScanPlanDigest([3; 32]),
|
||||
publication_epoch: 1,
|
||||
tier_registry_generation: 2,
|
||||
scan_mode: HealScanMode::Normal,
|
||||
};
|
||||
let mut saved = DataUsageCache {
|
||||
info: DataUsageCacheInfo {
|
||||
name: "completed".to_string(),
|
||||
next_cycle: 8,
|
||||
leader_epoch: 11,
|
||||
last_update: Some(SystemTime::now()),
|
||||
source: Some(source),
|
||||
snapshot_complete: true,
|
||||
scan_plan_digest: Some(plan),
|
||||
scan_coverage_digest: Some(coverage),
|
||||
scan_identity: Some(identity),
|
||||
tier_registry_generation: Some(2),
|
||||
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
saved.replace(
|
||||
"completed",
|
||||
"",
|
||||
DataUsageEntry {
|
||||
objects: 3,
|
||||
size: 48,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
for case in [
|
||||
"unpublished",
|
||||
"published",
|
||||
"missing_floor",
|
||||
"restart",
|
||||
"mutation",
|
||||
"legacy",
|
||||
"newer_cycle",
|
||||
"newer_leader",
|
||||
"same_leader",
|
||||
"tier",
|
||||
"incarnation",
|
||||
"source",
|
||||
"deep",
|
||||
"corrupt",
|
||||
"partial",
|
||||
] {
|
||||
let mut cache = saved.clone();
|
||||
let mut options = DataUsageCacheReuseOptions {
|
||||
require_source: true,
|
||||
tier_registry_generation: Some(2),
|
||||
checkpoint_identity: Some(identity),
|
||||
checkpoint_plan_digest: Some(coverage),
|
||||
completed_checkpoint_cycle_floor: Some(7),
|
||||
};
|
||||
match case {
|
||||
"published" => options.completed_checkpoint_cycle_floor = Some(8),
|
||||
"missing_floor" => options.completed_checkpoint_cycle_floor = None,
|
||||
"restart" => cache.info.scan_plan_digest = Some(DataUsageScanPlanDigest([4; 32])),
|
||||
"mutation" => cache.info.scan_coverage_digest = Some(DataUsageScanPlanDigest([4; 32])),
|
||||
"legacy" => cache.info.scan_coverage_digest = None,
|
||||
"newer_cycle" => cache.info.next_cycle = 10,
|
||||
"newer_leader" => cache.info.leader_epoch = 13,
|
||||
"same_leader" => cache.info.leader_epoch = 12,
|
||||
"tier" => options.tier_registry_generation = Some(3),
|
||||
"incarnation" => options.checkpoint_identity.as_mut().expect("identity").bucket_incarnation = Uuid::new_v4(),
|
||||
"source" => cache.info.source = Some(DataUsageCacheSource::new(1, 0)),
|
||||
"deep" => options.checkpoint_identity.as_mut().expect("identity").scan_mode = HealScanMode::Deep,
|
||||
"corrupt" => cache.cache.clear(),
|
||||
"partial" => cache.info.scan_resume_after = Some("completed/a".to_string()),
|
||||
"unpublished" => {}
|
||||
_ => unreachable!("all fixture cases are handled"),
|
||||
}
|
||||
let state = current_cache_root_or_prepare_with_generation(&mut cache, "completed", source, 9, 12, plan, options);
|
||||
assert_eq!(matches!(state, DataUsageCacheScanState::Current(_)), case == "unpublished", "{case}");
|
||||
if case == "unpublished" {
|
||||
assert_eq!(cache.info.next_cycle, 8, "adoption must not extend the saved receipt's lifetime");
|
||||
let DataUsageCacheScanState::Current(root) = state else { unreachable!() };
|
||||
assert_eq!(root.entry.objects, 3);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn completed_checkpoint_floor_requires_an_authoritative_baseline() {
|
||||
let source = DataUsageCacheSource::new(0, 0);
|
||||
let sources = HashSet::from([source]);
|
||||
let baseline: serde_json::Value =
|
||||
serde_json::from_slice(&complete_usage_baseline(source, DataUsageScanPlanDigest([1; 32]), 7, 11))
|
||||
.expect("complete baseline");
|
||||
for case in [
|
||||
"complete",
|
||||
"bootstrap",
|
||||
"observational",
|
||||
"partial",
|
||||
"incomplete",
|
||||
"legacy",
|
||||
"no_cycle",
|
||||
] {
|
||||
let mut value = baseline.clone();
|
||||
match case {
|
||||
"bootstrap" => {
|
||||
value = serde_json::to_value(crate::scanner::scanner_usage_bootstrap_marker(SystemTime::now(), Some(11)))
|
||||
.expect("bootstrap marker");
|
||||
}
|
||||
"observational" => value["usage_snapshot_observational"] = true.into(),
|
||||
"partial" => value["usage_snapshot_partial"] = true.into(),
|
||||
"incomplete" => value["usage_snapshot_complete"] = false.into(),
|
||||
"legacy" => value["scanner_epoch"] = serde_json::Value::Null,
|
||||
"no_cycle" => value["scanner_cycle"] = serde_json::Value::Null,
|
||||
"complete" => {}
|
||||
_ => unreachable!("all fixture cases are handled"),
|
||||
}
|
||||
let bytes = serde_json::to_vec(&value).expect("baseline fixture");
|
||||
assert_eq!(
|
||||
completed_checkpoint_cycle_floor(Some(&bytes), &sources, 9, 12),
|
||||
match case {
|
||||
"complete" => Some(7),
|
||||
"bootstrap" => Some(0),
|
||||
_ => None,
|
||||
},
|
||||
"{case}"
|
||||
);
|
||||
}
|
||||
assert_eq!(completed_checkpoint_cycle_floor(None, &sources, 9, 12), None);
|
||||
assert_eq!(completed_checkpoint_cycle_floor(Some(b"invalid"), &sources, 9, 12), None);
|
||||
}
|
||||
|
||||
async fn put_fixture_object(store: &Arc<ECStore>, bucket: &str, object: &str) {
|
||||
let mut reader = ScannerPutObjReader::from_vec(vec![7; 16]);
|
||||
store
|
||||
@@ -27,6 +164,18 @@ async fn run_full_cycle(
|
||||
cycle: u64,
|
||||
max_objects: Option<u64>,
|
||||
baseline: Option<&DataUsageInfo>,
|
||||
) -> (ScannerCycleResult, Option<DataUsageInfo>, Arc<ScannerCycleBudget>) {
|
||||
let baseline = baseline
|
||||
.map(|snapshot| Bytes::from(serde_json::to_vec(snapshot).expect("the actual candidate must encode as a baseline")));
|
||||
run_full_cycle_at_epoch(store, cycle, LEADER_EPOCH, max_objects, baseline).await
|
||||
}
|
||||
|
||||
async fn run_full_cycle_at_epoch(
|
||||
store: &Arc<ECStore>,
|
||||
cycle: u64,
|
||||
leader_epoch: u64,
|
||||
max_objects: Option<u64>,
|
||||
persisted_usage_baseline: Option<Bytes>,
|
||||
) -> (ScannerCycleResult, Option<DataUsageInfo>, Arc<ScannerCycleBudget>) {
|
||||
let ctx = CancellationToken::new();
|
||||
let budget = ScannerCycleBudget::new_with_progress_tracking(
|
||||
@@ -49,12 +198,10 @@ async fn run_full_cycle(
|
||||
budget: Arc::clone(&budget),
|
||||
updates,
|
||||
want_cycle: cycle,
|
||||
leader_epoch: LEADER_EPOCH,
|
||||
leader_epoch,
|
||||
scan_mode: HealScanMode::Normal,
|
||||
scan_scope: ScannerBucketScanScope::default(),
|
||||
persisted_usage_baseline: baseline.map(|snapshot| {
|
||||
Bytes::from(serde_json::to_vec(snapshot).expect("the actual candidate must encode as a baseline"))
|
||||
}),
|
||||
persisted_usage_baseline,
|
||||
observed_usage_candidate: None,
|
||||
requires_full_scan: true,
|
||||
service_cohort: None,
|
||||
@@ -73,6 +220,265 @@ async fn run_full_cycle(
|
||||
(result, snapshot, budget)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn checkpoint_small_budget_accumulates_completed_buckets_across_leaders() {
|
||||
let (_temp_dir, store) = setup_local_scanner_store(1, 1).await;
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
let names = ["bounded-a", "bounded-b", "bounded-c", "bounded-d"];
|
||||
for bucket in names {
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bounded fixture bucket");
|
||||
for object in ["a", "b", "c"] {
|
||||
put_fixture_object(&store, bucket, object).await;
|
||||
}
|
||||
record_dirty_usage_bucket(bucket);
|
||||
}
|
||||
wait_for_namespace_commit_tails(store.as_ref()).await;
|
||||
let baseline = Bytes::from(
|
||||
serde_json::to_vec(&crate::scanner::scanner_usage_bootstrap_marker(SystemTime::now(), Some(LEADER_EPOCH)))
|
||||
.expect("bootstrap baseline"),
|
||||
);
|
||||
let mut observations = Vec::new();
|
||||
for cycle in 1..=20 {
|
||||
let (result, usage, budget) =
|
||||
run_full_cycle_at_epoch(&store, cycle, LEADER_EPOCH + cycle, Some(2), Some(baseline.clone())).await;
|
||||
observations.push((cycle, result.status, budget.progress()));
|
||||
if result.status == ScannerCycleStatus::Complete {
|
||||
let usage = usage.expect("completed bounded sweep publishes usage");
|
||||
assert_eq!(usage.objects_total_count, 12);
|
||||
assert_eq!(usage.usage_snapshot_converged, Some(true));
|
||||
for bucket in names {
|
||||
assert_eq!(usage.buckets_usage[bucket].objects_count, 3);
|
||||
}
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
return;
|
||||
}
|
||||
}
|
||||
panic!("fixed budget must accumulate the completed buckets: {observations:?}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn checkpoint_small_budget_accumulates_across_process_restarts() {
|
||||
const ROOT_ENV: &str = "RUSTFS_CHECKPOINT_RESTART_ROOT";
|
||||
const CYCLE_ENV: &str = "RUSTFS_CHECKPOINT_RESTART_CYCLE";
|
||||
const TEST_NAME: &str =
|
||||
"scanner_io::tests::checkpoint_coverage_identity::checkpoint_small_budget_accumulates_across_process_restarts";
|
||||
let names = ["restart-a", "restart-b", "restart-c", "restart-d"];
|
||||
if let Some(root) = std::env::var_os(ROOT_ENV) {
|
||||
let root = std::path::PathBuf::from(root);
|
||||
let cycle: u64 = std::env::var(CYCLE_ENV)
|
||||
.expect("worker cycle")
|
||||
.parse()
|
||||
.expect("numeric cycle");
|
||||
let store = open_local_scanner_store(&root, 1, 1, true).await;
|
||||
let registry = scanner_set_disk_inventory(&store.pools[0].disk_set[0])
|
||||
.await
|
||||
.into_iter()
|
||||
.map(Some)
|
||||
.collect::<Vec<_>>();
|
||||
let previous = rustfs_heal::heal::consume_read_only_scanner_handoff("test-node", ®istry)
|
||||
.await
|
||||
.expect("consume previous receipt");
|
||||
if let Some(previous) = previous {
|
||||
assert!(crate::restore_scanner_checkpoint_instance_id(&previous));
|
||||
}
|
||||
if cycle == 0 {
|
||||
for bucket in names {
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("fixture bucket");
|
||||
for object in ["a", "b", "c"] {
|
||||
put_fixture_object(&store, bucket, object).await;
|
||||
}
|
||||
}
|
||||
wait_for_namespace_commit_tails(store.as_ref()).await;
|
||||
rustfs_heal::heal::clear_unclean_shutdown_markers().await;
|
||||
return;
|
||||
}
|
||||
let mutation_path = root.join("mutate.json");
|
||||
let mutated = root.join("mutated");
|
||||
if mutation_path.exists() && !mutated.exists() {
|
||||
let mutation: serde_json::Value =
|
||||
serde_json::from_slice(&tokio::fs::read(&mutation_path).await.expect("mutation request")).expect("mutation JSON");
|
||||
let bucket = mutation["bucket"].as_str().expect("mutation bucket");
|
||||
assert!(names.contains(&bucket));
|
||||
put_fixture_object(&store, bucket, "after-checkpoint").await;
|
||||
wait_for_namespace_commit_tails(store.as_ref()).await;
|
||||
assert!(store.scanner_namespace_mutation_generation() > 0 || !crate::scanner_process_was_read_only());
|
||||
if mutation["clean"] == true {
|
||||
rustfs_heal::heal::clear_unclean_shutdown_markers().await;
|
||||
}
|
||||
tokio::fs::write(&mutated, b"committed").await.expect("mutation receipt");
|
||||
let report = serde_json::json!({"instance": scanner_activity_epoch(), "mutated": true});
|
||||
tokio::fs::write(
|
||||
root.join(format!("round-{cycle}.json")),
|
||||
serde_json::to_vec(&report).expect("writer report"),
|
||||
)
|
||||
.await
|
||||
.expect("writer report file");
|
||||
return;
|
||||
}
|
||||
let baseline = if root.join("published.json").exists() {
|
||||
Bytes::from(tokio::fs::read(root.join("published.json")).await.expect("published floor"))
|
||||
} else {
|
||||
Bytes::from(
|
||||
serde_json::to_vec(&crate::scanner::scanner_usage_bootstrap_marker(SystemTime::now(), Some(LEADER_EPOCH)))
|
||||
.expect("bootstrap baseline"),
|
||||
)
|
||||
};
|
||||
let (result, usage, budget) = run_full_cycle_at_epoch(&store, cycle, LEADER_EPOCH + cycle, Some(2), Some(baseline)).await;
|
||||
let complete = result.status == ScannerCycleStatus::Complete;
|
||||
if complete {
|
||||
let usage = usage.expect("complete sweep must publish");
|
||||
let mutation: Option<serde_json::Value> = if mutated.exists() {
|
||||
Some(
|
||||
serde_json::from_slice(&tokio::fs::read(&mutation_path).await.expect("mutation request"))
|
||||
.expect("mutation JSON"),
|
||||
)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
assert_eq!(usage.objects_total_count, if mutation.is_some() { 13 } else { 12 });
|
||||
assert_eq!(usage.usage_snapshot_converged, Some(true));
|
||||
for bucket in names {
|
||||
let expected = if mutation.as_ref().is_some_and(|mutation| mutation["bucket"] == bucket) {
|
||||
4
|
||||
} else {
|
||||
3
|
||||
};
|
||||
assert_eq!(usage.buckets_usage[bucket].objects_count, expected, "{bucket}");
|
||||
}
|
||||
tokio::fs::write(root.join("published.json"), serde_json::to_vec(&usage).expect("complete usage"))
|
||||
.await
|
||||
.expect("published baseline");
|
||||
}
|
||||
let mut completed_buckets = Vec::new();
|
||||
for bucket in names {
|
||||
let mut cache = DataUsageCache::default();
|
||||
if cache
|
||||
.load(store.pools[0].disk_set[0].clone(), &path_join_buf(&[bucket, DATA_USAGE_CACHE_NAME]))
|
||||
.await
|
||||
.is_ok()
|
||||
&& cache.info.snapshot_complete
|
||||
&& !cache.has_scan_position()
|
||||
{
|
||||
completed_buckets.push(bucket);
|
||||
}
|
||||
}
|
||||
assert_eq!(
|
||||
store.scanner_namespace_mutation_generation(),
|
||||
0,
|
||||
"scanner must not mint namespace mutations"
|
||||
);
|
||||
assert!(
|
||||
crate::scanner_process_was_read_only(),
|
||||
"a scanner-only process must be eligible for a read-only handoff"
|
||||
);
|
||||
rustfs_heal::heal::save_read_only_scanner_handoff("test-node", crate::scanner_checkpoint_instance_id())
|
||||
.await
|
||||
.expect("persist read-only handoff");
|
||||
let report = serde_json::json!({
|
||||
"instance": scanner_activity_epoch(), "checkpoint_instance": crate::scanner_checkpoint_instance_id(),
|
||||
"complete": complete, "objects_processed": budget.progress().0, "completed_buckets": completed_buckets,
|
||||
});
|
||||
tokio::fs::write(
|
||||
root.join(format!("round-{cycle}.json")),
|
||||
serde_json::to_vec(&report).expect("worker report"),
|
||||
)
|
||||
.await
|
||||
.expect("worker report file");
|
||||
return;
|
||||
}
|
||||
for scenario in ["quiet", "clean-writer", "crashed-writer"] {
|
||||
let root = tempfile::tempdir().expect("restart fixture root");
|
||||
let mut instances = HashSet::new();
|
||||
let mut observations = Vec::new();
|
||||
let mut injected = false;
|
||||
let mut complete = false;
|
||||
let mut before_mutation = None;
|
||||
let mut expect_fresh = false;
|
||||
for cycle in 0..=40 {
|
||||
let log_path = root.path().join(format!("round-{cycle}.log"));
|
||||
let output = std::fs::File::create(&log_path).expect("worker log");
|
||||
let mut child = std::process::Command::new(std::env::current_exe().expect("test executable"))
|
||||
.args(["--exact", TEST_NAME, "--nocapture"])
|
||||
.env(ROOT_ENV, root.path())
|
||||
.env(CYCLE_ENV, cycle.to_string())
|
||||
.env("RUST_MIN_STACK", "4194304")
|
||||
.stdout(output.try_clone().expect("worker stdout"))
|
||||
.stderr(output)
|
||||
.spawn()
|
||||
.expect("restart worker");
|
||||
let status = tokio::time::timeout(Duration::from_secs(75), async {
|
||||
loop {
|
||||
if let Some(status) = child.try_wait().expect("worker status") {
|
||||
break status;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(25)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.unwrap_or_else(|_| {
|
||||
child.kill().expect("stop expired worker");
|
||||
child.wait().expect("reap expired worker");
|
||||
panic!("{scenario} round {cycle} exceeded its deadline");
|
||||
});
|
||||
assert!(
|
||||
status.success(),
|
||||
"{scenario} round {cycle}: {}",
|
||||
std::fs::read_to_string(log_path).expect("worker diagnostics")
|
||||
);
|
||||
if cycle == 0 {
|
||||
continue;
|
||||
}
|
||||
let report: serde_json::Value = serde_json::from_slice(
|
||||
&tokio::fs::read(root.path().join(format!("round-{cycle}.json")))
|
||||
.await
|
||||
.expect("worker report file"),
|
||||
)
|
||||
.expect("worker report JSON");
|
||||
assert!(instances.insert(report["instance"].as_str().expect("process identity").to_owned()));
|
||||
if report["mutated"] == true {
|
||||
expect_fresh = true;
|
||||
continue;
|
||||
}
|
||||
if expect_fresh {
|
||||
assert_ne!(
|
||||
before_mutation.as_ref(),
|
||||
Some(&report["checkpoint_instance"]),
|
||||
"writer must invalidate continuity"
|
||||
);
|
||||
assert_eq!(report["complete"], false, "old completed roots must not hide the mutation");
|
||||
expect_fresh = false;
|
||||
}
|
||||
if scenario != "quiet" && !injected && !report["completed_buckets"].as_array().expect("completed buckets").is_empty()
|
||||
{
|
||||
assert_eq!(report["complete"], false, "mutation must precede global publication");
|
||||
let mutation = serde_json::json!({"bucket": report["completed_buckets"][0], "clean": scenario == "clean-writer"});
|
||||
tokio::fs::write(root.path().join("mutate.json"), serde_json::to_vec(&mutation).expect("mutation request"))
|
||||
.await
|
||||
.expect("mutation request file");
|
||||
before_mutation = Some(report["checkpoint_instance"].clone());
|
||||
injected = true;
|
||||
}
|
||||
if report["complete"] == true {
|
||||
assert!(cycle > 1);
|
||||
assert!(scenario == "quiet" || injected);
|
||||
complete = true;
|
||||
eprintln!("{scenario}: exact convergence in {cycle} bounded fresh processes");
|
||||
break;
|
||||
}
|
||||
observations.push(report);
|
||||
}
|
||||
assert!(complete, "{scenario}: fixed budget did not converge in 40 restarts: {observations:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn checkpoint_resumes_across_a_maintenance_publication() {
|
||||
@@ -179,13 +585,13 @@ async fn checkpoint_resumes_across_a_maintenance_publication() {
|
||||
.await
|
||||
.expect("the fixture activity should be observable");
|
||||
assert_ne!(
|
||||
crate::scanner::scanner_activity_structural_digest(&before),
|
||||
crate::scanner::scanner_activity_structural_digest(&after),
|
||||
crate::scanner::scanner_activity_structural_digest(&before, crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE),
|
||||
crate::scanner::scanner_activity_structural_digest(&after, crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE),
|
||||
"a maintenance publication invalidates reusable results"
|
||||
);
|
||||
assert_eq!(
|
||||
crate::scanner::scanner_activity_coverage_digest(&before),
|
||||
crate::scanner::scanner_activity_coverage_digest(&after),
|
||||
crate::scanner::scanner_activity_coverage_digest(&before, crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE),
|
||||
crate::scanner::scanner_activity_coverage_digest(&after, crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE),
|
||||
"a maintenance publication keeps the coverage plan"
|
||||
);
|
||||
|
||||
@@ -223,6 +629,31 @@ async fn checkpoint_resumes_across_a_maintenance_publication() {
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn remote_checkpoint_resumes_across_restart_and_maintenance_identity_changes() {
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
struct Envelope {
|
||||
payload: Vec<u8>,
|
||||
}
|
||||
#[derive(Debug, serde::Deserialize)]
|
||||
struct Progress {
|
||||
objects_scanned: u64,
|
||||
}
|
||||
#[derive(Debug, serde::Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
enum FrameResult {
|
||||
Progress,
|
||||
Complete(serde::de::IgnoredAny),
|
||||
Partial,
|
||||
NamespaceNotFound,
|
||||
CycleAhead(serde::de::IgnoredAny),
|
||||
Error(serde::de::IgnoredAny),
|
||||
}
|
||||
#[derive(Debug, serde::Deserialize)]
|
||||
struct Frame {
|
||||
progress: Progress,
|
||||
result: FrameResult,
|
||||
}
|
||||
#[derive(serde::Serialize)]
|
||||
struct WorkerRequest<'a> {
|
||||
version: u16,
|
||||
@@ -245,20 +676,24 @@ async fn remote_checkpoint_resumes_across_restart_and_maintenance_identity_chang
|
||||
move || resolved.upgrade()
|
||||
)));
|
||||
rustfs_credentials::set_global_rpc_secret("remote-checkpoint-test-secret".to_string()).expect("test RPC secret");
|
||||
let bucket = "remote-checkpoint";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("remote bucket");
|
||||
for object in HOT_OBJECTS {
|
||||
put_fixture_object(&store, bucket, object).await;
|
||||
}
|
||||
wait_for_namespace_commit_tails(store.as_ref()).await;
|
||||
let set = store.pools[0].disk_set[0].clone();
|
||||
let disk = scanner_set_disk_inventory(&set).await.pop().expect("local worker disk");
|
||||
let cache_name = path_join_buf(&[bucket, DATA_USAGE_CACHE_NAME]);
|
||||
|
||||
for (index, restart) in [true, false].into_iter().enumerate() {
|
||||
let bucket = if restart {
|
||||
"remote-checkpoint-restart"
|
||||
} else {
|
||||
"remote-checkpoint-maintenance"
|
||||
};
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("remote bucket");
|
||||
for object in HOT_OBJECTS {
|
||||
put_fixture_object(&store, bucket, object).await;
|
||||
}
|
||||
wait_for_namespace_commit_tails(store.as_ref()).await;
|
||||
let cache_name = path_join_buf(&[bucket, DATA_USAGE_CACHE_NAME]);
|
||||
let before = if restart {
|
||||
std::collections::BTreeMap::from([(
|
||||
"<local>".to_string(),
|
||||
@@ -269,36 +704,64 @@ async fn remote_checkpoint_resumes_across_restart_and_maintenance_identity_chang
|
||||
.await
|
||||
.expect("activity before maintenance")
|
||||
};
|
||||
let coverage = DataUsageScanPlanDigest(crate::scanner::scanner_activity_coverage_digest(&before));
|
||||
let reuse = DataUsageScanPlanDigest(crate::scanner::scanner_activity_structural_digest(&before));
|
||||
let cycle = 100 + u64::try_from(index).expect("fixture index") * 2;
|
||||
for resumed in [false, true] {
|
||||
let coverage = DataUsageScanPlanDigest(crate::scanner::scanner_activity_coverage_digest(
|
||||
&before,
|
||||
crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE,
|
||||
));
|
||||
let reuse = DataUsageScanPlanDigest(crate::scanner::scanner_activity_structural_digest(
|
||||
&before,
|
||||
crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE,
|
||||
));
|
||||
let cycle = 100 + u64::try_from(index).expect("fixture index") * 4;
|
||||
for step in 0..4_u64 {
|
||||
let resumed = step > 0;
|
||||
let activity = if resumed && restart {
|
||||
std::collections::BTreeMap::from([(
|
||||
"<local>".to_string(),
|
||||
crate::scanner::scanner_node_activity_for_tests("instance-after", 0, 0, false),
|
||||
)])
|
||||
} else if resumed {
|
||||
record_scanner_maintenance_change(bucket);
|
||||
if step == 1 {
|
||||
record_scanner_maintenance_change(bucket);
|
||||
}
|
||||
crate::scanner::probe_scanner_activity(store.as_ref(), false)
|
||||
.await
|
||||
.expect("activity after maintenance")
|
||||
} else {
|
||||
before.clone()
|
||||
};
|
||||
let requested_reuse = DataUsageScanPlanDigest(crate::scanner::scanner_activity_structural_digest(&activity));
|
||||
let requested_reuse = DataUsageScanPlanDigest(crate::scanner::scanner_activity_structural_digest(
|
||||
&activity,
|
||||
crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE,
|
||||
));
|
||||
assert_eq!(
|
||||
coverage,
|
||||
DataUsageScanPlanDigest(crate::scanner::scanner_activity_coverage_digest(&activity))
|
||||
DataUsageScanPlanDigest(crate::scanner::scanner_activity_coverage_digest(
|
||||
&activity,
|
||||
crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE
|
||||
))
|
||||
);
|
||||
if resumed {
|
||||
assert_ne!(reuse, requested_reuse, "the event must invalidate completed cache reuse");
|
||||
}
|
||||
let next_cycle = cycle + u64::from(resumed);
|
||||
let next_cycle = cycle + step;
|
||||
let leader_epoch = LEADER_EPOCH + next_cycle;
|
||||
if step >= 2 {
|
||||
let baseline = if step == 2 {
|
||||
serde_json::to_vec(&crate::scanner::scanner_usage_bootstrap_marker(SystemTime::now(), Some(leader_epoch)))
|
||||
.expect("bootstrap baseline")
|
||||
} else {
|
||||
complete_usage_baseline(DataUsageCacheSource::new(0, 0), requested_reuse, cycle + 1, leader_epoch - 2)
|
||||
.to_vec()
|
||||
};
|
||||
save_config(store.clone(), crate::DATA_USAGE_OBJ_NAME_PATH.as_str(), baseline)
|
||||
.await
|
||||
.expect("authoritative aggregate floor");
|
||||
}
|
||||
save_config(
|
||||
store.clone(),
|
||||
DATA_USAGE_BLOOM_NAME_PATH.as_str(),
|
||||
crate::scanner::encode_scanner_cycle_fence_for_test(next_cycle, LEADER_EPOCH),
|
||||
crate::scanner::encode_scanner_cycle_fence_for_test(next_cycle, leader_epoch),
|
||||
)
|
||||
.await
|
||||
.expect("remote request cycle fence");
|
||||
@@ -310,28 +773,39 @@ async fn remote_checkpoint_resumes_across_restart_and_maintenance_identity_chang
|
||||
session_sequence: 0,
|
||||
bucket,
|
||||
next_cycle,
|
||||
leader_epoch: LEADER_EPOCH,
|
||||
leader_epoch,
|
||||
scan_plan_digest: requested_reuse,
|
||||
checkpoint_plan_digest: coverage,
|
||||
skip_healing: true,
|
||||
scan_mode: HealScanMode::Normal,
|
||||
budget: serde_json::json!({"max_objects": if resumed { None } else { Some(2_u64) }}),
|
||||
budget: serde_json::json!({"max_objects": if step == 1 { None } else { Some(2_u64) }}),
|
||||
};
|
||||
let body = rmp_serde::to_vec_named(&body).expect("remote worker request body");
|
||||
let request = crate::decode_remote_scanner_request(&body).expect("remote worker decodes coverage identity");
|
||||
let mut response = Vec::new();
|
||||
tokio::time::timeout(
|
||||
Duration::from_secs(60),
|
||||
crate::serve_remote_scanner_request(disk.clone(), request, tokio::io::sink(), CancellationToken::new()),
|
||||
crate::serve_remote_scanner_request(disk.clone(), request, &mut response, CancellationToken::new()),
|
||||
)
|
||||
.await
|
||||
.expect("bounded remote worker")
|
||||
.expect("remote worker stream");
|
||||
let mut frames = response.as_slice();
|
||||
let mut terminal: Option<Frame> = None;
|
||||
while !frames.is_empty() {
|
||||
let length = usize::try_from(frames.read_u32().await.expect("frame length")).expect("frame size");
|
||||
let envelope: Envelope = rmp_serde::from_slice(&frames[..length]).expect("response envelope");
|
||||
terminal = Some(rmp_serde::from_slice(&envelope.payload).expect("response frame"));
|
||||
frames = &frames[length..];
|
||||
}
|
||||
let terminal = terminal.expect("terminal worker response");
|
||||
let mut cache = DataUsageCache::default();
|
||||
cache
|
||||
.load(set.clone(), &cache_name)
|
||||
.await
|
||||
.expect("durable remote worker cache");
|
||||
if resumed {
|
||||
if step == 1 || step == 2 {
|
||||
assert!(matches!(terminal.result, FrameResult::Complete(_)), "step {step}: {terminal:?}");
|
||||
assert!(
|
||||
cache.info.snapshot_complete,
|
||||
"a resumed remote sweep must complete without mixed coverage"
|
||||
@@ -339,7 +813,12 @@ async fn remote_checkpoint_resumes_across_restart_and_maintenance_identity_chang
|
||||
assert!(cache.info.scan_progress.is_none());
|
||||
assert_eq!(cache.info.scan_plan_digest, Some(requested_reuse));
|
||||
assert_eq!(cache.flatten(cache.find(bucket).expect("bucket root")).objects, 6);
|
||||
if step == 2 {
|
||||
assert_eq!(terminal.progress.objects_scanned, 0);
|
||||
assert_eq!(cache.info.next_cycle, cycle + 1, "reuse must not renew the completed receipt");
|
||||
}
|
||||
} else {
|
||||
assert!(matches!(terminal.result, FrameResult::Partial), "step {step}: {terminal:?}");
|
||||
assert!(!cache.info.snapshot_complete);
|
||||
assert!(cache.has_scan_position(), "budget must leave real checkpoint progress");
|
||||
let progress = cache.info.scan_progress.expect("interrupted worker records coverage");
|
||||
|
||||
@@ -27,6 +27,7 @@ async fn checkpoint_plan(set: &Arc<SetDisks>, owner: &CancellationToken) -> Scan
|
||||
digest,
|
||||
bucket_coverage_digest: digest,
|
||||
checkpoint_coverage_digest: digest,
|
||||
completed_checkpoint_cycle_floor: None,
|
||||
requires_full_scan: true,
|
||||
service_cohort: None,
|
||||
execution_digest: digest,
|
||||
|
||||
@@ -130,6 +130,7 @@ async fn scan_plan(set: &Arc<SetDisks>, names: &[&str], cycle: u64) -> ScannerBu
|
||||
digest,
|
||||
bucket_coverage_digest: digest,
|
||||
checkpoint_coverage_digest: digest,
|
||||
completed_checkpoint_cycle_floor: None,
|
||||
requires_full_scan: true,
|
||||
service_cohort: Some(Arc::new(StdMutex::new(cohort))),
|
||||
execution_digest: digest,
|
||||
|
||||
@@ -15,6 +15,30 @@
|
||||
use super::*;
|
||||
use crate::data_usage_define::{DATA_USAGE_OBJ_NAME_PATH, read_config_with_revision};
|
||||
|
||||
// Two-pool EC fixtures keep many scan futures live at once and overflow
|
||||
// Linux libtest's default 2 MiB test-thread stack. Run each case on a
|
||||
// dedicated 32 MiB thread (same approach as scoped_entry_fallback).
|
||||
fn run_cohort_test<F, Fut>(thread_name: &'static str, test_fn: F)
|
||||
where
|
||||
F: FnOnce() -> Fut + Send + 'static,
|
||||
Fut: std::future::Future<Output = ()> + 'static,
|
||||
{
|
||||
let handle = std::thread::Builder::new()
|
||||
.name(thread_name.to_string())
|
||||
.stack_size(32 * 1024 * 1024)
|
||||
.spawn(move || {
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("service cohort test runtime should build");
|
||||
runtime.block_on(test_fn());
|
||||
})
|
||||
.expect("service cohort test thread should spawn");
|
||||
if let Err(payload) = handle.join() {
|
||||
std::panic::resume_unwind(payload);
|
||||
}
|
||||
}
|
||||
|
||||
async fn create_cohort_bucket(store: &ECStore, bucket: &str) {
|
||||
create_cohort_bucket_objects(store, bucket, 1).await;
|
||||
}
|
||||
@@ -104,9 +128,16 @@ async fn run_cohort_cycle(
|
||||
(result, usage)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[test]
|
||||
#[serial]
|
||||
async fn service_cohort_production_dispatch_services_waiters_across_sources() {
|
||||
fn service_cohort_production_dispatch_services_waiters_across_sources() {
|
||||
run_cohort_test(
|
||||
"cohort-prod-disp",
|
||||
service_cohort_production_dispatch_services_waiters_across_sources_case,
|
||||
);
|
||||
}
|
||||
|
||||
async fn service_cohort_production_dispatch_services_waiters_across_sources_case() {
|
||||
let (_dir, store) = setup_two_pool_scanner_store().await;
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
let hot = format!("a-hot-{}", Uuid::new_v4().simple());
|
||||
@@ -197,9 +228,16 @@ async fn service_cohort_production_dispatch_services_waiters_across_sources() {
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[test]
|
||||
#[serial]
|
||||
async fn service_cohort_flat_bucket_budget_does_not_publish_unscanned_small_bucket() {
|
||||
fn service_cohort_flat_bucket_budget_does_not_publish_unscanned_small_bucket() {
|
||||
run_cohort_test(
|
||||
"cohort-flat-bud",
|
||||
service_cohort_flat_bucket_budget_does_not_publish_unscanned_small_bucket_case,
|
||||
);
|
||||
}
|
||||
|
||||
async fn service_cohort_flat_bucket_budget_does_not_publish_unscanned_small_bucket_case() {
|
||||
let (_dir, store) = setup_two_pool_scanner_store().await;
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
let flat = format!("a-flat-{}", Uuid::new_v4().simple());
|
||||
@@ -283,9 +321,16 @@ async fn service_cohort_flat_bucket_budget_does_not_publish_unscanned_small_buck
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[test]
|
||||
#[serial]
|
||||
async fn service_cohort_fresh_complete_aggregate_preserves_reordered_sources() {
|
||||
fn service_cohort_fresh_complete_aggregate_preserves_reordered_sources() {
|
||||
run_cohort_test(
|
||||
"cohort-fresh-agg",
|
||||
service_cohort_fresh_complete_aggregate_preserves_reordered_sources_case,
|
||||
);
|
||||
}
|
||||
|
||||
async fn service_cohort_fresh_complete_aggregate_preserves_reordered_sources_case() {
|
||||
let (_dir, store) = setup_two_pool_scanner_store().await;
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
for bucket in ["cohort-first", "cohort-second"] {
|
||||
@@ -329,9 +374,16 @@ async fn service_cohort_fresh_complete_aggregate_preserves_reordered_sources() {
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[test]
|
||||
#[serial]
|
||||
async fn service_cohort_cancelled_dispatch_does_not_consume_waiters_or_leak_permits() {
|
||||
fn service_cohort_cancelled_dispatch_does_not_consume_waiters_or_leak_permits() {
|
||||
run_cohort_test(
|
||||
"cohort-cancelled",
|
||||
service_cohort_cancelled_dispatch_does_not_consume_waiters_or_leak_permits_case,
|
||||
);
|
||||
}
|
||||
|
||||
async fn service_cohort_cancelled_dispatch_does_not_consume_waiters_or_leak_permits_case() {
|
||||
let (_dir, store) = setup_two_pool_scanner_store().await;
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
let bucket = format!("cancel-{}", Uuid::new_v4().simple());
|
||||
@@ -360,9 +412,16 @@ async fn service_cohort_cancelled_dispatch_does_not_consume_waiters_or_leak_perm
|
||||
/// N=17, q=4: every bucket reaches a worker within ceil(17/4)=5 rounds while
|
||||
/// one bucket stays hot and another fails forever, with leader-local state
|
||||
/// discarded after each round as a restart would.
|
||||
#[tokio::test]
|
||||
#[test]
|
||||
#[serial]
|
||||
async fn cold_bucket_service_bound_survives_restart_with_hot_and_failing_bucket() {
|
||||
fn cold_bucket_service_bound_survives_restart_with_hot_and_failing_bucket() {
|
||||
run_cohort_test(
|
||||
"cold-svc-bound",
|
||||
cold_bucket_service_bound_survives_restart_with_hot_and_failing_bucket_case,
|
||||
);
|
||||
}
|
||||
|
||||
async fn cold_bucket_service_bound_survives_restart_with_hot_and_failing_bucket_case() {
|
||||
const BUCKETS: usize = 17;
|
||||
const QUANTA: u64 = 4;
|
||||
const ROUND_BOUND: usize = 5;
|
||||
@@ -434,9 +493,16 @@ fn quantum_budget(quantum: crate::scanner_budget::ScannerBucketQuantum) -> Arc<S
|
||||
)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[test]
|
||||
#[serial]
|
||||
async fn source_turn_dispatch_rotates_real_sources_and_retains_completed_roots() {
|
||||
fn source_turn_dispatch_rotates_real_sources_and_retains_completed_roots() {
|
||||
run_cohort_test(
|
||||
"src-turn-disp",
|
||||
source_turn_dispatch_rotates_real_sources_and_retains_completed_roots_case,
|
||||
);
|
||||
}
|
||||
|
||||
async fn source_turn_dispatch_rotates_real_sources_and_retains_completed_roots_case() {
|
||||
let mut evidence = None;
|
||||
temp_env::async_with_vars([(ENV_SCANNER_MAX_CONCURRENT_SET_SCANS, Some("1"))], async {
|
||||
crate::runtime_config::refresh_scanner_runtime_config_for_tests();
|
||||
@@ -503,9 +569,16 @@ async fn source_turn_dispatch_rotates_real_sources_and_retains_completed_roots()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[test]
|
||||
#[serial]
|
||||
async fn source_turn_zero_new_frontier_stops_requeuing_without_starving_other_sources() {
|
||||
fn source_turn_zero_new_frontier_stops_requeuing_without_starving_other_sources() {
|
||||
run_cohort_test(
|
||||
"src-turn-zero",
|
||||
source_turn_zero_new_frontier_stops_requeuing_without_starving_other_sources_case,
|
||||
);
|
||||
}
|
||||
|
||||
async fn source_turn_zero_new_frontier_stops_requeuing_without_starving_other_sources_case() {
|
||||
temp_env::async_with_vars([(ENV_SCANNER_MAX_CONCURRENT_SET_SCANS, Some("1"))], async {
|
||||
crate::runtime_config::refresh_scanner_runtime_config_for_tests();
|
||||
let (_dir, store) = setup_two_pool_scanner_store().await;
|
||||
@@ -533,9 +606,13 @@ async fn source_turn_zero_new_frontier_stops_requeuing_without_starving_other_so
|
||||
crate::runtime_config::refresh_scanner_runtime_config_for_tests();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[test]
|
||||
#[serial]
|
||||
async fn fair_prefix_quantum_spent_bucket_returns_behind_waiting_buckets() {
|
||||
fn fair_prefix_quantum_spent_bucket_returns_behind_waiting_buckets() {
|
||||
run_cohort_test("fair-spent-qm", fair_prefix_quantum_spent_bucket_returns_behind_waiting_buckets_case);
|
||||
}
|
||||
|
||||
async fn fair_prefix_quantum_spent_bucket_returns_behind_waiting_buckets_case() {
|
||||
let (_dir, store) = setup_local_scanner_store(1, 4).await;
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
create_cohort_bucket_objects(&store, "a-flat", 6).await;
|
||||
@@ -565,9 +642,13 @@ async fn fair_prefix_quantum_spent_bucket_returns_behind_waiting_buckets() {
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[test]
|
||||
#[serial]
|
||||
async fn fair_prefix_quantum_without_durable_motion_ends_the_bucket_turn() {
|
||||
fn fair_prefix_quantum_without_durable_motion_ends_the_bucket_turn() {
|
||||
run_cohort_test("fair-no-motion", fair_prefix_quantum_without_durable_motion_ends_the_bucket_turn_case);
|
||||
}
|
||||
|
||||
async fn fair_prefix_quantum_without_durable_motion_ends_the_bucket_turn_case() {
|
||||
let (_dir, store) = setup_local_scanner_store(1, 4).await;
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
create_cohort_bucket_objects(&store, "a-flat", 6).await;
|
||||
@@ -590,9 +671,16 @@ async fn fair_prefix_quantum_without_durable_motion_ends_the_bucket_turn() {
|
||||
|
||||
/// A one-object share stops inside its first child, so the frontier never
|
||||
/// moves. Requeuing that bucket would spin until the cycle deadline.
|
||||
#[tokio::test]
|
||||
#[test]
|
||||
#[serial]
|
||||
async fn fair_prefix_quantum_repeated_share_without_a_new_frontier_stops_requeuing() {
|
||||
fn fair_prefix_quantum_repeated_share_without_a_new_frontier_stops_requeuing() {
|
||||
run_cohort_test(
|
||||
"fair-rep-share",
|
||||
fair_prefix_quantum_repeated_share_without_a_new_frontier_stops_requeuing_case,
|
||||
);
|
||||
}
|
||||
|
||||
async fn fair_prefix_quantum_repeated_share_without_a_new_frontier_stops_requeuing_case() {
|
||||
let (_dir, store) = setup_local_scanner_store(1, 4).await;
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
create_cohort_bucket_objects(&store, "a-flat", 3).await;
|
||||
@@ -613,9 +701,16 @@ async fn fair_prefix_quantum_repeated_share_without_a_new_frontier_stops_requeui
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[test]
|
||||
#[serial]
|
||||
async fn cold_bucket_service_saved_order_keeps_coverage_and_rejects_older_owners() {
|
||||
fn cold_bucket_service_saved_order_keeps_coverage_and_rejects_older_owners() {
|
||||
run_cohort_test(
|
||||
"cold-saved-ord",
|
||||
cold_bucket_service_saved_order_keeps_coverage_and_rejects_older_owners_case,
|
||||
);
|
||||
}
|
||||
|
||||
async fn cold_bucket_service_saved_order_keeps_coverage_and_rejects_older_owners_case() {
|
||||
let (_dir, store) = setup_local_scanner_store(1, 4).await;
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
create_cohort_bucket(&store, "cold-only").await;
|
||||
|
||||
@@ -8,6 +8,22 @@ const EPOCH: u64 = 7;
|
||||
const RETIRED: &str = "retirement-generation";
|
||||
const LIVE: &str = "retirement-survivor";
|
||||
|
||||
async fn close_fixture(directory: tempfile::TempDir, store: Arc<ECStore>) {
|
||||
store.background_cancel_token().expect("fixture shutdown token").cancel();
|
||||
tokio::time::timeout(Duration::from_secs(30), async {
|
||||
for pool in &store.pools {
|
||||
pool.stop_endpoint_monitor_for_test().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("endpoint monitors must stop before removing fixture disks");
|
||||
wait_for_namespace_commit_tails(&store).await;
|
||||
drop(store);
|
||||
let path = directory.path().to_path_buf();
|
||||
directory.close().expect("remove source retirement fixture");
|
||||
assert!(!path.exists());
|
||||
}
|
||||
|
||||
async fn run_turn(
|
||||
set: &Arc<SetDisks>,
|
||||
buckets: &[BucketInfo],
|
||||
@@ -37,6 +53,7 @@ async fn run_turn(
|
||||
digest: DataUsageScanPlanDigest([31; 32]),
|
||||
bucket_coverage_digest: DataUsageScanPlanDigest([32; 32]),
|
||||
checkpoint_coverage_digest: DataUsageScanPlanDigest([33; 32]),
|
||||
completed_checkpoint_cycle_floor: None,
|
||||
requires_full_scan: true,
|
||||
execution_digest: DataUsageScanPlanDigest([34; 32]),
|
||||
leader_epoch: EPOCH,
|
||||
@@ -143,10 +160,7 @@ async fn assert_retired_source_turn_discards_only_deleted_generation_and_retains
|
||||
failures.partial.lock().await.contains(RETIRED),
|
||||
"responsibility remains until namespace replanning"
|
||||
);
|
||||
drop(store);
|
||||
let path = directory.path().to_path_buf();
|
||||
directory.close().expect("remove source retirement fixture");
|
||||
assert!(!path.exists());
|
||||
close_fixture(directory, store).await;
|
||||
}
|
||||
|
||||
async fn assert_recreated_source_turn_rescans_successor_without_old_value_or_identity() {
|
||||
@@ -188,10 +202,7 @@ async fn assert_recreated_source_turn_rescans_successor_without_old_value_or_ide
|
||||
assert_eq!((successor.objects, successor.size), (2, 28));
|
||||
assert_eq!(after.info.scan_bucket_incarnations[RETIRED], new_identity);
|
||||
assert_eq!(after.find(LIVE).expect("unrelated bucket").size, 13);
|
||||
drop(store);
|
||||
let path = directory.path().to_path_buf();
|
||||
directory.close().expect("remove source retirement fixture");
|
||||
assert!(!path.exists());
|
||||
close_fixture(directory, store).await;
|
||||
}
|
||||
|
||||
async fn assert_missing_retirement_proof_and_foreign_source_fences_cannot_clear_completion() {
|
||||
@@ -218,10 +229,7 @@ async fn assert_missing_retirement_proof_and_foreign_source_fences_cannot_clear_
|
||||
.expect_err("unconfirmed identity changes cannot retire completed work");
|
||||
assert!(error.to_string().contains("identity"), "{field}: {error}");
|
||||
}
|
||||
drop(store);
|
||||
let path = directory.path().to_path_buf();
|
||||
directory.close().expect("remove source retirement fixture");
|
||||
assert!(!path.exists());
|
||||
close_fixture(directory, store).await;
|
||||
}
|
||||
|
||||
async fn assert_missing_directories_and_unreadable_retirement_proof_remain_unconfirmed() {
|
||||
@@ -251,10 +259,7 @@ async fn assert_missing_directories_and_unreadable_retirement_proof_remain_uncon
|
||||
.await
|
||||
.expect_err("an unreadable retirement must preserve failed completion validation");
|
||||
assert!(error.to_string().contains("identity"));
|
||||
drop(store);
|
||||
let path = directory.path().to_path_buf();
|
||||
directory.close().expect("remove source retirement fixture");
|
||||
assert!(!path.exists());
|
||||
close_fixture(directory, store).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -168,8 +168,8 @@ fn watch_structure_ignores_unrelated_dirty_generations_but_requires_every_owner(
|
||||
unrelated.insert("peer".into(), crate::scanner::scanner_node_activity_for_tests("remote", 103, 104, false));
|
||||
assert_eq!(original, watch_structure_digest(&unrelated, &routes).unwrap());
|
||||
assert_ne!(
|
||||
crate::scanner::scanner_activity_coverage_digest(&before),
|
||||
crate::scanner::scanner_activity_coverage_digest(&unrelated)
|
||||
crate::scanner::scanner_activity_coverage_digest(&before, LOCAL_SCANNER_ACTIVITY_NODE),
|
||||
crate::scanner::scanner_activity_coverage_digest(&unrelated, LOCAL_SCANNER_ACTIVITY_NODE)
|
||||
);
|
||||
unrelated.remove("peer");
|
||||
assert!(watch_structure_digest(&unrelated, &routes).is_none());
|
||||
|
||||
@@ -170,6 +170,8 @@ pub(crate) use rustfs_ecstore::api::set_disk::SetDisks as EcstoreSetDisks;
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::storage::SCANNER_PUBLICATION_LEASE_TTL_MS as ECSTORE_SCANNER_PUBLICATION_LEASE_TTL_MS;
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::storage::init_local_disks as ecstore_init_local_disks;
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::storage::init_local_disks_with_instance_ctx as ecstore_init_local_disks_with_instance_ctx;
|
||||
pub(crate) use rustfs_ecstore::api::storage::{
|
||||
ECStore as EcstoreStore, ScannerDataMovementPauseStatus as EcstoreScannerDataMovementPauseStatus,
|
||||
@@ -410,9 +412,9 @@ pub(crate) mod scan {
|
||||
#[cfg(test)]
|
||||
pub(crate) use super::storage_contracts::{DeleteBucketOptions, MakeBucketOptions, ObjectIO, StorageAdminApi};
|
||||
pub use super::storage_contracts::{
|
||||
SCANNER_ACTIVITY_PROTOCOL_VERSION, SCANNER_ACTIVITY_V6_PROTOCOL_VERSION, SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES,
|
||||
SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION, SCANNER_DIRTY_USAGE_SNAPSHOT_RPC_MAX_MESSAGE_SIZE,
|
||||
SCANNER_SCOPED_DIRTY_USAGE_ACK_MAX_ENTRIES,
|
||||
SCANNER_ACTIVITY_PROTOCOL_VERSION, SCANNER_ACTIVITY_V6_PROTOCOL_VERSION, SCANNER_ACTIVITY_V7_PROTOCOL_VERSION,
|
||||
SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES, SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION,
|
||||
SCANNER_DIRTY_USAGE_SNAPSHOT_RPC_MAX_MESSAGE_SIZE, SCANNER_SCOPED_DIRTY_USAGE_ACK_MAX_ENTRIES,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use super::{
|
||||
@@ -422,8 +424,6 @@ pub(crate) mod scan {
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::acquire_scanner_bucket_incarnation_fence as ecstore_scanner_ack_fence;
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::disk::HEALING_MARKER_PATH;
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::notification::new_global_notification_sys;
|
||||
}
|
||||
|
||||
@@ -462,6 +462,9 @@ pub(crate) trait ScannerStorage:
|
||||
+ storage_contracts::NamespaceLocking<Error = EcstoreErrorType, NamespaceLock = NamespaceLockWrapper>
|
||||
{
|
||||
fn scanner_topology_digest(&self) -> [u8; 32];
|
||||
fn scanner_local_node_name(&self) -> Option<String> {
|
||||
None
|
||||
}
|
||||
fn scanner_namespace_mutation_generation(&self) -> u64;
|
||||
async fn scanner_data_movement_activity(&self) -> (bool, bool, u64);
|
||||
async fn scanner_data_usage_publication_blocked(&self) -> bool;
|
||||
@@ -514,6 +517,15 @@ impl ScannerStorage for EcstoreStore {
|
||||
crate::scanner::scanner_topology_digest(self)
|
||||
}
|
||||
|
||||
fn scanner_local_node_name(&self) -> Option<String> {
|
||||
self.endpoints()
|
||||
.0
|
||||
.iter()
|
||||
.flat_map(|pool| pool.endpoints.as_ref().iter())
|
||||
.find(|endpoint| endpoint.is_local)
|
||||
.map(|endpoint| endpoint.grid_host())
|
||||
}
|
||||
|
||||
fn scanner_namespace_mutation_generation(&self) -> u64 {
|
||||
EcstoreStore::scanner_namespace_mutation_generation(self)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
# SSE Crate Instructions
|
||||
|
||||
Applies to `crates/sse/`.
|
||||
|
||||
## Boundary
|
||||
|
||||
- This crate sits between the S3 layer and the storage engine and depends on neither. Its `rustfs-*` dependencies are pinned by `scripts/check_architecture_migration_rules.sh` to `rustfs-config`, `rustfs-kms`, `rustfs-policy`, `rustfs-s3-types`, and `rustfs-utils`; see [crate boundaries](../../docs/architecture/crate-boundaries.md).
|
||||
- Anything the crate needs from the running server arrives through `SseHost` (`src/host.rs`), passed as the first argument of every entry point that consults the server. Do not add a global that stands in for the host: a caller must not be able to reach key selection without naming the server it speaks for.
|
||||
- Translation to and from storage-engine types (`ObjectEncryptionResolver`, `WriteEncryption`, `MultipartCompletionAuthorization`) and request types belongs in `rustfs/src/storage/sse.rs`, not here.
|
||||
- Report failures as `SseError`, classified with the S3 error code the caller answers with. A failure that did not originate from the request goes through `SseError::internal`, which keeps its detail out of the response.
|
||||
|
||||
## Layout
|
||||
|
||||
- `src/startup.rs` settles the state that is fixed for the lifetime of the process and reports it. Nothing that follows the KMS service state belongs there: the service follows configuration and can be started, restarted, or stopped from the admin API, so every choice that depends on it is made per request.
|
||||
- `src/core/` holds the entry points (`write.rs`, `read.rs`, `rewrap.rs`) and the request principal (`request.rs`). An entry point only orders the steps; the decisions live in the two layers below it.
|
||||
- `src/core/s3/` is what S3 defines: the SSE headers of a request and a response, the scheme a write or a stored object resolves to, the SSE-C checks, and the authorization of a KMS key.
|
||||
- `src/core/keying/` is how RustFS forms, seals, and records an object's key. `direct.rs` and `object.rs` are the two modes, and each owns both directions of its stored record: the code that writes a key's metadata sits next to the code that reads it back. `mod.rs` is the only place that says which mode a build writes; `stored.rs` holds the order in which a read recognizes the modes and picks the provider that unwraps the data key; `metadata.rs` records the scheme and lists every stored key name.
|
||||
- `src/kms/` holds the data-key providers: they wrap and unwrap a data key and decide nothing about schemes. The write and rewrap entry points, `core/keying/stored.rs`, and the startup check of the local master key call them; `core/s3/` never does.
|
||||
|
||||
Rules that keep the layers apart:
|
||||
|
||||
- `core/keying/` never imports from `core/s3/`. `core/s3/` reads stored metadata only through `core/keying/metadata.rs`. Anything else that needs both layers is joined in an entry point.
|
||||
- `#[cfg(feature = "rio-v2")]` appears only under `core/keying/` and `kms/`. An entry point or `core/s3/` that needs a build-dependent answer asks `core/keying/` for it.
|
||||
- A new stored format, including a MinIO-compatible one, is added under `core/keying/` (and `kms/` when it brings its own wrapping) without changes under `core/s3/`: a mode file with its record and read functions, one entry in the recognition order of `stored.rs`, and its key names in the lists of `metadata.rs`.
|
||||
- A read decides from what the object recorded. The scheme a request advertises and whether a KMS is running now are write policy and runtime state; neither says how an existing object was keyed.
|
||||
|
||||
## Security
|
||||
|
||||
- Never log plaintext customer keys, decrypted key material, data-key envelopes, nonces, or SSE request payloads, including through error strings and `Debug` output.
|
||||
- KMS audit tags may carry only the fields documented on `KmsRequestAuditState`.
|
||||
- The `test-util` feature exposes authorization overrides and DEK-provider injection. It is a dev-dependency feature only; never enable it from a normal dependency.
|
||||
|
||||
## Change Coordination
|
||||
|
||||
When changing key resolution, metadata, or error classification, verify compatibility with:
|
||||
|
||||
- `rustfs/src/storage/sse.rs` (the host implementation and the engine-boundary adapters)
|
||||
- `crates/kms/`
|
||||
- `crates/utils/src/http/object_encryption_keys.rs` (the stored-metadata key vocabulary)
|
||||
- `rustfs/src/kms_rekey.rs` and `rustfs/src/admin/handlers/kms*.rs`
|
||||
|
||||
## Testing
|
||||
|
||||
Run both builds; the `rio-v2` feature swaps the object-key scheme and compiles different code paths:
|
||||
|
||||
```bash
|
||||
cargo nextest run -p rustfs-sse --no-tests fail
|
||||
cargo nextest run -p rustfs-sse --features rio-v2 --no-tests fail
|
||||
```
|
||||
|
||||
The adapter tests live with the adapter:
|
||||
|
||||
```bash
|
||||
cargo nextest run -p rustfs --no-tests fail -E 'test(/^storage::sse::/)'
|
||||
```
|
||||
|
||||
A change that should leave client-visible behavior or the stored form untouched (a refactor, a move between layers) is checked against a real server: run `scripts/test/sse_blackbox/` on the binary before and after the change, compare the two runs, and cross-read their data directories. Its README has the commands.
|
||||
@@ -0,0 +1,70 @@
|
||||
# Copyright 2024 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
[package]
|
||||
name = "rustfs-sse"
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
rust-version.workspace = true
|
||||
version.workspace = true
|
||||
homepage.workspace = true
|
||||
description = "S3 server-side encryption (SSE-S3, SSE-KMS, SSE-C) key resolution and metadata for RustFS."
|
||||
keywords = ["sse", "encryption", "s3", "kms", "rustfs"]
|
||||
categories = ["cryptography"]
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[features]
|
||||
default = []
|
||||
# MinIO-compatible sealed object keys (DARE). Mirrors the `rio-v2` feature of
|
||||
# the server binary, which enables both together.
|
||||
rio-v2 = ["dep:chacha20poly1305", "dep:hmac", "dep:sha2"]
|
||||
# Test seams for dependents: per-principal authorization overrides, DEK
|
||||
# provider injection and metadata fixtures. Never enable in a release build.
|
||||
test-util = []
|
||||
|
||||
[dependencies]
|
||||
# The only RustFS crates this one may reach. It sits between the S3 layer and
|
||||
# the storage engine and must not depend on either: what it needs from the
|
||||
# running server arrives through `SseHost`.
|
||||
rustfs-config = { workspace = true }
|
||||
rustfs-kms = { workspace = true }
|
||||
rustfs-policy = { workspace = true }
|
||||
rustfs-s3-types = { workspace = true }
|
||||
rustfs-utils = { workspace = true, features = ["http", "path"] }
|
||||
|
||||
aes-gcm = { workspace = true, features = ["rand_core", "zeroize"] }
|
||||
async-trait = { workspace = true }
|
||||
base64-simd = { workspace = true }
|
||||
chacha20poly1305 = { workspace = true, optional = true }
|
||||
hmac = { workspace = true, optional = true }
|
||||
http = { workspace = true }
|
||||
md-5 = { workspace = true }
|
||||
rand = { workspace = true }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
serde_json = { workspace = true }
|
||||
sha2 = { workspace = true, optional = true }
|
||||
tracing = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
# The fake AWS KMS endpoint the backend-selection tests talk to.
|
||||
axum = { workspace = true }
|
||||
temp-env = { workspace = true, features = ["async_closure"] }
|
||||
tempfile = { workspace = true }
|
||||
tokio = { workspace = true, features = ["macros", "net", "rt-multi-thread", "sync", "time"] }
|
||||
|
||||
[lib]
|
||||
doctest = false
|
||||
@@ -0,0 +1,292 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Per-request summary of data-plane KMS use, attached to the request's audit entry.
|
||||
|
||||
use crate::core::request::SseKmsPrincipal;
|
||||
use crate::error::SseError;
|
||||
use base64_simd::STANDARD as BASE64_STANDARD;
|
||||
use rustfs_kms::is_data_key_envelope;
|
||||
use rustfs_s3_types::S3ErrorCode;
|
||||
use rustfs_utils::http::object_encryption_keys::{INTERNAL_ENCRYPTION_KEY_HEADER, SSEType, normalize_managed_metadata};
|
||||
use serde_json::Value;
|
||||
use std::collections::{BTreeSet, HashMap};
|
||||
use std::sync::{Arc, LazyLock, Mutex, Weak};
|
||||
|
||||
/// Tag keys carrying the KMS summary on an S3 audit entry.
|
||||
///
|
||||
/// Audit consumers key off these strings, so they are a wire contract: append
|
||||
/// new keys rather than renaming existing ones.
|
||||
const KMS_AUDIT_TAG_SSE_TYPE: &str = "sseType";
|
||||
|
||||
const KMS_AUDIT_TAG_KEY_ID: &str = "kmsKeyId";
|
||||
|
||||
const KMS_AUDIT_TAG_KEY_VERSION: &str = "kmsKeyVersion";
|
||||
|
||||
const KMS_AUDIT_TAG_OUTCOME: &str = "kmsOutcome";
|
||||
|
||||
const KMS_AUDIT_TAG_ERROR_CLASS: &str = "kmsErrorClass";
|
||||
|
||||
/// Separator for the rare request that touched more than one key or scheme.
|
||||
const KMS_AUDIT_VALUE_SEPARATOR: &str = ",";
|
||||
|
||||
/// What the data path did with KMS while serving one S3 request.
|
||||
///
|
||||
/// This rides on the request's existing S3 audit entry instead of becoming its
|
||||
/// own event: the entry already carries the principal, request ID and API, so
|
||||
/// attaching the KMS outcome costs no extra event volume and needs no second
|
||||
/// correlation key.
|
||||
///
|
||||
/// # Redaction
|
||||
///
|
||||
/// Only the fields below may ever be recorded. The S3 audit entry fans out to
|
||||
/// every configured target, several of which sit outside the KMS trust
|
||||
/// boundary, so nothing derived from a data key — plaintext, ciphertext,
|
||||
/// envelope, nonce — nor any caller-supplied encryption-context value belongs
|
||||
/// here. Key identifiers and scheme names are configuration, not secrets.
|
||||
#[derive(Debug, Default)]
|
||||
struct KmsRequestAuditState {
|
||||
sse_types: BTreeSet<&'static str>,
|
||||
key_ids: BTreeSet<String>,
|
||||
key_versions: BTreeSet<u32>,
|
||||
/// Set by the first failure. A request that failed any KMS interaction is
|
||||
/// reported as failed even if others succeeded: a partially completed
|
||||
/// envelope operation is not a success for an audit reader.
|
||||
error_class: Option<&'static str>,
|
||||
recorded: bool,
|
||||
}
|
||||
|
||||
/// Shared accumulator for one request's KMS summary.
|
||||
///
|
||||
/// Written by the managed-SSE code paths through the request's principal, read
|
||||
/// once through [`KmsRequestAuditScope::audit_tags`] by whoever owns the
|
||||
/// request's audit entry.
|
||||
#[derive(Debug, Default)]
|
||||
pub(crate) struct KmsRequestAudit(Mutex<KmsRequestAuditState>);
|
||||
|
||||
impl KmsRequestAudit {
|
||||
pub(crate) fn record(
|
||||
&self,
|
||||
sse_type: SSEType,
|
||||
key_id: Option<&str>,
|
||||
key_version: Option<u32>,
|
||||
error_class: Option<&'static str>,
|
||||
) {
|
||||
// A poisoned lock costs the audit entry its KMS tags; it must never cost
|
||||
// the request its result.
|
||||
let Ok(mut state) = self.0.lock() else {
|
||||
return;
|
||||
};
|
||||
|
||||
state.recorded = true;
|
||||
state.sse_types.insert(sse_type.audit_label());
|
||||
if let Some(key_id) = key_id.filter(|value| !value.is_empty()) {
|
||||
state.key_ids.insert(key_id.to_string());
|
||||
}
|
||||
if let Some(key_version) = key_version {
|
||||
state.key_versions.insert(key_version);
|
||||
}
|
||||
if state.error_class.is_none() {
|
||||
state.error_class = error_class;
|
||||
}
|
||||
}
|
||||
|
||||
/// Render the summary as audit tags, or nothing when the request performed
|
||||
/// no KMS work at all (SSE-C and unencrypted objects never reach KMS).
|
||||
pub fn audit_tags(&self) -> Vec<(&'static str, Value)> {
|
||||
let Ok(state) = self.0.lock() else {
|
||||
return Vec::new();
|
||||
};
|
||||
if !state.recorded {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
let mut tags = Vec::with_capacity(5);
|
||||
tags.push((KMS_AUDIT_TAG_SSE_TYPE, join_audit_values(state.sse_types.iter().copied())));
|
||||
if !state.key_ids.is_empty() {
|
||||
tags.push((KMS_AUDIT_TAG_KEY_ID, join_audit_values(state.key_ids.iter().map(String::as_str))));
|
||||
}
|
||||
if !state.key_versions.is_empty() {
|
||||
tags.push((
|
||||
KMS_AUDIT_TAG_KEY_VERSION,
|
||||
join_audit_values(state.key_versions.iter().map(u32::to_string)),
|
||||
));
|
||||
}
|
||||
let outcome = if state.error_class.is_some() { "failure" } else { "success" };
|
||||
tags.push((KMS_AUDIT_TAG_OUTCOME, Value::String(outcome.to_string())));
|
||||
if let Some(error_class) = state.error_class {
|
||||
tags.push((KMS_AUDIT_TAG_ERROR_CLASS, Value::String(error_class.to_string())));
|
||||
}
|
||||
|
||||
tags
|
||||
}
|
||||
}
|
||||
|
||||
/// Render a set of values as one tag value.
|
||||
///
|
||||
/// Always a string, including for the single-value case: a consumer that would
|
||||
/// have to branch on the JSON type of a tag is a consumer that will get it
|
||||
/// wrong for the request that happens to touch two keys.
|
||||
fn join_audit_values(values: impl Iterator<Item = impl AsRef<str>>) -> Value {
|
||||
let joined = values.map(|value| value.as_ref().to_string()).collect::<Vec<_>>();
|
||||
Value::String(joined.join(KMS_AUDIT_VALUE_SEPARATOR))
|
||||
}
|
||||
|
||||
/// Slots of the requests currently being audited, keyed by canonical request ID.
|
||||
///
|
||||
/// Only a weak reference is held: the owning [`KmsRequestAuditScope`] decides the
|
||||
/// lifetime, so a scope that is somehow not dropped cannot keep a slot alive.
|
||||
static KMS_REQUEST_AUDITS: LazyLock<Mutex<HashMap<String, Weak<KmsRequestAudit>>>> = LazyLock::new(|| Mutex::new(HashMap::new()));
|
||||
|
||||
/// Registration of one request's KMS audit slot, dropped with the request's
|
||||
/// audit entry.
|
||||
pub struct KmsRequestAuditScope {
|
||||
request_id: String,
|
||||
audit: Arc<KmsRequestAudit>,
|
||||
}
|
||||
|
||||
impl KmsRequestAuditScope {
|
||||
/// Open a slot for `request_id`, so managed-SSE operations served under it can
|
||||
/// be summarised onto its audit entry.
|
||||
pub fn register(request_id: &str) -> Self {
|
||||
let audit = Arc::new(KmsRequestAudit::default());
|
||||
if let Ok(mut registry) = KMS_REQUEST_AUDITS.lock() {
|
||||
registry.insert(request_id.to_string(), Arc::downgrade(&audit));
|
||||
}
|
||||
|
||||
Self {
|
||||
request_id: request_id.to_string(),
|
||||
audit,
|
||||
}
|
||||
}
|
||||
|
||||
/// Tags describing the KMS work recorded under this request.
|
||||
pub fn audit_tags(&self) -> Vec<(&'static str, Value)> {
|
||||
self.audit.audit_tags()
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for KmsRequestAuditScope {
|
||||
fn drop(&mut self) {
|
||||
if let Ok(mut registry) = KMS_REQUEST_AUDITS.lock() {
|
||||
registry.remove(&self.request_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolve the audit slot opened for `request_id`, if the request is being audited.
|
||||
pub(crate) fn kms_request_audit(request_id: &str) -> Option<Arc<KmsRequestAudit>> {
|
||||
KMS_REQUEST_AUDITS.lock().ok()?.get(request_id)?.upgrade()
|
||||
}
|
||||
|
||||
/// Failure classes for data-path failures that never reached a KMS backend.
|
||||
///
|
||||
/// Backend failures carry the classification defined by the KMS audit contract;
|
||||
/// everything else is classified at the S3 boundary it surfaced through.
|
||||
pub(crate) fn kms_data_plane_error_class(error: &SseError) -> &'static str {
|
||||
if let Some(failure) = error
|
||||
.source
|
||||
.as_ref()
|
||||
.and_then(|source| source.downcast_ref::<KmsDataPlaneFailure>())
|
||||
{
|
||||
return failure.class;
|
||||
}
|
||||
|
||||
match error.code {
|
||||
S3ErrorCode::AccessDenied => "access_denied",
|
||||
S3ErrorCode::InvalidArgument | S3ErrorCode::InvalidRequest => "invalid_argument",
|
||||
_ => "sse_internal",
|
||||
}
|
||||
}
|
||||
|
||||
/// Carries a KMS failure class across the conversion to an S3-facing error.
|
||||
///
|
||||
/// The class is decided while the `KmsError` is still typed; re-deriving it from
|
||||
/// a rendered message downstream would be guesswork.
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct KmsDataPlaneFailure {
|
||||
pub(crate) class: &'static str,
|
||||
pub(crate) source: Option<Box<dyn std::error::Error + Send + Sync>>,
|
||||
}
|
||||
|
||||
impl std::fmt::Display for KmsDataPlaneFailure {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match &self.source {
|
||||
Some(source) => source.fmt(formatter),
|
||||
None => formatter.write_str(self.class),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for KmsDataPlaneFailure {
|
||||
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
|
||||
self.source
|
||||
.as_ref()
|
||||
.map(|source| source.as_ref() as &(dyn std::error::Error + 'static))
|
||||
}
|
||||
}
|
||||
|
||||
/// Record the outcome of one managed-SSE operation on the request's audit entry.
|
||||
///
|
||||
/// A `None` principal marks an internal caller — replication, lifecycle, heal —
|
||||
/// which has no S3 audit entry to attach to. `key_version` is a closure for
|
||||
/// exactly that caller: extracting the version means base64-decoding and
|
||||
/// parsing the stored envelope, work that must not run on the internal hot
|
||||
/// paths that discard it.
|
||||
pub(crate) fn record_managed_kms_outcome(
|
||||
principal: Option<&SseKmsPrincipal>,
|
||||
sse_type: SSEType,
|
||||
key_id: Option<&str>,
|
||||
key_version: impl FnOnce() -> Option<u32>,
|
||||
result: Result<(), &SseError>,
|
||||
) {
|
||||
let Some(audit) = principal.and_then(|principal| principal.request_audit.as_ref()) else {
|
||||
return;
|
||||
};
|
||||
|
||||
audit.record(sse_type, key_id, key_version(), result.err().map(kms_data_plane_error_class));
|
||||
}
|
||||
|
||||
/// Master-key version recorded in a managed-SSE data-key envelope, if the
|
||||
/// wrapping backend recorded one.
|
||||
///
|
||||
/// `None` is the honest answer for every other shape: Transit and AWS keep
|
||||
/// the version inside their own opaque ciphertext, Local records no version,
|
||||
/// and pre-versioning envelopes never carried the field. The single field is
|
||||
/// read through `serde_json::Value` rather than a full `DataKeyEnvelope`
|
||||
/// parse, so the audit path cannot double-count the envelope's unknown-field
|
||||
/// observability and touches nothing else in the envelope.
|
||||
pub(crate) fn envelope_master_key_version(envelope_bytes: &[u8]) -> Option<u32> {
|
||||
if !is_data_key_envelope(envelope_bytes) {
|
||||
return None;
|
||||
}
|
||||
u32::try_from(
|
||||
serde_json::from_slice::<Value>(envelope_bytes)
|
||||
.ok()?
|
||||
.get("master_key_version")?
|
||||
.as_u64()?,
|
||||
)
|
||||
.ok()
|
||||
}
|
||||
|
||||
/// Master-key version of the envelope stored on an object, for the audit
|
||||
/// summary of a read against that object.
|
||||
pub(crate) fn stored_envelope_master_key_version(metadata: &HashMap<String, String>) -> Option<u32> {
|
||||
// No context recoder: the recode only ever inserts the context key, which
|
||||
// this lookup never reads, so the normalized result is identical without it.
|
||||
let encoded = normalize_managed_metadata(metadata, None);
|
||||
let encoded = encoded.get(INTERNAL_ENCRYPTION_KEY_HEADER)?;
|
||||
let envelope = BASE64_STANDARD.decode_to_vec(encoded).ok()?;
|
||||
envelope_master_key_version(&envelope)
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! The encryption context a data key is bound to.
|
||||
|
||||
use crate::error::SseError;
|
||||
use base64_simd::STANDARD as BASE64_STANDARD;
|
||||
use rustfs_kms::types::ObjectEncryptionContext;
|
||||
use rustfs_utils::http::object_encryption_keys::MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER;
|
||||
use rustfs_utils::path::path_join_buf;
|
||||
use std::collections::HashMap;
|
||||
|
||||
pub(crate) fn canonical_kms_bucket_path(bucket: &str, key: &str) -> String {
|
||||
path_join_buf(&[bucket, key])
|
||||
}
|
||||
|
||||
pub(crate) fn build_kms_request_context(
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
provided_context: Option<&HashMap<String, String>>,
|
||||
) -> HashMap<String, String> {
|
||||
let mut context = provided_context.cloned().unwrap_or_default();
|
||||
context
|
||||
.entry(bucket.to_string())
|
||||
.or_insert_with(|| canonical_kms_bucket_path(bucket, key));
|
||||
context
|
||||
}
|
||||
|
||||
pub(crate) fn build_object_encryption_context(
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
provided_context: Option<&HashMap<String, String>>,
|
||||
) -> ObjectEncryptionContext {
|
||||
let mut context = ObjectEncryptionContext::new(bucket.to_string(), key.to_string());
|
||||
for (ctx_key, ctx_value) in build_kms_request_context(bucket, key, provided_context) {
|
||||
context = context.with_encryption_context(ctx_key, ctx_value);
|
||||
}
|
||||
context
|
||||
}
|
||||
|
||||
pub(crate) fn encode_minio_kms_context(context: &HashMap<String, String>) -> Result<String, SseError> {
|
||||
let encoded = serde_json::to_vec(context).map_err(|e| SseError::internal(format!("Failed to serialize KMS context: {e}")))?;
|
||||
Ok(BASE64_STANDARD.encode_to_string(encoded))
|
||||
}
|
||||
|
||||
pub(crate) fn decode_minio_kms_context(metadata: &HashMap<String, String>) -> Result<Option<HashMap<String, String>>, SseError> {
|
||||
let Some(encoded) = metadata.get(MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER) else {
|
||||
return Ok(None);
|
||||
};
|
||||
let decoded = BASE64_STANDARD
|
||||
.decode_to_vec(encoded)
|
||||
.map_err(|e| SseError::internal(format!("Failed to decode MinIO KMS context: {e}")))?;
|
||||
serde_json::from_slice(&decoded)
|
||||
.map(Some)
|
||||
.map_err(|e| SseError::internal(format!("Failed to parse MinIO KMS context: {e}")))
|
||||
}
|
||||
@@ -0,0 +1,212 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Direct keying: the data key encrypts the object itself, under a stored base nonce.
|
||||
|
||||
#[cfg(not(feature = "rio-v2"))]
|
||||
use crate::core::keying::NewObjectKey;
|
||||
#[cfg(not(feature = "rio-v2"))]
|
||||
use crate::core::keying::material::EncryptionKeyKind;
|
||||
use crate::core::keying::{StoredDataKey, StoredKeyMode};
|
||||
use crate::error::SseError;
|
||||
use crate::host::SseHost;
|
||||
use base64_simd::STANDARD as BASE64_STANDARD;
|
||||
use md5::{Digest as Md5Digest, Md5};
|
||||
#[cfg(not(feature = "rio-v2"))]
|
||||
use rand::Rng;
|
||||
#[cfg(not(feature = "rio-v2"))]
|
||||
use rustfs_kms::DataKey;
|
||||
#[cfg(not(feature = "rio-v2"))]
|
||||
use rustfs_utils::http::object_encryption_keys::SSEType;
|
||||
use rustfs_utils::http::object_encryption_keys::{
|
||||
INTERNAL_ENCRYPTION_ALGORITHM_HEADER, INTERNAL_ENCRYPTION_IV_HEADER, INTERNAL_ENCRYPTION_KEY_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_IV_HEADER,
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
|
||||
/// SSE-C: record the base nonce the object is encrypted under.
|
||||
///
|
||||
/// The nonce is random, so decrypt reads it back instead of recomputing a
|
||||
/// deterministic value. It is written under both the RustFS and the MinIO key per
|
||||
/// repo convention. This covers a single PUT and a multipart session alike:
|
||||
/// CreateMultipartUpload builds the session metadata through the same call.
|
||||
pub(crate) fn record_customer_key_nonce(metadata: &mut HashMap<String, String>, base_nonce: [u8; 12]) {
|
||||
metadata.insert(INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode_to_string(base_nonce));
|
||||
metadata.insert(
|
||||
MINIO_INTERNAL_ENCRYPTION_IV_HEADER.to_string(),
|
||||
BASE64_STANDARD.encode_to_string(base_nonce),
|
||||
);
|
||||
}
|
||||
|
||||
/// Managed SSE: record the wrapped data key, the nonce issued with it, and the cipher.
|
||||
pub(crate) fn record_wrapped_data_key(
|
||||
metadata: &mut HashMap<String, String>,
|
||||
wrapped_data_key: &[u8],
|
||||
base_nonce: [u8; 12],
|
||||
algorithm: &str,
|
||||
) {
|
||||
metadata.insert(
|
||||
INTERNAL_ENCRYPTION_KEY_HEADER.to_string(),
|
||||
BASE64_STANDARD.encode_to_string(wrapped_data_key),
|
||||
);
|
||||
metadata.insert(INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode_to_string(base_nonce));
|
||||
metadata.insert(INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(), algorithm.to_string());
|
||||
}
|
||||
|
||||
/// Managed SSE: read back what [`record_wrapped_data_key`] stored.
|
||||
///
|
||||
/// `normalized_metadata` is the object's metadata with the managed keys normalized.
|
||||
pub(crate) async fn read_wrapped_data_key(
|
||||
host: &dyn SseHost,
|
||||
normalized_metadata: &HashMap<String, String>,
|
||||
) -> Result<StoredDataKey, SseError> {
|
||||
let (wrapped, iv, algorithm) = if let Some(service) = host.encryption_service().await {
|
||||
// Production mode: use service for metadata parsing
|
||||
let parsed = service
|
||||
.headers_to_metadata(normalized_metadata)
|
||||
.map_err(|e| SseError::internal(format!("Failed to parse encryption metadata: {e}")))?;
|
||||
|
||||
(parsed.encrypted_data_key, parsed.iv, parsed.algorithm)
|
||||
} else {
|
||||
// Test mode: parse metadata manually
|
||||
let encrypted_key_b64 = normalized_metadata
|
||||
.get(INTERNAL_ENCRYPTION_KEY_HEADER)
|
||||
.ok_or_else(|| SseError::internal("Missing encrypted key in metadata"))?;
|
||||
let encrypted_data_key = BASE64_STANDARD
|
||||
.decode_to_vec(encrypted_key_b64)
|
||||
.map_err(|e| SseError::internal(format!("Failed to decode encrypted key: {e}")))?;
|
||||
|
||||
let iv_b64 = normalized_metadata
|
||||
.get(INTERNAL_ENCRYPTION_IV_HEADER)
|
||||
.ok_or_else(|| SseError::internal("Missing IV in metadata"))?;
|
||||
let iv = BASE64_STANDARD
|
||||
.decode_to_vec(iv_b64)
|
||||
.map_err(|e| SseError::internal(format!("Failed to decode IV: {e}")))?;
|
||||
|
||||
let algorithm = normalized_metadata
|
||||
.get(INTERNAL_ENCRYPTION_ALGORITHM_HEADER)
|
||||
.cloned()
|
||||
.unwrap_or_else(|| "AES256".to_string());
|
||||
|
||||
(encrypted_data_key, iv, algorithm)
|
||||
};
|
||||
|
||||
let base_nonce = <[u8; 12]>::try_from(iv.as_slice())
|
||||
.map_err(|_| SseError::internal("Invalid encryption nonce length; expected 12 bytes"))?;
|
||||
|
||||
Ok(StoredDataKey {
|
||||
wrapped,
|
||||
algorithm,
|
||||
mode: StoredKeyMode::Direct { base_nonce },
|
||||
})
|
||||
}
|
||||
|
||||
/// Resolve the SSE-C Direct base nonce for decryption.
|
||||
///
|
||||
/// New objects persist a random nonce at encryption time (see
|
||||
/// [`record_customer_key_nonce`]) under `INTERNAL_ENCRYPTION_IV_HEADER` (preferred) or,
|
||||
/// for MinIO interop, `MINIO_INTERNAL_ENCRYPTION_IV_HEADER`. Legacy objects written before
|
||||
/// random nonces were persisted carry no stored IV; for those we fall back to the
|
||||
/// deterministic `generate_ssec_nonce(bucket, key)` value they were originally encrypted
|
||||
/// with, so previously stored objects still decrypt correctly.
|
||||
pub(crate) fn read_stored_ssec_nonce(metadata: &HashMap<String, String>, bucket: &str, key: &str) -> [u8; 12] {
|
||||
metadata
|
||||
.get(INTERNAL_ENCRYPTION_IV_HEADER)
|
||||
.or_else(|| metadata.get(MINIO_INTERNAL_ENCRYPTION_IV_HEADER))
|
||||
.and_then(|encoded| BASE64_STANDARD.decode_to_vec(encoded).ok())
|
||||
.and_then(|bytes| <[u8; 12]>::try_from(bytes.as_slice()).ok())
|
||||
.unwrap_or_else(|| generate_ssec_nonce(bucket, key))
|
||||
}
|
||||
|
||||
/// Generate deterministic nonce for SSE-C encryption
|
||||
///
|
||||
/// The nonce is derived from the bucket and key to ensure:
|
||||
/// 1. Same object always gets the same nonce (required for SSE-C)
|
||||
/// 2. Different objects get different nonces
|
||||
pub(crate) fn generate_ssec_nonce(bucket: &str, key: &str) -> [u8; 12] {
|
||||
let nonce_source = format!("{bucket}-{key}");
|
||||
let nonce_hash: [u8; 16] = Md5::digest(nonce_source.as_bytes()).into();
|
||||
let mut nonce = [0u8; 12];
|
||||
nonce.copy_from_slice(&nonce_hash[..12]);
|
||||
nonce
|
||||
}
|
||||
|
||||
/// Whether a multipart session needs the customer key when it is created.
|
||||
///
|
||||
/// Direct mode records only a nonce for the session; the customer key is checked
|
||||
/// per part, against what the session recorded.
|
||||
#[cfg(not(feature = "rio-v2"))]
|
||||
pub(crate) const SESSION_KEY_NEEDS_CUSTOMER_KEY: bool = false;
|
||||
|
||||
#[cfg(not(feature = "rio-v2"))]
|
||||
fn random_base_nonce() -> [u8; 12] {
|
||||
let mut base_nonce = [0u8; 12];
|
||||
rand::rng().fill_bytes(&mut base_nonce);
|
||||
base_nonce
|
||||
}
|
||||
|
||||
/// SSE-C object: the customer key encrypts the object itself.
|
||||
///
|
||||
/// Uses a fresh random nonce per encryption. A deterministic nonce derived from
|
||||
/// (bucket, key) would repeat whenever the same object is overwritten under the same
|
||||
/// SSE-C key, reusing an identical (key, nonce) pair and catastrophically breaking
|
||||
/// AES-256-GCM. The nonce is persisted (see [`record_customer_key_nonce`]) and read
|
||||
/// back on decrypt (see [`read_stored_ssec_nonce`]).
|
||||
#[cfg(not(feature = "rio-v2"))]
|
||||
pub(crate) fn new_key_under_customer_key(customer_key: [u8; 32], _bucket: &str, _key: &str) -> Result<NewObjectKey, SseError> {
|
||||
Ok(NewObjectKey {
|
||||
key_bytes: customer_key,
|
||||
base_nonce: random_base_nonce(),
|
||||
key_kind: EncryptionKeyKind::Direct,
|
||||
sealed_key: None,
|
||||
})
|
||||
}
|
||||
|
||||
/// SSE-C multipart session: a nonce every part of the upload shares.
|
||||
///
|
||||
/// Generates a real random nonce for the session and persists it (see
|
||||
/// [`record_customer_key_nonce`]). Every part of this upload reads the same
|
||||
/// persisted nonce back, so all parts share one nonce without reintroducing the
|
||||
/// deterministic (bucket, key) reuse hazard. `key_bytes` stays a placeholder: it is
|
||||
/// neither used for encryption nor persisted; the real customer key is validated per
|
||||
/// part on upload.
|
||||
#[cfg(not(feature = "rio-v2"))]
|
||||
pub(crate) fn new_session_key_under_customer_key(
|
||||
_customer_key: Option<[u8; 32]>,
|
||||
_bucket: &str,
|
||||
_key: &str,
|
||||
) -> Result<NewObjectKey, SseError> {
|
||||
Ok(NewObjectKey {
|
||||
key_bytes: [0; 32],
|
||||
base_nonce: random_base_nonce(),
|
||||
key_kind: EncryptionKeyKind::Direct,
|
||||
sealed_key: None,
|
||||
})
|
||||
}
|
||||
|
||||
/// Managed SSE: the data key encrypts the object itself, under the nonce issued with it.
|
||||
#[cfg(not(feature = "rio-v2"))]
|
||||
pub(crate) fn new_key_under_data_key(
|
||||
data_key: &DataKey,
|
||||
_sse_type: SSEType,
|
||||
_bucket: &str,
|
||||
_key: &str,
|
||||
) -> Result<NewObjectKey, SseError> {
|
||||
Ok(NewObjectKey {
|
||||
key_bytes: data_key.plaintext_key,
|
||||
base_nonce: data_key.nonce,
|
||||
key_kind: EncryptionKeyKind::Direct,
|
||||
sealed_key: None,
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! The key material an encrypted write or read runs on.
|
||||
|
||||
#[cfg(feature = "rio-v2")]
|
||||
use crate::core::keying::object::{SEALED_KEY_IV_SIZE, SEALED_KEY_SIZE};
|
||||
use rustfs_utils::http::object_encryption_keys::SSEType;
|
||||
use std::collections::HashMap;
|
||||
|
||||
/// Encryption material returned by [`sse_encryption`] / [`sse_prepare_encryption`].
|
||||
pub struct EncryptionMaterial {
|
||||
#[allow(unused)]
|
||||
pub sse_type: SSEType,
|
||||
pub server_side_encryption: String,
|
||||
pub kms_key_id: Option<String>,
|
||||
|
||||
#[allow(unused)]
|
||||
pub algorithm: String,
|
||||
|
||||
/// Encryption key bytes
|
||||
pub key_bytes: [u8; 32],
|
||||
/// Base nonce/IV used by rio to derive block/part nonces.
|
||||
pub base_nonce: [u8; 12],
|
||||
/// Encrypted DEK for managed SSE. Absent for SSE-C.
|
||||
pub encrypted_data_key: Option<Vec<u8>>,
|
||||
/// SSE-C key MD5 if customer-managed encryption is in use.
|
||||
pub customer_key_md5: Option<String>,
|
||||
/// Original plaintext size when it should be persisted alongside metadata.
|
||||
pub original_size: Option<i64>,
|
||||
/// Indicates whether `key_bytes` is a direct stream key or a MinIO-style object key.
|
||||
pub key_kind: EncryptionKeyKind,
|
||||
/// Original client-provided SSE-KMS context, stored in MinIO-compatible metadata.
|
||||
pub managed_kms_context: Option<HashMap<String, String>>,
|
||||
/// MinIO-compatible sealed object-key metadata for managed SSE object-key mode.
|
||||
#[cfg_attr(not(feature = "rio-v2"), allow(dead_code))]
|
||||
pub managed_sealed_key: Option<ManagedSealedKey>,
|
||||
}
|
||||
|
||||
/// Decryption material returned by [`sse_decryption`].
|
||||
pub struct DecryptionMaterial {
|
||||
#[allow(unused)]
|
||||
pub sse_type: SSEType,
|
||||
pub server_side_encryption: String,
|
||||
pub kms_key_id: Option<String>,
|
||||
#[allow(unused)]
|
||||
pub algorithm: String,
|
||||
pub customer_key_md5: Option<String>, // if use SSE-C, check key md5
|
||||
|
||||
/// Decryption key bytes
|
||||
pub key_bytes: [u8; 32],
|
||||
/// Base nonce/IV used by rio to derive block/part nonces.
|
||||
pub base_nonce: [u8; 12],
|
||||
/// Indicates whether `key_bytes` is a direct stream key or a MinIO-style object key.
|
||||
pub key_kind: EncryptionKeyKind,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for EncryptionMaterial {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("EncryptionMaterial")
|
||||
.field("sse_type", &self.sse_type)
|
||||
.field("server_side_encryption", &self.server_side_encryption)
|
||||
.field("original_size", &self.original_size)
|
||||
.finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for DecryptionMaterial {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("DecryptionMaterial")
|
||||
.field("sse_type", &self.sse_type)
|
||||
.field("server_side_encryption", &self.server_side_encryption)
|
||||
.finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum EncryptionKeyKind {
|
||||
Direct,
|
||||
Object,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ManagedSealedKey {
|
||||
#[cfg(feature = "rio-v2")]
|
||||
pub iv: [u8; SEALED_KEY_IV_SIZE],
|
||||
#[cfg(feature = "rio-v2")]
|
||||
pub sealed_key: [u8; SEALED_KEY_SIZE],
|
||||
}
|
||||
|
||||
impl EncryptionMaterial {
|
||||
/// The KMS key id a write response may advertise.
|
||||
///
|
||||
/// `kms_key_id` is always set for managed SSE because SSE-S3 also wraps
|
||||
/// its data key under the service default key, but that key is an
|
||||
/// internal detail of an `AES256` object: only an `aws:kms` object names
|
||||
/// a key the caller can act on, and `x-amz-server-side-encryption-aws-kms-key-id`
|
||||
/// is defined only for that scheme.
|
||||
pub fn response_kms_key_id(&self) -> Option<String> {
|
||||
matches!(self.sse_type, SSEType::SseKms)
|
||||
.then(|| self.kms_key_id.clone())
|
||||
.flatten()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,270 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Encryption metadata as it is stored with the object.
|
||||
|
||||
use crate::core::keying::context::encode_minio_kms_context;
|
||||
use crate::core::keying::direct::{record_customer_key_nonce, record_wrapped_data_key};
|
||||
use crate::core::keying::material::{EncryptionKeyKind, EncryptionMaterial};
|
||||
#[cfg(feature = "rio-v2")]
|
||||
use crate::core::keying::object::{record_data_key_in_minio_slots, record_sealed_customer_key};
|
||||
use crate::error::SseError;
|
||||
use base64_simd::STANDARD as BASE64_STANDARD;
|
||||
use rustfs_utils::http::object_encryption_keys::{
|
||||
INTERNAL_ENCRYPTION_ALGORITHM_HEADER, INTERNAL_ENCRYPTION_CONTEXT_HEADER, INTERNAL_ENCRYPTION_IV_HEADER,
|
||||
INTERNAL_ENCRYPTION_KEY_HEADER, INTERNAL_ENCRYPTION_KEY_ID_HEADER, INTERNAL_ENCRYPTION_ORIGINAL_SIZE_HEADER,
|
||||
INTERNAL_ENCRYPTION_TAG_HEADER, MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER, MINIO_INTERNAL_ENCRYPTION_IV_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER, MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER, MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_MULTIPART_HEADER, MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER, SSEC_ORIGINAL_SIZE_HEADER, SSEType,
|
||||
};
|
||||
use std::borrow::Cow;
|
||||
use std::collections::HashMap;
|
||||
|
||||
/// Two spellings of one encryption metadata key carry different values.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct ConflictingEncryptionMetadata {
|
||||
/// The canonical key whose case-variants disagree.
|
||||
pub key: &'static str,
|
||||
}
|
||||
|
||||
impl std::fmt::Display for ConflictingEncryptionMetadata {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(formatter, "conflicting object encryption metadata for {}", self.key)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for ConflictingEncryptionMetadata {}
|
||||
|
||||
/// Fold case-variant spellings of the encryption metadata keys onto their canonical names.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// When two spellings of the same key hold different values: picking either
|
||||
/// would decrypt with material the object may not have been written under.
|
||||
pub fn normalize_encryption_metadata_case(
|
||||
metadata: &HashMap<String, String>,
|
||||
) -> Result<Cow<'_, HashMap<String, String>>, ConflictingEncryptionMetadata> {
|
||||
const CANONICAL_KEYS: &[&str] = &[
|
||||
"x-amz-server-side-encryption",
|
||||
"x-amz-server-side-encryption-aws-kms-key-id",
|
||||
"x-amz-server-side-encryption-customer-algorithm",
|
||||
"x-amz-server-side-encryption-customer-key-md5",
|
||||
SSEC_ORIGINAL_SIZE_HEADER,
|
||||
INTERNAL_ENCRYPTION_KEY_ID_HEADER,
|
||||
INTERNAL_ENCRYPTION_KEY_HEADER,
|
||||
INTERNAL_ENCRYPTION_ALGORITHM_HEADER,
|
||||
INTERNAL_ENCRYPTION_IV_HEADER,
|
||||
INTERNAL_ENCRYPTION_CONTEXT_HEADER,
|
||||
INTERNAL_ENCRYPTION_TAG_HEADER,
|
||||
INTERNAL_ENCRYPTION_ORIGINAL_SIZE_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_MULTIPART_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_IV_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER,
|
||||
];
|
||||
|
||||
let needs_normalization = metadata.keys().any(|key| {
|
||||
CANONICAL_KEYS
|
||||
.iter()
|
||||
.any(|canonical| key != canonical && key.eq_ignore_ascii_case(canonical))
|
||||
});
|
||||
if !needs_normalization {
|
||||
return Ok(Cow::Borrowed(metadata));
|
||||
}
|
||||
|
||||
let mut normalized = metadata.clone();
|
||||
for canonical in CANONICAL_KEYS {
|
||||
let mut matching_values = metadata
|
||||
.iter()
|
||||
.filter_map(|(key, value)| key.eq_ignore_ascii_case(canonical).then_some(value));
|
||||
let Some(value) = matching_values.next() else {
|
||||
continue;
|
||||
};
|
||||
if matching_values.any(|candidate| candidate != value) {
|
||||
return Err(ConflictingEncryptionMetadata { key: canonical });
|
||||
}
|
||||
if !normalized.contains_key(*canonical) {
|
||||
normalized.insert((*canonical).to_string(), value.clone());
|
||||
}
|
||||
}
|
||||
Ok(Cow::Owned(normalized))
|
||||
}
|
||||
|
||||
/// Turn the material of an encrypted write into the metadata stored with the object.
|
||||
///
|
||||
/// The scheme is recorded here. How the key itself is recorded belongs to the mode
|
||||
/// that formed it, in `direct.rs` and `object.rs`, next to the code that reads it back.
|
||||
pub fn encryption_material_to_metadata(material: &EncryptionMaterial) -> Result<HashMap<String, String>, SseError> {
|
||||
let mut metadata = HashMap::new();
|
||||
|
||||
match material.sse_type {
|
||||
SSEType::SseC => {
|
||||
record_ssec_scheme(&mut metadata, material);
|
||||
|
||||
// A `rio-v2` build seals an object key instead, so a Direct key there is a
|
||||
// multipart session created without its customer key.
|
||||
if material.key_kind == EncryptionKeyKind::Direct {
|
||||
record_customer_key_nonce(&mut metadata, material.base_nonce);
|
||||
}
|
||||
#[cfg(feature = "rio-v2")]
|
||||
if let Some(sealed) = &material.managed_sealed_key {
|
||||
record_sealed_customer_key(&mut metadata, sealed);
|
||||
}
|
||||
}
|
||||
SSEType::SseS3 | SSEType::SseKms => {
|
||||
let wrapped_data_key = material
|
||||
.encrypted_data_key
|
||||
.as_deref()
|
||||
.ok_or_else(|| SseError::internal("managed SSE materials must carry an encrypted data key"))?;
|
||||
record_managed_scheme(&mut metadata, material);
|
||||
|
||||
if material.key_kind == EncryptionKeyKind::Direct {
|
||||
record_wrapped_data_key(&mut metadata, wrapped_data_key, material.base_nonce, material.algorithm.as_str());
|
||||
}
|
||||
#[cfg(feature = "rio-v2")]
|
||||
record_data_key_in_minio_slots(&mut metadata, material, wrapped_data_key);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(metadata)
|
||||
}
|
||||
|
||||
/// SSE-C: the scheme, the fingerprint of the customer key, and the plaintext size.
|
||||
fn record_ssec_scheme(metadata: &mut HashMap<String, String>, material: &EncryptionMaterial) {
|
||||
metadata.insert(
|
||||
"x-amz-server-side-encryption".to_string(),
|
||||
material.server_side_encryption.as_str().to_string(),
|
||||
);
|
||||
metadata.insert(
|
||||
"x-amz-server-side-encryption-customer-algorithm".to_string(),
|
||||
material.algorithm.as_str().to_string(),
|
||||
);
|
||||
if let Some(customer_key_md5) = &material.customer_key_md5 {
|
||||
metadata.insert("x-amz-server-side-encryption-customer-key-md5".to_string(), customer_key_md5.to_string());
|
||||
}
|
||||
if let Some(original_size) = material.original_size {
|
||||
metadata.insert(SSEC_ORIGINAL_SIZE_HEADER.to_string(), original_size.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
/// Managed SSE: the scheme, the key the data key is wrapped under, the plaintext
|
||||
/// size, and the encryption context of the request.
|
||||
fn record_managed_scheme(metadata: &mut HashMap<String, String>, material: &EncryptionMaterial) {
|
||||
metadata.insert(
|
||||
"x-amz-server-side-encryption".to_string(),
|
||||
material.server_side_encryption.as_str().to_string(),
|
||||
);
|
||||
|
||||
let internal_key_id = material.kms_key_id.clone().unwrap_or_else(|| "default".to_string());
|
||||
metadata.insert(INTERNAL_ENCRYPTION_KEY_ID_HEADER.to_string(), internal_key_id.clone());
|
||||
|
||||
if matches!(material.sse_type, SSEType::SseKms) {
|
||||
metadata.insert("x-amz-server-side-encryption-aws-kms-key-id".to_string(), internal_key_id);
|
||||
}
|
||||
|
||||
if let Some(original_size) = material.original_size {
|
||||
metadata.insert(INTERNAL_ENCRYPTION_ORIGINAL_SIZE_HEADER.to_string(), original_size.to_string());
|
||||
}
|
||||
|
||||
if let Some(kms_context) = &material.managed_kms_context
|
||||
&& !kms_context.is_empty()
|
||||
{
|
||||
if let Ok(serialized) = serde_json::to_string(kms_context) {
|
||||
metadata.insert(INTERNAL_ENCRYPTION_CONTEXT_HEADER.to_string(), serialized);
|
||||
}
|
||||
if matches!(material.sse_type, SSEType::SseKms)
|
||||
&& let Ok(encoded) = encode_minio_kms_context(kms_context)
|
||||
{
|
||||
metadata.insert(MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER.to_string(), encoded);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Strip source encryption metadata before constructing metadata for a copy destination.
|
||||
///
|
||||
/// Encryption metadata describes the physical source representation and must never be
|
||||
/// inherited by a plaintext destination or by a destination using a different key.
|
||||
pub fn strip_managed_encryption_metadata(metadata: &mut HashMap<String, String>) {
|
||||
const KEYS: [&str; 21] = [
|
||||
"x-amz-server-side-encryption",
|
||||
"x-amz-server-side-encryption-aws-kms-key-id",
|
||||
"x-amz-server-side-encryption-customer-algorithm",
|
||||
"x-amz-server-side-encryption-customer-key-md5",
|
||||
SSEC_ORIGINAL_SIZE_HEADER,
|
||||
INTERNAL_ENCRYPTION_KEY_ID_HEADER,
|
||||
INTERNAL_ENCRYPTION_ALGORITHM_HEADER,
|
||||
INTERNAL_ENCRYPTION_IV_HEADER,
|
||||
INTERNAL_ENCRYPTION_TAG_HEADER,
|
||||
INTERNAL_ENCRYPTION_KEY_HEADER,
|
||||
INTERNAL_ENCRYPTION_CONTEXT_HEADER,
|
||||
INTERNAL_ENCRYPTION_ORIGINAL_SIZE_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_MULTIPART_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_IV_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER,
|
||||
];
|
||||
|
||||
for key in KEYS.iter() {
|
||||
metadata.remove(*key);
|
||||
}
|
||||
rustfs_utils::http::metadata_compat::remove_str(metadata, rustfs_utils::http::SUFFIX_OPAQUE_ENCRYPTED_ETAG);
|
||||
}
|
||||
|
||||
pub fn mark_encrypted_multipart_metadata(metadata: &mut HashMap<String, String>) {
|
||||
metadata.insert(MINIO_INTERNAL_ENCRYPTION_MULTIPART_HEADER.to_string(), String::new());
|
||||
}
|
||||
|
||||
/// True when the stored metadata marks the object as SSE-C protected.
|
||||
///
|
||||
/// RustFS persists the public `x-amz-server-side-encryption-customer-algorithm`
|
||||
/// key on SSE-C writes. MinIO persists only its internal sealed-key slot and
|
||||
/// synthesizes the public header onto responses (backlog#1638), so the slot is
|
||||
/// the only durable evidence on a MinIO-written object. That shape is readable
|
||||
/// solely through the rio-v2 sealed-object-key path, so it participates in
|
||||
/// detection only in builds that can actually read it — default builds keep
|
||||
/// the resolver's fail-closed diagnosis instead of demanding customer keys
|
||||
/// they could not use.
|
||||
pub fn stored_ssec_metadata(metadata: &HashMap<String, String>) -> bool {
|
||||
metadata.contains_key("x-amz-server-side-encryption-customer-algorithm") || minio_sealed_ssec_metadata(metadata)
|
||||
}
|
||||
|
||||
/// True when the object carries MinIO's SSE-C sealed-key slot — the shape that
|
||||
/// stores no customer-key MD5 and proves the key through the AEAD unseal
|
||||
/// instead.
|
||||
pub(crate) fn minio_sealed_ssec_metadata(metadata: &HashMap<String, String>) -> bool {
|
||||
cfg!(feature = "rio-v2") && metadata.contains_key(MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER)
|
||||
}
|
||||
|
||||
/// Recodes a stored MinIO KMS context value (base64-wrapped JSON) into the
|
||||
/// plain-JSON form RustFS stores under [`INTERNAL_ENCRYPTION_CONTEXT_HEADER`].
|
||||
///
|
||||
/// Injected into the shared [`normalize_managed_metadata`] because the shared
|
||||
/// crate carries no JSON codec; any decode failure returns `None`, which skips
|
||||
/// the context mapping exactly like the historical inline `if let Ok` chain.
|
||||
pub(crate) fn recode_minio_kms_context(value: &str) -> Option<String> {
|
||||
let decoded = BASE64_STANDARD.decode_to_vec(value).ok()?;
|
||||
let context = serde_json::from_slice::<HashMap<String, String>>(&decoded).ok()?;
|
||||
serde_json::to_string(&context).ok()
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! The internal keying modes: how an object's key is formed and how it is stored.
|
||||
|
||||
pub(crate) mod context;
|
||||
pub(crate) mod direct;
|
||||
pub(crate) mod material;
|
||||
pub(crate) mod metadata;
|
||||
#[cfg(feature = "rio-v2")]
|
||||
pub(crate) mod object;
|
||||
pub(crate) mod stored;
|
||||
|
||||
use crate::core::keying::material::{EncryptionKeyKind, ManagedSealedKey};
|
||||
|
||||
/// The key a new object is encrypted with, in the mode this build writes.
|
||||
pub(crate) struct NewObjectKey {
|
||||
/// Key handed to the stream cipher.
|
||||
pub(crate) key_bytes: [u8; 32],
|
||||
/// Base nonce the stream cipher derives its block nonces from.
|
||||
pub(crate) base_nonce: [u8; 12],
|
||||
/// The mode `key_bytes` belongs to.
|
||||
pub(crate) key_kind: EncryptionKeyKind,
|
||||
/// The sealed form to store, in a mode that seals a separate object key.
|
||||
pub(crate) sealed_key: Option<ManagedSealedKey>,
|
||||
}
|
||||
|
||||
/// A managed-SSE data key as a stored object recorded it, still wrapped.
|
||||
pub(crate) struct StoredDataKey {
|
||||
/// The data key, wrapped by the provider that issued it.
|
||||
pub(crate) wrapped: Vec<u8>,
|
||||
/// The cipher the object recorded.
|
||||
pub(crate) algorithm: String,
|
||||
/// What the data key is to the object once it is unwrapped.
|
||||
pub(crate) mode: StoredKeyMode,
|
||||
}
|
||||
|
||||
/// The mode a stored key was recorded in.
|
||||
pub(crate) enum StoredKeyMode {
|
||||
/// The data key is the object's key, under the nonce stored beside it.
|
||||
Direct { base_nonce: [u8; 12] },
|
||||
/// The data key unseals the object's key.
|
||||
#[cfg(feature = "rio-v2")]
|
||||
Object { sealed_key: ManagedSealedKey },
|
||||
}
|
||||
|
||||
// The mode new objects are written in. Every build reads every mode it can
|
||||
// recognize; writing picks exactly one, and this is the only place that says which.
|
||||
#[cfg(not(feature = "rio-v2"))]
|
||||
pub(crate) use direct::{
|
||||
SESSION_KEY_NEEDS_CUSTOMER_KEY, new_key_under_customer_key, new_key_under_data_key, new_session_key_under_customer_key,
|
||||
};
|
||||
#[cfg(feature = "rio-v2")]
|
||||
pub(crate) use object::{
|
||||
SESSION_KEY_NEEDS_CUSTOMER_KEY, new_key_under_customer_key, new_key_under_data_key, new_session_key_under_customer_key,
|
||||
};
|
||||
@@ -0,0 +1,436 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Object keying: a derived object key, sealed in the MinIO-compatible format.
|
||||
|
||||
use crate::core::keying::material::{EncryptionKeyKind, EncryptionMaterial, ManagedSealedKey};
|
||||
use crate::core::keying::{NewObjectKey, StoredDataKey, StoredKeyMode};
|
||||
use crate::error::SseError;
|
||||
use aes_gcm::aead::{Aead, KeyInit, Payload};
|
||||
use aes_gcm::{Aes256Gcm, Nonce};
|
||||
use base64_simd::STANDARD as BASE64_STANDARD;
|
||||
use chacha20poly1305::ChaCha20Poly1305;
|
||||
use hmac::{Hmac, Mac};
|
||||
use rand::RngExt;
|
||||
use rustfs_kms::DataKey;
|
||||
use rustfs_utils::http::object_encryption_keys::{
|
||||
INTERNAL_ENCRYPTION_ALGORITHM_HEADER, INTERNAL_ENCRYPTION_IV_HEADER, INTERNAL_ENCRYPTION_KEY_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER, MINIO_INTERNAL_ENCRYPTION_IV_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER, MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER, MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER, SSEType,
|
||||
};
|
||||
use sha2::Sha256;
|
||||
use std::collections::HashMap;
|
||||
|
||||
pub(crate) const MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM: &str = "DAREv2-HMAC-SHA256";
|
||||
|
||||
const DARE_VERSION_20: u8 = 0x20;
|
||||
|
||||
pub(crate) const DARE_CIPHER_AES_256_GCM: u8 = 0x00;
|
||||
|
||||
pub(crate) const DARE_CIPHER_CHACHA20_POLY1305: u8 = 0x01;
|
||||
|
||||
const DARE_HEADER_SIZE: usize = 16;
|
||||
|
||||
const DARE_TAG_SIZE: usize = 16;
|
||||
|
||||
pub(crate) const SEALED_KEY_IV_SIZE: usize = 32;
|
||||
|
||||
pub(crate) const SEALED_KEY_SIZE: usize = DARE_HEADER_SIZE + 32 + DARE_TAG_SIZE;
|
||||
|
||||
const OBJECT_KEY_DERIVATION_CONTEXT: &[u8] = b"object-encryption-key generation";
|
||||
|
||||
pub(crate) type HmacSha256 = Hmac<Sha256>;
|
||||
|
||||
fn managed_sse_domain(sse_type: SSEType) -> &'static str {
|
||||
match sse_type {
|
||||
SSEType::SseS3 => "SSE-S3",
|
||||
SSEType::SseKms => "SSE-KMS",
|
||||
SSEType::SseC => "SSE-C",
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn is_supported_sealed_object_key_cipher(cipher: u8) -> bool {
|
||||
matches!(cipher, DARE_CIPHER_AES_256_GCM | DARE_CIPHER_CHACHA20_POLY1305)
|
||||
}
|
||||
|
||||
fn decrypt_sealed_object_key_payload(sealing_key: [u8; 32], header: &[u8], sealed_key: &[u8]) -> Result<Vec<u8>, SseError> {
|
||||
let nonce = &header[4..16];
|
||||
let ciphertext = &sealed_key[DARE_HEADER_SIZE..];
|
||||
let aad = &header[..4];
|
||||
match header[1] {
|
||||
DARE_CIPHER_AES_256_GCM => {
|
||||
let cipher = Aes256Gcm::new_from_slice(&sealing_key)
|
||||
.map_err(|err| SseError::internal(format!("Invalid AES-GCM sealing key: {err}")))?;
|
||||
let nonce = Nonce::try_from(nonce).map_err(|_| SseError::internal("Invalid sealed object-key package nonce"))?;
|
||||
cipher.decrypt(&nonce, Payload { msg: ciphertext, aad })
|
||||
}
|
||||
DARE_CIPHER_CHACHA20_POLY1305 => {
|
||||
let cipher = ChaCha20Poly1305::new_from_slice(&sealing_key)
|
||||
.map_err(|err| SseError::internal(format!("Invalid ChaCha20-Poly1305 sealing key: {err}")))?;
|
||||
let nonce = chacha20poly1305::Nonce::try_from(nonce)
|
||||
.map_err(|_| SseError::internal("Invalid sealed object-key package nonce"))?;
|
||||
cipher.decrypt(&nonce, Payload { msg: ciphertext, aad })
|
||||
}
|
||||
_ => return Err(SseError::internal("Unsupported sealed object-key DARE header")),
|
||||
}
|
||||
.map_err(|err| SseError::internal(format!("Failed to unseal object key: {err}")))
|
||||
}
|
||||
|
||||
fn canonical_sse_path(bucket: &str, object: &str) -> String {
|
||||
let bucket = bucket.trim_matches('/');
|
||||
let object = object.trim_matches('/');
|
||||
if object.is_empty() {
|
||||
bucket.to_string()
|
||||
} else if bucket.is_empty() {
|
||||
object.to_string()
|
||||
} else {
|
||||
format!("{bucket}/{object}")
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn derive_object_key(external_key: [u8; 32]) -> Result<[u8; 32], SseError> {
|
||||
let mut random = [0u8; 32];
|
||||
rand::rng().fill(&mut random);
|
||||
|
||||
let mut mac = HmacSha256::new_from_slice(&external_key)
|
||||
.map_err(|err| SseError::internal(format!("Invalid HMAC key for object-key derivation: {err}")))?;
|
||||
mac.update(OBJECT_KEY_DERIVATION_CONTEXT);
|
||||
mac.update(&random);
|
||||
|
||||
let mut object_key = [0u8; 32];
|
||||
object_key.copy_from_slice(mac.finalize().into_bytes().as_slice());
|
||||
Ok(object_key)
|
||||
}
|
||||
|
||||
fn derive_sealing_key(
|
||||
external_key: [u8; 32],
|
||||
iv: [u8; SEALED_KEY_IV_SIZE],
|
||||
domain: &str,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
) -> Result<[u8; 32], SseError> {
|
||||
let mut mac = HmacSha256::new_from_slice(&external_key)
|
||||
.map_err(|err| SseError::internal(format!("Invalid HMAC key for sealing-key derivation: {err}")))?;
|
||||
mac.update(&iv);
|
||||
mac.update(domain.as_bytes());
|
||||
mac.update(MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM.as_bytes());
|
||||
mac.update(canonical_sse_path(bucket, object).as_bytes());
|
||||
|
||||
let mut sealing_key = [0u8; 32];
|
||||
sealing_key.copy_from_slice(mac.finalize().into_bytes().as_slice());
|
||||
Ok(sealing_key)
|
||||
}
|
||||
|
||||
pub(crate) fn seal_object_key(
|
||||
object_key: [u8; 32],
|
||||
external_key: [u8; 32],
|
||||
sse_type: SSEType,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
) -> Result<ManagedSealedKey, SseError> {
|
||||
let mut iv = [0u8; SEALED_KEY_IV_SIZE];
|
||||
rand::rng().fill(&mut iv);
|
||||
let sealing_key = derive_sealing_key(external_key, iv, managed_sse_domain(sse_type), bucket, object)?;
|
||||
|
||||
let mut header = [0u8; DARE_HEADER_SIZE];
|
||||
header[0] = DARE_VERSION_20;
|
||||
header[1] = DARE_CIPHER_AES_256_GCM;
|
||||
header[2..4].copy_from_slice(&(31u16).to_le_bytes());
|
||||
let mut stream_nonce = [0u8; 12];
|
||||
rand::rng().fill(&mut stream_nonce);
|
||||
stream_nonce[0] |= 0x80;
|
||||
header[4..16].copy_from_slice(&stream_nonce);
|
||||
|
||||
let cipher =
|
||||
Aes256Gcm::new_from_slice(&sealing_key).map_err(|err| SseError::internal(format!("Invalid sealing key: {err}")))?;
|
||||
let nonce =
|
||||
Nonce::try_from(stream_nonce.as_slice()).map_err(|_| SseError::internal("Invalid sealed object-key stream nonce"))?;
|
||||
let ciphertext = cipher
|
||||
.encrypt(
|
||||
&nonce,
|
||||
aes_gcm::aead::Payload {
|
||||
msg: &object_key,
|
||||
aad: &header[..4],
|
||||
},
|
||||
)
|
||||
.map_err(|err| SseError::internal(format!("Failed to seal object key: {err}")))?;
|
||||
|
||||
let mut sealed_key = [0u8; SEALED_KEY_SIZE];
|
||||
sealed_key[..DARE_HEADER_SIZE].copy_from_slice(&header);
|
||||
sealed_key[DARE_HEADER_SIZE..].copy_from_slice(&ciphertext);
|
||||
|
||||
Ok(ManagedSealedKey { iv, sealed_key })
|
||||
}
|
||||
|
||||
pub(crate) fn unseal_object_key(
|
||||
sealed: &ManagedSealedKey,
|
||||
external_key: [u8; 32],
|
||||
sse_type: SSEType,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
) -> Result<[u8; 32], SseError> {
|
||||
let header = &sealed.sealed_key[..DARE_HEADER_SIZE];
|
||||
if header[0] != DARE_VERSION_20 || !is_supported_sealed_object_key_cipher(header[1]) {
|
||||
return Err(SseError::internal("Unsupported sealed object-key DARE header"));
|
||||
}
|
||||
if u16::from_le_bytes([header[2], header[3]]) != 31 || header[4] & 0x80 == 0 {
|
||||
return Err(SseError::internal("Invalid sealed object-key payload header"));
|
||||
}
|
||||
|
||||
let sealing_key = derive_sealing_key(external_key, sealed.iv, managed_sse_domain(sse_type), bucket, object)?;
|
||||
let plaintext = decrypt_sealed_object_key_payload(sealing_key, header, &sealed.sealed_key)?;
|
||||
|
||||
let object_key: [u8; 32] = plaintext
|
||||
.as_slice()
|
||||
.try_into()
|
||||
.map_err(|_| SseError::internal("Sealed object key must decrypt to 32 bytes"))?;
|
||||
Ok(object_key)
|
||||
}
|
||||
|
||||
fn try_decode_minio_sealed_key(bytes: &str) -> Result<Option<[u8; SEALED_KEY_SIZE]>, SseError> {
|
||||
let decoded = BASE64_STANDARD
|
||||
.decode_to_vec(bytes)
|
||||
.map_err(|e| SseError::internal(format!("Failed to decode sealed object key: {e}")))?;
|
||||
match decoded.as_slice().try_into() {
|
||||
Ok(sealed_key) => Ok(Some(sealed_key)),
|
||||
Err(_) => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
fn try_decode_minio_sealing_iv(bytes: &str) -> Result<Option<[u8; SEALED_KEY_IV_SIZE]>, SseError> {
|
||||
let decoded = BASE64_STANDARD
|
||||
.decode_to_vec(bytes)
|
||||
.map_err(|e| SseError::internal(format!("Failed to decode sealing IV: {e}")))?;
|
||||
match decoded.as_slice().try_into() {
|
||||
Ok(iv) => Ok(Some(iv)),
|
||||
Err(_) => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn is_legacy_rustfs_managed_metadata(metadata: &HashMap<String, String>) -> bool {
|
||||
metadata.contains_key(INTERNAL_ENCRYPTION_KEY_HEADER)
|
||||
&& metadata.contains_key(INTERNAL_ENCRYPTION_IV_HEADER)
|
||||
&& !metadata.contains_key(MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER)
|
||||
&& !metadata.contains_key(MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER)
|
||||
}
|
||||
|
||||
/// Infer the managed SSE scheme from the MinIO sealed-key slot that is present.
|
||||
///
|
||||
/// Returns `None` when no managed MinIO slot is present, which keeps callers on
|
||||
/// their fail-closed path. SSE-C is not a managed scheme and is handled by the
|
||||
/// SSE-C read path, so its slot is not considered here.
|
||||
pub(crate) fn infer_minio_managed_sse_type(metadata: &HashMap<String, String>) -> Option<SSEType> {
|
||||
if metadata.contains_key(MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER) {
|
||||
Some(SSEType::SseS3)
|
||||
} else if metadata.contains_key(MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER) {
|
||||
Some(SSEType::SseKms)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn parse_minio_managed_sealed_key(
|
||||
metadata: &HashMap<String, String>,
|
||||
sse_type: SSEType,
|
||||
) -> Result<Option<ManagedSealedKey>, SseError> {
|
||||
let algorithm = match metadata.get(MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER) {
|
||||
Some(algorithm) => algorithm,
|
||||
None => return Ok(None),
|
||||
};
|
||||
if algorithm != MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let iv = match metadata.get(MINIO_INTERNAL_ENCRYPTION_IV_HEADER) {
|
||||
Some(iv) => match try_decode_minio_sealing_iv(iv)? {
|
||||
Some(iv) => iv,
|
||||
None => return Ok(None),
|
||||
},
|
||||
None => return Ok(None),
|
||||
};
|
||||
|
||||
let header_key = match sse_type {
|
||||
SSEType::SseS3 => MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER,
|
||||
SSEType::SseKms => MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER,
|
||||
SSEType::SseC => MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER,
|
||||
};
|
||||
let sealed_key = match metadata.get(header_key) {
|
||||
Some(value) => match try_decode_minio_sealed_key(value)? {
|
||||
Some(sealed_key) => sealed_key,
|
||||
None => return Ok(None),
|
||||
},
|
||||
None => return Ok(None),
|
||||
};
|
||||
|
||||
Ok(Some(ManagedSealedKey { iv, sealed_key }))
|
||||
}
|
||||
|
||||
/// SSE-C: record the object key sealed under the customer key.
|
||||
pub(crate) fn record_sealed_customer_key(metadata: &mut HashMap<String, String>, sealed: &ManagedSealedKey) {
|
||||
metadata.insert(
|
||||
MINIO_INTERNAL_ENCRYPTION_IV_HEADER.to_string(),
|
||||
BASE64_STANDARD.encode_to_string(sealed.iv),
|
||||
);
|
||||
metadata.insert(
|
||||
MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(),
|
||||
MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM.to_string(),
|
||||
);
|
||||
metadata.insert(
|
||||
MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER.to_string(),
|
||||
BASE64_STANDARD.encode_to_string(sealed.sealed_key),
|
||||
);
|
||||
}
|
||||
|
||||
/// Managed SSE: record the wrapped data key in MinIO's slots.
|
||||
///
|
||||
/// With a sealed object key the slots hold what MinIO itself writes. Without one
|
||||
/// the key was formed in Direct mode, and the slots mirror its wrapped data key and
|
||||
/// nonce: [`parse_minio_managed_sealed_key`] turns that shape down by its length, so
|
||||
/// the object is read back in Direct mode.
|
||||
pub(crate) fn record_data_key_in_minio_slots(
|
||||
metadata: &mut HashMap<String, String>,
|
||||
material: &EncryptionMaterial,
|
||||
wrapped_data_key: &[u8],
|
||||
) {
|
||||
metadata.insert(
|
||||
MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(),
|
||||
MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM.to_string(),
|
||||
);
|
||||
if let Some(kms_key_id) = &material.kms_key_id {
|
||||
metadata.insert(MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER.to_string(), kms_key_id.to_string());
|
||||
}
|
||||
|
||||
let encoded_data_key = BASE64_STANDARD.encode_to_string(wrapped_data_key);
|
||||
match &material.managed_sealed_key {
|
||||
Some(sealed) => {
|
||||
metadata.insert(
|
||||
MINIO_INTERNAL_ENCRYPTION_IV_HEADER.to_string(),
|
||||
BASE64_STANDARD.encode_to_string(sealed.iv),
|
||||
);
|
||||
match material.sse_type {
|
||||
SSEType::SseS3 => {
|
||||
metadata.insert(
|
||||
MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER.to_string(),
|
||||
BASE64_STANDARD.encode_to_string(sealed.sealed_key),
|
||||
);
|
||||
}
|
||||
SSEType::SseKms => {
|
||||
metadata.insert(
|
||||
MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER.to_string(),
|
||||
BASE64_STANDARD.encode_to_string(sealed.sealed_key),
|
||||
);
|
||||
}
|
||||
SSEType::SseC => {}
|
||||
}
|
||||
metadata.insert(MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER.to_string(), encoded_data_key);
|
||||
}
|
||||
None => {
|
||||
metadata.insert(
|
||||
MINIO_INTERNAL_ENCRYPTION_IV_HEADER.to_string(),
|
||||
BASE64_STANDARD.encode_to_string(material.base_nonce),
|
||||
);
|
||||
match material.sse_type {
|
||||
SSEType::SseS3 => {
|
||||
metadata.insert(MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER.to_string(), encoded_data_key);
|
||||
}
|
||||
SSEType::SseKms => {
|
||||
metadata.insert(MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER.to_string(), encoded_data_key.clone());
|
||||
metadata.insert(MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER.to_string(), encoded_data_key);
|
||||
}
|
||||
SSEType::SseC => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Managed SSE: read back the wrapped data key that seals `sealed_key`.
|
||||
///
|
||||
/// `normalized_metadata` is `metadata` with the managed keys normalized.
|
||||
pub(crate) fn read_sealed_data_key(
|
||||
metadata: &HashMap<String, String>,
|
||||
normalized_metadata: &HashMap<String, String>,
|
||||
sealed_key: ManagedSealedKey,
|
||||
) -> Result<StoredDataKey, SseError> {
|
||||
let encrypted_key_b64 = normalized_metadata
|
||||
.get(INTERNAL_ENCRYPTION_KEY_HEADER)
|
||||
.or_else(|| metadata.get(MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER))
|
||||
.ok_or_else(|| SseError::internal("Missing encrypted key in metadata"))?;
|
||||
let wrapped = BASE64_STANDARD
|
||||
.decode_to_vec(encrypted_key_b64)
|
||||
.map_err(|e| SseError::internal(format!("Failed to decode encrypted key: {e}")))?;
|
||||
|
||||
Ok(StoredDataKey {
|
||||
wrapped,
|
||||
algorithm: normalized_metadata
|
||||
.get(INTERNAL_ENCRYPTION_ALGORITHM_HEADER)
|
||||
.cloned()
|
||||
.unwrap_or_else(|| "AES256".to_string()),
|
||||
mode: StoredKeyMode::Object { sealed_key },
|
||||
})
|
||||
}
|
||||
|
||||
/// Whether a multipart session needs the customer key when it is created.
|
||||
///
|
||||
/// Object mode seals the session's object key under the customer key, so the key
|
||||
/// has to be present, and valid, at that point.
|
||||
pub(crate) const SESSION_KEY_NEEDS_CUSTOMER_KEY: bool = true;
|
||||
|
||||
/// Derive a fresh object key from `external_key` and seal it under the same key.
|
||||
fn new_sealed_object_key(external_key: [u8; 32], sse_type: SSEType, bucket: &str, key: &str) -> Result<NewObjectKey, SseError> {
|
||||
let object_key = derive_object_key(external_key)?;
|
||||
let sealed_key = seal_object_key(object_key, external_key, sse_type, bucket, key)?;
|
||||
Ok(NewObjectKey {
|
||||
key_bytes: object_key,
|
||||
base_nonce: [0; 12],
|
||||
key_kind: EncryptionKeyKind::Object,
|
||||
sealed_key: Some(sealed_key),
|
||||
})
|
||||
}
|
||||
|
||||
/// SSE-C object: an object key sealed under the customer key.
|
||||
pub(crate) fn new_key_under_customer_key(customer_key: [u8; 32], bucket: &str, key: &str) -> Result<NewObjectKey, SseError> {
|
||||
new_sealed_object_key(customer_key, SSEType::SseC, bucket, key)
|
||||
}
|
||||
|
||||
/// SSE-C multipart session: an object key sealed under the customer key.
|
||||
///
|
||||
/// A session created without its customer key has nothing to seal and records a
|
||||
/// placeholder instead.
|
||||
pub(crate) fn new_session_key_under_customer_key(
|
||||
customer_key: Option<[u8; 32]>,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
) -> Result<NewObjectKey, SseError> {
|
||||
match customer_key {
|
||||
Some(customer_key) => new_sealed_object_key(customer_key, SSEType::SseC, bucket, key),
|
||||
None => Ok(NewObjectKey {
|
||||
key_bytes: [0; 32],
|
||||
base_nonce: [0; 12],
|
||||
key_kind: EncryptionKeyKind::Direct,
|
||||
sealed_key: None,
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
/// Managed SSE: an object key sealed under the data key.
|
||||
pub(crate) fn new_key_under_data_key(
|
||||
data_key: &DataKey,
|
||||
sse_type: SSEType,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
) -> Result<NewObjectKey, SseError> {
|
||||
new_sealed_object_key(data_key.plaintext_key, sse_type, bucket, key)
|
||||
}
|
||||
@@ -0,0 +1,253 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Reading: recognize how a stored object was keyed, and recover its key.
|
||||
//!
|
||||
//! What an object recorded is the only source of truth here. The scheme a request
|
||||
//! advertises and whether a KMS is running now are write policy and runtime state;
|
||||
//! neither says how an object written earlier was keyed.
|
||||
|
||||
use crate::core::keying::context::{build_object_encryption_context, decode_minio_kms_context};
|
||||
use crate::core::keying::direct::{read_stored_ssec_nonce, read_wrapped_data_key};
|
||||
use crate::core::keying::material::EncryptionKeyKind;
|
||||
use crate::core::keying::metadata::recode_minio_kms_context;
|
||||
#[cfg(feature = "rio-v2")]
|
||||
use crate::core::keying::object::{
|
||||
infer_minio_managed_sse_type, is_legacy_rustfs_managed_metadata, parse_minio_managed_sealed_key, read_sealed_data_key,
|
||||
unseal_object_key,
|
||||
};
|
||||
use crate::core::keying::{StoredDataKey, StoredKeyMode};
|
||||
use crate::error::SseError;
|
||||
use crate::host::SseHost;
|
||||
use crate::kms::local::is_local_sse_dek;
|
||||
#[cfg(feature = "rio-v2")]
|
||||
use crate::kms::local::is_rustfs_managed_data_key;
|
||||
#[cfg(feature = "rio-v2")]
|
||||
use crate::kms::provider::decrypt_unenveloped_sealed_dek;
|
||||
use crate::kms::provider::{SseDekProvider, bare_data_key_kms_provider, get_local_sse_dek_provider, kms_sse_dek_provider};
|
||||
use rustfs_kms::is_data_key_envelope;
|
||||
use rustfs_utils::http::headers::AMZ_ENCRYPTION_KMS;
|
||||
use rustfs_utils::http::object_encryption_keys::{
|
||||
INTERNAL_ENCRYPTION_KEY_ID_HEADER, SSEType, contains_managed_encryption_metadata, normalize_managed_metadata,
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
/// The key a stored object is decrypted with, recovered from what the object recorded.
|
||||
pub(crate) struct OpenedObjectKey {
|
||||
/// Key handed to the stream cipher.
|
||||
pub(crate) key_bytes: [u8; 32],
|
||||
/// Base nonce the stream cipher derives its block nonces from.
|
||||
pub(crate) base_nonce: [u8; 12],
|
||||
/// The mode the object was written in.
|
||||
pub(crate) key_kind: EncryptionKeyKind,
|
||||
}
|
||||
|
||||
/// The managed scheme a stored object was written under, or `None` when this build
|
||||
/// cannot establish it.
|
||||
fn stored_managed_sse_type(metadata: &HashMap<String, String>) -> Option<SSEType> {
|
||||
match metadata.get("x-amz-server-side-encryption").map(String::as_str) {
|
||||
Some(AMZ_ENCRYPTION_KMS) => Some(SSEType::SseKms),
|
||||
Some(_) => Some(SSEType::SseS3),
|
||||
// MinIO never persists the public scheme header: `crypto.S3.CreateMetadata`
|
||||
// writes only the `X-Minio-Internal-*` family and the public header is
|
||||
// synthesized onto the response by `DecryptObjectInfo`. Requiring it here
|
||||
// is what made every MinIO-encrypted object unreadable (backlog#1638).
|
||||
//
|
||||
// Inferring from the sealed-key slot is self-consistent by construction:
|
||||
// the slot decides which header the unseal reads AND which domain string
|
||||
// the sealing key is derived under, so a scheme that disagrees with the
|
||||
// slot cannot silently derive a wrong key — it finds no key at all.
|
||||
// Inferring from the KMS key id would NOT be safe: MinIO writes
|
||||
// `-S3-Kms-Key-Id` on SSE-S3 objects too.
|
||||
//
|
||||
// An object whose scheme cannot be established stays fail-closed, and
|
||||
// deliberately not through an error raised here: the read plan independently
|
||||
// classifies the object as encrypted from its markers and refuses to serve
|
||||
// it without material, so it never degrades into a plaintext read.
|
||||
#[cfg(feature = "rio-v2")]
|
||||
None => infer_minio_managed_sse_type(metadata),
|
||||
// Without the rio-v2 reader there is no MinIO-format read path to serve
|
||||
// such an object with, so it stays on the fail-closed branch.
|
||||
#[cfg(not(feature = "rio-v2"))]
|
||||
None => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// SSE-C: recover the object's key from the customer key.
|
||||
///
|
||||
/// A sealed object key recorded in MinIO's slot is unsealed under the customer key;
|
||||
/// otherwise the customer key is the object's key, under the nonce the object recorded.
|
||||
pub(crate) fn open_under_customer_key(
|
||||
metadata: &HashMap<String, String>,
|
||||
customer_key: [u8; 32],
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
) -> Result<OpenedObjectKey, SseError> {
|
||||
#[cfg(feature = "rio-v2")]
|
||||
if let Some(sealed_key) = parse_minio_managed_sealed_key(metadata, SSEType::SseC)? {
|
||||
return Ok(OpenedObjectKey {
|
||||
key_bytes: unseal_object_key(&sealed_key, customer_key, SSEType::SseC, bucket, key)?,
|
||||
base_nonce: [0; 12],
|
||||
key_kind: EncryptionKeyKind::Object,
|
||||
});
|
||||
}
|
||||
|
||||
Ok(OpenedObjectKey {
|
||||
key_bytes: customer_key,
|
||||
base_nonce: read_stored_ssec_nonce(metadata, bucket, key),
|
||||
key_kind: EncryptionKeyKind::Direct,
|
||||
})
|
||||
}
|
||||
|
||||
/// A stored managed-SSE object, as recognized from its metadata alone.
|
||||
pub(crate) struct StoredManagedKey<'a> {
|
||||
/// The object's metadata as stored.
|
||||
metadata: &'a HashMap<String, String>,
|
||||
/// The same metadata with the managed keys normalized.
|
||||
normalized_metadata: HashMap<String, String>,
|
||||
/// The scheme the object was written under.
|
||||
pub(crate) sse_type: SSEType,
|
||||
/// The KMS key id the object recorded, if it recorded one.
|
||||
pub(crate) recorded_kms_key_id: Option<String>,
|
||||
}
|
||||
|
||||
/// Recognize a stored managed-SSE object.
|
||||
///
|
||||
/// `None` means the object carries no managed-SSE metadata, or carries some whose
|
||||
/// scheme this build cannot establish.
|
||||
pub(crate) fn recognize_stored_managed_key(metadata: &HashMap<String, String>) -> Option<StoredManagedKey<'_>> {
|
||||
if !contains_managed_encryption_metadata(metadata) {
|
||||
return None;
|
||||
}
|
||||
let sse_type = stored_managed_sse_type(metadata)?;
|
||||
|
||||
let normalized_metadata = normalize_managed_metadata(metadata, Some(recode_minio_kms_context));
|
||||
let recorded_kms_key_id = normalized_metadata
|
||||
.get(INTERNAL_ENCRYPTION_KEY_ID_HEADER)
|
||||
.or_else(|| metadata.get("x-amz-server-side-encryption-aws-kms-key-id"))
|
||||
.cloned();
|
||||
|
||||
Some(StoredManagedKey {
|
||||
metadata,
|
||||
normalized_metadata,
|
||||
sse_type,
|
||||
recorded_kms_key_id,
|
||||
})
|
||||
}
|
||||
|
||||
/// Managed SSE: unwrap the stored data key and recover the object's key from it.
|
||||
///
|
||||
/// `kms_key_id` is the key the read was authorized against. Returns the object's key
|
||||
/// together with the algorithm the object recorded.
|
||||
pub(crate) async fn open_under_stored_data_key(
|
||||
host: &dyn SseHost,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
stored: &StoredManagedKey<'_>,
|
||||
kms_key_id: &str,
|
||||
) -> Result<(OpenedObjectKey, String), SseError> {
|
||||
let record = read_stored_data_key(host, stored).await?;
|
||||
let data_key = unwrap_stored_data_key(host, bucket, key, stored, &record, kms_key_id).await?;
|
||||
|
||||
let opened = match record.mode {
|
||||
StoredKeyMode::Direct { base_nonce } => OpenedObjectKey {
|
||||
key_bytes: data_key,
|
||||
base_nonce,
|
||||
key_kind: EncryptionKeyKind::Direct,
|
||||
},
|
||||
#[cfg(feature = "rio-v2")]
|
||||
StoredKeyMode::Object { sealed_key } => OpenedObjectKey {
|
||||
key_bytes: unseal_object_key(&sealed_key, data_key, stored.sse_type, bucket, key)?,
|
||||
base_nonce: [0u8; 12],
|
||||
key_kind: EncryptionKeyKind::Object,
|
||||
},
|
||||
};
|
||||
|
||||
Ok((opened, record.algorithm))
|
||||
}
|
||||
|
||||
/// Read the stored data key in the mode the object recorded.
|
||||
///
|
||||
/// This is the recognition order: a mode listed earlier claims the object first, and
|
||||
/// Direct takes whatever no other mode recognizes. A mode this build cannot read is
|
||||
/// not listed, so its objects fail in the Direct reader instead of being misread.
|
||||
async fn read_stored_data_key(host: &dyn SseHost, stored: &StoredManagedKey<'_>) -> Result<StoredDataKey, SseError> {
|
||||
#[cfg(feature = "rio-v2")]
|
||||
if let Some(sealed_key) = parse_minio_managed_sealed_key(stored.metadata, stored.sse_type)? {
|
||||
return read_sealed_data_key(stored.metadata, &stored.normalized_metadata, sealed_key);
|
||||
}
|
||||
|
||||
read_wrapped_data_key(host, &stored.normalized_metadata).await
|
||||
}
|
||||
|
||||
/// Unwrap the stored data key with the provider its own bytes identify.
|
||||
async fn unwrap_stored_data_key(
|
||||
host: &dyn SseHost,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
stored: &StoredManagedKey<'_>,
|
||||
record: &StoredDataKey,
|
||||
kms_key_id: &str,
|
||||
) -> Result<[u8; 32], SseError> {
|
||||
let kms_context = if matches!(stored.sse_type, SSEType::SseKms) {
|
||||
decode_minio_kms_context(stored.metadata)?
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let object_context = build_object_encryption_context(bucket, key, kms_context.as_ref());
|
||||
let wrapped = record.wrapped.as_slice();
|
||||
|
||||
// Persisted wrapping format is the read-side source of truth. The
|
||||
// advertised SSE scheme and current KMS availability are write policy
|
||||
// and runtime state, neither of which identifies the historical provider.
|
||||
//
|
||||
// The one shape that cannot name its provider is a backend-native blob
|
||||
// stored without an envelope. It is offered to the KMS only when the
|
||||
// local provider could not have written those bytes and the active
|
||||
// backend is one whose released versions persisted them that way.
|
||||
let bare_kms = if is_data_key_envelope(wrapped) || is_local_sse_dek(wrapped) {
|
||||
None
|
||||
} else {
|
||||
bare_data_key_kms_provider(host).await
|
||||
};
|
||||
|
||||
// A MinIO sealed key alone does not mean MinIO wrote the object: RustFS's own
|
||||
// writer fills MinIO's metadata slots too, while still storing a RustFS
|
||||
// envelope in them, so neither the slot nor the header name distinguishes the
|
||||
// two. The data key's own shape does. RustFS envelopes are strictly-parsed
|
||||
// JSON; MinIO's builtin-KMS ciphertext is opaque bytes that match neither, so
|
||||
// recognizing RustFS positively — and treating only the remainder as MinIO —
|
||||
// keeps a RustFS envelope from ever reaching MinIO's decoder.
|
||||
#[cfg(feature = "rio-v2")]
|
||||
if matches!(record.mode, StoredKeyMode::Object { .. }) && !is_rustfs_managed_data_key(wrapped) {
|
||||
return decrypt_unenveloped_sealed_dek(bare_kms, get_local_sse_dek_provider(), wrapped, kms_key_id, &object_context)
|
||||
.await;
|
||||
}
|
||||
|
||||
let provider: Arc<dyn SseDekProvider> = if is_data_key_envelope(wrapped) {
|
||||
kms_sse_dek_provider(host).await?
|
||||
} else if let Some(kms_provider) = bare_kms {
|
||||
kms_provider
|
||||
} else {
|
||||
get_local_sse_dek_provider()?
|
||||
};
|
||||
|
||||
#[cfg(feature = "rio-v2")]
|
||||
if is_legacy_rustfs_managed_metadata(&stored.normalized_metadata) {
|
||||
return provider.decrypt_legacy_sse_dek(wrapped, kms_key_id, &object_context).await;
|
||||
}
|
||||
|
||||
provider.decrypt_sse_dek(wrapped, kms_key_id, &object_context).await
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! SSE key resolution: the S3-defined schemes, the internal keying modes, and the entry points that join them.
|
||||
|
||||
pub(crate) mod audit;
|
||||
pub(crate) mod keying;
|
||||
pub(crate) mod read;
|
||||
pub(crate) mod request;
|
||||
pub(crate) mod rewrap;
|
||||
pub(crate) mod s3;
|
||||
pub(crate) mod write;
|
||||
@@ -0,0 +1,303 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Read entry points: resolve the decryption material, or the response headers, for a stored object.
|
||||
|
||||
use crate::core::audit::{record_managed_kms_outcome, stored_envelope_master_key_version};
|
||||
use crate::core::keying::material::DecryptionMaterial;
|
||||
use crate::core::keying::metadata::stored_ssec_metadata;
|
||||
use crate::core::keying::stored::{
|
||||
StoredManagedKey, open_under_customer_key, open_under_stored_data_key, recognize_stored_managed_key,
|
||||
};
|
||||
use crate::core::request::SseKmsPrincipal;
|
||||
use crate::core::s3::authorization::authorize_sse_kms_key;
|
||||
use crate::core::s3::headers::{SseReadResponseHeaders, managed_sse_public_header};
|
||||
use crate::core::s3::scheme::resolve_stored_kms_key_id;
|
||||
use crate::core::s3::ssec::{SsecParams, require_matching_ssec_key, stored_ssec_algorithm, validate_ssec_params};
|
||||
use crate::error::SseError;
|
||||
use crate::host::SseHost;
|
||||
use rustfs_policy::policy::action::KmsAction;
|
||||
use rustfs_utils::http::headers::AMZ_ENCRYPTION_AES;
|
||||
use rustfs_utils::http::object_encryption_keys::{SSEType, contains_managed_encryption_metadata, stored_managed_encryption_key};
|
||||
use std::collections::HashMap;
|
||||
|
||||
impl std::fmt::Debug for DecryptionRequest<'_> {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("DecryptionRequest").finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
|
||||
/// Request parameters for unified decryption
|
||||
pub struct DecryptionRequest<'a> {
|
||||
/// Bucket name
|
||||
pub bucket: &'a str,
|
||||
/// Object key
|
||||
pub key: &'a str,
|
||||
/// Object metadata containing encryption headers
|
||||
pub metadata: &'a HashMap<String, String>,
|
||||
/// SSE-C key (Base64-encoded) - required if object was encrypted with SSE-C
|
||||
pub sse_customer_key: Option<&'a String>,
|
||||
/// SSE-C key MD5 (Base64-encoded) - required if object was encrypted with SSE-C
|
||||
pub sse_customer_key_md5: Option<&'a String>,
|
||||
/// Caller the SSE-KMS key usage is authorized as. `None` marks an internal caller.
|
||||
pub principal: Option<&'a SseKmsPrincipal>,
|
||||
}
|
||||
|
||||
/// **Core API**: Apply decryption based on stored metadata
|
||||
///
|
||||
/// This function automatically detects the encryption type from metadata:
|
||||
/// - SSE-C if customer key is provided
|
||||
/// - SSE-S3/SSE-KMS if managed encryption metadata is found
|
||||
/// - None if object is not encrypted
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `request` - Decryption request with metadata and optional customer key
|
||||
///
|
||||
/// # Returns
|
||||
/// * `Ok(Some(material))` - Decryption should be applied with the returned material
|
||||
/// * `Ok(None)` - Object is not encrypted
|
||||
/// * `Err` - Decryption configuration error or key mismatch
|
||||
///
|
||||
/// # Example
|
||||
/// ```rust,ignore
|
||||
/// let request = DecryptionRequest {
|
||||
/// bucket: &bucket,
|
||||
/// key: &key,
|
||||
/// metadata: &metadata,
|
||||
/// sse_customer_key: sse_customer_key.as_deref(),
|
||||
/// sse_customer_key_md5: sse_customer_key_md5.as_deref(),
|
||||
/// };
|
||||
///
|
||||
/// if let Some(material) = sse_decryption(host, request).await? {
|
||||
/// content_size = material.original_size.unwrap_or(actual_size);
|
||||
/// }
|
||||
/// ```
|
||||
pub async fn sse_decryption(host: &dyn SseHost, request: DecryptionRequest<'_>) -> Result<Option<DecryptionMaterial>, SseError> {
|
||||
// Check for SSE-C encryption
|
||||
if stored_ssec_metadata(request.metadata) {
|
||||
let (key, key_md5) = require_matching_ssec_key(request.metadata, request.sse_customer_key, request.sse_customer_key_md5)?;
|
||||
|
||||
let mut material = apply_ssec_decryption_material(request.bucket, request.key, request.metadata, key, key_md5).await?;
|
||||
material.customer_key_md5 = Some(key_md5.clone());
|
||||
return Ok(Some(material));
|
||||
}
|
||||
|
||||
// Check for managed SSE encryption
|
||||
if contains_managed_encryption_metadata(request.metadata) {
|
||||
return apply_managed_decryption_material(host, request.bucket, request.key, request.metadata, request.principal).await;
|
||||
}
|
||||
|
||||
// No encryption detected
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
/// Read-side response classification for an object whose payload the object
|
||||
/// layer's encryption resolver already decrypted.
|
||||
///
|
||||
/// [`sse_decryption`] both unwraps the data key and derives the response
|
||||
/// headers. The GET path only needs the latter — its stream comes out of the
|
||||
/// object layer decrypted (see [`SseObjectEncryptionResolver`]) — so calling
|
||||
/// [`sse_decryption`] there performed a second KMS `Decrypt` per request whose
|
||||
/// key bytes were discarded. This function reproduces that call's header,
|
||||
/// validation, authorization and audit behavior from stored metadata alone.
|
||||
///
|
||||
/// Contract, mirrored from [`sse_decryption`] item by item:
|
||||
/// - SSE-C: missing request key/MD5 and stored/request MD5 mismatch fail with
|
||||
/// the same errors and precedence, and the provided key is fully validated.
|
||||
/// - Managed SSE: per-key `kms:Decrypt` authorization runs ahead of every other
|
||||
/// failure mode, and the request's KMS audit summary records the same
|
||||
/// scheme/key/version/outcome fields the unwrap-based path recorded. The
|
||||
/// success outcome is honest because a failed unwrap aborts the read in the
|
||||
/// object layer before response classification is ever reached.
|
||||
/// - Objects without SSE metadata — and managed metadata whose scheme cannot
|
||||
/// be established in this build — classify as `None`, exactly where
|
||||
/// [`sse_decryption`] returned `None`.
|
||||
pub async fn classify_sse_read_response(
|
||||
host: &dyn SseHost,
|
||||
request: DecryptionRequest<'_>,
|
||||
) -> Result<Option<SseReadResponseHeaders>, SseError> {
|
||||
if stored_ssec_metadata(request.metadata) {
|
||||
let (key, key_md5) = require_matching_ssec_key(request.metadata, request.sse_customer_key, request.sse_customer_key_md5)?;
|
||||
|
||||
let algorithm = stored_ssec_algorithm(request.metadata);
|
||||
validate_ssec_params(SsecParams {
|
||||
algorithm: algorithm.to_string(),
|
||||
key: key.to_string(),
|
||||
key_md5: key_md5.to_string(),
|
||||
})?;
|
||||
|
||||
return Ok(Some(SseReadResponseHeaders {
|
||||
server_side_encryption: String::from(AMZ_ENCRYPTION_AES),
|
||||
sse_customer_algorithm: Some(String::from(algorithm)),
|
||||
sse_customer_key_md5: Some(key_md5.clone()),
|
||||
ssekms_key_id: None,
|
||||
}));
|
||||
}
|
||||
|
||||
// Same scheme and key resolution as the unwrap path, so authorization and the
|
||||
// response header name the same key.
|
||||
let Some((stored, kms_key_id)) = resolve_stored_managed_key(host, request.metadata).await else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
// Ahead of every other failure mode, so a denied caller learns nothing
|
||||
// about the key beyond "not yours".
|
||||
if let Err(error) =
|
||||
authorize_sse_kms_key(host, request.principal, stored.sse_type, KmsAction::DecryptAction, &kms_key_id).await
|
||||
{
|
||||
if let Some((sse_type, key_id)) = stored_managed_encryption_key(request.metadata) {
|
||||
record_managed_kms_outcome(
|
||||
request.principal,
|
||||
sse_type,
|
||||
Some(&key_id),
|
||||
|| stored_envelope_master_key_version(request.metadata),
|
||||
Err(&error),
|
||||
);
|
||||
}
|
||||
return Err(error);
|
||||
}
|
||||
|
||||
record_managed_kms_outcome(
|
||||
request.principal,
|
||||
stored.sse_type,
|
||||
Some(&kms_key_id),
|
||||
|| stored_envelope_master_key_version(request.metadata),
|
||||
Ok(()),
|
||||
);
|
||||
|
||||
Ok(Some(SseReadResponseHeaders {
|
||||
server_side_encryption: managed_sse_public_header(stored.sse_type).to_string(),
|
||||
sse_customer_algorithm: None,
|
||||
sse_customer_key_md5: None,
|
||||
// The key id was needed above to authorize the read, but an AES256
|
||||
// object's wrapping key is internal: only aws:kms objects advertise it.
|
||||
ssekms_key_id: matches!(stored.sse_type, SSEType::SseKms).then_some(kms_key_id),
|
||||
}))
|
||||
}
|
||||
|
||||
/// Recognize a stored managed-SSE object and name the key it is read under.
|
||||
///
|
||||
/// One resolution for every read: authorization, the response headers and the
|
||||
/// unwrap all name the key this returns. `None` means the object carries no
|
||||
/// managed-SSE metadata, or carries some whose scheme this build cannot establish.
|
||||
async fn resolve_stored_managed_key<'m>(
|
||||
host: &dyn SseHost,
|
||||
metadata: &'m HashMap<String, String>,
|
||||
) -> Option<(StoredManagedKey<'m>, String)> {
|
||||
let stored = recognize_stored_managed_key(metadata)?;
|
||||
let kms_key_id = resolve_stored_kms_key_id(host, stored.recorded_kms_key_id.clone()).await;
|
||||
Some((stored, kms_key_id))
|
||||
}
|
||||
|
||||
pub(crate) async fn apply_ssec_decryption_material(
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
metadata: &HashMap<String, String>,
|
||||
sse_key: &str,
|
||||
sse_key_md5: &str,
|
||||
) -> Result<DecryptionMaterial, SseError> {
|
||||
// Validate provided key
|
||||
let algorithm = stored_ssec_algorithm(metadata);
|
||||
|
||||
let params = SsecParams {
|
||||
algorithm: algorithm.to_string(),
|
||||
key: sse_key.to_string(),
|
||||
key_md5: sse_key_md5.to_string(),
|
||||
};
|
||||
|
||||
let validated = validate_ssec_params(params)?;
|
||||
let opened = open_under_customer_key(metadata, validated.key_bytes, bucket, key)?;
|
||||
|
||||
Ok(DecryptionMaterial {
|
||||
sse_type: SSEType::SseC,
|
||||
server_side_encryption: String::from(AMZ_ENCRYPTION_AES), // const
|
||||
kms_key_id: None,
|
||||
algorithm: String::from(algorithm),
|
||||
|
||||
customer_key_md5: None,
|
||||
key_bytes: opened.key_bytes,
|
||||
base_nonce: opened.base_nonce,
|
||||
key_kind: opened.key_kind,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) async fn apply_managed_decryption_material(
|
||||
host: &dyn SseHost,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
metadata: &HashMap<String, String>,
|
||||
principal: Option<&SseKmsPrincipal>,
|
||||
) -> Result<Option<DecryptionMaterial>, SseError> {
|
||||
let result = apply_managed_decryption_material_inner(host, bucket, key, metadata, principal).await;
|
||||
|
||||
match &result {
|
||||
// `None` means the object carries no managed-SSE metadata — SSE-C and
|
||||
// plaintext objects never reach KMS and must not appear in the summary.
|
||||
Ok(None) => {}
|
||||
Ok(Some(material)) => record_managed_kms_outcome(
|
||||
principal,
|
||||
material.sse_type,
|
||||
material.kms_key_id.as_deref(),
|
||||
|| stored_envelope_master_key_version(metadata),
|
||||
Ok(()),
|
||||
),
|
||||
Err(error) => {
|
||||
if let Some((sse_type, key_id)) = stored_managed_encryption_key(metadata) {
|
||||
record_managed_kms_outcome(
|
||||
principal,
|
||||
sse_type,
|
||||
Some(&key_id),
|
||||
|| stored_envelope_master_key_version(metadata),
|
||||
Err(error),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
result
|
||||
}
|
||||
|
||||
async fn apply_managed_decryption_material_inner(
|
||||
host: &dyn SseHost,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
metadata: &HashMap<String, String>,
|
||||
principal: Option<&SseKmsPrincipal>,
|
||||
) -> Result<Option<DecryptionMaterial>, SseError> {
|
||||
let Some((stored, kms_key_id)) = resolve_stored_managed_key(host, metadata).await else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
// Ahead of every other failure mode below, so a denied caller learns nothing about the
|
||||
// key beyond "not yours" — not whether it is disabled, pending deletion, or unreadable.
|
||||
authorize_sse_kms_key(host, principal, stored.sse_type, KmsAction::DecryptAction, &kms_key_id).await?;
|
||||
|
||||
let (opened, algorithm) = open_under_stored_data_key(host, bucket, key, &stored, &kms_key_id).await?;
|
||||
|
||||
Ok(Some(DecryptionMaterial {
|
||||
sse_type: stored.sse_type,
|
||||
// Synthesized from the resolved scheme rather than read back from
|
||||
// metadata: a MinIO-written object has no stored scheme header, which is
|
||||
// exactly why the scheme had to be inferred. MinIO synthesizes the same
|
||||
// header onto its own responses.
|
||||
server_side_encryption: managed_sse_public_header(stored.sse_type).to_string(),
|
||||
kms_key_id: Some(kms_key_id),
|
||||
algorithm,
|
||||
customer_key_md5: None,
|
||||
|
||||
key_bytes: opened.key_bytes,
|
||||
base_nonce: opened.base_nonce,
|
||||
key_kind: opened.key_kind,
|
||||
}))
|
||||
}
|
||||
@@ -0,0 +1,148 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! The caller an SSE operation is authorized as.
|
||||
|
||||
use crate::core::audit::{KmsRequestAudit, kms_request_audit};
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
use crate::host::KmsKeyAuthorizer;
|
||||
use serde_json::Value;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
/// Identity a data-path KMS key operation is attributed to.
|
||||
///
|
||||
/// Only requests that crossed the S3 authorization boundary carry one. Internal
|
||||
/// callers — replication, lifecycle transition, heal, scanner — enter the object
|
||||
/// layer directly and run with server credentials, so they never build a principal
|
||||
/// and are exempt by construction; `None` therefore means "system principal".
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct SseKmsPrincipal {
|
||||
pub(crate) account: String,
|
||||
groups: Option<Vec<String>>,
|
||||
is_owner: bool,
|
||||
claims: HashMap<String, Value>,
|
||||
conditions: HashMap<String, Vec<String>>,
|
||||
/// Audit slot of the S3 request this principal was built for, when the request
|
||||
/// is being audited. The principal is the only request-derived value that reaches
|
||||
/// the managed-SSE code paths, so it doubles as the carrier for the slot.
|
||||
pub(crate) request_audit: Option<Arc<KmsRequestAudit>>,
|
||||
/// Test-only decision overrides. Carried per principal rather than in a global slot so
|
||||
/// concurrent tests cannot observe each other's injection.
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub(crate) test_hooks: Option<TestAuthorizationHooks>,
|
||||
}
|
||||
|
||||
impl SseKmsPrincipal {
|
||||
/// Build the principal of a request that crossed the S3 authorization boundary.
|
||||
///
|
||||
/// Anonymous requests use an empty account, matching the bucket-policy authorization
|
||||
/// path. This keeps every request that crossed the S3 boundary distinct from trusted
|
||||
/// internal callers, which are represented by `None`.
|
||||
pub fn new(
|
||||
account: String,
|
||||
groups: Option<Vec<String>>,
|
||||
is_owner: bool,
|
||||
claims: HashMap<String, Value>,
|
||||
conditions: HashMap<String, Vec<String>>,
|
||||
) -> Self {
|
||||
Self {
|
||||
account,
|
||||
groups,
|
||||
is_owner,
|
||||
claims,
|
||||
conditions,
|
||||
request_audit: None,
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
test_hooks: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// The access key the request authenticated with; empty for an anonymous request.
|
||||
pub fn account(&self) -> &str {
|
||||
&self.account
|
||||
}
|
||||
|
||||
pub fn groups(&self) -> &Option<Vec<String>> {
|
||||
&self.groups
|
||||
}
|
||||
|
||||
pub fn is_owner(&self) -> bool {
|
||||
self.is_owner
|
||||
}
|
||||
|
||||
pub fn claims(&self) -> &HashMap<String, Value> {
|
||||
&self.claims
|
||||
}
|
||||
|
||||
/// Policy condition values of the request that do not depend on the resource.
|
||||
pub fn conditions(&self) -> &HashMap<String, Vec<String>> {
|
||||
&self.conditions
|
||||
}
|
||||
|
||||
/// Attach the audit slot opened for `request_id`, if that request is being audited
|
||||
/// (see [`KmsRequestAuditScope::register`]).
|
||||
#[must_use]
|
||||
pub fn with_request_id(mut self, request_id: &str) -> Self {
|
||||
self.request_audit = kms_request_audit(request_id);
|
||||
self
|
||||
}
|
||||
|
||||
/// Override the enforcement switch and the authorizer for this principal only.
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
#[must_use]
|
||||
pub fn with_test_hooks(mut self, hooks: Option<TestAuthorizationHooks>) -> Self {
|
||||
self.test_hooks = hooks;
|
||||
self
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn for_test(account: &str, enforced: bool, authorizer: Arc<dyn KmsKeyAuthorizer>) -> Self {
|
||||
Self {
|
||||
account: account.to_string(),
|
||||
groups: None,
|
||||
is_owner: false,
|
||||
claims: HashMap::new(),
|
||||
conditions: HashMap::new(),
|
||||
request_audit: None,
|
||||
test_hooks: Some(TestAuthorizationHooks { enforced, authorizer }),
|
||||
}
|
||||
}
|
||||
|
||||
/// Bind this principal to `audit` so managed-SSE operations performed for it are
|
||||
/// summarised onto the request's S3 audit entry.
|
||||
#[cfg(test)]
|
||||
pub(crate) fn with_request_audit(mut self, audit: Arc<KmsRequestAudit>) -> Self {
|
||||
self.request_audit = Some(audit);
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
/// Per-principal override of SSE-KMS key authorization, for tests.
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
#[derive(Clone)]
|
||||
pub struct TestAuthorizationHooks {
|
||||
pub enforced: bool,
|
||||
pub authorizer: Arc<dyn KmsKeyAuthorizer>,
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
impl std::fmt::Debug for TestAuthorizationHooks {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
formatter
|
||||
.debug_struct("TestAuthorizationHooks")
|
||||
.field("enforced", &self.enforced)
|
||||
.finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Re-wrap a stored data key onto its master key's current version.
|
||||
|
||||
use crate::core::keying::context::{build_object_encryption_context, decode_minio_kms_context};
|
||||
use crate::core::keying::metadata::recode_minio_kms_context;
|
||||
use crate::error::SseError;
|
||||
use crate::host::SseHost;
|
||||
use crate::kms::provider::kms_sse_dek_provider;
|
||||
use base64_simd::STANDARD as BASE64_STANDARD;
|
||||
use rustfs_kms::is_data_key_envelope;
|
||||
use rustfs_utils::http::headers::AMZ_ENCRYPTION_KMS;
|
||||
use rustfs_utils::http::object_encryption_keys::{
|
||||
INTERNAL_ENCRYPTION_KEY_HEADER, MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER, MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER, SSEType,
|
||||
contains_managed_encryption_metadata, normalize_managed_metadata,
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
|
||||
/// Outcome of a single object's DEK rewrap attempt.
|
||||
// Consumed by the bulk rekey sweep in the follow-up PR; tests exercise it now.
|
||||
#[allow(dead_code)]
|
||||
#[derive(Debug)]
|
||||
pub enum ObjectDekRewrapOutcome {
|
||||
/// The object carries no KMS-wrapped RustFS data-key envelope this build
|
||||
/// can rewrap: plaintext, SSE-C, or a MinIO-sealed data key.
|
||||
NotApplicable,
|
||||
/// The envelope is already on the current master key version and format;
|
||||
/// the caller must persist nothing, so a sweep re-run converges.
|
||||
AlreadyCurrent,
|
||||
/// The envelope was rewrapped. `metadata` holds the overrides to merge
|
||||
/// into the object's user-defined metadata — every stored copy of the old
|
||||
/// envelope, replaced — via `ObjectLayer::put_object_metadata`.
|
||||
Rewrapped { metadata: HashMap<String, String> },
|
||||
}
|
||||
|
||||
/// Rewrap the KMS-wrapped data key in an object's stored metadata onto its
|
||||
/// master key's current version (and current envelope format), without
|
||||
/// touching the object's data or its plaintext DEK.
|
||||
///
|
||||
/// Mirrors the managed decrypt path byte for byte where it matters: the
|
||||
/// envelope is located through the same normalized-header resolution, and the
|
||||
/// encryption context is rebuilt with the same helpers, so an envelope the
|
||||
/// read path can open is exactly an envelope this can rewrap. The KMS backend
|
||||
/// owns the format and the no-op decision.
|
||||
///
|
||||
/// The returned overrides replace every stored copy of the old envelope
|
||||
/// (RustFS's internal header and the MinIO-compatible slots that the writer
|
||||
/// fills with the same bytes). A copy left behind would win a read-path
|
||||
/// fallback and resurrect the old wrapping, so finding no replaceable copy is
|
||||
/// an error, never a silent success.
|
||||
pub async fn rewrap_object_encryption_metadata(
|
||||
host: &dyn SseHost,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
metadata: &HashMap<String, String>,
|
||||
) -> Result<ObjectDekRewrapOutcome, SseError> {
|
||||
if !contains_managed_encryption_metadata(metadata) {
|
||||
return Ok(ObjectDekRewrapOutcome::NotApplicable);
|
||||
}
|
||||
|
||||
let encryption_type = match metadata.get("x-amz-server-side-encryption").map(String::as_str) {
|
||||
Some(AMZ_ENCRYPTION_KMS) => SSEType::SseKms,
|
||||
Some(_) => SSEType::SseS3,
|
||||
// MinIO-written objects synthesize the public header on read; their
|
||||
// sealed data keys are not RustFS envelopes and fall out below.
|
||||
None => SSEType::SseS3,
|
||||
};
|
||||
|
||||
let normalized_metadata = normalize_managed_metadata(metadata, Some(recode_minio_kms_context));
|
||||
let Some(envelope_b64) = normalized_metadata
|
||||
.get(INTERNAL_ENCRYPTION_KEY_HEADER)
|
||||
.or_else(|| metadata.get(MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER))
|
||||
else {
|
||||
return Ok(ObjectDekRewrapOutcome::NotApplicable);
|
||||
};
|
||||
let encrypted_data_key = BASE64_STANDARD
|
||||
.decode_to_vec(envelope_b64)
|
||||
.map_err(|e| SseError::internal(format!("Failed to decode encrypted key: {e}")))?;
|
||||
// Only RustFS envelopes are rewrappable here; MinIO's builtin-KMS
|
||||
// ciphertext is opaque bytes owned by a different root of trust.
|
||||
if !is_data_key_envelope(&encrypted_data_key) {
|
||||
return Ok(ObjectDekRewrapOutcome::NotApplicable);
|
||||
}
|
||||
|
||||
let kms_context = if matches!(encryption_type, SSEType::SseKms) {
|
||||
decode_minio_kms_context(metadata)?
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let object_context = build_object_encryption_context(bucket, key, kms_context.as_ref());
|
||||
|
||||
// The same provider selection as the managed decrypt path.
|
||||
let provider = kms_sse_dek_provider(host).await?;
|
||||
let response = provider.rewrap_sse_dek(&encrypted_data_key, &object_context).await?;
|
||||
if !response.rewrapped {
|
||||
return Ok(ObjectDekRewrapOutcome::AlreadyCurrent);
|
||||
}
|
||||
|
||||
// Replace every stored copy of the old envelope, keyed by value and
|
||||
// matched case-insensitively: metadata key casing drifts through the
|
||||
// storage layer, and an override inserted under a differently-cased name
|
||||
// would sit beside the old copy instead of replacing it. The MinIO
|
||||
// sealed-key slots can instead hold a sealed *object* key (rio-v2 writer);
|
||||
// those bytes differ from the envelope and are untouched — the data key
|
||||
// they seal is unchanged by a rewrap.
|
||||
const REWRAP_ENVELOPE_HEADERS: [&str; 4] = [
|
||||
INTERNAL_ENCRYPTION_KEY_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER,
|
||||
];
|
||||
let old_envelope_b64 = envelope_b64.clone();
|
||||
let new_envelope_b64 = BASE64_STANDARD.encode_to_string(&response.ciphertext);
|
||||
let mut overrides = HashMap::new();
|
||||
for (stored_name, stored_value) in metadata {
|
||||
let is_envelope_slot = REWRAP_ENVELOPE_HEADERS
|
||||
.iter()
|
||||
.any(|header| stored_name.eq_ignore_ascii_case(header));
|
||||
if is_envelope_slot && *stored_value == old_envelope_b64 {
|
||||
overrides.insert(stored_name.clone(), new_envelope_b64.clone());
|
||||
}
|
||||
}
|
||||
if overrides.is_empty() {
|
||||
return Err(SseError::internal(
|
||||
"rewrapped a data-key envelope but found no stored metadata copy to replace; refusing a write that would \
|
||||
leave the old wrapping live",
|
||||
));
|
||||
}
|
||||
|
||||
Ok(ObjectDekRewrapOutcome::Rewrapped { metadata: overrides })
|
||||
}
|
||||
@@ -0,0 +1,177 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Per-key KMS authorization on the SSE-KMS data path.
|
||||
|
||||
use crate::core::audit::{record_managed_kms_outcome, stored_envelope_master_key_version};
|
||||
use crate::core::request::SseKmsPrincipal;
|
||||
use crate::error::SseError;
|
||||
use crate::host::SseHost;
|
||||
use crate::{LOG_COMPONENT_STORAGE, LOG_SUBSYSTEM_SSE};
|
||||
use rustfs_config::ENV_RUSTFS_KMS_ENFORCE_SSE_KEY_POLICY;
|
||||
use rustfs_policy::policy::action::KmsAction;
|
||||
use rustfs_s3_types::S3ErrorCode;
|
||||
use rustfs_utils::http::object_encryption_keys::{SSEType, stored_managed_encryption_key};
|
||||
use std::collections::HashMap;
|
||||
use tracing::debug;
|
||||
|
||||
/// Ask the authorizer that decides for `principal`: the host's, unless a test overrode it.
|
||||
async fn is_kms_key_allowed(
|
||||
host: &dyn SseHost,
|
||||
principal: &SseKmsPrincipal,
|
||||
action: KmsAction,
|
||||
key_id: &str,
|
||||
) -> Result<bool, SseError> {
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
if let Some(hooks) = principal.test_hooks.as_ref() {
|
||||
return hooks.authorizer.is_allowed(principal, action, key_id).await;
|
||||
}
|
||||
|
||||
host.kms_key_authorizer().is_allowed(principal, action, key_id).await
|
||||
}
|
||||
|
||||
/// Whether per-key KMS authorization is enforced on the SSE-KMS data path.
|
||||
fn sse_kms_key_policy_enforced(principal: Option<&SseKmsPrincipal>) -> bool {
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
if let Some(hooks) = principal.and_then(|candidate| candidate.test_hooks.as_ref()) {
|
||||
return hooks.enforced;
|
||||
}
|
||||
#[cfg(not(any(test, feature = "test-util")))]
|
||||
let _ = principal;
|
||||
|
||||
crate::startup::key_policy_enforced()
|
||||
}
|
||||
|
||||
/// The principal whose KMS key permissions this operation must satisfy, if any.
|
||||
///
|
||||
/// Scoping is limited to SSE-KMS, matching AWS: SSE-S3 wraps its data key with a
|
||||
/// server-owned key the caller never names, and SSE-C never reaches KMS at all.
|
||||
pub(crate) fn kms_authorization_subject(
|
||||
enforced: bool,
|
||||
principal: Option<&SseKmsPrincipal>,
|
||||
sse_type: SSEType,
|
||||
) -> Option<&SseKmsPrincipal> {
|
||||
if !enforced || !matches!(sse_type, SSEType::SseKms) {
|
||||
return None;
|
||||
}
|
||||
|
||||
principal
|
||||
}
|
||||
|
||||
/// Authorize `action` against `key_id` for the caller behind this SSE operation.
|
||||
///
|
||||
/// Runs before the key is used, so a denied request cannot distinguish an unauthorized
|
||||
/// key from a disabled or pending-deletion one.
|
||||
pub(crate) async fn authorize_sse_kms_key(
|
||||
host: &dyn SseHost,
|
||||
principal: Option<&SseKmsPrincipal>,
|
||||
sse_type: SSEType,
|
||||
action: KmsAction,
|
||||
key_id: &str,
|
||||
) -> Result<(), SseError> {
|
||||
let Some(principal) = kms_authorization_subject(sse_kms_key_policy_enforced(principal), principal, sse_type) else {
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
if is_kms_key_allowed(host, principal, action, key_id).await? {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
debug!(
|
||||
component = LOG_COMPONENT_STORAGE,
|
||||
subsystem = LOG_SUBSYSTEM_SSE,
|
||||
event = "sse_kms_key_authorization_denied",
|
||||
account = %principal.account,
|
||||
action = ?action,
|
||||
"Principal is not authorized for the KMS key resolved for this request"
|
||||
);
|
||||
|
||||
// One warn per process, not per request: anonymous denials are driven by
|
||||
// unauthenticated traffic, so a per-request warn would let anyone flood the
|
||||
// log. Per-request detail stays on the audit entry and the debug event above.
|
||||
if principal.account.is_empty() {
|
||||
static ANONYMOUS_DENIAL_WARNED: std::sync::Once = std::sync::Once::new();
|
||||
ANONYMOUS_DENIAL_WARNED.call_once(|| {
|
||||
tracing::warn!(
|
||||
component = LOG_COMPONENT_STORAGE,
|
||||
subsystem = LOG_SUBSYSTEM_SSE,
|
||||
event = "sse_kms_anonymous_key_authorization_denied",
|
||||
action = ?action,
|
||||
"Anonymous requests are being denied by SSE-KMS per-key authorization: anonymous \
|
||||
callers hold no kms grants, so a public bucket serving SSE-KMS objects is \
|
||||
incompatible with {ENV_RUSTFS_KMS_ENFORCE_SSE_KEY_POLICY}=true. Reported once per \
|
||||
process; per-request denials are on audit entries and at debug level."
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
Err(SseError {
|
||||
code: S3ErrorCode::AccessDenied,
|
||||
message: "Access Denied".to_string(),
|
||||
source: None,
|
||||
})
|
||||
}
|
||||
|
||||
/// Authorize `kms:Decrypt` for an object whose stored encryption material is unwrapped
|
||||
/// outside [`sse_decryption`].
|
||||
///
|
||||
/// The copy-source read resolves its material inside the object layer, which has no
|
||||
/// request identity, so the S3 layer has to run the check itself.
|
||||
pub async fn authorize_sse_kms_object_read(
|
||||
host: &dyn SseHost,
|
||||
principal: Option<&SseKmsPrincipal>,
|
||||
metadata: &HashMap<String, String>,
|
||||
) -> Result<(), SseError> {
|
||||
let Some((sse_type, key_id)) = stored_managed_encryption_key(metadata) else {
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
let result = authorize_sse_kms_key(host, principal, sse_type, KmsAction::DecryptAction, &key_id).await;
|
||||
// Only a denial is recorded here: an allowed read goes on to unwrap the key,
|
||||
// and that operation reports its own outcome.
|
||||
if let Err(error) = &result {
|
||||
record_managed_kms_outcome(
|
||||
principal,
|
||||
sse_type,
|
||||
Some(&key_id),
|
||||
|| stored_envelope_master_key_version(metadata),
|
||||
Err(error),
|
||||
);
|
||||
}
|
||||
|
||||
result
|
||||
}
|
||||
|
||||
/// Authorize the current request to write under the upload's stored KMS key.
|
||||
/// Upload IDs do not confer the creator's KMS permissions on another caller.
|
||||
pub async fn authorize_sse_kms_multipart_write(
|
||||
host: &dyn SseHost,
|
||||
principal: Option<&SseKmsPrincipal>,
|
||||
metadata: &HashMap<String, String>,
|
||||
) -> Result<(), SseError> {
|
||||
let Some((sse_type, key_id)) = stored_managed_encryption_key(metadata) else {
|
||||
return Ok(());
|
||||
};
|
||||
let result = authorize_sse_kms_key(host, principal, sse_type, KmsAction::GenerateDataKeyAction, &key_id).await;
|
||||
if let Err(error) = &result {
|
||||
record_managed_kms_outcome(
|
||||
principal,
|
||||
sse_type,
|
||||
Some(&key_id),
|
||||
|| stored_envelope_master_key_version(metadata),
|
||||
Err(error),
|
||||
);
|
||||
}
|
||||
result
|
||||
}
|
||||
@@ -0,0 +1,454 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! SSE request headers in, SSE response headers out.
|
||||
|
||||
use crate::core::keying::metadata::{normalize_encryption_metadata_case, recode_minio_kms_context, stored_ssec_metadata};
|
||||
use crate::core::s3::ssec::ssec_invalid_request;
|
||||
use crate::error::SseError;
|
||||
use base64_simd::STANDARD as BASE64_STANDARD;
|
||||
use http::{HeaderMap, HeaderName, HeaderValue};
|
||||
use rustfs_s3_types::S3ErrorCode;
|
||||
use rustfs_utils::http::headers::{
|
||||
AMZ_ENCRYPTION_AES, AMZ_ENCRYPTION_KMS, AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM,
|
||||
AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY, AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5, AMZ_SERVER_SIDE_ENCRYPTION_KMS_CONTEXT,
|
||||
SSEC_ALGORITHM_HEADER, SSEC_KEY_MD5_HEADER,
|
||||
};
|
||||
use rustfs_utils::http::object_encryption_keys::{
|
||||
INTERNAL_ENCRYPTION_CONTEXT_HEADER, INTERNAL_ENCRYPTION_KEY_ID_HEADER, MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER, MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER, MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER, SSEType,
|
||||
normalize_managed_metadata,
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
|
||||
// The same header names s3s publishes. Owned so this module does not import s3s.
|
||||
pub(crate) const X_AMZ_SERVER_SIDE_ENCRYPTION: HeaderName = HeaderName::from_static("x-amz-server-side-encryption");
|
||||
|
||||
pub(crate) const X_AMZ_SERVER_SIDE_ENCRYPTION_AWS_KMS_KEY_ID: HeaderName =
|
||||
HeaderName::from_static("x-amz-server-side-encryption-aws-kms-key-id");
|
||||
|
||||
pub(crate) const X_AMZ_SERVER_SIDE_ENCRYPTION_CONTEXT: HeaderName =
|
||||
HeaderName::from_static("x-amz-server-side-encryption-context");
|
||||
|
||||
pub(crate) const X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM: HeaderName =
|
||||
HeaderName::from_static("x-amz-server-side-encryption-customer-algorithm");
|
||||
|
||||
pub(crate) const X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5: HeaderName =
|
||||
HeaderName::from_static("x-amz-server-side-encryption-customer-key-md5");
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn sse_invalid_argument(message: &str) -> SseError {
|
||||
SseError {
|
||||
code: S3ErrorCode::InvalidArgument,
|
||||
message: message.to_string(),
|
||||
source: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// SSE-C parameters extracted from headers (algorithm, key, key MD5).
|
||||
pub type SsecParamsFromHeaders = (Option<String>, Option<String>, Option<String>);
|
||||
|
||||
/// Extract SSE-C parameters from request headers.
|
||||
/// Used as fallback when the S3 layer does not populate them in the input struct.
|
||||
///
|
||||
/// Returns an error if an SSE-C header is present but cannot be parsed as valid UTF-8,
|
||||
/// ensuring malformed headers do not bypass validation.
|
||||
pub fn extract_ssec_params_from_headers(headers: &HeaderMap) -> Result<SsecParamsFromHeaders, SseError> {
|
||||
let algorithm = match headers.get("x-amz-server-side-encryption-customer-algorithm") {
|
||||
None => Ok(None),
|
||||
Some(v) => v
|
||||
.to_str()
|
||||
.map(|s| Some(s.to_string()))
|
||||
.map_err(|_| sse_invalid_argument("The x-amz-server-side-encryption-customer-algorithm header must be valid UTF-8.")),
|
||||
}?;
|
||||
|
||||
let key = match headers.get("x-amz-server-side-encryption-customer-key") {
|
||||
None => Ok(None),
|
||||
Some(v) => v
|
||||
.to_str()
|
||||
.map(|s| Some(s.to_string()))
|
||||
.map_err(|_| sse_invalid_argument("The x-amz-server-side-encryption-customer-key header must be valid UTF-8.")),
|
||||
}?;
|
||||
|
||||
let key_md5 = match headers.get("x-amz-server-side-encryption-customer-key-md5") {
|
||||
None => Ok(None),
|
||||
Some(v) => v
|
||||
.to_str()
|
||||
.map(|s| Some(s.to_string()))
|
||||
.map_err(|_| sse_invalid_argument("The x-amz-server-side-encryption-customer-key-md5 header must be valid UTF-8.")),
|
||||
}?;
|
||||
|
||||
Ok((algorithm, key, key_md5))
|
||||
}
|
||||
|
||||
/// Extract x-amz-server-side-encryption from request headers.
|
||||
/// Used as fallback when the S3 layer does not populate it in the input struct.
|
||||
///
|
||||
/// Returns an error if the header is present but cannot be parsed as valid UTF-8,
|
||||
/// ensuring malformed headers do not bypass validation.
|
||||
pub fn extract_server_side_encryption_from_headers(headers: &HeaderMap) -> Result<Option<String>, SseError> {
|
||||
match headers.get("x-amz-server-side-encryption") {
|
||||
None => Ok(None),
|
||||
Some(v) => v
|
||||
.to_str()
|
||||
.map(|s| Some(s.to_string()))
|
||||
.map_err(|_| sse_invalid_argument("The x-amz-server-side-encryption header must be valid UTF-8.")),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn extract_ssekms_context_from_headers(headers: &HeaderMap) -> Result<Option<HashMap<String, String>>, SseError> {
|
||||
let Some(v) = headers.get(AMZ_SERVER_SIDE_ENCRYPTION_KMS_CONTEXT) else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
let value = v
|
||||
.to_str()
|
||||
.map_err(|_| sse_invalid_argument("The x-amz-server-side-encryption-context header must be valid UTF-8."))?;
|
||||
let decoded = BASE64_STANDARD.decode_to_vec(value).map_err(|_| {
|
||||
sse_invalid_argument("The x-amz-server-side-encryption-context header must be valid base64-encoded JSON.")
|
||||
})?;
|
||||
|
||||
serde_json::from_slice(&decoded).map(Some).map_err(|_| {
|
||||
sse_invalid_argument("The x-amz-server-side-encryption-context header must be a base64-encoded JSON object.")
|
||||
})
|
||||
}
|
||||
|
||||
/// Rejects a client encryption context that would replace the location binding.
|
||||
///
|
||||
/// The request context binds the data key to `{bucket: bucket/key}` only when the
|
||||
/// client supplied no entry under that name, and only the client context is
|
||||
/// persisted. A client entry keyed by the bucket name therefore travels with the
|
||||
/// object and replaces the binding on every read, so the envelope would no longer
|
||||
/// be tied to the object's location. Only the write side rejects it: objects stored
|
||||
/// with such an entry stay readable because the read side rebuilds the context
|
||||
/// unchanged.
|
||||
pub(crate) fn reject_kms_context_location_override(
|
||||
bucket: &str,
|
||||
ssekms_context: Option<&HashMap<String, String>>,
|
||||
) -> Result<(), SseError> {
|
||||
if ssekms_context.is_some_and(|context| context.contains_key(bucket)) {
|
||||
return Err(sse_invalid_argument(&format!(
|
||||
"The x-amz-server-side-encryption-context header contains the key \"{bucket}\", which equals the bucket name; \
|
||||
that key is reserved for binding the data key to the object's location."
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn validate_sse_headers_for_write(
|
||||
server_side_encryption: Option<&str>,
|
||||
ssekms_key_id: Option<&String>,
|
||||
ssekms_context: Option<&HashMap<String, String>>,
|
||||
sse_customer_algorithm: Option<&String>,
|
||||
sse_customer_key: Option<&String>,
|
||||
sse_customer_key_md5: Option<&String>,
|
||||
require_sse_customer_key: bool,
|
||||
) -> Result<(), SseError> {
|
||||
if let Some(sse) = server_side_encryption {
|
||||
let s = sse;
|
||||
if s != AMZ_ENCRYPTION_AES && s != AMZ_ENCRYPTION_KMS {
|
||||
return Err(sse_invalid_argument(
|
||||
"The SSE algorithm specified is not supported. The valid values are AES256 or aws:kms.",
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
let has_ssec_headers = sse_customer_algorithm.is_some() || sse_customer_key.is_some() || sse_customer_key_md5.is_some();
|
||||
let has_managed_headers = server_side_encryption.is_some() || ssekms_key_id.is_some() || ssekms_context.is_some();
|
||||
|
||||
if has_ssec_headers {
|
||||
if has_managed_headers {
|
||||
return Err(sse_invalid_argument(
|
||||
"The SSE-C and managed server-side encryption headers cannot be used together.",
|
||||
));
|
||||
}
|
||||
|
||||
let has_valid_ssec_headers = if require_sse_customer_key {
|
||||
matches!(
|
||||
(sse_customer_algorithm, sse_customer_key, sse_customer_key_md5),
|
||||
(Some(_), Some(_), Some(_))
|
||||
)
|
||||
} else {
|
||||
matches!((sse_customer_algorithm, sse_customer_key_md5), (Some(_), Some(_)))
|
||||
};
|
||||
|
||||
if !has_valid_ssec_headers {
|
||||
let message = if require_sse_customer_key {
|
||||
"Missing SSE-C parameters. Algorithm, customer key and customer key MD5 are all required."
|
||||
} else {
|
||||
"Missing SSE-C parameters. Algorithm and customer key MD5 are required."
|
||||
};
|
||||
|
||||
return Err(ssec_invalid_request(message));
|
||||
}
|
||||
}
|
||||
|
||||
if ssekms_key_id.is_some() && server_side_encryption.is_none_or(|sse| sse != AMZ_ENCRYPTION_KMS) {
|
||||
return Err(sse_invalid_argument(
|
||||
"The SSE-KMS key ID header can only be used when x-amz-server-side-encryption is set to aws:kms.",
|
||||
));
|
||||
}
|
||||
|
||||
if ssekms_context.is_some() && server_side_encryption.is_none_or(|sse| sse != AMZ_ENCRYPTION_KMS) {
|
||||
return Err(sse_invalid_argument(
|
||||
"The SSE-KMS context header can only be used when x-amz-server-side-encryption is set to aws:kms.",
|
||||
));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn validate_sse_headers_for_read(metadata: &HashMap<String, String>, headers: &HeaderMap) -> Result<(), SseError> {
|
||||
let has_req_ssec = headers.contains_key("x-amz-server-side-encryption-customer-algorithm")
|
||||
|| headers.contains_key("x-amz-server-side-encryption-customer-key")
|
||||
|| headers.contains_key("x-amz-server-side-encryption-customer-key-md5");
|
||||
|
||||
let has_req_sse = headers.contains_key("x-amz-server-side-encryption")
|
||||
|| headers.contains_key("x-amz-server-side-encryption-aws-kms-key-id")
|
||||
|| headers.contains_key("x-amz-server-side-encryption-context");
|
||||
|
||||
let is_object_ssec = stored_ssec_metadata(metadata);
|
||||
let is_object_sse = metadata.contains_key("x-amz-server-side-encryption");
|
||||
|
||||
if is_object_ssec {
|
||||
if has_req_sse {
|
||||
return Err(sse_invalid_argument(
|
||||
"Server-side encryption headers cannot be used with an object encrypted using SSE-C.",
|
||||
));
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if is_object_sse && has_req_ssec {
|
||||
return Err(sse_invalid_argument(
|
||||
"SSE-C headers cannot be used with an object encrypted using server-side managed encryption.",
|
||||
));
|
||||
}
|
||||
|
||||
if has_req_ssec {
|
||||
return Err(ssec_invalid_request(
|
||||
"The object was stored without SSE-C. The correct SSE-C parameters must not be provided.",
|
||||
));
|
||||
}
|
||||
|
||||
if has_req_sse {
|
||||
return Err(sse_invalid_argument(
|
||||
"The object is not encrypted with server-side encryption. Do not provide server-side encryption headers.",
|
||||
));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The public `x-amz-server-side-encryption` value a managed scheme reports.
|
||||
pub(crate) fn managed_sse_public_header(sse_type: SSEType) -> &'static str {
|
||||
match sse_type {
|
||||
SSEType::SseKms => AMZ_ENCRYPTION_KMS,
|
||||
// SSE-C never reaches the managed path; reporting AES256 keeps this
|
||||
// total without inventing a third public value.
|
||||
SSEType::SseS3 | SSEType::SseC => AMZ_ENCRYPTION_AES,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn build_ssec_read_headers(algorithm: Option<&String>, key: Option<&String>, key_md5: Option<&String>) -> HeaderMap {
|
||||
let mut headers = HeaderMap::new();
|
||||
|
||||
if let Some(algorithm) = algorithm
|
||||
&& let Ok(mut value) = HeaderValue::from_str(algorithm.as_str())
|
||||
{
|
||||
value.set_sensitive(true);
|
||||
headers.insert(AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM, value);
|
||||
}
|
||||
|
||||
if let Some(key) = key
|
||||
&& let Ok(mut value) = HeaderValue::from_str(key.as_str())
|
||||
{
|
||||
value.set_sensitive(true);
|
||||
headers.insert(AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY, value);
|
||||
}
|
||||
|
||||
if let Some(key_md5) = key_md5
|
||||
&& let Ok(mut value) = HeaderValue::from_str(key_md5.as_str())
|
||||
{
|
||||
value.set_sensitive(true);
|
||||
headers.insert(AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5, value);
|
||||
}
|
||||
|
||||
headers
|
||||
}
|
||||
|
||||
/// Response headers derived for a decrypted read without unwrapping the data key.
|
||||
#[derive(Debug)]
|
||||
pub struct SseReadResponseHeaders {
|
||||
pub server_side_encryption: String,
|
||||
pub sse_customer_algorithm: Option<String>,
|
||||
pub sse_customer_key_md5: Option<String>,
|
||||
pub ssekms_key_id: Option<String>,
|
||||
}
|
||||
|
||||
fn invalid_sse_response_metadata() -> SseError {
|
||||
SseError {
|
||||
code: S3ErrorCode::InternalError,
|
||||
message: "Persisted object encryption metadata is invalid.".to_string(),
|
||||
source: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn insert_sse_response_header(headers: &mut HeaderMap, name: HeaderName, value: &str) -> Result<(), SseError> {
|
||||
let value = HeaderValue::from_str(value).map_err(|_| invalid_sse_response_metadata())?;
|
||||
headers.insert(name, value);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Project response headers after the caller has completed its existing read
|
||||
/// validation and KMS authorization path.
|
||||
pub fn project_sse_read_response_headers(
|
||||
metadata: &HashMap<String, String>,
|
||||
sse_customer_algorithm: Option<&String>,
|
||||
sse_customer_key_md5: Option<&String>,
|
||||
) -> Result<HeaderMap, SseError> {
|
||||
if metadata.keys().any(|key| {
|
||||
key.eq_ignore_ascii_case(AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY)
|
||||
|| key.eq_ignore_ascii_case(AMZ_SERVER_SIDE_ENCRYPTION_KMS_CONTEXT)
|
||||
}) {
|
||||
return Err(invalid_sse_response_metadata());
|
||||
}
|
||||
|
||||
let metadata = normalize_encryption_metadata_case(metadata).map_err(|_| invalid_sse_response_metadata())?;
|
||||
let public_mode = metadata.get("x-amz-server-side-encryption").map(String::as_str);
|
||||
let customer_algorithm = metadata.get(SSEC_ALGORITHM_HEADER).map(String::as_str);
|
||||
|
||||
let public_mode = match public_mode {
|
||||
Some(AMZ_ENCRYPTION_AES) => Some(SSEType::SseS3),
|
||||
Some(AMZ_ENCRYPTION_KMS) => Some(SSEType::SseKms),
|
||||
Some(_) => return Err(invalid_sse_response_metadata()),
|
||||
None => None,
|
||||
};
|
||||
if customer_algorithm.is_some_and(|algorithm| algorithm != AMZ_ENCRYPTION_AES) {
|
||||
return Err(invalid_sse_response_metadata());
|
||||
}
|
||||
|
||||
let mode = if customer_algorithm.is_some() {
|
||||
if public_mode == Some(SSEType::SseKms) {
|
||||
return Err(invalid_sse_response_metadata());
|
||||
}
|
||||
Some(SSEType::SseC)
|
||||
} else {
|
||||
public_mode
|
||||
};
|
||||
for marker_mode in [
|
||||
metadata
|
||||
.contains_key(MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER)
|
||||
.then_some(SSEType::SseC),
|
||||
metadata
|
||||
.contains_key(MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER)
|
||||
.then_some(SSEType::SseS3),
|
||||
metadata
|
||||
.contains_key(MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER)
|
||||
.then_some(SSEType::SseKms),
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
{
|
||||
if mode != Some(marker_mode) {
|
||||
return Err(invalid_sse_response_metadata());
|
||||
}
|
||||
}
|
||||
let Some(mode) = mode else {
|
||||
if metadata
|
||||
.keys()
|
||||
.any(|key| rustfs_utils::http::is_object_encryption_marker(key))
|
||||
{
|
||||
return Err(invalid_sse_response_metadata());
|
||||
}
|
||||
return Ok(HeaderMap::new());
|
||||
};
|
||||
|
||||
let mut key_id = None;
|
||||
for value in [
|
||||
metadata.get("x-amz-server-side-encryption-aws-kms-key-id"),
|
||||
metadata.get(INTERNAL_ENCRYPTION_KEY_ID_HEADER),
|
||||
metadata.get(MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER),
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
{
|
||||
if key_id.is_some_and(|current| current != value) {
|
||||
return Err(invalid_sse_response_metadata());
|
||||
}
|
||||
key_id = Some(value);
|
||||
}
|
||||
|
||||
let normalized_managed = normalize_managed_metadata(metadata.as_ref(), Some(recode_minio_kms_context));
|
||||
let mut response_headers = HeaderMap::with_capacity(3);
|
||||
match mode {
|
||||
SSEType::SseS3 => {
|
||||
if metadata.contains_key("x-amz-server-side-encryption-aws-kms-key-id") || metadata.contains_key(SSEC_KEY_MD5_HEADER)
|
||||
{
|
||||
return Err(invalid_sse_response_metadata());
|
||||
}
|
||||
response_headers.insert(X_AMZ_SERVER_SIDE_ENCRYPTION, HeaderValue::from_static(AMZ_ENCRYPTION_AES));
|
||||
}
|
||||
SSEType::SseKms => {
|
||||
if metadata.contains_key(SSEC_KEY_MD5_HEADER) {
|
||||
return Err(invalid_sse_response_metadata());
|
||||
}
|
||||
let key_id = key_id
|
||||
.map(String::as_str)
|
||||
.filter(|value| !value.is_empty())
|
||||
.ok_or_else(invalid_sse_response_metadata)?;
|
||||
response_headers.insert(X_AMZ_SERVER_SIDE_ENCRYPTION, HeaderValue::from_static(AMZ_ENCRYPTION_KMS));
|
||||
insert_sse_response_header(&mut response_headers, X_AMZ_SERVER_SIDE_ENCRYPTION_AWS_KMS_KEY_ID, key_id)?;
|
||||
|
||||
if metadata.contains_key(MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER)
|
||||
&& !normalized_managed.contains_key(INTERNAL_ENCRYPTION_CONTEXT_HEADER)
|
||||
{
|
||||
return Err(invalid_sse_response_metadata());
|
||||
}
|
||||
if let Some(context) = normalized_managed.get(INTERNAL_ENCRYPTION_CONTEXT_HEADER) {
|
||||
serde_json::from_str::<HashMap<String, String>>(context).map_err(|_| invalid_sse_response_metadata())?;
|
||||
let encoded = BASE64_STANDARD.encode_to_string(context);
|
||||
insert_sse_response_header(&mut response_headers, X_AMZ_SERVER_SIDE_ENCRYPTION_CONTEXT, &encoded)?;
|
||||
}
|
||||
}
|
||||
SSEType::SseC => {
|
||||
if key_id.is_some() || normalized_managed.contains_key(INTERNAL_ENCRYPTION_CONTEXT_HEADER) {
|
||||
return Err(invalid_sse_response_metadata());
|
||||
}
|
||||
let algorithm = sse_customer_algorithm
|
||||
.map(|value| value.as_ref())
|
||||
.filter(|value| *value == AMZ_ENCRYPTION_AES)
|
||||
.ok_or_else(|| {
|
||||
ssec_invalid_request(
|
||||
"The object was stored using a form of Server Side Encryption. The correct parameters must be provided to retrieve the object.",
|
||||
)
|
||||
})?;
|
||||
let key_md5 = sse_customer_key_md5
|
||||
.map(|value| value.as_ref())
|
||||
.ok_or_else(|| ssec_invalid_request("Missing SSE-C customer key MD5."))?;
|
||||
let stored_md5 = metadata.get(SSEC_KEY_MD5_HEADER).ok_or_else(invalid_sse_response_metadata)?;
|
||||
if stored_md5 != key_md5 {
|
||||
return Err(ssec_invalid_request(
|
||||
"The provided encryption parameters did not match the ones used originally to encrypt the object.",
|
||||
));
|
||||
}
|
||||
insert_sse_response_header(&mut response_headers, X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM, algorithm)?;
|
||||
insert_sse_response_header(&mut response_headers, X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5, key_md5)?;
|
||||
}
|
||||
}
|
||||
Ok(response_headers)
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! The schemes S3 defines: SSE-S3, SSE-KMS and SSE-C, as a request names them and a response reports them.
|
||||
|
||||
pub(crate) mod authorization;
|
||||
pub(crate) mod headers;
|
||||
pub(crate) mod scheme;
|
||||
pub(crate) mod ssec;
|
||||
@@ -0,0 +1,288 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Which SSE scheme and which key a request resolves to: the request first, then the bucket default.
|
||||
|
||||
use crate::core::s3::headers::validate_sse_headers_for_write;
|
||||
use crate::error::SseError;
|
||||
use crate::host::SseHost;
|
||||
use crate::{LOG_COMPONENT_STORAGE, LOG_SUBSYSTEM_SSE};
|
||||
use rustfs_utils::http::headers::{AMZ_ENCRYPTION_AES, AMZ_ENCRYPTION_KMS};
|
||||
use rustfs_utils::http::object_encryption_keys::SSEType;
|
||||
use std::collections::HashMap;
|
||||
use tracing::debug;
|
||||
|
||||
pub const DEFAULT_SSE_ALGORITHM: &str = "AES256";
|
||||
|
||||
pub(crate) const SUPPORT_SSE_ALGORITHMS: &[&str] = &[DEFAULT_SSE_ALGORITHM];
|
||||
|
||||
/// SSE configuration resolved from request and bucket defaults
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct SseConfiguration {
|
||||
/// Effective server-side encryption algorithm (after considering bucket defaults)
|
||||
pub effective_sse: String,
|
||||
/// Effective KMS key ID (after considering bucket defaults)
|
||||
pub effective_kms_key_id: Option<String>,
|
||||
}
|
||||
|
||||
/// Managed SSE resolved from a bucket default encryption rule on a write path.
|
||||
///
|
||||
/// The single mapping shared by every writer: the resolver in this crate, and
|
||||
/// the server's PUT, COPY and extract paths. Unknown algorithms fall back to AES256 rather
|
||||
/// than to `None`. Resolving `None` instead lets a same-name copy under a
|
||||
/// malformed bucket default pass the `copy_changes_encryption` guard and take
|
||||
/// the metadata-only shortcut while this layer still encrypts: fresh DEK
|
||||
/// metadata is committed beside the untouched plaintext blocks and the object
|
||||
/// becomes unreadable. PutBucketEncryption refuses unknown algorithms, so this
|
||||
/// is reachable only through a configuration stored before that check or
|
||||
/// through hand-edited bucket metadata (backlog#1826).
|
||||
pub fn bucket_default_write_sse(sse_algorithm: &str) -> String {
|
||||
match sse_algorithm {
|
||||
"AES256" => String::from(AMZ_ENCRYPTION_AES),
|
||||
"aws:kms" => String::from(AMZ_ENCRYPTION_KMS),
|
||||
_ => String::from(AMZ_ENCRYPTION_AES),
|
||||
}
|
||||
}
|
||||
|
||||
/// Prepare SSE configuration by resolving request parameters with bucket defaults
|
||||
///
|
||||
/// This function:
|
||||
/// 1. Queries bucket default encryption configuration
|
||||
/// 2. Resolves effective encryption (request overrides bucket default)
|
||||
/// 3. Prepares metadata headers for managed SSE
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `bucket` - Bucket name
|
||||
/// * `server_side_encryption` - SSE algorithm from request (SSE-S3 or SSE-KMS)
|
||||
/// * `ssekms_key_id` - KMS key ID from request
|
||||
/// * `sse_customer_algorithm` - SSE-C algorithm from request
|
||||
///
|
||||
/// # Returns
|
||||
/// `SseConfiguration` with resolved encryption parameters and metadata headers
|
||||
pub(crate) async fn prepare_sse_configuration(
|
||||
host: &dyn SseHost,
|
||||
bucket: &str,
|
||||
server_side_encryption: Option<String>,
|
||||
ssekms_key_id: Option<String>,
|
||||
) -> Result<Option<SseConfiguration>, SseError> {
|
||||
if let Some(server_side_encryption) = server_side_encryption.clone()
|
||||
&& server_side_encryption.as_str() == AMZ_ENCRYPTION_AES
|
||||
{
|
||||
return Ok(Some(SseConfiguration {
|
||||
effective_sse: server_side_encryption,
|
||||
effective_kms_key_id: None,
|
||||
}));
|
||||
}
|
||||
|
||||
if let Some(server_side_encryption) = server_side_encryption.clone()
|
||||
&& let Some(ssekms_key_id) = ssekms_key_id
|
||||
{
|
||||
return Ok(Some(SseConfiguration {
|
||||
effective_sse: server_side_encryption,
|
||||
effective_kms_key_id: Some(ssekms_key_id),
|
||||
}));
|
||||
}
|
||||
|
||||
// Get bucket default encryption configuration.
|
||||
let bucket_encryption = host.bucket_encryption(bucket).await;
|
||||
debug!(
|
||||
component = LOG_COMPONENT_STORAGE,
|
||||
subsystem = LOG_SUBSYSTEM_SSE,
|
||||
event = "bucket_sse_config_lookup",
|
||||
bucket = %bucket,
|
||||
found = matches!(bucket_encryption, Ok(Some(_))),
|
||||
"Bucket SSE configuration lookup completed"
|
||||
);
|
||||
|
||||
let Some(bucket_encryption) = bucket_encryption? else {
|
||||
// The bucket has no SSE config. If the user explicitly requested
|
||||
// aws:kms, we must honor that — return the explicit SSE header so
|
||||
// downstream logic can try (and fail if KMS is unavailable).
|
||||
return Ok(server_side_encryption.map(|sse| SseConfiguration {
|
||||
effective_sse: sse,
|
||||
effective_kms_key_id: ssekms_key_id,
|
||||
}));
|
||||
};
|
||||
|
||||
let default_rule = bucket_encryption.default_rule.as_ref();
|
||||
let effective_sse = server_side_encryption.clone().or_else(|| {
|
||||
default_rule.map(|sse| {
|
||||
debug!(
|
||||
component = LOG_COMPONENT_STORAGE,
|
||||
subsystem = LOG_SUBSYSTEM_SSE,
|
||||
event = "bucket_sse_default_applied",
|
||||
bucket = %bucket,
|
||||
algorithm = sse.sse_algorithm.as_str(),
|
||||
has_kms_key_id = sse.kms_master_key_id.is_some(),
|
||||
"Bucket SSE default resolved"
|
||||
);
|
||||
bucket_default_write_sse(&sse.sse_algorithm)
|
||||
})
|
||||
});
|
||||
let Some(effective_sse) = effective_sse else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
debug!(
|
||||
component = LOG_COMPONENT_STORAGE,
|
||||
subsystem = LOG_SUBSYSTEM_SSE,
|
||||
event = "effective_sse_resolved",
|
||||
bucket = %bucket,
|
||||
requested = ?server_side_encryption,
|
||||
effective = ?effective_sse,
|
||||
"Resolved effective SSE configuration"
|
||||
);
|
||||
|
||||
let effective_kms_key_id = resolve_effective_kms_key_id(Some(&effective_sse), ssekms_key_id, || {
|
||||
default_rule.and_then(|sse| sse.kms_master_key_id.clone())
|
||||
});
|
||||
|
||||
Ok(Some(SseConfiguration {
|
||||
effective_sse,
|
||||
effective_kms_key_id,
|
||||
}))
|
||||
}
|
||||
|
||||
pub(crate) fn resolve_effective_kms_key_id<F>(
|
||||
effective_sse: Option<&str>,
|
||||
requested_kms_key_id: Option<String>,
|
||||
bucket_default_kms_key_id: F,
|
||||
) -> Option<String>
|
||||
where
|
||||
F: FnOnce() -> Option<String>,
|
||||
{
|
||||
if effective_sse.is_none_or(|sse| sse != AMZ_ENCRYPTION_KMS) {
|
||||
return requested_kms_key_id;
|
||||
}
|
||||
|
||||
requested_kms_key_id.or_else(bucket_default_kms_key_id)
|
||||
}
|
||||
|
||||
/// The SSE headers of a write request, as the S3 layer received them.
|
||||
pub(crate) struct WriteSchemeHeaders<'a> {
|
||||
pub(crate) server_side_encryption: Option<String>,
|
||||
pub(crate) ssekms_key_id: Option<String>,
|
||||
pub(crate) ssekms_context: Option<&'a HashMap<String, String>>,
|
||||
pub(crate) sse_customer_algorithm: Option<String>,
|
||||
pub(crate) sse_customer_key: Option<String>,
|
||||
pub(crate) sse_customer_key_md5: Option<String>,
|
||||
}
|
||||
|
||||
/// The scheme a write resolved to, in S3's terms.
|
||||
#[derive(Clone)]
|
||||
pub(crate) enum WriteScheme {
|
||||
/// `AES256`: the server chooses the wrapping key and the caller names none.
|
||||
SseS3,
|
||||
/// `aws:kms`, under the key the request or the bucket default named, if either did.
|
||||
SseKms { key_id: Option<String> },
|
||||
/// A key the client supplies; only its MD5 is ever stored.
|
||||
SseC {
|
||||
algorithm: String,
|
||||
key: String,
|
||||
key_md5: String,
|
||||
},
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for WriteScheme {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.write_str(match self {
|
||||
Self::SseS3 => "SseS3",
|
||||
Self::SseKms { .. } => "SseKms(..)",
|
||||
Self::SseC { .. } => "SseC(..)",
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Decide the scheme of a write: validate the headers, then the request first and the
|
||||
/// bucket default second.
|
||||
///
|
||||
/// Every write goes through here, an object and a multipart session alike, so the two
|
||||
/// cannot resolve the same headers differently.
|
||||
pub(crate) async fn resolve_write_scheme(
|
||||
host: &dyn SseHost,
|
||||
bucket: &str,
|
||||
headers: WriteSchemeHeaders<'_>,
|
||||
) -> Result<Option<WriteScheme>, SseError> {
|
||||
validate_sse_headers_for_write(
|
||||
headers.server_side_encryption.as_deref(),
|
||||
headers.ssekms_key_id.as_ref(),
|
||||
headers.ssekms_context,
|
||||
headers.sse_customer_algorithm.as_ref(),
|
||||
headers.sse_customer_key.as_ref(),
|
||||
headers.sse_customer_key_md5.as_ref(),
|
||||
true,
|
||||
)?;
|
||||
|
||||
// Validation admits SSE-C headers only as a complete set, and never beside the
|
||||
// managed ones, so their presence alone decides.
|
||||
if let (Some(algorithm), Some(key), Some(key_md5)) =
|
||||
(headers.sse_customer_algorithm, headers.sse_customer_key, headers.sse_customer_key_md5)
|
||||
{
|
||||
return Ok(Some(WriteScheme::SseC { algorithm, key, key_md5 }));
|
||||
}
|
||||
|
||||
let Some(config) = prepare_sse_configuration(host, bucket, headers.server_side_encryption, headers.ssekms_key_id).await?
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
Ok(match config.effective_sse.as_str() {
|
||||
AMZ_ENCRYPTION_AES => {
|
||||
// Validation admits a key id only beside `aws:kms`, and the bucket default's
|
||||
// key is consulted only for `aws:kms`.
|
||||
debug_assert!(config.effective_kms_key_id.is_none(), "SSE-S3 never names a KMS key");
|
||||
Some(WriteScheme::SseS3)
|
||||
}
|
||||
AMZ_ENCRYPTION_KMS => Some(WriteScheme::SseKms {
|
||||
key_id: config.effective_kms_key_id,
|
||||
}),
|
||||
_ => None,
|
||||
})
|
||||
}
|
||||
|
||||
/// Scheme a managed-SSE header names, defaulting to SSE-S3 the same way the
|
||||
/// encryption and decryption paths do.
|
||||
pub(crate) fn managed_sse_type(server_side_encryption: &str) -> SSEType {
|
||||
match server_side_encryption {
|
||||
AMZ_ENCRYPTION_KMS => SSEType::SseKms,
|
||||
_ => SSEType::SseS3,
|
||||
}
|
||||
}
|
||||
|
||||
/// Key id an SSE-KMS read authorizes and reports when the stored metadata
|
||||
/// carries none (legacy envelopes and MinIO-written objects predate the
|
||||
/// explicit key-id headers).
|
||||
///
|
||||
/// The KMS service's configured default key is the key such envelopes were
|
||||
/// actually wrapped under, so per-key authorization must target it — the bare
|
||||
/// literal `default` would silently escape any policy narrowed to real key
|
||||
/// ids. The literal remains only as the last resort when no service is
|
||||
/// running, matching what the legacy write path recorded implicitly.
|
||||
pub(crate) async fn resolve_stored_kms_key_id(host: &dyn SseHost, stored: Option<String>) -> String {
|
||||
if let Some(key_id) = stored {
|
||||
return key_id;
|
||||
}
|
||||
if let Some(service) = host.encryption_service().await
|
||||
&& let Some(key_id) = service.get_default_key_id()
|
||||
{
|
||||
return key_id.clone();
|
||||
}
|
||||
"default".to_string()
|
||||
}
|
||||
|
||||
/// Check if the server_side_encryption is a managed SSE type (SSE-S3 or SSE-KMS)
|
||||
#[inline]
|
||||
pub(crate) fn is_managed_sse(server_side_encryption: &str) -> bool {
|
||||
matches!(server_side_encryption, "AES256" | "aws:kms")
|
||||
}
|
||||
@@ -0,0 +1,217 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! SSE-C: validating the customer-provided key and matching it against the stored object.
|
||||
|
||||
use crate::core::keying::metadata::{minio_sealed_ssec_metadata, stored_ssec_metadata};
|
||||
use crate::core::s3::scheme::{DEFAULT_SSE_ALGORITHM, SUPPORT_SSE_ALGORITHMS};
|
||||
use crate::error::SseError;
|
||||
use base64_simd::STANDARD as BASE64_STANDARD;
|
||||
use md5::{Digest as Md5Digest, Md5};
|
||||
use rustfs_s3_types::S3ErrorCode;
|
||||
use std::collections::HashMap;
|
||||
use tracing::error;
|
||||
|
||||
fn md5_bytes(input: impl AsRef<[u8]>) -> [u8; 16] {
|
||||
let mut hasher = Md5::new();
|
||||
hasher.update(input.as_ref());
|
||||
hasher.finalize().into()
|
||||
}
|
||||
|
||||
pub(crate) fn md5_base64(input: impl AsRef<[u8]>) -> String {
|
||||
BASE64_STANDARD.encode_to_string(md5_bytes(input))
|
||||
}
|
||||
|
||||
/// Validated SSE-C parameters
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct ValidatedSsecParams {
|
||||
/// Encryption algorithm (always "AES256" for SSE-C)
|
||||
pub algorithm: String,
|
||||
/// Decoded encryption key bytes (32 bytes for AES-256)
|
||||
pub key_bytes: [u8; 32],
|
||||
/// Base64-encoded MD5 of the key
|
||||
pub key_md5: String,
|
||||
}
|
||||
|
||||
/// SSE-C parameters from client request
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct SsecParams {
|
||||
/// Encryption algorithm
|
||||
pub algorithm: String,
|
||||
/// Base64-encoded encryption key
|
||||
pub key: String,
|
||||
/// Base64-encoded MD5 of the key
|
||||
pub key_md5: String,
|
||||
}
|
||||
|
||||
/// Validate SSE-C parameters from client request
|
||||
///
|
||||
/// Validates:
|
||||
/// 1. Algorithm is "AES256"
|
||||
/// 2. Key is valid Base64 and exactly 32 bytes
|
||||
/// 3. MD5 hash matches the key
|
||||
///
|
||||
/// # Returns
|
||||
/// `ValidatedSsecParams` with decoded key bytes
|
||||
pub(crate) fn validate_ssec_params(params: SsecParams) -> Result<ValidatedSsecParams, SseError> {
|
||||
if !SUPPORT_SSE_ALGORITHMS.contains(¶ms.algorithm.as_str()) {
|
||||
return Err(ssec_invalid_request(&format!(
|
||||
"Unsupported SSE-C algorithm. Only {DEFAULT_SSE_ALGORITHM} is supported."
|
||||
)));
|
||||
}
|
||||
|
||||
let key_bytes = BASE64_STANDARD.decode_to_vec(¶ms.key).map_err(|e| {
|
||||
error!("Failed to decode SSE-C key: {}", e);
|
||||
ssec_invalid_request("Invalid SSE-C key: not valid Base64.")
|
||||
})?;
|
||||
|
||||
if key_bytes.len() != 32 {
|
||||
return Err(ssec_invalid_request(&format!(
|
||||
"SSE-C key must be 32 bytes (256 bits), got {} bytes.",
|
||||
key_bytes.len()
|
||||
)));
|
||||
}
|
||||
|
||||
let computed_md5 = md5_base64(&key_bytes);
|
||||
if computed_md5 != params.key_md5 {
|
||||
return Err(ssec_invalid_request(
|
||||
"The calculated MD5 hash of the key did not match the hash that was provided.",
|
||||
));
|
||||
}
|
||||
|
||||
let key_array: [u8; 32] = key_bytes
|
||||
.try_into()
|
||||
.map_err(|_| ssec_invalid_request("SSE-C key must be exactly 32 bytes."))?;
|
||||
|
||||
Ok(ValidatedSsecParams {
|
||||
algorithm: params.algorithm,
|
||||
key_bytes: key_array,
|
||||
key_md5: params.key_md5,
|
||||
})
|
||||
}
|
||||
|
||||
/// Verify SSE-C key matches the stored metadata.
|
||||
///
|
||||
/// Used during GetObject/HeadObject to ensure the client provided the correct key.
|
||||
/// Returns 400 InvalidRequest on mismatch, consistent with AWS S3 behavior.
|
||||
pub(crate) fn verify_ssec_key_match(
|
||||
provided_md5: &str,
|
||||
stored_md5: Option<&String>,
|
||||
minio_sealed_ssec: bool,
|
||||
) -> Result<(), SseError> {
|
||||
match stored_md5 {
|
||||
Some(stored) if stored == provided_md5 => Ok(()),
|
||||
Some(_) => Err(ssec_invalid_request(
|
||||
"The provided encryption parameters did not match the ones used originally to encrypt the object.",
|
||||
)),
|
||||
// MinIO never persists the customer-key MD5, so this early check has
|
||||
// nothing to compare against on a migrated object. The key itself is
|
||||
// still proven: a wrong key fails the AEAD unseal of the sealed object
|
||||
// key. The check exists only for the friendlier early error, so its
|
||||
// absence is tolerated for exactly that shape and no other.
|
||||
None if minio_sealed_ssec => Ok(()),
|
||||
None => Err(ssec_invalid_request("Object has no stored SSE-C key metadata.")),
|
||||
}
|
||||
}
|
||||
|
||||
/// Algorithm an SSE-C object recorded; objects that recorded none were written as `AES256`.
|
||||
pub(crate) fn stored_ssec_algorithm(metadata: &HashMap<String, String>) -> &str {
|
||||
metadata
|
||||
.get("x-amz-server-side-encryption-customer-algorithm")
|
||||
.map(|s| s.as_str())
|
||||
.unwrap_or("AES256")
|
||||
}
|
||||
|
||||
/// The customer key and MD5 of a read of an SSE-C object, once both are present and
|
||||
/// the MD5 matches what the object recorded.
|
||||
///
|
||||
/// The key itself is not validated here; the caller does that next, so a request
|
||||
/// that names the wrong key learns of the mismatch before anything else.
|
||||
pub(crate) fn require_matching_ssec_key<'r>(
|
||||
metadata: &HashMap<String, String>,
|
||||
sse_customer_key: Option<&'r String>,
|
||||
sse_customer_key_md5: Option<&'r String>,
|
||||
) -> Result<(&'r String, &'r String), SseError> {
|
||||
let (key, key_md5) = match (sse_customer_key, sse_customer_key_md5) {
|
||||
(Some(k), Some(md5)) => (k, md5),
|
||||
_ => {
|
||||
return Err(ssec_invalid_request(
|
||||
"The object was stored using a form of Server Side Encryption. \
|
||||
The correct parameters must be provided to retrieve the object.",
|
||||
));
|
||||
}
|
||||
};
|
||||
|
||||
// Verify that the provided key MD5 matches the stored MD5 for security
|
||||
let stored_md5 = metadata.get("x-amz-server-side-encryption-customer-key-md5");
|
||||
verify_ssec_key_match(key_md5, stored_md5, minio_sealed_ssec_metadata(metadata))?;
|
||||
|
||||
Ok((key, key_md5))
|
||||
}
|
||||
|
||||
/// Validate that the SSE-C headers required for reading an SSE-C encrypted object
|
||||
/// are present in the request. This is used by HeadObject which does not decrypt
|
||||
/// the data but still must verify the caller holds the correct key.
|
||||
///
|
||||
/// Performs full validation: decodes the customer key, recomputes its MD5,
|
||||
/// verifies the client-provided MD5 header matches the key, then compares
|
||||
/// the computed MD5 against the stored metadata. This prevents a client from
|
||||
/// bypassing validation by guessing/obtaining only the stored MD5 without
|
||||
/// possessing the actual encryption key.
|
||||
///
|
||||
/// Returns `Ok(())` if either the object is not SSE-C encrypted, or valid SSE-C
|
||||
/// headers are provided and the key matches. Returns 400 InvalidRequest otherwise.
|
||||
pub fn validate_ssec_for_read(
|
||||
metadata: &HashMap<String, String>,
|
||||
sse_customer_key: Option<&String>,
|
||||
sse_customer_key_md5: Option<&String>,
|
||||
) -> Result<(), SseError> {
|
||||
if !stored_ssec_metadata(metadata) {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let (key, key_md5) = match (sse_customer_key, sse_customer_key_md5) {
|
||||
(Some(k), Some(md5)) => (k, md5),
|
||||
_ => {
|
||||
return Err(ssec_invalid_request(
|
||||
"The object was stored using a form of Server Side Encryption. \
|
||||
The correct parameters must be provided to retrieve the object.",
|
||||
));
|
||||
}
|
||||
};
|
||||
|
||||
// Full param validation: decode key, verify 32 bytes, recompute MD5
|
||||
// from actual key bytes and compare to the client-provided MD5 header.
|
||||
let algorithm = metadata
|
||||
.get("x-amz-server-side-encryption-customer-algorithm")
|
||||
.cloned()
|
||||
.unwrap_or_else(|| DEFAULT_SSE_ALGORITHM.to_string());
|
||||
let validated = validate_ssec_params(SsecParams {
|
||||
algorithm,
|
||||
key: key.to_string(),
|
||||
key_md5: key_md5.clone(),
|
||||
})?;
|
||||
|
||||
let stored_md5 = metadata.get("x-amz-server-side-encryption-customer-key-md5");
|
||||
verify_ssec_key_match(&validated.key_md5, stored_md5, minio_sealed_ssec_metadata(metadata))
|
||||
}
|
||||
|
||||
/// Build an `SseError` with `InvalidRequest` (HTTP 400) for SSE-C related errors.
|
||||
pub fn ssec_invalid_request(message: &str) -> SseError {
|
||||
SseError {
|
||||
code: S3ErrorCode::InvalidRequest,
|
||||
message: message.to_string(),
|
||||
source: None,
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user