mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-22 07:53:57 +00:00
Compare commits
5 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 1321e15988 | |||
| f1312f9bd1 | |||
| 149f5700b4 | |||
| 38aa318fb2 | |||
| e7c5d7c198 |
@@ -57,7 +57,6 @@ rustfs-scanner 1392 20
|
||||
rustfs-scanner-metrics 90 0
|
||||
rustfs-security-governance 20 0
|
||||
rustfs-signer 34 0
|
||||
rustfs-sse 145 0
|
||||
rustfs-storage-api 34 0
|
||||
rustfs-targets 612 26
|
||||
rustfs-test-utils 1 0
|
||||
|
||||
@@ -231,7 +231,7 @@ jobs:
|
||||
# --all-targets links the same test binaries as the reader lane.
|
||||
CARGO_BUILD_JOBS: "2"
|
||||
run: |
|
||||
cargo build -p rustfs -p rustfs-ecstore -p rustfs-sse --all-targets --features rio-v2
|
||||
cargo build -p rustfs -p rustfs-ecstore --all-targets --features rio-v2
|
||||
cargo build -p rustfs --bins --features rio-v2,e2e-test-hooks
|
||||
|
||||
# Readers: the swift and sftp legs of test-and-lint-protocols. Built in
|
||||
|
||||
@@ -563,7 +563,7 @@ jobs:
|
||||
run: chmod go-w "$(realpath "$HOME")"
|
||||
|
||||
- name: Run rio-v2 clippy lints
|
||||
run: cargo clippy -p rustfs -p rustfs-ecstore -p rustfs-sse --all-targets --features rio-v2 -- -D warnings
|
||||
run: cargo clippy -p rustfs -p rustfs-ecstore --all-targets --features rio-v2 -- -D warnings
|
||||
|
||||
- name: Run rio-v2 feature tests
|
||||
shell: bash
|
||||
@@ -580,7 +580,7 @@ jobs:
|
||||
rm -f target/nextest/ci/junit.xml
|
||||
./scripts/ci/resource_sampler.sh start nextest-rio-v2
|
||||
trap './scripts/ci/resource_sampler.sh stop' EXIT
|
||||
cargo nextest run --profile ci -p rustfs -p rustfs-ecstore -p rustfs-sse --features rio-v2 \
|
||||
cargo nextest run --profile ci -p rustfs -p rustfs-ecstore --features rio-v2 \
|
||||
2>&1 | tee artifacts/workspace/nextest.log
|
||||
cargo test -p rustfs --doc --features rio-v2
|
||||
|
||||
|
||||
+1
-1
@@ -94,7 +94,7 @@ refactors.
|
||||
|--------|--------------------------|----------------|
|
||||
| Foundation | `checksums`, `common`, `config`, `data-usage`, `heal-contracts`, `scanner-metrics`, `utils` | Shared configuration, data-usage models, heal domain contracts, scanner telemetry types, utilities, and checksums. |
|
||||
| I/O and storage | `concurrency`, `ecstore`, `filemeta`, `heal`, `io-core`, `io-metrics`, `lifecycle`, `lock`, `object-capacity`, `object-data-cache`, `replication`, `rio`, `rio-v2`, `s3-client`, `scanner`, `storage-api` | Erasure-coded object storage, metadata, recovery, lifecycle, replication, locking, cache, I/O pipelines, and the engine-side S3 client for remote tier/transition targets. |
|
||||
| Security and identity | `credentials`, `crypto`, `iam`, `keystone`, `kms`, `policy`, `security-governance`, `signer`, `sse`, `tls-runtime`, `trusted-proxies` | Credentials, authentication, authorization, encryption, key management, server-side encryption key resolution, TLS, and security contracts. |
|
||||
| Security and identity | `credentials`, `crypto`, `iam`, `keystone`, `kms`, `policy`, `security-governance`, `signer`, `tls-runtime`, `trusted-proxies` | Credentials, authentication, authorization, encryption, key management, TLS, and security contracts. |
|
||||
| Protocols and contracts | `extension-schema`, `madmin`, `protos`, `protocols`, `s3-ops`, `s3-types`, `s3select-api`, `s3select-query` | Admin, inter-node, S3, S3 Select, and optional protocol contracts. |
|
||||
| Operations and integration | `audit`, `notify`, `obs`, `targets`, `zip` | Auditing, observability, event delivery, notification targets, and archive support. |
|
||||
| Test support | `e2e_test`, `test-utils` | End-to-end validation and shared test bootstrap utilities. |
|
||||
|
||||
Generated
+187
-212
File diff suppressed because it is too large
Load Diff
+19
-24
@@ -60,7 +60,6 @@ members = [
|
||||
"crates/security-governance", # Security governance contracts
|
||||
"crates/extension-schema", # Extension schema contracts
|
||||
"crates/signer", # client signer
|
||||
"crates/sse", # S3 server-side encryption key resolution and metadata
|
||||
"crates/storage-api", # Storage API contracts
|
||||
"crates/test-utils", # Shared test bootstrap helpers (dev-dependency only)
|
||||
"crates/targets", # Target-specific configurations and utilities
|
||||
@@ -141,7 +140,6 @@ rustfs-scanner = { path = "crates/scanner", version = "1.0.5" }
|
||||
rustfs-security-governance = { path = "crates/security-governance", version = "1.0.5" }
|
||||
rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.5" }
|
||||
rustfs-signer = { path = "crates/signer", version = "1.0.5" }
|
||||
rustfs-sse = { path = "crates/sse", version = "1.0.5" }
|
||||
rustfs-storage-api = { path = "crates/storage-api", version = "1.0.5" }
|
||||
rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.5" }
|
||||
rustfs-targets = { path = "crates/targets", version = "1.0.5" }
|
||||
@@ -177,9 +175,9 @@ futures = "0.3.34"
|
||||
futures-core = "0.3.34"
|
||||
futures-lite = "2.6.1"
|
||||
futures-util = "0.3.34"
|
||||
pollster = "1.0.3"
|
||||
pollster = "1.0.2"
|
||||
pulsar = { default-features = false, version = "=6.9.0" }
|
||||
lapin = { default-features = false, version = "4.13.0" }
|
||||
lapin = { default-features = false, version = "4.12.3" }
|
||||
hyper = { version = "1.12.0" }
|
||||
h2 = { version = "0.4.20" }
|
||||
hyper-rustls = { default-features = false, version = "0.27.10" }
|
||||
@@ -218,7 +216,7 @@ csv-core = "0.1.13"
|
||||
bytesize = "2.7.0"
|
||||
byteorder = "1.5.0"
|
||||
flatbuffers = "25.12.19"
|
||||
form_urlencoded = "1.2.3"
|
||||
form_urlencoded = "1.2.2"
|
||||
prost = "0.14.6"
|
||||
quick-xml = "0.42.0"
|
||||
rmp = { version = "0.8.15" }
|
||||
@@ -260,7 +258,7 @@ proptest = "1"
|
||||
# Time and Date
|
||||
chrono = { version = "0.4.45" }
|
||||
humantime = "2.4.0"
|
||||
jiff = { version = "1.0.2" }
|
||||
jiff = { version = "1.0.1" }
|
||||
time = { version = "0.3.55" }
|
||||
|
||||
# Database
|
||||
@@ -280,15 +278,15 @@ atomic_enum = "0.3.0"
|
||||
aws-config = { version = "1.13.3" }
|
||||
aws-credential-types = { version = "1.3.1" }
|
||||
aws-sdk-kms = { default-features = false, version = "1.133.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.161.0" }
|
||||
aws-sdk-sts = { default-features = false, version = "1.128.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.160.0" }
|
||||
aws-sdk-sts = { default-features = false, version = "1.127.0" }
|
||||
aws-smithy-async = { version = "1.3.1" }
|
||||
aws-smithy-http-client = { default-features = false, version = "1.5.3" }
|
||||
aws-smithy-runtime-api = { version = "1.22.0" }
|
||||
aws-smithy-types = { version = "1.9.4" }
|
||||
base64-simd = "0.8.0"
|
||||
brotli = "9.1.3"
|
||||
clap = { version = "4.6.9" }
|
||||
clap = { version = "4.6.8" }
|
||||
const-str = { version = "1.1.0" }
|
||||
convert_case = "0.12.0"
|
||||
criterion = { version = "0.8" }
|
||||
@@ -300,12 +298,12 @@ datafusion = { default-features = false, version = "55.3.0" }
|
||||
derive_builder = "0.20.2"
|
||||
ed25519-dalek = "3.0.0"
|
||||
enumset = "1.1.15"
|
||||
faster-hex = "2.0.0"
|
||||
faster-hex = "1.0.0"
|
||||
flate2 = "1.1.11"
|
||||
glob = "0.3.4"
|
||||
google-cloud-storage = "1.22.0"
|
||||
google-cloud-auth = "1.19.0"
|
||||
hashbrown = { version = "0.17.2" }
|
||||
hashbrown = { version = "0.17.1" }
|
||||
# Base32 for RFC 6238 TOTP shared secrets (RFC 4648 unpadded, the alphabet
|
||||
# every authenticator app expects). Already in the graph transitively.
|
||||
data-encoding = "2.11.1"
|
||||
@@ -339,7 +337,7 @@ percent-encoding = "2.3.2"
|
||||
# CLI needs its own QR encoder. No default features: the image/render backends
|
||||
# pull in an image stack this only needs SVG and text output from.
|
||||
qrcode-rs = { version = "2.2.0", default-features = false, features = ["std", "svg"] }
|
||||
pin-project-lite = "0.2.18"
|
||||
pin-project-lite = "0.2.17"
|
||||
pretty_assertions = "1.4.1"
|
||||
rand = { version = "0.10.4" }
|
||||
ratelimit = "2.0.0"
|
||||
@@ -362,7 +360,7 @@ legacy-s3s = { package = "s3s", git = "https://github.com/s3s-project/s3s", rev
|
||||
serial_test = "4.0.1"
|
||||
shadow-rs = { default-features = false, version = "2.0.0" }
|
||||
siphasher = "1.0.4"
|
||||
smallvec = { version = "1.16.6" }
|
||||
smallvec = { version = "1.16.5" }
|
||||
compact_str = "0.10.0"
|
||||
snap = "1.1.2"
|
||||
starshard = { version = "2.4.1" }
|
||||
@@ -380,23 +378,23 @@ tracing-error = "0.2.1"
|
||||
tracing-opentelemetry = { version = "0.34.0" }
|
||||
tracing-subscriber = { version = "0.3.23" }
|
||||
transform-stream = "0.3.1"
|
||||
url = "2.5.9"
|
||||
url = "2.5.8"
|
||||
urlencoding = "2.1.3"
|
||||
uuid = { version = "1.29.3" }
|
||||
vaultrs = { version = "0.8.0" }
|
||||
tar = "0.4.46"
|
||||
walkdir = "2.5.1"
|
||||
walkdir = "2.5.0"
|
||||
winapi-util = "0.1.11"
|
||||
windows = { version = "0.62.2" }
|
||||
windows-sys = "0.61.2"
|
||||
xxhash-rust = { version = "0.8.21" }
|
||||
zip = "9.8.1"
|
||||
zip = "9.7.0"
|
||||
zstd = "0.14.4"
|
||||
|
||||
# Observability and Metrics
|
||||
metrics = "0.24.6"
|
||||
metrics-util = "0.20"
|
||||
dial9 = { version = "0.5.9", default-features = false }
|
||||
dial9 = { version = "0.5.8", default-features = false }
|
||||
opentelemetry = { version = "0.33.1" }
|
||||
opentelemetry-appender-tracing = { version = "0.33.1" }
|
||||
opentelemetry-otlp = { version = "0.33.1" }
|
||||
@@ -407,12 +405,12 @@ opentelemetry-stdout = { version = "0.33.1" }
|
||||
pyroscope = { version = "2.1.5" }
|
||||
|
||||
# FTP and SFTP
|
||||
libunftp = { version = "0.25.1" }
|
||||
libunftp = { version = "0.25.0" }
|
||||
unftp-core = "0.1.1"
|
||||
suppaftp = { version = "13.0.0" }
|
||||
rcgen = { version = "0.14.10", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
|
||||
russh = { version = "0.65.4" }
|
||||
russh-sftp = "4.0.0"
|
||||
russh = { version = "0.65.2" }
|
||||
russh-sftp = "3.0.1"
|
||||
|
||||
# WebDAV
|
||||
dav-server = "0.12.0"
|
||||
@@ -425,10 +423,7 @@ hotpath = { version = "0.32.1", default-features = false }
|
||||
insta = { version = "1.49" }
|
||||
|
||||
[workspace.metadata.cargo-shear]
|
||||
# The `#[path]` test modules under `rustfs/src/server/http/tests/` are linked by
|
||||
# rustc (including the `http.rs` + `http/` sibling-directory rule), but cargo-shear
|
||||
# resolves those nested `#[path]` attributes against the wrong base directory.
|
||||
ignored-paths = ["rustfs/src/server/http/tests/**/*.rs"]
|
||||
ignored = ["hotpath", "rustfs"]
|
||||
|
||||
[profile.dev]
|
||||
# Full debuginfo roughly doubles compile+link time and produces multi-GB
|
||||
|
||||
@@ -15,11 +15,12 @@ workspace = true
|
||||
|
||||
[features]
|
||||
default = []
|
||||
hotpath = ["rustfs-io-core/hotpath"]
|
||||
hotpath-alloc = ["hotpath", "rustfs-io-core/hotpath-alloc"]
|
||||
hotpath-cpu = ["hotpath", "rustfs-io-core/hotpath-cpu"]
|
||||
hotpath = ["hotpath/hotpath", "hotpath/tokio", "rustfs-io-core/hotpath"]
|
||||
hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc", "rustfs-io-core/hotpath-alloc"]
|
||||
hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu", "rustfs-io-core/hotpath-cpu"]
|
||||
|
||||
[dependencies]
|
||||
hotpath.workspace = true
|
||||
# Internal crates
|
||||
rustfs-io-core = { workspace = true }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
|
||||
@@ -34,7 +34,10 @@
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
|
||||
use crate::common::{
|
||||
AdminTransport, RustFSTestEnvironment, admin_attach_user_policy_via, admin_create_user, admin_request_with_session_token,
|
||||
init_logging, local_http_client, parse_assume_role_credentials,
|
||||
};
|
||||
use hmac::{Hmac, KeyInit as _, Mac};
|
||||
use http::header::HOST;
|
||||
use rustfs_s3_types::Body;
|
||||
@@ -396,6 +399,54 @@ mod tests {
|
||||
let (status, _) = signed_request(&env.url, http::Method::GET, ACCOUNT_INFO_PATH, None, user_ak, old_sk).await?;
|
||||
assert!(status.is_client_error(), "the old secret must stop working, status: {status}");
|
||||
|
||||
env.stop_server();
|
||||
drop(env);
|
||||
console_session_rotates_the_iam_password_and_revokes_the_old_session().await
|
||||
}
|
||||
|
||||
async fn console_session_rotates_the_iam_password_and_revokes_the_old_session() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let user = "console-password-user";
|
||||
let old_password = uuid::Uuid::new_v4().to_string();
|
||||
let new_password = uuid::Uuid::new_v4().to_string();
|
||||
admin_create_user(&env, user, &old_password).await?;
|
||||
admin_attach_user_policy_via(AdminTransport::Signed, &env.url, &env.access_key, &env.secret_key, "readonly", user)
|
||||
.await?;
|
||||
|
||||
let (status, response) = assume_role(&env.url, user, &old_password, None).await?;
|
||||
assert_eq!(status, reqwest::StatusCode::OK, "the current password must sign in");
|
||||
let (session_key, session_secret, token) = parse_assume_role_credentials(&response)?;
|
||||
let session_request = |method, path, body| {
|
||||
admin_request_with_session_token(&env.url, method, path, body, &session_key, &session_secret, Some(&token))
|
||||
};
|
||||
let password_request = |current: &str| {
|
||||
Some(serde_json::json!({ "current_secret_key": current, "new_secret_key": new_password }).to_string())
|
||||
};
|
||||
|
||||
let (status, body) =
|
||||
session_request(http::Method::POST, ACCOUNT_PASSWORD_PATH, password_request(&session_secret)).await?;
|
||||
assert_eq!(
|
||||
status,
|
||||
reqwest::StatusCode::BAD_REQUEST,
|
||||
"the session secret must not be accepted as the IAM password"
|
||||
);
|
||||
assert!(body.contains("InvalidRequest") && body.contains("the current secret key is incorrect"));
|
||||
|
||||
let (status, body) = session_request(http::Method::POST, ACCOUNT_PASSWORD_PATH, password_request(&old_password)).await?;
|
||||
assert_eq!(status, reqwest::StatusCode::OK, "the same session must accept the current IAM password");
|
||||
assert_eq!(serde_json::from_str::<serde_json::Value>(&body)?["sessions_revoked"], 1);
|
||||
|
||||
let (status, _) = session_request(http::Method::GET, ACCOUNT_INFO_PATH, None).await?;
|
||||
assert_eq!(status, reqwest::StatusCode::FORBIDDEN, "the pre-rotation session must be revoked");
|
||||
let (status, _) = assume_role(&env.url, user, &old_password, None).await?;
|
||||
assert_eq!(status, reqwest::StatusCode::FORBIDDEN, "the old password must no longer sign in");
|
||||
let (status, response) = assume_role(&env.url, user, &new_password, None).await?;
|
||||
assert_eq!(status, reqwest::StatusCode::OK, "the new password must sign in");
|
||||
parse_assume_role_credentials(&response)?;
|
||||
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -183,6 +183,7 @@ rmp.workspace = true
|
||||
rmp-serde.workspace = true
|
||||
tokio-util = { workspace = true, features = ["io", "compat", "rt"] }
|
||||
tokio-stream = { workspace = true, features = ["sync"] }
|
||||
hmac = { workspace = true }
|
||||
sha2 = { workspace = true }
|
||||
hex-simd = { workspace = true }
|
||||
tempfile.workspace = true
|
||||
|
||||
@@ -1786,7 +1786,7 @@ where
|
||||
}
|
||||
|
||||
pub async fn update_user_secret_key(&self, access_key: &str, secret_key: &str) -> Result<(OffsetDateTime, AccountStatus)> {
|
||||
self.update_user_secret_key_with_guard(access_key, secret_key, None)
|
||||
self.update_user_secret_key_with_guard(access_key, secret_key, None, None)
|
||||
.await
|
||||
.map(|(updated_at, status, _guard)| (updated_at, status))
|
||||
}
|
||||
@@ -1796,6 +1796,7 @@ where
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
expected_secret: Option<&str>,
|
||||
session: Option<&Credentials>,
|
||||
) -> Result<(
|
||||
OffsetDateTime,
|
||||
AccountStatus,
|
||||
@@ -1824,6 +1825,51 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
// Lock order is parent, then the presenting STS credential. Deletion and
|
||||
// session refresh use the same STS lock. Release it before the revocation sweep.
|
||||
let _session_guard = match session {
|
||||
Some(session) => {
|
||||
let guard = self.cache.sts_account_mutation_lock(&session.access_key).lock_owned().await;
|
||||
let current = self
|
||||
.api
|
||||
.load_user_identity(&session.access_key, UserType::Sts)
|
||||
.await
|
||||
.map_err(|error| {
|
||||
if is_err_no_such_user(&error) {
|
||||
Error::IAMActionNotAllowed
|
||||
} else {
|
||||
error
|
||||
}
|
||||
})?;
|
||||
let parent = if current.credentials.parent_user.is_empty() {
|
||||
current.credentials.claims_or_empty().get("parent").and_then(Value::as_str)
|
||||
} else {
|
||||
Some(current.credentials.parent_user.as_str())
|
||||
};
|
||||
use subtle::ConstantTimeEq as _;
|
||||
if !session.is_valid()
|
||||
|| !current.credentials.is_valid()
|
||||
|| !current.credentials.is_temp()
|
||||
|| current.credentials.is_service_account()
|
||||
|| current.credentials.access_key != session.access_key
|
||||
|| parent != Some(access_key)
|
||||
|| !bool::from(current.credentials.secret_key.as_bytes().ct_eq(session.secret_key.as_bytes()))
|
||||
|| !bool::from(
|
||||
current
|
||||
.credentials
|
||||
.session_token
|
||||
.as_bytes()
|
||||
.ct_eq(session.session_token.as_bytes()),
|
||||
)
|
||||
|| current.sts_generation != u.sts_generation
|
||||
{
|
||||
return Err(Error::IAMActionNotAllowed);
|
||||
}
|
||||
Some(guard)
|
||||
}
|
||||
None => None,
|
||||
};
|
||||
|
||||
let mut cred = u.credentials.clone();
|
||||
cred.secret_key = secret_key.to_string();
|
||||
|
||||
|
||||
+183
-6
@@ -1421,11 +1421,11 @@ impl<T: Store> IamSys<T> {
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
) -> Result<(OffsetDateTime, rustfs_madmin::AccountStatus, usize)> {
|
||||
self.rotate_user_secret_key_checked(access_key, secret_key, None).await
|
||||
self.rotate_user_secret_key_checked(access_key, secret_key, None, None).await
|
||||
}
|
||||
|
||||
/// Change an enabled regular caller's secret only if its signing secret is
|
||||
/// still current under the parent mutation lock. Status and policies are kept.
|
||||
/// Change an enabled regular user's secret only if the supplied current
|
||||
/// secret still matches under the parent mutation lock. Status and policies are kept.
|
||||
/// Persistence and revocation continue after caller cancellation.
|
||||
pub async fn change_own_secret_key(
|
||||
&self,
|
||||
@@ -1433,7 +1433,27 @@ impl<T: Store> IamSys<T> {
|
||||
expected_secret: &str,
|
||||
secret_key: &str,
|
||||
) -> Result<(OffsetDateTime, rustfs_madmin::AccountStatus, usize)> {
|
||||
self.rotate_user_secret_key_checked(access_key, secret_key, Some(expected_secret))
|
||||
self.rotate_user_secret_key_checked(access_key, secret_key, Some(expected_secret), None)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Change a local IAM password using an authenticated STS session and its parent password.
|
||||
///
|
||||
/// The parent and session mutation locks protect revalidation through the secret write.
|
||||
/// Persistence and session revocation continue after request cancellation.
|
||||
/// A revocation failure is returned after the new secret has been persisted.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns `IAMActionNotAllowed` for inactive sessions, parent mismatches, or an
|
||||
/// incorrect current password. Storage and revocation errors are propagated.
|
||||
pub async fn change_own_secret_key_from_session(
|
||||
&self,
|
||||
access_key: &str,
|
||||
expected_secret: &str,
|
||||
secret_key: &str,
|
||||
session: &Credentials,
|
||||
) -> Result<(OffsetDateTime, rustfs_madmin::AccountStatus, usize)> {
|
||||
self.rotate_user_secret_key_checked(access_key, secret_key, Some(expected_secret), Some(session))
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -1442,16 +1462,18 @@ impl<T: Store> IamSys<T> {
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
expected_secret: Option<&str>,
|
||||
session: Option<&Credentials>,
|
||||
) -> Result<(OffsetDateTime, rustfs_madmin::AccountStatus, usize)> {
|
||||
let runtime = tokio::runtime::Handle::try_current().map_err(Error::other)?;
|
||||
let iam = self.clone();
|
||||
let access_key = access_key.to_string();
|
||||
let secret_key = secret_key.to_string();
|
||||
let expected_secret = expected_secret.map(str::to_string);
|
||||
let session = session.cloned();
|
||||
#[cfg(test)]
|
||||
let notification_probe = crate::LOAD_USER_NOTIFICATION_PROBE.try_with(Arc::clone).ok();
|
||||
let operation = async move {
|
||||
iam.rotate_user_secret_key_inner(&access_key, &secret_key, expected_secret.as_deref())
|
||||
iam.rotate_user_secret_key_inner(&access_key, &secret_key, expected_secret.as_deref(), session.as_ref())
|
||||
.await
|
||||
};
|
||||
#[cfg(test)]
|
||||
@@ -1471,6 +1493,7 @@ impl<T: Store> IamSys<T> {
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
expected_secret: Option<&str>,
|
||||
session: Option<&Credentials>,
|
||||
) -> Result<(OffsetDateTime, rustfs_madmin::AccountStatus, usize)> {
|
||||
if !is_access_key_valid(access_key) {
|
||||
return Err(IamError::InvalidAccessKeyLength);
|
||||
@@ -1482,7 +1505,7 @@ impl<T: Store> IamSys<T> {
|
||||
|
||||
let (updated_at, status, parent_guard) = self
|
||||
.store
|
||||
.update_user_secret_key_with_guard(access_key, secret_key, expected_secret)
|
||||
.update_user_secret_key_with_guard(access_key, secret_key, expected_secret, session)
|
||||
.await?;
|
||||
let revoked = self.revoke_sts_sessions_for_parent(access_key).await?;
|
||||
drop(parent_guard);
|
||||
@@ -4815,6 +4838,160 @@ mod tests {
|
||||
assert!(matches!(issuer.check_key(&legacy.access_key).await, Ok((None, false))));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn own_secret_change_revalidates_the_presenting_session_under_the_parent_lock() {
|
||||
ensure_test_global_credentials();
|
||||
for mutation in ["revoke", "expire", "disable", "parent", "secret", "token", "generation"] {
|
||||
let store = StsTestMockStore::new(false);
|
||||
let iam = Arc::new(IamSys::new(IamCache::new(store.clone()).await.expect("IAM")));
|
||||
let parent = "session-change-parent";
|
||||
let password = "currentParentSecret123";
|
||||
iam.create_user(
|
||||
parent,
|
||||
&AddOrUpdateUserReq {
|
||||
secret_key: password.to_string(),
|
||||
policy: None,
|
||||
status: rustfs_madmin::AccountStatus::Enabled,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create parent");
|
||||
let session = rotation_test_session(parent, "password-session");
|
||||
let owner = iam.get_user(parent).await.expect("parent").credentials;
|
||||
iam.set_temp_user_from_parent(&session, &owner)
|
||||
.await
|
||||
.expect("publish session");
|
||||
|
||||
let guard = iam.store.cache.parent_mutation_lock(parent).lock_owned().await;
|
||||
let pending = {
|
||||
let iam = Arc::clone(&iam);
|
||||
let session = session.clone();
|
||||
tokio::spawn(async move {
|
||||
iam.change_own_secret_key_from_session(parent, password, "replacementParentSecret123", &session)
|
||||
.await
|
||||
})
|
||||
};
|
||||
if mutation == "revoke" {
|
||||
iam.delete_temp_account(&session.access_key, false)
|
||||
.await
|
||||
.expect("revoke presenting session");
|
||||
} else {
|
||||
let mut saved = store.saved_sts_users.lock().expect("saved identities");
|
||||
let current = saved.get_mut(&session.access_key).expect("durable session");
|
||||
match mutation {
|
||||
"expire" => current.credentials.expiration = Some(OffsetDateTime::UNIX_EPOCH),
|
||||
"disable" => current.credentials.status = "off".to_string(),
|
||||
"parent" => current.credentials.parent_user = "another-parent".to_string(),
|
||||
"secret" => current.credentials.secret_key = "replacementSessionSecret123".to_string(),
|
||||
"token" => current.credentials.session_token = "replacement-token".to_string(),
|
||||
"generation" => {
|
||||
assert!(current.sts_generation.take().is_some());
|
||||
}
|
||||
_ => unreachable!("known fixture mutation"),
|
||||
}
|
||||
}
|
||||
drop(guard);
|
||||
let error = tokio::time::timeout(std::time::Duration::from_secs(5), pending)
|
||||
.await
|
||||
.expect("queued rotation finishes")
|
||||
.expect("rotation task")
|
||||
.expect_err("stale session must fail");
|
||||
assert_eq!(error, Error::IAMActionNotAllowed, "{mutation}");
|
||||
let saved = store.saved_sts_users.lock().expect("saved identities");
|
||||
assert_eq!(saved.get(parent).expect("parent survives").credentials.secret_key, password, "{mutation}");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn session_secret_change_uses_the_durable_parent_password_with_a_stale_cache() {
|
||||
let iam = test_iam_sys().await;
|
||||
let parent = "stale-password-parent";
|
||||
iam.create_user(
|
||||
parent,
|
||||
&AddOrUpdateUserReq {
|
||||
secret_key: "cachedParentSecret123".to_string(),
|
||||
policy: None,
|
||||
status: rustfs_madmin::AccountStatus::Enabled,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create parent");
|
||||
let session = rotation_test_session(parent, "fresh-password-session");
|
||||
let owner = iam.get_user(parent).await.expect("cached parent").credentials;
|
||||
iam.set_temp_user_from_parent(&session, &owner)
|
||||
.await
|
||||
.expect("publish session");
|
||||
iam.store
|
||||
.api
|
||||
.saved_sts_users
|
||||
.lock()
|
||||
.expect("saved identities")
|
||||
.get_mut(parent)
|
||||
.expect("durable parent")
|
||||
.credentials
|
||||
.secret_key = "durableParentSecret123".to_string();
|
||||
assert_eq!(
|
||||
iam.store
|
||||
.get_user(parent)
|
||||
.await
|
||||
.expect("stale cached parent")
|
||||
.credentials
|
||||
.secret_key,
|
||||
"cachedParentSecret123"
|
||||
);
|
||||
let probe = IamSys::<StsTestMockStore>::load_user_notification_probe(0, false, false);
|
||||
crate::LOAD_USER_NOTIFICATION_PROBE
|
||||
.scope(
|
||||
probe,
|
||||
iam.change_own_secret_key_from_session(parent, "durableParentSecret123", "replacementParentSecret123", &session),
|
||||
)
|
||||
.await
|
||||
.expect("durable current password authorizes the change");
|
||||
assert_eq!(
|
||||
iam.get_user(parent).await.expect("updated parent").credentials.secret_key,
|
||||
"replacementParentSecret123"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn current_session_can_change_its_parent_secret_and_revoke_itself() {
|
||||
let iam = test_iam_sys().await;
|
||||
let parent = "current-session-parent";
|
||||
let password = "currentParentSecret123";
|
||||
iam.create_user(
|
||||
parent,
|
||||
&AddOrUpdateUserReq {
|
||||
secret_key: password.to_string(),
|
||||
policy: None,
|
||||
status: rustfs_madmin::AccountStatus::Enabled,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create parent");
|
||||
let session = rotation_test_session(parent, "current-password-session");
|
||||
let owner = iam.get_user(parent).await.expect("parent").credentials;
|
||||
iam.set_temp_user_from_parent(&session, &owner)
|
||||
.await
|
||||
.expect("publish session");
|
||||
let probe = IamSys::<StsTestMockStore>::load_user_notification_probe(0, false, false);
|
||||
let (_, _, revoked) = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(5),
|
||||
crate::LOAD_USER_NOTIFICATION_PROBE.scope(
|
||||
probe,
|
||||
iam.change_own_secret_key_from_session(parent, password, "replacementParentSecret123", &session),
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("rotation must release its session lock before revocation")
|
||||
.expect("rotate current session");
|
||||
assert_eq!(revoked, 1);
|
||||
assert!(iam.get_user(&session.access_key).await.is_none());
|
||||
assert_eq!(
|
||||
iam.get_user(parent).await.expect("rotated parent").credentials.secret_key,
|
||||
"replacementParentSecret123"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn self_secret_change_cannot_undo_concurrent_disable_rotation_or_deletion() {
|
||||
ensure_test_global_credentials();
|
||||
|
||||
@@ -29,11 +29,12 @@ workspace = true
|
||||
|
||||
[features]
|
||||
default = []
|
||||
hotpath = ["rustfs-io-metrics/hotpath"]
|
||||
hotpath-alloc = ["hotpath", "rustfs-io-metrics/hotpath-alloc"]
|
||||
hotpath-cpu = ["hotpath", "rustfs-io-metrics/hotpath-cpu"]
|
||||
hotpath = ["hotpath/hotpath", "hotpath/tokio", "rustfs-io-metrics/hotpath"]
|
||||
hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc", "rustfs-io-metrics/hotpath-alloc"]
|
||||
hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu", "rustfs-io-metrics/hotpath-cpu"]
|
||||
|
||||
[dependencies]
|
||||
hotpath.workspace = true
|
||||
bytes = { workspace = true, features = ["serde"] }
|
||||
thiserror = { workspace = true }
|
||||
tokio = { workspace = true, features = ["io-util", "fs", "sync", "rt-multi-thread"] }
|
||||
|
||||
@@ -6,8 +6,7 @@ Applies to `crates/kms/`.
|
||||
|
||||
When changing key-management behavior, verify compatibility with:
|
||||
|
||||
- `crates/sse/` (SSE key resolution; consumes this crate on the data path)
|
||||
- `rustfs/src/storage/sse.rs` (binds `crates/sse/` to the running server)
|
||||
- `rustfs/src/storage/sse.rs`
|
||||
- `rustfs/src/admin/handlers/kms.rs`
|
||||
- `rustfs/src/admin/handlers/kms_dynamic.rs`
|
||||
- `rustfs/src/admin/handlers/kms_keys.rs`
|
||||
|
||||
@@ -410,10 +410,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
|
||||
}
|
||||
};
|
||||
let handle = self.allocate_handle(HandleState::Dir(cursor))?;
|
||||
Ok(Handle {
|
||||
id,
|
||||
handle: handle.into(),
|
||||
})
|
||||
Ok(Handle { id, handle })
|
||||
}
|
||||
|
||||
/// READDIR body shared with the Handler trait wrapper. Removes the
|
||||
@@ -669,11 +666,8 @@ mod tests {
|
||||
assert_eq!(auth.continuation_token, call.continuation_token);
|
||||
}
|
||||
assert_eq!(authorized[2].continuation_token.as_ref().map(|value| value.as_str()), Some("next-page"));
|
||||
let denied = with_test_auth_override(
|
||||
|_, _, prefix| prefix != Some("private/"),
|
||||
driver.readdir_inner(2, String::from_utf8(handle.handle.into_bytes()).expect("utf8 handle")),
|
||||
)
|
||||
.await;
|
||||
let denied =
|
||||
with_test_auth_override(|_, _, prefix| prefix != Some("private/"), driver.readdir_inner(2, handle.handle)).await;
|
||||
assert!(denied.is_err(), "policy change must deny the next READDIR");
|
||||
assert_eq!(backend.list_objects_calls().len(), 2, "denied READDIR cannot reach storage");
|
||||
}
|
||||
@@ -722,7 +716,7 @@ mod tests {
|
||||
};
|
||||
let handle_id = driver.allocate_handle(HandleState::Dir(cursor)).expect("allocate");
|
||||
|
||||
let readdir_fut = driver.readdir(1, handle_id.clone().into());
|
||||
let readdir_fut = driver.readdir(1, handle_id.clone());
|
||||
|
||||
with_test_auth_override(|_, _, _| true, async {
|
||||
tokio::select! {
|
||||
@@ -764,7 +758,7 @@ mod tests {
|
||||
// token, no skipped entries).
|
||||
backend.clear_stall_list_objects_v2();
|
||||
backend.queue_list_objects_v2_ok_empty();
|
||||
let result = with_test_auth_override(|_, _, _| true, driver.readdir(2, handle_id.into())).await;
|
||||
let result = with_test_auth_override(|_, _, _| true, driver.readdir(2, handle_id)).await;
|
||||
// Empty page returns Eof per readdir's empty-Name-to-Eof translation.
|
||||
let err = result.expect_err("re-issued READDIR against an empty listing must return Eof, not Ok");
|
||||
assert!(
|
||||
@@ -791,7 +785,7 @@ mod tests {
|
||||
let handle_id = driver.allocate_handle(HandleState::Dir(cursor)).expect("allocate");
|
||||
|
||||
let (result, captured) =
|
||||
capture_tracing_at(Level::ERROR, with_test_auth_override(|_, _, _| true, driver.readdir(7, handle_id.into()))).await;
|
||||
capture_tracing_at(Level::ERROR, with_test_auth_override(|_, _, _| true, driver.readdir(7, handle_id))).await;
|
||||
let err = result.expect_err("exhausted cursor must return Eof");
|
||||
assert!(matches!(StatusCode::from(err), StatusCode::Eof));
|
||||
assert!(
|
||||
@@ -824,7 +818,7 @@ mod tests {
|
||||
let handle_id = driver.allocate_handle(HandleState::Dir(cursor)).expect("allocate");
|
||||
|
||||
let (result, captured) =
|
||||
capture_tracing_at(Level::ERROR, with_test_auth_override(|_, _, _| true, driver.readdir(8, handle_id.into()))).await;
|
||||
capture_tracing_at(Level::ERROR, with_test_auth_override(|_, _, _| true, driver.readdir(8, handle_id))).await;
|
||||
let err = result.expect_err("backend error must propagate as Err");
|
||||
assert!(!matches!(StatusCode::from(err), StatusCode::Eof), "backend error must not be Eof");
|
||||
assert!(
|
||||
|
||||
@@ -33,9 +33,7 @@ use super::write::{build_write_tombstone, fstat_reported_size, rejects_excl_or_t
|
||||
use crate::common::client::s3::StorageBackend;
|
||||
use crate::common::gateway::{AuthorizationError, S3Action};
|
||||
use crate::common::session::SessionContext;
|
||||
use russh_sftp::protocol::{
|
||||
Attrs, Data, File, FileAttributes, Handle, Name, OpenFlags, Packet, SftpHandle, Status, StatusCode, Version,
|
||||
};
|
||||
use russh_sftp::protocol::{Attrs, Data, File, FileAttributes, Handle, Name, OpenFlags, Packet, Status, StatusCode, Version};
|
||||
use rustfs_credentials::Credentials;
|
||||
use rustfs_gateway_types::dto::{AbortMultipartUploadInput, CopyObjectInput, ListObjectsV2Input};
|
||||
use rustfs_gateway_types::{BucketName, ObjectKey, UploadIdClaim};
|
||||
@@ -210,12 +208,6 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
|
||||
}
|
||||
}
|
||||
|
||||
/// russh-sftp 4 hands the Handler opaque `SftpHandle` bytes; the table
|
||||
/// keys remain the UTF-8 UUID strings this driver mints.
|
||||
fn handle_table_key(handle: SftpHandle) -> Result<String, SftpError> {
|
||||
String::from_utf8(handle.into_bytes()).map_err(|_| SftpError::code(StatusCode::Failure))
|
||||
}
|
||||
|
||||
/// Generate a fresh UUID v4 handle, insert the given state into the
|
||||
/// per-session handle table, and return the handle string. Enforces
|
||||
/// self.handles_per_session before any UUID generation. Cap-exceeded
|
||||
@@ -502,9 +494,8 @@ impl<S: StorageBackend + Send + Sync + 'static> russh_sftp::server::Handler for
|
||||
/// Streaming returns (next_part_number - 1) * part_size + buffer
|
||||
/// length. Failed returns the size recorded at the most recent
|
||||
/// successful write. An unknown handle returns Failure.
|
||||
#[tracing::instrument(level = "debug", skip(self), fields(id, handle = ?handle), err(Debug))]
|
||||
async fn fstat(&mut self, id: u32, handle: SftpHandle) -> Result<Attrs, Self::Error> {
|
||||
let handle = Self::handle_table_key(handle)?;
|
||||
#[tracing::instrument(level = "debug", skip(self), fields(id, handle = %handle), err(Debug))]
|
||||
async fn fstat(&mut self, id: u32, handle: String) -> Result<Attrs, Self::Error> {
|
||||
self.session_diag.stamp();
|
||||
let part_size = self.part_size;
|
||||
let result = self.with_handle_ref(&handle, |state| match state {
|
||||
@@ -550,9 +541,8 @@ impl<S: StorageBackend + Send + Sync + 'static> russh_sftp::server::Handler for
|
||||
/// exhaustion, so it is normal control flow on this handler. The
|
||||
/// instrument attribute therefore omits err(Debug) and non-Eof
|
||||
/// failures are surfaced via the explicit error log below.
|
||||
#[tracing::instrument(level = "debug", skip(self), fields(id, handle = ?handle))]
|
||||
async fn readdir(&mut self, id: u32, handle: SftpHandle) -> Result<Name, Self::Error> {
|
||||
let handle = Self::handle_table_key(handle)?;
|
||||
#[tracing::instrument(level = "debug", skip(self), fields(id, handle = %handle))]
|
||||
async fn readdir(&mut self, id: u32, handle: String) -> Result<Name, Self::Error> {
|
||||
self.session_diag.stamp();
|
||||
let result = self.readdir_inner(id, handle.clone()).await;
|
||||
self.session_diag.stamp();
|
||||
@@ -627,9 +617,8 @@ impl<S: StorageBackend + Send + Sync + 'static> russh_sftp::server::Handler for
|
||||
/// read-past-end-of-file, so it is normal control flow on this
|
||||
/// handler. The instrument attribute therefore omits err(Debug) and
|
||||
/// non-Eof failures are surfaced via the explicit error log below.
|
||||
#[tracing::instrument(level = "debug", skip(self), fields(id, handle = ?handle, offset, len))]
|
||||
async fn read(&mut self, id: u32, handle: SftpHandle, offset: u64, len: u32) -> Result<Data, Self::Error> {
|
||||
let handle = Self::handle_table_key(handle)?;
|
||||
#[tracing::instrument(level = "debug", skip(self), fields(id, handle = %handle, offset, len))]
|
||||
async fn read(&mut self, id: u32, handle: String, offset: u64, len: u32) -> Result<Data, Self::Error> {
|
||||
self.session_diag.stamp();
|
||||
let result = self.read_inner(id, handle.clone(), offset, len).await;
|
||||
self.session_diag.stamp();
|
||||
@@ -661,9 +650,8 @@ impl<S: StorageBackend + Send + Sync + 'static> russh_sftp::server::Handler for
|
||||
///
|
||||
/// A missing handle is treated as Ok to tolerate clients that
|
||||
/// double-close on session teardown.
|
||||
#[tracing::instrument(level = "info", skip(self), fields(id, handle = ?handle), err(Debug))]
|
||||
async fn close(&mut self, id: u32, handle: SftpHandle) -> Result<Status, Self::Error> {
|
||||
let handle = Self::handle_table_key(handle)?;
|
||||
#[tracing::instrument(level = "info", skip(self), fields(id, handle = %handle), err(Debug))]
|
||||
async fn close(&mut self, id: u32, handle: String) -> Result<Status, Self::Error> {
|
||||
self.session_diag.stamp();
|
||||
let removed = self.handles.remove(&handle);
|
||||
let Some(HandleState::Write {
|
||||
@@ -767,9 +755,8 @@ impl<S: StorageBackend + Send + Sync + 'static> russh_sftp::server::Handler for
|
||||
/// to Streaming by issuing CreateMultipartUpload. A Failed handle
|
||||
/// rejects every subsequent write with the same status that caused
|
||||
/// the failure.
|
||||
#[tracing::instrument(level = "debug", skip(self, data), fields(id, handle = ?handle, offset, len = data.len()), err(Debug))]
|
||||
async fn write(&mut self, id: u32, handle: SftpHandle, offset: u64, data: Vec<u8>) -> Result<Status, Self::Error> {
|
||||
let handle = Self::handle_table_key(handle)?;
|
||||
#[tracing::instrument(level = "debug", skip(self, data), fields(id, handle = %handle, offset, len = data.len()), err(Debug))]
|
||||
async fn write(&mut self, id: u32, handle: String, offset: u64, data: Vec<u8>) -> Result<Status, Self::Error> {
|
||||
self.session_diag.stamp();
|
||||
self.enforce_server_readonly()?;
|
||||
|
||||
@@ -996,8 +983,8 @@ impl<S: StorageBackend + Send + Sync + 'static> russh_sftp::server::Handler for
|
||||
/// SSH_FXP_FSETSTAT, SFTP Internet Draft section 6.6. Same rationale
|
||||
/// as setstat: S3 cannot honour POSIX attributes, and clients use
|
||||
/// FSETSTAT during transfers to stamp the in-flight handle.
|
||||
#[tracing::instrument(level = "debug", skip(self, _attrs), fields(id, handle = ?_handle), err(Debug))]
|
||||
async fn fsetstat(&mut self, id: u32, _handle: SftpHandle, _attrs: FileAttributes) -> Result<Status, Self::Error> {
|
||||
#[tracing::instrument(level = "debug", skip(self, _attrs), fields(id, handle = %_handle), err(Debug))]
|
||||
async fn fsetstat(&mut self, id: u32, _handle: String, _attrs: FileAttributes) -> Result<Status, Self::Error> {
|
||||
self.enforce_server_readonly()?;
|
||||
Ok(ok_status(id))
|
||||
}
|
||||
@@ -1267,7 +1254,7 @@ mod tests {
|
||||
let handle_id = driver
|
||||
.allocate_handle(file_handle("bucket", "k", 1234, attrs.clone()))
|
||||
.expect("allocate");
|
||||
let out = driver.fstat(4, handle_id.into()).await.expect("fstat on File must succeed");
|
||||
let out = driver.fstat(4, handle_id).await.expect("fstat on File must succeed");
|
||||
assert_eq!(out.attrs.size, Some(1234));
|
||||
assert_eq!(out.attrs.mtime, Some(1_700_000_000));
|
||||
}
|
||||
@@ -1280,7 +1267,7 @@ mod tests {
|
||||
part_buffer: vec![0u8; 4096],
|
||||
};
|
||||
let handle_id = driver.allocate_handle(write_handle("bucket", "k", phase)).expect("allocate");
|
||||
let out = driver.fstat(5, handle_id.into()).await.expect("fstat on Write must succeed");
|
||||
let out = driver.fstat(5, handle_id).await.expect("fstat on Write must succeed");
|
||||
assert_eq!(
|
||||
out.attrs.size,
|
||||
Some(4096),
|
||||
@@ -1296,7 +1283,7 @@ mod tests {
|
||||
.allocate_handle(file_handle("bucket", "k", 0, FileAttributes::default()))
|
||||
.expect("allocate");
|
||||
let status = driver
|
||||
.fsetstat(6, handle_id.into(), FileAttributes::default())
|
||||
.fsetstat(6, handle_id, FileAttributes::default())
|
||||
.await
|
||||
.expect("fsetstat must succeed on any attrs");
|
||||
assert!(matches!(status.status_code, StatusCode::Ok));
|
||||
@@ -1430,7 +1417,7 @@ mod tests {
|
||||
let handle_id = driver
|
||||
.allocate_handle(file_handle("bucket", "k", 0, FileAttributes::default()))
|
||||
.expect("allocate");
|
||||
let result = driver.fsetstat(10, handle_id.into(), FileAttributes::default()).await;
|
||||
let result = driver.fsetstat(10, handle_id, FileAttributes::default()).await;
|
||||
match result {
|
||||
Err(err) => assert!(matches!(err.0, StatusCode::PermissionDenied)),
|
||||
Ok(_) => panic!("fsetstat must error in read-only mode"),
|
||||
|
||||
@@ -69,10 +69,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
|
||||
attrs,
|
||||
read_cache,
|
||||
})?;
|
||||
Ok(Handle {
|
||||
id,
|
||||
handle: handle.into(),
|
||||
})
|
||||
Ok(Handle { id, handle })
|
||||
}
|
||||
|
||||
/// Body of the SSH_FXP_READ handler. Returns up to len bytes starting
|
||||
@@ -259,7 +256,7 @@ mod tests {
|
||||
.allocate_handle(file_handle("bucket", "secret.txt", 4096, FileAttributes::default()))
|
||||
.expect("allocate read handle");
|
||||
driver.try_populate_read_cache(&handle, 0, vec![42; 4096]);
|
||||
let error = with_test_auth_override(|_, _, _| true, driver.read(1, handle.into(), 0, 1024))
|
||||
let error = with_test_auth_override(|_, _, _| true, driver.read(1, handle, 0, 1024))
|
||||
.await
|
||||
.expect_err("backend policy denial must prevent cached content disclosure");
|
||||
assert!(matches!(StatusCode::from(error), StatusCode::PermissionDenied));
|
||||
@@ -273,7 +270,7 @@ mod tests {
|
||||
.allocate_handle(file_handle("bucket", "k", 100, FileAttributes::default()))
|
||||
.expect("allocate");
|
||||
let err = driver
|
||||
.read(1, handle_id.into(), 0, 0)
|
||||
.read(1, handle_id, 0, 0)
|
||||
.await
|
||||
.expect_err("len=0 must return BadMessage");
|
||||
assert!(matches!(StatusCode::from(err), StatusCode::BadMessage));
|
||||
@@ -287,7 +284,7 @@ mod tests {
|
||||
.allocate_handle(file_handle("bucket", "k", 10, FileAttributes::default()))
|
||||
.expect("allocate");
|
||||
let err = driver
|
||||
.read(2, handle_id.into(), 10, 4)
|
||||
.read(2, handle_id, 10, 4)
|
||||
.await
|
||||
.expect_err("offset==size must return Eof");
|
||||
assert!(matches!(StatusCode::from(err), StatusCode::Eof));
|
||||
@@ -302,7 +299,7 @@ mod tests {
|
||||
.allocate_handle(file_handle("bucket", "k", 5, FileAttributes::default()))
|
||||
.expect("allocate");
|
||||
|
||||
let data = with_test_auth_override(|_, _, _| true, driver.read(3, handle_id.into(), 0, 1024))
|
||||
let data = with_test_auth_override(|_, _, _| true, driver.read(3, handle_id, 0, 1024))
|
||||
.await
|
||||
.expect("read must succeed");
|
||||
assert_eq!(data.data, b"hello".to_vec());
|
||||
@@ -320,7 +317,7 @@ mod tests {
|
||||
.allocate_handle(file_handle("bucket", "k", 10, FileAttributes::default()))
|
||||
.expect("allocate");
|
||||
|
||||
let (result, captured) = capture_tracing_at(Level::ERROR, async { driver.read(11, handle_id.into(), 10, 4).await }).await;
|
||||
let (result, captured) = capture_tracing_at(Level::ERROR, async { driver.read(11, handle_id, 10, 4).await }).await;
|
||||
let err = result.expect_err("offset==size must return Eof");
|
||||
assert!(matches!(StatusCode::from(err), StatusCode::Eof));
|
||||
assert!(
|
||||
@@ -345,11 +342,8 @@ mod tests {
|
||||
.allocate_handle(file_handle("bucket", "k", 1024, FileAttributes::default()))
|
||||
.expect("allocate");
|
||||
|
||||
let (result, captured) = capture_tracing_at(
|
||||
Level::ERROR,
|
||||
with_test_auth_override(|_, _, _| true, driver.read(12, handle_id.into(), 0, 256)),
|
||||
)
|
||||
.await;
|
||||
let (result, captured) =
|
||||
capture_tracing_at(Level::ERROR, with_test_auth_override(|_, _, _| true, driver.read(12, handle_id, 0, 256))).await;
|
||||
let err = result.expect_err("backend error must propagate as Err");
|
||||
assert!(!matches!(StatusCode::from(err), StatusCode::Eof), "backend error must not be Eof");
|
||||
assert!(
|
||||
@@ -385,7 +379,7 @@ mod tests {
|
||||
let start = Instant::now();
|
||||
let outcome = tokio::time::timeout(
|
||||
Duration::from_secs(10),
|
||||
with_test_auth_override(|_, _, _| true, driver.read(14, handle_id.into(), 0, 4096)),
|
||||
with_test_auth_override(|_, _, _| true, driver.read(14, handle_id, 0, 4096)),
|
||||
)
|
||||
.await;
|
||||
let elapsed = start.elapsed();
|
||||
@@ -431,7 +425,7 @@ mod tests {
|
||||
let mut assembled: Vec<u8> = Vec::with_capacity(object_size as usize);
|
||||
let mut reads: u32 = 0;
|
||||
while offset < object_size {
|
||||
let data = with_test_auth_override(|_, _, _| true, driver.read(20 + reads, handle_id.clone().into(), offset, chunk))
|
||||
let data = with_test_auth_override(|_, _, _| true, driver.read(20 + reads, handle_id.clone(), offset, chunk))
|
||||
.await
|
||||
.expect("read inside the cached window must succeed without a second backend call");
|
||||
assert!(!data.data.is_empty(), "non-empty hit");
|
||||
@@ -467,21 +461,21 @@ mod tests {
|
||||
.expect("allocate");
|
||||
|
||||
// First read fetches from the backend and populates window 1.
|
||||
let r1 = with_test_auth_override(|_, _, _| true, driver.read(30, handle_id.clone().into(), 0, 1024))
|
||||
let r1 = with_test_auth_override(|_, _, _| true, driver.read(30, handle_id.clone(), 0, 1024))
|
||||
.await
|
||||
.expect("first read must succeed");
|
||||
assert!(r1.data.iter().all(|b| *b == 0xAA), "first read must come from window 1");
|
||||
|
||||
// Second read inside the cached chunk is served from the
|
||||
// buffer. No second backend call yet.
|
||||
let r2 = with_test_auth_override(|_, _, _| true, driver.read(31, handle_id.clone().into(), 1024, 1024))
|
||||
let r2 = with_test_auth_override(|_, _, _| true, driver.read(31, handle_id.clone(), 1024, 1024))
|
||||
.await
|
||||
.expect("mid-window read must succeed from cache");
|
||||
assert!(r2.data.iter().all(|b| *b == 0xAA), "mid-window read still in window 1");
|
||||
|
||||
// Reading at offset == window falls outside the cached chunk
|
||||
// and triggers the second backend call.
|
||||
let r3 = with_test_auth_override(|_, _, _| true, driver.read(32, handle_id.clone().into(), window, 1024))
|
||||
let r3 = with_test_auth_override(|_, _, _| true, driver.read(32, handle_id.clone(), window, 1024))
|
||||
.await
|
||||
.expect("read at offset=window must succeed via second backend call");
|
||||
assert!(r3.data.iter().all(|b| *b == 0xBB), "read at window boundary must come from window 2");
|
||||
@@ -489,7 +483,7 @@ mod tests {
|
||||
// A read inside the second cached chunk is served from the
|
||||
// buffer. The queue is empty by now, so any third backend
|
||||
// call would fail.
|
||||
let r4 = with_test_auth_override(|_, _, _| true, driver.read(33, handle_id.into(), window + 1024, 1024))
|
||||
let r4 = with_test_auth_override(|_, _, _| true, driver.read(33, handle_id, window + 1024, 1024))
|
||||
.await
|
||||
.expect("mid-window-2 read must succeed from cache");
|
||||
assert!(r4.data.iter().all(|b| *b == 0xBB), "mid-window-2 read still in window 2");
|
||||
@@ -517,13 +511,13 @@ mod tests {
|
||||
.expect("allocate");
|
||||
|
||||
// Populate window 1 with a full read.
|
||||
let _ = with_test_auth_override(|_, _, _| true, driver.read(40, handle_id.clone().into(), 0, window as u32))
|
||||
let _ = with_test_auth_override(|_, _, _| true, driver.read(40, handle_id.clone(), 0, window as u32))
|
||||
.await
|
||||
.expect("populate window 1");
|
||||
|
||||
// Ask for 256 bytes starting 64 bytes before window end. Only
|
||||
// 64 bytes are in the window. The driver must return 64.
|
||||
let edge = with_test_auth_override(|_, _, _| true, driver.read(41, handle_id.into(), window - 64, 256))
|
||||
let edge = with_test_auth_override(|_, _, _| true, driver.read(41, handle_id, window - 64, 256))
|
||||
.await
|
||||
.expect("partial-hit read must succeed");
|
||||
assert_eq!(edge.data.len(), 64, "partial hit must return only the in-window portion");
|
||||
@@ -558,10 +552,9 @@ mod tests {
|
||||
|
||||
for i in 0..read_count {
|
||||
let offset = (chunk_size as u64) * (i as u64);
|
||||
let data =
|
||||
with_test_auth_override(|_, _, _| true, driver.read(50 + i, handle_id.clone().into(), offset, chunk_size as u32))
|
||||
.await
|
||||
.expect("each read must succeed via the backend");
|
||||
let data = with_test_auth_override(|_, _, _| true, driver.read(50 + i, handle_id.clone(), offset, chunk_size as u32))
|
||||
.await
|
||||
.expect("each read must succeed via the backend");
|
||||
assert_eq!(data.data.len(), chunk_size, "read must return full requested length");
|
||||
let expected_byte = (i + 1) as u8;
|
||||
assert!(
|
||||
|
||||
@@ -336,10 +336,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
|
||||
open_attrs,
|
||||
phase: WritePhase::Buffering { part_buffer: Vec::new() },
|
||||
})?;
|
||||
Ok(Handle {
|
||||
id,
|
||||
handle: handle.into(),
|
||||
})
|
||||
Ok(Handle { id, handle })
|
||||
}
|
||||
|
||||
/// Upload the buffered bytes to S3 with a single PutObject call.
|
||||
@@ -2108,7 +2105,7 @@ mod tests {
|
||||
)
|
||||
.await
|
||||
.expect("WRITE | CREATE | TRUNCATE on a missing file must allocate a handle");
|
||||
assert!(!handle.handle.as_bytes().is_empty());
|
||||
assert!(!handle.handle.is_empty());
|
||||
assert!(backend.head_object_calls().is_empty(), "the non-EXCL accept path must not HEAD");
|
||||
}
|
||||
|
||||
@@ -2133,7 +2130,7 @@ mod tests {
|
||||
)
|
||||
.await
|
||||
.expect("WRITE | CREATE | TRUNCATE on an existing file must allocate a handle");
|
||||
assert!(!handle.handle.as_bytes().is_empty());
|
||||
assert!(!handle.handle.is_empty());
|
||||
assert!(backend.head_object_calls().is_empty(), "the non-EXCL accept path must not HEAD");
|
||||
}
|
||||
|
||||
@@ -2180,7 +2177,7 @@ mod tests {
|
||||
)
|
||||
.await
|
||||
.expect("EXCL on a missing key must allow creation");
|
||||
assert!(!handle.handle.as_bytes().is_empty());
|
||||
assert!(!handle.handle.is_empty());
|
||||
assert_eq!(backend.head_object_calls().len(), 1, "EXCL must HEAD exactly once");
|
||||
}
|
||||
|
||||
|
||||
@@ -38,6 +38,8 @@ default = []
|
||||
# event. Requires `RUSTFLAGS="--cfg hyper_unstable_tracing"`.
|
||||
diagnostic-internode-http2-ping = ["dep:hyper", "rustfs-io-metrics/diagnostic-internode-http2-ping"]
|
||||
hotpath = [
|
||||
"hotpath/hotpath",
|
||||
"hotpath/tokio",
|
||||
"rustfs-common/hotpath",
|
||||
"rustfs-config/hotpath",
|
||||
"rustfs-io-metrics/hotpath",
|
||||
@@ -46,6 +48,7 @@ hotpath = [
|
||||
]
|
||||
hotpath-alloc = [
|
||||
"hotpath",
|
||||
"hotpath/hotpath-alloc",
|
||||
"rustfs-common/hotpath-alloc",
|
||||
"rustfs-config/hotpath-alloc",
|
||||
"rustfs-io-metrics/hotpath-alloc",
|
||||
@@ -54,6 +57,7 @@ hotpath-alloc = [
|
||||
]
|
||||
hotpath-cpu = [
|
||||
"hotpath",
|
||||
"hotpath/hotpath-cpu",
|
||||
"rustfs-common/hotpath-cpu",
|
||||
"rustfs-config/hotpath-cpu",
|
||||
"rustfs-io-metrics/hotpath-cpu",
|
||||
@@ -62,6 +66,7 @@ hotpath-cpu = [
|
||||
]
|
||||
|
||||
[dependencies]
|
||||
hotpath.workspace = true
|
||||
rustfs-common.workspace = true
|
||||
rustfs-heal-contracts.workspace = true
|
||||
rustfs-io-metrics.workspace = true
|
||||
|
||||
@@ -31,24 +31,29 @@ workspace = true
|
||||
[features]
|
||||
default = []
|
||||
hotpath = [
|
||||
"hotpath/hotpath",
|
||||
"hotpath/tokio",
|
||||
"rustfs-rio/hotpath",
|
||||
"rustfs-utils/hotpath",
|
||||
"rustfs-filemeta/hotpath",
|
||||
]
|
||||
hotpath-alloc = [
|
||||
"hotpath",
|
||||
"hotpath/hotpath-alloc",
|
||||
"rustfs-rio/hotpath-alloc",
|
||||
"rustfs-utils/hotpath-alloc",
|
||||
"rustfs-filemeta/hotpath-alloc",
|
||||
]
|
||||
hotpath-cpu = [
|
||||
"hotpath",
|
||||
"hotpath/hotpath-cpu",
|
||||
"rustfs-rio/hotpath-cpu",
|
||||
"rustfs-utils/hotpath-cpu",
|
||||
"rustfs-filemeta/hotpath-cpu",
|
||||
]
|
||||
|
||||
[dependencies]
|
||||
hotpath.workspace = true
|
||||
aes-gcm = { workspace = true, features = ["rand_core"] }
|
||||
bytes = { workspace = true, features = ["serde"] }
|
||||
chacha20poly1305.workspace = true
|
||||
|
||||
@@ -32,7 +32,6 @@ workspace = true
|
||||
internode-transport = []
|
||||
default = []
|
||||
hotpath = [
|
||||
"dep:hotpath",
|
||||
"hotpath/hotpath",
|
||||
"hotpath/tokio",
|
||||
"hotpath/futures",
|
||||
@@ -60,8 +59,7 @@ hotpath-cpu = [
|
||||
]
|
||||
|
||||
[dependencies]
|
||||
# Feature-gated in src (measure_block!); optional so default builds do not warn.
|
||||
hotpath = { workspace = true, optional = true }
|
||||
hotpath.workspace = true
|
||||
arc-swap.workspace = true
|
||||
tokio = { workspace = true, features = ["io-util", "macros", "net", "rt-multi-thread", "sync", "time"] }
|
||||
rand = { workspace = true, features = ["serde"] }
|
||||
@@ -81,6 +79,7 @@ rustfs-io-metrics.workspace = true
|
||||
rustfs-tls-runtime.workspace = true
|
||||
rustfs-utils = { workspace = true, features = ["io", "hash", "compress"] }
|
||||
serde_json = { workspace = true, features = ["raw_value"] }
|
||||
md-5 = { workspace = true }
|
||||
hmac.workspace = true
|
||||
minlz.workspace = true
|
||||
tracing.workspace = true
|
||||
@@ -92,7 +91,6 @@ xxhash-rust = { workspace = true, features = ["xxh64", "xxh3"] }
|
||||
hex-simd.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
md-5 = { workspace = true }
|
||||
faster-hex.workspace = true
|
||||
rcgen.workspace = true
|
||||
tokio-rustls.workspace = true
|
||||
|
||||
@@ -15,30 +15,6 @@
|
||||
use super::*;
|
||||
use crate::data_usage_define::{DATA_USAGE_OBJ_NAME_PATH, read_config_with_revision};
|
||||
|
||||
// Two-pool EC fixtures keep many scan futures live at once and overflow
|
||||
// Linux libtest's default 2 MiB test-thread stack. Run each case on a
|
||||
// dedicated 32 MiB thread (same approach as scoped_entry_fallback).
|
||||
fn run_cohort_test<F, Fut>(thread_name: &'static str, test_fn: F)
|
||||
where
|
||||
F: FnOnce() -> Fut + Send + 'static,
|
||||
Fut: std::future::Future<Output = ()> + 'static,
|
||||
{
|
||||
let handle = std::thread::Builder::new()
|
||||
.name(thread_name.to_string())
|
||||
.stack_size(32 * 1024 * 1024)
|
||||
.spawn(move || {
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("service cohort test runtime should build");
|
||||
runtime.block_on(test_fn());
|
||||
})
|
||||
.expect("service cohort test thread should spawn");
|
||||
if let Err(payload) = handle.join() {
|
||||
std::panic::resume_unwind(payload);
|
||||
}
|
||||
}
|
||||
|
||||
async fn create_cohort_bucket(store: &ECStore, bucket: &str) {
|
||||
create_cohort_bucket_objects(store, bucket, 1).await;
|
||||
}
|
||||
@@ -128,16 +104,9 @@ async fn run_cohort_cycle(
|
||||
(result, usage)
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
fn service_cohort_production_dispatch_services_waiters_across_sources() {
|
||||
run_cohort_test(
|
||||
"cohort-prod-disp",
|
||||
service_cohort_production_dispatch_services_waiters_across_sources_case,
|
||||
);
|
||||
}
|
||||
|
||||
async fn service_cohort_production_dispatch_services_waiters_across_sources_case() {
|
||||
async fn service_cohort_production_dispatch_services_waiters_across_sources() {
|
||||
let (_dir, store) = setup_two_pool_scanner_store().await;
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
let hot = format!("a-hot-{}", Uuid::new_v4().simple());
|
||||
@@ -228,16 +197,9 @@ async fn service_cohort_production_dispatch_services_waiters_across_sources_case
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
fn service_cohort_flat_bucket_budget_does_not_publish_unscanned_small_bucket() {
|
||||
run_cohort_test(
|
||||
"cohort-flat-bud",
|
||||
service_cohort_flat_bucket_budget_does_not_publish_unscanned_small_bucket_case,
|
||||
);
|
||||
}
|
||||
|
||||
async fn service_cohort_flat_bucket_budget_does_not_publish_unscanned_small_bucket_case() {
|
||||
async fn service_cohort_flat_bucket_budget_does_not_publish_unscanned_small_bucket() {
|
||||
let (_dir, store) = setup_two_pool_scanner_store().await;
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
let flat = format!("a-flat-{}", Uuid::new_v4().simple());
|
||||
@@ -321,16 +283,9 @@ async fn service_cohort_flat_bucket_budget_does_not_publish_unscanned_small_buck
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
fn service_cohort_fresh_complete_aggregate_preserves_reordered_sources() {
|
||||
run_cohort_test(
|
||||
"cohort-fresh-agg",
|
||||
service_cohort_fresh_complete_aggregate_preserves_reordered_sources_case,
|
||||
);
|
||||
}
|
||||
|
||||
async fn service_cohort_fresh_complete_aggregate_preserves_reordered_sources_case() {
|
||||
async fn service_cohort_fresh_complete_aggregate_preserves_reordered_sources() {
|
||||
let (_dir, store) = setup_two_pool_scanner_store().await;
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
for bucket in ["cohort-first", "cohort-second"] {
|
||||
@@ -374,16 +329,9 @@ async fn service_cohort_fresh_complete_aggregate_preserves_reordered_sources_cas
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
fn service_cohort_cancelled_dispatch_does_not_consume_waiters_or_leak_permits() {
|
||||
run_cohort_test(
|
||||
"cohort-cancelled",
|
||||
service_cohort_cancelled_dispatch_does_not_consume_waiters_or_leak_permits_case,
|
||||
);
|
||||
}
|
||||
|
||||
async fn service_cohort_cancelled_dispatch_does_not_consume_waiters_or_leak_permits_case() {
|
||||
async fn service_cohort_cancelled_dispatch_does_not_consume_waiters_or_leak_permits() {
|
||||
let (_dir, store) = setup_two_pool_scanner_store().await;
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
let bucket = format!("cancel-{}", Uuid::new_v4().simple());
|
||||
@@ -412,16 +360,9 @@ async fn service_cohort_cancelled_dispatch_does_not_consume_waiters_or_leak_perm
|
||||
/// N=17, q=4: every bucket reaches a worker within ceil(17/4)=5 rounds while
|
||||
/// one bucket stays hot and another fails forever, with leader-local state
|
||||
/// discarded after each round as a restart would.
|
||||
#[test]
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
fn cold_bucket_service_bound_survives_restart_with_hot_and_failing_bucket() {
|
||||
run_cohort_test(
|
||||
"cold-svc-bound",
|
||||
cold_bucket_service_bound_survives_restart_with_hot_and_failing_bucket_case,
|
||||
);
|
||||
}
|
||||
|
||||
async fn cold_bucket_service_bound_survives_restart_with_hot_and_failing_bucket_case() {
|
||||
async fn cold_bucket_service_bound_survives_restart_with_hot_and_failing_bucket() {
|
||||
const BUCKETS: usize = 17;
|
||||
const QUANTA: u64 = 4;
|
||||
const ROUND_BOUND: usize = 5;
|
||||
@@ -493,16 +434,9 @@ fn quantum_budget(quantum: crate::scanner_budget::ScannerBucketQuantum) -> Arc<S
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
fn source_turn_dispatch_rotates_real_sources_and_retains_completed_roots() {
|
||||
run_cohort_test(
|
||||
"src-turn-disp",
|
||||
source_turn_dispatch_rotates_real_sources_and_retains_completed_roots_case,
|
||||
);
|
||||
}
|
||||
|
||||
async fn source_turn_dispatch_rotates_real_sources_and_retains_completed_roots_case() {
|
||||
async fn source_turn_dispatch_rotates_real_sources_and_retains_completed_roots() {
|
||||
let mut evidence = None;
|
||||
temp_env::async_with_vars([(ENV_SCANNER_MAX_CONCURRENT_SET_SCANS, Some("1"))], async {
|
||||
crate::runtime_config::refresh_scanner_runtime_config_for_tests();
|
||||
@@ -569,16 +503,9 @@ async fn source_turn_dispatch_rotates_real_sources_and_retains_completed_roots_c
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
fn source_turn_zero_new_frontier_stops_requeuing_without_starving_other_sources() {
|
||||
run_cohort_test(
|
||||
"src-turn-zero",
|
||||
source_turn_zero_new_frontier_stops_requeuing_without_starving_other_sources_case,
|
||||
);
|
||||
}
|
||||
|
||||
async fn source_turn_zero_new_frontier_stops_requeuing_without_starving_other_sources_case() {
|
||||
async fn source_turn_zero_new_frontier_stops_requeuing_without_starving_other_sources() {
|
||||
temp_env::async_with_vars([(ENV_SCANNER_MAX_CONCURRENT_SET_SCANS, Some("1"))], async {
|
||||
crate::runtime_config::refresh_scanner_runtime_config_for_tests();
|
||||
let (_dir, store) = setup_two_pool_scanner_store().await;
|
||||
@@ -606,13 +533,9 @@ async fn source_turn_zero_new_frontier_stops_requeuing_without_starving_other_so
|
||||
crate::runtime_config::refresh_scanner_runtime_config_for_tests();
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
fn fair_prefix_quantum_spent_bucket_returns_behind_waiting_buckets() {
|
||||
run_cohort_test("fair-spent-qm", fair_prefix_quantum_spent_bucket_returns_behind_waiting_buckets_case);
|
||||
}
|
||||
|
||||
async fn fair_prefix_quantum_spent_bucket_returns_behind_waiting_buckets_case() {
|
||||
async fn fair_prefix_quantum_spent_bucket_returns_behind_waiting_buckets() {
|
||||
let (_dir, store) = setup_local_scanner_store(1, 4).await;
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
create_cohort_bucket_objects(&store, "a-flat", 6).await;
|
||||
@@ -642,13 +565,9 @@ async fn fair_prefix_quantum_spent_bucket_returns_behind_waiting_buckets_case()
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
fn fair_prefix_quantum_without_durable_motion_ends_the_bucket_turn() {
|
||||
run_cohort_test("fair-no-motion", fair_prefix_quantum_without_durable_motion_ends_the_bucket_turn_case);
|
||||
}
|
||||
|
||||
async fn fair_prefix_quantum_without_durable_motion_ends_the_bucket_turn_case() {
|
||||
async fn fair_prefix_quantum_without_durable_motion_ends_the_bucket_turn() {
|
||||
let (_dir, store) = setup_local_scanner_store(1, 4).await;
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
create_cohort_bucket_objects(&store, "a-flat", 6).await;
|
||||
@@ -671,16 +590,9 @@ async fn fair_prefix_quantum_without_durable_motion_ends_the_bucket_turn_case()
|
||||
|
||||
/// A one-object share stops inside its first child, so the frontier never
|
||||
/// moves. Requeuing that bucket would spin until the cycle deadline.
|
||||
#[test]
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
fn fair_prefix_quantum_repeated_share_without_a_new_frontier_stops_requeuing() {
|
||||
run_cohort_test(
|
||||
"fair-rep-share",
|
||||
fair_prefix_quantum_repeated_share_without_a_new_frontier_stops_requeuing_case,
|
||||
);
|
||||
}
|
||||
|
||||
async fn fair_prefix_quantum_repeated_share_without_a_new_frontier_stops_requeuing_case() {
|
||||
async fn fair_prefix_quantum_repeated_share_without_a_new_frontier_stops_requeuing() {
|
||||
let (_dir, store) = setup_local_scanner_store(1, 4).await;
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
create_cohort_bucket_objects(&store, "a-flat", 3).await;
|
||||
@@ -701,16 +613,9 @@ async fn fair_prefix_quantum_repeated_share_without_a_new_frontier_stops_requeui
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
fn cold_bucket_service_saved_order_keeps_coverage_and_rejects_older_owners() {
|
||||
run_cohort_test(
|
||||
"cold-saved-ord",
|
||||
cold_bucket_service_saved_order_keeps_coverage_and_rejects_older_owners_case,
|
||||
);
|
||||
}
|
||||
|
||||
async fn cold_bucket_service_saved_order_keeps_coverage_and_rejects_older_owners_case() {
|
||||
async fn cold_bucket_service_saved_order_keeps_coverage_and_rejects_older_owners() {
|
||||
let (_dir, store) = setup_local_scanner_store(1, 4).await;
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
create_cohort_bucket(&store, "cold-only").await;
|
||||
|
||||
@@ -1,40 +0,0 @@
|
||||
# SSE Crate Instructions
|
||||
|
||||
Applies to `crates/sse/`.
|
||||
|
||||
## Boundary
|
||||
|
||||
- This crate sits between the S3 layer and the storage engine and depends on neither. Its `rustfs-*` dependencies are pinned by `scripts/check_architecture_migration_rules.sh` to `rustfs-config`, `rustfs-kms`, `rustfs-policy`, `rustfs-s3-types`, and `rustfs-utils`; see [crate boundaries](../../docs/architecture/crate-boundaries.md).
|
||||
- Anything the crate needs from the running server arrives through `SseHost` (`src/host.rs`), passed as the first argument of every entry point that consults the server. Do not add a global that stands in for the host: a caller must not be able to reach key selection without naming the server it speaks for.
|
||||
- Translation to and from storage-engine types (`ObjectEncryptionResolver`, `WriteEncryption`, `MultipartCompletionAuthorization`) and request types belongs in `rustfs/src/storage/sse.rs`, not here.
|
||||
- Report failures as `SseError`, classified with the S3 error code the caller answers with. A failure that did not originate from the request goes through `SseError::internal`, which keeps its detail out of the response.
|
||||
|
||||
## Security
|
||||
|
||||
- Never log plaintext customer keys, decrypted key material, data-key envelopes, nonces, or SSE request payloads, including through error strings and `Debug` output.
|
||||
- KMS audit tags may carry only the fields documented on `KmsRequestAuditState`.
|
||||
- The `test-util` feature exposes authorization overrides and DEK-provider injection. It is a dev-dependency feature only; never enable it from a normal dependency.
|
||||
|
||||
## Change Coordination
|
||||
|
||||
When changing key resolution, metadata, or error classification, verify compatibility with:
|
||||
|
||||
- `rustfs/src/storage/sse.rs` (the host implementation and the engine-boundary adapters)
|
||||
- `crates/kms/`
|
||||
- `crates/utils/src/http/object_encryption_keys.rs` (the stored-metadata key vocabulary)
|
||||
- `rustfs/src/kms_rekey.rs` and `rustfs/src/admin/handlers/kms*.rs`
|
||||
|
||||
## Testing
|
||||
|
||||
Run both builds; the `rio-v2` feature swaps the object-key scheme and compiles different code paths:
|
||||
|
||||
```bash
|
||||
cargo nextest run -p rustfs-sse --no-tests fail
|
||||
cargo nextest run -p rustfs-sse --features rio-v2 --no-tests fail
|
||||
```
|
||||
|
||||
The adapter tests live with the adapter:
|
||||
|
||||
```bash
|
||||
cargo nextest run -p rustfs --no-tests fail -E 'test(/^storage::sse::/)'
|
||||
```
|
||||
@@ -1,70 +0,0 @@
|
||||
# Copyright 2024 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
[package]
|
||||
name = "rustfs-sse"
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
rust-version.workspace = true
|
||||
version.workspace = true
|
||||
homepage.workspace = true
|
||||
description = "S3 server-side encryption (SSE-S3, SSE-KMS, SSE-C) key resolution and metadata for RustFS."
|
||||
keywords = ["sse", "encryption", "s3", "kms", "rustfs"]
|
||||
categories = ["cryptography"]
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[features]
|
||||
default = []
|
||||
# MinIO-compatible sealed object keys (DARE). Mirrors the `rio-v2` feature of
|
||||
# the server binary, which enables both together.
|
||||
rio-v2 = ["dep:chacha20poly1305", "dep:hmac", "dep:sha2"]
|
||||
# Test seams for dependents: per-principal authorization overrides, DEK
|
||||
# provider injection and metadata fixtures. Never enable in a release build.
|
||||
test-util = []
|
||||
|
||||
[dependencies]
|
||||
# The only RustFS crates this one may reach. It sits between the S3 layer and
|
||||
# the storage engine and must not depend on either: what it needs from the
|
||||
# running server arrives through `SseHost`.
|
||||
rustfs-config = { workspace = true }
|
||||
rustfs-kms = { workspace = true }
|
||||
rustfs-policy = { workspace = true }
|
||||
rustfs-s3-types = { workspace = true }
|
||||
rustfs-utils = { workspace = true, features = ["http", "path"] }
|
||||
|
||||
aes-gcm = { workspace = true, features = ["rand_core", "zeroize"] }
|
||||
async-trait = { workspace = true }
|
||||
base64-simd = { workspace = true }
|
||||
chacha20poly1305 = { workspace = true, optional = true }
|
||||
hmac = { workspace = true, optional = true }
|
||||
http = { workspace = true }
|
||||
md-5 = { workspace = true }
|
||||
rand = { workspace = true }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
serde_json = { workspace = true }
|
||||
sha2 = { workspace = true, optional = true }
|
||||
tracing = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
# The fake AWS KMS endpoint the backend-selection tests talk to.
|
||||
axum = { workspace = true }
|
||||
temp-env = { workspace = true, features = ["async_closure"] }
|
||||
tempfile = { workspace = true }
|
||||
tokio = { workspace = true, features = ["macros", "net", "rt-multi-thread", "sync", "time"] }
|
||||
|
||||
[lib]
|
||||
doctest = false
|
||||
@@ -1,176 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! The error every SSE operation reports, already classified for the S3 data path.
|
||||
|
||||
use rustfs_kms::{KmsError, KmsUnavailableError};
|
||||
use rustfs_s3_types::S3ErrorCode;
|
||||
|
||||
type BoxError = Box<dyn std::error::Error + Send + Sync>;
|
||||
|
||||
/// S3 error code a KMS key that does not exist is reported under.
|
||||
pub const KMS_KEY_NOT_FOUND_ERROR_CODE: &str = "KMS.NotFoundException";
|
||||
|
||||
pub(crate) const INTERNAL_ERROR_MESSAGE: &str = "We encountered an internal error, please try again.";
|
||||
const SERVICE_UNAVAILABLE_MESSAGE: &str = "The service is unavailable. Please retry.";
|
||||
const ACCESS_DENIED_MESSAGE: &str = "Access Denied.";
|
||||
|
||||
/// A failed SSE operation, carrying the S3 error code and message the caller answers with.
|
||||
///
|
||||
/// `source` keeps the underlying failure for logs and for classification by
|
||||
/// callers; it is never part of the response.
|
||||
#[derive(Debug)]
|
||||
pub struct SseError {
|
||||
pub code: S3ErrorCode,
|
||||
pub message: String,
|
||||
pub source: Option<BoxError>,
|
||||
}
|
||||
|
||||
impl SseError {
|
||||
/// Classify a failure that did not originate from the request itself.
|
||||
///
|
||||
/// A [`KmsUnavailableError`] is a retryable `ServiceUnavailable`, a
|
||||
/// [`KmsError`] follows [`data_plane_kms_error`], and everything else is an
|
||||
/// `InternalError` whose detail stays in `source` and out of the response.
|
||||
pub fn internal(error: impl Into<BoxError>) -> Self {
|
||||
let source = error.into();
|
||||
let (code, message) = if source.downcast_ref::<KmsUnavailableError>().is_some() {
|
||||
(S3ErrorCode::ServiceUnavailable, SERVICE_UNAVAILABLE_MESSAGE.to_string())
|
||||
} else if let Some(classified) = source.downcast_ref::<KmsError>().and_then(data_plane_kms_error) {
|
||||
classified
|
||||
} else {
|
||||
(S3ErrorCode::InternalError, INTERNAL_ERROR_MESSAGE.to_string())
|
||||
};
|
||||
Self {
|
||||
code,
|
||||
message,
|
||||
source: Some(source),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Display for SseError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.write_str(&self.message)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for SseError {
|
||||
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
|
||||
self.source.as_deref().map(|source| source as _)
|
||||
}
|
||||
}
|
||||
|
||||
/// Map a KMS failure that surfaced on the S3 data path to its S3 error code.
|
||||
///
|
||||
/// The contract deliberately differs from the admin key-lifecycle handlers:
|
||||
/// there a key id is the resource being addressed, so a missing key is `404`.
|
||||
/// Here the key id arrives inside a request header or a bucket default, so a
|
||||
/// key that is missing, disabled, or otherwise unusable is configuration the
|
||||
/// caller has to correct — AWS answers `400`, and reporting `500` instead both
|
||||
/// misfiles the failure as a server fault and makes SDKs back off and retry a
|
||||
/// request that cannot succeed.
|
||||
///
|
||||
/// `None` keeps the caller's fallthrough, which is the `500` that integrity
|
||||
/// faults — damaged, unreadable, or unknown-format key material — must keep.
|
||||
///
|
||||
/// Messages either name what the caller asked for or stay generic; detail that
|
||||
/// belongs to the deployment rather than the request stays in `source`, which
|
||||
/// the caller attaches and the logs retain.
|
||||
pub fn data_plane_kms_error(error: &KmsError) -> Option<(S3ErrorCode, String)> {
|
||||
use KmsError as Kms;
|
||||
|
||||
match error {
|
||||
Kms::KeyNotFound { key_id } => Some((
|
||||
S3ErrorCode::Custom(KMS_KEY_NOT_FOUND_ERROR_CODE.into()),
|
||||
format!("KMS key not found: {key_id}"),
|
||||
)),
|
||||
Kms::KeyVersionNotFound { key_id, version } => Some((
|
||||
S3ErrorCode::Custom(KMS_KEY_NOT_FOUND_ERROR_CODE.into()),
|
||||
format!("KMS key version {version} not found for key {key_id}"),
|
||||
)),
|
||||
// The key exists but its state forbids the operation (disabled,
|
||||
// pending deletion). AWS treats an invalid key state as a request
|
||||
// error, not a server fault.
|
||||
Kms::InvalidOperation { .. } => Some((S3ErrorCode::InvalidRequest, error.to_string())),
|
||||
// A policy decision needs a human, so it must be distinguishable from
|
||||
// the transient classes an SDK retries.
|
||||
Kms::AccessDenied { .. } => Some((S3ErrorCode::AccessDenied, ACCESS_DENIED_MESSAGE.to_string())),
|
||||
// Request-side faults: what was asked for cannot be served as asked.
|
||||
Kms::ContextMismatch { .. }
|
||||
| Kms::InvalidKey { .. }
|
||||
| Kms::ValidationError { .. }
|
||||
| Kms::UnsupportedAlgorithm { .. }
|
||||
| Kms::InvalidKeySize { .. } => Some((S3ErrorCode::InvalidRequest, error.to_string())),
|
||||
// A deployment whose KMS configuration cannot serve the request (for
|
||||
// example no key named and no default). Actionable, but the detail
|
||||
// describes the deployment, so it stays out of the response body.
|
||||
Kms::ConfigurationError { .. } => {
|
||||
Some((S3ErrorCode::InvalidRequest, "The KMS configuration cannot serve this request".to_string()))
|
||||
}
|
||||
// Transient: worth retrying, and must be counted against availability
|
||||
// rather than against the caller. `IoError` belongs here because it is
|
||||
// how a backend reports that its key store itself was unreachable —
|
||||
// rustfs/rustfs#7470 separated that from a missing key precisely so the
|
||||
// two stop looking alike, and leaving it on the 500 fallthrough would
|
||||
// erase that distinction again at the S3 boundary.
|
||||
Kms::BackendError { .. }
|
||||
| Kms::IoError { .. }
|
||||
| Kms::OperationTimedOut { .. }
|
||||
| Kms::OperationCancelled { .. }
|
||||
| Kms::CredentialsUnavailable { .. }
|
||||
| Kms::CacheError { .. } => Some((S3ErrorCode::ServiceUnavailable, SERVICE_UNAVAILABLE_MESSAGE.to_string())),
|
||||
// A permanent gap in the configured backend, never a missing resource.
|
||||
Kms::UnsupportedCapability { .. } => Some((S3ErrorCode::NotImplemented, error.to_string())),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn internal_keeps_request_free_failures_generic() {
|
||||
let error = SseError::internal("Failed to decode IV: bad base64");
|
||||
|
||||
assert_eq!(error.code, S3ErrorCode::InternalError);
|
||||
assert_eq!(error.message, INTERNAL_ERROR_MESSAGE);
|
||||
assert!(
|
||||
error
|
||||
.source
|
||||
.as_ref()
|
||||
.is_some_and(|source| source.to_string().contains("Failed to decode IV"))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn internal_reports_an_unavailable_kms_as_retryable() {
|
||||
let error = SseError::internal(KmsUnavailableError);
|
||||
|
||||
assert_eq!(error.code, S3ErrorCode::ServiceUnavailable);
|
||||
assert_eq!(error.message, SERVICE_UNAVAILABLE_MESSAGE);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn internal_classifies_kms_errors_for_the_data_path() {
|
||||
let missing = SseError::internal(KmsError::key_not_found("absent-key"));
|
||||
assert_eq!(missing.code, S3ErrorCode::Custom(KMS_KEY_NOT_FOUND_ERROR_CODE.into()));
|
||||
assert_eq!(missing.message, "KMS key not found: absent-key");
|
||||
|
||||
let damaged = SseError::internal(KmsError::cryptographic_error("decrypt", "authentication failed"));
|
||||
assert_eq!(damaged.code, S3ErrorCode::InternalError);
|
||||
assert_eq!(damaged.message, INTERNAL_ERROR_MESSAGE);
|
||||
}
|
||||
}
|
||||
@@ -1,85 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! What SSE needs from the server it runs in.
|
||||
//!
|
||||
//! This crate reaches the running server only through [`SseHost`]: it holds no
|
||||
//! handle to the storage engine, to IAM, or to the bucket metadata system. Every
|
||||
//! entry point that consults the server takes the host as its first argument, so
|
||||
//! a caller cannot reach key selection without saying which server it speaks for.
|
||||
|
||||
use crate::{SseError, SseKmsPrincipal};
|
||||
use async_trait::async_trait;
|
||||
use rustfs_kms::KmsServiceManager;
|
||||
use rustfs_kms::service::ObjectEncryptionService;
|
||||
use rustfs_policy::policy::action::KmsAction;
|
||||
use std::sync::Arc;
|
||||
|
||||
/// The server capabilities SSE key resolution depends on.
|
||||
#[async_trait]
|
||||
pub trait SseHost: Send + Sync {
|
||||
/// The KMS object-encryption service currently serving, if one is.
|
||||
///
|
||||
/// Resolved on every call: the KMS backend can be reconfigured while the
|
||||
/// server runs, so a service handle must not outlive the operation it serves.
|
||||
async fn encryption_service(&self) -> Option<Arc<ObjectEncryptionService>>;
|
||||
|
||||
/// The KMS service manager, when the server has one.
|
||||
///
|
||||
/// Only its status is read, to tell a configured KMS that is not serving
|
||||
/// (retryable) from a deployment that never configured one (a request error).
|
||||
fn kms_service_manager(&self) -> Option<Arc<KmsServiceManager>>;
|
||||
|
||||
/// The bucket's default-encryption configuration.
|
||||
///
|
||||
/// `Ok(None)` means the bucket has no encryption configuration.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Any failure to read the configuration, already classified for the S3
|
||||
/// response. A write must not proceed as if the bucket had no default when
|
||||
/// the configuration could not be read.
|
||||
async fn bucket_encryption(&self, bucket: &str) -> Result<Option<BucketEncryption>, SseError>;
|
||||
|
||||
/// The authorizer deciding per-key KMS permissions on the data path.
|
||||
fn kms_key_authorizer(&self) -> &dyn KmsKeyAuthorizer;
|
||||
}
|
||||
|
||||
/// A bucket's stored encryption configuration, reduced to what a write consults.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct BucketEncryption {
|
||||
/// The default-encryption rule of the first configuration rule, if it has one.
|
||||
pub default_rule: Option<BucketDefaultEncryption>,
|
||||
}
|
||||
|
||||
/// The `ApplyServerSideEncryptionByDefault` element of a bucket encryption rule.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct BucketDefaultEncryption {
|
||||
/// The stored algorithm, exactly as persisted (`AES256`, `aws:kms`, or an unknown value).
|
||||
pub sse_algorithm: String,
|
||||
pub kms_master_key_id: Option<String>,
|
||||
}
|
||||
|
||||
/// Decides whether a principal may use a specific KMS key on the data path.
|
||||
///
|
||||
/// Behind a trait so tests can pin the allow/deny contract of the SSE call sites
|
||||
/// without a running IAM subsystem.
|
||||
#[async_trait]
|
||||
pub trait KmsKeyAuthorizer: Send + Sync {
|
||||
/// # Errors
|
||||
///
|
||||
/// When the decision itself could not be made, for example because the
|
||||
/// policy subsystem is not ready. A denial is `Ok(false)`.
|
||||
async fn is_allowed(&self, principal: &SseKmsPrincipal, action: KmsAction, key_id: &str) -> Result<bool, SseError>;
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,70 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Fixtures for tests of SSE call sites, in this crate and in its dependents.
|
||||
|
||||
use crate::{KmsKeyAuthorizer, SseError, SseKmsPrincipal};
|
||||
use async_trait::async_trait;
|
||||
use base64_simd::STANDARD as BASE64_STANDARD;
|
||||
use rustfs_policy::policy::action::KmsAction;
|
||||
use rustfs_utils::http::headers::AMZ_ENCRYPTION_KMS;
|
||||
use rustfs_utils::http::object_encryption_keys::{
|
||||
INTERNAL_ENCRYPTION_ALGORITHM_HEADER, INTERNAL_ENCRYPTION_IV_HEADER, INTERNAL_ENCRYPTION_KEY_HEADER,
|
||||
INTERNAL_ENCRYPTION_KEY_ID_HEADER,
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
use std::sync::{Arc, Mutex, PoisonError};
|
||||
|
||||
/// An authorizer that answers every request the same way and records what it was asked.
|
||||
pub struct RecordingKmsKeyAuthorizer {
|
||||
allowed: bool,
|
||||
calls: Mutex<Vec<(KmsAction, String)>>,
|
||||
}
|
||||
|
||||
impl RecordingKmsKeyAuthorizer {
|
||||
pub fn new(allowed: bool) -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
allowed,
|
||||
calls: Mutex::new(Vec::new()),
|
||||
})
|
||||
}
|
||||
|
||||
/// The `(action, key id)` pairs this authorizer was asked about, in order.
|
||||
pub fn calls(&self) -> Vec<(KmsAction, String)> {
|
||||
self.calls.lock().unwrap_or_else(PoisonError::into_inner).clone()
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl KmsKeyAuthorizer for RecordingKmsKeyAuthorizer {
|
||||
async fn is_allowed(&self, _principal: &SseKmsPrincipal, action: KmsAction, key_id: &str) -> Result<bool, SseError> {
|
||||
self.calls
|
||||
.lock()
|
||||
.unwrap_or_else(PoisonError::into_inner)
|
||||
.push((action, key_id.to_string()));
|
||||
Ok(self.allowed)
|
||||
}
|
||||
}
|
||||
|
||||
/// Stored metadata of an SSE-KMS object wrapped under the key `finance-key`.
|
||||
pub fn sse_kms_object_metadata() -> HashMap<String, String> {
|
||||
HashMap::from([
|
||||
("x-amz-server-side-encryption".to_string(), AMZ_ENCRYPTION_KMS.to_string()),
|
||||
("x-amz-server-side-encryption-aws-kms-key-id".to_string(), "finance-key".to_string()),
|
||||
(INTERNAL_ENCRYPTION_KEY_ID_HEADER.to_string(), "finance-key".to_string()),
|
||||
(INTERNAL_ENCRYPTION_KEY_HEADER.to_string(), BASE64_STANDARD.encode_to_string([7u8; 48])),
|
||||
(INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode_to_string([9u8; 12])),
|
||||
(INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(), "aws:kms".to_string()),
|
||||
])
|
||||
}
|
||||
@@ -30,11 +30,12 @@ doctest = false
|
||||
|
||||
[features]
|
||||
default = []
|
||||
hotpath = ["rustfs-rio/hotpath"]
|
||||
hotpath-alloc = ["hotpath", "rustfs-rio/hotpath-alloc"]
|
||||
hotpath-cpu = ["hotpath", "rustfs-rio/hotpath-cpu"]
|
||||
hotpath = ["hotpath/hotpath", "hotpath/tokio", "rustfs-rio/hotpath"]
|
||||
hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc", "rustfs-rio/hotpath-alloc"]
|
||||
hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu", "rustfs-rio/hotpath-cpu"]
|
||||
|
||||
[dependencies]
|
||||
hotpath.workspace = true
|
||||
async-compression = { workspace = true, features = [
|
||||
"tokio",
|
||||
"bzip2",
|
||||
@@ -48,6 +49,10 @@ rustfs-rio.workspace = true
|
||||
tokio = { workspace = true, features = ["io-util", "macros", "rt"] }
|
||||
thiserror = { workspace = true }
|
||||
|
||||
[target.'cfg(target_os = "macos")'.dependencies]
|
||||
# Keep XZ decoding self-contained in macOS binaries instead of linking Homebrew liblzma.
|
||||
liblzma = { version = "0.4.9", features = ["static"] }
|
||||
|
||||
[dev-dependencies]
|
||||
rustfs-tokio-tar = { workspace = true }
|
||||
futures = { workspace = true }
|
||||
|
||||
@@ -29,16 +29,14 @@ zdict_builder = ["zstd_fixed/zdict_builder"]
|
||||
zstdmt = ["zstd_fixed/zstdmt"]
|
||||
|
||||
[dev-dependencies]
|
||||
# Enables zstd-safe's `seekable` feature for tests/security.rs. The test imports
|
||||
# `zstd_safe::seekable` directly so cargo-shear treats this as a used dependency.
|
||||
zstd-safe = { version = "8.3.0", features = ["seekable"] }
|
||||
toml = "1.2.2"
|
||||
toml = "1.2.1"
|
||||
semver = "1.0.28"
|
||||
apache-avro.workspace = true
|
||||
arrow-array = "60.2.0"
|
||||
arrow-ipc = { version = "60.2.0", features = ["zstd"] }
|
||||
arrow-schema = "60.2.0"
|
||||
parquet = { version = "60.2.0", default-features = false, features = ["arrow", "zstd"] }
|
||||
arrow-array = "60.1.0"
|
||||
arrow-ipc = { version = "60.1.0", features = ["zstd"] }
|
||||
arrow-schema = "60.1.0"
|
||||
parquet = { version = "60.1.0", default-features = false, features = ["arrow", "zstd"] }
|
||||
kafka-protocol = { version = "0.18.0", default-features = false, features = ["zstd"] }
|
||||
bytes.workspace = true
|
||||
|
||||
|
||||
@@ -86,7 +86,7 @@ fn corrupt_legacy_frame_cannot_emit_bytes_when_writer_is_dropped() {
|
||||
|
||||
#[test]
|
||||
fn seekable_offsets_past_eof_keep_vec_length_within_capacity() {
|
||||
use zstd_safe::seekable::{Seekable, SeekableCStream};
|
||||
use zstd::zstd_safe::seekable::{Seekable, SeekableCStream};
|
||||
|
||||
let source = b"seekable frame fixture with enough data for several independent frames";
|
||||
let mut encoded = [0; 1024];
|
||||
|
||||
@@ -30,11 +30,9 @@ Contract crates stay below implementation crates. Forbidden edges:
|
||||
| `security-governance -> rustfs` | Governance contracts stay below the binary crate |
|
||||
| `extension-schema -> rustfs` | The extension schema is consumed by the binary, never the reverse |
|
||||
| `extension-schema -> ecstore` | The extension schema must not reach storage internals |
|
||||
| `sse -> ecstore`, `sse -> iam`, `sse -> rustfs` | SSE key resolution reaches the running server only through its `SseHost` port |
|
||||
|
||||
- `rustfs-storage-api` exposes storage-facing replication status/state contracts only through `crates/storage-api/src/replication.rs`, so its temporary dependency on `rustfs-filemeta` wire types stays centralized and no `rustfs-replication` / `rustfs-storage-api` cycle appears.
|
||||
- Leaf crates (`config`, `credentials`, `crypto`, `io-metrics`, `madmin`) may not depend on other `rustfs-*` crates, in either TOML spelling, except the adjudicated edges pinned in the guard's leaf allowlist: `io-metrics -> rustfs-s3-ops` (pure contract crates sharing the `S3Operation` vocabulary) and `madmin -> rustfs-signer` (the SigV4-signed admin SDK client). A new leaf exception must be a pure contract dependency (types and enums only, no I/O, no globals, no non-contract internal dependencies) and land together with its allowlist entry.
|
||||
- `rustfs-sse` sits between the S3 layer and the storage engine and depends on neither. Its internal dependencies are pinned in the same guard to `rustfs-config`, `rustfs-kms`, `rustfs-policy`, `rustfs-s3-types`, and `rustfs-utils`; the KMS service, a bucket's default encryption, and key-permission decisions arrive through `rustfs_sse::SseHost`, which `rustfs/src/storage/sse.rs` implements. That file also owns every translation across the engine boundary (`ObjectEncryptionResolver`, `WriteEncryption`, `MultipartCompletionAuthorization`), so neither side names the other's types.
|
||||
- Compile-time source reads follow the same direction: `include_str!` / `include!` of a `.rs` file must not resolve outside the including crate (`scripts/check_layer_dependencies.sh`). Shared source-text expectations belong in a contract surface such as `rustfs_protos::compat_manifest` (`crates/protos/src/compat_manifest.rs`) and are asserted by each owning crate.
|
||||
|
||||
## ECStore Access Boundary
|
||||
|
||||
@@ -47,7 +47,7 @@ RustFS-side statics that matter architecturally because other modules are tempte
|
||||
| `KEYSTONE_AUTH`, `KEYSTONE_MAPPER`, `KEYSTONE_CONFIG` | `rustfs/src/auth_keystone.rs` | Keystone provider, mapper, and config stay private to the Keystone owner. |
|
||||
| `DEADLOCK_DETECTOR` | `rustfs/src/storage/deadlock_detector.rs` | Detector lifecycle stays private to the storage deadlock detector. |
|
||||
| `CONCURRENCY_MANAGER` | `rustfs/src/storage/concurrency/manager.rs` | Storage concurrency scheduler state stays inside the concurrency owner. |
|
||||
| `GLOBAL_KMS_DEK_PROVIDER`, `GLOBAL_SSE_DEK_PROVIDER` | `crates/sse/src/lib.rs` | DEK provider caches stay private to the SSE owner. |
|
||||
| `GLOBAL_KMS_DEK_PROVIDER`, `GLOBAL_SSE_DEK_PROVIDER` | `rustfs/src/storage/sse.rs` | DEK provider caches stay private to the SSE owner. |
|
||||
| `ECSTORE_EVENT_DISPATCH_HOOK` | `rustfs/src/server/event.rs` | Event bridge registration goes through the storage facade. |
|
||||
| `AUDIT_MODULE_ENABLED`, `NOTIFY_MODULE_ENABLED` | `rustfs/src/module_switches.rs` | Module toggles are read through module-switch helpers; `MODULE_SWITCH_RMW_LOCK` (`rustfs/src/server/module_switch.rs`) serializes persisted updates. |
|
||||
| `RUNTIME_CONFIG_RELOAD_MUTEX` | `rustfs/src/admin/service/config.rs` | Serializes dynamic config reload fanout. |
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# KMS Bulk Rekey Job Contract
|
||||
|
||||
**Use this when:** changing the bulk envelope re-wrap sweep (`rustfs/src/kms_rekey.rs`), its admin endpoints (`rustfs/src/admin/handlers/kms_rekey.rs`), the re-wrap primitive, or anything that decides which objects a rekey may touch.
|
||||
**Source of truth:** `rustfs/src/kms_rekey.rs`, `rustfs/src/admin/handlers/kms_rekey.rs`, `rewrap_object_encryption_metadata` in `crates/sse/src/lib.rs`, `KmsManager::rewrap_data_key` / `KmsManager::describe_data_key_wrapping` in `crates/kms/src/manager.rs`, `put_object_metadata` in `crates/ecstore/src/set_disk/ops/object.rs`.
|
||||
**Source of truth:** `rustfs/src/kms_rekey.rs`, `rustfs/src/admin/handlers/kms_rekey.rs`, `rewrap_object_encryption_metadata` in `rustfs/src/storage/sse.rs`, `KmsManager::rewrap_data_key` / `KmsManager::describe_data_key_wrapping` in `crates/kms/src/manager.rs`, `put_object_metadata` in `crates/ecstore/src/set_disk/ops/object.rs`.
|
||||
|
||||
The bulk rekey job re-wraps stored data-key envelopes under the current key-encryption key (KEK) without rewriting object bodies. This document is the acceptance bar; where the shipped v1 sweep deliberately narrows it, [Implementation Status](#implementation-status-v1-sweep) records the deviation.
|
||||
|
||||
@@ -52,7 +52,7 @@ Idempotency comes from object metadata itself: the envelope's state is the targe
|
||||
|
||||
### Reading the wrapping KEK version
|
||||
|
||||
There is no key-version metadata key. Object metadata carries the key id (`x-rustfs-encryption-key-id`) and the sealed blob under `x-rustfs-encryption-key`; `DecryptResponse` in `crates/kms/src/types.rs` does not report a version either. The version is recoverable because the sealed blob is structured: for every backend that builds one, the ciphertext is the JSON of `DataKeyEnvelope` (`crates/kms/src/encryption/dek.rs`), and the read path already discriminates on it via `is_data_key_envelope` in `crates/sse/src/lib.rs`.
|
||||
There is no key-version metadata key. Object metadata carries the key id (`x-rustfs-encryption-key-id`) and the sealed blob under `x-rustfs-encryption-key`; `DecryptResponse` in `crates/kms/src/types.rs` does not report a version either. The version is recoverable because the sealed blob is structured: for every backend that builds one, the ciphertext is the JSON of `DataKeyEnvelope` (`crates/kms/src/encryption/dek.rs`), and the read path already discriminates on it via `is_data_key_envelope` in `rustfs/src/storage/sse.rs`.
|
||||
|
||||
| Backend | Rotates | Where the wrapping version lives | Recoverable by a scan |
|
||||
|---|---|---|---|
|
||||
@@ -95,7 +95,7 @@ Replication destinations are unresolved: propagation depends on `rustfs/backlog#
|
||||
|
||||
Three properties of `put_object_metadata` (`crates/ecstore/src/set_disk/ops/object.rs`) constrain the re-wrap write:
|
||||
|
||||
- **The merge is additive; it cannot remove keys.** Overwriting a key that keeps its name is safe; a re-wrap that changes *which* keys describe the envelope leaves the old keys behind. This is a live hazard: `parse_minio_managed_sealed_key` in `crates/sse/src/lib.rs` selects the MinIO decrypt branch on the mere presence of the MinIO seal-algorithm header, so a RustFS-native envelope written onto MinIO-compatible headers without neutralizing them steers reads down the stale branch. Any envelope-shape change must neutralize superseded keys in the same write.
|
||||
- **The merge is additive; it cannot remove keys.** Overwriting a key that keeps its name is safe; a re-wrap that changes *which* keys describe the envelope leaves the old keys behind. This is a live hazard: `parse_minio_managed_sealed_key` in `rustfs/src/storage/sse.rs` selects the MinIO decrypt branch on the mere presence of the MinIO seal-algorithm header, so a RustFS-native envelope written onto MinIO-compatible headers without neutralizing them steers reads down the stale branch. Any envelope-shape change must neutralize superseded keys in the same write.
|
||||
- **Object-lock retention is enforced before the merge.** `check_object_lock_retention_update` (`crates/ecstore/src/set_disk/mod.rs`) runs first; rekey inherits its decision and must not acquire a bypass.
|
||||
- **`mod_time` is preserved unless the caller sets it.** The re-wrap path leaves it unset so age-based lifecycle rules are not perturbed.
|
||||
|
||||
|
||||
@@ -1,13 +1,13 @@
|
||||
# MinIO On-Disk Format Interoperability
|
||||
|
||||
**Use this when:** deciding whether a MinIO drive set, bucket-metadata blob, or SSE object can be read or imported by a given RustFS build, or before touching any constant or codec listed under Version Anchors.
|
||||
**Source of truth:** `crates/filemeta/src/filemeta.rs`, `crates/filemeta/src/filemeta/codec.rs`, `crates/ecstore/src/bucket/metadata.rs`, `crates/ecstore/src/bucket/migration.rs`, `crates/sse/src/lib.rs`, `rustfs/Cargo.toml` `[features]`, `.github/workflows/ci.yml`, `.github/workflows/minio-interop.yml`.
|
||||
**Source of truth:** `crates/filemeta/src/filemeta.rs`, `crates/filemeta/src/filemeta/codec.rs`, `crates/ecstore/src/bucket/metadata.rs`, `crates/ecstore/src/bucket/migration.rs`, `rustfs/src/storage/sse.rs`, `rustfs/Cargo.toml` `[features]`, `.github/workflows/ci.yml`, `.github/workflows/minio-interop.yml`.
|
||||
|
||||
This is an interop contract, not a plan. Migration is one-way (MinIO to RustFS). Erasure-coding internals are owned by [erasure-coding.md](erasure-coding.md); this document owns the interop claim, the fixture evidence, and the out-of-scope list.
|
||||
|
||||
## Scope Matrix By Build Variant
|
||||
|
||||
Build variants are the `rustfs` crate features in `rustfs/Cargo.toml`: `default`, `full`, and `rio-v2` (which enables `rustfs-ecstore/rio-v2` and `rustfs-sse/rio-v2`, and pulls in `crates/rio-v2`). `rio-v2` is absent from both `default` and `full`.
|
||||
Build variants are the `rustfs` crate features in `rustfs/Cargo.toml`: `default`, `full`, and `rio-v2` (which enables `rustfs-ecstore/rio-v2` and pulls in `crates/rio-v2`). `rio-v2` is absent from both `default` and `full`.
|
||||
|
||||
| MinIO artifact | `default` / `full` build | `rio-v2` build | Notes |
|
||||
|---|:--:|:--:|---|
|
||||
@@ -81,7 +81,7 @@ The `xl.meta` around a MinIO SSE object parses in every build, so such objects l
|
||||
|
||||
### Seams
|
||||
|
||||
The cryptography (DARE v2 stream format, object-key derivation, sealing) was never the gap. Three metadata seams above it rejected MinIO objects; all are closed in `rio-v2` builds. Symbols are in `crates/sse/src/lib.rs` unless noted.
|
||||
The cryptography (DARE v2 stream format, object-key derivation, sealing) was never the gap. Three metadata seams above it rejected MinIO objects; all are closed in `rio-v2` builds. Symbols are in `rustfs/src/storage/sse.rs` unless noted.
|
||||
|
||||
| Seam | Resolution |
|
||||
|---|---|
|
||||
@@ -113,10 +113,10 @@ Inventory the source first: bucket default encryption means objects can be encry
|
||||
| Fact | Value |
|
||||
|---|---|
|
||||
| Shipping status | Ships in no default build: absent from `default` and `full` in `rustfs/Cargo.toml`; released binaries and container images never include it. Enable with `--features rio-v2`. |
|
||||
| Pull-request coverage | `test-and-lint-rio-v2` in `.github/workflows/ci.yml`: clippy plus `cargo nextest` for `rustfs`, `rustfs-ecstore`, and `rustfs-sse` with `--features rio-v2`. This is the cfg-seam guard; it keeps the feature compiling and its unit suite green on every pull request. |
|
||||
| Pull-request coverage | `test-and-lint-rio-v2` in `.github/workflows/ci.yml`: clippy plus `cargo nextest` for `rustfs` and `rustfs-ecstore` with `--features rio-v2`. This is the cfg-seam guard; it keeps the feature compiling and its unit suite green on every pull request. |
|
||||
| Full-suite lane | `build-rustfs-debug-binary-rio-v2` and `e2e-tests-rio-v2` in `ci.yml` run only on the nightly schedule and manual dispatch (`cache-warm.yml` keeps the `ci-feat-rio` cache warm so the scheduled build fits its timeout). |
|
||||
| Interop evidence | `.github/workflows/minio-interop.yml` (nightly plus manual) regenerates real MinIO backend trees via `crates/rio-v2/tests/minio_fixture_lab/` and runs the `#[ignore]` reader tests in `rustfs/src/storage/minio_generated_read_test.rs` with `--features rio-v2`. Its freshness is tracked in `.github/scheduled-validations.json`. |
|
||||
| Promote-or-delete condition | The variant stays dormant until one of two things happens. **Promote**: a release commits to shipping MinIO SSE migration as a supported capability; then `rio-v2` joins `default`/`full`, the scheduled lanes run on every pull request, and this section is rewritten. **Delete**: no release commits to it and the scheduled lanes are not kept green; then the feature flag, `crates/rio-v2`, the cfg seams in `crates/sse/src/lib.rs`, the three `ci.yml` jobs, the `cache-warm.yml` warm step, `minio-interop.yml`, and its `scheduled-validations.json` entry are removed in one change. Either outcome must update `ARCHITECTURE.md` and the `ci.yml` job comments that cite this section. |
|
||||
| Promote-or-delete condition | The variant stays dormant until one of two things happens. **Promote**: a release commits to shipping MinIO SSE migration as a supported capability; then `rio-v2` joins `default`/`full`, the scheduled lanes run on every pull request, and this section is rewritten. **Delete**: no release commits to it and the scheduled lanes are not kept green; then the feature flag, `crates/rio-v2`, the cfg seams in `rustfs/src/storage/sse.rs`, the three `ci.yml` jobs, the `cache-warm.yml` warm step, `minio-interop.yml`, and its `scheduled-validations.json` entry are removed in one change. Either outcome must update `ARCHITECTURE.md` and the `ci.yml` job comments that cite this section. |
|
||||
|
||||
## Fixture Evidence
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# AWS KMS backend: setup, verification, and operations
|
||||
|
||||
**Use this when:** connecting RustFS to AWS KMS for the first time, verifying an existing AWS-backed deployment, upgrading one from 1.0.4 or 1.0.5, or deciding how to run and protect the AWS keys RustFS depends on.
|
||||
**Source of truth:** `crates/kms/src/backends/aws.rs` (backend behavior and its deviations), `crates/kms/src/config.rs` (`ENV_KMS_AWS_*`), `crates/kms/src/policy.rs` (retry, deadline and circuit breaker), `crates/sse/src/lib.rs` (SSE data path). 中文版: [kms-aws-backend-guide.zh-CN.md](kms-aws-backend-guide.zh-CN.md).
|
||||
**Source of truth:** `crates/kms/src/backends/aws.rs` (backend behavior and its deviations), `crates/kms/src/config.rs` (`ENV_KMS_AWS_*`), `crates/kms/src/policy.rs` (retry, deadline and circuit breaker), `rustfs/src/storage/sse.rs` (SSE data path). 中文版: [kms-aws-backend-guide.zh-CN.md](kms-aws-backend-guide.zh-CN.md).
|
||||
|
||||
The statements about behavior on this page were observed against the real AWS KMS service (region `eu-north-1`) and, for failures AWS cannot be asked to produce on demand, against a loopback stand-in that speaks the AWS KMS wire protocol. Where a statement is guidance that was not exercised, it says so. For the contract-level differences between AWS and the other backends, see [KMS backend security properties](kms-backend-security.md#aws-kms-deviations-from-the-shared-backend-contract).
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# AWS KMS 后端:接入、验证与运维
|
||||
|
||||
**适用场景:** 首次把 RustFS 接入 AWS KMS、验证已有的 AWS 后端部署、从 1.0.4 或 1.0.5 升级,或者规划如何运维并保护 RustFS 所依赖的 AWS 密钥。
|
||||
**事实来源:** `crates/kms/src/backends/aws.rs`(后端行为及其差异)、`crates/kms/src/config.rs`(`ENV_KMS_AWS_*`)、`crates/kms/src/policy.rs`(重试、超时与熔断)、`crates/sse/src/lib.rs`(SSE 数据面)。English: [kms-aws-backend-guide.md](kms-aws-backend-guide.md)。
|
||||
**事实来源:** `crates/kms/src/backends/aws.rs`(后端行为及其差异)、`crates/kms/src/config.rs`(`ENV_KMS_AWS_*`)、`crates/kms/src/policy.rs`(重试、超时与熔断)、`rustfs/src/storage/sse.rs`(SSE 数据面)。English: [kms-aws-backend-guide.md](kms-aws-backend-guide.md)。
|
||||
|
||||
本页关于行为的描述,均来自对真实 AWS KMS 服务(区域 `eu-north-1`)的实测;对于无法让 AWS 按需产生的故障,则使用一个实现了 AWS KMS 线上协议的本机替身来实测。属于未经实测的建议之处,文中会明确标注。AWS 与其它后端在契约层面的差异,见 [KMS backend security properties](kms-backend-security.md#aws-kms-deviations-from-the-shared-backend-contract)。
|
||||
|
||||
|
||||
+1
-3
@@ -80,7 +80,7 @@ e2e-test-hooks = ["rustfs-ecstore/e2e-test-hooks", "rustfs-scanner/e2e-test-hook
|
||||
connect-e2e-short-credentials = []
|
||||
# Builds the dedicated rustfs-cli-e2e target with a build-time public enrollment root.
|
||||
offline-enrollment-e2e-root = []
|
||||
rio-v2 = ["rustfs-ecstore/rio-v2", "rustfs-sse/rio-v2"]
|
||||
rio-v2 = ["rustfs-ecstore/rio-v2"]
|
||||
# Linux-only cuObject S3-over-RDMA GET into client (GPU) memory; needs libcuobjserver.
|
||||
cuobject = ["dep:rustfs-cuobject", "rustfs-cuobject/server"]
|
||||
# NIXL/UCX transport for S3-over-RDMA; replaces cuobject when both are enabled.
|
||||
@@ -281,7 +281,6 @@ rustfs-rio = { workspace = true }
|
||||
rustfs-s3-client = { workspace = true }
|
||||
rustfs-s3-types = { workspace = true, features = ["compat-s3s"] }
|
||||
rustfs-s3-ops = { workspace = true }
|
||||
rustfs-sse = { workspace = true }
|
||||
rustfs-security-governance = { workspace = true }
|
||||
rustfs-data-usage = { workspace = true }
|
||||
rustfs-s3select-api = { workspace = true }
|
||||
@@ -425,7 +424,6 @@ h2.workspace = true
|
||||
rustfs-targets = { workspace = true, features = ["test-support"] }
|
||||
uuid = { workspace = true, features = ["v4", "v5", "fast-rng", "macro-diagnostics"] }
|
||||
rustfs-iam = { workspace = true, features = ["test-util"] }
|
||||
rustfs-sse = { workspace = true, features = ["test-util"] }
|
||||
serial_test = { workspace = true }
|
||||
tempfile = { workspace = true }
|
||||
aws-config = { workspace = true }
|
||||
|
||||
@@ -195,27 +195,14 @@ impl Operation for ChangeOwnPasswordHandler {
|
||||
let request: ChangePasswordRequest = serde_json::from_slice(&body)
|
||||
.map_err(|e| s3::error(S3ErrorCode::InvalidRequest, format!("invalid change-password request: {e}")))?;
|
||||
|
||||
let iam_store =
|
||||
current_ready_iam_handle().map_err(|_| s3::error(S3ErrorCode::InternalError, "iam is not initialized"))?;
|
||||
|
||||
let Some(stored) = iam_store.get_user(&caller.access_key).await else {
|
||||
// Reached only if the identity was deleted between authentication
|
||||
// and this lookup.
|
||||
emit_audit(
|
||||
&audit,
|
||||
AccountAuditRecord::failure(
|
||||
AccountAuditOperation::ChangeOwnPassword,
|
||||
&caller.access_key,
|
||||
caller.identity_type,
|
||||
AccountAuditFailure::Internal,
|
||||
),
|
||||
);
|
||||
return Err(s3::error(S3ErrorCode::InvalidRequest, "the calling identity no longer exists"));
|
||||
// An STS session signs with its own secret. A direct IAM request must
|
||||
// have been signed with the same password the caller is replacing.
|
||||
let expected_signing_secret = if caller.identity_type == IdentityType::Sts {
|
||||
&caller.credentials.secret_key
|
||||
} else {
|
||||
&request.current_secret_key
|
||||
};
|
||||
|
||||
if !constant_time_eq(&request.current_secret_key, &stored.credentials.secret_key)
|
||||
|| !constant_time_eq(&request.current_secret_key, &signing_secret)
|
||||
{
|
||||
if !constant_time_eq(expected_signing_secret, &signing_secret) {
|
||||
warn!(
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_ACCOUNT,
|
||||
@@ -256,8 +243,10 @@ impl Operation for ChangeOwnPasswordHandler {
|
||||
}
|
||||
|
||||
let access_key = caller.access_key.clone();
|
||||
let new_secret_key = request.new_secret_key.clone();
|
||||
let current_secret_key = request.current_secret_key;
|
||||
let new_secret_key = request.new_secret_key;
|
||||
let identity_type = caller.identity_type;
|
||||
let session_credentials = (identity_type == IdentityType::Sts).then(|| caller.credentials.clone());
|
||||
let session_access_key = caller.session_access_key.clone();
|
||||
let audit_for_task = audit.clone();
|
||||
|
||||
@@ -268,16 +257,35 @@ impl Operation for ChangeOwnPasswordHandler {
|
||||
let iam_store =
|
||||
current_ready_iam_handle().map_err(|_| s3::error(S3ErrorCode::InternalError, "iam is not initialized"))?;
|
||||
|
||||
let (updated_at, status, revoked) = iam_store
|
||||
.change_own_secret_key(&access_key, &signing_secret, &new_secret_key)
|
||||
.await
|
||||
.map_err(|error| {
|
||||
if matches!(error, rustfs_iam::error::Error::IAMActionNotAllowed) {
|
||||
s3::error(S3ErrorCode::InvalidRequest, "the current secret key is incorrect")
|
||||
} else {
|
||||
iam_error_to_s3_error(error)
|
||||
}
|
||||
})?;
|
||||
let rotation = match session_credentials.as_ref() {
|
||||
Some(session) => {
|
||||
iam_store
|
||||
.change_own_secret_key_from_session(&access_key, ¤t_secret_key, &new_secret_key, session)
|
||||
.await
|
||||
}
|
||||
None => {
|
||||
iam_store
|
||||
.change_own_secret_key(&access_key, ¤t_secret_key, &new_secret_key)
|
||||
.await
|
||||
}
|
||||
};
|
||||
let (updated_at, status, revoked) = rotation.map_err(|error| {
|
||||
if matches!(error, rustfs_iam::error::Error::IAMActionNotAllowed) {
|
||||
emit_audit(
|
||||
&audit_for_task,
|
||||
AccountAuditRecord::failure(
|
||||
AccountAuditOperation::ChangeOwnPassword,
|
||||
&access_key,
|
||||
identity_type,
|
||||
AccountAuditFailure::InvalidCurrentSecret,
|
||||
)
|
||||
.with_session_access_key(session_access_key.as_deref()),
|
||||
);
|
||||
s3::error(S3ErrorCode::InvalidRequest, "the current secret key is incorrect")
|
||||
} else {
|
||||
iam_error_to_s3_error(error)
|
||||
}
|
||||
})?;
|
||||
|
||||
emit_audit(
|
||||
&audit_for_task,
|
||||
|
||||
+26
-25
@@ -1019,32 +1019,33 @@ impl<T> S3Router<T> {
|
||||
pub(crate) fn registered_routes(&self) -> &[String] {
|
||||
self.table.registered_routes()
|
||||
}
|
||||
}
|
||||
|
||||
/// Dispatches one legacy admin request. Callers keep their assertions and do not name s3s.
|
||||
/// Free function (not an inherent impl) so the `OwnedOperation` bound stays `pub(crate)`.
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn call_legacy<T: OwnedOperation>(
|
||||
router: &S3Router<T>,
|
||||
method: Method,
|
||||
uri: &str,
|
||||
extensions: http::Extensions,
|
||||
) -> s3s::S3Result<S3Response<Body>> {
|
||||
S3Route::call(
|
||||
router,
|
||||
S3Request {
|
||||
input: Body::empty(),
|
||||
method,
|
||||
uri: uri.parse().expect("test URI"),
|
||||
headers: HeaderMap::new(),
|
||||
extensions,
|
||||
credentials: None,
|
||||
region: None,
|
||||
service: None,
|
||||
trailing_headers: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
/// Dispatches one legacy admin request. Callers keep their assertions and do not name s3s.
|
||||
pub(crate) async fn call_legacy(
|
||||
&self,
|
||||
method: Method,
|
||||
uri: &str,
|
||||
extensions: http::Extensions,
|
||||
) -> s3s::S3Result<S3Response<Body>>
|
||||
where
|
||||
T: OwnedOperation,
|
||||
{
|
||||
S3Route::call(
|
||||
self,
|
||||
S3Request {
|
||||
input: Body::empty(),
|
||||
method,
|
||||
uri: uri.parse().expect("test URI"),
|
||||
headers: HeaderMap::new(),
|
||||
extensions,
|
||||
credentials: None,
|
||||
region: None,
|
||||
service: None,
|
||||
trailing_headers: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> Default for S3Router<T> {
|
||||
|
||||
@@ -141,7 +141,7 @@ mod tests {
|
||||
object_store_from_envelope, publish_test_app_context,
|
||||
};
|
||||
use crate::admin::operation::{AdminRequest, AdminResponse, Operation};
|
||||
use crate::admin::router::{S3Router, call_legacy};
|
||||
use crate::admin::router::S3Router;
|
||||
use crate::app::{RequestEnvelope, RequestHead};
|
||||
use hyper::{Method, StatusCode};
|
||||
use matchit::Params;
|
||||
@@ -323,13 +323,15 @@ mod tests {
|
||||
.insert(Method::GET, "/context-probe", ContextDependentAdminRoute)
|
||||
.expect("register test route");
|
||||
|
||||
let err = call_legacy(&router, Method::GET, "/context-probe", http::Extensions::new())
|
||||
let err = router
|
||||
.call_legacy(Method::GET, "/context-probe", http::Extensions::new())
|
||||
.await
|
||||
.expect_err("empty slot must fail closed");
|
||||
assert_eq!(err.code(), &S3ErrorCode::ServiceUnavailable.into());
|
||||
|
||||
assert!(slot.install(distinct_context(&ambient)));
|
||||
let response = call_legacy(&router, Method::GET, "/context-probe", http::Extensions::new())
|
||||
let response = router
|
||||
.call_legacy(Method::GET, "/context-probe", http::Extensions::new())
|
||||
.await
|
||||
.expect("installed slot must serve request");
|
||||
assert_eq!(response.status, Some(StatusCode::NO_CONTENT));
|
||||
|
||||
@@ -87,6 +87,8 @@ pub(crate) enum CredentialMutationDenial {
|
||||
ServiceAccount,
|
||||
/// A derived credential whose parent could not be determined.
|
||||
UnresolvedParent,
|
||||
/// A storage delegation has no account-management authority.
|
||||
TableStorageSession,
|
||||
}
|
||||
|
||||
impl CredentialMutationDenial {
|
||||
@@ -97,6 +99,7 @@ impl CredentialMutationDenial {
|
||||
}
|
||||
Self::FederatedIdentity => "federated identities are managed by their identity provider",
|
||||
Self::ServiceAccount => "service account credentials cannot change the credentials of their parent identity",
|
||||
Self::TableStorageSession => "table-scoped storage credentials cannot manage account credentials",
|
||||
Self::UnresolvedParent => "the parent identity of this session could not be resolved",
|
||||
}
|
||||
}
|
||||
@@ -187,7 +190,7 @@ impl CallerIdentity {
|
||||
// root", and an unresolved parent must never fall through to a
|
||||
// permissive answer.
|
||||
//
|
||||
// These three denials apply to both capabilities: a machine credential
|
||||
// These denials apply to both capabilities: a machine credential
|
||||
// must not take over its parent, a federated identity is owned by its
|
||||
// IdP, and an unresolvable parent fails closed.
|
||||
let shared_denial = if unresolved_parent {
|
||||
@@ -196,6 +199,11 @@ impl CallerIdentity {
|
||||
Some(CredentialMutationDenial::ServiceAccount)
|
||||
} else if federated {
|
||||
Some(CredentialMutationDenial::FederatedIdentity)
|
||||
} else if ["rustfs:table-bucket", "rustfs:table-id", "rustfs:credential-scope-prefix"]
|
||||
.iter()
|
||||
.any(|claim| credentials.claims_or_empty().contains_key(*claim))
|
||||
{
|
||||
Some(CredentialMutationDenial::TableStorageSession)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
@@ -501,6 +509,20 @@ mod tests {
|
||||
assert_eq!(caller.mutation_denial, Some(CredentialMutationDenial::FederatedIdentity));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn table_storage_sessions_cannot_manage_parent_credentials() {
|
||||
for claim in ["rustfs:table-bucket", "rustfs:table-id", "rustfs:credential-scope-prefix"] {
|
||||
let mut credentials = sts_session("TABLEKEY", "table-owner");
|
||||
credentials.claims = Some(HashMap::from([(claim.to_string(), Value::String("scope".to_string()))]));
|
||||
let caller = CallerIdentity::from_credentials(credentials, false);
|
||||
|
||||
assert_eq!(caller.mutation_denial, Some(CredentialMutationDenial::TableStorageSession));
|
||||
assert!(!caller.mutability().password);
|
||||
assert!(caller.ensure_credential_mutation_allowed().is_err());
|
||||
assert!(caller.ensure_mfa_management_allowed().is_err());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn derived_credential_without_a_parent_is_denied_rather_than_allowed() {
|
||||
// Fail closed: an unresolvable parent must not be treated as a
|
||||
|
||||
@@ -1320,22 +1320,23 @@ mod tests {
|
||||
};
|
||||
for (index, expected_key) in ["a&local", "z-local"].into_iter().enumerate() {
|
||||
let request_marker = input.marker.clone().unwrap_or_default();
|
||||
let list_req = S3Request {
|
||||
input: input.clone(),
|
||||
method: http::Method::GET,
|
||||
uri: http::Uri::from_static("/"),
|
||||
headers: HeaderMap::new(),
|
||||
extensions: http::Extensions::new(),
|
||||
credentials: None,
|
||||
region: None,
|
||||
service: None,
|
||||
trailing_headers: None,
|
||||
};
|
||||
let envelope = crate::app::RequestEnvelope::from_s3s(&list_req);
|
||||
let gateway_input =
|
||||
rustfs_gateway_types::compat::s3s_5761ddfe::generated::ops::list_objects::input_from_s3s(
|
||||
input.clone(),
|
||||
list_req.input,
|
||||
)
|
||||
.expect("convert ListObjectsInput");
|
||||
let envelope = crate::app::RequestEnvelope::from_parts(
|
||||
crate::app::RequestHead {
|
||||
method: http::Method::GET,
|
||||
uri: http::Uri::from_static("/"),
|
||||
headers: HeaderMap::new(),
|
||||
credentials: None,
|
||||
region: None,
|
||||
service: None,
|
||||
},
|
||||
http::Extensions::new(),
|
||||
);
|
||||
let gateway_output = tokio::time::timeout(
|
||||
Duration::from_secs(10),
|
||||
DefaultBucketUsecase::from_global().execute_list_objects(envelope, gateway_input),
|
||||
|
||||
@@ -509,7 +509,7 @@ impl DefaultBucketUsecase {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::app::bucket_usecase::test_support::{build_request_with_req_info, owned_envelope};
|
||||
use crate::app::bucket_usecase::test_support::{build_request, build_request_with_req_info};
|
||||
use crate::app::storage_api::s3::dto::CreateBucketInput;
|
||||
use http::Method;
|
||||
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
||||
@@ -1005,10 +1005,11 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_list_buckets_returns_internal_error_when_store_uninitialized() {
|
||||
let req = build_request((), Method::GET);
|
||||
let usecase = DefaultBucketUsecase::without_context();
|
||||
|
||||
let err = usecase
|
||||
.execute_list_buckets(owned_envelope(Method::GET), gateway_dto::ListBucketsInput::default())
|
||||
.execute_list_buckets(RequestEnvelope::from_s3s(&req), gateway_dto::ListBucketsInput::default())
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(err.code(), &rustfs_s3_types::S3ErrorCode::InternalError);
|
||||
|
||||
@@ -366,7 +366,7 @@ impl DefaultBucketUsecase {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::app::bucket_usecase::test_support::{build_request_with_req_info, owned_envelope, s3_op_total};
|
||||
use crate::app::bucket_usecase::test_support::{build_request, build_request_with_req_info, s3_op_total};
|
||||
use http::Method;
|
||||
|
||||
#[test]
|
||||
@@ -399,10 +399,11 @@ mod tests {
|
||||
|
||||
/// The envelope of a request whose access phase installed `req_info`, or none.
|
||||
fn envelope(req_info: Option<ReqInfo>) -> RequestEnvelope {
|
||||
match req_info {
|
||||
Some(req_info) => RequestEnvelope::from_s3s(&build_request_with_req_info((), Method::PUT, req_info)),
|
||||
None => owned_envelope(Method::PUT),
|
||||
}
|
||||
let req = match req_info {
|
||||
Some(req_info) => build_request_with_req_info((), Method::PUT, req_info),
|
||||
None => build_request((), Method::PUT),
|
||||
};
|
||||
RequestEnvelope::from_s3s(&req)
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -203,7 +203,7 @@ impl DefaultBucketUsecase {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::app::bucket_usecase::test_support::owned_envelope;
|
||||
use crate::app::bucket_usecase::test_support::build_request;
|
||||
use crate::app::storage_api::s3::dto::{self, PutBucketEncryptionInput, ServerSideEncryption};
|
||||
use http::Method;
|
||||
use rustfs_gateway_types::persistence::{
|
||||
@@ -292,19 +292,20 @@ mod tests {
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let prepared = crate::storage::persisted_from_s3s(&input.server_side_encryption_configuration);
|
||||
let req = build_request(input, Method::PUT);
|
||||
let prepared = crate::storage::persisted_from_s3s(&req.input.server_side_encryption_configuration);
|
||||
let usecase = DefaultBucketUsecase::without_context();
|
||||
|
||||
let err = usecase
|
||||
.execute_put_bucket_encryption(owned_envelope(Method::PUT), &input.bucket, prepared.map_err(Into::into))
|
||||
.execute_put_bucket_encryption(RequestEnvelope::from_s3s(&req), &req.input.bucket, prepared.map_err(Into::into))
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
|
||||
let err = usecase
|
||||
.execute_put_bucket_encryption(
|
||||
owned_envelope(Method::PUT),
|
||||
&input.bucket,
|
||||
RequestEnvelope::from_s3s(&req),
|
||||
&req.input.bucket,
|
||||
Ok(PersistedBucketEncryptionConfiguration { rules: vec![] }),
|
||||
)
|
||||
.await
|
||||
|
||||
@@ -405,7 +405,7 @@ mod tests {
|
||||
// waiter again on the single-thread test runtime.
|
||||
assert_eq!(admission.available_permits(), 1);
|
||||
}
|
||||
use crate::app::bucket_usecase::test_support::{build_request, owned_envelope};
|
||||
use crate::app::bucket_usecase::test_support::build_request;
|
||||
use crate::app::gating_test_env::{run_large_stack_test, shared_gating_ambient};
|
||||
use crate::app::runtime_sources::ServerContextSlot;
|
||||
use crate::app::storage_api::bucket_usecase::helper::MutationCompletionProbe;
|
||||
@@ -775,12 +775,13 @@ mod tests {
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let req = build_request(input, Method::PUT);
|
||||
let usecase = DefaultBucketUsecase::without_context();
|
||||
|
||||
let err = usecase
|
||||
.execute_put_bucket_lifecycle_configuration(
|
||||
owned_envelope(Method::PUT),
|
||||
&input.bucket,
|
||||
RequestEnvelope::from_s3s(&req),
|
||||
&req.input.bucket,
|
||||
PreparedLifecycleConfiguration::Missing,
|
||||
)
|
||||
.await
|
||||
|
||||
@@ -159,7 +159,7 @@ fn list_object_versions_output(
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::app::bucket_usecase::test_support::owned_envelope;
|
||||
use crate::app::bucket_usecase::test_support::build_request;
|
||||
use rustfs_gateway_types::BucketName;
|
||||
use rustfs_s3_types::S3ErrorCode;
|
||||
|
||||
@@ -319,7 +319,7 @@ mod tests {
|
||||
};
|
||||
|
||||
let err = DefaultBucketUsecase::without_context()
|
||||
.execute_list_object_versions(owned_envelope(http::Method::GET), input)
|
||||
.execute_list_object_versions(RequestEnvelope::from_s3s(&build_request((), http::Method::GET)), input)
|
||||
.await
|
||||
.expect_err("a negative page size");
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
|
||||
@@ -333,7 +333,7 @@ mod tests {
|
||||
};
|
||||
|
||||
let err = DefaultBucketUsecase::without_context()
|
||||
.execute_list_object_versions(owned_envelope(http::Method::GET), input)
|
||||
.execute_list_object_versions(RequestEnvelope::from_s3s(&build_request((), http::Method::GET)), input)
|
||||
.await
|
||||
.expect_err("no store");
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
|
||||
@@ -372,12 +372,13 @@ fn list_objects_next_marker(info: &ListObjectsV2Info, cursor: Option<&str>) -> O
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::app::bucket_usecase::test_support::owned_envelope;
|
||||
use crate::app::bucket_usecase::test_support::build_request;
|
||||
use rustfs_gateway_types::BucketName;
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn list_apis_report_storage_read_quorum_as_unavailable() {
|
||||
use crate::app::bucket_usecase::test_support::owned_envelope;
|
||||
use crate::app::gating_test_env::{run_large_stack_test, shared_gating_ambient, shared_gating_ecstore};
|
||||
run_large_stack_test("list-quorum-surface", || async {
|
||||
let usecase = DefaultBucketUsecase::with_context(Some(shared_gating_ambient().await));
|
||||
@@ -484,7 +485,7 @@ mod tests {
|
||||
}
|
||||
|
||||
fn envelope() -> RequestEnvelope {
|
||||
owned_envelope(http::Method::GET)
|
||||
RequestEnvelope::from_s3s(&build_request((), http::Method::GET))
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -56,7 +56,7 @@ impl DefaultBucketUsecase {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::app::bucket_usecase::test_support::{build_request, owned_envelope};
|
||||
use crate::app::bucket_usecase::test_support::build_request;
|
||||
use crate::app::bucket_usecase::{BucketOperations, DeleteBucketOptions, MakeBucketOptions};
|
||||
use crate::app::gating_test_env::{run_large_stack_test, shared_gating_ambient};
|
||||
use crate::app::runtime_sources::ServerContextSlot;
|
||||
@@ -75,8 +75,10 @@ mod tests {
|
||||
let context = shared_gating_ambient().await;
|
||||
let bucket = format!("logging-missing-{}", uuid::Uuid::new_v4().simple());
|
||||
let usecase = DefaultBucketUsecase::with_context(Some(context));
|
||||
let request = build_request((), Method::PUT);
|
||||
let envelope = RequestEnvelope::from_s3s(&request);
|
||||
let error = usecase
|
||||
.execute_put_bucket_logging(owned_envelope(Method::PUT), &bucket, || {
|
||||
.execute_put_bucket_logging(envelope, &bucket, || {
|
||||
Err(rustfs_s3_types::s3_error!(MalformedXML, "invalid logging configuration"))
|
||||
})
|
||||
.await
|
||||
|
||||
@@ -243,15 +243,16 @@ impl DefaultBucketUsecase {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::app::bucket_usecase::test_support::owned_envelope;
|
||||
use crate::app::bucket_usecase::test_support::build_request;
|
||||
use http::Method;
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_delete_bucket_policy_returns_internal_error_when_store_uninitialized() {
|
||||
let req = build_request((), Method::DELETE);
|
||||
let usecase = DefaultBucketUsecase::without_context();
|
||||
|
||||
let err = usecase
|
||||
.execute_delete_bucket_policy(owned_envelope(Method::DELETE), "test-bucket")
|
||||
.execute_delete_bucket_policy(RequestEnvelope::from_s3s(&req), "test-bucket")
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(err.code(), &rustfs_s3_types::S3ErrorCode::InternalError);
|
||||
@@ -279,10 +280,11 @@ mod tests {
|
||||
bucket: bucket("test-bucket"),
|
||||
..Default::default()
|
||||
};
|
||||
let req = build_request((), Method::GET);
|
||||
let usecase = DefaultBucketUsecase::without_context();
|
||||
|
||||
let err = usecase
|
||||
.execute_get_bucket_policy_status(owned_envelope(Method::GET), input)
|
||||
.execute_get_bucket_policy_status(RequestEnvelope::from_s3s(&req), input)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(err.code(), &rustfs_s3_types::S3ErrorCode::InternalError);
|
||||
@@ -290,10 +292,11 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_put_bucket_policy_returns_internal_error_when_store_uninitialized() {
|
||||
let req = build_request((), Method::PUT);
|
||||
let usecase = DefaultBucketUsecase::without_context();
|
||||
|
||||
let err = usecase
|
||||
.execute_put_bucket_policy(owned_envelope(Method::PUT), "test-bucket", "{}")
|
||||
.execute_put_bucket_policy(RequestEnvelope::from_s3s(&req), "test-bucket", "{}")
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(err.code(), &rustfs_s3_types::S3ErrorCode::InternalError);
|
||||
|
||||
@@ -112,7 +112,7 @@ impl DefaultBucketUsecase {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::app::bucket_usecase::test_support::{build_request, owned_envelope};
|
||||
use crate::app::bucket_usecase::test_support::build_request;
|
||||
use crate::app::gating_test_env::{run_large_stack_test, shared_gating_ambient};
|
||||
use crate::app::runtime_sources::ServerContextSlot;
|
||||
use crate::app::storage_api::bucket_usecase::helper::MutationCompletionProbe;
|
||||
@@ -278,7 +278,7 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_delete_public_access_block_returns_internal_error_when_store_uninitialized() {
|
||||
let envelope = owned_envelope(Method::DELETE);
|
||||
let envelope = RequestEnvelope::from_s3s(&build_request((), Method::DELETE));
|
||||
let usecase = DefaultBucketUsecase::without_context();
|
||||
let err = usecase
|
||||
.execute_delete_public_access_block(envelope, "test-bucket")
|
||||
@@ -296,7 +296,7 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_put_public_access_block_returns_internal_error_when_store_uninitialized() {
|
||||
let envelope = owned_envelope(Method::PUT);
|
||||
let envelope = RequestEnvelope::from_s3s(&build_request((), Method::PUT));
|
||||
let usecase = DefaultBucketUsecase::without_context();
|
||||
let err = usecase
|
||||
.execute_put_public_access_block(envelope, "test-bucket", Ok(PersistedPublicAccessBlockConfiguration::default()))
|
||||
|
||||
@@ -138,7 +138,7 @@ impl DefaultBucketUsecase {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::app::bucket_usecase::test_support::{build_request, owned_envelope};
|
||||
use crate::app::bucket_usecase::test_support::build_request;
|
||||
use crate::app::storage_api::s3::dto::{self, PutBucketTaggingInput, Tag, Tagging};
|
||||
use http::Method;
|
||||
use rustfs_s3_types::S3ErrorCode;
|
||||
@@ -249,10 +249,11 @@ mod tests {
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let req = build_request(input, Method::PUT);
|
||||
let usecase = DefaultBucketUsecase::without_context();
|
||||
|
||||
let err = usecase
|
||||
.execute_put_bucket_tagging(owned_envelope(Method::PUT), &input.bucket, PersistedTagging::default())
|
||||
.execute_put_bucket_tagging(RequestEnvelope::from_s3s(&req), &req.input.bucket, PersistedTagging::default())
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
@@ -266,7 +267,7 @@ mod tests {
|
||||
(vec![tag; MAX_BUCKET_TAGS + 1], "Cannot have more than 50 tags per bucket"),
|
||||
] {
|
||||
let err = usecase
|
||||
.execute_put_bucket_tagging(owned_envelope(Method::PUT), &input.bucket, PersistedTagging { tag_set })
|
||||
.execute_put_bucket_tagging(RequestEnvelope::from_s3s(&req), &req.input.bucket, PersistedTagging { tag_set })
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidTag);
|
||||
|
||||
@@ -47,17 +47,7 @@ pub(super) fn gateway_bucket(name: &str) -> rustfs_gateway_types::BucketName {
|
||||
|
||||
/// The envelope of a request that carries nothing but `method`.
|
||||
pub(super) fn owned_envelope(method: Method) -> RequestEnvelope {
|
||||
RequestEnvelope::from_parts(
|
||||
crate::app::RequestHead {
|
||||
method,
|
||||
uri: Uri::from_static("/"),
|
||||
headers: HeaderMap::new(),
|
||||
credentials: None,
|
||||
region: None,
|
||||
service: None,
|
||||
},
|
||||
Extensions::new(),
|
||||
)
|
||||
RequestEnvelope::from_s3s(&build_request((), method))
|
||||
}
|
||||
|
||||
pub(super) fn build_request_with_req_info<T>(input: T, method: Method, req_info: ReqInfo) -> S3Request<T> {
|
||||
|
||||
@@ -81,7 +81,7 @@ impl DefaultBucketUsecase {
|
||||
mod tests {
|
||||
use super::super::{BUCKET_ACCELERATE_CONFIG, BUCKET_REQUEST_PAYMENT_CONFIG};
|
||||
use super::*;
|
||||
use crate::app::bucket_usecase::test_support::{build_request, build_request_with_req_info, owned_envelope};
|
||||
use crate::app::bucket_usecase::test_support::{build_request, build_request_with_req_info};
|
||||
use crate::app::gating_test_env::{run_large_stack_test, shared_gating_ambient};
|
||||
use crate::app::runtime_sources::ServerContextSlot;
|
||||
use crate::app::storage_api::object_usecase::helper::MutationCompletionProbe;
|
||||
@@ -190,21 +190,22 @@ mod tests {
|
||||
let ambient = shared_gating_ambient().await;
|
||||
let usecase = DefaultBucketUsecase::with_context(Some(ambient));
|
||||
let bucket = format!("core-config-missing-{}", uuid::Uuid::new_v4().simple());
|
||||
let request = build_request((), http::Method::PUT);
|
||||
let mut converted = [false; 3];
|
||||
let accelerate = usecase
|
||||
.execute_put_bucket_accelerate_configuration(owned_envelope(http::Method::PUT), &bucket, || {
|
||||
.execute_put_bucket_accelerate_configuration(RequestEnvelope::from_s3s(&request), &bucket, || {
|
||||
converted[0] = true;
|
||||
Err(rustfs_s3_types::s3_error!(MalformedXML, "invalid acceleration"))
|
||||
})
|
||||
.await;
|
||||
let payment = usecase
|
||||
.execute_put_bucket_request_payment(owned_envelope(http::Method::PUT), &bucket, || {
|
||||
.execute_put_bucket_request_payment(RequestEnvelope::from_s3s(&request), &bucket, || {
|
||||
converted[1] = true;
|
||||
Err(rustfs_s3_types::s3_error!(MalformedXML, "invalid request payment"))
|
||||
})
|
||||
.await;
|
||||
let website = usecase
|
||||
.execute_put_bucket_website(owned_envelope(http::Method::PUT), &bucket, || {
|
||||
.execute_put_bucket_website(RequestEnvelope::from_s3s(&request), &bucket, || {
|
||||
converted[2] = true;
|
||||
Err(rustfs_s3_types::s3_error!(MalformedXML, "invalid website"))
|
||||
})
|
||||
|
||||
@@ -52,19 +52,8 @@ async fn object_lock_handlers_schedule_replication() {
|
||||
.expect("Failed to create object-lock bucket");
|
||||
upload_test_object(&ecstore, bucket.as_str(), object, b"worm payload").await;
|
||||
|
||||
let envelope = || {
|
||||
RequestEnvelope::from_parts(
|
||||
crate::app::RequestHead {
|
||||
method: Method::PUT,
|
||||
uri: Uri::from_static("/"),
|
||||
headers: HeaderMap::new(),
|
||||
credentials: None,
|
||||
region: None,
|
||||
service: None,
|
||||
},
|
||||
Extensions::new(),
|
||||
)
|
||||
};
|
||||
let req = build_request((), Method::PUT);
|
||||
let envelope = || RequestEnvelope::from_s3s(&req);
|
||||
|
||||
// A bucket without a replication configuration can skip the decision.
|
||||
MUST_REPLICATE_OBJECT_CALLS.store(0, Ordering::SeqCst);
|
||||
|
||||
@@ -22,17 +22,7 @@ const RESTORE_SUSPENDED_WAIT_TIMEOUT: Duration = Duration::from_secs(15);
|
||||
async fn suspend_test_bucket(bucket: &str) {
|
||||
DefaultBucketUsecase::from_global()
|
||||
.execute_put_bucket_versioning(
|
||||
crate::app::RequestEnvelope::from_parts(
|
||||
crate::app::RequestHead {
|
||||
method: Method::PUT,
|
||||
uri: Uri::from_static("/"),
|
||||
headers: HeaderMap::new(),
|
||||
credentials: None,
|
||||
region: None,
|
||||
service: None,
|
||||
},
|
||||
Extensions::new(),
|
||||
),
|
||||
crate::app::RequestEnvelope::from_s3s(&build_request((), Method::PUT)),
|
||||
bucket,
|
||||
Ok(rustfs_gateway_types::persistence::PersistedVersioningConfiguration {
|
||||
status: Some("Suspended".to_owned()),
|
||||
|
||||
@@ -62,7 +62,7 @@ use super::storage_api::multipart_usecase::s3_api::multipart::{
|
||||
};
|
||||
use super::storage_api::multipart_usecase::set_disk::is_valid_storage_class;
|
||||
use super::storage_api::multipart_usecase::sse::{
|
||||
DecryptionRequest, EncryptionKeyKind, EncryptionRequest, PrepareEncryptionRequest, SseKmsPrincipal, SseKmsPrincipalExt,
|
||||
DecryptionRequest, EncryptionKeyKind, EncryptionRequest, PrepareEncryptionRequest, SseKmsPrincipal,
|
||||
apply_bucket_default_lock_retention, authorize_sse_kms_multipart_write, authorize_sse_kms_object_read,
|
||||
build_ssec_read_headers, encryption_material_to_metadata, extract_server_side_encryption_from_headers,
|
||||
extract_ssec_params_from_headers, extract_ssekms_context_from_headers, get_buffer_size_opt_in,
|
||||
@@ -545,17 +545,10 @@ mod tests {
|
||||
.key(key.to_owned())
|
||||
.build()
|
||||
.expect("valid create request DTO");
|
||||
let error = match make_usecase()
|
||||
.execute_create_multipart_upload(
|
||||
owned_envelope(Method::POST),
|
||||
super::create_values::CreateMultipartUploadFields::from_legacy(create),
|
||||
None,
|
||||
)
|
||||
let error = make_usecase()
|
||||
.execute_create_multipart_upload_from_s3s(build_request(create, Method::POST))
|
||||
.await
|
||||
{
|
||||
Err(error) => crate::app::storage_api::s3::S3Error::from(error),
|
||||
Ok(_) => panic!("reserved key must be rejected before accessing storage"),
|
||||
};
|
||||
.expect_err("reserved key must be rejected before accessing storage");
|
||||
assert_eq!(error.code(), &crate::app::storage_api::s3::S3ErrorCode::InvalidArgument);
|
||||
|
||||
let complete = CompleteMultipartUploadInput::builder()
|
||||
@@ -565,15 +558,9 @@ mod tests {
|
||||
.multipart_upload(Some(CompletedMultipartUpload::default()))
|
||||
.build()
|
||||
.expect("valid complete request DTO");
|
||||
let error =
|
||||
match rustfs_gateway_types::compat::s3s_5761ddfe::generated::ops::complete_multipart_upload::input_from_s3s(
|
||||
complete,
|
||||
) {
|
||||
Err(error) => crate::app::dto_values::seam_error(error),
|
||||
Ok(input) => Box::pin(make_usecase().execute_complete_multipart_upload(owned_envelope(Method::POST), input))
|
||||
.await
|
||||
.expect_err("uploads created before the fix must not publish reserved keys"),
|
||||
};
|
||||
let error = Box::pin(make_usecase().complete_for_s3s_request(build_request(complete, Method::POST)))
|
||||
.await
|
||||
.expect_err("uploads created before the fix must not publish reserved keys");
|
||||
assert_eq!(error.code(), &rustfs_s3_types::S3ErrorCode::InvalidArgument);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -24,7 +24,8 @@ use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_abort_multipart_upload_returns_internal_error_when_store_uninitialized() {
|
||||
let envelope = owned_envelope(Method::DELETE);
|
||||
let request = build_request((), Method::DELETE);
|
||||
let envelope = RequestEnvelope::from_s3s(&request);
|
||||
let before = s3_op_total(S3Operation::AbortMultipartUpload);
|
||||
let target = AbortMultipartUploadTarget::Legacy {
|
||||
bucket: "bucket".to_owned(),
|
||||
|
||||
@@ -943,72 +943,21 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_complete_multipart_upload_rejects_missing_parts_payload() {
|
||||
use rustfs_gateway_types::compat::s3s_5761ddfe::generated::ops::complete_multipart_upload;
|
||||
|
||||
let input = CompleteMultipartUploadInput::builder()
|
||||
.bucket("bucket".to_string())
|
||||
.key("object".to_string())
|
||||
.upload_id("upload-id".to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
let mut s3s_input = input;
|
||||
let has_upload = s3s_input.multipart_upload.is_some();
|
||||
s3s_input.multipart_upload.get_or_insert_with(Default::default);
|
||||
let tags: Vec<Option<String>> = s3s_input
|
||||
.multipart_upload
|
||||
.iter_mut()
|
||||
.flat_map(|upload| upload.parts.iter_mut().flatten())
|
||||
.map(|part| {
|
||||
part.part_number.get_or_insert(0);
|
||||
part.e_tag.take().map(|tag| tag.value().to_owned())
|
||||
})
|
||||
.collect();
|
||||
for checksum in [
|
||||
&mut s3s_input.checksum_crc32,
|
||||
&mut s3s_input.checksum_crc32c,
|
||||
&mut s3s_input.checksum_crc64nvme,
|
||||
&mut s3s_input.checksum_md5,
|
||||
&mut s3s_input.checksum_sha1,
|
||||
&mut s3s_input.checksum_sha256,
|
||||
&mut s3s_input.checksum_sha512,
|
||||
&mut s3s_input.checksum_xxhash128,
|
||||
&mut s3s_input.checksum_xxhash3,
|
||||
&mut s3s_input.checksum_xxhash64,
|
||||
] {
|
||||
*checksum = None;
|
||||
}
|
||||
let err = match complete_multipart_upload::input_from_s3s(s3s_input) {
|
||||
Err(error) => crate::app::dto_values::seam_error(error),
|
||||
Ok(mut owned) => {
|
||||
for (part, tag) in owned.multipart_upload.parts.iter_mut().zip(tags) {
|
||||
part.e_tag = tag.map(rustfs_gateway_types::ETag::from_stored);
|
||||
}
|
||||
let taken = std::mem::take(&mut owned.multipart_upload);
|
||||
let parts = has_upload.then_some(taken);
|
||||
let result = match parts {
|
||||
Some(parts) => {
|
||||
owned.multipart_upload = parts;
|
||||
Box::pin(make_usecase().execute_complete_multipart_upload(owned_envelope(Method::POST), owned)).await
|
||||
}
|
||||
None => {
|
||||
Box::pin(make_usecase().execute_complete_multipart_upload_parts(
|
||||
owned_envelope(Method::POST),
|
||||
owned,
|
||||
None,
|
||||
))
|
||||
.await
|
||||
}
|
||||
};
|
||||
result.unwrap_err()
|
||||
}
|
||||
};
|
||||
let req = build_request(input, Method::POST);
|
||||
|
||||
let err = Box::pin(make_usecase().complete_for_s3s_request(req)).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidPart);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_complete_multipart_upload_rejects_missing_parts_list() {
|
||||
use crate::app::storage_api::s3::xml::Deserialize as _;
|
||||
use rustfs_gateway_types::compat::s3s_5761ddfe::generated::ops::complete_multipart_upload;
|
||||
|
||||
let mut deserializer = crate::app::storage_api::s3::xml::Deserializer::new(b"<CompleteMultipartUpload/>");
|
||||
let multipart_upload =
|
||||
@@ -1025,57 +974,11 @@ mod tests {
|
||||
.multipart_upload(Some(multipart_upload))
|
||||
.build()
|
||||
.expect("complete multipart input should build");
|
||||
let mut s3s_input = input;
|
||||
let has_upload = s3s_input.multipart_upload.is_some();
|
||||
s3s_input.multipart_upload.get_or_insert_with(Default::default);
|
||||
let tags: Vec<Option<String>> = s3s_input
|
||||
.multipart_upload
|
||||
.iter_mut()
|
||||
.flat_map(|upload| upload.parts.iter_mut().flatten())
|
||||
.map(|part| {
|
||||
part.part_number.get_or_insert(0);
|
||||
part.e_tag.take().map(|tag| tag.value().to_owned())
|
||||
})
|
||||
.collect();
|
||||
for checksum in [
|
||||
&mut s3s_input.checksum_crc32,
|
||||
&mut s3s_input.checksum_crc32c,
|
||||
&mut s3s_input.checksum_crc64nvme,
|
||||
&mut s3s_input.checksum_md5,
|
||||
&mut s3s_input.checksum_sha1,
|
||||
&mut s3s_input.checksum_sha256,
|
||||
&mut s3s_input.checksum_sha512,
|
||||
&mut s3s_input.checksum_xxhash128,
|
||||
&mut s3s_input.checksum_xxhash3,
|
||||
&mut s3s_input.checksum_xxhash64,
|
||||
] {
|
||||
*checksum = None;
|
||||
}
|
||||
let err = match complete_multipart_upload::input_from_s3s(s3s_input) {
|
||||
Err(error) => crate::app::dto_values::seam_error(error),
|
||||
Ok(mut owned) => {
|
||||
for (part, tag) in owned.multipart_upload.parts.iter_mut().zip(tags) {
|
||||
part.e_tag = tag.map(rustfs_gateway_types::ETag::from_stored);
|
||||
}
|
||||
let taken = std::mem::take(&mut owned.multipart_upload);
|
||||
let parts = has_upload.then_some(taken);
|
||||
let result = match parts {
|
||||
Some(parts) => {
|
||||
owned.multipart_upload = parts;
|
||||
Box::pin(make_usecase().execute_complete_multipart_upload(owned_envelope(Method::POST), owned)).await
|
||||
}
|
||||
None => {
|
||||
Box::pin(make_usecase().execute_complete_multipart_upload_parts(
|
||||
owned_envelope(Method::POST),
|
||||
owned,
|
||||
None,
|
||||
))
|
||||
.await
|
||||
}
|
||||
};
|
||||
result.expect_err("missing parts list must be rejected")
|
||||
}
|
||||
};
|
||||
let req = build_request(input, Method::POST);
|
||||
|
||||
let err = Box::pin(make_usecase().complete_for_s3s_request(req))
|
||||
.await
|
||||
.expect_err("missing parts list must be rejected");
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
|
||||
}
|
||||
|
||||
@@ -1221,9 +1124,10 @@ mod tests {
|
||||
crate::app::gating_test_env::run_large_stack_test("ssec-completion-checksums", || async {
|
||||
let (_store, bucket) =
|
||||
crate::app::gating_test_env::durable_quota_test_bucket("ssec-completion-checksums", 16_384).await;
|
||||
let versioning_req = build_request((), Method::PUT);
|
||||
crate::app::bucket_usecase::DefaultBucketUsecase::from_global()
|
||||
.execute_put_bucket_versioning(
|
||||
super::super::test_support::owned_envelope(Method::PUT),
|
||||
RequestEnvelope::from_s3s(&versioning_req),
|
||||
&bucket,
|
||||
Ok(rustfs_gateway_types::persistence::PersistedVersioningConfiguration {
|
||||
status: Some("Enabled".to_owned()),
|
||||
@@ -1400,13 +1304,16 @@ mod tests {
|
||||
|
||||
let bucket_usecase = crate::app::bucket_usecase::DefaultBucketUsecase::from_global();
|
||||
// Listings carry neither the SSE-C key nor a per-object decryption request.
|
||||
let versions_input = ListObjectVersionsInput {
|
||||
bucket: bucket.to_string(),
|
||||
prefix: Some(key.to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
let versions_env = super::super::test_support::owned_envelope(Method::GET);
|
||||
let versions_in = list_object_versions::input_from_s3s(versions_input).expect("convert ListObjectVersionsInput");
|
||||
let versions_req = build_request(
|
||||
ListObjectVersionsInput {
|
||||
bucket: bucket.to_string(),
|
||||
prefix: Some(key.to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
Method::GET,
|
||||
);
|
||||
let versions_env = RequestEnvelope::from_s3s(&versions_req);
|
||||
let versions_in = list_object_versions::input_from_s3s(versions_req.input).expect("convert ListObjectVersionsInput");
|
||||
let versions = list_object_versions::output_to_s3s(
|
||||
bucket_usecase
|
||||
.execute_list_object_versions(versions_env, versions_in)
|
||||
@@ -1421,13 +1328,16 @@ mod tests {
|
||||
assert_eq!(entries[0].e_tag.as_ref(), Some(etag));
|
||||
assert_xml_etag(&versions, etag);
|
||||
|
||||
let v1_input = ListObjectsInput {
|
||||
bucket: bucket.to_string(),
|
||||
prefix: Some(key.to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
let v1_env = super::super::test_support::owned_envelope(Method::GET);
|
||||
let v1_in = list_objects::input_from_s3s(v1_input).expect("convert ListObjectsInput");
|
||||
let v1_req = build_request(
|
||||
ListObjectsInput {
|
||||
bucket: bucket.to_string(),
|
||||
prefix: Some(key.to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
Method::GET,
|
||||
);
|
||||
let v1_env = RequestEnvelope::from_s3s(&v1_req);
|
||||
let v1_in = list_objects::input_from_s3s(v1_req.input).expect("convert ListObjectsInput");
|
||||
let v1 = list_objects::output_to_s3s(
|
||||
bucket_usecase
|
||||
.execute_list_objects(v1_env, v1_in)
|
||||
@@ -1501,6 +1411,7 @@ mod tests {
|
||||
.await
|
||||
.expect("create versioned Object Lock bucket");
|
||||
let key = "kms-encrypted";
|
||||
let enc_req = build_request((), Method::PUT);
|
||||
let enc_cfg = crate::app::storage_api::s3::dto::ServerSideEncryptionConfiguration {
|
||||
rules: vec![crate::app::storage_api::s3::dto::ServerSideEncryptionRule {
|
||||
apply_server_side_encryption_by_default: Some(
|
||||
@@ -1514,7 +1425,7 @@ mod tests {
|
||||
};
|
||||
let prepared = crate::storage::persisted_from_s3s(&enc_cfg).map_err(Into::into);
|
||||
crate::app::bucket_usecase::DefaultBucketUsecase::from_global()
|
||||
.execute_put_bucket_encryption(super::super::test_support::owned_envelope(Method::PUT), &bucket, prepared)
|
||||
.execute_put_bucket_encryption(RequestEnvelope::from_s3s(&enc_req), &bucket, prepared)
|
||||
.await
|
||||
.expect("configure bucket-default SSE-KMS");
|
||||
let usecase = DefaultMultipartUsecase::from_global();
|
||||
@@ -2240,8 +2151,6 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_complete_multipart_upload_allows_duplicate_part_numbers_by_using_last_occurrence() {
|
||||
use rustfs_gateway_types::compat::s3s_5761ddfe::generated::ops::complete_multipart_upload;
|
||||
|
||||
let multipart_upload = CompletedMultipartUpload {
|
||||
parts: Some(vec![
|
||||
CompletedPart {
|
||||
@@ -2261,67 +2170,14 @@ mod tests {
|
||||
.multipart_upload(Some(multipart_upload))
|
||||
.build()
|
||||
.unwrap();
|
||||
let mut s3s_input = input;
|
||||
let has_upload = s3s_input.multipart_upload.is_some();
|
||||
s3s_input.multipart_upload.get_or_insert_with(Default::default);
|
||||
let tags: Vec<Option<String>> = s3s_input
|
||||
.multipart_upload
|
||||
.iter_mut()
|
||||
.flat_map(|upload| upload.parts.iter_mut().flatten())
|
||||
.map(|part| {
|
||||
part.part_number.get_or_insert(0);
|
||||
part.e_tag.take().map(|tag| tag.value().to_owned())
|
||||
})
|
||||
.collect();
|
||||
for checksum in [
|
||||
&mut s3s_input.checksum_crc32,
|
||||
&mut s3s_input.checksum_crc32c,
|
||||
&mut s3s_input.checksum_crc64nvme,
|
||||
&mut s3s_input.checksum_md5,
|
||||
&mut s3s_input.checksum_sha1,
|
||||
&mut s3s_input.checksum_sha256,
|
||||
&mut s3s_input.checksum_sha512,
|
||||
&mut s3s_input.checksum_xxhash128,
|
||||
&mut s3s_input.checksum_xxhash3,
|
||||
&mut s3s_input.checksum_xxhash64,
|
||||
] {
|
||||
*checksum = None;
|
||||
}
|
||||
let err = match complete_multipart_upload::input_from_s3s(s3s_input) {
|
||||
Err(error) => crate::app::dto_values::seam_error(error),
|
||||
Ok(mut owned) => {
|
||||
for (part, tag) in owned.multipart_upload.parts.iter_mut().zip(tags) {
|
||||
part.e_tag = tag.map(rustfs_gateway_types::ETag::from_stored);
|
||||
}
|
||||
let taken = std::mem::take(&mut owned.multipart_upload);
|
||||
let parts = has_upload.then_some(taken);
|
||||
let result = match parts {
|
||||
Some(parts) => {
|
||||
owned.multipart_upload = parts;
|
||||
Box::pin(make_usecase().execute_complete_multipart_upload(owned_envelope(Method::POST), owned)).await
|
||||
}
|
||||
None => {
|
||||
Box::pin(make_usecase().execute_complete_multipart_upload_parts(
|
||||
owned_envelope(Method::POST),
|
||||
owned,
|
||||
None,
|
||||
))
|
||||
.await
|
||||
}
|
||||
};
|
||||
match result {
|
||||
Err(error) => error,
|
||||
Ok(_output) => panic!("duplicate part numbers must not succeed before the order check"),
|
||||
}
|
||||
}
|
||||
};
|
||||
let req = build_request(input, Method::POST);
|
||||
|
||||
let err = Box::pin(make_usecase().complete_for_s3s_request(req)).await.unwrap_err();
|
||||
assert_ne!(err.code(), &S3ErrorCode::InvalidPartOrder);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_complete_multipart_upload_rejects_out_of_order_parts() {
|
||||
use rustfs_gateway_types::compat::s3s_5761ddfe::generated::ops::complete_multipart_upload;
|
||||
|
||||
let multipart_upload = CompletedMultipartUpload {
|
||||
parts: Some(vec![
|
||||
CompletedPart {
|
||||
@@ -2341,60 +2197,9 @@ mod tests {
|
||||
.multipart_upload(Some(multipart_upload))
|
||||
.build()
|
||||
.unwrap();
|
||||
let mut s3s_input = input;
|
||||
let has_upload = s3s_input.multipart_upload.is_some();
|
||||
s3s_input.multipart_upload.get_or_insert_with(Default::default);
|
||||
let tags: Vec<Option<String>> = s3s_input
|
||||
.multipart_upload
|
||||
.iter_mut()
|
||||
.flat_map(|upload| upload.parts.iter_mut().flatten())
|
||||
.map(|part| {
|
||||
part.part_number.get_or_insert(0);
|
||||
part.e_tag.take().map(|tag| tag.value().to_owned())
|
||||
})
|
||||
.collect();
|
||||
for checksum in [
|
||||
&mut s3s_input.checksum_crc32,
|
||||
&mut s3s_input.checksum_crc32c,
|
||||
&mut s3s_input.checksum_crc64nvme,
|
||||
&mut s3s_input.checksum_md5,
|
||||
&mut s3s_input.checksum_sha1,
|
||||
&mut s3s_input.checksum_sha256,
|
||||
&mut s3s_input.checksum_sha512,
|
||||
&mut s3s_input.checksum_xxhash128,
|
||||
&mut s3s_input.checksum_xxhash3,
|
||||
&mut s3s_input.checksum_xxhash64,
|
||||
] {
|
||||
*checksum = None;
|
||||
}
|
||||
let err = match complete_multipart_upload::input_from_s3s(s3s_input) {
|
||||
Err(error) => crate::app::dto_values::seam_error(error),
|
||||
Ok(mut owned) => {
|
||||
for (part, tag) in owned.multipart_upload.parts.iter_mut().zip(tags) {
|
||||
part.e_tag = tag.map(rustfs_gateway_types::ETag::from_stored);
|
||||
}
|
||||
let taken = std::mem::take(&mut owned.multipart_upload);
|
||||
let parts = has_upload.then_some(taken);
|
||||
let result = match parts {
|
||||
Some(parts) => {
|
||||
owned.multipart_upload = parts;
|
||||
Box::pin(make_usecase().execute_complete_multipart_upload(owned_envelope(Method::POST), owned)).await
|
||||
}
|
||||
None => {
|
||||
Box::pin(make_usecase().execute_complete_multipart_upload_parts(
|
||||
owned_envelope(Method::POST),
|
||||
owned,
|
||||
None,
|
||||
))
|
||||
.await
|
||||
}
|
||||
};
|
||||
match result {
|
||||
Err(error) => error,
|
||||
Ok(_output) => panic!("out-of-order parts must be rejected"),
|
||||
}
|
||||
}
|
||||
};
|
||||
let req = build_request(input, Method::POST);
|
||||
|
||||
let err = Box::pin(make_usecase().complete_for_s3s_request(req)).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidPartOrder);
|
||||
}
|
||||
|
||||
|
||||
@@ -545,17 +545,10 @@ mod tests {
|
||||
.object_lock_retain_until_date(date.map(Timestamp::from))
|
||||
.build()
|
||||
.expect("direct fixture DTO");
|
||||
let error = match make_usecase()
|
||||
.execute_create_multipart_upload(
|
||||
owned_envelope(Method::POST),
|
||||
CreateMultipartUploadFields::from_legacy(input),
|
||||
None,
|
||||
)
|
||||
let error = make_usecase()
|
||||
.execute_create_multipart_upload_from_s3s(build_request(input, Method::POST))
|
||||
.await
|
||||
{
|
||||
Err(error) => crate::app::storage_api::s3::S3Error::from(error),
|
||||
Ok(_) => panic!("direct fixture does not create an upload"),
|
||||
};
|
||||
.expect_err("direct fixture does not create an upload");
|
||||
assert_eq!(
|
||||
(error.code().as_str(), error.message()),
|
||||
(expected.0, Some(expected.1)),
|
||||
@@ -597,13 +590,12 @@ mod tests {
|
||||
.storage_class(Some(StorageClass::from("invalid".to_string())))
|
||||
.build()
|
||||
.unwrap();
|
||||
let err = match make_usecase()
|
||||
.execute_create_multipart_upload(owned_envelope(Method::POST), CreateMultipartUploadFields::from_legacy(input), None)
|
||||
let req = build_request(input, Method::POST);
|
||||
|
||||
let err = make_usecase()
|
||||
.execute_create_multipart_upload_from_s3s(req)
|
||||
.await
|
||||
{
|
||||
Err(error) => crate::app::storage_api::s3::S3Error::from(error),
|
||||
Ok(_) => panic!("invalid storage class must be rejected"),
|
||||
};
|
||||
.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidStorageClass);
|
||||
}
|
||||
|
||||
|
||||
@@ -295,7 +295,7 @@ mod tests {
|
||||
|
||||
async fn list_uploads(input: gateway_dto::ListMultipartUploadsInput) -> rustfs_s3_types::S3Error {
|
||||
make_usecase()
|
||||
.execute_list_multipart_uploads(owned_envelope(Method::GET), input)
|
||||
.execute_list_multipart_uploads(RequestEnvelope::from_s3s(&build_request((), Method::GET)), input)
|
||||
.await
|
||||
.expect_err("refused")
|
||||
}
|
||||
@@ -393,7 +393,7 @@ mod tests {
|
||||
max_parts,
|
||||
};
|
||||
make_usecase()
|
||||
.execute_list_parts(owned_envelope(Method::GET), fields)
|
||||
.execute_list_parts(RequestEnvelope::from_s3s(&build_request((), Method::GET)), fields)
|
||||
.await
|
||||
.expect_err("refused")
|
||||
}
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
//! ListParts execution against real multipart metadata and the native claim exchange.
|
||||
|
||||
use super::super::test_support::{Method, build_request, owned_envelope};
|
||||
use super::super::test_support::{Method, build_request};
|
||||
use super::*;
|
||||
use crate::app::gating_test_env::{run_large_stack_test, shared_gating_ambient, shared_gating_ecstore_and_disk_paths};
|
||||
use crate::app::storage_api::multipart_usecase::access::{AuthorizationRequest, load_bucket_generation_from_store};
|
||||
@@ -81,7 +81,7 @@ fn list_parts_resolves_real_owners_and_retains_storage_errors() {
|
||||
let key = "fixture/one";
|
||||
let foreign_key = "fixture/two";
|
||||
let opts = ObjectOptions::default();
|
||||
let envelope = || owned_envelope(Method::GET);
|
||||
let envelope = || RequestEnvelope::from_s3s(&build_request((), Method::GET));
|
||||
for name in [&bucket, &foreign_bucket] {
|
||||
store
|
||||
.make_bucket(name, &MakeBucketOptions::default())
|
||||
|
||||
@@ -18,7 +18,6 @@ use super::*;
|
||||
use crate::app::storage_api::s3::S3Request;
|
||||
pub(super) use crate::app::storage_api::s3::dto::CompletedMultipartUpload;
|
||||
pub(super) use crate::app::storage_api::s3::dto::StorageClass;
|
||||
use crate::app::{RequestEnvelope, RequestHead};
|
||||
pub(super) use http::Extensions;
|
||||
pub(super) use http::Method;
|
||||
pub(super) use rustfs_filemeta::ObjectPartInfo;
|
||||
@@ -51,21 +50,6 @@ pub(super) fn build_request<T>(input: T, method: Method) -> S3Request<T> {
|
||||
}
|
||||
}
|
||||
|
||||
/// The envelope of a request that carries nothing but `method`.
|
||||
pub(super) fn owned_envelope(method: Method) -> RequestEnvelope {
|
||||
RequestEnvelope::from_parts(
|
||||
RequestHead {
|
||||
method,
|
||||
uri: Uri::from_static("/"),
|
||||
headers: HeaderMap::new(),
|
||||
credentials: None,
|
||||
region: None,
|
||||
service: None,
|
||||
},
|
||||
Extensions::new(),
|
||||
)
|
||||
}
|
||||
|
||||
pub(super) fn make_usecase() -> DefaultMultipartUsecase {
|
||||
DefaultMultipartUsecase::without_context()
|
||||
}
|
||||
|
||||
@@ -382,8 +382,7 @@ impl DefaultMultipartUsecase {
|
||||
content_size: actual_size,
|
||||
principal: None,
|
||||
}
|
||||
.validate_multipart_ssec(&fi.user_defined)
|
||||
.map_err(ApiError::from)?;
|
||||
.validate_multipart_ssec(&fi.user_defined)?;
|
||||
}
|
||||
let (requested_sse, requested_kms_key_id) = if has_ssec {
|
||||
let ssec_material = sse_decryption(DecryptionRequest {
|
||||
@@ -742,13 +741,9 @@ mod tests {
|
||||
.part_number(1)
|
||||
.build()
|
||||
.unwrap();
|
||||
let err = match make_usecase()
|
||||
.execute_upload_part_with_audit(owned_envelope(Method::PUT), UploadPartWriteInput::from_legacy(input, None))
|
||||
.await
|
||||
{
|
||||
Err(error) => crate::app::storage_api::s3::S3Error::from(error),
|
||||
Ok(_) => panic!("missing body must be rejected"),
|
||||
};
|
||||
let req = build_request(input, Method::PUT);
|
||||
|
||||
let err = make_usecase().execute_upload_part(req).await.unwrap_err();
|
||||
assert_eq!(err.code(), &LegacyErrorCode::IncompleteBody);
|
||||
}
|
||||
|
||||
@@ -770,13 +765,9 @@ mod tests {
|
||||
.content_length(Some(declared))
|
||||
.build()
|
||||
.unwrap();
|
||||
let err = match make_usecase()
|
||||
.execute_upload_part_with_audit(owned_envelope(Method::PUT), UploadPartWriteInput::from_legacy(input, None))
|
||||
.await
|
||||
{
|
||||
Err(error) => crate::app::storage_api::s3::S3Error::from(error),
|
||||
Ok(_) => panic!("declared part must be rejected"),
|
||||
};
|
||||
let req = build_request(input, Method::PUT);
|
||||
|
||||
let err = make_usecase().execute_upload_part(req).await.unwrap_err();
|
||||
if expect_too_large {
|
||||
assert_eq!(err.code(), &LegacyErrorCode::EntityTooLarge, "declared {declared}");
|
||||
assert_eq!(polls.load(std::sync::atomic::Ordering::SeqCst), 0, "body must not be polled");
|
||||
@@ -796,13 +787,9 @@ mod tests {
|
||||
.part_number(part_number)
|
||||
.build()
|
||||
.unwrap();
|
||||
let err = match make_usecase()
|
||||
.execute_upload_part_with_audit(owned_envelope(Method::PUT), UploadPartWriteInput::from_legacy(input, None))
|
||||
.await
|
||||
{
|
||||
Err(error) => crate::app::storage_api::s3::S3Error::from(error),
|
||||
Ok(_) => panic!("invalid part number must be rejected"),
|
||||
};
|
||||
let req = build_request(input, Method::PUT);
|
||||
|
||||
let err = make_usecase().execute_upload_part(req).await.unwrap_err();
|
||||
assert_eq!(err.code(), &LegacyErrorCode::InvalidArgument);
|
||||
assert_eq!(err.message(), Some("partNumber must be between 1 and 10000"));
|
||||
}
|
||||
@@ -984,13 +971,10 @@ mod tests {
|
||||
.build()
|
||||
.expect("upload part input should build");
|
||||
|
||||
let err = match usecase
|
||||
.execute_upload_part_with_audit(owned_envelope(Method::PUT), UploadPartWriteInput::from_legacy(input, None))
|
||||
let err = usecase
|
||||
.execute_upload_part(build_request(input, Method::PUT))
|
||||
.await
|
||||
{
|
||||
Err(error) => crate::app::storage_api::s3::S3Error::from(error),
|
||||
Ok(_) => panic!("full foreground write admission should reject before body ingest"),
|
||||
};
|
||||
.expect_err("full foreground write admission should reject before body ingest");
|
||||
assert_eq!(err.code(), &LegacyErrorCode::SlowDown);
|
||||
drop(held);
|
||||
}
|
||||
|
||||
@@ -14,7 +14,6 @@
|
||||
|
||||
//! Upload target ownership and refusal order against real multipart state.
|
||||
|
||||
use super::super::super::test_support::owned_envelope;
|
||||
use super::*;
|
||||
use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions};
|
||||
use rustfs_gateway_types::{BucketName, ObjectKey, UploadIdClaim};
|
||||
@@ -102,7 +101,7 @@ async fn native_claim_checks_order_ownership_and_one_active_lookup() {
|
||||
Body::from(Bytes::from_static(b"abc"))
|
||||
}
|
||||
});
|
||||
let envelope = owned_envelope(Method::PUT);
|
||||
let envelope = RequestEnvelope::from_s3s(&build_request((), Method::PUT));
|
||||
LOOKUP
|
||||
.scope((Cell::new(0), mode), async {
|
||||
let result = usecase.execute_upload_part_with_audit(envelope, input).await;
|
||||
@@ -212,7 +211,7 @@ fn native_copy_part_rejects_unproved_owner_without_changing_parts() {
|
||||
copy_source_sse_customer_key: None,
|
||||
copy_source_sse_customer_key_md5: None,
|
||||
};
|
||||
let envelope = owned_envelope(Method::PUT);
|
||||
let envelope = RequestEnvelope::from_s3s(&build_request((), Method::PUT));
|
||||
LOOKUP
|
||||
.scope((Cell::new(0), mode), async {
|
||||
let result = Box::pin(usecase.execute_upload_part_copy_with_audit(envelope, input)).await;
|
||||
|
||||
@@ -313,8 +313,7 @@ impl DefaultMultipartUsecase {
|
||||
content_size: 0,
|
||||
principal: None,
|
||||
}
|
||||
.validate_multipart_ssec(&mp_info.user_defined)
|
||||
.map_err(ApiError::from)?;
|
||||
.validate_multipart_ssec(&mp_info.user_defined)?;
|
||||
|
||||
let src_versioned = BucketVersioningSys::prefix_enabled(&src_bucket, &src_key).await;
|
||||
let src_version_suspended = BucketVersioningSys::prefix_suspended(&src_bucket, &src_key).await;
|
||||
@@ -562,9 +561,6 @@ impl DefaultMultipartUsecase {
|
||||
mod tests {
|
||||
use super::super::test_support::*;
|
||||
use super::*;
|
||||
use crate::app::multipart_usecase::UploadPartTarget;
|
||||
use crate::app::object_usecase::CopySourceCondition;
|
||||
use crate::app::storage_api::s3::dto::ETag;
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_upload_part_copy_returns_internal_error_when_store_uninitialized() {
|
||||
@@ -580,37 +576,9 @@ mod tests {
|
||||
.upload_id("upload-id".to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
let condition = |value| match value {
|
||||
crate::app::storage_api::s3::dto::ETagCondition::Any => CopySourceCondition::Any,
|
||||
crate::app::storage_api::s3::dto::ETagCondition::ETag(ETag::Strong(value)) => CopySourceCondition::Strong(value),
|
||||
crate::app::storage_api::s3::dto::ETagCondition::ETag(ETag::Weak(_)) => CopySourceCondition::Weak,
|
||||
};
|
||||
let err = match Box::pin(make_usecase().execute_upload_part_copy_with_audit(
|
||||
owned_envelope(Method::PUT),
|
||||
UploadPartCopyWriteInput {
|
||||
target: UploadPartTarget::Legacy {
|
||||
bucket: input.bucket,
|
||||
key: input.key,
|
||||
upload_id: input.upload_id,
|
||||
},
|
||||
part_number: input.part_number,
|
||||
copy_source: rustfs_gateway_types::compat::s3s_5761ddfe::copy_source::from_s3s(input.copy_source),
|
||||
copy_source_range: input.copy_source_range,
|
||||
copy_source_if_match: input.copy_source_if_match.map(&condition),
|
||||
copy_source_if_none_match: input.copy_source_if_none_match.map(&condition),
|
||||
sse_customer_algorithm: input.sse_customer_algorithm,
|
||||
sse_customer_key: input.sse_customer_key,
|
||||
sse_customer_key_md5: input.sse_customer_key_md5,
|
||||
copy_source_sse_customer_algorithm: input.copy_source_sse_customer_algorithm,
|
||||
copy_source_sse_customer_key: input.copy_source_sse_customer_key,
|
||||
copy_source_sse_customer_key_md5: input.copy_source_sse_customer_key_md5,
|
||||
},
|
||||
))
|
||||
.await
|
||||
{
|
||||
Err(error) => crate::app::storage_api::s3::S3Error::from(error),
|
||||
Ok(_) => panic!("uninitialized store must be an internal error"),
|
||||
};
|
||||
let req = build_request(input, Method::PUT);
|
||||
|
||||
let err = Box::pin(make_usecase().execute_upload_part_copy(req)).await.unwrap_err();
|
||||
assert_eq!(err.code(), &crate::app::storage_api::s3::S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
@@ -629,37 +597,9 @@ mod tests {
|
||||
.upload_id("upload-id".to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
let condition = |value| match value {
|
||||
crate::app::storage_api::s3::dto::ETagCondition::Any => CopySourceCondition::Any,
|
||||
crate::app::storage_api::s3::dto::ETagCondition::ETag(ETag::Strong(value)) => CopySourceCondition::Strong(value),
|
||||
crate::app::storage_api::s3::dto::ETagCondition::ETag(ETag::Weak(_)) => CopySourceCondition::Weak,
|
||||
};
|
||||
let err = match Box::pin(make_usecase().execute_upload_part_copy_with_audit(
|
||||
owned_envelope(Method::PUT),
|
||||
UploadPartCopyWriteInput {
|
||||
target: UploadPartTarget::Legacy {
|
||||
bucket: input.bucket,
|
||||
key: input.key,
|
||||
upload_id: input.upload_id,
|
||||
},
|
||||
part_number: input.part_number,
|
||||
copy_source: rustfs_gateway_types::compat::s3s_5761ddfe::copy_source::from_s3s(input.copy_source),
|
||||
copy_source_range: input.copy_source_range,
|
||||
copy_source_if_match: input.copy_source_if_match.map(&condition),
|
||||
copy_source_if_none_match: input.copy_source_if_none_match.map(&condition),
|
||||
sse_customer_algorithm: input.sse_customer_algorithm,
|
||||
sse_customer_key: input.sse_customer_key,
|
||||
sse_customer_key_md5: input.sse_customer_key_md5,
|
||||
copy_source_sse_customer_algorithm: input.copy_source_sse_customer_algorithm,
|
||||
copy_source_sse_customer_key: input.copy_source_sse_customer_key,
|
||||
copy_source_sse_customer_key_md5: input.copy_source_sse_customer_key_md5,
|
||||
},
|
||||
))
|
||||
.await
|
||||
{
|
||||
Err(error) => crate::app::storage_api::s3::S3Error::from(error),
|
||||
Ok(_) => panic!("invalid part number must be rejected"),
|
||||
};
|
||||
let req = build_request(input, Method::PUT);
|
||||
|
||||
let err = Box::pin(make_usecase().execute_upload_part_copy(req)).await.unwrap_err();
|
||||
assert_eq!(err.code(), &crate::app::storage_api::s3::S3ErrorCode::InvalidArgument);
|
||||
assert_eq!(err.message(), Some("partNumber must be between 1 and 10000"));
|
||||
}
|
||||
|
||||
@@ -76,38 +76,9 @@ fn upload_part_copy_shares_foreground_admission_with_upload_part() {
|
||||
.any(|label| label.key() == "size_bucket" && label.value() == source_size_bucket)
|
||||
})
|
||||
};
|
||||
let saturated = input.clone();
|
||||
let condition = |value| match value {
|
||||
crate::app::storage_api::s3::dto::ETagCondition::Any => CopySourceCondition::Any,
|
||||
crate::app::storage_api::s3::dto::ETagCondition::ETag(ETag::Strong(value)) => CopySourceCondition::Strong(value),
|
||||
crate::app::storage_api::s3::dto::ETagCondition::ETag(ETag::Weak(_)) => CopySourceCondition::Weak,
|
||||
};
|
||||
let error = match Box::pin(usecase.execute_upload_part_copy_with_audit(
|
||||
owned_envelope(Method::PUT),
|
||||
UploadPartCopyWriteInput {
|
||||
target: UploadPartTarget::Legacy {
|
||||
bucket: saturated.bucket,
|
||||
key: saturated.key,
|
||||
upload_id: saturated.upload_id,
|
||||
},
|
||||
part_number: saturated.part_number,
|
||||
copy_source: rustfs_gateway_types::compat::s3s_5761ddfe::copy_source::from_s3s(saturated.copy_source),
|
||||
copy_source_range: saturated.copy_source_range,
|
||||
copy_source_if_match: saturated.copy_source_if_match.map(&condition),
|
||||
copy_source_if_none_match: saturated.copy_source_if_none_match.map(&condition),
|
||||
sse_customer_algorithm: saturated.sse_customer_algorithm,
|
||||
sse_customer_key: saturated.sse_customer_key,
|
||||
sse_customer_key_md5: saturated.sse_customer_key_md5,
|
||||
copy_source_sse_customer_algorithm: saturated.copy_source_sse_customer_algorithm,
|
||||
copy_source_sse_customer_key: saturated.copy_source_sse_customer_key,
|
||||
copy_source_sse_customer_key_md5: saturated.copy_source_sse_customer_key_md5,
|
||||
},
|
||||
))
|
||||
.await
|
||||
{
|
||||
Err(error) => crate::app::storage_api::s3::S3Error::from(error),
|
||||
Ok(_) => panic!("copy must not bypass the occupied UploadPart pool"),
|
||||
};
|
||||
let error = Box::pin(usecase.execute_upload_part_copy(build_request(input.clone(), Method::PUT)))
|
||||
.await
|
||||
.expect_err("copy must not bypass the occupied UploadPart pool");
|
||||
assert_eq!(error.code(), &S3ErrorCode::SlowDown);
|
||||
assert!(!source_reader_observed(), "saturated copy must not construct a source body reader");
|
||||
let parts = store
|
||||
@@ -194,37 +165,9 @@ fn upload_part_copy_shares_foreground_admission_with_upload_part() {
|
||||
.upload_id(deleted_upload.upload_id.clone())
|
||||
.build()
|
||||
.expect("deleted-source copy request");
|
||||
let condition = |value| match value {
|
||||
crate::app::storage_api::s3::dto::ETagCondition::Any => CopySourceCondition::Any,
|
||||
crate::app::storage_api::s3::dto::ETagCondition::ETag(ETag::Strong(value)) => CopySourceCondition::Strong(value),
|
||||
crate::app::storage_api::s3::dto::ETagCondition::ETag(ETag::Weak(_)) => CopySourceCondition::Weak,
|
||||
};
|
||||
let error = match Box::pin(usecase.execute_upload_part_copy_with_audit(
|
||||
owned_envelope(Method::PUT),
|
||||
UploadPartCopyWriteInput {
|
||||
target: UploadPartTarget::Legacy {
|
||||
bucket: input.bucket,
|
||||
key: input.key,
|
||||
upload_id: input.upload_id,
|
||||
},
|
||||
part_number: input.part_number,
|
||||
copy_source: rustfs_gateway_types::compat::s3s_5761ddfe::copy_source::from_s3s(input.copy_source),
|
||||
copy_source_range: input.copy_source_range,
|
||||
copy_source_if_match: input.copy_source_if_match.map(&condition),
|
||||
copy_source_if_none_match: input.copy_source_if_none_match.map(&condition),
|
||||
sse_customer_algorithm: input.sse_customer_algorithm,
|
||||
sse_customer_key: input.sse_customer_key,
|
||||
sse_customer_key_md5: input.sse_customer_key_md5,
|
||||
copy_source_sse_customer_algorithm: input.copy_source_sse_customer_algorithm,
|
||||
copy_source_sse_customer_key: input.copy_source_sse_customer_key,
|
||||
copy_source_sse_customer_key_md5: input.copy_source_sse_customer_key_md5,
|
||||
},
|
||||
))
|
||||
.await
|
||||
{
|
||||
Err(error) => crate::app::storage_api::s3::S3Error::from(error),
|
||||
Ok(_) => panic!("deleted source must fail before conditions or admission"),
|
||||
};
|
||||
let error = Box::pin(usecase.execute_upload_part_copy(build_request(input, Method::PUT)))
|
||||
.await
|
||||
.expect_err("deleted source must fail before conditions or admission");
|
||||
assert_eq!(error.code(), &expected);
|
||||
}
|
||||
});
|
||||
@@ -268,32 +211,7 @@ fn upload_part_copy_range_error_and_cancellation_release_foreground_permit() {
|
||||
.upload_id(cancelled_upload.upload_id.clone())
|
||||
.build()
|
||||
.expect("cancelled copy request");
|
||||
let condition = |value| match value {
|
||||
crate::app::storage_api::s3::dto::ETagCondition::Any => CopySourceCondition::Any,
|
||||
crate::app::storage_api::s3::dto::ETagCondition::ETag(ETag::Strong(value)) => CopySourceCondition::Strong(value),
|
||||
crate::app::storage_api::s3::dto::ETagCondition::ETag(ETag::Weak(_)) => CopySourceCondition::Weak,
|
||||
};
|
||||
let mut cancelled_copy = Box::pin(usecase.execute_upload_part_copy_with_audit(
|
||||
owned_envelope(Method::PUT),
|
||||
UploadPartCopyWriteInput {
|
||||
target: UploadPartTarget::Legacy {
|
||||
bucket: cancelled_input.bucket,
|
||||
key: cancelled_input.key,
|
||||
upload_id: cancelled_input.upload_id,
|
||||
},
|
||||
part_number: cancelled_input.part_number,
|
||||
copy_source: rustfs_gateway_types::compat::s3s_5761ddfe::copy_source::from_s3s(cancelled_input.copy_source),
|
||||
copy_source_range: cancelled_input.copy_source_range,
|
||||
copy_source_if_match: cancelled_input.copy_source_if_match.map(&condition),
|
||||
copy_source_if_none_match: cancelled_input.copy_source_if_none_match.map(&condition),
|
||||
sse_customer_algorithm: cancelled_input.sse_customer_algorithm,
|
||||
sse_customer_key: cancelled_input.sse_customer_key,
|
||||
sse_customer_key_md5: cancelled_input.sse_customer_key_md5,
|
||||
copy_source_sse_customer_algorithm: cancelled_input.copy_source_sse_customer_algorithm,
|
||||
copy_source_sse_customer_key: cancelled_input.copy_source_sse_customer_key,
|
||||
copy_source_sse_customer_key_md5: cancelled_input.copy_source_sse_customer_key_md5,
|
||||
},
|
||||
));
|
||||
let mut cancelled_copy = Box::pin(usecase.execute_upload_part_copy(build_request(cancelled_input, Method::PUT)));
|
||||
tokio::time::timeout(Duration::from_secs(10), async {
|
||||
tokio::select! {
|
||||
biased;
|
||||
@@ -302,10 +220,7 @@ fn upload_part_copy_range_error_and_cancellation_release_foreground_permit() {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
} => {}
|
||||
result = &mut cancelled_copy => match result {
|
||||
Ok(_) => panic!("copy completed before cancellation"),
|
||||
Err(error) => panic!("copy completed before cancellation: {error:?}"),
|
||||
},
|
||||
result = &mut cancelled_copy => panic!("copy completed before cancellation: {result:?}"),
|
||||
}
|
||||
})
|
||||
.await
|
||||
@@ -345,39 +260,17 @@ fn upload_part_copy_range_error_and_cancellation_release_foreground_permit() {
|
||||
.upload_id(upload.upload_id.clone())
|
||||
.build()
|
||||
.expect("range request");
|
||||
let response = Box::pin(usecase.execute_upload_part_copy_with_audit(
|
||||
owned_envelope(Method::PUT),
|
||||
UploadPartCopyWriteInput {
|
||||
target: UploadPartTarget::Legacy {
|
||||
bucket: input.bucket,
|
||||
key: input.key,
|
||||
upload_id: input.upload_id,
|
||||
},
|
||||
part_number: input.part_number,
|
||||
copy_source: rustfs_gateway_types::compat::s3s_5761ddfe::copy_source::from_s3s(input.copy_source),
|
||||
copy_source_range: input.copy_source_range,
|
||||
copy_source_if_match: input.copy_source_if_match.map(&condition),
|
||||
copy_source_if_none_match: input.copy_source_if_none_match.map(&condition),
|
||||
sse_customer_algorithm: input.sse_customer_algorithm,
|
||||
sse_customer_key: input.sse_customer_key,
|
||||
sse_customer_key_md5: input.sse_customer_key_md5,
|
||||
copy_source_sse_customer_algorithm: input.copy_source_sse_customer_algorithm,
|
||||
copy_source_sse_customer_key: input.copy_source_sse_customer_key,
|
||||
copy_source_sse_customer_key_md5: input.copy_source_sse_customer_key_md5,
|
||||
},
|
||||
))
|
||||
.await;
|
||||
let response = Box::pin(usecase.execute_upload_part_copy(build_request(input, Method::PUT))).await;
|
||||
assert_eq!(
|
||||
manager.put_object_admission_snapshot().active,
|
||||
Some(0),
|
||||
"copy releases its permit on every outcome"
|
||||
);
|
||||
if capped {
|
||||
let error = match response {
|
||||
Err(error) => crate::app::storage_api::s3::S3Error::from(error),
|
||||
Ok(_) => panic!("range exceeds destination size limit"),
|
||||
};
|
||||
assert_eq!(error.code(), &S3ErrorCode::EntityTooLarge);
|
||||
assert_eq!(
|
||||
response.expect_err("range exceeds destination size limit").code(),
|
||||
&S3ErrorCode::EntityTooLarge
|
||||
);
|
||||
let parts = store
|
||||
.list_object_parts(&bucket, "object", &upload.upload_id, None, 1000, &ObjectOptions::default())
|
||||
.await
|
||||
@@ -388,16 +281,16 @@ fn upload_part_copy_range_error_and_cancellation_release_foreground_permit() {
|
||||
.await
|
||||
.expect("abort capped session");
|
||||
} else {
|
||||
let copied = match response {
|
||||
Ok(output) => output,
|
||||
Err(error) => panic!("range copy after failed request released its source and permit: {error:?}"),
|
||||
};
|
||||
let copied = response.expect("range copy after failed request released its source and permit");
|
||||
let etag = copied
|
||||
.output
|
||||
.copy_part_result
|
||||
.expect("range result")
|
||||
.e_tag
|
||||
.as_deref()
|
||||
.map(crate::storage::storage_api::to_s3s_xml_etag)
|
||||
.expect("range ETag");
|
||||
let etag = etag.value().trim_matches('"').to_owned();
|
||||
.expect("range ETag")
|
||||
.value()
|
||||
.trim_matches('"')
|
||||
.to_owned();
|
||||
Arc::clone(&store)
|
||||
.complete_multipart_upload(
|
||||
&bucket,
|
||||
|
||||
@@ -258,17 +258,8 @@ mod tests {
|
||||
sse_customer_key: None,
|
||||
sse_customer_key_md5: None,
|
||||
};
|
||||
let envelope = RequestEnvelope::from_parts(
|
||||
crate::app::RequestHead {
|
||||
method: http::Method::GET,
|
||||
uri: http::Uri::from_static("/"),
|
||||
headers: http::HeaderMap::new(),
|
||||
credentials: None,
|
||||
region: None,
|
||||
service: None,
|
||||
},
|
||||
http::Extensions::new(),
|
||||
);
|
||||
let request = build_request((), http::Method::GET);
|
||||
let envelope = RequestEnvelope::from_s3s(&request);
|
||||
let error = DefaultObjectUsecase::without_context()
|
||||
.execute_get_object_attributes(envelope, fields)
|
||||
.await
|
||||
|
||||
@@ -108,11 +108,11 @@ use super::storage_api::object_usecase::set_disk::{
|
||||
get_lock_acquire_timeout, get_object_disk_read_timeout, is_valid_storage_class,
|
||||
};
|
||||
use super::storage_api::object_usecase::sse::{
|
||||
DecryptionRequest, EncryptionMaterialExt, EncryptionRequest, SseKmsPrincipal, SseKmsPrincipalExt,
|
||||
apply_bucket_default_lock_retention, authorize_sse_kms_object_read, bucket_default_write_sse, build_ssec_read_headers,
|
||||
classify_sse_read_response, encryption_material_to_metadata, extract_server_side_encryption_from_headers,
|
||||
extract_ssec_params_from_headers, extract_ssekms_context_from_headers, get_buffer_size_opt_in,
|
||||
load_bucket_object_lock_config_state, map_get_object_reader_error, sse_encryption, validate_bucket_object_lock_enabled_state,
|
||||
DecryptionRequest, EncryptionRequest, SseKmsPrincipal, apply_bucket_default_lock_retention, authorize_sse_kms_object_read,
|
||||
bucket_default_write_sse, build_ssec_read_headers, classify_sse_read_response, encryption_material_to_metadata,
|
||||
extract_server_side_encryption_from_headers, extract_ssec_params_from_headers, extract_ssekms_context_from_headers,
|
||||
get_buffer_size_opt_in, load_bucket_object_lock_config_state, map_get_object_reader_error, sse_encryption,
|
||||
validate_bucket_object_lock_enabled_state,
|
||||
};
|
||||
use super::storage_api::object_usecase::storage_class as storageclass;
|
||||
use super::storage_api::object_usecase::timeout_wrapper::{GetObjectTimeoutPolicy, RequestTimeoutWrapper};
|
||||
|
||||
@@ -2,9 +2,7 @@
|
||||
use super::storage_api::select_object::StorageError;
|
||||
use super::storage_api::select_object::options::get_opts;
|
||||
use super::storage_api::select_object::request_context::spawn_traced;
|
||||
use super::storage_api::select_object::sse::{
|
||||
SseKmsPrincipal, SseKmsPrincipalExt, authorize_sse_kms_object_read, project_sse_read_response_headers,
|
||||
};
|
||||
use super::storage_api::select_object::sse::{SseKmsPrincipal, authorize_sse_kms_object_read, project_sse_read_response_headers};
|
||||
use super::storage_api::select_object::{
|
||||
StoragePrepareSelectObjectSnapshotError, StorageSelectObjectSnapshot, get_validated_store, validate_sse_headers_for_read,
|
||||
validate_ssec_for_read,
|
||||
|
||||
@@ -1675,9 +1675,9 @@ pub(crate) mod sse {
|
||||
validate_bucket_object_lock_enabled_state,
|
||||
};
|
||||
pub(crate) use crate::storage::storage_api::sse_consumer::{
|
||||
EncryptionKeyKind, EncryptionMaterialExt, SseKmsPrincipalExt, bucket_default_write_sse, build_ssec_read_headers,
|
||||
encryption_material_to_metadata, extract_ssec_params_from_headers, extract_ssekms_context_from_headers,
|
||||
map_get_object_reader_error, mark_encrypted_multipart_metadata, multipart_completion_authorization, stored_ssec_metadata,
|
||||
EncryptionKeyKind, bucket_default_write_sse, build_ssec_read_headers, encryption_material_to_metadata,
|
||||
extract_ssec_params_from_headers, extract_ssekms_context_from_headers, map_get_object_reader_error,
|
||||
mark_encrypted_multipart_metadata, multipart_completion_authorization, stored_ssec_metadata,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
+72
-13
@@ -18,7 +18,6 @@ use http::StatusCode;
|
||||
use rustfs_kms::KmsUnavailableError;
|
||||
use rustfs_lock::LockError;
|
||||
use rustfs_s3_types::S3ErrorCode;
|
||||
use rustfs_sse::{SseError, data_plane_kms_error};
|
||||
use s3s::S3Error as LegacyS3Error;
|
||||
use s3s::stream::{aws_chunked_stream::AwsChunkedStreamError, upload_stream::UploadStreamError};
|
||||
|
||||
@@ -52,7 +51,7 @@ pub(crate) fn rpc_replay_admission_exhausted_status() -> tonic::Status {
|
||||
const UNSUPPORTED_PARTICIPANT_MESSAGE: &str = "A storage participant does not support this operation's protocol version. If the cluster is being upgraded, finish upgrading every node and retry.";
|
||||
|
||||
/// S3 error code for a request that names a KMS key the KMS does not hold.
|
||||
pub use rustfs_sse::KMS_KEY_NOT_FOUND_ERROR_CODE;
|
||||
pub const KMS_KEY_NOT_FOUND_ERROR_CODE: &str = "KMS.NotFoundException";
|
||||
|
||||
fn lock_resource_class(resource: Option<&str>) -> &'static str {
|
||||
let Some(resource) = resource else {
|
||||
@@ -123,6 +122,77 @@ fn record_s3_lock_failure(err: &StorageError) {
|
||||
}
|
||||
}
|
||||
|
||||
/// Map a KMS failure that surfaced on the S3 data path to its S3 error code.
|
||||
///
|
||||
/// The contract deliberately differs from the admin lifecycle handlers
|
||||
/// (`kms_key_lifecycle::lifecycle_error_status`): there a key id is the
|
||||
/// resource being addressed, so a missing key is `404`. Here the key id
|
||||
/// arrives inside a request header or a bucket default, so a key that is
|
||||
/// missing, disabled, or otherwise unusable is configuration the caller has to
|
||||
/// correct — AWS answers `400`, and reporting `500` instead both misfiles the
|
||||
/// failure as a server fault and makes SDKs back off and retry a request that
|
||||
/// cannot succeed.
|
||||
///
|
||||
/// `None` keeps the caller's fallthrough, which is the `500` that integrity
|
||||
/// faults — damaged, unreadable, or unknown-format key material — must keep.
|
||||
///
|
||||
/// Messages either name what the caller asked for or stay generic; detail that
|
||||
/// belongs to the deployment rather than the request stays in `source`, which
|
||||
/// the caller attaches and the logs retain.
|
||||
fn data_plane_kms_error(error: &rustfs_kms::KmsError) -> Option<(S3ErrorCode, String)> {
|
||||
use rustfs_kms::KmsError as Kms;
|
||||
|
||||
let generic = |code: S3ErrorCode| {
|
||||
let message = ApiError::error_code_to_message(&code);
|
||||
Some((code, message))
|
||||
};
|
||||
|
||||
match error {
|
||||
Kms::KeyNotFound { key_id } => Some((
|
||||
S3ErrorCode::Custom(KMS_KEY_NOT_FOUND_ERROR_CODE.into()),
|
||||
format!("KMS key not found: {key_id}"),
|
||||
)),
|
||||
Kms::KeyVersionNotFound { key_id, version } => Some((
|
||||
S3ErrorCode::Custom(KMS_KEY_NOT_FOUND_ERROR_CODE.into()),
|
||||
format!("KMS key version {version} not found for key {key_id}"),
|
||||
)),
|
||||
// The key exists but its state forbids the operation (disabled,
|
||||
// pending deletion). AWS treats an invalid key state as a request
|
||||
// error, not a server fault.
|
||||
Kms::InvalidOperation { .. } => Some((S3ErrorCode::InvalidRequest, error.to_string())),
|
||||
// A policy decision needs a human, so it must be distinguishable from
|
||||
// the transient classes an SDK retries.
|
||||
Kms::AccessDenied { .. } => generic(S3ErrorCode::AccessDenied),
|
||||
// Request-side faults: what was asked for cannot be served as asked.
|
||||
Kms::ContextMismatch { .. }
|
||||
| Kms::InvalidKey { .. }
|
||||
| Kms::ValidationError { .. }
|
||||
| Kms::UnsupportedAlgorithm { .. }
|
||||
| Kms::InvalidKeySize { .. } => Some((S3ErrorCode::InvalidRequest, error.to_string())),
|
||||
// A deployment whose KMS configuration cannot serve the request (for
|
||||
// example no key named and no default). Actionable, but the detail
|
||||
// describes the deployment, so it stays out of the response body.
|
||||
Kms::ConfigurationError { .. } => {
|
||||
Some((S3ErrorCode::InvalidRequest, "The KMS configuration cannot serve this request".to_string()))
|
||||
}
|
||||
// Transient: worth retrying, and must be counted against availability
|
||||
// rather than against the caller. `IoError` belongs here because it is
|
||||
// how a backend reports that its key store itself was unreachable —
|
||||
// rustfs/rustfs#7470 separated that from a missing key precisely so the
|
||||
// two stop looking alike, and leaving it on the 500 fallthrough would
|
||||
// erase that distinction again at the S3 boundary.
|
||||
Kms::BackendError { .. }
|
||||
| Kms::IoError { .. }
|
||||
| Kms::OperationTimedOut { .. }
|
||||
| Kms::OperationCancelled { .. }
|
||||
| Kms::CredentialsUnavailable { .. }
|
||||
| Kms::CacheError { .. } => generic(S3ErrorCode::ServiceUnavailable),
|
||||
// A permanent gap in the configured backend, never a missing resource.
|
||||
Kms::UnsupportedCapability { .. } => Some((S3ErrorCode::NotImplemented, error.to_string())),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// HTTP status of the error codes s3s cannot derive on its own.
|
||||
///
|
||||
/// s3s answers `None` for every `Custom` code, which the response layer turns
|
||||
@@ -693,17 +763,6 @@ impl From<crate::table_catalog::TableCatalogStoreError> for ApiError {
|
||||
}
|
||||
}
|
||||
|
||||
/// An SSE failure arrives already classified for the S3 data path; carry it over as is.
|
||||
impl From<SseError> for ApiError {
|
||||
fn from(err: SseError) -> Self {
|
||||
ApiError {
|
||||
code: err.code,
|
||||
message: err.message,
|
||||
source: err.source,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<StorageError> for ApiError {
|
||||
fn from(err: StorageError) -> Self {
|
||||
record_s3_lock_failure(&err);
|
||||
|
||||
@@ -15,14 +15,13 @@
|
||||
//! PutObject edge authorization and native frame tests against real storage.
|
||||
|
||||
use super::*;
|
||||
use crate::app::RequestHead;
|
||||
use crate::runtime_sources::ServerContextSlot;
|
||||
use crate::storage_api::server::http::access::test_support::{
|
||||
AuthorizationRequest, BucketOperations as _, CompletePart, DeleteBucketOptions, MakeBucketOptions, MultipartOperations as _,
|
||||
ObjectIO as _, ObjectOperations as _, ObjectOptions, ReqInfo, get_bucket_metadata, set_bucket_metadata,
|
||||
ObjectIO as _, ObjectOperations as _, ObjectOptions, ReqInfo, S3Request, get_bucket_metadata, set_bucket_metadata,
|
||||
};
|
||||
use bytes::Bytes;
|
||||
use http::{Extensions, HeaderMap, HeaderValue, Method, Uri};
|
||||
use http::{HeaderMap, HeaderValue, Method};
|
||||
use rustfs_gateway::{Limits, MetaView, SseConfig, TargetKind, TransportSecurity, WireRequest};
|
||||
use rustfs_gateway_stream::{ByteStream, PayloadCaps, PayloadRead, PayloadStream, StreamError, TrailingHeaders};
|
||||
use serial_test::serial;
|
||||
@@ -75,20 +74,11 @@ async fn set_policy(bucket: &str, statement: serde_json::Value) {
|
||||
}
|
||||
|
||||
fn envelope(server: &Arc<ServerContextSlot>, headers: HeaderMap) -> RequestEnvelope {
|
||||
let mut extensions = Extensions::new();
|
||||
extensions.insert(Arc::clone(server));
|
||||
extensions.insert(ReqInfo::default());
|
||||
RequestEnvelope::from_parts(
|
||||
RequestHead {
|
||||
method: Method::PUT,
|
||||
uri: Uri::from_static("/"),
|
||||
headers,
|
||||
credentials: None,
|
||||
region: None,
|
||||
service: None,
|
||||
},
|
||||
extensions,
|
||||
)
|
||||
let mut legacy = build_request((), Method::PUT);
|
||||
legacy.extensions.insert(Arc::clone(server));
|
||||
legacy.extensions.insert(ReqInfo::default());
|
||||
legacy.headers = headers;
|
||||
RequestEnvelope::from_s3s(&legacy)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -385,6 +375,20 @@ async fn native_put_frames_reach_checksum_validation_and_storage() {
|
||||
}
|
||||
}
|
||||
|
||||
fn build_request<T>(input: T, method: Method) -> S3Request<T> {
|
||||
S3Request {
|
||||
input,
|
||||
method,
|
||||
uri: http::Uri::from_static("/"),
|
||||
headers: HeaderMap::new(),
|
||||
extensions: http::Extensions::new(),
|
||||
credentials: None,
|
||||
region: None,
|
||||
service: None,
|
||||
trailing_headers: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn put_source_version_selects_versioned_read_permission() {
|
||||
@@ -393,26 +397,16 @@ async fn put_source_version_selects_versioned_read_permission() {
|
||||
for (read, allowed) in [("s3:GetObject", false), ("s3:GetObjectVersion", true)] {
|
||||
let (server, bucket) = fixture(&["s3:PutObject", read, "s3:ReplicateObject"]).await;
|
||||
let mut headers = HeaderMap::new();
|
||||
let uri = if query {
|
||||
format!("/?versionId={VERSION}").parse().unwrap()
|
||||
let mut legacy = build_request((), Method::PUT);
|
||||
legacy.extensions.insert(Arc::clone(&server));
|
||||
legacy.extensions.insert(ReqInfo::default());
|
||||
if query {
|
||||
legacy.uri = format!("/?versionId={VERSION}").parse().unwrap();
|
||||
} else {
|
||||
headers.insert("x-minio-source-version-id", HeaderValue::from_static(VERSION));
|
||||
Uri::from_static("/")
|
||||
};
|
||||
let mut extensions = Extensions::new();
|
||||
extensions.insert(Arc::clone(&server));
|
||||
extensions.insert(ReqInfo::default());
|
||||
let mut envelope = RequestEnvelope::from_parts(
|
||||
RequestHead {
|
||||
method: Method::PUT,
|
||||
uri,
|
||||
headers,
|
||||
credentials: None,
|
||||
region: None,
|
||||
service: None,
|
||||
},
|
||||
extensions,
|
||||
);
|
||||
}
|
||||
legacy.headers = headers;
|
||||
let mut envelope = RequestEnvelope::from_s3s(&legacy);
|
||||
let req = request(dto::PutObjectInput {
|
||||
bucket: rustfs_gateway_types::BucketName::new(bucket).unwrap(),
|
||||
key: rustfs_gateway_types::ObjectKey::new("allowed").unwrap(),
|
||||
|
||||
@@ -12,9 +12,7 @@ Applies to `rustfs/src/storage/`.
|
||||
|
||||
- Never log plaintext customer keys, decrypted key material, or sensitive SSE request payloads.
|
||||
- Keep SSE behavior aligned with KMS and admin handler integrations.
|
||||
- SSE key resolution lives in `crates/sse/`; `sse.rs` here only binds it to the running server (`RustfsSseHost`) and translates across the storage-engine boundary. Put new SSE logic in the crate unless it needs ecstore, IAM, or request types.
|
||||
- Coordinate related changes with:
|
||||
- `crates/sse/`
|
||||
- `crates/kms/`
|
||||
- `rustfs/src/admin/handlers/kms*.rs`
|
||||
- `crates/ecstore/`
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Authorization shapes that only migration batch A2 (rustfs/backlog#2765) uses: bucket policy, policy status, versioning, notification and ACL.
|
||||
//!
|
||||
//! Responsible for: the owned authorization function of each such operation, generic over
|
||||
//! `AuthorizationRequest`, so the legacy `S3Access` hook and `Served::authorize` call the same code.
|
||||
//! Not responsible for: shapes more than one batch shares (those stay in `storage/access.rs`), the
|
||||
//! policy evaluation itself (`authorize_request`), or the operation's use case.
|
||||
//! Upstream: the legacy hook in `storage/access.rs` and the batch's `server/gateway_handlers/bucket_access.rs`.
|
||||
//! Downstream: the shared shapes in `storage/access.rs`.
|
||||
@@ -0,0 +1,22 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Authorization shapes that only migration batch A1 (rustfs/backlog#2765) uses: bucket CORS, tagging, public access block and encryption.
|
||||
//!
|
||||
//! Responsible for: the owned authorization function of each such operation, generic over
|
||||
//! `AuthorizationRequest`, so the legacy `S3Access` hook and `Served::authorize` call the same code.
|
||||
//! Not responsible for: shapes more than one batch shares (those stay in `storage/access.rs`), the
|
||||
//! policy evaluation itself (`authorize_request`), or the operation's use case.
|
||||
//! Upstream: the legacy hook in `storage/access.rs` and the batch's `server/gateway_handlers/bucket_configs.rs`.
|
||||
//! Downstream: the shared shapes in `storage/access.rs`.
|
||||
@@ -15,6 +15,8 @@
|
||||
//! The authorization shapes one migration batch owns (rustfs/backlog#2765), one file per batch so
|
||||
//! parallel batches edit disjoint files. Shapes several batches share stay in `storage/access.rs`.
|
||||
|
||||
pub(crate) mod bucket_access;
|
||||
pub(crate) mod bucket_configs;
|
||||
pub(crate) mod bucket_core;
|
||||
pub(crate) mod bucket_replication_lifecycle;
|
||||
pub(crate) mod listing_multipart;
|
||||
|
||||
@@ -1954,7 +1954,6 @@ mod tests {
|
||||
// Drive the real data path: the principal built from this request is
|
||||
// what carries the audit slot into the SSE code.
|
||||
let metadata = unreadable_sse_kms_metadata();
|
||||
use crate::storage::sse::SseKmsPrincipalExt as _;
|
||||
let envelope = crate::app::RequestEnvelope::from_s3s(&req);
|
||||
let principal =
|
||||
crate::storage::sse::SseKmsPrincipal::from_request(&envelope).expect("authenticated request has a principal");
|
||||
|
||||
+9243
-606
File diff suppressed because it is too large
Load Diff
@@ -252,237 +252,4 @@ mod tests {
|
||||
|
||||
println!("✅ Full cycle (generate -> encrypt DEK -> decrypt DEK -> decrypt data) test passed!");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_simple_sse_dek_provider_encrypt_decrypt() {
|
||||
use std::io::Cursor;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
// 1. Setup: Create SimpleSseDekProvider with test master key
|
||||
let provider = TestSseDekProvider::new_with_key([42u8; 32]);
|
||||
|
||||
// 2. Generate a data encryption key
|
||||
let bucket = "test-bucket";
|
||||
let key = "test-key";
|
||||
let kms_key_id = "default"; // Key ID is ignored in simple provider
|
||||
let context = ObjectEncryptionContext::new(bucket.to_string(), key.to_string());
|
||||
|
||||
let (data_key, _encrypted_dek) = provider
|
||||
.generate_sse_dek(&context, kms_key_id)
|
||||
.await
|
||||
.expect("Failed to generate DEK");
|
||||
|
||||
// 3. Prepare test data (plaintext)
|
||||
let plaintext = b"Hello, World! This is a test message for encryption and decryption.";
|
||||
println!("Original plaintext: {:?}", String::from_utf8_lossy(plaintext));
|
||||
println!("Plaintext length: {} bytes", plaintext.len());
|
||||
|
||||
// 4. Encrypt with EncryptReader.
|
||||
let plaintext_reader = Cursor::new(plaintext.to_vec());
|
||||
let mut encrypt_reader = EncryptReader::new(plaintext_reader, data_key.plaintext_key, data_key.nonce);
|
||||
|
||||
// Read encrypted data
|
||||
let mut encrypted_data = Vec::new();
|
||||
encrypt_reader
|
||||
.read_to_end(&mut encrypted_data)
|
||||
.await
|
||||
.expect("Failed to read encrypted data");
|
||||
|
||||
println!("Encrypted data length: {} bytes", encrypted_data.len());
|
||||
println!(
|
||||
"First 16 bytes of encrypted data: {:02x?}",
|
||||
&encrypted_data[..16.min(encrypted_data.len())]
|
||||
);
|
||||
|
||||
// Verify encrypted data is different from plaintext
|
||||
assert_ne!(
|
||||
&encrypted_data[..plaintext.len()],
|
||||
plaintext,
|
||||
"Encrypted data should be different from plaintext"
|
||||
);
|
||||
|
||||
// 5. Decrypt with DecryptReader.
|
||||
let encrypted_reader = Cursor::new(encrypted_data);
|
||||
let mut decrypt_reader = DecryptReader::new(encrypted_reader, data_key.plaintext_key, data_key.nonce);
|
||||
|
||||
// Read decrypted data
|
||||
let mut decrypted_data = Vec::new();
|
||||
decrypt_reader
|
||||
.read_to_end(&mut decrypted_data)
|
||||
.await
|
||||
.expect("Failed to read decrypted data");
|
||||
|
||||
println!("Decrypted data: {:?}", String::from_utf8_lossy(&decrypted_data));
|
||||
println!("Decrypted length: {} bytes", decrypted_data.len());
|
||||
|
||||
// 6. Verify decrypted data matches original plaintext
|
||||
assert_eq!(decrypted_data, plaintext, "Decrypted data should match original plaintext");
|
||||
|
||||
println!("✅ Encryption/Decryption test passed!");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_simple_sse_dek_provider_encrypt_decrypt_large_data() {
|
||||
use std::io::Cursor;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
// 1. Setup: Create SimpleSseDekProvider with test master key
|
||||
let provider = TestSseDekProvider::new_with_key([42u8; 32]);
|
||||
|
||||
let bucket = "test-bucket";
|
||||
let key = "test-key-large";
|
||||
let kms_key_id = "default";
|
||||
let context = ObjectEncryptionContext::new(bucket.to_string(), key.to_string());
|
||||
|
||||
let (data_key, _encrypted_dek) = provider
|
||||
.generate_sse_dek(&context, kms_key_id)
|
||||
.await
|
||||
.expect("Failed to generate DEK");
|
||||
|
||||
// Create 1MB of test data
|
||||
let plaintext_size = 1024 * 1024; // 1MB
|
||||
let plaintext: Vec<u8> = (0..plaintext_size).map(|i| (i % 256) as u8).collect();
|
||||
println!("Testing with {} bytes of data", plaintext.len());
|
||||
|
||||
// Encrypt.
|
||||
let plaintext_reader = Cursor::new(plaintext.clone());
|
||||
let mut encrypt_reader = EncryptReader::new(plaintext_reader, data_key.plaintext_key, data_key.nonce);
|
||||
|
||||
let mut encrypted_data = Vec::new();
|
||||
encrypt_reader
|
||||
.read_to_end(&mut encrypted_data)
|
||||
.await
|
||||
.expect("Failed to encrypt large data");
|
||||
|
||||
println!("Encrypted {} bytes to {} bytes", plaintext.len(), encrypted_data.len());
|
||||
|
||||
// Decrypt.
|
||||
let encrypted_reader = Cursor::new(encrypted_data);
|
||||
let mut decrypt_reader = DecryptReader::new(encrypted_reader, data_key.plaintext_key, data_key.nonce);
|
||||
|
||||
let mut decrypted_data = Vec::new();
|
||||
decrypt_reader
|
||||
.read_to_end(&mut decrypted_data)
|
||||
.await
|
||||
.expect("Failed to decrypt large data");
|
||||
|
||||
// Verify
|
||||
assert_eq!(decrypted_data.len(), plaintext.len(), "Decrypted size should match original");
|
||||
assert_eq!(decrypted_data, plaintext, "Decrypted data should match original plaintext");
|
||||
|
||||
println!("✅ Large data encryption/decryption test passed!");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_simple_sse_dek_provider_different_nonces() {
|
||||
use std::io::Cursor;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
// 1. Setup: Create SimpleSseDekProvider with test master key
|
||||
let provider = TestSseDekProvider::new_with_key([42u8; 32]);
|
||||
|
||||
let bucket = "test-bucket";
|
||||
let key = "test-key";
|
||||
let kms_key_id = "default";
|
||||
let context = ObjectEncryptionContext::new(bucket.to_string(), key.to_string());
|
||||
|
||||
// Generate two different keys (with different nonces)
|
||||
let (data_key1, _) = provider
|
||||
.generate_sse_dek(&context, kms_key_id)
|
||||
.await
|
||||
.expect("Failed to generate DEK 1");
|
||||
|
||||
let (data_key2, _) = provider
|
||||
.generate_sse_dek(&context, kms_key_id)
|
||||
.await
|
||||
.expect("Failed to generate DEK 2");
|
||||
|
||||
// Verify nonces are different
|
||||
assert_ne!(data_key1.nonce, data_key2.nonce, "Different keys should have different nonces");
|
||||
|
||||
// Same plaintext
|
||||
let plaintext = b"Same plaintext";
|
||||
|
||||
// Encrypt with first key.
|
||||
let reader1 = Cursor::new(plaintext.to_vec());
|
||||
let mut encrypt_reader1 = EncryptReader::new(reader1, data_key1.plaintext_key, data_key1.nonce);
|
||||
let mut encrypted1 = Vec::new();
|
||||
encrypt_reader1.read_to_end(&mut encrypted1).await.unwrap();
|
||||
|
||||
// Encrypt with second key.
|
||||
let reader2 = Cursor::new(plaintext.to_vec());
|
||||
let mut encrypt_reader2 = EncryptReader::new(reader2, data_key2.plaintext_key, data_key2.nonce);
|
||||
let mut encrypted2 = Vec::new();
|
||||
encrypt_reader2.read_to_end(&mut encrypted2).await.unwrap();
|
||||
|
||||
// Verify ciphertexts are different (due to different nonces/keys)
|
||||
assert_ne!(
|
||||
encrypted1, encrypted2,
|
||||
"Same plaintext with different nonces should produce different ciphertext"
|
||||
);
|
||||
|
||||
println!("✅ Different nonces produce different ciphertext - test passed!");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_simple_sse_dek_provider_decrypt_with_encrypted_dek() {
|
||||
use std::io::Cursor;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
// 1. Setup: Create SimpleSseDekProvider with test master key
|
||||
let provider = TestSseDekProvider::new_with_key([42u8; 32]);
|
||||
|
||||
let bucket = "test-bucket";
|
||||
let key = "test-key";
|
||||
let kms_key_id = "default";
|
||||
let context = ObjectEncryptionContext::new(bucket.to_string(), key.to_string());
|
||||
|
||||
// 1. Generate DEK and get encrypted DEK
|
||||
let (data_key, encrypted_dek) = provider
|
||||
.generate_sse_dek(&context, kms_key_id)
|
||||
.await
|
||||
.expect("Failed to generate DEK");
|
||||
|
||||
let original_plaintext_key = data_key.plaintext_key;
|
||||
let original_nonce = data_key.nonce;
|
||||
|
||||
// 2. Simulate storing encrypted_dek and nonce in metadata
|
||||
// In real scenario, nonce would be stored separately in metadata
|
||||
|
||||
// 3. Later, decrypt the DEK
|
||||
let decrypted_plaintext_key = provider
|
||||
.decrypt_sse_dek(&encrypted_dek, kms_key_id, &context)
|
||||
.await
|
||||
.expect("Failed to decrypt DEK");
|
||||
|
||||
// 4. Verify decrypted key matches original
|
||||
assert_eq!(
|
||||
decrypted_plaintext_key, original_plaintext_key,
|
||||
"Decrypted DEK should match original plaintext key"
|
||||
);
|
||||
|
||||
// 5. Use decrypted key to encrypt/decrypt data
|
||||
let plaintext = b"Test data with decrypted DEK";
|
||||
|
||||
// Encrypt with original key.
|
||||
let reader = Cursor::new(plaintext.to_vec());
|
||||
let mut encrypt_reader = EncryptReader::new(reader, original_plaintext_key, original_nonce);
|
||||
let mut encrypted_data = Vec::new();
|
||||
encrypt_reader.read_to_end(&mut encrypted_data).await.unwrap();
|
||||
|
||||
// Decrypt with recovered key (simulating GET operation).
|
||||
let reader = Cursor::new(encrypted_data);
|
||||
let mut decrypt_reader = DecryptReader::new(
|
||||
reader,
|
||||
decrypted_plaintext_key,
|
||||
original_nonce, // In real scenario, read from metadata
|
||||
);
|
||||
let mut decrypted_data = Vec::new();
|
||||
decrypt_reader.read_to_end(&mut decrypted_data).await.unwrap();
|
||||
|
||||
// Verify
|
||||
assert_eq!(decrypted_data, plaintext, "Data decrypted with recovered key should match original");
|
||||
|
||||
println!("✅ Full cycle (generate -> encrypt DEK -> decrypt DEK -> decrypt data) test passed!");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -448,10 +448,9 @@ pub(crate) mod s3_api_consumer {
|
||||
|
||||
pub(crate) mod sse_consumer {
|
||||
pub(crate) use super::super::sse::{
|
||||
EncryptionKeyKind, EncryptionMaterialExt, SseKmsPrincipalExt, bucket_default_write_sse, build_ssec_read_headers,
|
||||
encryption_material_to_metadata, extract_ssec_params_from_headers, extract_ssekms_context_from_headers,
|
||||
log_sse_kms_key_policy_mode, map_get_object_reader_error, mark_encrypted_multipart_metadata,
|
||||
multipart_completion_authorization, stored_ssec_metadata,
|
||||
EncryptionKeyKind, bucket_default_write_sse, build_ssec_read_headers, encryption_material_to_metadata,
|
||||
extract_ssec_params_from_headers, extract_ssekms_context_from_headers, log_sse_kms_key_policy_mode,
|
||||
map_get_object_reader_error, mark_encrypted_multipart_metadata, multipart_completion_authorization, stored_ssec_metadata,
|
||||
};
|
||||
pub(crate) use super::{
|
||||
DecryptionRequest, EncryptionRequest, PrepareEncryptionRequest, SseKmsPrincipal, apply_bucket_default_lock_retention,
|
||||
@@ -480,7 +479,8 @@ pub(crate) mod tonic_service_consumer {
|
||||
pub(crate) mod test_consumer {
|
||||
pub(crate) use super::super::access_owned::object_read::authorize_head_object;
|
||||
pub(crate) mod sse {
|
||||
pub(crate) use super::super::super::sse::{RecordingKmsKeyAuthorizer, TestAuthorizationHooks, sse_kms_object_metadata};
|
||||
pub(crate) use super::super::super::sse::TestAuthorizationHooks;
|
||||
pub(crate) use super::super::super::sse::tests::{RecordingKmsKeyAuthorizer, sse_kms_object_metadata};
|
||||
}
|
||||
pub(crate) use super::super::{
|
||||
apply_cors_headers, apply_default_lock_retention_metadata, check_preconditions, decode_tags_to_map,
|
||||
|
||||
@@ -233,6 +233,7 @@ pub(crate) mod server {
|
||||
StorageObjectOptions as ObjectOptions, StoragePutObjReader, get_bucket_metadata, is_err_object_not_found,
|
||||
set_bucket_metadata,
|
||||
};
|
||||
pub(crate) use s3s::S3Request;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -5497,18 +5497,13 @@ require_source_contains \
|
||||
# its internal dependency surface is pinned to exactly rustfs-signer so it cannot
|
||||
# quietly grow storage-side dependencies. Adding any other rustfs-* dependency to
|
||||
# any of these five crates needs its own adjudication, not a quiet Cargo.toml edit.
|
||||
# sse sits between the S3 layer and the storage engine and reaches the running
|
||||
# server only through its `SseHost` port, so its internal dependency surface is
|
||||
# pinned as well: the KMS it calls, the policy and S3 vocabularies it speaks, and
|
||||
# the two foundation crates. An edge to ecstore, iam, or the binary would let key
|
||||
# resolution reach engine or identity state behind the port.
|
||||
# The pattern matches both TOML dependency spellings: `rustfs-x = ...` and the
|
||||
# dotted `rustfs-x.workspace = true` form (which previously escaped this guard).
|
||||
LEAF_CRATE_DEP_HITS_FILE="${TMP_DIR}/leaf_crate_dep_hits.txt"
|
||||
: >"$LEAF_CRATE_DEP_HITS_FILE"
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
for leaf in config credentials crypto io-metrics madmin sse; do
|
||||
for leaf in config credentials crypto io-metrics madmin; do
|
||||
manifest="crates/${leaf}/Cargo.toml"
|
||||
[[ -f "$manifest" ]] || continue
|
||||
leaf_dep_status=0
|
||||
@@ -5523,9 +5518,6 @@ LEAF_CRATE_DEP_HITS_FILE="${TMP_DIR}/leaf_crate_dep_hits.txt"
|
||||
madmin)
|
||||
rg -v '^[^:]*:[0-9]+:rustfs-signer(\.[a-zA-Z_-]+)* *=' "${TMP_DIR}/leaf_dep_raw.txt" >>"$LEAF_CRATE_DEP_HITS_FILE" || true
|
||||
;;
|
||||
sse)
|
||||
rg -v '^[^:]*:[0-9]+:rustfs-(config|kms|policy|s3-types|utils)(\.[a-zA-Z_-]+)* *=' "${TMP_DIR}/leaf_dep_raw.txt" >>"$LEAF_CRATE_DEP_HITS_FILE" || true
|
||||
;;
|
||||
*)
|
||||
cat "${TMP_DIR}/leaf_dep_raw.txt" >>"$LEAF_CRATE_DEP_HITS_FILE"
|
||||
;;
|
||||
@@ -5534,7 +5526,7 @@ LEAF_CRATE_DEP_HITS_FILE="${TMP_DIR}/leaf_crate_dep_hits.txt"
|
||||
)
|
||||
|
||||
if [[ -s "$LEAF_CRATE_DEP_HITS_FILE" ]]; then
|
||||
report_failure "leaf/pinned-dep crates: config/credentials/crypto take no internal rustfs-* dependency; io-metrics only rustfs-s3-ops (pure contract crate, backlog#1834); madmin only rustfs-signer (admin SDK SigV4 client, #6166); sse only rustfs-config/kms/policy/s3-types/utils (it reaches the server through SseHost); any new edge needs its own adjudication: $(paste -sd '; ' "$LEAF_CRATE_DEP_HITS_FILE")"
|
||||
report_failure "leaf/pinned-dep crates: config/credentials/crypto take no internal rustfs-* dependency; io-metrics only rustfs-s3-ops (pure contract crate, backlog#1834); madmin only rustfs-signer (admin SDK SigV4 client, #6166); any new edge needs its own adjudication: $(paste -sd '; ' "$LEAF_CRATE_DEP_HITS_FILE")"
|
||||
fi
|
||||
|
||||
# --- ecstore module-level lint blankets (backlog#1823 step 9) ---
|
||||
|
||||
Reference in New Issue
Block a user