mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-22 07:53:57 +00:00
Compare commits
5 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 1321e15988 | |||
| f1312f9bd1 | |||
| 149f5700b4 | |||
| 38aa318fb2 | |||
| e7c5d7c198 |
Generated
+187
-184
File diff suppressed because it is too large
Load Diff
+19
-22
@@ -175,9 +175,9 @@ futures = "0.3.34"
|
||||
futures-core = "0.3.34"
|
||||
futures-lite = "2.6.1"
|
||||
futures-util = "0.3.34"
|
||||
pollster = "1.0.3"
|
||||
pollster = "1.0.2"
|
||||
pulsar = { default-features = false, version = "=6.9.0" }
|
||||
lapin = { default-features = false, version = "4.13.0" }
|
||||
lapin = { default-features = false, version = "4.12.3" }
|
||||
hyper = { version = "1.12.0" }
|
||||
h2 = { version = "0.4.20" }
|
||||
hyper-rustls = { default-features = false, version = "0.27.10" }
|
||||
@@ -216,7 +216,7 @@ csv-core = "0.1.13"
|
||||
bytesize = "2.7.0"
|
||||
byteorder = "1.5.0"
|
||||
flatbuffers = "25.12.19"
|
||||
form_urlencoded = "1.2.3"
|
||||
form_urlencoded = "1.2.2"
|
||||
prost = "0.14.6"
|
||||
quick-xml = "0.42.0"
|
||||
rmp = { version = "0.8.15" }
|
||||
@@ -258,7 +258,7 @@ proptest = "1"
|
||||
# Time and Date
|
||||
chrono = { version = "0.4.45" }
|
||||
humantime = "2.4.0"
|
||||
jiff = { version = "1.0.2" }
|
||||
jiff = { version = "1.0.1" }
|
||||
time = { version = "0.3.55" }
|
||||
|
||||
# Database
|
||||
@@ -278,15 +278,15 @@ atomic_enum = "0.3.0"
|
||||
aws-config = { version = "1.13.3" }
|
||||
aws-credential-types = { version = "1.3.1" }
|
||||
aws-sdk-kms = { default-features = false, version = "1.133.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.161.0" }
|
||||
aws-sdk-sts = { default-features = false, version = "1.128.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.160.0" }
|
||||
aws-sdk-sts = { default-features = false, version = "1.127.0" }
|
||||
aws-smithy-async = { version = "1.3.1" }
|
||||
aws-smithy-http-client = { default-features = false, version = "1.5.3" }
|
||||
aws-smithy-runtime-api = { version = "1.22.0" }
|
||||
aws-smithy-types = { version = "1.9.4" }
|
||||
base64-simd = "0.8.0"
|
||||
brotli = "9.1.3"
|
||||
clap = { version = "4.6.9" }
|
||||
clap = { version = "4.6.8" }
|
||||
const-str = { version = "1.1.0" }
|
||||
convert_case = "0.12.0"
|
||||
criterion = { version = "0.8" }
|
||||
@@ -298,12 +298,12 @@ datafusion = { default-features = false, version = "55.3.0" }
|
||||
derive_builder = "0.20.2"
|
||||
ed25519-dalek = "3.0.0"
|
||||
enumset = "1.1.15"
|
||||
faster-hex = "2.0.0"
|
||||
faster-hex = "1.0.0"
|
||||
flate2 = "1.1.11"
|
||||
glob = "0.3.4"
|
||||
google-cloud-storage = "1.22.0"
|
||||
google-cloud-auth = "1.19.0"
|
||||
hashbrown = { version = "0.17.2" }
|
||||
hashbrown = { version = "0.17.1" }
|
||||
# Base32 for RFC 6238 TOTP shared secrets (RFC 4648 unpadded, the alphabet
|
||||
# every authenticator app expects). Already in the graph transitively.
|
||||
data-encoding = "2.11.1"
|
||||
@@ -337,7 +337,7 @@ percent-encoding = "2.3.2"
|
||||
# CLI needs its own QR encoder. No default features: the image/render backends
|
||||
# pull in an image stack this only needs SVG and text output from.
|
||||
qrcode-rs = { version = "2.2.0", default-features = false, features = ["std", "svg"] }
|
||||
pin-project-lite = "0.2.18"
|
||||
pin-project-lite = "0.2.17"
|
||||
pretty_assertions = "1.4.1"
|
||||
rand = { version = "0.10.4" }
|
||||
ratelimit = "2.0.0"
|
||||
@@ -360,7 +360,7 @@ legacy-s3s = { package = "s3s", git = "https://github.com/s3s-project/s3s", rev
|
||||
serial_test = "4.0.1"
|
||||
shadow-rs = { default-features = false, version = "2.0.0" }
|
||||
siphasher = "1.0.4"
|
||||
smallvec = { version = "1.16.6" }
|
||||
smallvec = { version = "1.16.5" }
|
||||
compact_str = "0.10.0"
|
||||
snap = "1.1.2"
|
||||
starshard = { version = "2.4.1" }
|
||||
@@ -378,23 +378,23 @@ tracing-error = "0.2.1"
|
||||
tracing-opentelemetry = { version = "0.34.0" }
|
||||
tracing-subscriber = { version = "0.3.23" }
|
||||
transform-stream = "0.3.1"
|
||||
url = "2.5.9"
|
||||
url = "2.5.8"
|
||||
urlencoding = "2.1.3"
|
||||
uuid = { version = "1.29.3" }
|
||||
vaultrs = { version = "0.8.0" }
|
||||
tar = "0.4.46"
|
||||
walkdir = "2.5.1"
|
||||
walkdir = "2.5.0"
|
||||
winapi-util = "0.1.11"
|
||||
windows = { version = "0.62.2" }
|
||||
windows-sys = "0.61.2"
|
||||
xxhash-rust = { version = "0.8.21" }
|
||||
zip = "9.8.1"
|
||||
zip = "9.7.0"
|
||||
zstd = "0.14.4"
|
||||
|
||||
# Observability and Metrics
|
||||
metrics = "0.24.6"
|
||||
metrics-util = "0.20"
|
||||
dial9 = { version = "0.5.9", default-features = false }
|
||||
dial9 = { version = "0.5.8", default-features = false }
|
||||
opentelemetry = { version = "0.33.1" }
|
||||
opentelemetry-appender-tracing = { version = "0.33.1" }
|
||||
opentelemetry-otlp = { version = "0.33.1" }
|
||||
@@ -405,12 +405,12 @@ opentelemetry-stdout = { version = "0.33.1" }
|
||||
pyroscope = { version = "2.1.5" }
|
||||
|
||||
# FTP and SFTP
|
||||
libunftp = { version = "0.25.1" }
|
||||
libunftp = { version = "0.25.0" }
|
||||
unftp-core = "0.1.1"
|
||||
suppaftp = { version = "13.0.0" }
|
||||
rcgen = { version = "0.14.10", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
|
||||
russh = { version = "0.65.4" }
|
||||
russh-sftp = "4.0.0"
|
||||
russh = { version = "0.65.2" }
|
||||
russh-sftp = "3.0.1"
|
||||
|
||||
# WebDAV
|
||||
dav-server = "0.12.0"
|
||||
@@ -423,10 +423,7 @@ hotpath = { version = "0.32.1", default-features = false }
|
||||
insta = { version = "1.49" }
|
||||
|
||||
[workspace.metadata.cargo-shear]
|
||||
# The `#[path]` test modules under `rustfs/src/server/http/tests/` are linked by
|
||||
# rustc (including the `http.rs` + `http/` sibling-directory rule), but cargo-shear
|
||||
# resolves those nested `#[path]` attributes against the wrong base directory.
|
||||
ignored-paths = ["rustfs/src/server/http/tests/**/*.rs"]
|
||||
ignored = ["hotpath", "rustfs"]
|
||||
|
||||
[profile.dev]
|
||||
# Full debuginfo roughly doubles compile+link time and produces multi-GB
|
||||
|
||||
@@ -15,11 +15,12 @@ workspace = true
|
||||
|
||||
[features]
|
||||
default = []
|
||||
hotpath = ["rustfs-io-core/hotpath"]
|
||||
hotpath-alloc = ["hotpath", "rustfs-io-core/hotpath-alloc"]
|
||||
hotpath-cpu = ["hotpath", "rustfs-io-core/hotpath-cpu"]
|
||||
hotpath = ["hotpath/hotpath", "hotpath/tokio", "rustfs-io-core/hotpath"]
|
||||
hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc", "rustfs-io-core/hotpath-alloc"]
|
||||
hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu", "rustfs-io-core/hotpath-cpu"]
|
||||
|
||||
[dependencies]
|
||||
hotpath.workspace = true
|
||||
# Internal crates
|
||||
rustfs-io-core = { workspace = true }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
|
||||
@@ -34,7 +34,10 @@
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
|
||||
use crate::common::{
|
||||
AdminTransport, RustFSTestEnvironment, admin_attach_user_policy_via, admin_create_user, admin_request_with_session_token,
|
||||
init_logging, local_http_client, parse_assume_role_credentials,
|
||||
};
|
||||
use hmac::{Hmac, KeyInit as _, Mac};
|
||||
use http::header::HOST;
|
||||
use rustfs_s3_types::Body;
|
||||
@@ -396,6 +399,54 @@ mod tests {
|
||||
let (status, _) = signed_request(&env.url, http::Method::GET, ACCOUNT_INFO_PATH, None, user_ak, old_sk).await?;
|
||||
assert!(status.is_client_error(), "the old secret must stop working, status: {status}");
|
||||
|
||||
env.stop_server();
|
||||
drop(env);
|
||||
console_session_rotates_the_iam_password_and_revokes_the_old_session().await
|
||||
}
|
||||
|
||||
async fn console_session_rotates_the_iam_password_and_revokes_the_old_session() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let user = "console-password-user";
|
||||
let old_password = uuid::Uuid::new_v4().to_string();
|
||||
let new_password = uuid::Uuid::new_v4().to_string();
|
||||
admin_create_user(&env, user, &old_password).await?;
|
||||
admin_attach_user_policy_via(AdminTransport::Signed, &env.url, &env.access_key, &env.secret_key, "readonly", user)
|
||||
.await?;
|
||||
|
||||
let (status, response) = assume_role(&env.url, user, &old_password, None).await?;
|
||||
assert_eq!(status, reqwest::StatusCode::OK, "the current password must sign in");
|
||||
let (session_key, session_secret, token) = parse_assume_role_credentials(&response)?;
|
||||
let session_request = |method, path, body| {
|
||||
admin_request_with_session_token(&env.url, method, path, body, &session_key, &session_secret, Some(&token))
|
||||
};
|
||||
let password_request = |current: &str| {
|
||||
Some(serde_json::json!({ "current_secret_key": current, "new_secret_key": new_password }).to_string())
|
||||
};
|
||||
|
||||
let (status, body) =
|
||||
session_request(http::Method::POST, ACCOUNT_PASSWORD_PATH, password_request(&session_secret)).await?;
|
||||
assert_eq!(
|
||||
status,
|
||||
reqwest::StatusCode::BAD_REQUEST,
|
||||
"the session secret must not be accepted as the IAM password"
|
||||
);
|
||||
assert!(body.contains("InvalidRequest") && body.contains("the current secret key is incorrect"));
|
||||
|
||||
let (status, body) = session_request(http::Method::POST, ACCOUNT_PASSWORD_PATH, password_request(&old_password)).await?;
|
||||
assert_eq!(status, reqwest::StatusCode::OK, "the same session must accept the current IAM password");
|
||||
assert_eq!(serde_json::from_str::<serde_json::Value>(&body)?["sessions_revoked"], 1);
|
||||
|
||||
let (status, _) = session_request(http::Method::GET, ACCOUNT_INFO_PATH, None).await?;
|
||||
assert_eq!(status, reqwest::StatusCode::FORBIDDEN, "the pre-rotation session must be revoked");
|
||||
let (status, _) = assume_role(&env.url, user, &old_password, None).await?;
|
||||
assert_eq!(status, reqwest::StatusCode::FORBIDDEN, "the old password must no longer sign in");
|
||||
let (status, response) = assume_role(&env.url, user, &new_password, None).await?;
|
||||
assert_eq!(status, reqwest::StatusCode::OK, "the new password must sign in");
|
||||
parse_assume_role_credentials(&response)?;
|
||||
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -183,6 +183,7 @@ rmp.workspace = true
|
||||
rmp-serde.workspace = true
|
||||
tokio-util = { workspace = true, features = ["io", "compat", "rt"] }
|
||||
tokio-stream = { workspace = true, features = ["sync"] }
|
||||
hmac = { workspace = true }
|
||||
sha2 = { workspace = true }
|
||||
hex-simd = { workspace = true }
|
||||
tempfile.workspace = true
|
||||
|
||||
@@ -1786,7 +1786,7 @@ where
|
||||
}
|
||||
|
||||
pub async fn update_user_secret_key(&self, access_key: &str, secret_key: &str) -> Result<(OffsetDateTime, AccountStatus)> {
|
||||
self.update_user_secret_key_with_guard(access_key, secret_key, None)
|
||||
self.update_user_secret_key_with_guard(access_key, secret_key, None, None)
|
||||
.await
|
||||
.map(|(updated_at, status, _guard)| (updated_at, status))
|
||||
}
|
||||
@@ -1796,6 +1796,7 @@ where
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
expected_secret: Option<&str>,
|
||||
session: Option<&Credentials>,
|
||||
) -> Result<(
|
||||
OffsetDateTime,
|
||||
AccountStatus,
|
||||
@@ -1824,6 +1825,51 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
// Lock order is parent, then the presenting STS credential. Deletion and
|
||||
// session refresh use the same STS lock. Release it before the revocation sweep.
|
||||
let _session_guard = match session {
|
||||
Some(session) => {
|
||||
let guard = self.cache.sts_account_mutation_lock(&session.access_key).lock_owned().await;
|
||||
let current = self
|
||||
.api
|
||||
.load_user_identity(&session.access_key, UserType::Sts)
|
||||
.await
|
||||
.map_err(|error| {
|
||||
if is_err_no_such_user(&error) {
|
||||
Error::IAMActionNotAllowed
|
||||
} else {
|
||||
error
|
||||
}
|
||||
})?;
|
||||
let parent = if current.credentials.parent_user.is_empty() {
|
||||
current.credentials.claims_or_empty().get("parent").and_then(Value::as_str)
|
||||
} else {
|
||||
Some(current.credentials.parent_user.as_str())
|
||||
};
|
||||
use subtle::ConstantTimeEq as _;
|
||||
if !session.is_valid()
|
||||
|| !current.credentials.is_valid()
|
||||
|| !current.credentials.is_temp()
|
||||
|| current.credentials.is_service_account()
|
||||
|| current.credentials.access_key != session.access_key
|
||||
|| parent != Some(access_key)
|
||||
|| !bool::from(current.credentials.secret_key.as_bytes().ct_eq(session.secret_key.as_bytes()))
|
||||
|| !bool::from(
|
||||
current
|
||||
.credentials
|
||||
.session_token
|
||||
.as_bytes()
|
||||
.ct_eq(session.session_token.as_bytes()),
|
||||
)
|
||||
|| current.sts_generation != u.sts_generation
|
||||
{
|
||||
return Err(Error::IAMActionNotAllowed);
|
||||
}
|
||||
Some(guard)
|
||||
}
|
||||
None => None,
|
||||
};
|
||||
|
||||
let mut cred = u.credentials.clone();
|
||||
cred.secret_key = secret_key.to_string();
|
||||
|
||||
|
||||
+183
-6
@@ -1421,11 +1421,11 @@ impl<T: Store> IamSys<T> {
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
) -> Result<(OffsetDateTime, rustfs_madmin::AccountStatus, usize)> {
|
||||
self.rotate_user_secret_key_checked(access_key, secret_key, None).await
|
||||
self.rotate_user_secret_key_checked(access_key, secret_key, None, None).await
|
||||
}
|
||||
|
||||
/// Change an enabled regular caller's secret only if its signing secret is
|
||||
/// still current under the parent mutation lock. Status and policies are kept.
|
||||
/// Change an enabled regular user's secret only if the supplied current
|
||||
/// secret still matches under the parent mutation lock. Status and policies are kept.
|
||||
/// Persistence and revocation continue after caller cancellation.
|
||||
pub async fn change_own_secret_key(
|
||||
&self,
|
||||
@@ -1433,7 +1433,27 @@ impl<T: Store> IamSys<T> {
|
||||
expected_secret: &str,
|
||||
secret_key: &str,
|
||||
) -> Result<(OffsetDateTime, rustfs_madmin::AccountStatus, usize)> {
|
||||
self.rotate_user_secret_key_checked(access_key, secret_key, Some(expected_secret))
|
||||
self.rotate_user_secret_key_checked(access_key, secret_key, Some(expected_secret), None)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Change a local IAM password using an authenticated STS session and its parent password.
|
||||
///
|
||||
/// The parent and session mutation locks protect revalidation through the secret write.
|
||||
/// Persistence and session revocation continue after request cancellation.
|
||||
/// A revocation failure is returned after the new secret has been persisted.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns `IAMActionNotAllowed` for inactive sessions, parent mismatches, or an
|
||||
/// incorrect current password. Storage and revocation errors are propagated.
|
||||
pub async fn change_own_secret_key_from_session(
|
||||
&self,
|
||||
access_key: &str,
|
||||
expected_secret: &str,
|
||||
secret_key: &str,
|
||||
session: &Credentials,
|
||||
) -> Result<(OffsetDateTime, rustfs_madmin::AccountStatus, usize)> {
|
||||
self.rotate_user_secret_key_checked(access_key, secret_key, Some(expected_secret), Some(session))
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -1442,16 +1462,18 @@ impl<T: Store> IamSys<T> {
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
expected_secret: Option<&str>,
|
||||
session: Option<&Credentials>,
|
||||
) -> Result<(OffsetDateTime, rustfs_madmin::AccountStatus, usize)> {
|
||||
let runtime = tokio::runtime::Handle::try_current().map_err(Error::other)?;
|
||||
let iam = self.clone();
|
||||
let access_key = access_key.to_string();
|
||||
let secret_key = secret_key.to_string();
|
||||
let expected_secret = expected_secret.map(str::to_string);
|
||||
let session = session.cloned();
|
||||
#[cfg(test)]
|
||||
let notification_probe = crate::LOAD_USER_NOTIFICATION_PROBE.try_with(Arc::clone).ok();
|
||||
let operation = async move {
|
||||
iam.rotate_user_secret_key_inner(&access_key, &secret_key, expected_secret.as_deref())
|
||||
iam.rotate_user_secret_key_inner(&access_key, &secret_key, expected_secret.as_deref(), session.as_ref())
|
||||
.await
|
||||
};
|
||||
#[cfg(test)]
|
||||
@@ -1471,6 +1493,7 @@ impl<T: Store> IamSys<T> {
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
expected_secret: Option<&str>,
|
||||
session: Option<&Credentials>,
|
||||
) -> Result<(OffsetDateTime, rustfs_madmin::AccountStatus, usize)> {
|
||||
if !is_access_key_valid(access_key) {
|
||||
return Err(IamError::InvalidAccessKeyLength);
|
||||
@@ -1482,7 +1505,7 @@ impl<T: Store> IamSys<T> {
|
||||
|
||||
let (updated_at, status, parent_guard) = self
|
||||
.store
|
||||
.update_user_secret_key_with_guard(access_key, secret_key, expected_secret)
|
||||
.update_user_secret_key_with_guard(access_key, secret_key, expected_secret, session)
|
||||
.await?;
|
||||
let revoked = self.revoke_sts_sessions_for_parent(access_key).await?;
|
||||
drop(parent_guard);
|
||||
@@ -4815,6 +4838,160 @@ mod tests {
|
||||
assert!(matches!(issuer.check_key(&legacy.access_key).await, Ok((None, false))));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn own_secret_change_revalidates_the_presenting_session_under_the_parent_lock() {
|
||||
ensure_test_global_credentials();
|
||||
for mutation in ["revoke", "expire", "disable", "parent", "secret", "token", "generation"] {
|
||||
let store = StsTestMockStore::new(false);
|
||||
let iam = Arc::new(IamSys::new(IamCache::new(store.clone()).await.expect("IAM")));
|
||||
let parent = "session-change-parent";
|
||||
let password = "currentParentSecret123";
|
||||
iam.create_user(
|
||||
parent,
|
||||
&AddOrUpdateUserReq {
|
||||
secret_key: password.to_string(),
|
||||
policy: None,
|
||||
status: rustfs_madmin::AccountStatus::Enabled,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create parent");
|
||||
let session = rotation_test_session(parent, "password-session");
|
||||
let owner = iam.get_user(parent).await.expect("parent").credentials;
|
||||
iam.set_temp_user_from_parent(&session, &owner)
|
||||
.await
|
||||
.expect("publish session");
|
||||
|
||||
let guard = iam.store.cache.parent_mutation_lock(parent).lock_owned().await;
|
||||
let pending = {
|
||||
let iam = Arc::clone(&iam);
|
||||
let session = session.clone();
|
||||
tokio::spawn(async move {
|
||||
iam.change_own_secret_key_from_session(parent, password, "replacementParentSecret123", &session)
|
||||
.await
|
||||
})
|
||||
};
|
||||
if mutation == "revoke" {
|
||||
iam.delete_temp_account(&session.access_key, false)
|
||||
.await
|
||||
.expect("revoke presenting session");
|
||||
} else {
|
||||
let mut saved = store.saved_sts_users.lock().expect("saved identities");
|
||||
let current = saved.get_mut(&session.access_key).expect("durable session");
|
||||
match mutation {
|
||||
"expire" => current.credentials.expiration = Some(OffsetDateTime::UNIX_EPOCH),
|
||||
"disable" => current.credentials.status = "off".to_string(),
|
||||
"parent" => current.credentials.parent_user = "another-parent".to_string(),
|
||||
"secret" => current.credentials.secret_key = "replacementSessionSecret123".to_string(),
|
||||
"token" => current.credentials.session_token = "replacement-token".to_string(),
|
||||
"generation" => {
|
||||
assert!(current.sts_generation.take().is_some());
|
||||
}
|
||||
_ => unreachable!("known fixture mutation"),
|
||||
}
|
||||
}
|
||||
drop(guard);
|
||||
let error = tokio::time::timeout(std::time::Duration::from_secs(5), pending)
|
||||
.await
|
||||
.expect("queued rotation finishes")
|
||||
.expect("rotation task")
|
||||
.expect_err("stale session must fail");
|
||||
assert_eq!(error, Error::IAMActionNotAllowed, "{mutation}");
|
||||
let saved = store.saved_sts_users.lock().expect("saved identities");
|
||||
assert_eq!(saved.get(parent).expect("parent survives").credentials.secret_key, password, "{mutation}");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn session_secret_change_uses_the_durable_parent_password_with_a_stale_cache() {
|
||||
let iam = test_iam_sys().await;
|
||||
let parent = "stale-password-parent";
|
||||
iam.create_user(
|
||||
parent,
|
||||
&AddOrUpdateUserReq {
|
||||
secret_key: "cachedParentSecret123".to_string(),
|
||||
policy: None,
|
||||
status: rustfs_madmin::AccountStatus::Enabled,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create parent");
|
||||
let session = rotation_test_session(parent, "fresh-password-session");
|
||||
let owner = iam.get_user(parent).await.expect("cached parent").credentials;
|
||||
iam.set_temp_user_from_parent(&session, &owner)
|
||||
.await
|
||||
.expect("publish session");
|
||||
iam.store
|
||||
.api
|
||||
.saved_sts_users
|
||||
.lock()
|
||||
.expect("saved identities")
|
||||
.get_mut(parent)
|
||||
.expect("durable parent")
|
||||
.credentials
|
||||
.secret_key = "durableParentSecret123".to_string();
|
||||
assert_eq!(
|
||||
iam.store
|
||||
.get_user(parent)
|
||||
.await
|
||||
.expect("stale cached parent")
|
||||
.credentials
|
||||
.secret_key,
|
||||
"cachedParentSecret123"
|
||||
);
|
||||
let probe = IamSys::<StsTestMockStore>::load_user_notification_probe(0, false, false);
|
||||
crate::LOAD_USER_NOTIFICATION_PROBE
|
||||
.scope(
|
||||
probe,
|
||||
iam.change_own_secret_key_from_session(parent, "durableParentSecret123", "replacementParentSecret123", &session),
|
||||
)
|
||||
.await
|
||||
.expect("durable current password authorizes the change");
|
||||
assert_eq!(
|
||||
iam.get_user(parent).await.expect("updated parent").credentials.secret_key,
|
||||
"replacementParentSecret123"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn current_session_can_change_its_parent_secret_and_revoke_itself() {
|
||||
let iam = test_iam_sys().await;
|
||||
let parent = "current-session-parent";
|
||||
let password = "currentParentSecret123";
|
||||
iam.create_user(
|
||||
parent,
|
||||
&AddOrUpdateUserReq {
|
||||
secret_key: password.to_string(),
|
||||
policy: None,
|
||||
status: rustfs_madmin::AccountStatus::Enabled,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create parent");
|
||||
let session = rotation_test_session(parent, "current-password-session");
|
||||
let owner = iam.get_user(parent).await.expect("parent").credentials;
|
||||
iam.set_temp_user_from_parent(&session, &owner)
|
||||
.await
|
||||
.expect("publish session");
|
||||
let probe = IamSys::<StsTestMockStore>::load_user_notification_probe(0, false, false);
|
||||
let (_, _, revoked) = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(5),
|
||||
crate::LOAD_USER_NOTIFICATION_PROBE.scope(
|
||||
probe,
|
||||
iam.change_own_secret_key_from_session(parent, password, "replacementParentSecret123", &session),
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("rotation must release its session lock before revocation")
|
||||
.expect("rotate current session");
|
||||
assert_eq!(revoked, 1);
|
||||
assert!(iam.get_user(&session.access_key).await.is_none());
|
||||
assert_eq!(
|
||||
iam.get_user(parent).await.expect("rotated parent").credentials.secret_key,
|
||||
"replacementParentSecret123"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn self_secret_change_cannot_undo_concurrent_disable_rotation_or_deletion() {
|
||||
ensure_test_global_credentials();
|
||||
|
||||
@@ -29,11 +29,12 @@ workspace = true
|
||||
|
||||
[features]
|
||||
default = []
|
||||
hotpath = ["rustfs-io-metrics/hotpath"]
|
||||
hotpath-alloc = ["hotpath", "rustfs-io-metrics/hotpath-alloc"]
|
||||
hotpath-cpu = ["hotpath", "rustfs-io-metrics/hotpath-cpu"]
|
||||
hotpath = ["hotpath/hotpath", "hotpath/tokio", "rustfs-io-metrics/hotpath"]
|
||||
hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc", "rustfs-io-metrics/hotpath-alloc"]
|
||||
hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu", "rustfs-io-metrics/hotpath-cpu"]
|
||||
|
||||
[dependencies]
|
||||
hotpath.workspace = true
|
||||
bytes = { workspace = true, features = ["serde"] }
|
||||
thiserror = { workspace = true }
|
||||
tokio = { workspace = true, features = ["io-util", "fs", "sync", "rt-multi-thread"] }
|
||||
|
||||
@@ -410,10 +410,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
|
||||
}
|
||||
};
|
||||
let handle = self.allocate_handle(HandleState::Dir(cursor))?;
|
||||
Ok(Handle {
|
||||
id,
|
||||
handle: handle.into(),
|
||||
})
|
||||
Ok(Handle { id, handle })
|
||||
}
|
||||
|
||||
/// READDIR body shared with the Handler trait wrapper. Removes the
|
||||
@@ -669,11 +666,8 @@ mod tests {
|
||||
assert_eq!(auth.continuation_token, call.continuation_token);
|
||||
}
|
||||
assert_eq!(authorized[2].continuation_token.as_ref().map(|value| value.as_str()), Some("next-page"));
|
||||
let denied = with_test_auth_override(
|
||||
|_, _, prefix| prefix != Some("private/"),
|
||||
driver.readdir_inner(2, String::from_utf8(handle.handle.into_bytes()).expect("utf8 handle")),
|
||||
)
|
||||
.await;
|
||||
let denied =
|
||||
with_test_auth_override(|_, _, prefix| prefix != Some("private/"), driver.readdir_inner(2, handle.handle)).await;
|
||||
assert!(denied.is_err(), "policy change must deny the next READDIR");
|
||||
assert_eq!(backend.list_objects_calls().len(), 2, "denied READDIR cannot reach storage");
|
||||
}
|
||||
@@ -722,7 +716,7 @@ mod tests {
|
||||
};
|
||||
let handle_id = driver.allocate_handle(HandleState::Dir(cursor)).expect("allocate");
|
||||
|
||||
let readdir_fut = driver.readdir(1, handle_id.clone().into());
|
||||
let readdir_fut = driver.readdir(1, handle_id.clone());
|
||||
|
||||
with_test_auth_override(|_, _, _| true, async {
|
||||
tokio::select! {
|
||||
@@ -764,7 +758,7 @@ mod tests {
|
||||
// token, no skipped entries).
|
||||
backend.clear_stall_list_objects_v2();
|
||||
backend.queue_list_objects_v2_ok_empty();
|
||||
let result = with_test_auth_override(|_, _, _| true, driver.readdir(2, handle_id.into())).await;
|
||||
let result = with_test_auth_override(|_, _, _| true, driver.readdir(2, handle_id)).await;
|
||||
// Empty page returns Eof per readdir's empty-Name-to-Eof translation.
|
||||
let err = result.expect_err("re-issued READDIR against an empty listing must return Eof, not Ok");
|
||||
assert!(
|
||||
@@ -791,7 +785,7 @@ mod tests {
|
||||
let handle_id = driver.allocate_handle(HandleState::Dir(cursor)).expect("allocate");
|
||||
|
||||
let (result, captured) =
|
||||
capture_tracing_at(Level::ERROR, with_test_auth_override(|_, _, _| true, driver.readdir(7, handle_id.into()))).await;
|
||||
capture_tracing_at(Level::ERROR, with_test_auth_override(|_, _, _| true, driver.readdir(7, handle_id))).await;
|
||||
let err = result.expect_err("exhausted cursor must return Eof");
|
||||
assert!(matches!(StatusCode::from(err), StatusCode::Eof));
|
||||
assert!(
|
||||
@@ -824,7 +818,7 @@ mod tests {
|
||||
let handle_id = driver.allocate_handle(HandleState::Dir(cursor)).expect("allocate");
|
||||
|
||||
let (result, captured) =
|
||||
capture_tracing_at(Level::ERROR, with_test_auth_override(|_, _, _| true, driver.readdir(8, handle_id.into()))).await;
|
||||
capture_tracing_at(Level::ERROR, with_test_auth_override(|_, _, _| true, driver.readdir(8, handle_id))).await;
|
||||
let err = result.expect_err("backend error must propagate as Err");
|
||||
assert!(!matches!(StatusCode::from(err), StatusCode::Eof), "backend error must not be Eof");
|
||||
assert!(
|
||||
|
||||
@@ -33,9 +33,7 @@ use super::write::{build_write_tombstone, fstat_reported_size, rejects_excl_or_t
|
||||
use crate::common::client::s3::StorageBackend;
|
||||
use crate::common::gateway::{AuthorizationError, S3Action};
|
||||
use crate::common::session::SessionContext;
|
||||
use russh_sftp::protocol::{
|
||||
Attrs, Data, File, FileAttributes, Handle, Name, OpenFlags, Packet, SftpHandle, Status, StatusCode, Version,
|
||||
};
|
||||
use russh_sftp::protocol::{Attrs, Data, File, FileAttributes, Handle, Name, OpenFlags, Packet, Status, StatusCode, Version};
|
||||
use rustfs_credentials::Credentials;
|
||||
use rustfs_gateway_types::dto::{AbortMultipartUploadInput, CopyObjectInput, ListObjectsV2Input};
|
||||
use rustfs_gateway_types::{BucketName, ObjectKey, UploadIdClaim};
|
||||
@@ -210,12 +208,6 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
|
||||
}
|
||||
}
|
||||
|
||||
/// russh-sftp 4 hands the Handler opaque `SftpHandle` bytes; the table
|
||||
/// keys remain the UTF-8 UUID strings this driver mints.
|
||||
fn handle_table_key(handle: SftpHandle) -> Result<String, SftpError> {
|
||||
String::from_utf8(handle.into_bytes()).map_err(|_| SftpError::code(StatusCode::Failure))
|
||||
}
|
||||
|
||||
/// Generate a fresh UUID v4 handle, insert the given state into the
|
||||
/// per-session handle table, and return the handle string. Enforces
|
||||
/// self.handles_per_session before any UUID generation. Cap-exceeded
|
||||
@@ -502,9 +494,8 @@ impl<S: StorageBackend + Send + Sync + 'static> russh_sftp::server::Handler for
|
||||
/// Streaming returns (next_part_number - 1) * part_size + buffer
|
||||
/// length. Failed returns the size recorded at the most recent
|
||||
/// successful write. An unknown handle returns Failure.
|
||||
#[tracing::instrument(level = "debug", skip(self), fields(id, handle = ?handle), err(Debug))]
|
||||
async fn fstat(&mut self, id: u32, handle: SftpHandle) -> Result<Attrs, Self::Error> {
|
||||
let handle = Self::handle_table_key(handle)?;
|
||||
#[tracing::instrument(level = "debug", skip(self), fields(id, handle = %handle), err(Debug))]
|
||||
async fn fstat(&mut self, id: u32, handle: String) -> Result<Attrs, Self::Error> {
|
||||
self.session_diag.stamp();
|
||||
let part_size = self.part_size;
|
||||
let result = self.with_handle_ref(&handle, |state| match state {
|
||||
@@ -550,9 +541,8 @@ impl<S: StorageBackend + Send + Sync + 'static> russh_sftp::server::Handler for
|
||||
/// exhaustion, so it is normal control flow on this handler. The
|
||||
/// instrument attribute therefore omits err(Debug) and non-Eof
|
||||
/// failures are surfaced via the explicit error log below.
|
||||
#[tracing::instrument(level = "debug", skip(self), fields(id, handle = ?handle))]
|
||||
async fn readdir(&mut self, id: u32, handle: SftpHandle) -> Result<Name, Self::Error> {
|
||||
let handle = Self::handle_table_key(handle)?;
|
||||
#[tracing::instrument(level = "debug", skip(self), fields(id, handle = %handle))]
|
||||
async fn readdir(&mut self, id: u32, handle: String) -> Result<Name, Self::Error> {
|
||||
self.session_diag.stamp();
|
||||
let result = self.readdir_inner(id, handle.clone()).await;
|
||||
self.session_diag.stamp();
|
||||
@@ -627,9 +617,8 @@ impl<S: StorageBackend + Send + Sync + 'static> russh_sftp::server::Handler for
|
||||
/// read-past-end-of-file, so it is normal control flow on this
|
||||
/// handler. The instrument attribute therefore omits err(Debug) and
|
||||
/// non-Eof failures are surfaced via the explicit error log below.
|
||||
#[tracing::instrument(level = "debug", skip(self), fields(id, handle = ?handle, offset, len))]
|
||||
async fn read(&mut self, id: u32, handle: SftpHandle, offset: u64, len: u32) -> Result<Data, Self::Error> {
|
||||
let handle = Self::handle_table_key(handle)?;
|
||||
#[tracing::instrument(level = "debug", skip(self), fields(id, handle = %handle, offset, len))]
|
||||
async fn read(&mut self, id: u32, handle: String, offset: u64, len: u32) -> Result<Data, Self::Error> {
|
||||
self.session_diag.stamp();
|
||||
let result = self.read_inner(id, handle.clone(), offset, len).await;
|
||||
self.session_diag.stamp();
|
||||
@@ -661,9 +650,8 @@ impl<S: StorageBackend + Send + Sync + 'static> russh_sftp::server::Handler for
|
||||
///
|
||||
/// A missing handle is treated as Ok to tolerate clients that
|
||||
/// double-close on session teardown.
|
||||
#[tracing::instrument(level = "info", skip(self), fields(id, handle = ?handle), err(Debug))]
|
||||
async fn close(&mut self, id: u32, handle: SftpHandle) -> Result<Status, Self::Error> {
|
||||
let handle = Self::handle_table_key(handle)?;
|
||||
#[tracing::instrument(level = "info", skip(self), fields(id, handle = %handle), err(Debug))]
|
||||
async fn close(&mut self, id: u32, handle: String) -> Result<Status, Self::Error> {
|
||||
self.session_diag.stamp();
|
||||
let removed = self.handles.remove(&handle);
|
||||
let Some(HandleState::Write {
|
||||
@@ -767,9 +755,8 @@ impl<S: StorageBackend + Send + Sync + 'static> russh_sftp::server::Handler for
|
||||
/// to Streaming by issuing CreateMultipartUpload. A Failed handle
|
||||
/// rejects every subsequent write with the same status that caused
|
||||
/// the failure.
|
||||
#[tracing::instrument(level = "debug", skip(self, data), fields(id, handle = ?handle, offset, len = data.len()), err(Debug))]
|
||||
async fn write(&mut self, id: u32, handle: SftpHandle, offset: u64, data: Vec<u8>) -> Result<Status, Self::Error> {
|
||||
let handle = Self::handle_table_key(handle)?;
|
||||
#[tracing::instrument(level = "debug", skip(self, data), fields(id, handle = %handle, offset, len = data.len()), err(Debug))]
|
||||
async fn write(&mut self, id: u32, handle: String, offset: u64, data: Vec<u8>) -> Result<Status, Self::Error> {
|
||||
self.session_diag.stamp();
|
||||
self.enforce_server_readonly()?;
|
||||
|
||||
@@ -996,8 +983,8 @@ impl<S: StorageBackend + Send + Sync + 'static> russh_sftp::server::Handler for
|
||||
/// SSH_FXP_FSETSTAT, SFTP Internet Draft section 6.6. Same rationale
|
||||
/// as setstat: S3 cannot honour POSIX attributes, and clients use
|
||||
/// FSETSTAT during transfers to stamp the in-flight handle.
|
||||
#[tracing::instrument(level = "debug", skip(self, _attrs), fields(id, handle = ?_handle), err(Debug))]
|
||||
async fn fsetstat(&mut self, id: u32, _handle: SftpHandle, _attrs: FileAttributes) -> Result<Status, Self::Error> {
|
||||
#[tracing::instrument(level = "debug", skip(self, _attrs), fields(id, handle = %_handle), err(Debug))]
|
||||
async fn fsetstat(&mut self, id: u32, _handle: String, _attrs: FileAttributes) -> Result<Status, Self::Error> {
|
||||
self.enforce_server_readonly()?;
|
||||
Ok(ok_status(id))
|
||||
}
|
||||
@@ -1267,7 +1254,7 @@ mod tests {
|
||||
let handle_id = driver
|
||||
.allocate_handle(file_handle("bucket", "k", 1234, attrs.clone()))
|
||||
.expect("allocate");
|
||||
let out = driver.fstat(4, handle_id.into()).await.expect("fstat on File must succeed");
|
||||
let out = driver.fstat(4, handle_id).await.expect("fstat on File must succeed");
|
||||
assert_eq!(out.attrs.size, Some(1234));
|
||||
assert_eq!(out.attrs.mtime, Some(1_700_000_000));
|
||||
}
|
||||
@@ -1280,7 +1267,7 @@ mod tests {
|
||||
part_buffer: vec![0u8; 4096],
|
||||
};
|
||||
let handle_id = driver.allocate_handle(write_handle("bucket", "k", phase)).expect("allocate");
|
||||
let out = driver.fstat(5, handle_id.into()).await.expect("fstat on Write must succeed");
|
||||
let out = driver.fstat(5, handle_id).await.expect("fstat on Write must succeed");
|
||||
assert_eq!(
|
||||
out.attrs.size,
|
||||
Some(4096),
|
||||
@@ -1296,7 +1283,7 @@ mod tests {
|
||||
.allocate_handle(file_handle("bucket", "k", 0, FileAttributes::default()))
|
||||
.expect("allocate");
|
||||
let status = driver
|
||||
.fsetstat(6, handle_id.into(), FileAttributes::default())
|
||||
.fsetstat(6, handle_id, FileAttributes::default())
|
||||
.await
|
||||
.expect("fsetstat must succeed on any attrs");
|
||||
assert!(matches!(status.status_code, StatusCode::Ok));
|
||||
@@ -1430,7 +1417,7 @@ mod tests {
|
||||
let handle_id = driver
|
||||
.allocate_handle(file_handle("bucket", "k", 0, FileAttributes::default()))
|
||||
.expect("allocate");
|
||||
let result = driver.fsetstat(10, handle_id.into(), FileAttributes::default()).await;
|
||||
let result = driver.fsetstat(10, handle_id, FileAttributes::default()).await;
|
||||
match result {
|
||||
Err(err) => assert!(matches!(err.0, StatusCode::PermissionDenied)),
|
||||
Ok(_) => panic!("fsetstat must error in read-only mode"),
|
||||
|
||||
@@ -69,10 +69,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
|
||||
attrs,
|
||||
read_cache,
|
||||
})?;
|
||||
Ok(Handle {
|
||||
id,
|
||||
handle: handle.into(),
|
||||
})
|
||||
Ok(Handle { id, handle })
|
||||
}
|
||||
|
||||
/// Body of the SSH_FXP_READ handler. Returns up to len bytes starting
|
||||
@@ -259,7 +256,7 @@ mod tests {
|
||||
.allocate_handle(file_handle("bucket", "secret.txt", 4096, FileAttributes::default()))
|
||||
.expect("allocate read handle");
|
||||
driver.try_populate_read_cache(&handle, 0, vec![42; 4096]);
|
||||
let error = with_test_auth_override(|_, _, _| true, driver.read(1, handle.into(), 0, 1024))
|
||||
let error = with_test_auth_override(|_, _, _| true, driver.read(1, handle, 0, 1024))
|
||||
.await
|
||||
.expect_err("backend policy denial must prevent cached content disclosure");
|
||||
assert!(matches!(StatusCode::from(error), StatusCode::PermissionDenied));
|
||||
@@ -273,7 +270,7 @@ mod tests {
|
||||
.allocate_handle(file_handle("bucket", "k", 100, FileAttributes::default()))
|
||||
.expect("allocate");
|
||||
let err = driver
|
||||
.read(1, handle_id.into(), 0, 0)
|
||||
.read(1, handle_id, 0, 0)
|
||||
.await
|
||||
.expect_err("len=0 must return BadMessage");
|
||||
assert!(matches!(StatusCode::from(err), StatusCode::BadMessage));
|
||||
@@ -287,7 +284,7 @@ mod tests {
|
||||
.allocate_handle(file_handle("bucket", "k", 10, FileAttributes::default()))
|
||||
.expect("allocate");
|
||||
let err = driver
|
||||
.read(2, handle_id.into(), 10, 4)
|
||||
.read(2, handle_id, 10, 4)
|
||||
.await
|
||||
.expect_err("offset==size must return Eof");
|
||||
assert!(matches!(StatusCode::from(err), StatusCode::Eof));
|
||||
@@ -302,7 +299,7 @@ mod tests {
|
||||
.allocate_handle(file_handle("bucket", "k", 5, FileAttributes::default()))
|
||||
.expect("allocate");
|
||||
|
||||
let data = with_test_auth_override(|_, _, _| true, driver.read(3, handle_id.into(), 0, 1024))
|
||||
let data = with_test_auth_override(|_, _, _| true, driver.read(3, handle_id, 0, 1024))
|
||||
.await
|
||||
.expect("read must succeed");
|
||||
assert_eq!(data.data, b"hello".to_vec());
|
||||
@@ -320,7 +317,7 @@ mod tests {
|
||||
.allocate_handle(file_handle("bucket", "k", 10, FileAttributes::default()))
|
||||
.expect("allocate");
|
||||
|
||||
let (result, captured) = capture_tracing_at(Level::ERROR, async { driver.read(11, handle_id.into(), 10, 4).await }).await;
|
||||
let (result, captured) = capture_tracing_at(Level::ERROR, async { driver.read(11, handle_id, 10, 4).await }).await;
|
||||
let err = result.expect_err("offset==size must return Eof");
|
||||
assert!(matches!(StatusCode::from(err), StatusCode::Eof));
|
||||
assert!(
|
||||
@@ -345,11 +342,8 @@ mod tests {
|
||||
.allocate_handle(file_handle("bucket", "k", 1024, FileAttributes::default()))
|
||||
.expect("allocate");
|
||||
|
||||
let (result, captured) = capture_tracing_at(
|
||||
Level::ERROR,
|
||||
with_test_auth_override(|_, _, _| true, driver.read(12, handle_id.into(), 0, 256)),
|
||||
)
|
||||
.await;
|
||||
let (result, captured) =
|
||||
capture_tracing_at(Level::ERROR, with_test_auth_override(|_, _, _| true, driver.read(12, handle_id, 0, 256))).await;
|
||||
let err = result.expect_err("backend error must propagate as Err");
|
||||
assert!(!matches!(StatusCode::from(err), StatusCode::Eof), "backend error must not be Eof");
|
||||
assert!(
|
||||
@@ -385,7 +379,7 @@ mod tests {
|
||||
let start = Instant::now();
|
||||
let outcome = tokio::time::timeout(
|
||||
Duration::from_secs(10),
|
||||
with_test_auth_override(|_, _, _| true, driver.read(14, handle_id.into(), 0, 4096)),
|
||||
with_test_auth_override(|_, _, _| true, driver.read(14, handle_id, 0, 4096)),
|
||||
)
|
||||
.await;
|
||||
let elapsed = start.elapsed();
|
||||
@@ -431,7 +425,7 @@ mod tests {
|
||||
let mut assembled: Vec<u8> = Vec::with_capacity(object_size as usize);
|
||||
let mut reads: u32 = 0;
|
||||
while offset < object_size {
|
||||
let data = with_test_auth_override(|_, _, _| true, driver.read(20 + reads, handle_id.clone().into(), offset, chunk))
|
||||
let data = with_test_auth_override(|_, _, _| true, driver.read(20 + reads, handle_id.clone(), offset, chunk))
|
||||
.await
|
||||
.expect("read inside the cached window must succeed without a second backend call");
|
||||
assert!(!data.data.is_empty(), "non-empty hit");
|
||||
@@ -467,21 +461,21 @@ mod tests {
|
||||
.expect("allocate");
|
||||
|
||||
// First read fetches from the backend and populates window 1.
|
||||
let r1 = with_test_auth_override(|_, _, _| true, driver.read(30, handle_id.clone().into(), 0, 1024))
|
||||
let r1 = with_test_auth_override(|_, _, _| true, driver.read(30, handle_id.clone(), 0, 1024))
|
||||
.await
|
||||
.expect("first read must succeed");
|
||||
assert!(r1.data.iter().all(|b| *b == 0xAA), "first read must come from window 1");
|
||||
|
||||
// Second read inside the cached chunk is served from the
|
||||
// buffer. No second backend call yet.
|
||||
let r2 = with_test_auth_override(|_, _, _| true, driver.read(31, handle_id.clone().into(), 1024, 1024))
|
||||
let r2 = with_test_auth_override(|_, _, _| true, driver.read(31, handle_id.clone(), 1024, 1024))
|
||||
.await
|
||||
.expect("mid-window read must succeed from cache");
|
||||
assert!(r2.data.iter().all(|b| *b == 0xAA), "mid-window read still in window 1");
|
||||
|
||||
// Reading at offset == window falls outside the cached chunk
|
||||
// and triggers the second backend call.
|
||||
let r3 = with_test_auth_override(|_, _, _| true, driver.read(32, handle_id.clone().into(), window, 1024))
|
||||
let r3 = with_test_auth_override(|_, _, _| true, driver.read(32, handle_id.clone(), window, 1024))
|
||||
.await
|
||||
.expect("read at offset=window must succeed via second backend call");
|
||||
assert!(r3.data.iter().all(|b| *b == 0xBB), "read at window boundary must come from window 2");
|
||||
@@ -489,7 +483,7 @@ mod tests {
|
||||
// A read inside the second cached chunk is served from the
|
||||
// buffer. The queue is empty by now, so any third backend
|
||||
// call would fail.
|
||||
let r4 = with_test_auth_override(|_, _, _| true, driver.read(33, handle_id.into(), window + 1024, 1024))
|
||||
let r4 = with_test_auth_override(|_, _, _| true, driver.read(33, handle_id, window + 1024, 1024))
|
||||
.await
|
||||
.expect("mid-window-2 read must succeed from cache");
|
||||
assert!(r4.data.iter().all(|b| *b == 0xBB), "mid-window-2 read still in window 2");
|
||||
@@ -517,13 +511,13 @@ mod tests {
|
||||
.expect("allocate");
|
||||
|
||||
// Populate window 1 with a full read.
|
||||
let _ = with_test_auth_override(|_, _, _| true, driver.read(40, handle_id.clone().into(), 0, window as u32))
|
||||
let _ = with_test_auth_override(|_, _, _| true, driver.read(40, handle_id.clone(), 0, window as u32))
|
||||
.await
|
||||
.expect("populate window 1");
|
||||
|
||||
// Ask for 256 bytes starting 64 bytes before window end. Only
|
||||
// 64 bytes are in the window. The driver must return 64.
|
||||
let edge = with_test_auth_override(|_, _, _| true, driver.read(41, handle_id.into(), window - 64, 256))
|
||||
let edge = with_test_auth_override(|_, _, _| true, driver.read(41, handle_id, window - 64, 256))
|
||||
.await
|
||||
.expect("partial-hit read must succeed");
|
||||
assert_eq!(edge.data.len(), 64, "partial hit must return only the in-window portion");
|
||||
@@ -558,10 +552,9 @@ mod tests {
|
||||
|
||||
for i in 0..read_count {
|
||||
let offset = (chunk_size as u64) * (i as u64);
|
||||
let data =
|
||||
with_test_auth_override(|_, _, _| true, driver.read(50 + i, handle_id.clone().into(), offset, chunk_size as u32))
|
||||
.await
|
||||
.expect("each read must succeed via the backend");
|
||||
let data = with_test_auth_override(|_, _, _| true, driver.read(50 + i, handle_id.clone(), offset, chunk_size as u32))
|
||||
.await
|
||||
.expect("each read must succeed via the backend");
|
||||
assert_eq!(data.data.len(), chunk_size, "read must return full requested length");
|
||||
let expected_byte = (i + 1) as u8;
|
||||
assert!(
|
||||
|
||||
@@ -336,10 +336,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
|
||||
open_attrs,
|
||||
phase: WritePhase::Buffering { part_buffer: Vec::new() },
|
||||
})?;
|
||||
Ok(Handle {
|
||||
id,
|
||||
handle: handle.into(),
|
||||
})
|
||||
Ok(Handle { id, handle })
|
||||
}
|
||||
|
||||
/// Upload the buffered bytes to S3 with a single PutObject call.
|
||||
@@ -2108,7 +2105,7 @@ mod tests {
|
||||
)
|
||||
.await
|
||||
.expect("WRITE | CREATE | TRUNCATE on a missing file must allocate a handle");
|
||||
assert!(!handle.handle.as_bytes().is_empty());
|
||||
assert!(!handle.handle.is_empty());
|
||||
assert!(backend.head_object_calls().is_empty(), "the non-EXCL accept path must not HEAD");
|
||||
}
|
||||
|
||||
@@ -2133,7 +2130,7 @@ mod tests {
|
||||
)
|
||||
.await
|
||||
.expect("WRITE | CREATE | TRUNCATE on an existing file must allocate a handle");
|
||||
assert!(!handle.handle.as_bytes().is_empty());
|
||||
assert!(!handle.handle.is_empty());
|
||||
assert!(backend.head_object_calls().is_empty(), "the non-EXCL accept path must not HEAD");
|
||||
}
|
||||
|
||||
@@ -2180,7 +2177,7 @@ mod tests {
|
||||
)
|
||||
.await
|
||||
.expect("EXCL on a missing key must allow creation");
|
||||
assert!(!handle.handle.as_bytes().is_empty());
|
||||
assert!(!handle.handle.is_empty());
|
||||
assert_eq!(backend.head_object_calls().len(), 1, "EXCL must HEAD exactly once");
|
||||
}
|
||||
|
||||
|
||||
@@ -38,6 +38,8 @@ default = []
|
||||
# event. Requires `RUSTFLAGS="--cfg hyper_unstable_tracing"`.
|
||||
diagnostic-internode-http2-ping = ["dep:hyper", "rustfs-io-metrics/diagnostic-internode-http2-ping"]
|
||||
hotpath = [
|
||||
"hotpath/hotpath",
|
||||
"hotpath/tokio",
|
||||
"rustfs-common/hotpath",
|
||||
"rustfs-config/hotpath",
|
||||
"rustfs-io-metrics/hotpath",
|
||||
@@ -46,6 +48,7 @@ hotpath = [
|
||||
]
|
||||
hotpath-alloc = [
|
||||
"hotpath",
|
||||
"hotpath/hotpath-alloc",
|
||||
"rustfs-common/hotpath-alloc",
|
||||
"rustfs-config/hotpath-alloc",
|
||||
"rustfs-io-metrics/hotpath-alloc",
|
||||
@@ -54,6 +57,7 @@ hotpath-alloc = [
|
||||
]
|
||||
hotpath-cpu = [
|
||||
"hotpath",
|
||||
"hotpath/hotpath-cpu",
|
||||
"rustfs-common/hotpath-cpu",
|
||||
"rustfs-config/hotpath-cpu",
|
||||
"rustfs-io-metrics/hotpath-cpu",
|
||||
@@ -62,6 +66,7 @@ hotpath-cpu = [
|
||||
]
|
||||
|
||||
[dependencies]
|
||||
hotpath.workspace = true
|
||||
rustfs-common.workspace = true
|
||||
rustfs-heal-contracts.workspace = true
|
||||
rustfs-io-metrics.workspace = true
|
||||
|
||||
@@ -31,24 +31,29 @@ workspace = true
|
||||
[features]
|
||||
default = []
|
||||
hotpath = [
|
||||
"hotpath/hotpath",
|
||||
"hotpath/tokio",
|
||||
"rustfs-rio/hotpath",
|
||||
"rustfs-utils/hotpath",
|
||||
"rustfs-filemeta/hotpath",
|
||||
]
|
||||
hotpath-alloc = [
|
||||
"hotpath",
|
||||
"hotpath/hotpath-alloc",
|
||||
"rustfs-rio/hotpath-alloc",
|
||||
"rustfs-utils/hotpath-alloc",
|
||||
"rustfs-filemeta/hotpath-alloc",
|
||||
]
|
||||
hotpath-cpu = [
|
||||
"hotpath",
|
||||
"hotpath/hotpath-cpu",
|
||||
"rustfs-rio/hotpath-cpu",
|
||||
"rustfs-utils/hotpath-cpu",
|
||||
"rustfs-filemeta/hotpath-cpu",
|
||||
]
|
||||
|
||||
[dependencies]
|
||||
hotpath.workspace = true
|
||||
aes-gcm = { workspace = true, features = ["rand_core"] }
|
||||
bytes = { workspace = true, features = ["serde"] }
|
||||
chacha20poly1305.workspace = true
|
||||
|
||||
@@ -32,7 +32,6 @@ workspace = true
|
||||
internode-transport = []
|
||||
default = []
|
||||
hotpath = [
|
||||
"dep:hotpath",
|
||||
"hotpath/hotpath",
|
||||
"hotpath/tokio",
|
||||
"hotpath/futures",
|
||||
@@ -60,8 +59,7 @@ hotpath-cpu = [
|
||||
]
|
||||
|
||||
[dependencies]
|
||||
# Feature-gated in src (measure_block!); optional so default builds do not warn.
|
||||
hotpath = { workspace = true, optional = true }
|
||||
hotpath.workspace = true
|
||||
arc-swap.workspace = true
|
||||
tokio = { workspace = true, features = ["io-util", "macros", "net", "rt-multi-thread", "sync", "time"] }
|
||||
rand = { workspace = true, features = ["serde"] }
|
||||
@@ -81,6 +79,7 @@ rustfs-io-metrics.workspace = true
|
||||
rustfs-tls-runtime.workspace = true
|
||||
rustfs-utils = { workspace = true, features = ["io", "hash", "compress"] }
|
||||
serde_json = { workspace = true, features = ["raw_value"] }
|
||||
md-5 = { workspace = true }
|
||||
hmac.workspace = true
|
||||
minlz.workspace = true
|
||||
tracing.workspace = true
|
||||
@@ -92,7 +91,6 @@ xxhash-rust = { workspace = true, features = ["xxh64", "xxh3"] }
|
||||
hex-simd.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
md-5 = { workspace = true }
|
||||
faster-hex.workspace = true
|
||||
rcgen.workspace = true
|
||||
tokio-rustls.workspace = true
|
||||
|
||||
@@ -15,30 +15,6 @@
|
||||
use super::*;
|
||||
use crate::data_usage_define::{DATA_USAGE_OBJ_NAME_PATH, read_config_with_revision};
|
||||
|
||||
// Two-pool EC fixtures keep many scan futures live at once and overflow
|
||||
// Linux libtest's default 2 MiB test-thread stack. Run each case on a
|
||||
// dedicated 32 MiB thread (same approach as scoped_entry_fallback).
|
||||
fn run_cohort_test<F, Fut>(thread_name: &'static str, test_fn: F)
|
||||
where
|
||||
F: FnOnce() -> Fut + Send + 'static,
|
||||
Fut: std::future::Future<Output = ()> + 'static,
|
||||
{
|
||||
let handle = std::thread::Builder::new()
|
||||
.name(thread_name.to_string())
|
||||
.stack_size(32 * 1024 * 1024)
|
||||
.spawn(move || {
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("service cohort test runtime should build");
|
||||
runtime.block_on(test_fn());
|
||||
})
|
||||
.expect("service cohort test thread should spawn");
|
||||
if let Err(payload) = handle.join() {
|
||||
std::panic::resume_unwind(payload);
|
||||
}
|
||||
}
|
||||
|
||||
async fn create_cohort_bucket(store: &ECStore, bucket: &str) {
|
||||
create_cohort_bucket_objects(store, bucket, 1).await;
|
||||
}
|
||||
@@ -128,16 +104,9 @@ async fn run_cohort_cycle(
|
||||
(result, usage)
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
fn service_cohort_production_dispatch_services_waiters_across_sources() {
|
||||
run_cohort_test(
|
||||
"cohort-prod-disp",
|
||||
service_cohort_production_dispatch_services_waiters_across_sources_case,
|
||||
);
|
||||
}
|
||||
|
||||
async fn service_cohort_production_dispatch_services_waiters_across_sources_case() {
|
||||
async fn service_cohort_production_dispatch_services_waiters_across_sources() {
|
||||
let (_dir, store) = setup_two_pool_scanner_store().await;
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
let hot = format!("a-hot-{}", Uuid::new_v4().simple());
|
||||
@@ -228,16 +197,9 @@ async fn service_cohort_production_dispatch_services_waiters_across_sources_case
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
fn service_cohort_flat_bucket_budget_does_not_publish_unscanned_small_bucket() {
|
||||
run_cohort_test(
|
||||
"cohort-flat-bud",
|
||||
service_cohort_flat_bucket_budget_does_not_publish_unscanned_small_bucket_case,
|
||||
);
|
||||
}
|
||||
|
||||
async fn service_cohort_flat_bucket_budget_does_not_publish_unscanned_small_bucket_case() {
|
||||
async fn service_cohort_flat_bucket_budget_does_not_publish_unscanned_small_bucket() {
|
||||
let (_dir, store) = setup_two_pool_scanner_store().await;
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
let flat = format!("a-flat-{}", Uuid::new_v4().simple());
|
||||
@@ -321,16 +283,9 @@ async fn service_cohort_flat_bucket_budget_does_not_publish_unscanned_small_buck
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
fn service_cohort_fresh_complete_aggregate_preserves_reordered_sources() {
|
||||
run_cohort_test(
|
||||
"cohort-fresh-agg",
|
||||
service_cohort_fresh_complete_aggregate_preserves_reordered_sources_case,
|
||||
);
|
||||
}
|
||||
|
||||
async fn service_cohort_fresh_complete_aggregate_preserves_reordered_sources_case() {
|
||||
async fn service_cohort_fresh_complete_aggregate_preserves_reordered_sources() {
|
||||
let (_dir, store) = setup_two_pool_scanner_store().await;
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
for bucket in ["cohort-first", "cohort-second"] {
|
||||
@@ -374,16 +329,9 @@ async fn service_cohort_fresh_complete_aggregate_preserves_reordered_sources_cas
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
fn service_cohort_cancelled_dispatch_does_not_consume_waiters_or_leak_permits() {
|
||||
run_cohort_test(
|
||||
"cohort-cancelled",
|
||||
service_cohort_cancelled_dispatch_does_not_consume_waiters_or_leak_permits_case,
|
||||
);
|
||||
}
|
||||
|
||||
async fn service_cohort_cancelled_dispatch_does_not_consume_waiters_or_leak_permits_case() {
|
||||
async fn service_cohort_cancelled_dispatch_does_not_consume_waiters_or_leak_permits() {
|
||||
let (_dir, store) = setup_two_pool_scanner_store().await;
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
let bucket = format!("cancel-{}", Uuid::new_v4().simple());
|
||||
@@ -412,16 +360,9 @@ async fn service_cohort_cancelled_dispatch_does_not_consume_waiters_or_leak_perm
|
||||
/// N=17, q=4: every bucket reaches a worker within ceil(17/4)=5 rounds while
|
||||
/// one bucket stays hot and another fails forever, with leader-local state
|
||||
/// discarded after each round as a restart would.
|
||||
#[test]
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
fn cold_bucket_service_bound_survives_restart_with_hot_and_failing_bucket() {
|
||||
run_cohort_test(
|
||||
"cold-svc-bound",
|
||||
cold_bucket_service_bound_survives_restart_with_hot_and_failing_bucket_case,
|
||||
);
|
||||
}
|
||||
|
||||
async fn cold_bucket_service_bound_survives_restart_with_hot_and_failing_bucket_case() {
|
||||
async fn cold_bucket_service_bound_survives_restart_with_hot_and_failing_bucket() {
|
||||
const BUCKETS: usize = 17;
|
||||
const QUANTA: u64 = 4;
|
||||
const ROUND_BOUND: usize = 5;
|
||||
@@ -493,16 +434,9 @@ fn quantum_budget(quantum: crate::scanner_budget::ScannerBucketQuantum) -> Arc<S
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
fn source_turn_dispatch_rotates_real_sources_and_retains_completed_roots() {
|
||||
run_cohort_test(
|
||||
"src-turn-disp",
|
||||
source_turn_dispatch_rotates_real_sources_and_retains_completed_roots_case,
|
||||
);
|
||||
}
|
||||
|
||||
async fn source_turn_dispatch_rotates_real_sources_and_retains_completed_roots_case() {
|
||||
async fn source_turn_dispatch_rotates_real_sources_and_retains_completed_roots() {
|
||||
let mut evidence = None;
|
||||
temp_env::async_with_vars([(ENV_SCANNER_MAX_CONCURRENT_SET_SCANS, Some("1"))], async {
|
||||
crate::runtime_config::refresh_scanner_runtime_config_for_tests();
|
||||
@@ -569,16 +503,9 @@ async fn source_turn_dispatch_rotates_real_sources_and_retains_completed_roots_c
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
fn source_turn_zero_new_frontier_stops_requeuing_without_starving_other_sources() {
|
||||
run_cohort_test(
|
||||
"src-turn-zero",
|
||||
source_turn_zero_new_frontier_stops_requeuing_without_starving_other_sources_case,
|
||||
);
|
||||
}
|
||||
|
||||
async fn source_turn_zero_new_frontier_stops_requeuing_without_starving_other_sources_case() {
|
||||
async fn source_turn_zero_new_frontier_stops_requeuing_without_starving_other_sources() {
|
||||
temp_env::async_with_vars([(ENV_SCANNER_MAX_CONCURRENT_SET_SCANS, Some("1"))], async {
|
||||
crate::runtime_config::refresh_scanner_runtime_config_for_tests();
|
||||
let (_dir, store) = setup_two_pool_scanner_store().await;
|
||||
@@ -606,13 +533,9 @@ async fn source_turn_zero_new_frontier_stops_requeuing_without_starving_other_so
|
||||
crate::runtime_config::refresh_scanner_runtime_config_for_tests();
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
fn fair_prefix_quantum_spent_bucket_returns_behind_waiting_buckets() {
|
||||
run_cohort_test("fair-spent-qm", fair_prefix_quantum_spent_bucket_returns_behind_waiting_buckets_case);
|
||||
}
|
||||
|
||||
async fn fair_prefix_quantum_spent_bucket_returns_behind_waiting_buckets_case() {
|
||||
async fn fair_prefix_quantum_spent_bucket_returns_behind_waiting_buckets() {
|
||||
let (_dir, store) = setup_local_scanner_store(1, 4).await;
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
create_cohort_bucket_objects(&store, "a-flat", 6).await;
|
||||
@@ -642,13 +565,9 @@ async fn fair_prefix_quantum_spent_bucket_returns_behind_waiting_buckets_case()
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
fn fair_prefix_quantum_without_durable_motion_ends_the_bucket_turn() {
|
||||
run_cohort_test("fair-no-motion", fair_prefix_quantum_without_durable_motion_ends_the_bucket_turn_case);
|
||||
}
|
||||
|
||||
async fn fair_prefix_quantum_without_durable_motion_ends_the_bucket_turn_case() {
|
||||
async fn fair_prefix_quantum_without_durable_motion_ends_the_bucket_turn() {
|
||||
let (_dir, store) = setup_local_scanner_store(1, 4).await;
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
create_cohort_bucket_objects(&store, "a-flat", 6).await;
|
||||
@@ -671,16 +590,9 @@ async fn fair_prefix_quantum_without_durable_motion_ends_the_bucket_turn_case()
|
||||
|
||||
/// A one-object share stops inside its first child, so the frontier never
|
||||
/// moves. Requeuing that bucket would spin until the cycle deadline.
|
||||
#[test]
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
fn fair_prefix_quantum_repeated_share_without_a_new_frontier_stops_requeuing() {
|
||||
run_cohort_test(
|
||||
"fair-rep-share",
|
||||
fair_prefix_quantum_repeated_share_without_a_new_frontier_stops_requeuing_case,
|
||||
);
|
||||
}
|
||||
|
||||
async fn fair_prefix_quantum_repeated_share_without_a_new_frontier_stops_requeuing_case() {
|
||||
async fn fair_prefix_quantum_repeated_share_without_a_new_frontier_stops_requeuing() {
|
||||
let (_dir, store) = setup_local_scanner_store(1, 4).await;
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
create_cohort_bucket_objects(&store, "a-flat", 3).await;
|
||||
@@ -701,16 +613,9 @@ async fn fair_prefix_quantum_repeated_share_without_a_new_frontier_stops_requeui
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
fn cold_bucket_service_saved_order_keeps_coverage_and_rejects_older_owners() {
|
||||
run_cohort_test(
|
||||
"cold-saved-ord",
|
||||
cold_bucket_service_saved_order_keeps_coverage_and_rejects_older_owners_case,
|
||||
);
|
||||
}
|
||||
|
||||
async fn cold_bucket_service_saved_order_keeps_coverage_and_rejects_older_owners_case() {
|
||||
async fn cold_bucket_service_saved_order_keeps_coverage_and_rejects_older_owners() {
|
||||
let (_dir, store) = setup_local_scanner_store(1, 4).await;
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
create_cohort_bucket(&store, "cold-only").await;
|
||||
|
||||
@@ -30,11 +30,12 @@ doctest = false
|
||||
|
||||
[features]
|
||||
default = []
|
||||
hotpath = ["rustfs-rio/hotpath"]
|
||||
hotpath-alloc = ["hotpath", "rustfs-rio/hotpath-alloc"]
|
||||
hotpath-cpu = ["hotpath", "rustfs-rio/hotpath-cpu"]
|
||||
hotpath = ["hotpath/hotpath", "hotpath/tokio", "rustfs-rio/hotpath"]
|
||||
hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc", "rustfs-rio/hotpath-alloc"]
|
||||
hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu", "rustfs-rio/hotpath-cpu"]
|
||||
|
||||
[dependencies]
|
||||
hotpath.workspace = true
|
||||
async-compression = { workspace = true, features = [
|
||||
"tokio",
|
||||
"bzip2",
|
||||
@@ -48,6 +49,10 @@ rustfs-rio.workspace = true
|
||||
tokio = { workspace = true, features = ["io-util", "macros", "rt"] }
|
||||
thiserror = { workspace = true }
|
||||
|
||||
[target.'cfg(target_os = "macos")'.dependencies]
|
||||
# Keep XZ decoding self-contained in macOS binaries instead of linking Homebrew liblzma.
|
||||
liblzma = { version = "0.4.9", features = ["static"] }
|
||||
|
||||
[dev-dependencies]
|
||||
rustfs-tokio-tar = { workspace = true }
|
||||
futures = { workspace = true }
|
||||
|
||||
@@ -29,16 +29,14 @@ zdict_builder = ["zstd_fixed/zdict_builder"]
|
||||
zstdmt = ["zstd_fixed/zstdmt"]
|
||||
|
||||
[dev-dependencies]
|
||||
# Enables zstd-safe's `seekable` feature for tests/security.rs. The test imports
|
||||
# `zstd_safe::seekable` directly so cargo-shear treats this as a used dependency.
|
||||
zstd-safe = { version = "8.3.0", features = ["seekable"] }
|
||||
toml = "1.2.2"
|
||||
toml = "1.2.1"
|
||||
semver = "1.0.28"
|
||||
apache-avro.workspace = true
|
||||
arrow-array = "60.2.0"
|
||||
arrow-ipc = { version = "60.2.0", features = ["zstd"] }
|
||||
arrow-schema = "60.2.0"
|
||||
parquet = { version = "60.2.0", default-features = false, features = ["arrow", "zstd"] }
|
||||
arrow-array = "60.1.0"
|
||||
arrow-ipc = { version = "60.1.0", features = ["zstd"] }
|
||||
arrow-schema = "60.1.0"
|
||||
parquet = { version = "60.1.0", default-features = false, features = ["arrow", "zstd"] }
|
||||
kafka-protocol = { version = "0.18.0", default-features = false, features = ["zstd"] }
|
||||
bytes.workspace = true
|
||||
|
||||
|
||||
@@ -86,7 +86,7 @@ fn corrupt_legacy_frame_cannot_emit_bytes_when_writer_is_dropped() {
|
||||
|
||||
#[test]
|
||||
fn seekable_offsets_past_eof_keep_vec_length_within_capacity() {
|
||||
use zstd_safe::seekable::{Seekable, SeekableCStream};
|
||||
use zstd::zstd_safe::seekable::{Seekable, SeekableCStream};
|
||||
|
||||
let source = b"seekable frame fixture with enough data for several independent frames";
|
||||
let mut encoded = [0; 1024];
|
||||
|
||||
Generated
+6
-6
@@ -2,11 +2,11 @@
|
||||
"nodes": {
|
||||
"nixpkgs": {
|
||||
"locked": {
|
||||
"lastModified": 1795249095,
|
||||
"narHash": "sha256-8JDgyVvWUZg5m9hNlL8Ge9aT647ng3Ukc8NPMiKNTPQ=",
|
||||
"lastModified": 1794495034,
|
||||
"narHash": "sha256-E0hYoioqUtM2XmPPvYYpJgdADF5AW9oOoPSEiM9dTHQ=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "93a9ed0a866099a230ed5146a5bb7d16fd5c9b14",
|
||||
"rev": "2074cc672349f14240fb7ea90d509e0b643e9dd4",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
@@ -29,11 +29,11 @@
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1795253373,
|
||||
"narHash": "sha256-Vg6Sg8gJCDG9qpVEdOZX+72Z50MLwdnwtfZ8FNBwiYI=",
|
||||
"lastModified": 1794646145,
|
||||
"narHash": "sha256-rpqDyUM/d5fACYB8W+wKxxO57ICWjjalzb2X25oFFkY=",
|
||||
"owner": "oxalica",
|
||||
"repo": "rust-overlay",
|
||||
"rev": "52db20539ec0ba063479d66092992d9efcf456ff",
|
||||
"rev": "5b87ee0513c29127a01ad06e9cb362ea5ce91e78",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
||||
@@ -195,27 +195,14 @@ impl Operation for ChangeOwnPasswordHandler {
|
||||
let request: ChangePasswordRequest = serde_json::from_slice(&body)
|
||||
.map_err(|e| s3::error(S3ErrorCode::InvalidRequest, format!("invalid change-password request: {e}")))?;
|
||||
|
||||
let iam_store =
|
||||
current_ready_iam_handle().map_err(|_| s3::error(S3ErrorCode::InternalError, "iam is not initialized"))?;
|
||||
|
||||
let Some(stored) = iam_store.get_user(&caller.access_key).await else {
|
||||
// Reached only if the identity was deleted between authentication
|
||||
// and this lookup.
|
||||
emit_audit(
|
||||
&audit,
|
||||
AccountAuditRecord::failure(
|
||||
AccountAuditOperation::ChangeOwnPassword,
|
||||
&caller.access_key,
|
||||
caller.identity_type,
|
||||
AccountAuditFailure::Internal,
|
||||
),
|
||||
);
|
||||
return Err(s3::error(S3ErrorCode::InvalidRequest, "the calling identity no longer exists"));
|
||||
// An STS session signs with its own secret. A direct IAM request must
|
||||
// have been signed with the same password the caller is replacing.
|
||||
let expected_signing_secret = if caller.identity_type == IdentityType::Sts {
|
||||
&caller.credentials.secret_key
|
||||
} else {
|
||||
&request.current_secret_key
|
||||
};
|
||||
|
||||
if !constant_time_eq(&request.current_secret_key, &stored.credentials.secret_key)
|
||||
|| !constant_time_eq(&request.current_secret_key, &signing_secret)
|
||||
{
|
||||
if !constant_time_eq(expected_signing_secret, &signing_secret) {
|
||||
warn!(
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_ACCOUNT,
|
||||
@@ -256,8 +243,10 @@ impl Operation for ChangeOwnPasswordHandler {
|
||||
}
|
||||
|
||||
let access_key = caller.access_key.clone();
|
||||
let new_secret_key = request.new_secret_key.clone();
|
||||
let current_secret_key = request.current_secret_key;
|
||||
let new_secret_key = request.new_secret_key;
|
||||
let identity_type = caller.identity_type;
|
||||
let session_credentials = (identity_type == IdentityType::Sts).then(|| caller.credentials.clone());
|
||||
let session_access_key = caller.session_access_key.clone();
|
||||
let audit_for_task = audit.clone();
|
||||
|
||||
@@ -268,16 +257,35 @@ impl Operation for ChangeOwnPasswordHandler {
|
||||
let iam_store =
|
||||
current_ready_iam_handle().map_err(|_| s3::error(S3ErrorCode::InternalError, "iam is not initialized"))?;
|
||||
|
||||
let (updated_at, status, revoked) = iam_store
|
||||
.change_own_secret_key(&access_key, &signing_secret, &new_secret_key)
|
||||
.await
|
||||
.map_err(|error| {
|
||||
if matches!(error, rustfs_iam::error::Error::IAMActionNotAllowed) {
|
||||
s3::error(S3ErrorCode::InvalidRequest, "the current secret key is incorrect")
|
||||
} else {
|
||||
iam_error_to_s3_error(error)
|
||||
}
|
||||
})?;
|
||||
let rotation = match session_credentials.as_ref() {
|
||||
Some(session) => {
|
||||
iam_store
|
||||
.change_own_secret_key_from_session(&access_key, ¤t_secret_key, &new_secret_key, session)
|
||||
.await
|
||||
}
|
||||
None => {
|
||||
iam_store
|
||||
.change_own_secret_key(&access_key, ¤t_secret_key, &new_secret_key)
|
||||
.await
|
||||
}
|
||||
};
|
||||
let (updated_at, status, revoked) = rotation.map_err(|error| {
|
||||
if matches!(error, rustfs_iam::error::Error::IAMActionNotAllowed) {
|
||||
emit_audit(
|
||||
&audit_for_task,
|
||||
AccountAuditRecord::failure(
|
||||
AccountAuditOperation::ChangeOwnPassword,
|
||||
&access_key,
|
||||
identity_type,
|
||||
AccountAuditFailure::InvalidCurrentSecret,
|
||||
)
|
||||
.with_session_access_key(session_access_key.as_deref()),
|
||||
);
|
||||
s3::error(S3ErrorCode::InvalidRequest, "the current secret key is incorrect")
|
||||
} else {
|
||||
iam_error_to_s3_error(error)
|
||||
}
|
||||
})?;
|
||||
|
||||
emit_audit(
|
||||
&audit_for_task,
|
||||
|
||||
+26
-25
@@ -1019,32 +1019,33 @@ impl<T> S3Router<T> {
|
||||
pub(crate) fn registered_routes(&self) -> &[String] {
|
||||
self.table.registered_routes()
|
||||
}
|
||||
}
|
||||
|
||||
/// Dispatches one legacy admin request. Callers keep their assertions and do not name s3s.
|
||||
/// Free function (not an inherent impl) so the `OwnedOperation` bound stays `pub(crate)`.
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn call_legacy<T: OwnedOperation>(
|
||||
router: &S3Router<T>,
|
||||
method: Method,
|
||||
uri: &str,
|
||||
extensions: http::Extensions,
|
||||
) -> s3s::S3Result<S3Response<Body>> {
|
||||
S3Route::call(
|
||||
router,
|
||||
S3Request {
|
||||
input: Body::empty(),
|
||||
method,
|
||||
uri: uri.parse().expect("test URI"),
|
||||
headers: HeaderMap::new(),
|
||||
extensions,
|
||||
credentials: None,
|
||||
region: None,
|
||||
service: None,
|
||||
trailing_headers: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
/// Dispatches one legacy admin request. Callers keep their assertions and do not name s3s.
|
||||
pub(crate) async fn call_legacy(
|
||||
&self,
|
||||
method: Method,
|
||||
uri: &str,
|
||||
extensions: http::Extensions,
|
||||
) -> s3s::S3Result<S3Response<Body>>
|
||||
where
|
||||
T: OwnedOperation,
|
||||
{
|
||||
S3Route::call(
|
||||
self,
|
||||
S3Request {
|
||||
input: Body::empty(),
|
||||
method,
|
||||
uri: uri.parse().expect("test URI"),
|
||||
headers: HeaderMap::new(),
|
||||
extensions,
|
||||
credentials: None,
|
||||
region: None,
|
||||
service: None,
|
||||
trailing_headers: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> Default for S3Router<T> {
|
||||
|
||||
@@ -141,7 +141,7 @@ mod tests {
|
||||
object_store_from_envelope, publish_test_app_context,
|
||||
};
|
||||
use crate::admin::operation::{AdminRequest, AdminResponse, Operation};
|
||||
use crate::admin::router::{S3Router, call_legacy};
|
||||
use crate::admin::router::S3Router;
|
||||
use crate::app::{RequestEnvelope, RequestHead};
|
||||
use hyper::{Method, StatusCode};
|
||||
use matchit::Params;
|
||||
@@ -323,13 +323,15 @@ mod tests {
|
||||
.insert(Method::GET, "/context-probe", ContextDependentAdminRoute)
|
||||
.expect("register test route");
|
||||
|
||||
let err = call_legacy(&router, Method::GET, "/context-probe", http::Extensions::new())
|
||||
let err = router
|
||||
.call_legacy(Method::GET, "/context-probe", http::Extensions::new())
|
||||
.await
|
||||
.expect_err("empty slot must fail closed");
|
||||
assert_eq!(err.code(), &S3ErrorCode::ServiceUnavailable.into());
|
||||
|
||||
assert!(slot.install(distinct_context(&ambient)));
|
||||
let response = call_legacy(&router, Method::GET, "/context-probe", http::Extensions::new())
|
||||
let response = router
|
||||
.call_legacy(Method::GET, "/context-probe", http::Extensions::new())
|
||||
.await
|
||||
.expect("installed slot must serve request");
|
||||
assert_eq!(response.status, Some(StatusCode::NO_CONTENT));
|
||||
|
||||
@@ -87,6 +87,8 @@ pub(crate) enum CredentialMutationDenial {
|
||||
ServiceAccount,
|
||||
/// A derived credential whose parent could not be determined.
|
||||
UnresolvedParent,
|
||||
/// A storage delegation has no account-management authority.
|
||||
TableStorageSession,
|
||||
}
|
||||
|
||||
impl CredentialMutationDenial {
|
||||
@@ -97,6 +99,7 @@ impl CredentialMutationDenial {
|
||||
}
|
||||
Self::FederatedIdentity => "federated identities are managed by their identity provider",
|
||||
Self::ServiceAccount => "service account credentials cannot change the credentials of their parent identity",
|
||||
Self::TableStorageSession => "table-scoped storage credentials cannot manage account credentials",
|
||||
Self::UnresolvedParent => "the parent identity of this session could not be resolved",
|
||||
}
|
||||
}
|
||||
@@ -187,7 +190,7 @@ impl CallerIdentity {
|
||||
// root", and an unresolved parent must never fall through to a
|
||||
// permissive answer.
|
||||
//
|
||||
// These three denials apply to both capabilities: a machine credential
|
||||
// These denials apply to both capabilities: a machine credential
|
||||
// must not take over its parent, a federated identity is owned by its
|
||||
// IdP, and an unresolvable parent fails closed.
|
||||
let shared_denial = if unresolved_parent {
|
||||
@@ -196,6 +199,11 @@ impl CallerIdentity {
|
||||
Some(CredentialMutationDenial::ServiceAccount)
|
||||
} else if federated {
|
||||
Some(CredentialMutationDenial::FederatedIdentity)
|
||||
} else if ["rustfs:table-bucket", "rustfs:table-id", "rustfs:credential-scope-prefix"]
|
||||
.iter()
|
||||
.any(|claim| credentials.claims_or_empty().contains_key(*claim))
|
||||
{
|
||||
Some(CredentialMutationDenial::TableStorageSession)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
@@ -501,6 +509,20 @@ mod tests {
|
||||
assert_eq!(caller.mutation_denial, Some(CredentialMutationDenial::FederatedIdentity));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn table_storage_sessions_cannot_manage_parent_credentials() {
|
||||
for claim in ["rustfs:table-bucket", "rustfs:table-id", "rustfs:credential-scope-prefix"] {
|
||||
let mut credentials = sts_session("TABLEKEY", "table-owner");
|
||||
credentials.claims = Some(HashMap::from([(claim.to_string(), Value::String("scope".to_string()))]));
|
||||
let caller = CallerIdentity::from_credentials(credentials, false);
|
||||
|
||||
assert_eq!(caller.mutation_denial, Some(CredentialMutationDenial::TableStorageSession));
|
||||
assert!(!caller.mutability().password);
|
||||
assert!(caller.ensure_credential_mutation_allowed().is_err());
|
||||
assert!(caller.ensure_mfa_management_allowed().is_err());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn derived_credential_without_a_parent_is_denied_rather_than_allowed() {
|
||||
// Fail closed: an unresolvable parent must not be treated as a
|
||||
|
||||
@@ -1320,22 +1320,23 @@ mod tests {
|
||||
};
|
||||
for (index, expected_key) in ["a&local", "z-local"].into_iter().enumerate() {
|
||||
let request_marker = input.marker.clone().unwrap_or_default();
|
||||
let list_req = S3Request {
|
||||
input: input.clone(),
|
||||
method: http::Method::GET,
|
||||
uri: http::Uri::from_static("/"),
|
||||
headers: HeaderMap::new(),
|
||||
extensions: http::Extensions::new(),
|
||||
credentials: None,
|
||||
region: None,
|
||||
service: None,
|
||||
trailing_headers: None,
|
||||
};
|
||||
let envelope = crate::app::RequestEnvelope::from_s3s(&list_req);
|
||||
let gateway_input =
|
||||
rustfs_gateway_types::compat::s3s_5761ddfe::generated::ops::list_objects::input_from_s3s(
|
||||
input.clone(),
|
||||
list_req.input,
|
||||
)
|
||||
.expect("convert ListObjectsInput");
|
||||
let envelope = crate::app::RequestEnvelope::from_parts(
|
||||
crate::app::RequestHead {
|
||||
method: http::Method::GET,
|
||||
uri: http::Uri::from_static("/"),
|
||||
headers: HeaderMap::new(),
|
||||
credentials: None,
|
||||
region: None,
|
||||
service: None,
|
||||
},
|
||||
http::Extensions::new(),
|
||||
);
|
||||
let gateway_output = tokio::time::timeout(
|
||||
Duration::from_secs(10),
|
||||
DefaultBucketUsecase::from_global().execute_list_objects(envelope, gateway_input),
|
||||
|
||||
@@ -509,7 +509,7 @@ impl DefaultBucketUsecase {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::app::bucket_usecase::test_support::{build_request_with_req_info, owned_envelope};
|
||||
use crate::app::bucket_usecase::test_support::{build_request, build_request_with_req_info};
|
||||
use crate::app::storage_api::s3::dto::CreateBucketInput;
|
||||
use http::Method;
|
||||
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
||||
@@ -1005,10 +1005,11 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_list_buckets_returns_internal_error_when_store_uninitialized() {
|
||||
let req = build_request((), Method::GET);
|
||||
let usecase = DefaultBucketUsecase::without_context();
|
||||
|
||||
let err = usecase
|
||||
.execute_list_buckets(owned_envelope(Method::GET), gateway_dto::ListBucketsInput::default())
|
||||
.execute_list_buckets(RequestEnvelope::from_s3s(&req), gateway_dto::ListBucketsInput::default())
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(err.code(), &rustfs_s3_types::S3ErrorCode::InternalError);
|
||||
|
||||
@@ -366,7 +366,7 @@ impl DefaultBucketUsecase {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::app::bucket_usecase::test_support::{build_request_with_req_info, owned_envelope, s3_op_total};
|
||||
use crate::app::bucket_usecase::test_support::{build_request, build_request_with_req_info, s3_op_total};
|
||||
use http::Method;
|
||||
|
||||
#[test]
|
||||
@@ -399,10 +399,11 @@ mod tests {
|
||||
|
||||
/// The envelope of a request whose access phase installed `req_info`, or none.
|
||||
fn envelope(req_info: Option<ReqInfo>) -> RequestEnvelope {
|
||||
match req_info {
|
||||
Some(req_info) => RequestEnvelope::from_s3s(&build_request_with_req_info((), Method::PUT, req_info)),
|
||||
None => owned_envelope(Method::PUT),
|
||||
}
|
||||
let req = match req_info {
|
||||
Some(req_info) => build_request_with_req_info((), Method::PUT, req_info),
|
||||
None => build_request((), Method::PUT),
|
||||
};
|
||||
RequestEnvelope::from_s3s(&req)
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -203,7 +203,7 @@ impl DefaultBucketUsecase {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::app::bucket_usecase::test_support::owned_envelope;
|
||||
use crate::app::bucket_usecase::test_support::build_request;
|
||||
use crate::app::storage_api::s3::dto::{self, PutBucketEncryptionInput, ServerSideEncryption};
|
||||
use http::Method;
|
||||
use rustfs_gateway_types::persistence::{
|
||||
@@ -292,19 +292,20 @@ mod tests {
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let prepared = crate::storage::persisted_from_s3s(&input.server_side_encryption_configuration);
|
||||
let req = build_request(input, Method::PUT);
|
||||
let prepared = crate::storage::persisted_from_s3s(&req.input.server_side_encryption_configuration);
|
||||
let usecase = DefaultBucketUsecase::without_context();
|
||||
|
||||
let err = usecase
|
||||
.execute_put_bucket_encryption(owned_envelope(Method::PUT), &input.bucket, prepared.map_err(Into::into))
|
||||
.execute_put_bucket_encryption(RequestEnvelope::from_s3s(&req), &req.input.bucket, prepared.map_err(Into::into))
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
|
||||
let err = usecase
|
||||
.execute_put_bucket_encryption(
|
||||
owned_envelope(Method::PUT),
|
||||
&input.bucket,
|
||||
RequestEnvelope::from_s3s(&req),
|
||||
&req.input.bucket,
|
||||
Ok(PersistedBucketEncryptionConfiguration { rules: vec![] }),
|
||||
)
|
||||
.await
|
||||
|
||||
@@ -405,7 +405,7 @@ mod tests {
|
||||
// waiter again on the single-thread test runtime.
|
||||
assert_eq!(admission.available_permits(), 1);
|
||||
}
|
||||
use crate::app::bucket_usecase::test_support::{build_request, owned_envelope};
|
||||
use crate::app::bucket_usecase::test_support::build_request;
|
||||
use crate::app::gating_test_env::{run_large_stack_test, shared_gating_ambient};
|
||||
use crate::app::runtime_sources::ServerContextSlot;
|
||||
use crate::app::storage_api::bucket_usecase::helper::MutationCompletionProbe;
|
||||
@@ -775,12 +775,13 @@ mod tests {
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let req = build_request(input, Method::PUT);
|
||||
let usecase = DefaultBucketUsecase::without_context();
|
||||
|
||||
let err = usecase
|
||||
.execute_put_bucket_lifecycle_configuration(
|
||||
owned_envelope(Method::PUT),
|
||||
&input.bucket,
|
||||
RequestEnvelope::from_s3s(&req),
|
||||
&req.input.bucket,
|
||||
PreparedLifecycleConfiguration::Missing,
|
||||
)
|
||||
.await
|
||||
|
||||
@@ -159,7 +159,7 @@ fn list_object_versions_output(
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::app::bucket_usecase::test_support::owned_envelope;
|
||||
use crate::app::bucket_usecase::test_support::build_request;
|
||||
use rustfs_gateway_types::BucketName;
|
||||
use rustfs_s3_types::S3ErrorCode;
|
||||
|
||||
@@ -319,7 +319,7 @@ mod tests {
|
||||
};
|
||||
|
||||
let err = DefaultBucketUsecase::without_context()
|
||||
.execute_list_object_versions(owned_envelope(http::Method::GET), input)
|
||||
.execute_list_object_versions(RequestEnvelope::from_s3s(&build_request((), http::Method::GET)), input)
|
||||
.await
|
||||
.expect_err("a negative page size");
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
|
||||
@@ -333,7 +333,7 @@ mod tests {
|
||||
};
|
||||
|
||||
let err = DefaultBucketUsecase::without_context()
|
||||
.execute_list_object_versions(owned_envelope(http::Method::GET), input)
|
||||
.execute_list_object_versions(RequestEnvelope::from_s3s(&build_request((), http::Method::GET)), input)
|
||||
.await
|
||||
.expect_err("no store");
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
|
||||
@@ -372,12 +372,13 @@ fn list_objects_next_marker(info: &ListObjectsV2Info, cursor: Option<&str>) -> O
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::app::bucket_usecase::test_support::owned_envelope;
|
||||
use crate::app::bucket_usecase::test_support::build_request;
|
||||
use rustfs_gateway_types::BucketName;
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn list_apis_report_storage_read_quorum_as_unavailable() {
|
||||
use crate::app::bucket_usecase::test_support::owned_envelope;
|
||||
use crate::app::gating_test_env::{run_large_stack_test, shared_gating_ambient, shared_gating_ecstore};
|
||||
run_large_stack_test("list-quorum-surface", || async {
|
||||
let usecase = DefaultBucketUsecase::with_context(Some(shared_gating_ambient().await));
|
||||
@@ -484,7 +485,7 @@ mod tests {
|
||||
}
|
||||
|
||||
fn envelope() -> RequestEnvelope {
|
||||
owned_envelope(http::Method::GET)
|
||||
RequestEnvelope::from_s3s(&build_request((), http::Method::GET))
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -56,7 +56,7 @@ impl DefaultBucketUsecase {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::app::bucket_usecase::test_support::{build_request, owned_envelope};
|
||||
use crate::app::bucket_usecase::test_support::build_request;
|
||||
use crate::app::bucket_usecase::{BucketOperations, DeleteBucketOptions, MakeBucketOptions};
|
||||
use crate::app::gating_test_env::{run_large_stack_test, shared_gating_ambient};
|
||||
use crate::app::runtime_sources::ServerContextSlot;
|
||||
@@ -75,8 +75,10 @@ mod tests {
|
||||
let context = shared_gating_ambient().await;
|
||||
let bucket = format!("logging-missing-{}", uuid::Uuid::new_v4().simple());
|
||||
let usecase = DefaultBucketUsecase::with_context(Some(context));
|
||||
let request = build_request((), Method::PUT);
|
||||
let envelope = RequestEnvelope::from_s3s(&request);
|
||||
let error = usecase
|
||||
.execute_put_bucket_logging(owned_envelope(Method::PUT), &bucket, || {
|
||||
.execute_put_bucket_logging(envelope, &bucket, || {
|
||||
Err(rustfs_s3_types::s3_error!(MalformedXML, "invalid logging configuration"))
|
||||
})
|
||||
.await
|
||||
|
||||
@@ -243,15 +243,16 @@ impl DefaultBucketUsecase {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::app::bucket_usecase::test_support::owned_envelope;
|
||||
use crate::app::bucket_usecase::test_support::build_request;
|
||||
use http::Method;
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_delete_bucket_policy_returns_internal_error_when_store_uninitialized() {
|
||||
let req = build_request((), Method::DELETE);
|
||||
let usecase = DefaultBucketUsecase::without_context();
|
||||
|
||||
let err = usecase
|
||||
.execute_delete_bucket_policy(owned_envelope(Method::DELETE), "test-bucket")
|
||||
.execute_delete_bucket_policy(RequestEnvelope::from_s3s(&req), "test-bucket")
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(err.code(), &rustfs_s3_types::S3ErrorCode::InternalError);
|
||||
@@ -279,10 +280,11 @@ mod tests {
|
||||
bucket: bucket("test-bucket"),
|
||||
..Default::default()
|
||||
};
|
||||
let req = build_request((), Method::GET);
|
||||
let usecase = DefaultBucketUsecase::without_context();
|
||||
|
||||
let err = usecase
|
||||
.execute_get_bucket_policy_status(owned_envelope(Method::GET), input)
|
||||
.execute_get_bucket_policy_status(RequestEnvelope::from_s3s(&req), input)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(err.code(), &rustfs_s3_types::S3ErrorCode::InternalError);
|
||||
@@ -290,10 +292,11 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_put_bucket_policy_returns_internal_error_when_store_uninitialized() {
|
||||
let req = build_request((), Method::PUT);
|
||||
let usecase = DefaultBucketUsecase::without_context();
|
||||
|
||||
let err = usecase
|
||||
.execute_put_bucket_policy(owned_envelope(Method::PUT), "test-bucket", "{}")
|
||||
.execute_put_bucket_policy(RequestEnvelope::from_s3s(&req), "test-bucket", "{}")
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(err.code(), &rustfs_s3_types::S3ErrorCode::InternalError);
|
||||
|
||||
@@ -112,7 +112,7 @@ impl DefaultBucketUsecase {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::app::bucket_usecase::test_support::{build_request, owned_envelope};
|
||||
use crate::app::bucket_usecase::test_support::build_request;
|
||||
use crate::app::gating_test_env::{run_large_stack_test, shared_gating_ambient};
|
||||
use crate::app::runtime_sources::ServerContextSlot;
|
||||
use crate::app::storage_api::bucket_usecase::helper::MutationCompletionProbe;
|
||||
@@ -278,7 +278,7 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_delete_public_access_block_returns_internal_error_when_store_uninitialized() {
|
||||
let envelope = owned_envelope(Method::DELETE);
|
||||
let envelope = RequestEnvelope::from_s3s(&build_request((), Method::DELETE));
|
||||
let usecase = DefaultBucketUsecase::without_context();
|
||||
let err = usecase
|
||||
.execute_delete_public_access_block(envelope, "test-bucket")
|
||||
@@ -296,7 +296,7 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_put_public_access_block_returns_internal_error_when_store_uninitialized() {
|
||||
let envelope = owned_envelope(Method::PUT);
|
||||
let envelope = RequestEnvelope::from_s3s(&build_request((), Method::PUT));
|
||||
let usecase = DefaultBucketUsecase::without_context();
|
||||
let err = usecase
|
||||
.execute_put_public_access_block(envelope, "test-bucket", Ok(PersistedPublicAccessBlockConfiguration::default()))
|
||||
|
||||
@@ -138,7 +138,7 @@ impl DefaultBucketUsecase {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::app::bucket_usecase::test_support::{build_request, owned_envelope};
|
||||
use crate::app::bucket_usecase::test_support::build_request;
|
||||
use crate::app::storage_api::s3::dto::{self, PutBucketTaggingInput, Tag, Tagging};
|
||||
use http::Method;
|
||||
use rustfs_s3_types::S3ErrorCode;
|
||||
@@ -249,10 +249,11 @@ mod tests {
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let req = build_request(input, Method::PUT);
|
||||
let usecase = DefaultBucketUsecase::without_context();
|
||||
|
||||
let err = usecase
|
||||
.execute_put_bucket_tagging(owned_envelope(Method::PUT), &input.bucket, PersistedTagging::default())
|
||||
.execute_put_bucket_tagging(RequestEnvelope::from_s3s(&req), &req.input.bucket, PersistedTagging::default())
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
@@ -266,7 +267,7 @@ mod tests {
|
||||
(vec![tag; MAX_BUCKET_TAGS + 1], "Cannot have more than 50 tags per bucket"),
|
||||
] {
|
||||
let err = usecase
|
||||
.execute_put_bucket_tagging(owned_envelope(Method::PUT), &input.bucket, PersistedTagging { tag_set })
|
||||
.execute_put_bucket_tagging(RequestEnvelope::from_s3s(&req), &req.input.bucket, PersistedTagging { tag_set })
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidTag);
|
||||
|
||||
@@ -47,17 +47,7 @@ pub(super) fn gateway_bucket(name: &str) -> rustfs_gateway_types::BucketName {
|
||||
|
||||
/// The envelope of a request that carries nothing but `method`.
|
||||
pub(super) fn owned_envelope(method: Method) -> RequestEnvelope {
|
||||
RequestEnvelope::from_parts(
|
||||
crate::app::RequestHead {
|
||||
method,
|
||||
uri: Uri::from_static("/"),
|
||||
headers: HeaderMap::new(),
|
||||
credentials: None,
|
||||
region: None,
|
||||
service: None,
|
||||
},
|
||||
Extensions::new(),
|
||||
)
|
||||
RequestEnvelope::from_s3s(&build_request((), method))
|
||||
}
|
||||
|
||||
pub(super) fn build_request_with_req_info<T>(input: T, method: Method, req_info: ReqInfo) -> S3Request<T> {
|
||||
|
||||
@@ -81,7 +81,7 @@ impl DefaultBucketUsecase {
|
||||
mod tests {
|
||||
use super::super::{BUCKET_ACCELERATE_CONFIG, BUCKET_REQUEST_PAYMENT_CONFIG};
|
||||
use super::*;
|
||||
use crate::app::bucket_usecase::test_support::{build_request, build_request_with_req_info, owned_envelope};
|
||||
use crate::app::bucket_usecase::test_support::{build_request, build_request_with_req_info};
|
||||
use crate::app::gating_test_env::{run_large_stack_test, shared_gating_ambient};
|
||||
use crate::app::runtime_sources::ServerContextSlot;
|
||||
use crate::app::storage_api::object_usecase::helper::MutationCompletionProbe;
|
||||
@@ -190,21 +190,22 @@ mod tests {
|
||||
let ambient = shared_gating_ambient().await;
|
||||
let usecase = DefaultBucketUsecase::with_context(Some(ambient));
|
||||
let bucket = format!("core-config-missing-{}", uuid::Uuid::new_v4().simple());
|
||||
let request = build_request((), http::Method::PUT);
|
||||
let mut converted = [false; 3];
|
||||
let accelerate = usecase
|
||||
.execute_put_bucket_accelerate_configuration(owned_envelope(http::Method::PUT), &bucket, || {
|
||||
.execute_put_bucket_accelerate_configuration(RequestEnvelope::from_s3s(&request), &bucket, || {
|
||||
converted[0] = true;
|
||||
Err(rustfs_s3_types::s3_error!(MalformedXML, "invalid acceleration"))
|
||||
})
|
||||
.await;
|
||||
let payment = usecase
|
||||
.execute_put_bucket_request_payment(owned_envelope(http::Method::PUT), &bucket, || {
|
||||
.execute_put_bucket_request_payment(RequestEnvelope::from_s3s(&request), &bucket, || {
|
||||
converted[1] = true;
|
||||
Err(rustfs_s3_types::s3_error!(MalformedXML, "invalid request payment"))
|
||||
})
|
||||
.await;
|
||||
let website = usecase
|
||||
.execute_put_bucket_website(owned_envelope(http::Method::PUT), &bucket, || {
|
||||
.execute_put_bucket_website(RequestEnvelope::from_s3s(&request), &bucket, || {
|
||||
converted[2] = true;
|
||||
Err(rustfs_s3_types::s3_error!(MalformedXML, "invalid website"))
|
||||
})
|
||||
|
||||
@@ -52,19 +52,8 @@ async fn object_lock_handlers_schedule_replication() {
|
||||
.expect("Failed to create object-lock bucket");
|
||||
upload_test_object(&ecstore, bucket.as_str(), object, b"worm payload").await;
|
||||
|
||||
let envelope = || {
|
||||
RequestEnvelope::from_parts(
|
||||
crate::app::RequestHead {
|
||||
method: Method::PUT,
|
||||
uri: Uri::from_static("/"),
|
||||
headers: HeaderMap::new(),
|
||||
credentials: None,
|
||||
region: None,
|
||||
service: None,
|
||||
},
|
||||
Extensions::new(),
|
||||
)
|
||||
};
|
||||
let req = build_request((), Method::PUT);
|
||||
let envelope = || RequestEnvelope::from_s3s(&req);
|
||||
|
||||
// A bucket without a replication configuration can skip the decision.
|
||||
MUST_REPLICATE_OBJECT_CALLS.store(0, Ordering::SeqCst);
|
||||
|
||||
@@ -22,17 +22,7 @@ const RESTORE_SUSPENDED_WAIT_TIMEOUT: Duration = Duration::from_secs(15);
|
||||
async fn suspend_test_bucket(bucket: &str) {
|
||||
DefaultBucketUsecase::from_global()
|
||||
.execute_put_bucket_versioning(
|
||||
crate::app::RequestEnvelope::from_parts(
|
||||
crate::app::RequestHead {
|
||||
method: Method::PUT,
|
||||
uri: Uri::from_static("/"),
|
||||
headers: HeaderMap::new(),
|
||||
credentials: None,
|
||||
region: None,
|
||||
service: None,
|
||||
},
|
||||
Extensions::new(),
|
||||
),
|
||||
crate::app::RequestEnvelope::from_s3s(&build_request((), Method::PUT)),
|
||||
bucket,
|
||||
Ok(rustfs_gateway_types::persistence::PersistedVersioningConfiguration {
|
||||
status: Some("Suspended".to_owned()),
|
||||
|
||||
@@ -545,17 +545,10 @@ mod tests {
|
||||
.key(key.to_owned())
|
||||
.build()
|
||||
.expect("valid create request DTO");
|
||||
let error = match make_usecase()
|
||||
.execute_create_multipart_upload(
|
||||
owned_envelope(Method::POST),
|
||||
super::create_values::CreateMultipartUploadFields::from_legacy(create),
|
||||
None,
|
||||
)
|
||||
let error = make_usecase()
|
||||
.execute_create_multipart_upload_from_s3s(build_request(create, Method::POST))
|
||||
.await
|
||||
{
|
||||
Err(error) => crate::app::storage_api::s3::S3Error::from(error),
|
||||
Ok(_) => panic!("reserved key must be rejected before accessing storage"),
|
||||
};
|
||||
.expect_err("reserved key must be rejected before accessing storage");
|
||||
assert_eq!(error.code(), &crate::app::storage_api::s3::S3ErrorCode::InvalidArgument);
|
||||
|
||||
let complete = CompleteMultipartUploadInput::builder()
|
||||
@@ -565,15 +558,9 @@ mod tests {
|
||||
.multipart_upload(Some(CompletedMultipartUpload::default()))
|
||||
.build()
|
||||
.expect("valid complete request DTO");
|
||||
let error =
|
||||
match rustfs_gateway_types::compat::s3s_5761ddfe::generated::ops::complete_multipart_upload::input_from_s3s(
|
||||
complete,
|
||||
) {
|
||||
Err(error) => crate::app::dto_values::seam_error(error),
|
||||
Ok(input) => Box::pin(make_usecase().execute_complete_multipart_upload(owned_envelope(Method::POST), input))
|
||||
.await
|
||||
.expect_err("uploads created before the fix must not publish reserved keys"),
|
||||
};
|
||||
let error = Box::pin(make_usecase().complete_for_s3s_request(build_request(complete, Method::POST)))
|
||||
.await
|
||||
.expect_err("uploads created before the fix must not publish reserved keys");
|
||||
assert_eq!(error.code(), &rustfs_s3_types::S3ErrorCode::InvalidArgument);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -24,7 +24,8 @@ use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_abort_multipart_upload_returns_internal_error_when_store_uninitialized() {
|
||||
let envelope = owned_envelope(Method::DELETE);
|
||||
let request = build_request((), Method::DELETE);
|
||||
let envelope = RequestEnvelope::from_s3s(&request);
|
||||
let before = s3_op_total(S3Operation::AbortMultipartUpload);
|
||||
let target = AbortMultipartUploadTarget::Legacy {
|
||||
bucket: "bucket".to_owned(),
|
||||
|
||||
@@ -943,72 +943,21 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_complete_multipart_upload_rejects_missing_parts_payload() {
|
||||
use rustfs_gateway_types::compat::s3s_5761ddfe::generated::ops::complete_multipart_upload;
|
||||
|
||||
let input = CompleteMultipartUploadInput::builder()
|
||||
.bucket("bucket".to_string())
|
||||
.key("object".to_string())
|
||||
.upload_id("upload-id".to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
let mut s3s_input = input;
|
||||
let has_upload = s3s_input.multipart_upload.is_some();
|
||||
s3s_input.multipart_upload.get_or_insert_with(Default::default);
|
||||
let tags: Vec<Option<String>> = s3s_input
|
||||
.multipart_upload
|
||||
.iter_mut()
|
||||
.flat_map(|upload| upload.parts.iter_mut().flatten())
|
||||
.map(|part| {
|
||||
part.part_number.get_or_insert(0);
|
||||
part.e_tag.take().map(|tag| tag.value().to_owned())
|
||||
})
|
||||
.collect();
|
||||
for checksum in [
|
||||
&mut s3s_input.checksum_crc32,
|
||||
&mut s3s_input.checksum_crc32c,
|
||||
&mut s3s_input.checksum_crc64nvme,
|
||||
&mut s3s_input.checksum_md5,
|
||||
&mut s3s_input.checksum_sha1,
|
||||
&mut s3s_input.checksum_sha256,
|
||||
&mut s3s_input.checksum_sha512,
|
||||
&mut s3s_input.checksum_xxhash128,
|
||||
&mut s3s_input.checksum_xxhash3,
|
||||
&mut s3s_input.checksum_xxhash64,
|
||||
] {
|
||||
*checksum = None;
|
||||
}
|
||||
let err = match complete_multipart_upload::input_from_s3s(s3s_input) {
|
||||
Err(error) => crate::app::dto_values::seam_error(error),
|
||||
Ok(mut owned) => {
|
||||
for (part, tag) in owned.multipart_upload.parts.iter_mut().zip(tags) {
|
||||
part.e_tag = tag.map(rustfs_gateway_types::ETag::from_stored);
|
||||
}
|
||||
let taken = std::mem::take(&mut owned.multipart_upload);
|
||||
let parts = has_upload.then_some(taken);
|
||||
let result = match parts {
|
||||
Some(parts) => {
|
||||
owned.multipart_upload = parts;
|
||||
Box::pin(make_usecase().execute_complete_multipart_upload(owned_envelope(Method::POST), owned)).await
|
||||
}
|
||||
None => {
|
||||
Box::pin(make_usecase().execute_complete_multipart_upload_parts(
|
||||
owned_envelope(Method::POST),
|
||||
owned,
|
||||
None,
|
||||
))
|
||||
.await
|
||||
}
|
||||
};
|
||||
result.unwrap_err()
|
||||
}
|
||||
};
|
||||
let req = build_request(input, Method::POST);
|
||||
|
||||
let err = Box::pin(make_usecase().complete_for_s3s_request(req)).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidPart);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_complete_multipart_upload_rejects_missing_parts_list() {
|
||||
use crate::app::storage_api::s3::xml::Deserialize as _;
|
||||
use rustfs_gateway_types::compat::s3s_5761ddfe::generated::ops::complete_multipart_upload;
|
||||
|
||||
let mut deserializer = crate::app::storage_api::s3::xml::Deserializer::new(b"<CompleteMultipartUpload/>");
|
||||
let multipart_upload =
|
||||
@@ -1025,57 +974,11 @@ mod tests {
|
||||
.multipart_upload(Some(multipart_upload))
|
||||
.build()
|
||||
.expect("complete multipart input should build");
|
||||
let mut s3s_input = input;
|
||||
let has_upload = s3s_input.multipart_upload.is_some();
|
||||
s3s_input.multipart_upload.get_or_insert_with(Default::default);
|
||||
let tags: Vec<Option<String>> = s3s_input
|
||||
.multipart_upload
|
||||
.iter_mut()
|
||||
.flat_map(|upload| upload.parts.iter_mut().flatten())
|
||||
.map(|part| {
|
||||
part.part_number.get_or_insert(0);
|
||||
part.e_tag.take().map(|tag| tag.value().to_owned())
|
||||
})
|
||||
.collect();
|
||||
for checksum in [
|
||||
&mut s3s_input.checksum_crc32,
|
||||
&mut s3s_input.checksum_crc32c,
|
||||
&mut s3s_input.checksum_crc64nvme,
|
||||
&mut s3s_input.checksum_md5,
|
||||
&mut s3s_input.checksum_sha1,
|
||||
&mut s3s_input.checksum_sha256,
|
||||
&mut s3s_input.checksum_sha512,
|
||||
&mut s3s_input.checksum_xxhash128,
|
||||
&mut s3s_input.checksum_xxhash3,
|
||||
&mut s3s_input.checksum_xxhash64,
|
||||
] {
|
||||
*checksum = None;
|
||||
}
|
||||
let err = match complete_multipart_upload::input_from_s3s(s3s_input) {
|
||||
Err(error) => crate::app::dto_values::seam_error(error),
|
||||
Ok(mut owned) => {
|
||||
for (part, tag) in owned.multipart_upload.parts.iter_mut().zip(tags) {
|
||||
part.e_tag = tag.map(rustfs_gateway_types::ETag::from_stored);
|
||||
}
|
||||
let taken = std::mem::take(&mut owned.multipart_upload);
|
||||
let parts = has_upload.then_some(taken);
|
||||
let result = match parts {
|
||||
Some(parts) => {
|
||||
owned.multipart_upload = parts;
|
||||
Box::pin(make_usecase().execute_complete_multipart_upload(owned_envelope(Method::POST), owned)).await
|
||||
}
|
||||
None => {
|
||||
Box::pin(make_usecase().execute_complete_multipart_upload_parts(
|
||||
owned_envelope(Method::POST),
|
||||
owned,
|
||||
None,
|
||||
))
|
||||
.await
|
||||
}
|
||||
};
|
||||
result.expect_err("missing parts list must be rejected")
|
||||
}
|
||||
};
|
||||
let req = build_request(input, Method::POST);
|
||||
|
||||
let err = Box::pin(make_usecase().complete_for_s3s_request(req))
|
||||
.await
|
||||
.expect_err("missing parts list must be rejected");
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
|
||||
}
|
||||
|
||||
@@ -1221,9 +1124,10 @@ mod tests {
|
||||
crate::app::gating_test_env::run_large_stack_test("ssec-completion-checksums", || async {
|
||||
let (_store, bucket) =
|
||||
crate::app::gating_test_env::durable_quota_test_bucket("ssec-completion-checksums", 16_384).await;
|
||||
let versioning_req = build_request((), Method::PUT);
|
||||
crate::app::bucket_usecase::DefaultBucketUsecase::from_global()
|
||||
.execute_put_bucket_versioning(
|
||||
super::super::test_support::owned_envelope(Method::PUT),
|
||||
RequestEnvelope::from_s3s(&versioning_req),
|
||||
&bucket,
|
||||
Ok(rustfs_gateway_types::persistence::PersistedVersioningConfiguration {
|
||||
status: Some("Enabled".to_owned()),
|
||||
@@ -1400,13 +1304,16 @@ mod tests {
|
||||
|
||||
let bucket_usecase = crate::app::bucket_usecase::DefaultBucketUsecase::from_global();
|
||||
// Listings carry neither the SSE-C key nor a per-object decryption request.
|
||||
let versions_input = ListObjectVersionsInput {
|
||||
bucket: bucket.to_string(),
|
||||
prefix: Some(key.to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
let versions_env = super::super::test_support::owned_envelope(Method::GET);
|
||||
let versions_in = list_object_versions::input_from_s3s(versions_input).expect("convert ListObjectVersionsInput");
|
||||
let versions_req = build_request(
|
||||
ListObjectVersionsInput {
|
||||
bucket: bucket.to_string(),
|
||||
prefix: Some(key.to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
Method::GET,
|
||||
);
|
||||
let versions_env = RequestEnvelope::from_s3s(&versions_req);
|
||||
let versions_in = list_object_versions::input_from_s3s(versions_req.input).expect("convert ListObjectVersionsInput");
|
||||
let versions = list_object_versions::output_to_s3s(
|
||||
bucket_usecase
|
||||
.execute_list_object_versions(versions_env, versions_in)
|
||||
@@ -1421,13 +1328,16 @@ mod tests {
|
||||
assert_eq!(entries[0].e_tag.as_ref(), Some(etag));
|
||||
assert_xml_etag(&versions, etag);
|
||||
|
||||
let v1_input = ListObjectsInput {
|
||||
bucket: bucket.to_string(),
|
||||
prefix: Some(key.to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
let v1_env = super::super::test_support::owned_envelope(Method::GET);
|
||||
let v1_in = list_objects::input_from_s3s(v1_input).expect("convert ListObjectsInput");
|
||||
let v1_req = build_request(
|
||||
ListObjectsInput {
|
||||
bucket: bucket.to_string(),
|
||||
prefix: Some(key.to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
Method::GET,
|
||||
);
|
||||
let v1_env = RequestEnvelope::from_s3s(&v1_req);
|
||||
let v1_in = list_objects::input_from_s3s(v1_req.input).expect("convert ListObjectsInput");
|
||||
let v1 = list_objects::output_to_s3s(
|
||||
bucket_usecase
|
||||
.execute_list_objects(v1_env, v1_in)
|
||||
@@ -1501,6 +1411,7 @@ mod tests {
|
||||
.await
|
||||
.expect("create versioned Object Lock bucket");
|
||||
let key = "kms-encrypted";
|
||||
let enc_req = build_request((), Method::PUT);
|
||||
let enc_cfg = crate::app::storage_api::s3::dto::ServerSideEncryptionConfiguration {
|
||||
rules: vec![crate::app::storage_api::s3::dto::ServerSideEncryptionRule {
|
||||
apply_server_side_encryption_by_default: Some(
|
||||
@@ -1514,7 +1425,7 @@ mod tests {
|
||||
};
|
||||
let prepared = crate::storage::persisted_from_s3s(&enc_cfg).map_err(Into::into);
|
||||
crate::app::bucket_usecase::DefaultBucketUsecase::from_global()
|
||||
.execute_put_bucket_encryption(super::super::test_support::owned_envelope(Method::PUT), &bucket, prepared)
|
||||
.execute_put_bucket_encryption(RequestEnvelope::from_s3s(&enc_req), &bucket, prepared)
|
||||
.await
|
||||
.expect("configure bucket-default SSE-KMS");
|
||||
let usecase = DefaultMultipartUsecase::from_global();
|
||||
@@ -2240,8 +2151,6 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_complete_multipart_upload_allows_duplicate_part_numbers_by_using_last_occurrence() {
|
||||
use rustfs_gateway_types::compat::s3s_5761ddfe::generated::ops::complete_multipart_upload;
|
||||
|
||||
let multipart_upload = CompletedMultipartUpload {
|
||||
parts: Some(vec![
|
||||
CompletedPart {
|
||||
@@ -2261,67 +2170,14 @@ mod tests {
|
||||
.multipart_upload(Some(multipart_upload))
|
||||
.build()
|
||||
.unwrap();
|
||||
let mut s3s_input = input;
|
||||
let has_upload = s3s_input.multipart_upload.is_some();
|
||||
s3s_input.multipart_upload.get_or_insert_with(Default::default);
|
||||
let tags: Vec<Option<String>> = s3s_input
|
||||
.multipart_upload
|
||||
.iter_mut()
|
||||
.flat_map(|upload| upload.parts.iter_mut().flatten())
|
||||
.map(|part| {
|
||||
part.part_number.get_or_insert(0);
|
||||
part.e_tag.take().map(|tag| tag.value().to_owned())
|
||||
})
|
||||
.collect();
|
||||
for checksum in [
|
||||
&mut s3s_input.checksum_crc32,
|
||||
&mut s3s_input.checksum_crc32c,
|
||||
&mut s3s_input.checksum_crc64nvme,
|
||||
&mut s3s_input.checksum_md5,
|
||||
&mut s3s_input.checksum_sha1,
|
||||
&mut s3s_input.checksum_sha256,
|
||||
&mut s3s_input.checksum_sha512,
|
||||
&mut s3s_input.checksum_xxhash128,
|
||||
&mut s3s_input.checksum_xxhash3,
|
||||
&mut s3s_input.checksum_xxhash64,
|
||||
] {
|
||||
*checksum = None;
|
||||
}
|
||||
let err = match complete_multipart_upload::input_from_s3s(s3s_input) {
|
||||
Err(error) => crate::app::dto_values::seam_error(error),
|
||||
Ok(mut owned) => {
|
||||
for (part, tag) in owned.multipart_upload.parts.iter_mut().zip(tags) {
|
||||
part.e_tag = tag.map(rustfs_gateway_types::ETag::from_stored);
|
||||
}
|
||||
let taken = std::mem::take(&mut owned.multipart_upload);
|
||||
let parts = has_upload.then_some(taken);
|
||||
let result = match parts {
|
||||
Some(parts) => {
|
||||
owned.multipart_upload = parts;
|
||||
Box::pin(make_usecase().execute_complete_multipart_upload(owned_envelope(Method::POST), owned)).await
|
||||
}
|
||||
None => {
|
||||
Box::pin(make_usecase().execute_complete_multipart_upload_parts(
|
||||
owned_envelope(Method::POST),
|
||||
owned,
|
||||
None,
|
||||
))
|
||||
.await
|
||||
}
|
||||
};
|
||||
match result {
|
||||
Err(error) => error,
|
||||
Ok(_output) => panic!("duplicate part numbers must not succeed before the order check"),
|
||||
}
|
||||
}
|
||||
};
|
||||
let req = build_request(input, Method::POST);
|
||||
|
||||
let err = Box::pin(make_usecase().complete_for_s3s_request(req)).await.unwrap_err();
|
||||
assert_ne!(err.code(), &S3ErrorCode::InvalidPartOrder);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_complete_multipart_upload_rejects_out_of_order_parts() {
|
||||
use rustfs_gateway_types::compat::s3s_5761ddfe::generated::ops::complete_multipart_upload;
|
||||
|
||||
let multipart_upload = CompletedMultipartUpload {
|
||||
parts: Some(vec![
|
||||
CompletedPart {
|
||||
@@ -2341,60 +2197,9 @@ mod tests {
|
||||
.multipart_upload(Some(multipart_upload))
|
||||
.build()
|
||||
.unwrap();
|
||||
let mut s3s_input = input;
|
||||
let has_upload = s3s_input.multipart_upload.is_some();
|
||||
s3s_input.multipart_upload.get_or_insert_with(Default::default);
|
||||
let tags: Vec<Option<String>> = s3s_input
|
||||
.multipart_upload
|
||||
.iter_mut()
|
||||
.flat_map(|upload| upload.parts.iter_mut().flatten())
|
||||
.map(|part| {
|
||||
part.part_number.get_or_insert(0);
|
||||
part.e_tag.take().map(|tag| tag.value().to_owned())
|
||||
})
|
||||
.collect();
|
||||
for checksum in [
|
||||
&mut s3s_input.checksum_crc32,
|
||||
&mut s3s_input.checksum_crc32c,
|
||||
&mut s3s_input.checksum_crc64nvme,
|
||||
&mut s3s_input.checksum_md5,
|
||||
&mut s3s_input.checksum_sha1,
|
||||
&mut s3s_input.checksum_sha256,
|
||||
&mut s3s_input.checksum_sha512,
|
||||
&mut s3s_input.checksum_xxhash128,
|
||||
&mut s3s_input.checksum_xxhash3,
|
||||
&mut s3s_input.checksum_xxhash64,
|
||||
] {
|
||||
*checksum = None;
|
||||
}
|
||||
let err = match complete_multipart_upload::input_from_s3s(s3s_input) {
|
||||
Err(error) => crate::app::dto_values::seam_error(error),
|
||||
Ok(mut owned) => {
|
||||
for (part, tag) in owned.multipart_upload.parts.iter_mut().zip(tags) {
|
||||
part.e_tag = tag.map(rustfs_gateway_types::ETag::from_stored);
|
||||
}
|
||||
let taken = std::mem::take(&mut owned.multipart_upload);
|
||||
let parts = has_upload.then_some(taken);
|
||||
let result = match parts {
|
||||
Some(parts) => {
|
||||
owned.multipart_upload = parts;
|
||||
Box::pin(make_usecase().execute_complete_multipart_upload(owned_envelope(Method::POST), owned)).await
|
||||
}
|
||||
None => {
|
||||
Box::pin(make_usecase().execute_complete_multipart_upload_parts(
|
||||
owned_envelope(Method::POST),
|
||||
owned,
|
||||
None,
|
||||
))
|
||||
.await
|
||||
}
|
||||
};
|
||||
match result {
|
||||
Err(error) => error,
|
||||
Ok(_output) => panic!("out-of-order parts must be rejected"),
|
||||
}
|
||||
}
|
||||
};
|
||||
let req = build_request(input, Method::POST);
|
||||
|
||||
let err = Box::pin(make_usecase().complete_for_s3s_request(req)).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidPartOrder);
|
||||
}
|
||||
|
||||
|
||||
@@ -545,17 +545,10 @@ mod tests {
|
||||
.object_lock_retain_until_date(date.map(Timestamp::from))
|
||||
.build()
|
||||
.expect("direct fixture DTO");
|
||||
let error = match make_usecase()
|
||||
.execute_create_multipart_upload(
|
||||
owned_envelope(Method::POST),
|
||||
CreateMultipartUploadFields::from_legacy(input),
|
||||
None,
|
||||
)
|
||||
let error = make_usecase()
|
||||
.execute_create_multipart_upload_from_s3s(build_request(input, Method::POST))
|
||||
.await
|
||||
{
|
||||
Err(error) => crate::app::storage_api::s3::S3Error::from(error),
|
||||
Ok(_) => panic!("direct fixture does not create an upload"),
|
||||
};
|
||||
.expect_err("direct fixture does not create an upload");
|
||||
assert_eq!(
|
||||
(error.code().as_str(), error.message()),
|
||||
(expected.0, Some(expected.1)),
|
||||
@@ -597,13 +590,12 @@ mod tests {
|
||||
.storage_class(Some(StorageClass::from("invalid".to_string())))
|
||||
.build()
|
||||
.unwrap();
|
||||
let err = match make_usecase()
|
||||
.execute_create_multipart_upload(owned_envelope(Method::POST), CreateMultipartUploadFields::from_legacy(input), None)
|
||||
let req = build_request(input, Method::POST);
|
||||
|
||||
let err = make_usecase()
|
||||
.execute_create_multipart_upload_from_s3s(req)
|
||||
.await
|
||||
{
|
||||
Err(error) => crate::app::storage_api::s3::S3Error::from(error),
|
||||
Ok(_) => panic!("invalid storage class must be rejected"),
|
||||
};
|
||||
.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidStorageClass);
|
||||
}
|
||||
|
||||
|
||||
@@ -295,7 +295,7 @@ mod tests {
|
||||
|
||||
async fn list_uploads(input: gateway_dto::ListMultipartUploadsInput) -> rustfs_s3_types::S3Error {
|
||||
make_usecase()
|
||||
.execute_list_multipart_uploads(owned_envelope(Method::GET), input)
|
||||
.execute_list_multipart_uploads(RequestEnvelope::from_s3s(&build_request((), Method::GET)), input)
|
||||
.await
|
||||
.expect_err("refused")
|
||||
}
|
||||
@@ -393,7 +393,7 @@ mod tests {
|
||||
max_parts,
|
||||
};
|
||||
make_usecase()
|
||||
.execute_list_parts(owned_envelope(Method::GET), fields)
|
||||
.execute_list_parts(RequestEnvelope::from_s3s(&build_request((), Method::GET)), fields)
|
||||
.await
|
||||
.expect_err("refused")
|
||||
}
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
//! ListParts execution against real multipart metadata and the native claim exchange.
|
||||
|
||||
use super::super::test_support::{Method, build_request, owned_envelope};
|
||||
use super::super::test_support::{Method, build_request};
|
||||
use super::*;
|
||||
use crate::app::gating_test_env::{run_large_stack_test, shared_gating_ambient, shared_gating_ecstore_and_disk_paths};
|
||||
use crate::app::storage_api::multipart_usecase::access::{AuthorizationRequest, load_bucket_generation_from_store};
|
||||
@@ -81,7 +81,7 @@ fn list_parts_resolves_real_owners_and_retains_storage_errors() {
|
||||
let key = "fixture/one";
|
||||
let foreign_key = "fixture/two";
|
||||
let opts = ObjectOptions::default();
|
||||
let envelope = || owned_envelope(Method::GET);
|
||||
let envelope = || RequestEnvelope::from_s3s(&build_request((), Method::GET));
|
||||
for name in [&bucket, &foreign_bucket] {
|
||||
store
|
||||
.make_bucket(name, &MakeBucketOptions::default())
|
||||
|
||||
@@ -18,7 +18,6 @@ use super::*;
|
||||
use crate::app::storage_api::s3::S3Request;
|
||||
pub(super) use crate::app::storage_api::s3::dto::CompletedMultipartUpload;
|
||||
pub(super) use crate::app::storage_api::s3::dto::StorageClass;
|
||||
use crate::app::{RequestEnvelope, RequestHead};
|
||||
pub(super) use http::Extensions;
|
||||
pub(super) use http::Method;
|
||||
pub(super) use rustfs_filemeta::ObjectPartInfo;
|
||||
@@ -51,21 +50,6 @@ pub(super) fn build_request<T>(input: T, method: Method) -> S3Request<T> {
|
||||
}
|
||||
}
|
||||
|
||||
/// The envelope of a request that carries nothing but `method`.
|
||||
pub(super) fn owned_envelope(method: Method) -> RequestEnvelope {
|
||||
RequestEnvelope::from_parts(
|
||||
RequestHead {
|
||||
method,
|
||||
uri: Uri::from_static("/"),
|
||||
headers: HeaderMap::new(),
|
||||
credentials: None,
|
||||
region: None,
|
||||
service: None,
|
||||
},
|
||||
Extensions::new(),
|
||||
)
|
||||
}
|
||||
|
||||
pub(super) fn make_usecase() -> DefaultMultipartUsecase {
|
||||
DefaultMultipartUsecase::without_context()
|
||||
}
|
||||
|
||||
@@ -741,13 +741,9 @@ mod tests {
|
||||
.part_number(1)
|
||||
.build()
|
||||
.unwrap();
|
||||
let err = match make_usecase()
|
||||
.execute_upload_part_with_audit(owned_envelope(Method::PUT), UploadPartWriteInput::from_legacy(input, None))
|
||||
.await
|
||||
{
|
||||
Err(error) => crate::app::storage_api::s3::S3Error::from(error),
|
||||
Ok(_) => panic!("missing body must be rejected"),
|
||||
};
|
||||
let req = build_request(input, Method::PUT);
|
||||
|
||||
let err = make_usecase().execute_upload_part(req).await.unwrap_err();
|
||||
assert_eq!(err.code(), &LegacyErrorCode::IncompleteBody);
|
||||
}
|
||||
|
||||
@@ -769,13 +765,9 @@ mod tests {
|
||||
.content_length(Some(declared))
|
||||
.build()
|
||||
.unwrap();
|
||||
let err = match make_usecase()
|
||||
.execute_upload_part_with_audit(owned_envelope(Method::PUT), UploadPartWriteInput::from_legacy(input, None))
|
||||
.await
|
||||
{
|
||||
Err(error) => crate::app::storage_api::s3::S3Error::from(error),
|
||||
Ok(_) => panic!("declared part must be rejected"),
|
||||
};
|
||||
let req = build_request(input, Method::PUT);
|
||||
|
||||
let err = make_usecase().execute_upload_part(req).await.unwrap_err();
|
||||
if expect_too_large {
|
||||
assert_eq!(err.code(), &LegacyErrorCode::EntityTooLarge, "declared {declared}");
|
||||
assert_eq!(polls.load(std::sync::atomic::Ordering::SeqCst), 0, "body must not be polled");
|
||||
@@ -795,13 +787,9 @@ mod tests {
|
||||
.part_number(part_number)
|
||||
.build()
|
||||
.unwrap();
|
||||
let err = match make_usecase()
|
||||
.execute_upload_part_with_audit(owned_envelope(Method::PUT), UploadPartWriteInput::from_legacy(input, None))
|
||||
.await
|
||||
{
|
||||
Err(error) => crate::app::storage_api::s3::S3Error::from(error),
|
||||
Ok(_) => panic!("invalid part number must be rejected"),
|
||||
};
|
||||
let req = build_request(input, Method::PUT);
|
||||
|
||||
let err = make_usecase().execute_upload_part(req).await.unwrap_err();
|
||||
assert_eq!(err.code(), &LegacyErrorCode::InvalidArgument);
|
||||
assert_eq!(err.message(), Some("partNumber must be between 1 and 10000"));
|
||||
}
|
||||
@@ -983,13 +971,10 @@ mod tests {
|
||||
.build()
|
||||
.expect("upload part input should build");
|
||||
|
||||
let err = match usecase
|
||||
.execute_upload_part_with_audit(owned_envelope(Method::PUT), UploadPartWriteInput::from_legacy(input, None))
|
||||
let err = usecase
|
||||
.execute_upload_part(build_request(input, Method::PUT))
|
||||
.await
|
||||
{
|
||||
Err(error) => crate::app::storage_api::s3::S3Error::from(error),
|
||||
Ok(_) => panic!("full foreground write admission should reject before body ingest"),
|
||||
};
|
||||
.expect_err("full foreground write admission should reject before body ingest");
|
||||
assert_eq!(err.code(), &LegacyErrorCode::SlowDown);
|
||||
drop(held);
|
||||
}
|
||||
|
||||
@@ -14,7 +14,6 @@
|
||||
|
||||
//! Upload target ownership and refusal order against real multipart state.
|
||||
|
||||
use super::super::super::test_support::owned_envelope;
|
||||
use super::*;
|
||||
use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions};
|
||||
use rustfs_gateway_types::{BucketName, ObjectKey, UploadIdClaim};
|
||||
@@ -102,7 +101,7 @@ async fn native_claim_checks_order_ownership_and_one_active_lookup() {
|
||||
Body::from(Bytes::from_static(b"abc"))
|
||||
}
|
||||
});
|
||||
let envelope = owned_envelope(Method::PUT);
|
||||
let envelope = RequestEnvelope::from_s3s(&build_request((), Method::PUT));
|
||||
LOOKUP
|
||||
.scope((Cell::new(0), mode), async {
|
||||
let result = usecase.execute_upload_part_with_audit(envelope, input).await;
|
||||
@@ -212,7 +211,7 @@ fn native_copy_part_rejects_unproved_owner_without_changing_parts() {
|
||||
copy_source_sse_customer_key: None,
|
||||
copy_source_sse_customer_key_md5: None,
|
||||
};
|
||||
let envelope = owned_envelope(Method::PUT);
|
||||
let envelope = RequestEnvelope::from_s3s(&build_request((), Method::PUT));
|
||||
LOOKUP
|
||||
.scope((Cell::new(0), mode), async {
|
||||
let result = Box::pin(usecase.execute_upload_part_copy_with_audit(envelope, input)).await;
|
||||
|
||||
@@ -561,9 +561,6 @@ impl DefaultMultipartUsecase {
|
||||
mod tests {
|
||||
use super::super::test_support::*;
|
||||
use super::*;
|
||||
use crate::app::multipart_usecase::UploadPartTarget;
|
||||
use crate::app::object_usecase::CopySourceCondition;
|
||||
use crate::app::storage_api::s3::dto::ETag;
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_upload_part_copy_returns_internal_error_when_store_uninitialized() {
|
||||
@@ -579,37 +576,9 @@ mod tests {
|
||||
.upload_id("upload-id".to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
let condition = |value| match value {
|
||||
crate::app::storage_api::s3::dto::ETagCondition::Any => CopySourceCondition::Any,
|
||||
crate::app::storage_api::s3::dto::ETagCondition::ETag(ETag::Strong(value)) => CopySourceCondition::Strong(value),
|
||||
crate::app::storage_api::s3::dto::ETagCondition::ETag(ETag::Weak(_)) => CopySourceCondition::Weak,
|
||||
};
|
||||
let err = match Box::pin(make_usecase().execute_upload_part_copy_with_audit(
|
||||
owned_envelope(Method::PUT),
|
||||
UploadPartCopyWriteInput {
|
||||
target: UploadPartTarget::Legacy {
|
||||
bucket: input.bucket,
|
||||
key: input.key,
|
||||
upload_id: input.upload_id,
|
||||
},
|
||||
part_number: input.part_number,
|
||||
copy_source: rustfs_gateway_types::compat::s3s_5761ddfe::copy_source::from_s3s(input.copy_source),
|
||||
copy_source_range: input.copy_source_range,
|
||||
copy_source_if_match: input.copy_source_if_match.map(&condition),
|
||||
copy_source_if_none_match: input.copy_source_if_none_match.map(&condition),
|
||||
sse_customer_algorithm: input.sse_customer_algorithm,
|
||||
sse_customer_key: input.sse_customer_key,
|
||||
sse_customer_key_md5: input.sse_customer_key_md5,
|
||||
copy_source_sse_customer_algorithm: input.copy_source_sse_customer_algorithm,
|
||||
copy_source_sse_customer_key: input.copy_source_sse_customer_key,
|
||||
copy_source_sse_customer_key_md5: input.copy_source_sse_customer_key_md5,
|
||||
},
|
||||
))
|
||||
.await
|
||||
{
|
||||
Err(error) => crate::app::storage_api::s3::S3Error::from(error),
|
||||
Ok(_) => panic!("uninitialized store must be an internal error"),
|
||||
};
|
||||
let req = build_request(input, Method::PUT);
|
||||
|
||||
let err = Box::pin(make_usecase().execute_upload_part_copy(req)).await.unwrap_err();
|
||||
assert_eq!(err.code(), &crate::app::storage_api::s3::S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
@@ -628,37 +597,9 @@ mod tests {
|
||||
.upload_id("upload-id".to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
let condition = |value| match value {
|
||||
crate::app::storage_api::s3::dto::ETagCondition::Any => CopySourceCondition::Any,
|
||||
crate::app::storage_api::s3::dto::ETagCondition::ETag(ETag::Strong(value)) => CopySourceCondition::Strong(value),
|
||||
crate::app::storage_api::s3::dto::ETagCondition::ETag(ETag::Weak(_)) => CopySourceCondition::Weak,
|
||||
};
|
||||
let err = match Box::pin(make_usecase().execute_upload_part_copy_with_audit(
|
||||
owned_envelope(Method::PUT),
|
||||
UploadPartCopyWriteInput {
|
||||
target: UploadPartTarget::Legacy {
|
||||
bucket: input.bucket,
|
||||
key: input.key,
|
||||
upload_id: input.upload_id,
|
||||
},
|
||||
part_number: input.part_number,
|
||||
copy_source: rustfs_gateway_types::compat::s3s_5761ddfe::copy_source::from_s3s(input.copy_source),
|
||||
copy_source_range: input.copy_source_range,
|
||||
copy_source_if_match: input.copy_source_if_match.map(&condition),
|
||||
copy_source_if_none_match: input.copy_source_if_none_match.map(&condition),
|
||||
sse_customer_algorithm: input.sse_customer_algorithm,
|
||||
sse_customer_key: input.sse_customer_key,
|
||||
sse_customer_key_md5: input.sse_customer_key_md5,
|
||||
copy_source_sse_customer_algorithm: input.copy_source_sse_customer_algorithm,
|
||||
copy_source_sse_customer_key: input.copy_source_sse_customer_key,
|
||||
copy_source_sse_customer_key_md5: input.copy_source_sse_customer_key_md5,
|
||||
},
|
||||
))
|
||||
.await
|
||||
{
|
||||
Err(error) => crate::app::storage_api::s3::S3Error::from(error),
|
||||
Ok(_) => panic!("invalid part number must be rejected"),
|
||||
};
|
||||
let req = build_request(input, Method::PUT);
|
||||
|
||||
let err = Box::pin(make_usecase().execute_upload_part_copy(req)).await.unwrap_err();
|
||||
assert_eq!(err.code(), &crate::app::storage_api::s3::S3ErrorCode::InvalidArgument);
|
||||
assert_eq!(err.message(), Some("partNumber must be between 1 and 10000"));
|
||||
}
|
||||
|
||||
@@ -76,38 +76,9 @@ fn upload_part_copy_shares_foreground_admission_with_upload_part() {
|
||||
.any(|label| label.key() == "size_bucket" && label.value() == source_size_bucket)
|
||||
})
|
||||
};
|
||||
let saturated = input.clone();
|
||||
let condition = |value| match value {
|
||||
crate::app::storage_api::s3::dto::ETagCondition::Any => CopySourceCondition::Any,
|
||||
crate::app::storage_api::s3::dto::ETagCondition::ETag(ETag::Strong(value)) => CopySourceCondition::Strong(value),
|
||||
crate::app::storage_api::s3::dto::ETagCondition::ETag(ETag::Weak(_)) => CopySourceCondition::Weak,
|
||||
};
|
||||
let error = match Box::pin(usecase.execute_upload_part_copy_with_audit(
|
||||
owned_envelope(Method::PUT),
|
||||
UploadPartCopyWriteInput {
|
||||
target: UploadPartTarget::Legacy {
|
||||
bucket: saturated.bucket,
|
||||
key: saturated.key,
|
||||
upload_id: saturated.upload_id,
|
||||
},
|
||||
part_number: saturated.part_number,
|
||||
copy_source: rustfs_gateway_types::compat::s3s_5761ddfe::copy_source::from_s3s(saturated.copy_source),
|
||||
copy_source_range: saturated.copy_source_range,
|
||||
copy_source_if_match: saturated.copy_source_if_match.map(&condition),
|
||||
copy_source_if_none_match: saturated.copy_source_if_none_match.map(&condition),
|
||||
sse_customer_algorithm: saturated.sse_customer_algorithm,
|
||||
sse_customer_key: saturated.sse_customer_key,
|
||||
sse_customer_key_md5: saturated.sse_customer_key_md5,
|
||||
copy_source_sse_customer_algorithm: saturated.copy_source_sse_customer_algorithm,
|
||||
copy_source_sse_customer_key: saturated.copy_source_sse_customer_key,
|
||||
copy_source_sse_customer_key_md5: saturated.copy_source_sse_customer_key_md5,
|
||||
},
|
||||
))
|
||||
.await
|
||||
{
|
||||
Err(error) => crate::app::storage_api::s3::S3Error::from(error),
|
||||
Ok(_) => panic!("copy must not bypass the occupied UploadPart pool"),
|
||||
};
|
||||
let error = Box::pin(usecase.execute_upload_part_copy(build_request(input.clone(), Method::PUT)))
|
||||
.await
|
||||
.expect_err("copy must not bypass the occupied UploadPart pool");
|
||||
assert_eq!(error.code(), &S3ErrorCode::SlowDown);
|
||||
assert!(!source_reader_observed(), "saturated copy must not construct a source body reader");
|
||||
let parts = store
|
||||
@@ -194,37 +165,9 @@ fn upload_part_copy_shares_foreground_admission_with_upload_part() {
|
||||
.upload_id(deleted_upload.upload_id.clone())
|
||||
.build()
|
||||
.expect("deleted-source copy request");
|
||||
let condition = |value| match value {
|
||||
crate::app::storage_api::s3::dto::ETagCondition::Any => CopySourceCondition::Any,
|
||||
crate::app::storage_api::s3::dto::ETagCondition::ETag(ETag::Strong(value)) => CopySourceCondition::Strong(value),
|
||||
crate::app::storage_api::s3::dto::ETagCondition::ETag(ETag::Weak(_)) => CopySourceCondition::Weak,
|
||||
};
|
||||
let error = match Box::pin(usecase.execute_upload_part_copy_with_audit(
|
||||
owned_envelope(Method::PUT),
|
||||
UploadPartCopyWriteInput {
|
||||
target: UploadPartTarget::Legacy {
|
||||
bucket: input.bucket,
|
||||
key: input.key,
|
||||
upload_id: input.upload_id,
|
||||
},
|
||||
part_number: input.part_number,
|
||||
copy_source: rustfs_gateway_types::compat::s3s_5761ddfe::copy_source::from_s3s(input.copy_source),
|
||||
copy_source_range: input.copy_source_range,
|
||||
copy_source_if_match: input.copy_source_if_match.map(&condition),
|
||||
copy_source_if_none_match: input.copy_source_if_none_match.map(&condition),
|
||||
sse_customer_algorithm: input.sse_customer_algorithm,
|
||||
sse_customer_key: input.sse_customer_key,
|
||||
sse_customer_key_md5: input.sse_customer_key_md5,
|
||||
copy_source_sse_customer_algorithm: input.copy_source_sse_customer_algorithm,
|
||||
copy_source_sse_customer_key: input.copy_source_sse_customer_key,
|
||||
copy_source_sse_customer_key_md5: input.copy_source_sse_customer_key_md5,
|
||||
},
|
||||
))
|
||||
.await
|
||||
{
|
||||
Err(error) => crate::app::storage_api::s3::S3Error::from(error),
|
||||
Ok(_) => panic!("deleted source must fail before conditions or admission"),
|
||||
};
|
||||
let error = Box::pin(usecase.execute_upload_part_copy(build_request(input, Method::PUT)))
|
||||
.await
|
||||
.expect_err("deleted source must fail before conditions or admission");
|
||||
assert_eq!(error.code(), &expected);
|
||||
}
|
||||
});
|
||||
@@ -268,32 +211,7 @@ fn upload_part_copy_range_error_and_cancellation_release_foreground_permit() {
|
||||
.upload_id(cancelled_upload.upload_id.clone())
|
||||
.build()
|
||||
.expect("cancelled copy request");
|
||||
let condition = |value| match value {
|
||||
crate::app::storage_api::s3::dto::ETagCondition::Any => CopySourceCondition::Any,
|
||||
crate::app::storage_api::s3::dto::ETagCondition::ETag(ETag::Strong(value)) => CopySourceCondition::Strong(value),
|
||||
crate::app::storage_api::s3::dto::ETagCondition::ETag(ETag::Weak(_)) => CopySourceCondition::Weak,
|
||||
};
|
||||
let mut cancelled_copy = Box::pin(usecase.execute_upload_part_copy_with_audit(
|
||||
owned_envelope(Method::PUT),
|
||||
UploadPartCopyWriteInput {
|
||||
target: UploadPartTarget::Legacy {
|
||||
bucket: cancelled_input.bucket,
|
||||
key: cancelled_input.key,
|
||||
upload_id: cancelled_input.upload_id,
|
||||
},
|
||||
part_number: cancelled_input.part_number,
|
||||
copy_source: rustfs_gateway_types::compat::s3s_5761ddfe::copy_source::from_s3s(cancelled_input.copy_source),
|
||||
copy_source_range: cancelled_input.copy_source_range,
|
||||
copy_source_if_match: cancelled_input.copy_source_if_match.map(&condition),
|
||||
copy_source_if_none_match: cancelled_input.copy_source_if_none_match.map(&condition),
|
||||
sse_customer_algorithm: cancelled_input.sse_customer_algorithm,
|
||||
sse_customer_key: cancelled_input.sse_customer_key,
|
||||
sse_customer_key_md5: cancelled_input.sse_customer_key_md5,
|
||||
copy_source_sse_customer_algorithm: cancelled_input.copy_source_sse_customer_algorithm,
|
||||
copy_source_sse_customer_key: cancelled_input.copy_source_sse_customer_key,
|
||||
copy_source_sse_customer_key_md5: cancelled_input.copy_source_sse_customer_key_md5,
|
||||
},
|
||||
));
|
||||
let mut cancelled_copy = Box::pin(usecase.execute_upload_part_copy(build_request(cancelled_input, Method::PUT)));
|
||||
tokio::time::timeout(Duration::from_secs(10), async {
|
||||
tokio::select! {
|
||||
biased;
|
||||
@@ -302,10 +220,7 @@ fn upload_part_copy_range_error_and_cancellation_release_foreground_permit() {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
} => {}
|
||||
result = &mut cancelled_copy => match result {
|
||||
Ok(_) => panic!("copy completed before cancellation"),
|
||||
Err(error) => panic!("copy completed before cancellation: {error:?}"),
|
||||
},
|
||||
result = &mut cancelled_copy => panic!("copy completed before cancellation: {result:?}"),
|
||||
}
|
||||
})
|
||||
.await
|
||||
@@ -345,39 +260,17 @@ fn upload_part_copy_range_error_and_cancellation_release_foreground_permit() {
|
||||
.upload_id(upload.upload_id.clone())
|
||||
.build()
|
||||
.expect("range request");
|
||||
let response = Box::pin(usecase.execute_upload_part_copy_with_audit(
|
||||
owned_envelope(Method::PUT),
|
||||
UploadPartCopyWriteInput {
|
||||
target: UploadPartTarget::Legacy {
|
||||
bucket: input.bucket,
|
||||
key: input.key,
|
||||
upload_id: input.upload_id,
|
||||
},
|
||||
part_number: input.part_number,
|
||||
copy_source: rustfs_gateway_types::compat::s3s_5761ddfe::copy_source::from_s3s(input.copy_source),
|
||||
copy_source_range: input.copy_source_range,
|
||||
copy_source_if_match: input.copy_source_if_match.map(&condition),
|
||||
copy_source_if_none_match: input.copy_source_if_none_match.map(&condition),
|
||||
sse_customer_algorithm: input.sse_customer_algorithm,
|
||||
sse_customer_key: input.sse_customer_key,
|
||||
sse_customer_key_md5: input.sse_customer_key_md5,
|
||||
copy_source_sse_customer_algorithm: input.copy_source_sse_customer_algorithm,
|
||||
copy_source_sse_customer_key: input.copy_source_sse_customer_key,
|
||||
copy_source_sse_customer_key_md5: input.copy_source_sse_customer_key_md5,
|
||||
},
|
||||
))
|
||||
.await;
|
||||
let response = Box::pin(usecase.execute_upload_part_copy(build_request(input, Method::PUT))).await;
|
||||
assert_eq!(
|
||||
manager.put_object_admission_snapshot().active,
|
||||
Some(0),
|
||||
"copy releases its permit on every outcome"
|
||||
);
|
||||
if capped {
|
||||
let error = match response {
|
||||
Err(error) => crate::app::storage_api::s3::S3Error::from(error),
|
||||
Ok(_) => panic!("range exceeds destination size limit"),
|
||||
};
|
||||
assert_eq!(error.code(), &S3ErrorCode::EntityTooLarge);
|
||||
assert_eq!(
|
||||
response.expect_err("range exceeds destination size limit").code(),
|
||||
&S3ErrorCode::EntityTooLarge
|
||||
);
|
||||
let parts = store
|
||||
.list_object_parts(&bucket, "object", &upload.upload_id, None, 1000, &ObjectOptions::default())
|
||||
.await
|
||||
@@ -388,16 +281,16 @@ fn upload_part_copy_range_error_and_cancellation_release_foreground_permit() {
|
||||
.await
|
||||
.expect("abort capped session");
|
||||
} else {
|
||||
let copied = match response {
|
||||
Ok(output) => output,
|
||||
Err(error) => panic!("range copy after failed request released its source and permit: {error:?}"),
|
||||
};
|
||||
let copied = response.expect("range copy after failed request released its source and permit");
|
||||
let etag = copied
|
||||
.output
|
||||
.copy_part_result
|
||||
.expect("range result")
|
||||
.e_tag
|
||||
.as_deref()
|
||||
.map(crate::storage::storage_api::to_s3s_xml_etag)
|
||||
.expect("range ETag");
|
||||
let etag = etag.value().trim_matches('"').to_owned();
|
||||
.expect("range ETag")
|
||||
.value()
|
||||
.trim_matches('"')
|
||||
.to_owned();
|
||||
Arc::clone(&store)
|
||||
.complete_multipart_upload(
|
||||
&bucket,
|
||||
|
||||
@@ -258,17 +258,8 @@ mod tests {
|
||||
sse_customer_key: None,
|
||||
sse_customer_key_md5: None,
|
||||
};
|
||||
let envelope = RequestEnvelope::from_parts(
|
||||
crate::app::RequestHead {
|
||||
method: http::Method::GET,
|
||||
uri: http::Uri::from_static("/"),
|
||||
headers: http::HeaderMap::new(),
|
||||
credentials: None,
|
||||
region: None,
|
||||
service: None,
|
||||
},
|
||||
http::Extensions::new(),
|
||||
);
|
||||
let request = build_request((), http::Method::GET);
|
||||
let envelope = RequestEnvelope::from_s3s(&request);
|
||||
let error = DefaultObjectUsecase::without_context()
|
||||
.execute_get_object_attributes(envelope, fields)
|
||||
.await
|
||||
|
||||
@@ -15,14 +15,13 @@
|
||||
//! PutObject edge authorization and native frame tests against real storage.
|
||||
|
||||
use super::*;
|
||||
use crate::app::RequestHead;
|
||||
use crate::runtime_sources::ServerContextSlot;
|
||||
use crate::storage_api::server::http::access::test_support::{
|
||||
AuthorizationRequest, BucketOperations as _, CompletePart, DeleteBucketOptions, MakeBucketOptions, MultipartOperations as _,
|
||||
ObjectIO as _, ObjectOperations as _, ObjectOptions, ReqInfo, get_bucket_metadata, set_bucket_metadata,
|
||||
ObjectIO as _, ObjectOperations as _, ObjectOptions, ReqInfo, S3Request, get_bucket_metadata, set_bucket_metadata,
|
||||
};
|
||||
use bytes::Bytes;
|
||||
use http::{Extensions, HeaderMap, HeaderValue, Method, Uri};
|
||||
use http::{HeaderMap, HeaderValue, Method};
|
||||
use rustfs_gateway::{Limits, MetaView, SseConfig, TargetKind, TransportSecurity, WireRequest};
|
||||
use rustfs_gateway_stream::{ByteStream, PayloadCaps, PayloadRead, PayloadStream, StreamError, TrailingHeaders};
|
||||
use serial_test::serial;
|
||||
@@ -75,20 +74,11 @@ async fn set_policy(bucket: &str, statement: serde_json::Value) {
|
||||
}
|
||||
|
||||
fn envelope(server: &Arc<ServerContextSlot>, headers: HeaderMap) -> RequestEnvelope {
|
||||
let mut extensions = Extensions::new();
|
||||
extensions.insert(Arc::clone(server));
|
||||
extensions.insert(ReqInfo::default());
|
||||
RequestEnvelope::from_parts(
|
||||
RequestHead {
|
||||
method: Method::PUT,
|
||||
uri: Uri::from_static("/"),
|
||||
headers,
|
||||
credentials: None,
|
||||
region: None,
|
||||
service: None,
|
||||
},
|
||||
extensions,
|
||||
)
|
||||
let mut legacy = build_request((), Method::PUT);
|
||||
legacy.extensions.insert(Arc::clone(server));
|
||||
legacy.extensions.insert(ReqInfo::default());
|
||||
legacy.headers = headers;
|
||||
RequestEnvelope::from_s3s(&legacy)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -385,6 +375,20 @@ async fn native_put_frames_reach_checksum_validation_and_storage() {
|
||||
}
|
||||
}
|
||||
|
||||
fn build_request<T>(input: T, method: Method) -> S3Request<T> {
|
||||
S3Request {
|
||||
input,
|
||||
method,
|
||||
uri: http::Uri::from_static("/"),
|
||||
headers: HeaderMap::new(),
|
||||
extensions: http::Extensions::new(),
|
||||
credentials: None,
|
||||
region: None,
|
||||
service: None,
|
||||
trailing_headers: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn put_source_version_selects_versioned_read_permission() {
|
||||
@@ -393,26 +397,16 @@ async fn put_source_version_selects_versioned_read_permission() {
|
||||
for (read, allowed) in [("s3:GetObject", false), ("s3:GetObjectVersion", true)] {
|
||||
let (server, bucket) = fixture(&["s3:PutObject", read, "s3:ReplicateObject"]).await;
|
||||
let mut headers = HeaderMap::new();
|
||||
let uri = if query {
|
||||
format!("/?versionId={VERSION}").parse().unwrap()
|
||||
let mut legacy = build_request((), Method::PUT);
|
||||
legacy.extensions.insert(Arc::clone(&server));
|
||||
legacy.extensions.insert(ReqInfo::default());
|
||||
if query {
|
||||
legacy.uri = format!("/?versionId={VERSION}").parse().unwrap();
|
||||
} else {
|
||||
headers.insert("x-minio-source-version-id", HeaderValue::from_static(VERSION));
|
||||
Uri::from_static("/")
|
||||
};
|
||||
let mut extensions = Extensions::new();
|
||||
extensions.insert(Arc::clone(&server));
|
||||
extensions.insert(ReqInfo::default());
|
||||
let mut envelope = RequestEnvelope::from_parts(
|
||||
RequestHead {
|
||||
method: Method::PUT,
|
||||
uri,
|
||||
headers,
|
||||
credentials: None,
|
||||
region: None,
|
||||
service: None,
|
||||
},
|
||||
extensions,
|
||||
);
|
||||
}
|
||||
legacy.headers = headers;
|
||||
let mut envelope = RequestEnvelope::from_s3s(&legacy);
|
||||
let req = request(dto::PutObjectInput {
|
||||
bucket: rustfs_gateway_types::BucketName::new(bucket).unwrap(),
|
||||
key: rustfs_gateway_types::ObjectKey::new("allowed").unwrap(),
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Authorization shapes that only migration batch A2 (rustfs/backlog#2765) uses: bucket policy, policy status, versioning, notification and ACL.
|
||||
//!
|
||||
//! Responsible for: the owned authorization function of each such operation, generic over
|
||||
//! `AuthorizationRequest`, so the legacy `S3Access` hook and `Served::authorize` call the same code.
|
||||
//! Not responsible for: shapes more than one batch shares (those stay in `storage/access.rs`), the
|
||||
//! policy evaluation itself (`authorize_request`), or the operation's use case.
|
||||
//! Upstream: the legacy hook in `storage/access.rs` and the batch's `server/gateway_handlers/bucket_access.rs`.
|
||||
//! Downstream: the shared shapes in `storage/access.rs`.
|
||||
@@ -0,0 +1,22 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Authorization shapes that only migration batch A1 (rustfs/backlog#2765) uses: bucket CORS, tagging, public access block and encryption.
|
||||
//!
|
||||
//! Responsible for: the owned authorization function of each such operation, generic over
|
||||
//! `AuthorizationRequest`, so the legacy `S3Access` hook and `Served::authorize` call the same code.
|
||||
//! Not responsible for: shapes more than one batch shares (those stay in `storage/access.rs`), the
|
||||
//! policy evaluation itself (`authorize_request`), or the operation's use case.
|
||||
//! Upstream: the legacy hook in `storage/access.rs` and the batch's `server/gateway_handlers/bucket_configs.rs`.
|
||||
//! Downstream: the shared shapes in `storage/access.rs`.
|
||||
@@ -15,6 +15,8 @@
|
||||
//! The authorization shapes one migration batch owns (rustfs/backlog#2765), one file per batch so
|
||||
//! parallel batches edit disjoint files. Shapes several batches share stay in `storage/access.rs`.
|
||||
|
||||
pub(crate) mod bucket_access;
|
||||
pub(crate) mod bucket_configs;
|
||||
pub(crate) mod bucket_core;
|
||||
pub(crate) mod bucket_replication_lifecycle;
|
||||
pub(crate) mod listing_multipart;
|
||||
|
||||
@@ -233,6 +233,7 @@ pub(crate) mod server {
|
||||
StorageObjectOptions as ObjectOptions, StoragePutObjReader, get_bucket_metadata, is_err_object_not_found,
|
||||
set_bucket_metadata,
|
||||
};
|
||||
pub(crate) use s3s::S3Request;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user