Compare commits

..

2 Commits

Author SHA1 Message Date
overtrue 635746534b test(ci): allow equal timestamps for parallel scanner probes 2026-11-21 06:48:16 +08:00
overtrue 6a94097b97 test(ci): make storage fixtures wait for their prerequisites 2026-11-21 06:37:09 +08:00
8 changed files with 52 additions and 346 deletions
+1 -52
View File
@@ -34,10 +34,7 @@
#[cfg(test)]
mod tests {
use crate::common::{
AdminTransport, RustFSTestEnvironment, admin_attach_user_policy_via, admin_create_user, admin_request_with_session_token,
init_logging, local_http_client, parse_assume_role_credentials,
};
use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
use hmac::{Hmac, KeyInit as _, Mac};
use http::header::HOST;
use rustfs_s3_types::Body;
@@ -399,54 +396,6 @@ mod tests {
let (status, _) = signed_request(&env.url, http::Method::GET, ACCOUNT_INFO_PATH, None, user_ak, old_sk).await?;
assert!(status.is_client_error(), "the old secret must stop working, status: {status}");
env.stop_server();
drop(env);
console_session_rotates_the_iam_password_and_revokes_the_old_session().await
}
async fn console_session_rotates_the_iam_password_and_revokes_the_old_session() -> Result<(), Box<dyn Error + Send + Sync>> {
init_logging();
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server(vec![]).await?;
let user = "console-password-user";
let old_password = uuid::Uuid::new_v4().to_string();
let new_password = uuid::Uuid::new_v4().to_string();
admin_create_user(&env, user, &old_password).await?;
admin_attach_user_policy_via(AdminTransport::Signed, &env.url, &env.access_key, &env.secret_key, "readonly", user)
.await?;
let (status, response) = assume_role(&env.url, user, &old_password, None).await?;
assert_eq!(status, reqwest::StatusCode::OK, "the current password must sign in");
let (session_key, session_secret, token) = parse_assume_role_credentials(&response)?;
let session_request = |method, path, body| {
admin_request_with_session_token(&env.url, method, path, body, &session_key, &session_secret, Some(&token))
};
let password_request = |current: &str| {
Some(serde_json::json!({ "current_secret_key": current, "new_secret_key": new_password }).to_string())
};
let (status, body) =
session_request(http::Method::POST, ACCOUNT_PASSWORD_PATH, password_request(&session_secret)).await?;
assert_eq!(
status,
reqwest::StatusCode::BAD_REQUEST,
"the session secret must not be accepted as the IAM password"
);
assert!(body.contains("InvalidRequest") && body.contains("the current secret key is incorrect"));
let (status, body) = session_request(http::Method::POST, ACCOUNT_PASSWORD_PATH, password_request(&old_password)).await?;
assert_eq!(status, reqwest::StatusCode::OK, "the same session must accept the current IAM password");
assert_eq!(serde_json::from_str::<serde_json::Value>(&body)?["sessions_revoked"], 1);
let (status, _) = session_request(http::Method::GET, ACCOUNT_INFO_PATH, None).await?;
assert_eq!(status, reqwest::StatusCode::FORBIDDEN, "the pre-rotation session must be revoked");
let (status, _) = assume_role(&env.url, user, &old_password, None).await?;
assert_eq!(status, reqwest::StatusCode::FORBIDDEN, "the old password must no longer sign in");
let (status, response) = assume_role(&env.url, user, &new_password, None).await?;
assert_eq!(status, reqwest::StatusCode::OK, "the new password must sign in");
parse_assume_role_credentials(&response)?;
env.stop_server();
Ok(())
}
@@ -5082,7 +5082,14 @@ mod tests {
let cert = generate_simple_self_signed(vec!["localhost".to_string()]).expect("certificate should generate");
let (port, server) = spawn_https_server(&cert, 2);
let endpoint = format!("https://localhost:{port}");
let strict = target_client_for_test("arn:strict", endpoint.clone(), s3_client_for_test(port, None));
let strict_client = s3_client_for_test(port, None);
// A failed trust check must consume just one of the two fixture connections.
let strict_config = strict_client
.config()
.to_builder()
.retry_config(aws_sdk_s3::config::retry::RetryConfig::disabled())
.build();
let strict = target_client_for_test("arn:strict", endpoint.clone(), S3Client::from_conf(strict_config));
let insecure = target_client_for_test(
"arn:insecure",
endpoint,
+1 -47
View File
@@ -1786,7 +1786,7 @@ where
}
pub async fn update_user_secret_key(&self, access_key: &str, secret_key: &str) -> Result<(OffsetDateTime, AccountStatus)> {
self.update_user_secret_key_with_guard(access_key, secret_key, None, None)
self.update_user_secret_key_with_guard(access_key, secret_key, None)
.await
.map(|(updated_at, status, _guard)| (updated_at, status))
}
@@ -1796,7 +1796,6 @@ where
access_key: &str,
secret_key: &str,
expected_secret: Option<&str>,
session: Option<&Credentials>,
) -> Result<(
OffsetDateTime,
AccountStatus,
@@ -1825,51 +1824,6 @@ where
}
}
// Lock order is parent, then the presenting STS credential. Deletion and
// session refresh use the same STS lock. Release it before the revocation sweep.
let _session_guard = match session {
Some(session) => {
let guard = self.cache.sts_account_mutation_lock(&session.access_key).lock_owned().await;
let current = self
.api
.load_user_identity(&session.access_key, UserType::Sts)
.await
.map_err(|error| {
if is_err_no_such_user(&error) {
Error::IAMActionNotAllowed
} else {
error
}
})?;
let parent = if current.credentials.parent_user.is_empty() {
current.credentials.claims_or_empty().get("parent").and_then(Value::as_str)
} else {
Some(current.credentials.parent_user.as_str())
};
use subtle::ConstantTimeEq as _;
if !session.is_valid()
|| !current.credentials.is_valid()
|| !current.credentials.is_temp()
|| current.credentials.is_service_account()
|| current.credentials.access_key != session.access_key
|| parent != Some(access_key)
|| !bool::from(current.credentials.secret_key.as_bytes().ct_eq(session.secret_key.as_bytes()))
|| !bool::from(
current
.credentials
.session_token
.as_bytes()
.ct_eq(session.session_token.as_bytes()),
)
|| current.sts_generation != u.sts_generation
{
return Err(Error::IAMActionNotAllowed);
}
Some(guard)
}
None => None,
};
let mut cred = u.credentials.clone();
cred.secret_key = secret_key.to_string();
+6 -183
View File
@@ -1421,11 +1421,11 @@ impl<T: Store> IamSys<T> {
access_key: &str,
secret_key: &str,
) -> Result<(OffsetDateTime, rustfs_madmin::AccountStatus, usize)> {
self.rotate_user_secret_key_checked(access_key, secret_key, None, None).await
self.rotate_user_secret_key_checked(access_key, secret_key, None).await
}
/// Change an enabled regular user's secret only if the supplied current
/// secret still matches under the parent mutation lock. Status and policies are kept.
/// Change an enabled regular caller's secret only if its signing secret is
/// still current under the parent mutation lock. Status and policies are kept.
/// Persistence and revocation continue after caller cancellation.
pub async fn change_own_secret_key(
&self,
@@ -1433,27 +1433,7 @@ impl<T: Store> IamSys<T> {
expected_secret: &str,
secret_key: &str,
) -> Result<(OffsetDateTime, rustfs_madmin::AccountStatus, usize)> {
self.rotate_user_secret_key_checked(access_key, secret_key, Some(expected_secret), None)
.await
}
/// Change a local IAM password using an authenticated STS session and its parent password.
///
/// The parent and session mutation locks protect revalidation through the secret write.
/// Persistence and session revocation continue after request cancellation.
/// A revocation failure is returned after the new secret has been persisted.
///
/// # Errors
/// Returns `IAMActionNotAllowed` for inactive sessions, parent mismatches, or an
/// incorrect current password. Storage and revocation errors are propagated.
pub async fn change_own_secret_key_from_session(
&self,
access_key: &str,
expected_secret: &str,
secret_key: &str,
session: &Credentials,
) -> Result<(OffsetDateTime, rustfs_madmin::AccountStatus, usize)> {
self.rotate_user_secret_key_checked(access_key, secret_key, Some(expected_secret), Some(session))
self.rotate_user_secret_key_checked(access_key, secret_key, Some(expected_secret))
.await
}
@@ -1462,18 +1442,16 @@ impl<T: Store> IamSys<T> {
access_key: &str,
secret_key: &str,
expected_secret: Option<&str>,
session: Option<&Credentials>,
) -> Result<(OffsetDateTime, rustfs_madmin::AccountStatus, usize)> {
let runtime = tokio::runtime::Handle::try_current().map_err(Error::other)?;
let iam = self.clone();
let access_key = access_key.to_string();
let secret_key = secret_key.to_string();
let expected_secret = expected_secret.map(str::to_string);
let session = session.cloned();
#[cfg(test)]
let notification_probe = crate::LOAD_USER_NOTIFICATION_PROBE.try_with(Arc::clone).ok();
let operation = async move {
iam.rotate_user_secret_key_inner(&access_key, &secret_key, expected_secret.as_deref(), session.as_ref())
iam.rotate_user_secret_key_inner(&access_key, &secret_key, expected_secret.as_deref())
.await
};
#[cfg(test)]
@@ -1493,7 +1471,6 @@ impl<T: Store> IamSys<T> {
access_key: &str,
secret_key: &str,
expected_secret: Option<&str>,
session: Option<&Credentials>,
) -> Result<(OffsetDateTime, rustfs_madmin::AccountStatus, usize)> {
if !is_access_key_valid(access_key) {
return Err(IamError::InvalidAccessKeyLength);
@@ -1505,7 +1482,7 @@ impl<T: Store> IamSys<T> {
let (updated_at, status, parent_guard) = self
.store
.update_user_secret_key_with_guard(access_key, secret_key, expected_secret, session)
.update_user_secret_key_with_guard(access_key, secret_key, expected_secret)
.await?;
let revoked = self.revoke_sts_sessions_for_parent(access_key).await?;
drop(parent_guard);
@@ -4838,160 +4815,6 @@ mod tests {
assert!(matches!(issuer.check_key(&legacy.access_key).await, Ok((None, false))));
}
#[tokio::test]
async fn own_secret_change_revalidates_the_presenting_session_under_the_parent_lock() {
ensure_test_global_credentials();
for mutation in ["revoke", "expire", "disable", "parent", "secret", "token", "generation"] {
let store = StsTestMockStore::new(false);
let iam = Arc::new(IamSys::new(IamCache::new(store.clone()).await.expect("IAM")));
let parent = "session-change-parent";
let password = "currentParentSecret123";
iam.create_user(
parent,
&AddOrUpdateUserReq {
secret_key: password.to_string(),
policy: None,
status: rustfs_madmin::AccountStatus::Enabled,
},
)
.await
.expect("create parent");
let session = rotation_test_session(parent, "password-session");
let owner = iam.get_user(parent).await.expect("parent").credentials;
iam.set_temp_user_from_parent(&session, &owner)
.await
.expect("publish session");
let guard = iam.store.cache.parent_mutation_lock(parent).lock_owned().await;
let pending = {
let iam = Arc::clone(&iam);
let session = session.clone();
tokio::spawn(async move {
iam.change_own_secret_key_from_session(parent, password, "replacementParentSecret123", &session)
.await
})
};
if mutation == "revoke" {
iam.delete_temp_account(&session.access_key, false)
.await
.expect("revoke presenting session");
} else {
let mut saved = store.saved_sts_users.lock().expect("saved identities");
let current = saved.get_mut(&session.access_key).expect("durable session");
match mutation {
"expire" => current.credentials.expiration = Some(OffsetDateTime::UNIX_EPOCH),
"disable" => current.credentials.status = "off".to_string(),
"parent" => current.credentials.parent_user = "another-parent".to_string(),
"secret" => current.credentials.secret_key = "replacementSessionSecret123".to_string(),
"token" => current.credentials.session_token = "replacement-token".to_string(),
"generation" => {
assert!(current.sts_generation.take().is_some());
}
_ => unreachable!("known fixture mutation"),
}
}
drop(guard);
let error = tokio::time::timeout(std::time::Duration::from_secs(5), pending)
.await
.expect("queued rotation finishes")
.expect("rotation task")
.expect_err("stale session must fail");
assert_eq!(error, Error::IAMActionNotAllowed, "{mutation}");
let saved = store.saved_sts_users.lock().expect("saved identities");
assert_eq!(saved.get(parent).expect("parent survives").credentials.secret_key, password, "{mutation}");
}
}
#[tokio::test]
async fn session_secret_change_uses_the_durable_parent_password_with_a_stale_cache() {
let iam = test_iam_sys().await;
let parent = "stale-password-parent";
iam.create_user(
parent,
&AddOrUpdateUserReq {
secret_key: "cachedParentSecret123".to_string(),
policy: None,
status: rustfs_madmin::AccountStatus::Enabled,
},
)
.await
.expect("create parent");
let session = rotation_test_session(parent, "fresh-password-session");
let owner = iam.get_user(parent).await.expect("cached parent").credentials;
iam.set_temp_user_from_parent(&session, &owner)
.await
.expect("publish session");
iam.store
.api
.saved_sts_users
.lock()
.expect("saved identities")
.get_mut(parent)
.expect("durable parent")
.credentials
.secret_key = "durableParentSecret123".to_string();
assert_eq!(
iam.store
.get_user(parent)
.await
.expect("stale cached parent")
.credentials
.secret_key,
"cachedParentSecret123"
);
let probe = IamSys::<StsTestMockStore>::load_user_notification_probe(0, false, false);
crate::LOAD_USER_NOTIFICATION_PROBE
.scope(
probe,
iam.change_own_secret_key_from_session(parent, "durableParentSecret123", "replacementParentSecret123", &session),
)
.await
.expect("durable current password authorizes the change");
assert_eq!(
iam.get_user(parent).await.expect("updated parent").credentials.secret_key,
"replacementParentSecret123"
);
}
#[tokio::test]
async fn current_session_can_change_its_parent_secret_and_revoke_itself() {
let iam = test_iam_sys().await;
let parent = "current-session-parent";
let password = "currentParentSecret123";
iam.create_user(
parent,
&AddOrUpdateUserReq {
secret_key: password.to_string(),
policy: None,
status: rustfs_madmin::AccountStatus::Enabled,
},
)
.await
.expect("create parent");
let session = rotation_test_session(parent, "current-password-session");
let owner = iam.get_user(parent).await.expect("parent").credentials;
iam.set_temp_user_from_parent(&session, &owner)
.await
.expect("publish session");
let probe = IamSys::<StsTestMockStore>::load_user_notification_probe(0, false, false);
let (_, _, revoked) = tokio::time::timeout(
std::time::Duration::from_secs(5),
crate::LOAD_USER_NOTIFICATION_PROBE.scope(
probe,
iam.change_own_secret_key_from_session(parent, password, "replacementParentSecret123", &session),
),
)
.await
.expect("rotation must release its session lock before revocation")
.expect("rotate current session");
assert_eq!(revoked, 1);
assert!(iam.get_user(&session.access_key).await.is_none());
assert_eq!(
iam.get_user(parent).await.expect("rotated parent").credentials.secret_key,
"replacementParentSecret123"
);
}
#[tokio::test]
async fn self_secret_change_cannot_undo_concurrent_disable_rotation_or_deletion() {
ensure_test_global_credentials();
+31 -39
View File
@@ -195,14 +195,27 @@ impl Operation for ChangeOwnPasswordHandler {
let request: ChangePasswordRequest = serde_json::from_slice(&body)
.map_err(|e| s3::error(S3ErrorCode::InvalidRequest, format!("invalid change-password request: {e}")))?;
// An STS session signs with its own secret. A direct IAM request must
// have been signed with the same password the caller is replacing.
let expected_signing_secret = if caller.identity_type == IdentityType::Sts {
&caller.credentials.secret_key
} else {
&request.current_secret_key
let iam_store =
current_ready_iam_handle().map_err(|_| s3::error(S3ErrorCode::InternalError, "iam is not initialized"))?;
let Some(stored) = iam_store.get_user(&caller.access_key).await else {
// Reached only if the identity was deleted between authentication
// and this lookup.
emit_audit(
&audit,
AccountAuditRecord::failure(
AccountAuditOperation::ChangeOwnPassword,
&caller.access_key,
caller.identity_type,
AccountAuditFailure::Internal,
),
);
return Err(s3::error(S3ErrorCode::InvalidRequest, "the calling identity no longer exists"));
};
if !constant_time_eq(expected_signing_secret, &signing_secret) {
if !constant_time_eq(&request.current_secret_key, &stored.credentials.secret_key)
|| !constant_time_eq(&request.current_secret_key, &signing_secret)
{
warn!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_ACCOUNT,
@@ -243,10 +256,8 @@ impl Operation for ChangeOwnPasswordHandler {
}
let access_key = caller.access_key.clone();
let current_secret_key = request.current_secret_key;
let new_secret_key = request.new_secret_key;
let new_secret_key = request.new_secret_key.clone();
let identity_type = caller.identity_type;
let session_credentials = (identity_type == IdentityType::Sts).then(|| caller.credentials.clone());
let session_access_key = caller.session_access_key.clone();
let audit_for_task = audit.clone();
@@ -257,35 +268,16 @@ impl Operation for ChangeOwnPasswordHandler {
let iam_store =
current_ready_iam_handle().map_err(|_| s3::error(S3ErrorCode::InternalError, "iam is not initialized"))?;
let rotation = match session_credentials.as_ref() {
Some(session) => {
iam_store
.change_own_secret_key_from_session(&access_key, &current_secret_key, &new_secret_key, session)
.await
}
None => {
iam_store
.change_own_secret_key(&access_key, &current_secret_key, &new_secret_key)
.await
}
};
let (updated_at, status, revoked) = rotation.map_err(|error| {
if matches!(error, rustfs_iam::error::Error::IAMActionNotAllowed) {
emit_audit(
&audit_for_task,
AccountAuditRecord::failure(
AccountAuditOperation::ChangeOwnPassword,
&access_key,
identity_type,
AccountAuditFailure::InvalidCurrentSecret,
)
.with_session_access_key(session_access_key.as_deref()),
);
s3::error(S3ErrorCode::InvalidRequest, "the current secret key is incorrect")
} else {
iam_error_to_s3_error(error)
}
})?;
let (updated_at, status, revoked) = iam_store
.change_own_secret_key(&access_key, &signing_secret, &new_secret_key)
.await
.map_err(|error| {
if matches!(error, rustfs_iam::error::Error::IAMActionNotAllowed) {
s3::error(S3ErrorCode::InvalidRequest, "the current secret key is incorrect")
} else {
iam_error_to_s3_error(error)
}
})?;
emit_audit(
&audit_for_task,
+1 -23
View File
@@ -87,8 +87,6 @@ pub(crate) enum CredentialMutationDenial {
ServiceAccount,
/// A derived credential whose parent could not be determined.
UnresolvedParent,
/// A storage delegation has no account-management authority.
TableStorageSession,
}
impl CredentialMutationDenial {
@@ -99,7 +97,6 @@ impl CredentialMutationDenial {
}
Self::FederatedIdentity => "federated identities are managed by their identity provider",
Self::ServiceAccount => "service account credentials cannot change the credentials of their parent identity",
Self::TableStorageSession => "table-scoped storage credentials cannot manage account credentials",
Self::UnresolvedParent => "the parent identity of this session could not be resolved",
}
}
@@ -190,7 +187,7 @@ impl CallerIdentity {
// root", and an unresolved parent must never fall through to a
// permissive answer.
//
// These denials apply to both capabilities: a machine credential
// These three denials apply to both capabilities: a machine credential
// must not take over its parent, a federated identity is owned by its
// IdP, and an unresolvable parent fails closed.
let shared_denial = if unresolved_parent {
@@ -199,11 +196,6 @@ impl CallerIdentity {
Some(CredentialMutationDenial::ServiceAccount)
} else if federated {
Some(CredentialMutationDenial::FederatedIdentity)
} else if ["rustfs:table-bucket", "rustfs:table-id", "rustfs:credential-scope-prefix"]
.iter()
.any(|claim| credentials.claims_or_empty().contains_key(*claim))
{
Some(CredentialMutationDenial::TableStorageSession)
} else {
None
};
@@ -509,20 +501,6 @@ mod tests {
assert_eq!(caller.mutation_denial, Some(CredentialMutationDenial::FederatedIdentity));
}
#[test]
fn table_storage_sessions_cannot_manage_parent_credentials() {
for claim in ["rustfs:table-bucket", "rustfs:table-id", "rustfs:credential-scope-prefix"] {
let mut credentials = sts_session("TABLEKEY", "table-owner");
credentials.claims = Some(HashMap::from([(claim.to_string(), Value::String("scope".to_string()))]));
let caller = CallerIdentity::from_credentials(credentials, false);
assert_eq!(caller.mutation_denial, Some(CredentialMutationDenial::TableStorageSession));
assert!(!caller.mutability().password);
assert!(caller.ensure_credential_mutation_allowed().is_err());
assert!(caller.ensure_mfa_management_allowed().is_err());
}
}
#[test]
fn derived_credential_without_a_parent_is_denied_rather_than_allowed() {
// Fail closed: an unresolvable parent must not be treated as a
@@ -175,6 +175,7 @@ fn lock_write_rejects_bucket_replacement(write: LockWrite) {
bucket::{BucketOperations, DeleteBucketOptions, MakeBucketOptions},
object::{ObjectIO, ObjectOperations},
};
use crate::storage::storage_api::ecstore_object::WriteCompletion;
use crate::storage::storage_api::{StorageObjectOptions, StoragePutObjReader};
use rustfs_scanner::{SegmentInvalidationProducerIdentity, set_scanner_dirty_usage_mutation_observer};
use std::sync::{Arc, Mutex};
@@ -205,6 +206,8 @@ fn lock_write_rejects_bucket_replacement(write: LockWrite) {
&mut StoragePutObjReader::from_vec(body.to_vec()),
&StorageObjectOptions {
versioned: true,
// The test snapshots every disk immediately after this write.
write_completion: WriteCompletion::TailDrained,
..Default::default()
},
)
+1 -1
View File
@@ -333,7 +333,7 @@ root = Path(sys.argv[1])
records = [(path, json.loads(path.read_text())) for path in (root / "requests").glob("*.timing.json")]
assert len(records) == 5, records
assert len({row["clock_id"] for _, row in records}) == 1
assert len({row["start_ns"] for _, row in records}) == 5
# Concurrent requests can start in the same monotonic clock tick.
for _, row in records:
assert 0 < row["start_ns"] < row["end_ns"]
assert row["start_unix_ns"] > 0 and row["end_unix_ns"] > 0