Merge branch 'main' into feat/kms-vault-transit2

crates/iam/src/manager.rs

@@ -13,7 +13,7 @@
 // limitations under the License.
 
 use crate::error::{Error, Result, is_err_config_not_found};
-use crate::sys::get_claims_from_token_with_secret;
+use crate::sys::{get_claims_from_token_with_secret, get_claims_from_token_with_secret_allow_missing_exp};
 use crate::{
     cache::{Cache, CacheEntity},
     error::{Error as IamError, is_err_no_such_group, is_err_no_such_policy, is_err_no_such_user},
@@ -687,6 +687,8 @@ where
             cr.description = opts.description;
         }
 
+        let token_without_expiration = cr.expiration.is_none();
+
         if opts.expiration.is_some() {
             // TODO: check expiration
             cr.expiration = opts.expiration;
@@ -702,7 +704,11 @@ where
             }
         }
 
-        let mut m: HashMap<String, Value> = get_claims_from_token_with_secret(&cr.session_token, &current_secret_key)?;
+        let mut m: HashMap<String, Value> = if token_without_expiration {
+            get_claims_from_token_with_secret_allow_missing_exp(&cr.session_token, &current_secret_key)?
+        } else {
+            get_claims_from_token_with_secret(&cr.session_token, &current_secret_key)?
+        };
         m.remove(SESSION_POLICY_NAME_EXTRACTED);
 
         let nosp = if let Some(policy) = &opts.session_policy {
@@ -732,6 +738,10 @@ where
             }
         }
 
+        if let Some(expiration) = opts.expiration {
+            m.insert("exp".to_owned(), Value::Number(serde_json::Number::from(expiration.unix_timestamp())));
+        }
+
         m.insert("accessKey".to_owned(), Value::String(name.to_owned()));
 
         cr.session_token = jwt_sign(&m, &cr.secret_key)?;
@@ -755,7 +765,11 @@ where
 
         if let Some(groups) = groups {
             for group in groups.iter() {
-                let (gp, _) = self.policy_db_get_internal(group, true, present).await?;
+                let (gp, _) = match self.policy_db_get_internal(group, true, present).await {
+                    Ok(result) => result,
+                    Err(err) if is_err_no_such_group(&err) => continue,
+                    Err(err) => return Err(err),
+                };
                 gp.iter().for_each(|v| {
                     policies.push(v.clone());
                 });
@@ -1333,7 +1347,11 @@ where
     fn update_user_with_claims(&self, k: &str, u: UserIdentity) -> Result<()> {
         let mut u = u;
         if !u.credentials.session_token.is_empty() {
-            u.credentials.claims = Some(extract_jwt_claims(&u)?);
+            u.credentials.claims = Some(if u.credentials.expiration.is_none() {
+                extract_jwt_claims_allow_missing_exp(&u)?
+            } else {
+                extract_jwt_claims(&u)?
+            });
         }
 
         if u.credentials.is_temp() && !u.credentials.is_service_account() {
@@ -1879,6 +1897,21 @@ pub fn extract_jwt_claims(u: &UserIdentity) -> Result<HashMap<String, Value>> {
     Err(Error::other("unable to extract claims"))
 }
 
+pub fn extract_jwt_claims_allow_missing_exp(u: &UserIdentity) -> Result<HashMap<String, Value>> {
+    let Some(sys_key) = get_token_signing_key() else {
+        return Err(Error::other("global active sk not init"));
+    };
+
+    let keys = vec![&sys_key, &u.credentials.secret_key];
+
+    for key in keys {
+        if let Ok(claims) = get_claims_from_token_with_secret_allow_missing_exp(&u.credentials.session_token, key) {
+            return Ok(claims);
+        }
+    }
+    Err(Error::other("unable to extract claims"))
+}
+
 fn filter_policies(cache: &Cache, policy_name: &str, bucket_name: &str) -> (String, Policy) {
     let mp = MappedPolicy::new(policy_name).to_slice();
 

crates/iam/src/store/object.rs

@@ -17,7 +17,7 @@ use crate::error::{Error, Result, is_err_config_not_found, is_err_no_such_group}
 use crate::{
     cache::{Cache, CacheEntity},
     error::{is_err_no_such_policy, is_err_no_such_user},
-    manager::{extract_jwt_claims, get_default_policyes},
+    manager::{extract_jwt_claims, extract_jwt_claims_allow_missing_exp, get_default_policyes},
 };
 use futures::future::join_all;
 use rustfs_credentials::get_global_action_cred;
@@ -565,7 +565,13 @@ impl Store for ObjectStore {
         }
 
         if !u.credentials.session_token.is_empty() {
-            match extract_jwt_claims(&u) {
+            let claims_result = if user_type == UserType::Svc && u.credentials.expiration.is_none() {
+                extract_jwt_claims_allow_missing_exp(&u)
+            } else {
+                extract_jwt_claims(&u)
+            };
+
+            match claims_result {
                 Ok(claims) => {
                     u.credentials.claims = Some(claims);
                 }

crates/iam/src/sys.rs

@@ -23,7 +23,7 @@ use crate::store::GroupInfo;
 use crate::store::MappedPolicy;
 use crate::store::Store;
 use crate::store::UserType;
-use crate::utils::extract_claims;
+use crate::utils::{extract_claims, extract_claims_allow_missing_exp};
 use rustfs_credentials::{Credentials, EMBEDDED_POLICY_TYPE, INHERITED_POLICY_TYPE, get_global_action_cred};
 use rustfs_ecstore::notification_sys::get_global_notification_sys;
 use rustfs_madmin::AddOrUpdateUserReq;
@@ -468,14 +468,9 @@ impl<T: Store> IamSys<T> {
             }
         }
 
-        // set expiration time default to 1 hour
-        m.insert(
-            "exp".to_string(),
-            Value::Number(serde_json::Number::from(
-                opts.expiration
-                    .map_or(OffsetDateTime::now_utc().unix_timestamp() + 3600, |t| t.unix_timestamp()),
-            )),
-        );
+        if let Some(expiration) = opts.expiration {
+            m.insert("exp".to_string(), Value::Number(serde_json::Number::from(expiration.unix_timestamp())));
+        }
 
         let (access_key, secret_key) = if !opts.access_key.is_empty() || !opts.secret_key.is_empty() {
             (opts.access_key, opts.secret_key)
@@ -489,7 +484,6 @@ impl<T: Store> IamSys<T> {
         cred.status = ACCOUNT_ON.to_owned();
         cred.name = opts.name;
         cred.description = opts.description;
-        cred.expiration = opts.expiration;
 
         let create_at = self.store.add_service_account(cred.clone()).await?;
 
@@ -528,7 +522,7 @@ impl<T: Store> IamSys<T> {
     }
 
     async fn get_service_account_internal(&self, access_key: &str) -> Result<(UserIdentity, Option<Policy>)> {
-        let (sa, claims) = match self.get_account_with_claims(access_key).await {
+        let (sa, claims) = match self.get_account_with_claims_allow_missing_exp(access_key).await {
             Ok(res) => res,
             Err(err) => {
                 if is_err_no_such_account(&err) {
@@ -566,6 +560,19 @@ impl<T: Store> IamSys<T> {
         Ok((acc, m))
     }
 
+    async fn get_account_with_claims_allow_missing_exp(
+        &self,
+        access_key: &str,
+    ) -> Result<(UserIdentity, HashMap<String, Value>)> {
+        let Some(acc) = self.store.get_user(access_key).await else {
+            return Err(IamError::NoSuchAccount(access_key.to_string()));
+        };
+
+        let m = crate::manager::extract_jwt_claims_allow_missing_exp(&acc)?;
+
+        Ok((acc, m))
+    }
+
     pub async fn get_temporary_account(&self, access_key: &str) -> Result<(Credentials, Option<Policy>)> {
         let (mut sa, policy) = match self.get_temp_account(access_key).await {
             Ok(res) => res,
@@ -625,7 +632,7 @@ impl<T: Store> IamSys<T> {
             return Err(IamError::NoSuchServiceAccount(access_key.to_string()));
         }
 
-        extract_jwt_claims(&u)
+        crate::manager::extract_jwt_claims_allow_missing_exp(&u)
     }
 
     pub async fn delete_service_account(&self, access_key: &str, notify: bool) -> Result<()> {
@@ -1248,6 +1255,23 @@ pub fn get_claims_from_token_with_secret(token: &str, secret: &str) -> Result<Ha
     Ok(ms.claims)
 }
 
+pub fn get_claims_from_token_with_secret_allow_missing_exp(token: &str, secret: &str) -> Result<HashMap<String, Value>> {
+    let mut ms = extract_claims_allow_missing_exp::<HashMap<String, Value>>(token, secret)
+        .map_err(|e| Error::other(format!("extract claims err {e}")))?;
+
+    if let Some(session_policy) = ms.claims.get(SESSION_POLICY_NAME) {
+        let policy_str = session_policy.as_str().unwrap_or_default();
+        let policy = base64_simd::URL_SAFE_NO_PAD
+            .decode_to_vec(policy_str.as_bytes())
+            .map_err(|e| Error::other(format!("base64 decode err {e}")))?;
+        ms.claims.insert(
+            SESSION_POLICY_NAME_EXTRACTED.to_string(),
+            Value::String(String::from_utf8(policy).map_err(|e| Error::other(format!("utf8 decode err {e}")))?),
+        );
+    }
+    Ok(ms.claims)
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -1255,7 +1279,7 @@ mod tests {
     use crate::error::Error;
     use crate::manager::get_default_policyes;
     use crate::store::{GroupInfo, MappedPolicy, Store, UserType};
-    use rustfs_credentials::Credentials;
+    use rustfs_credentials::{Credentials, get_global_action_cred, init_global_action_credentials};
     use rustfs_policy::auth::UserIdentity;
     use rustfs_policy::policy::Args;
     use rustfs_policy::policy::action::{Action, AdminAction, S3Action};
@@ -1296,7 +1320,7 @@ mod tests {
             _item: UserIdentity,
             _ttl: Option<usize>,
         ) -> Result<()> {
-            Err(Error::InvalidArgument)
+            Ok(())
         }
 
         async fn delete_user_identity(&self, _name: &str, _user_type: UserType) -> Result<()> {
@@ -1337,7 +1361,7 @@ mod tests {
         }
 
         async fn load_group(&self, _name: &str, _m: &mut HashMap<String, GroupInfo>) -> Result<()> {
-            Err(Error::InvalidArgument)
+            Ok(())
         }
 
         async fn load_groups(&self, _m: &mut HashMap<String, GroupInfo>) -> Result<()> {
@@ -1486,6 +1510,129 @@ mod tests {
         }
     }
 
+    fn ensure_test_global_credentials() {
+        if get_global_action_cred().is_none() {
+            let _ = init_global_action_credentials(Some("TESTROOTACCESSKEY".to_string()), Some("TESTROOTSECRET123".to_string()));
+        }
+    }
+
+    #[tokio::test]
+    async fn test_new_service_account_without_expiration_omits_exp_claim() {
+        ensure_test_global_credentials();
+
+        let store = StsTestMockStore { empty_policies: false };
+        let cache_manager = IamCache::new(store).await;
+        let iam_sys = IamSys::new(cache_manager);
+
+        let (cred, _) = iam_sys
+            .new_service_account("svc-parent-user", None, NewServiceAccountOpts::default())
+            .await
+            .expect("service account should be created without expiration");
+
+        assert!(cred.expiration.is_none());
+
+        let claims = get_claims_from_token_with_secret_allow_missing_exp(&cred.session_token, &cred.secret_key)
+            .expect("service account JWT without expiration should decode");
+        assert!(
+            !claims.contains_key("exp"),
+            "service account without explicit expiration should not get a default JWT exp"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_update_service_account_updates_exp_claim() {
+        ensure_test_global_credentials();
+
+        let store = StsTestMockStore { empty_policies: false };
+        let cache_manager = IamCache::new(store).await;
+        let iam_sys = IamSys::new(cache_manager);
+
+        let initial_expiration = OffsetDateTime::now_utc() + time::Duration::hours(2);
+        let (cred, _) = iam_sys
+            .new_service_account(
+                "svc-parent-user",
+                None,
+                NewServiceAccountOpts {
+                    expiration: Some(initial_expiration),
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("service account with explicit expiration should be created");
+
+        let updated_expiration = OffsetDateTime::now_utc() + time::Duration::hours(4);
+        iam_sys
+            .update_service_account(
+                &cred.access_key,
+                UpdateServiceAccountOpts {
+                    session_policy: None,
+                    secret_key: None,
+                    name: None,
+                    description: None,
+                    expiration: Some(updated_expiration),
+                    status: None,
+                },
+            )
+            .await
+            .expect("service account expiration should update");
+
+        let updated_user = iam_sys
+            .get_user(&cred.access_key)
+            .await
+            .expect("updated service account should exist");
+        assert_eq!(updated_user.credentials.expiration, Some(updated_expiration));
+
+        let claims =
+            get_claims_from_token_with_secret(&updated_user.credentials.session_token, &updated_user.credentials.secret_key)
+                .expect("updated service account JWT should decode");
+        assert_eq!(
+            claims.get("exp").and_then(|v| v.as_i64()),
+            Some(updated_expiration.unix_timestamp()),
+            "updating service account expiration must rewrite the JWT exp claim"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_update_service_account_adds_exp_claim_to_non_expiring_account() {
+        ensure_test_global_credentials();
+
+        let store = StsTestMockStore { empty_policies: false };
+        let cache_manager = IamCache::new(store).await;
+        let iam_sys = IamSys::new(cache_manager);
+
+        let (cred, _) = iam_sys
+            .new_service_account("svc-parent-user", None, NewServiceAccountOpts::default())
+            .await
+            .expect("service account without explicit expiration should be created");
+
+        let updated_expiration = OffsetDateTime::now_utc() + time::Duration::hours(3);
+        iam_sys
+            .update_service_account(
+                &cred.access_key,
+                UpdateServiceAccountOpts {
+                    session_policy: None,
+                    secret_key: None,
+                    name: None,
+                    description: None,
+                    expiration: Some(updated_expiration),
+                    status: None,
+                },
+            )
+            .await
+            .expect("service account without expiration should accept a new expiration");
+
+        let updated_user = iam_sys
+            .get_user(&cred.access_key)
+            .await
+            .expect("updated service account should exist");
+        assert_eq!(updated_user.credentials.expiration, Some(updated_expiration));
+
+        let claims =
+            get_claims_from_token_with_secret(&updated_user.credentials.session_token, &updated_user.credentials.secret_key)
+                .expect("updated service account JWT should decode after adding expiration");
+        assert_eq!(claims.get("exp").and_then(|v| v.as_i64()), Some(updated_expiration.unix_timestamp()));
+    }
+
     /// Regression test: temp credentials without groups in args still receive group-attached
     /// policies via the parent user (groups fallback). Without the fallback, policy_db_get
     /// would get None for groups and the user would have no group policies, so the action
@@ -1873,4 +2020,29 @@ mod tests {
             "inherited service account should not require object tag fetch based on session policy hint"
         );
     }
+
+    /// Regression test for rustfs#2392: `policy_db_get` must skip non-existent groups
+    /// instead of aborting the entire policy resolution. When a JWT contains groups
+    /// that exist in the IdP but not in IAM, policies from the remaining valid groups
+    /// must still be returned.
+    #[tokio::test]
+    async fn test_policy_db_get_skips_nonexistent_groups() {
+        let store = StsTestMockStore { empty_policies: false };
+        let cache_manager = IamCache::new(store).await;
+        let iam_sys = IamSys::new(cache_manager);
+
+        // "testgroup" exists with "readwrite" policy; "nonexistent-group" does not exist in IAM.
+        let groups = Some(vec!["testgroup".to_string(), "nonexistent-group".to_string()]);
+
+        let policies = iam_sys
+            .policy_db_get("sts-fallback-test-parent", &groups)
+            .await
+            .expect("policy_db_get should not fail when some groups are missing");
+
+        assert!(
+            policies.iter().any(|p| p == "readwrite"),
+            "policies from existing group 'testgroup' should be returned even when other groups are missing; got: {:?}",
+            policies
+        );
+    }
 }

crates/iam/src/utils.rs

@@ -15,6 +15,7 @@
 use jsonwebtoken::{Algorithm, DecodingKey, EncodingKey, Header};
 use rand::{Rng, RngExt};
 use serde::{Serialize, de::DeserializeOwned};
+use std::collections::HashSet;
 use std::io::{Error, Result};
 
 /// Generates a random access key of the specified length.
@@ -98,6 +99,16 @@ pub fn extract_claims<T: DeserializeOwned + Clone>(
     )
 }
 
+pub fn extract_claims_allow_missing_exp<T: DeserializeOwned + Clone>(
+    token: &str,
+    secret: &str,
+) -> std::result::Result<jsonwebtoken::TokenData<T>, jsonwebtoken::errors::Error> {
+    let mut validation = jsonwebtoken::Validation::new(Algorithm::HS512);
+    validation.required_spec_claims = HashSet::new();
+
+    jsonwebtoken::decode::<T>(token, &DecodingKey::from_secret(secret.as_bytes()), &validation)
+}
+
 #[cfg(test)]
 mod tests {
     use super::{extract_claims, gen_access_key, gen_secret_key, generate_jwt};

helm/README.md

@@ -23,7 +23,6 @@ RustFS helm chart supports **standalone and distributed mode**. For standalone m
 | config.rustfs.console_address | string | `":9001"` |  |
 | config.rustfs.console_enable | string | `"true"` |  |
 | config.rustfs.domains | string | `""` | Enable virtual host mode. |
-| config.rustfs.ec.storage_class_standard | string | `EC:4` | Standard storage class environment variable. |
 | config.rustfs.log_level | string | `"info"` |  |
 | config.rustfs.obs_environment | string | `"development"` |  |
 | config.rustfs.obs_log_directory | string | `"/logs"` |  |

helm/rustfs/templates/configmap.yaml

@@ -82,9 +82,3 @@ data:
   RUSTFS_SCANNER_IDLE_MODE: {{ .idle_mode | quote }}
     {{- end }}
   {{- end }}
-  {{- if .Values.mode.distributed.enabled }}
-  {{- with .Values.config.rustfs.ec }}
-  RUSTFS_ERASURE_SET_DRIVE_COUNT: {{ 16 | quote }}
-  RUSTFS_STORAGE_CLASS_STANDARD: {{ .storage_class_standard | quote }}
-  {{- end }}
-  {{- end }}

helm/rustfs/values.yaml

@@ -100,8 +100,11 @@ config:
 
 extraEnv: [] # This is for setting extra environment variables in the rustfs container. It should be a list of key value pairs. For example:
 # extraEnv:
-#   - name: RUSTFS_EXTRA_ENV
-#     value: "extra_value"
+#   - name: RUSTFS_ERASURE_SET_DRIVE_COUNT
+#     value: "16"
+#   - name: RUSTFS_STORAGE_CLASS_STANDARD
+#     value: "EC:4"
+
 
 # This section builds out the service account more information can be found here: https://kubernetes.io/docs/concepts/security/service-accounts/
 serviceAccount:

rustfs/src/auth.rs

@@ -16,8 +16,9 @@ use http::HeaderMap;
 use http::Uri;
 use rustfs_credentials::{Credentials, get_global_action_cred};
 use rustfs_iam::error::Error as IamError;
-use rustfs_iam::sys::SESSION_POLICY_NAME;
-use rustfs_iam::sys::get_claims_from_token_with_secret;
+use rustfs_iam::sys::{
+    SESSION_POLICY_NAME, get_claims_from_token_with_secret, get_claims_from_token_with_secret_allow_missing_exp,
+};
 use rustfs_utils::http::ip::get_source_ip_raw;
 use s3s::S3Error;
 use s3s::S3ErrorCode;
@@ -353,8 +354,12 @@ pub fn check_claims_from_token(token: &str, cred: &Credentials) -> S3Result<Hash
     };
 
     if !token.is_empty() {
-        let claims: HashMap<String, Value> =
-            get_claims_from_token_with_secret(token, secret).map_err(|_e| s3_error!(InvalidRequest, "invalid token"))?;
+        let claims: HashMap<String, Value> = if cred.is_service_account() {
+            get_claims_from_token_with_secret_allow_missing_exp(token, secret)
+                .map_err(|_e| s3_error!(InvalidRequest, "invalid token"))?
+        } else {
+            get_claims_from_token_with_secret(token, secret).map_err(|_e| s3_error!(InvalidRequest, "invalid token"))?
+        };
         return Ok(claims);
     }
 

rustfs/src/storage/access.rs

@@ -1876,11 +1876,29 @@ impl S3Access for FS {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use http::{HeaderMap, Method, Uri};
+    use http::{Extensions, HeaderMap, Method, Uri};
     use rustfs_policy::policy::{BucketPolicy, bucket_policy_uses_existing_object_tag_conditions};
     use std::collections::HashMap;
     use time::OffsetDateTime;
 
+    fn build_request<T>(input: T, method: Method) -> S3Request<T> {
+        S3Request {
+            input,
+            method,
+            uri: Uri::from_static("/"),
+            headers: HeaderMap::new(),
+            extensions: Extensions::new(),
+            credentials: None,
+            region: None,
+            service: None,
+            trailing_headers: None,
+        }
+    }
+
+    fn ensure_req_info<T>(req: &mut S3Request<T>) {
+        req.extensions.insert(ReqInfo::default());
+    }
+
     #[test]
     fn get_bucket_policy_uses_get_bucket_policy_action() {
         assert_eq!(get_bucket_policy_authorize_action(), Action::S3Action(S3Action::GetBucketPolicyAction));
@@ -2315,4 +2333,74 @@ mod tests {
         assert_eq!(req_info.object.as_deref(), Some("test-key"));
         assert_eq!(req_info.version_id, None);
     }
+
+    #[tokio::test]
+    async fn abort_multipart_upload_rejects_unauthorized_request() {
+        let fs = FS::new();
+        let mut req = build_request(
+            AbortMultipartUploadInput::builder()
+                .bucket("bucket".to_string())
+                .key("object".to_string())
+                .upload_id("upload-id".to_string())
+                .build()
+                .unwrap(),
+            Method::DELETE,
+        );
+        ensure_req_info(&mut req);
+
+        let err = fs
+            .abort_multipart_upload(&mut req)
+            .await
+            .expect_err("missing credentials should reject access");
+        assert_eq!(err.code(), &S3ErrorCode::AccessDenied);
+    }
+
+    #[tokio::test]
+    async fn complete_multipart_upload_rejects_unauthorized_request() {
+        let fs = FS::new();
+        let mut req = build_request(
+            CompleteMultipartUploadInput::builder()
+                .bucket("bucket".to_string())
+                .key("object".to_string())
+                .upload_id("upload-id".to_string())
+                .multipart_upload(Some(CompletedMultipartUpload::default()))
+                .build()
+                .unwrap(),
+            Method::POST,
+        );
+        ensure_req_info(&mut req);
+
+        let err = fs
+            .complete_multipart_upload(&mut req)
+            .await
+            .expect_err("missing credentials should reject access");
+        assert_eq!(err.code(), &S3ErrorCode::AccessDenied);
+    }
+
+    #[tokio::test]
+    async fn upload_part_copy_rejects_unauthorized_request() {
+        let fs = FS::new();
+        let mut req = build_request(
+            UploadPartCopyInput::builder()
+                .bucket("dst-bucket".to_string())
+                .key("dst-object".to_string())
+                .upload_id("upload-id".to_string())
+                .part_number(1)
+                .copy_source(CopySource::Bucket {
+                    bucket: "src-bucket".into(),
+                    key: "src-object".into(),
+                    version_id: None,
+                })
+                .build()
+                .unwrap(),
+            Method::PUT,
+        );
+        ensure_req_info(&mut req);
+
+        let err = fs
+            .upload_part_copy(&mut req)
+            .await
+            .expect_err("missing credentials should reject access");
+        assert_eq!(err.code(), &S3ErrorCode::AccessDenied);
+    }
 }