// Copyright 2024 RustFS Team // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //! Negative / lifecycle e2e coverage for the central admin authorization gate //! (`rustfs/src/admin/auth.rs`), tracking rustfs/backlog#1151 sec-4. //! //! These tests exercise the gate end-to-end against a real server, using raw //! SigV4-signed HTTP (no `awscurl` dependency) so the exact HTTP status and S3 //! error code are asserted: //! //! 1. A fully authenticated but non-admin credential is rejected with //! `403 AccessDenied` on an admin API (`GET /rustfs/admin/v3/info`), while //! the root credential is accepted — proving the rejection is authorization, //! not a broken endpoint. //! 2. Root-credential rotation takes effect across a restart: the new //! credential is accepted and the old credential is rejected, on both the //! S3 data plane and the admin plane. //! 3. Starting with the default `rustfsadmin` credentials emits the //! default-credentials startup warning (observable operator signal). #[cfg(test)] mod tests { use crate::common::{RustFSTestEnvironment, init_logging, local_http_client, rustfs_binary_path}; use aws_sdk_s3::config::{Credentials, Region}; use aws_sdk_s3::{Client, Config}; use http::header::HOST; use rustfs_signer::constants::UNSIGNED_PAYLOAD; use rustfs_signer::sign_v4; use s3s::Body; use serial_test::serial; use std::error::Error; use std::io::Read; use std::process::{Command, Stdio}; use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; const ADMIN_INFO_PATH: &str = "/rustfs/admin/v3/info"; const ADMIN_MANUAL_TRANSITION_BUCKET: &str = "auth-deny-manual-transition"; const ADMIN_MANUAL_TRANSITION_PATH: &str = "/rustfs/admin/v3/ilm/transition/run?bucket=auth-deny-manual-transition&maxObjects=1&mode=async"; fn assert_no_raw_manual_transition_markers(body: &str, context: &str) { assert!( !body.contains("\"marker\"") && !body.contains("\"versionMarker\"") && !body.contains("\"version_marker\""), "{context} must not expose raw manual transition resume markers, body: {body}" ); } async fn wait_for_terminal_manual_transition_job( env: &RustFSTestEnvironment, status_endpoint: &str, ) -> Result> { let deadline = Instant::now() + Duration::from_secs(30); loop { let (status, body) = signed_request(&env.url, http::Method::GET, status_endpoint, None, &env.access_key, &env.secret_key).await?; assert_eq!( status, reqwest::StatusCode::OK, "root credential must query manual transition job status, body: {body}" ); assert_no_raw_manual_transition_markers(&body, "manual transition status response"); let value: serde_json::Value = serde_json::from_str(&body)?; let job_status = value .get("status") .and_then(serde_json::Value::as_str) .ok_or("manual transition job status response must include status")?; if matches!(job_status, "completed" | "partial" | "cancelled" | "failed" | "unknown") { return Ok(body); } if Instant::now() >= deadline { return Err(format!("manual transition job did not reach terminal status within 30s; last={body}").into()); } tokio::time::sleep(Duration::from_millis(250)).await; } } /// Send a SigV4-signed request to `path` (optionally with a JSON `body`) and /// return `(status, body)`. Uses the `UNSIGNED_PAYLOAD` content hash so a /// request body can be attached without the caller pre-hashing it — the /// server verifies the signature against the same sentinel, exactly as the /// AWS SDKs / MinIO client do for streaming/unsigned payloads. async fn signed_request( base_url: &str, method: http::Method, path: &str, body: Option<&str>, access_key: &str, secret_key: &str, ) -> Result<(reqwest::StatusCode, String), Box> { let url = format!("{base_url}{path}"); let uri = url.parse::()?; let authority = uri.authority().ok_or("missing authority")?.to_string(); let body_bytes = body.map(|b| b.as_bytes().to_vec()).unwrap_or_default(); // The signature is computed over `UNSIGNED_PAYLOAD`, so the body bytes do // not participate in the SigV4 hash — sign over an empty body and attach // the real payload to the wire request below. let request = http::Request::builder() .method(method.clone()) .uri(uri) .header(HOST, authority) .header("x-amz-content-sha256", UNSIGNED_PAYLOAD); let signed = sign_v4(request.body(Body::empty())?, 0, access_key, secret_key, "", "us-east-1"); let client = local_http_client(); let mut rb = client.request(method, url.as_str()); for (name, value) in signed.headers() { rb = rb.header(name, value); } if !body_bytes.is_empty() { rb = rb.body(body_bytes); } let resp = rb.send().await?; let status = resp.status(); let text = resp.text().await?; Ok((status, text)) } /// Build an S3 client bound to explicit credentials (used to exercise the S3 /// data plane with rotated / stale root credentials). fn s3_client_with(env: &RustFSTestEnvironment, access_key: &str, secret_key: &str) -> Client { let credentials = Credentials::new(access_key, secret_key, None, None, "sec4-admin-auth"); let config = Config::builder() .credentials_provider(credentials) .region(Region::new("us-east-1")) .endpoint_url(&env.url) .force_path_style(true) .behavior_version_latest() .build(); Client::from_conf(config) } /// Create a non-admin IAM user via the admin `add-user` API using the root /// credentials. The user is created with no policy attached, so it is a /// valid (authenticatable) principal that is nonetheless unauthorized for /// admin actions. async fn create_limited_user( env: &RustFSTestEnvironment, access_key: &str, secret_key: &str, ) -> Result<(), Box> { let path = format!("/rustfs/admin/v3/add-user?accessKey={access_key}"); let body = serde_json::json!({ "secretKey": secret_key, "status": "enabled" }).to_string(); let (status, resp) = signed_request(&env.url, http::Method::PUT, &path, Some(&body), &env.access_key, &env.secret_key).await?; assert!(status.is_success(), "add-user should succeed (status={status}, body={resp})"); Ok(()) } /// A fully authenticated but non-admin credential must be rejected with /// `403 AccessDenied` on an admin API, while the root credential succeeds. #[tokio::test(flavor = "multi_thread")] #[serial] async fn non_admin_credential_denied_on_admin_api() -> Result<(), Box> { init_logging(); let mut env = RustFSTestEnvironment::new().await?; env.start_rustfs_server(vec![]).await?; let user_ak = "sec4limited"; let user_sk = "sec4limitedsecret"; create_limited_user(&env, user_ak, user_sk).await?; // Root credential is authorized: proves the endpoint works. let (root_status, root_body) = signed_request(&env.url, http::Method::GET, ADMIN_INFO_PATH, None, &env.access_key, &env.secret_key).await?; assert_eq!( root_status, reqwest::StatusCode::OK, "root credential must be authorized on the admin API, body: {root_body}" ); // Non-admin credential is rejected by the admin authorization gate. let (status, body) = signed_request(&env.url, http::Method::GET, ADMIN_INFO_PATH, None, user_ak, user_sk).await?; assert_eq!( status, reqwest::StatusCode::FORBIDDEN, "non-admin credential must get 403 on the admin API, body: {body}" ); assert!( body.contains("AccessDenied"), "admin gate rejection must carry the AccessDenied S3 error code, body: {body}" ); env.stop_server(); Ok(()) } #[tokio::test(flavor = "multi_thread")] #[serial] async fn non_admin_credential_denied_on_manual_transition_run() -> Result<(), Box> { init_logging(); let mut env = RustFSTestEnvironment::new().await?; env.start_rustfs_server(vec![]).await?; let user_ak = "ilmtransitionlimited"; let user_sk = "ilmtransitionlimitedsecret"; create_limited_user(&env, user_ak, user_sk).await?; env.create_s3_client() .create_bucket() .bucket(ADMIN_MANUAL_TRANSITION_BUCKET) .send() .await?; let (root_status, root_body) = signed_request( &env.url, http::Method::POST, ADMIN_MANUAL_TRANSITION_PATH, None, &env.access_key, &env.secret_key, ) .await?; assert_eq!( root_status, reqwest::StatusCode::ACCEPTED, "root credential must reach the manual transition handler, body: {root_body}" ); assert!( root_body.contains("\"mode\":\"durable_job\""), "root response should be the durable manual transition JSON contract, body: {root_body}" ); assert_no_raw_manual_transition_markers(&root_body, "manual transition run response"); let root_value: serde_json::Value = serde_json::from_str(&root_body)?; let job_id = root_value .get("job_id") .and_then(serde_json::Value::as_str) .ok_or("manual transition async response must include job_id")?; let status_endpoint = root_value .get("status_endpoint") .and_then(serde_json::Value::as_str) .ok_or("manual transition async response must include status_endpoint")?; let cancel_endpoint = root_value .get("cancel_endpoint") .and_then(serde_json::Value::as_str) .ok_or("manual transition async response must include cancel_endpoint")?; assert_eq!( cancel_endpoint, status_endpoint, "manual transition durable jobs currently use the same status/cancel endpoint" ); assert!( status_endpoint.ends_with(job_id), "status endpoint must address the returned job id, job_id={job_id}, status_endpoint={status_endpoint}" ); let terminal_body = wait_for_terminal_manual_transition_job(&env, status_endpoint).await?; let terminal: serde_json::Value = serde_json::from_str(&terminal_body)?; assert_eq!(terminal.get("job_id").and_then(serde_json::Value::as_str), Some(job_id)); assert_eq!( terminal .get("report") .and_then(|report| report.get("bucket")) .and_then(serde_json::Value::as_str), Some(ADMIN_MANUAL_TRANSITION_BUCKET) ); let (root_status, root_body) = signed_request(&env.url, http::Method::DELETE, status_endpoint, None, &env.access_key, &env.secret_key).await?; assert_eq!( root_status, reqwest::StatusCode::OK, "root credential must cancel/query a terminal manual transition job idempotently, body: {root_body}" ); assert_no_raw_manual_transition_markers(&root_body, "manual transition cancel response"); let root_cancel: serde_json::Value = serde_json::from_str(&root_body)?; assert_eq!(root_cancel.get("job_id").and_then(serde_json::Value::as_str), Some(job_id)); assert!( matches!( root_cancel.get("status").and_then(serde_json::Value::as_str), Some("completed" | "partial" | "failed" | "unknown") ), "terminal cancel must not rewrite the job into cancelled state, body: {root_body}" ); let (status, body) = signed_request(&env.url, http::Method::POST, ADMIN_MANUAL_TRANSITION_PATH, None, user_ak, user_sk).await?; assert_eq!( status, reqwest::StatusCode::FORBIDDEN, "non-admin credential must get 403 on manual transition run, body: {body}" ); assert!( body.contains("AccessDenied"), "manual transition rejection must carry the AccessDenied S3 error code, body: {body}" ); let (status, body) = signed_request(&env.url, http::Method::GET, status_endpoint, None, user_ak, user_sk).await?; assert_eq!( status, reqwest::StatusCode::FORBIDDEN, "non-admin credential must get 403 on manual transition status, body: {body}" ); assert_no_raw_manual_transition_markers(&body, "manual transition status rejection"); assert!( body.contains("AccessDenied"), "manual transition status rejection must carry the AccessDenied S3 error code, body: {body}" ); let (status, body) = signed_request(&env.url, http::Method::DELETE, status_endpoint, None, user_ak, user_sk).await?; assert_eq!( status, reqwest::StatusCode::FORBIDDEN, "non-admin credential must get 403 on manual transition cancel, body: {body}" ); assert_no_raw_manual_transition_markers(&body, "manual transition cancel rejection"); assert!( body.contains("AccessDenied"), "manual transition cancel rejection must carry the AccessDenied S3 error code, body: {body}" ); env.stop_server(); Ok(()) } /// Rotating the root credentials (restart with new `--access-key` / /// `--secret-key` on the same data directory) takes effect: the new /// credential is accepted and the old one is rejected, on both the S3 data /// plane and the admin plane. #[tokio::test(flavor = "multi_thread")] #[serial] async fn root_credential_rotation_takes_effect() -> Result<(), Box> { init_logging(); let mut env = RustFSTestEnvironment::new().await?; let old_ak = env.access_key.clone(); let old_sk = env.secret_key.clone(); // --- Boot with the original root credential. --- env.start_rustfs_server(vec![]).await?; // Admin plane accepts the original root credential. let (status, body) = signed_request(&env.url, http::Method::GET, ADMIN_INFO_PATH, None, &old_ak, &old_sk).await?; assert_eq!(status, reqwest::StatusCode::OK, "original root must reach admin API, body: {body}"); // S3 plane accepts the original root credential. s3_client_with(&env, &old_ak, &old_sk) .list_buckets() .send() .await .expect("original root must be able to list buckets on the S3 plane"); env.stop_server(); // --- Rotate: restart on the same data dir with a new root credential. --- let new_ak = "rotatedroot"; let new_sk = "rotatedrootsecret"; env.access_key = new_ak.to_string(); env.secret_key = new_sk.to_string(); env.start_rustfs_server(vec![]).await?; // New root credential works on both planes. let (status, body) = signed_request(&env.url, http::Method::GET, ADMIN_INFO_PATH, None, new_ak, new_sk).await?; assert_eq!(status, reqwest::StatusCode::OK, "rotated root must reach admin API, body: {body}"); s3_client_with(&env, new_ak, new_sk) .list_buckets() .send() .await .expect("rotated root must be able to list buckets on the S3 plane"); // Old root credential is now rejected on both planes. let (status, body) = signed_request(&env.url, http::Method::GET, ADMIN_INFO_PATH, None, &old_ak, &old_sk).await?; assert_eq!( status, reqwest::StatusCode::FORBIDDEN, "stale root must be rejected on the admin API after rotation, body: {body}" ); let s3_old = s3_client_with(&env, &old_ak, &old_sk).list_buckets().send().await; assert!( s3_old.is_err(), "stale root must be rejected on the S3 plane after rotation, got: {s3_old:?}" ); env.stop_server(); Ok(()) } /// Booting with the default `rustfsadmin` credentials must emit the /// default-credentials startup warning so operators get an observable /// signal. The predicate + message text are unit-tested in /// `rustfs::startup_server`; this pins that the warning actually fires at /// runtime. We capture the child's stdout/stderr directly (the shared /// harness inherits stdio) and poll for the warning until it appears. #[tokio::test(flavor = "multi_thread")] #[serial] async fn default_credentials_emit_startup_warning() -> Result<(), Box> { init_logging(); let port = RustFSTestEnvironment::find_available_port().await?; let address = format!("127.0.0.1:{port}"); let temp_dir = std::env::temp_dir().join(format!("rustfs_sec4_defcred_{}_{}", std::process::id(), port)); std::fs::create_dir_all(&temp_dir)?; let temp_dir_str = temp_dir.display().to_string(); let mut child = Command::new(rustfs_binary_path()) .env("RUST_LOG", "rustfs=info") .env("RUSTFS_CONSOLE_ENABLE", "false") .args([ "--address", &address, // Explicitly the compiled-in defaults, so the warning must fire. "--access-key", "rustfsadmin", "--secret-key", "rustfsadmin", &temp_dir_str, ]) .stdout(Stdio::piped()) .stderr(Stdio::piped()) .spawn()?; // Drain stdout + stderr concurrently into a shared buffer so a full // pipe never blocks the child. let captured = Arc::new(Mutex::new(String::new())); let mut readers = Vec::new(); for stream in [ child.stdout.take().map(|s| Box::new(s) as Box), child.stderr.take().map(|s| Box::new(s) as Box), ] .into_iter() .flatten() { let sink = Arc::clone(&captured); readers.push(std::thread::spawn(move || { let mut stream = stream; let mut buf = [0u8; 4096]; loop { match stream.read(&mut buf) { Ok(0) | Err(_) => break, Ok(n) => { if let Ok(mut guard) = sink.lock() { guard.push_str(&String::from_utf8_lossy(&buf[..n])); } } } } })); } const WARNING_NEEDLE: &str = "Detected default root credentials"; let deadline = Instant::now() + Duration::from_secs(30); let mut seen = false; while Instant::now() < deadline { if let Ok(Some(status)) = child.try_wait() { // Process exited; make a final check of what we captured. seen = captured.lock().map(|g| g.contains(WARNING_NEEDLE)).unwrap_or(false); assert!( seen, "server exited ({status}) before emitting the default-credentials warning; captured: {}", captured.lock().map(|g| g.clone()).unwrap_or_default() ); break; } if captured.lock().map(|g| g.contains(WARNING_NEEDLE)).unwrap_or(false) { seen = true; break; } tokio::time::sleep(Duration::from_millis(200)).await; } let _ = child.kill(); let _ = child.wait(); for r in readers { let _ = r.join(); } let _ = std::fs::remove_dir_all(&temp_dir); assert!( seen, "starting with default credentials must emit the default-credentials warning; captured: {}", captured.lock().map(|g| g.clone()).unwrap_or_default() ); Ok(()) } }