diff --git a/crates/e2e_test/src/sts_query_compat_test.rs b/crates/e2e_test/src/sts_query_compat_test.rs index 7626972ba..ecd7771aa 100644 --- a/crates/e2e_test/src/sts_query_compat_test.rs +++ b/crates/e2e_test/src/sts_query_compat_test.rs @@ -360,12 +360,26 @@ impl OpaMock { pub(crate) async fn expect_bucket_tags(&mut self, action: &str, bucket: &str, department: Option<&str>) -> TestResult { let payload = self.next_request().await?; + Self::assert_bucket_tags(&payload, action, bucket, department); + Ok(()) + } + + async fn expect_versioned_tag_permission(&mut self, bucket: &str, object: &str, version: &str) -> TestResult { + let payload = self.next_request().await?; + Self::assert_bucket_tags(&payload, "s3:GetObjectVersionTagging", bucket, Some("finance")); + assert_eq!(payload["input"]["resource"]["object"], object); + assert_eq!(payload["input"]["context"]["conditions"]["versionid"], serde_json::json!([version])); + Ok(()) + } + + fn assert_bucket_tags(payload: &Value, action: &str, bucket: &str, department: Option<&str>) { let input = &payload["input"]; + assert_eq!(input["identity"]["account"], "opabuckettags"); + assert_eq!(input["identity"]["is_owner"], false); assert_eq!(input["action"], action); assert_eq!(input["resource"]["bucket"], bucket); let expected = department.map(|value| serde_json::json!([value])); assert_eq!(input["context"]["conditions"].get("ExistingBucketTag/department"), expected.as_ref()); - Ok(()) } async fn wait_for_validation(&mut self) -> TestResult { @@ -720,7 +734,7 @@ async fn seed_legacy_duplicate_bucket_tags(env: &mut RustFSTestEnvironment, buck async fn test_bucket_tags_opa_contract() -> TestResult { use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::types::{ - BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Tag, Tagging, VersioningConfiguration, + BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Tag, Tagging, TaggingDirective, VersioningConfiguration, }; init_logging(); @@ -772,7 +786,7 @@ async fn test_bucket_tags_opa_contract() -> TestResult { .await?; assert_eq!(read.body.collect().await?.into_bytes().as_ref(), b"report"); opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?; - client + let head = client .head_object() .bucket(source) .key("report") @@ -780,6 +794,8 @@ async fn test_bucket_tags_opa_contract() -> TestResult { .send() .await?; opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?; + assert_eq!(head.tag_count(), Some(1)); + opa.expect_versioned_tag_permission(source, "report", version).await?; client.list_objects_v2().bucket(source).send().await?; opa.expect_bucket_tags("s3:ListBucket", source, Some("finance")).await?; client.list_object_versions().bucket(source).send().await?; @@ -789,28 +805,65 @@ async fn test_bucket_tags_opa_contract() -> TestResult { admin.create_bucket().bucket(destination).send().await?; set_department(&admin, destination, "engineering").await?; let copy_source = format!("{source}/report?versionId={version}"); - let error = client - .copy_object() - .bucket(destination) - .key("copied") - .copy_source(©_source) - .send() - .await - .expect_err("source tags must not authorize a different destination"); - assert_eq!(error.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied")); - opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?; - opa.expect_bucket_tags("s3:PutObject", destination, Some("engineering")) - .await?; + for directive in [TaggingDirective::Copy, TaggingDirective::Replace] { + let error = client + .copy_object() + .bucket(destination) + .key("copied") + .copy_source(©_source) + .set_tagging_directive((directive == TaggingDirective::Replace).then_some(TaggingDirective::Replace)) + .set_tagging((directive == TaggingDirective::Replace).then(|| "department=legal".to_owned())) + .send() + .await + .expect_err("source tags must not authorize a different destination"); + assert_eq!(error.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied")); + opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?; + if directive == TaggingDirective::Copy { + opa.expect_versioned_tag_permission(source, "report", version).await?; + } + opa.expect_bucket_tags("s3:PutObject", destination, Some("engineering")) + .await?; + let missing = admin + .head_object() + .bucket(destination) + .key("copied") + .send() + .await + .expect_err("denied COPY and REPLACE must leave no destination object"); + assert_eq!(missing.raw_response().map(|response| response.status().as_u16()), Some(404)); + } set_department(&admin, destination, "finance").await?; - client - .copy_object() - .bucket(destination) - .key("copied") - .copy_source(©_source) - .send() - .await?; - opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?; - opa.expect_bucket_tags("s3:PutObject", destination, Some("finance")).await?; + for directive in [TaggingDirective::Copy, TaggingDirective::Replace] { + client + .copy_object() + .bucket(destination) + .key("copied") + .copy_source(©_source) + .set_tagging_directive((directive == TaggingDirective::Replace).then_some(TaggingDirective::Replace)) + .set_tagging((directive == TaggingDirective::Replace).then(|| "department=legal".to_owned())) + .send() + .await?; + opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?; + // The access hook checks preliminary tags, then the copy use case checks + // the exact source reader's tags before publishing the destination. + for _ in 0..2 { + if directive == TaggingDirective::Copy { + opa.expect_versioned_tag_permission(source, "report", version).await?; + } + opa.expect_bucket_tags("s3:PutObject", destination, Some("finance")).await?; + } + let tags = admin.get_object_tagging().bucket(destination).key("copied").send().await?; + assert_eq!(tags.tag_set().len(), 1); + assert_eq!(tags.tag_set()[0].key(), "department"); + assert_eq!( + tags.tag_set()[0].value(), + if directive == TaggingDirective::Copy { + "engineering" + } else { + "legal" + } + ); + } let upload = client .create_multipart_upload() @@ -886,6 +939,14 @@ async fn test_bucket_tags_opa_contract() -> TestResult { assert_eq!(error.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied")); opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("engineering")) .await?; + let missing = admin + .head_object() + .bucket(destination) + .key("denied-source") + .send() + .await + .expect_err("a denied source must leave no destination object"); + assert_eq!(missing.raw_response().map(|response| response.status().as_u16()), Some(404)); set_department(&client, source, "finance") .await .expect_err("requested tags must not authorize their own mutation"); @@ -924,8 +985,6 @@ async fn test_bucket_tags_opa_contract() -> TestResult { assert_eq!(read.body.collect().await?.into_bytes().as_ref(), b"report"); opa.expect_bucket_tags("s3:GetObject", destination, Some("engineering")) .await?; - // Duplicate keys are refused on write; the ambiguous record below is seeded - // offline the way an upgraded deployment may still hold it on disk. let invalid_write = admin .put_bucket_tagging() .bucket(destination) @@ -938,6 +997,7 @@ async fn test_bucket_tags_opa_contract() -> TestResult { .send() .await .expect_err("new duplicate tags must be rejected before publication"); + assert_eq!(invalid_write.as_service_error().and_then(ProvideErrorMetadata::code), Some("InvalidTag")); assert_eq!(invalid_write.raw_response().map(|response| response.status().as_u16()), Some(400)); set_department(&admin, source, "finance").await?; diff --git a/rustfs/src/admin/router.rs b/rustfs/src/admin/router.rs index 3f454dff8..8c3819863 100644 --- a/rustfs/src/admin/router.rs +++ b/rustfs/src/admin/router.rs @@ -474,14 +474,20 @@ fn parse_misc_extension_request(method: &Method, uri: &Uri) -> Option S3Result<()> { let lambda_arns = query_values_exact(uri, "lambdaArn"); if lambda_arns.len() != 1 || lambda_arns[0].trim().is_empty() { - return Err(s3_error!(InvalidRequest, "lambdaArn query parameter must be provided exactly once")); + return Err(S3Error::with_message( + S3ErrorCode::InvalidRequest, + "lambdaArn query parameter must be provided exactly once", + )); } let lambda_arn = lambda_arns[0].trim(); let arn_parts = lambda_arn.split(':').collect::>(); let is_valid_arn = arn_parts.len() >= 6 && arn_parts[0] == "arn" && !arn_parts[1].is_empty() && !arn_parts[2].is_empty(); if !is_valid_arn { - return Err(s3_error!(InvalidRequest, "lambdaArn query parameter must be a valid ARN string")); + return Err(S3Error::with_message( + S3ErrorCode::InvalidRequest, + "lambdaArn query parameter must be a valid ARN string", + )); } Ok(()) } @@ -489,43 +495,56 @@ fn validate_object_lambda_query(uri: &Uri) -> S3Result<()> { fn validate_listen_notification_query(uri: &Uri) -> S3Result<()> { let events = query_values_exact(uri, "events"); if events.is_empty() { - return Err(s3_error!(InvalidArgument, "events query parameter is required")); + return Err(S3Error::with_message(S3ErrorCode::InvalidArgument, "events query parameter is required")); } for event in events { - EventName::parse(&event).map_err(|_| s3_error!(InvalidArgument, "invalid event in events query parameter"))?; + EventName::parse(&event) + .map_err(|_| S3Error::with_message(S3ErrorCode::InvalidArgument, "invalid event in events query parameter"))?; } let prefixes = query_values_exact(uri, "prefix"); if prefixes.len() > 1 { - return Err(s3_error!(InvalidArgument, "prefix query parameter must not be repeated")); + return Err(S3Error::with_message( + S3ErrorCode::InvalidArgument, + "prefix query parameter must not be repeated", + )); } if let Some(prefix) = prefixes.first() && !is_valid_filter_rule_value(prefix) { - return Err(s3_error!(InvalidArgument, "invalid prefix filter value")); + return Err(S3Error::with_message(S3ErrorCode::InvalidArgument, "invalid prefix filter value")); } let suffixes = query_values_exact(uri, "suffix"); if suffixes.len() > 1 { - return Err(s3_error!(InvalidArgument, "suffix query parameter must not be repeated")); + return Err(S3Error::with_message( + S3ErrorCode::InvalidArgument, + "suffix query parameter must not be repeated", + )); } if let Some(suffix) = suffixes.first() && !is_valid_filter_rule_value(suffix) { - return Err(s3_error!(InvalidArgument, "invalid suffix filter value")); + return Err(S3Error::with_message(S3ErrorCode::InvalidArgument, "invalid suffix filter value")); } let pings = query_values_exact(uri, "ping"); if pings.len() > 1 { - return Err(s3_error!(InvalidArgument, "ping query parameter must not be repeated")); + return Err(S3Error::with_message( + S3ErrorCode::InvalidArgument, + "ping query parameter must not be repeated", + )); } if let Some(ping) = pings.first() { - let ping_interval = ping - .parse::() - .map_err(|_| s3_error!(InvalidArgument, "ping query parameter must be a positive integer"))?; + let ping_interval = ping.parse::().map_err(|_| { + S3Error::with_message(S3ErrorCode::InvalidArgument, "ping query parameter must be a positive integer") + })?; if ping_interval == 0 { - return Err(s3_error!(InvalidArgument, "ping query parameter must be greater than zero")); + return Err(S3Error::with_message( + S3ErrorCode::InvalidArgument, + "ping query parameter must be greater than zero", + )); } } @@ -536,7 +555,7 @@ fn parse_listen_notification_filter(uri: &Uri, bucket: Option<&str>) -> S3Result let mut event_mask = 0_u64; for event in query_values_exact(uri, "events") { event_mask |= EventName::parse(&event) - .map_err(|_| s3_error!(InvalidArgument, "invalid event in events query parameter"))? + .map_err(|_| S3Error::with_message(S3ErrorCode::InvalidArgument, "invalid event in events query parameter"))? .mask(); } @@ -569,7 +588,7 @@ fn uri_without_query_key(uri: &Uri, excluded_key: &str) -> S3Result { Some( uri.path() .parse() - .map_err(|_| s3_error!(InvalidRequest, "failed to rebuild request URI"))?, + .map_err(|_| S3Error::with_message(S3ErrorCode::InvalidRequest, "failed to rebuild request URI"))?, ) } else { let query = form_urlencoded::Serializer::new(String::new()) @@ -578,10 +597,10 @@ fn uri_without_query_key(uri: &Uri, excluded_key: &str) -> S3Result { Some( format!("{}?{}", uri.path(), query) .parse() - .map_err(|_| s3_error!(InvalidRequest, "failed to rebuild request URI"))?, + .map_err(|_| S3Error::with_message(S3ErrorCode::InvalidRequest, "failed to rebuild request URI"))?, ) }; - Uri::from_parts(parts).map_err(|_| s3_error!(InvalidRequest, "failed to rebuild request URI")) + Uri::from_parts(parts).map_err(|_| S3Error::with_message(S3ErrorCode::InvalidRequest, "failed to rebuild request URI")) } fn parse_optional_header(headers: &HeaderMap, name: HeaderName) -> S3Result> { @@ -591,7 +610,7 @@ fn parse_optional_header(headers: &HeaderMap, name: HeaderName) -> S3Result S3R parse_optional_header(headers, name)? .map(|value| { Timestamp::parse(TimestampFormat::HttpDate, &value) - .map_err(|_| s3_error!(InvalidRequest, "request timestamp header is invalid")) + .map_err(|_| S3Error::with_message(S3ErrorCode::InvalidRequest, "request timestamp header is invalid")) }) .transpose() } @@ -613,7 +632,7 @@ where .map(|value| { value .parse::() - .map_err(|_| s3_error!(InvalidRequest, "request etag condition header is invalid")) + .map_err(|_| S3Error::with_message(S3ErrorCode::InvalidRequest, "request etag condition header is invalid")) }) .transpose() } @@ -623,14 +642,16 @@ fn build_object_lambda_get_request(req: &S3Request, bucket: &str, object: let part_number = query_value_exact(&filtered_uri, "partNumber") .filter(|value| !value.is_empty()) .map(|value| { - value - .parse::() - .map_err(|_| s3_error!(InvalidArgument, "partNumber query parameter must be a positive integer")) + value.parse::().map_err(|_| { + S3Error::with_message(S3ErrorCode::InvalidArgument, "partNumber query parameter must be a positive integer") + }) }) .transpose()?; let version_id = query_value_exact(&filtered_uri, "versionId").filter(|value| !value.is_empty()); let range = parse_optional_header(&req.headers, header::RANGE)? - .map(|value| Range::parse(&value).map_err(|_| s3_error!(InvalidArgument, "Range header is invalid"))) + .map(|value| { + Range::parse(&value).map_err(|_| S3Error::with_message(S3ErrorCode::InvalidArgument, "Range header is invalid")) + }) .transpose()?; let mut builder = GetObjectInput::builder() @@ -657,9 +678,9 @@ fn build_object_lambda_get_request(req: &S3Request, bucket: &str, object: HeaderName::from_static("x-amz-server-side-encryption-customer-key-md5"), )?); - let input = builder - .build() - .map_err(|err| s3_error!(InvalidRequest, "failed to build object lambda get request: {err}"))?; + let input = builder.build().map_err(|err| { + S3Error::with_message(S3ErrorCode::InvalidRequest, format!("failed to build object lambda get request: {err}")) + })?; Ok(S3Request { input, @@ -677,11 +698,16 @@ fn build_object_lambda_get_request(req: &S3Request, bucket: &str, object: fn parse_object_lambda_arn(uri: &Uri) -> S3Result { let lambda_arn = query_value_exact(uri, "lambdaArn") .filter(|value| !value.trim().is_empty()) - .ok_or_else(|| s3_error!(InvalidRequest, "lambdaArn query parameter must be provided exactly once"))?; + .ok_or_else(|| { + S3Error::with_message(S3ErrorCode::InvalidRequest, "lambdaArn query parameter must be provided exactly once") + })?; - lambda_arn - .parse::() - .map_err(|_| s3_error!(InvalidRequest, "lambdaArn query parameter must reference a supported target ARN")) + lambda_arn.parse::().map_err(|_| { + S3Error::with_message( + S3ErrorCode::InvalidRequest, + "lambdaArn query parameter must reference a supported target ARN", + ) + }) } fn config_enable_is_on(value: &str) -> bool { @@ -694,38 +720,43 @@ fn resolve_object_lambda_webhook_config_from_server_config( ) -> S3Result { let target_name = arn.target_id.name.to_ascii_lowercase(); if target_name != "webhook" && !target_name.starts_with("webhook-") { - return Err(s3_error!(NotImplemented, "object lambda target type is not supported")); + return Err(S3Error::with_message( + S3ErrorCode::NotImplemented, + "object lambda target type is not supported", + )); } let subsystem = config .0 .get(LAMBDA_WEBHOOK_SUB_SYS) .or_else(|| config.0.get(NOTIFY_WEBHOOK_SUB_SYS)) - .ok_or_else(|| s3_error!(InvalidRequest, "object lambda webhook subsystem is not configured"))?; + .ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "object lambda webhook subsystem is not configured"))?; let kvs = subsystem .get(&arn.target_id.id) - .ok_or_else(|| s3_error!(InvalidRequest, "object lambda target is not configured"))?; + .ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "object lambda target is not configured"))?; if !config_enable_is_on(&kvs.get(ENABLE_KEY)) { - return Err(s3_error!(InvalidRequest, "object lambda target is disabled")); + return Err(S3Error::with_message(S3ErrorCode::InvalidRequest, "object lambda target is disabled")); } let endpoint = kvs.lookup(WEBHOOK_ENDPOINT).unwrap_or_default(); if endpoint.trim().is_empty() { - return Err(s3_error!(InvalidRequest, "object lambda target endpoint is empty")); + return Err(S3Error::with_message( + S3ErrorCode::InvalidRequest, + "object lambda target endpoint is empty", + )); } let response_header_timeout = match kvs.lookup(WEBHOOK_RESPONSE_HEADER_TIMEOUT) { Some(value) if value.trim().is_empty() => None, - Some(value) => Some( - parse_duration(&value) - .map_err(|_| s3_error!(InvalidRequest, "object lambda target response_header_timeout is invalid"))?, - ), + Some(value) => Some(parse_duration(&value).map_err(|_| { + S3Error::with_message(S3ErrorCode::InvalidRequest, "object lambda target response_header_timeout is invalid") + })?), None => None, }; - let parsed_endpoint = - Url::parse(&endpoint).map_err(|_| s3_error!(InvalidRequest, "object lambda target endpoint is invalid"))?; + let parsed_endpoint = Url::parse(&endpoint) + .map_err(|_| S3Error::with_message(S3ErrorCode::InvalidRequest, "object lambda target endpoint is invalid"))?; let outbound_resolver = outbound_policy()? .resolver_for(&parsed_endpoint) .map_err(|err| s3_error!(InvalidRequest, "object lambda target endpoint is not allowed: {}", err))?; @@ -763,7 +794,8 @@ async fn load_current_server_config() -> S3Result { } } - let config = current_server_config().ok_or_else(|| s3_error!(InternalError, "server config is not initialized"))?; + let config = current_server_config() + .ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "server config is not initialized"))?; Ok(config) } @@ -802,10 +834,12 @@ fn build_object_lambda_http_client_with_resolver( ); builder = builder.danger_accept_invalid_certs(true); } else if !config.client_ca.is_empty() { - let ca_pem = std::fs::read(&config.client_ca) - .map_err(|e| s3_error!(InternalError, "failed to read object lambda client_ca: {e}"))?; - let ca = reqwest::Certificate::from_pem(&ca_pem) - .map_err(|e| s3_error!(InternalError, "failed to parse object lambda client_ca: {e}"))?; + let ca_pem = std::fs::read(&config.client_ca).map_err(|e| { + S3Error::with_message(S3ErrorCode::InternalError, format!("failed to read object lambda client_ca: {e}")) + })?; + let ca = reqwest::Certificate::from_pem(&ca_pem).map_err(|e| { + S3Error::with_message(S3ErrorCode::InternalError, format!("failed to parse object lambda client_ca: {e}")) + })?; builder = builder.add_root_certificate(ca); } @@ -817,25 +851,28 @@ fn build_object_lambda_http_client_with_resolver( )); } - let cert = std::fs::read(&config.client_cert) - .map_err(|e| s3_error!(InternalError, "failed to read object lambda client_cert: {e}"))?; - let key = std::fs::read(&config.client_key) - .map_err(|e| s3_error!(InternalError, "failed to read object lambda client_key: {e}"))?; - let identity = reqwest::Identity::from_pem(&[cert, key].concat()) - .map_err(|e| s3_error!(InternalError, "failed to build object lambda client identity: {e}"))?; + let cert = std::fs::read(&config.client_cert).map_err(|e| { + S3Error::with_message(S3ErrorCode::InternalError, format!("failed to read object lambda client_cert: {e}")) + })?; + let key = std::fs::read(&config.client_key).map_err(|e| { + S3Error::with_message(S3ErrorCode::InternalError, format!("failed to read object lambda client_key: {e}")) + })?; + let identity = reqwest::Identity::from_pem(&[cert, key].concat()).map_err(|e| { + S3Error::with_message(S3ErrorCode::InternalError, format!("failed to build object lambda client identity: {e}")) + })?; builder = builder.identity(identity); } builder .build() - .map_err(|e| s3_error!(InternalError, "failed to build object lambda http client: {e}")) + .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("failed to build object lambda http client: {e}"))) } async fn send_object_lambda_request(request: reqwest::RequestBuilder) -> S3Result { request .send() .await - .map_err(|_| s3_error!(InternalError, "object lambda target request failed")) + .map_err(|_| S3Error::with_message(S3ErrorCode::InternalError, "object lambda target request failed")) } fn outbound_policy() -> S3Result<&'static OutboundPolicy> { @@ -868,21 +905,21 @@ fn extract_request_host(headers: &HeaderMap, uri: &Uri) -> Option { } fn build_object_lambda_source_url(req: &S3Request) -> S3Result { - let credentials = req - .credentials - .as_ref() - .ok_or_else(|| s3_error!(AccessDenied, "object lambda source URL requires authenticated credentials"))?; - let host = extract_request_host(&req.headers, &req.uri) - .ok_or_else(|| s3_error!(InvalidRequest, "object lambda source URL requires a valid host header"))?; + let credentials = req.credentials.as_ref().ok_or_else(|| { + S3Error::with_message(S3ErrorCode::AccessDenied, "object lambda source URL requires authenticated credentials") + })?; + let host = extract_request_host(&req.headers, &req.uri).ok_or_else(|| { + S3Error::with_message(S3ErrorCode::InvalidRequest, "object lambda source URL requires a valid host header") + })?; let scheme = extract_request_scheme(&req.headers, &req.uri); let filtered_uri = uri_without_query_key(&req.uri, "lambdaArn")?; let path_and_query = filtered_uri .path_and_query() .map(|value| value.as_str().to_string()) .unwrap_or_else(|| filtered_uri.path().to_string()); - let source_uri = format!("{scheme}://{host}{path_and_query}") - .parse::() - .map_err(|e| s3_error!(InvalidRequest, "failed to construct object lambda source URL: {e}"))?; + let source_uri = format!("{scheme}://{host}{path_and_query}").parse::().map_err(|e| { + S3Error::with_message(S3ErrorCode::InvalidRequest, format!("failed to construct object lambda source URL: {e}")) + })?; let region = req .region .clone() @@ -897,7 +934,9 @@ fn build_object_lambda_source_url(req: &S3Request) -> S3Result { .uri(source_uri) .header(http::header::HOST, host) .body(Body::default()) - .map_err(|e| s3_error!(InvalidRequest, "failed to build object lambda source request: {e}"))?, + .map_err(|e| { + S3Error::with_message(S3ErrorCode::InvalidRequest, format!("failed to build object lambda source request: {e}")) + })?, &credentials.access_key, credentials.secret_key.expose(), &session_token, @@ -937,7 +976,7 @@ fn build_object_lambda_event_payload( }, "protocolVersion": "rustfs-object-lambda-1.0", })) - .map_err(|e| s3_error!(InternalError, "failed to serialize object lambda payload: {e}")) + .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("failed to serialize object lambda payload: {e}"))) } fn validate_object_lambda_response_auth_headers(headers: &HeaderMap, output_route: &str, output_token: &str) -> S3Result<()> { @@ -966,13 +1005,13 @@ fn format_timestamp_http_date(value: &Timestamp) -> S3Result { let mut buf = Vec::new(); value .format(TimestampFormat::HttpDate, &mut buf) - .map_err(|_| s3_error!(InternalError, "failed to format timestamp header"))?; - String::from_utf8(buf).map_err(|_| s3_error!(InternalError, "failed to format timestamp header")) + .map_err(|_| S3Error::with_message(S3ErrorCode::InternalError, "failed to format timestamp header"))?; + String::from_utf8(buf).map_err(|_| S3Error::with_message(S3ErrorCode::InternalError, "failed to format timestamp header")) } fn insert_string_header(headers: &mut HeaderMap, name: HeaderName, value: String) -> S3Result<()> { - let header_value = - HeaderValue::from_str(&value).map_err(|_| s3_error!(InternalError, "failed to build response header value"))?; + let header_value = HeaderValue::from_str(&value) + .map_err(|_| S3Error::with_message(S3ErrorCode::InternalError, "failed to build response header value"))?; headers.insert(name, header_value); Ok(()) } @@ -1007,7 +1046,8 @@ fn build_get_object_response_headers(output: &GetObjectOutput, base_headers: &He if let Some(etag) = &output.e_tag { headers.insert( http::header::ETAG, - etag.to_http_header().map_err(|_| s3_error!(InternalError, "invalid etag"))?, + etag.to_http_header() + .map_err(|_| S3Error::with_message(S3ErrorCode::InternalError, "invalid etag"))?, ); } if let Some(last_modified) = &output.last_modified { @@ -1091,7 +1131,7 @@ fn build_get_object_response_headers(output: &GetObjectOutput, base_headers: &He let header_name = format!("x-amz-meta-{key}"); if let Ok(parsed_name) = HeaderName::from_bytes(header_name.as_bytes()) { let parsed_value = HeaderValue::from_str(value) - .map_err(|_| s3_error!(InternalError, "failed to build metadata response header"))?; + .map_err(|_| S3Error::with_message(S3ErrorCode::InternalError, "failed to build metadata response header"))?; headers.insert(parsed_name, parsed_value); } } @@ -1195,7 +1235,9 @@ async fn invoke_object_lambda_target( let client = build_object_lambda_http_client(&lambda_config)?; let lambda_arn = query_value_exact(&req.uri, "lambdaArn") .filter(|value| !value.trim().is_empty()) - .ok_or_else(|| s3_error!(InvalidRequest, "lambdaArn query parameter must be provided exactly once"))?; + .ok_or_else(|| { + S3Error::with_message(S3ErrorCode::InvalidRequest, "lambdaArn query parameter must be provided exactly once") + })?; let input_s3_url = build_object_lambda_source_url(req)?; let output_route = Uuid::new_v4().to_string(); let output_token = Uuid::new_v4().to_string(); @@ -1316,7 +1358,7 @@ fn serialize_listen_notification_event(event: &NotificationEvent) -> S3Result Vec { @@ -1340,13 +1382,174 @@ fn deserialize_peer_live_events(payload: &[u8]) -> Result serde_json::from_slice(payload) } +const LISTEN_AUTH_REFRESH_INTERVAL: Duration = Duration::from_secs(1); +const LISTEN_AUTH_TIMEOUT: Duration = Duration::from_secs(5); + +// No Debug implementation: the admitted request retains its signing credentials. +struct ListenNotificationAuthorization { + request: S3Request<()>, + action: Action, + bucket_incarnation: Option<(String, uuid::Uuid)>, + keystone_credentials: Option, + refresh_at: tokio::time::Instant, +} + +impl ListenNotificationAuthorization { + async fn capture(req: &S3Request, bucket: Option<&str>) -> S3Result { + let bucket_incarnation = if let Some(bucket) = bucket { + let store = crate::admin::runtime_sources::object_store_from_req(req) + .ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "Object store is unavailable"))?; + Some(( + bucket.to_string(), + store.bucket_incarnation_id_from_disk(bucket).await.map_err(ApiError::from)?, + )) + } else { + None + }; + Ok(Self { + request: S3Request { + input: (), + method: req.method.clone(), + uri: req.uri.clone(), + headers: req.headers.clone(), + extensions: req.extensions.clone(), + credentials: req.credentials.clone(), + region: req.region.clone(), + service: req.service.clone(), + trailing_headers: req.trailing_headers.clone(), + }, + action: Action::S3Action(if bucket.is_some() { + S3Action::ListenBucketNotificationAction + } else { + S3Action::ListenNotificationAction + }), + bucket_incarnation, + keystone_credentials: rustfs_keystone::KEYSTONE_CREDENTIALS.try_with(Clone::clone).ok().flatten(), + refresh_at: tokio::time::Instant::now(), + }) + } + + async fn still_allowed(&mut self) -> bool { + matches!(tokio::time::timeout(LISTEN_AUTH_TIMEOUT, self.reauthorize()).await, Ok(Ok(()))) + } + + async fn reauthorize(&mut self) -> S3Result<()> { + let now = tokio::time::Instant::now(); + if now >= self.refresh_at { + if let Some(credentials) = &self.keystone_credentials { + let provider = + crate::auth_keystone::get_keystone_auth().ok_or_else(|| S3Error::new(S3ErrorCode::AccessDenied))?; + let token = self + .request + .headers + .get("x-auth-token") + .and_then(|v| v.to_str().ok()) + .or_else(|| self.request.headers.get("x-storage-token").and_then(|v| v.to_str().ok())) + .ok_or_else(|| S3Error::new(S3ErrorCode::AccessDenied))?; + let current = provider + .revalidate_subscription_token(token) + .await + .map_err(|_| S3Error::new(S3ErrorCode::AccessDenied))?; + if current.access_key != credentials.access_key { + return Err(S3Error::new(S3ErrorCode::AccessDenied)); + } + self.keystone_credentials = Some(current); + } + if let Some((bucket, expected)) = &self.bucket_incarnation { + let store = crate::admin::runtime_sources::object_store_from_req(&self.request) + .ok_or_else(|| S3Error::new(S3ErrorCode::AccessDenied))?; + let actual = store.bucket_incarnation_id_from_disk(bucket).await.map_err(ApiError::from)?; + if actual != *expected { + return Err(S3Error::new(S3ErrorCode::AccessDenied)); + } + } + self.refresh_at = now + LISTEN_AUTH_REFRESH_INTERVAL; + } + if self + .keystone_credentials + .as_ref() + .and_then(|cred| cred.expiration) + .is_some_and(|expiry| expiry <= time::OffsetDateTime::now_utc()) + { + return Err(S3Error::new(S3ErrorCode::AccessDenied)); + } + let context = app_context_from_req(&self.request); + if context.is_none() && self.request.extensions.get::>().is_some() { + return Err(S3Error::new(S3ErrorCode::AccessDenied)); + } + let keystone = self.keystone_credentials.clone(); + let input = self + .request + .credentials + .clone() + .or_else(|| { + keystone.as_ref().map(|credentials| s3s::auth::Credentials { + access_key: credentials.access_key.clone(), + secret_key: "".into(), + }) + }) + .ok_or_else(|| S3Error::new(S3ErrorCode::AccessDenied))?; + rustfs_keystone::KEYSTONE_CREDENTIALS + .scope(keystone, async { + let (cred, is_owner) = crate::auth::check_key_valid_with_context( + get_session_token(&self.request.uri, &self.request.headers).unwrap_or_default(), + &input.access_key, + context.as_deref(), + ) + .await?; + if self.keystone_credentials.is_none() && !constant_time_eq(&cred.secret_key, input.secret_key.expose()) { + return Err(S3Error::new(S3ErrorCode::AccessDenied)); + } + let mut info = self + .request + .extensions + .get::() + .cloned() + .ok_or_else(|| S3Error::new(S3ErrorCode::AccessDenied))?; + info.cred = Some(cred); + info.is_owner = is_owner; + self.request.extensions.insert(info); + authorize_request(&mut self.request, self.action).await + }) + .await + } + + async fn still_allowed_until_closed(&mut self, tx: &mpsc::Sender>) -> bool { + tokio::select! { + biased; + _ = tx.closed() => false, + allowed = self.still_allowed() => allowed, + } + } + + async fn send(&mut self, tx: &mpsc::Sender>, bytes: Bytes) -> bool { + loop { + tokio::select! { + permit = tx.reserve() => { + let Ok(permit) = permit else { return false; }; + if !self.still_allowed_until_closed(tx).await { return false; } + permit.send(Ok(bytes)); + return true; + } + _ = tokio::time::sleep(LISTEN_AUTH_REFRESH_INTERVAL) => { + if !self.still_allowed_until_closed(tx).await { return false; } + } + } + } + } +} + async fn fan_in_remote_live_events( peers: &mut [PeerLiveEventCursor], filter: &ListenNotificationFilter, tx: &mpsc::Sender>, + authorization: &mut ListenNotificationAuthorization, ) -> bool { for peer in peers.iter_mut() { loop { + if !authorization.still_allowed().await { + return false; + } let batch = match tokio::time::timeout( Duration::from_secs(2), peer.client @@ -1392,7 +1595,7 @@ async fn fan_in_remote_live_events( } match serialize_listen_notification_event(&event) { Ok(serialized) => { - if tx.send(Ok(serialized)).await.is_err() { + if !authorization.send(tx, serialized).await { return false; } } @@ -1434,7 +1637,11 @@ async fn fan_in_remote_live_events( true } -fn build_listen_notification_response(uri: &Uri, bucket: Option<&str>) -> S3Result> { +fn build_listen_notification_response( + uri: &Uri, + bucket: Option<&str>, + mut authorization: ListenNotificationAuthorization, +) -> S3Result> { let (interval_duration, payload) = listen_notification_keepalive_plan(uri); let filter = parse_listen_notification_filter(uri, bucket)?; let mut live_events = notification_system().map(|system| system.subscribe_live_events()); @@ -1446,6 +1653,7 @@ fn build_listen_notification_response(uri: &Uri, bucket: Option<&str>) -> S3Resu }); spawn_traced(async move { + let mut authorization_ticker = tokio::time::interval(LISTEN_AUTH_REFRESH_INTERVAL); let mut ticker = tokio::time::interval(interval_duration); let mut peer_ticker = tokio::time::interval(LISTEN_NOTIFICATION_PEER_POLL_INTERVAL); // Skip the immediate first tick so behavior starts after interval duration. @@ -1455,8 +1663,11 @@ fn build_listen_notification_response(uri: &Uri, bucket: Option<&str>) -> S3Resu if let Some(events_rx) = live_events.as_mut() { tokio::select! { _ = tx.closed() => break, + _ = authorization_ticker.tick() => { + if !authorization.still_allowed_until_closed(&tx).await { break; } + }, _ = ticker.tick() => { - if tx.send(Ok(payload.clone())).await.is_err() { + if !authorization.send(&tx, payload.clone()).await { break; } } @@ -1468,7 +1679,7 @@ fn build_listen_notification_response(uri: &Uri, bucket: Option<&str>) -> S3Resu } match serialize_listen_notification_event(&event) { Ok(serialized) => { - if tx.send(Ok(serialized)).await.is_err() { + if !authorization.send(&tx, serialized).await { break; } } @@ -1499,7 +1710,11 @@ fn build_listen_notification_response(uri: &Uri, bucket: Option<&str>) -> S3Resu } } _ = peer_ticker.tick(), if !peer_live_events.is_empty() => { - if !fan_in_remote_live_events(&mut peer_live_events, &filter, &tx).await { + if !tokio::select! { + biased; + _ = tx.closed() => false, + result = fan_in_remote_live_events(&mut peer_live_events, &filter, &tx, &mut authorization) => result, + } { break; } } @@ -1507,13 +1722,20 @@ fn build_listen_notification_response(uri: &Uri, bucket: Option<&str>) -> S3Resu } else { tokio::select! { _ = tx.closed() => break, + _ = authorization_ticker.tick() => { + if !authorization.still_allowed_until_closed(&tx).await { break; } + }, _ = ticker.tick() => { - if tx.send(Ok(payload.clone())).await.is_err() { + if !authorization.send(&tx, payload.clone()).await { break; } } _ = peer_ticker.tick(), if !peer_live_events.is_empty() => { - if !fan_in_remote_live_events(&mut peer_live_events, &filter, &tx).await { + if !tokio::select! { + biased; + _ = tx.closed() => false, + result = fan_in_remote_live_events(&mut peer_live_events, &filter, &tx, &mut authorization) => result, + } { break; } } @@ -1547,7 +1769,7 @@ async fn ensure_replication_bucket_exists(bucket: &str) -> S3Result<()> { async fn ensure_replication_config_exists(bucket: &str) -> S3Result<()> { match metadata_sys::get_replication_config(bucket).await { Ok(_) => Ok(()), - Err(StorageError::ConfigNotFound) => Err(s3_error!(ReplicationConfigurationNotFoundError)), + Err(StorageError::ConfigNotFound) => Err(S3Error::new(S3ErrorCode::ReplicationConfigurationNotFoundError)), Err(err) => Err(ApiError::from(err).into()), } } @@ -1634,21 +1856,19 @@ fn serialize_replication_metrics_body( ) -> S3Result> { use crate::admin::replication_metrics_wire::{MetricsV2Wire, MetricsWire}; match route { - ReplicationExtRoute::MetricsV1 => { - serde_json::to_vec(&MetricsWire::from(&bucket_stats.replication_stats)).map_err(|e| s3_error!(InternalError, "{e}")) - } - ReplicationExtRoute::MetricsV2 => { - serde_json::to_vec(&MetricsV2Wire::from_stats(bucket_stats, node_name)).map_err(|e| s3_error!(InternalError, "{e}")) - } + ReplicationExtRoute::MetricsV1 => serde_json::to_vec(&MetricsWire::from(&bucket_stats.replication_stats)) + .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("{e}"))), + ReplicationExtRoute::MetricsV2 => serde_json::to_vec(&MetricsV2Wire::from_stats(bucket_stats, node_name)) + .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("{e}"))), ReplicationExtRoute::Check | ReplicationExtRoute::ResetStart | ReplicationExtRoute::ResetStatus => { - Err(s3_error!(InternalError, "invalid route for metrics response")) + Err(S3Error::with_message(S3ErrorCode::InternalError, "invalid route for metrics response")) } } } async fn authorize_replication_extension_request(req: &mut S3Request, ext_req: &ReplicationExtRequest) -> S3Result<()> { let Some(input_cred) = req.credentials.as_ref() else { - return Err(s3_error!(AccessDenied, "Signature is required")); + return Err(S3Error::with_message(S3ErrorCode::AccessDenied, "Signature is required")); }; let (cred, is_owner) = @@ -1665,7 +1885,7 @@ async fn authorize_replication_extension_request(req: &mut S3Request, ext_ }); license_check().map_err(|er| match er.kind() { - std::io::ErrorKind::PermissionDenied => s3_error!(AccessDenied, "{er}"), + std::io::ErrorKind::PermissionDenied => S3Error::with_message(S3ErrorCode::AccessDenied, format!("{er}")), _ => { error!( event = EVENT_ADMIN_ROUTER_STATE, @@ -1675,7 +1895,7 @@ async fn authorize_replication_extension_request(req: &mut S3Request, ext_ error = %er, "admin router state" ); - s3_error!(InternalError, "License validation failed") + S3Error::with_message(S3ErrorCode::InternalError, "License validation failed") } })?; @@ -1700,10 +1920,12 @@ fn parse_reset_start_target(uri: &Uri) -> S3Result let now = OffsetDateTime::now_utc(); let reset_before = match query_value_exact(uri, "older-than").filter(|v| !v.is_empty()) { Some(older_than) => { - let duration = parse_duration(&older_than) - .map_err(|err| s3_error!(InvalidRequest, "invalid older-than query parameter: {err}"))?; - let duration = time::Duration::try_from(duration) - .map_err(|err| s3_error!(InvalidRequest, "invalid older-than query parameter: {err}"))?; + let duration = parse_duration(&older_than).map_err(|err| { + S3Error::with_message(S3ErrorCode::InvalidRequest, format!("invalid older-than query parameter: {err}")) + })?; + let duration = time::Duration::try_from(duration).map_err(|err| { + S3Error::with_message(S3ErrorCode::InvalidRequest, format!("invalid older-than query parameter: {err}")) + })?; Some(now - duration) } None => Some(now), @@ -1774,7 +1996,10 @@ fn resolve_replication_reset_target_arn(config: &s3s::dto::ReplicationConfigurat let (has_arn, existing_object_enabled) = config.has_existing_object_replication(requested_arn); if !has_arn { - return Err(s3_error!(InvalidRequest, "replication reset arn is not configured for this bucket")); + return Err(S3Error::with_message( + S3ErrorCode::InvalidRequest, + "replication reset arn is not configured for this bucket", + )); } if !existing_object_enabled { return Err(s3_error!( @@ -1787,7 +2012,8 @@ fn resolve_replication_reset_target_arn(config: &s3s::dto::ReplicationConfigurat } fn build_replication_reset_response(targets: Vec) -> S3Result> { - let data = serde_json::to_vec(&ReplicationResetResponse { targets }).map_err(|e| s3_error!(InternalError, "{e}"))?; + let data = serde_json::to_vec(&ReplicationResetResponse { targets }) + .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("{e}")))?; let mut resp = S3Response::with_status(Body::from(data), StatusCode::OK); resp.headers .insert(header::CONTENT_TYPE, HeaderValue::from_static("application/json")); @@ -1797,7 +2023,7 @@ fn build_replication_reset_response(targets: Vec) -> S3R fn map_replication_resync_start_error(error: StorageError) -> S3Error { match replication::resync_start_conflict_id(&error) { Some(active_resync_id) => replication_resync_active_conflict_error(active_resync_id), - None => s3_error!(InternalError, "{error}"), + None => S3Error::with_message(S3ErrorCode::InternalError, format!("{error}")), } } @@ -1849,7 +2075,7 @@ fn build_replication_reset_status_response( let data = serde_json::to_vec(&ReplicationResetStatusResponse { targets: build_replication_reset_status_targets(&status, arn_filter), }) - .map_err(|e| s3_error!(InternalError, "{e}"))?; + .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("{e}")))?; let mut resp = S3Response::with_status(Body::from(data), StatusCode::OK); resp.headers .insert(header::CONTENT_TYPE, HeaderValue::from_static("application/json")); @@ -1870,7 +2096,7 @@ fn build_replication_check_response(mut targets: Vec S3Result { let Some(pool) = current_replication_pool_handle() else { - return Err(s3_error!(InternalError, "replication pool is not initialized")); + return Err(S3Error::with_message(S3ErrorCode::InternalError, "replication pool is not initialized")); }; let _targets_guard = lock_bucket_targets_metadata(bucket).await; @@ -3000,12 +3229,12 @@ async fn start_replication_resync(bucket: &str, reset: &ReplicationResetStartReq async fn load_replication_resync_status(bucket: &str) -> S3Result { let Some(pool) = current_replication_pool_handle() else { - return Err(s3_error!(InternalError, "replication pool is not initialized")); + return Err(S3Error::with_message(S3ErrorCode::InternalError, "replication pool is not initialized")); }; pool.get_bucket_resync_status(bucket) .await - .map_err(|e| s3_error!(InternalError, "{e}")) + .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("{e}"))) } async fn handle_replication_extension_request( @@ -3050,7 +3279,7 @@ async fn handle_replication_extension_request( async fn authorize_misc_extension_request(req: &mut S3Request, route: &MiscExtRoute) -> S3Result<()> { let Some(input_cred) = req.credentials.as_ref() else { - return Err(s3_error!(AccessDenied, "Signature is required")); + return Err(S3Error::with_message(S3ErrorCode::AccessDenied, "Signature is required")); }; let (cred, is_owner) = @@ -3077,7 +3306,7 @@ async fn authorize_misc_extension_request(req: &mut S3Request, route: &Mis }); license_check().map_err(|er| match er.kind() { - std::io::ErrorKind::PermissionDenied => s3_error!(AccessDenied, "{er}"), + std::io::ErrorKind::PermissionDenied => S3Error::with_message(S3ErrorCode::AccessDenied, format!("{er}")), _ => { error!( event = EVENT_ADMIN_ROUTER_STATE, @@ -3087,20 +3316,39 @@ async fn authorize_misc_extension_request(req: &mut S3Request, route: &Mis error = %er, "admin router state" ); - s3_error!(InternalError, "License validation failed") + S3Error::with_message(S3ErrorCode::InternalError, "License validation failed") } })?; + if matches!(route, MiscExtRoute::ObjectLambda { .. }) { + // The bridged GetObject hook evaluates the actual version and all read conditions. + return Ok(()); + } authorize_request(req, action).await } +async fn authorized_object_lambda_get_request( + req: &S3Request, + bucket: &str, + object: &str, +) -> S3Result> { + let mut get_req = build_object_lambda_get_request(req, bucket, object)?; + let slot = req + .extensions + .get::>() + .cloned() + .ok_or_else(server_context_not_ready_error)?; + s3s::access::S3Access::get_object(&crate::storage::ecfs::FS::with_server_ctx(slot), &mut get_req).await?; + Ok(get_req) +} + async fn handle_misc_extension_request(req: &mut S3Request, route: &MiscExtRoute) -> S3Result> { authorize_misc_extension_request(req, route).await?; validate_misc_extension_request(&req.uri, route)?; match route { MiscExtRoute::ObjectLambda { bucket, object } => { - let get_req = build_object_lambda_get_request(req, bucket, object)?; + let get_req = authorized_object_lambda_get_request(req, bucket, object).await?; let usecase = default_object_usecase(); let get_resp = usecase.execute_get_object(get_req).await?; invoke_object_lambda_target(req, bucket, object, get_resp).await @@ -3115,7 +3363,8 @@ async fn handle_misc_extension_request(req: &mut S3Request, route: &MiscEx .await .map_err(ApiError::from)?; } - build_listen_notification_response(&req.uri, bucket.as_deref()) + let authorization = ListenNotificationAuthorization::capture(req, bucket.as_deref()).await?; + build_listen_notification_response(&req.uri, bucket.as_deref(), authorization) } } } @@ -3136,8 +3385,40 @@ fn is_public_health_path(path: &str) -> bool { path == HEALTH_PREFIX || path == HEALTH_READY_PATH } +fn reject_unbound_custom_route_auth(req: &S3Request) -> S3Result<()> { + // V2 does not bind custom route query arguments or payloads to the signer. + let signed_v2 = req + .headers + .get_all(http::header::AUTHORIZATION) + .iter() + .any(|value| value.to_str().is_ok_and(|value| value.starts_with("AWS "))); + let presigned_v2 = req + .uri + .query() + .is_some_and(|query| url::form_urlencoded::parse(query.as_bytes()).any(|(key, _)| key == "Signature")); + let post_policy = req.method == Method::POST + && req.credentials.is_some() + && req + .headers + .get(http::header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| { + value + .split(';') + .next() + .is_some_and(|media_type| media_type.trim().eq_ignore_ascii_case("multipart/form-data")) + }); + if signed_v2 || presigned_v2 || post_policy { + return Err(s3_error!( + AccessDenied, + "Header or query Signature Version 4 is required for this operation" + )); + } + Ok(()) +} + fn server_context_not_ready_error() -> S3Error { - s3_error!(ServiceUnavailable, "server context is not ready") + S3Error::with_message(S3ErrorCode::ServiceUnavailable, "server context is not ready") } fn is_object_zip_download_token_path(method: &Method, uri: &Uri) -> bool { @@ -3287,9 +3568,10 @@ where if parse_replication_extension_request(&req.method, &req.uri).is_some() || parse_misc_extension_request(&req.method, &req.uri).is_some() { + reject_unbound_custom_route_auth(req)?; return match req.credentials { Some(_) => Ok(()), - None => Err(s3_error!(AccessDenied, "Signature is required")), + None => Err(S3Error::with_message(S3ErrorCode::AccessDenied, "Signature is required")), }; } @@ -3327,9 +3609,10 @@ where } // For non-RPC admin requests, check credentials + reject_unbound_custom_route_auth(req)?; match req.credentials { Some(_) => Ok(()), - None => Err(s3_error!(AccessDenied, "Signature is required")), + None => Err(S3Error::with_message(S3ErrorCode::AccessDenied, "Signature is required")), } } @@ -3359,7 +3642,7 @@ where Err(e) => Err(s3_error!(InternalError, "{}", e)), }; } - return Err(s3_error!(InternalError, "console is not enabled")); + return Err(S3Error::with_message(S3ErrorCode::InternalError, "console is not enabled")); } let canonical_path = canonicalize_admin_path(req.uri.path()); @@ -3380,7 +3663,7 @@ where return Ok(resp); } - Err(s3_error!(NotImplemented)) + Err(S3Error::new(S3ErrorCode::NotImplemented)) } } @@ -3402,6 +3685,74 @@ impl Operation for AdminOperation { #[cfg(test)] mod tests { + #[test] + fn object_lambda_bridge_enforces_version_read_permissions() { + crate::app::gating_test_env::run_large_stack_test("lambda-version-permissions", || async { + use crate::admin::runtime_sources::AppContext; + use crate::admin::storage_api::contract::bucket::MakeBucketOptions; + let store = crate::app::gating_test_env::shared_gating_ecstore().await; + let ambient = crate::app::gating_test_env::shared_gating_ambient().await; + let slot = ServerContextSlot::new(); + assert!(slot.install(Arc::new(AppContext::new(store.clone(), ambient.iam(), ambient.kms())))); + let bucket = format!("lambda-version-{}", uuid::Uuid::new_v4()); + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create bucket"); + for (grant_current, grant_versions, deny_versions) in [ + (true, false, false), + (true, true, true), + (true, true, false), + (false, true, false), + ] { + let mut statements = Vec::new(); + if grant_current { + statements.push(serde_json::json!({"Effect":"Allow","Principal":{"AWS":"*"},"Action":"s3:GetObject","Resource":format!("arn:aws:s3:::{bucket}/*")})); + } + for (enabled, effect) in [(grant_versions, "Allow"), (deny_versions, "Deny")] { + if enabled { + statements.push(serde_json::json!({"Effect":effect,"Principal":{"AWS":"*"},"Action":"s3:GetObjectVersion","Resource":format!("arn:aws:s3:::{bucket}/*")})); + } + } + store + .update_bucket_metadata_config( + &bucket, + crate::storage::storage_api::ecstore_bucket::metadata::BUCKET_POLICY_CONFIG, + serde_json::to_vec(&serde_json::json!({"Version":"2012-10-17","Statement":statements})).unwrap(), + ) + .await + .expect("publish policy"); + for version in [None, Some("null"), Some("12345678-1234-1234-1234-123456789abc")] { + let mut req = router_request(Method::GET, "/"); + req.uri = format!( + "/{bucket}/object?lambdaArn=arn:aws:s3-object-lambda:us-east-1:123456789012:accesspoint/test{}", + version.map(|v| format!("&versionId={v}")).unwrap_or_default() + ) + .parse() + .unwrap(); + req.extensions.insert(slot.clone()); + req.extensions.insert(ReqInfo::default()); + req.headers.insert(http::header::IF_NONE_MATCH, HeaderValue::from_static("*")); + let result = authorized_object_lambda_get_request(&req, &bucket, "object").await; + // Version-only Allow covers current reads; Version Deny overrides + // the explicit current-read Allow as well as version grants. + if deny_versions || (version.is_some() && !grant_versions) { + assert_eq!( + result + .expect_err("version read must be denied before usecase or webhook") + .code(), + &s3s::S3ErrorCode::AccessDenied + ); + } else { + let request = result.expect("authorized read remains available"); + assert_eq!(request.input.version_id.as_deref(), version); + assert_eq!(request.extensions.get::().unwrap().version_id.as_deref(), version); + } + } + } + }); + } + use super::*; use http::HeaderMap; use http::Method; @@ -5700,6 +6051,89 @@ mod tests { assert_eq!(err.code(), &S3ErrorCode::AccessDenied); } + #[tokio::test] + async fn custom_routes_reject_signature_v2_arguments() { + let router: S3Router = S3Router::new(false); + for uri in [ + "/rustfs/admin/v3/add-user?accessKey=target", + "/minio/admin/v3/set-user-or-group-policy?userOrGroup=target", + "/?Action=AssumeRole", + "/bucket?replication-metrics", + "/iceberg/v1/config", + ] { + for presigned in [false, true] { + let method = if uri.contains("replication-metrics") { + Method::GET + } else { + Method::POST + }; + let mut req = router_request(method, uri); + req.credentials = Some(s3s::auth::Credentials { + access_key: "test".into(), + secret_key: "secret".into(), + }); + req.headers + .insert(http::header::CONTENT_TYPE, HeaderValue::from_static("application/x-www-form-urlencoded")); + if presigned { + let separator = if uri.contains('?') { '&' } else { '?' }; + req.uri = format!("{uri}{separator}%53ignature=captured&AWSAccessKeyId=test&Expires=1790000000") + .parse() + .unwrap(); + } else { + req.headers + .insert(http::header::AUTHORIZATION, HeaderValue::from_static("AWS test:captured")); + } + let error = router + .check_access(&mut req) + .await + .expect_err("V2 custom arguments are not signed"); + assert_eq!(error.code(), &S3ErrorCode::AccessDenied, "{uri}"); + assert_eq!( + error.message(), + Some("Header or query Signature Version 4 is required for this operation") + ); + } + } + } + + #[tokio::test] + async fn custom_routes_preserve_verified_sigv4_access() { + let router: S3Router = S3Router::new(false); + for uri in ["/rustfs/admin/v3/info", "/bucket?replication-metrics", "/?Action=AssumeRole"] { + let mut req = router_request(Method::GET, uri); + req.credentials = Some(s3s::auth::Credentials { + access_key: "test".into(), + secret_key: "secret".into(), + }); + req.headers.insert(http::header::AUTHORIZATION, HeaderValue::from_static( + "AWS4-HMAC-SHA256 Credential=test/20260827/us-east-1/s3/aws4_request, SignedHeaders=host, Signature=0000000000000000000000000000000000000000000000000000000000000000" + )); + router + .check_access(&mut req) + .await + .expect("verified V4 requests remain accepted"); + } + } + + #[tokio::test] + async fn custom_routes_reject_post_policy_authentication() { + let router: S3Router = S3Router::new(false); + for content_type in ["multipart/form-data; boundary=upload", "Multipart/Form-Data; boundary=upload"] { + let mut req = router_request(Method::POST, "/rustfs/admin/v3/service?action=stop"); + req.credentials = Some(s3s::auth::Credentials { + access_key: "test".into(), + secret_key: "secret".into(), + }); + req.headers + .insert(http::header::CONTENT_TYPE, HeaderValue::from_static(content_type)); + let error = router + .check_access(&mut req) + .await + .expect_err("upload form signatures do not authorize admin routes"); + assert_eq!(error.code(), &S3ErrorCode::AccessDenied); + } + } + /// GHSA-g8w9-qw9q-fghr: custom routes must apply the presigned /// signed-header rule too, since they never reach `S3Access::check`. #[tokio::test] @@ -6038,13 +6472,212 @@ mod tests { assert!(body.ends_with('\n')); } + #[test] + #[serial_test::serial] + fn listen_notification_streams_close_after_identity_policy_or_bucket_revocation() { + crate::app::gating_test_env::run_large_stack_test("listen-revocation", || async { + use crate::admin::runtime_sources::{AppContext, IamInterface}; + use crate::admin::storage_api::contract::bucket::{DeleteBucketOptions, MakeBucketOptions}; + use rustfs_iam::store::{Store as _, UserType, object::ObjectStore}; + use rustfs_madmin::{AccountStatus, AddOrUpdateUserReq}; + use rustfs_policy::policy::Policy; + struct ReadyIam(Arc>); + impl IamInterface for ReadyIam { + fn handle(&self) -> Arc> { + Arc::clone(&self.0) + } + fn is_ready(&self) -> bool { + true + } + } + let (_directory, _paths, store) = crate::app::gating_test_env::isolated_multi_pool_ecstore().await; + ObjectStore::new(store.clone()) + .save_iam_config( + serde_json::json!({"version":1}), + format!("{}/format.json", *rustfs_iam::store::object::IAM_CONFIG_PREFIX), + ) + .await + .expect("IAM format"); + let iam = rustfs_iam::build_iam_sys(store.clone()).await.expect("IAM"); + let ambient = crate::app::gating_test_env::shared_gating_ambient().await; + let context = Arc::new(AppContext::new(store.clone(), Arc::new(ReadyIam(iam.clone())), ambient.kms())); + assert!(context.publish_action_credentials(rustfs_credentials::Credentials { + access_key: "listen-test-root".into(), + secret_key: "listen-root-secret".into(), + status: "on".into(), + ..Default::default() + })); + let slot = ServerContextSlot::new(); + assert!(slot.install(context)); + for (case, bucket_scoped) in [ + ("detach", false), + ("disable", true), + ("delete", false), + ("rotate", true), + ("recreate", true), + ("expire", true), + ] { + let key = format!("listen-{case}"); + let bucket = format!("listen-{case}-{}", uuid::Uuid::new_v4()); + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("bucket"); + iam.create_user( + &key, + &AddOrUpdateUserReq { + secret_key: "listen-user-secret".into(), + policy: None, + status: AccountStatus::Enabled, + }, + ) + .await + .expect("listener"); + let policy = serde_json::json!({"Version":"2012-10-17","Statement":[{"Effect":"Allow", + "Action":["s3:ListenNotification","s3:ListenBucketNotification"],"Resource":"arn:aws:s3:::*"}]}); + iam.set_policy( + &key, + Policy::parse_config(&serde_json::to_vec(&policy).expect("policy JSON")).expect("policy"), + ) + .await + .expect("set policy"); + iam.policy_db_set(&key, UserType::Reg, false, &key) + .await + .expect("attach policy"); + let temporary = if case == "expire" { + let claims = std::collections::HashMap::from([ + ( + "exp".to_string(), + serde_json::json!((time::OffsetDateTime::now_utc() + time::Duration::seconds(5)).unix_timestamp()), + ), + ("parent".to_string(), serde_json::json!(key)), + ]); + let mut cred = rustfs_policy::auth::get_new_credentials_with_metadata(&claims, "listen-root-secret") + .expect("signed temporary credentials"); + cred.parent_user = key.clone(); + iam.set_temp_user(&cred.access_key, &cred, None) + .await + .expect("temporary listener"); + Some(cred) + } else { + None + }; + let uri: Uri = if bucket_scoped { + format!("/{bucket}?events=s3:ObjectCreated:*&ping=1") + } else { + "/?events=s3:ObjectCreated:*&ping=1".to_string() + } + .parse() + .expect("listen URI"); + let mut request = S3Request { + input: Body::empty(), + method: Method::GET, + uri: uri.clone(), + headers: HeaderMap::new(), + extensions: http::Extensions::new(), + credentials: Some(s3s::auth::Credentials { + access_key: key.clone(), + secret_key: "listen-user-secret".into(), + }), + region: None, + service: None, + trailing_headers: None, + }; + if let Some(cred) = &temporary { + request.credentials = Some(s3s::auth::Credentials { + access_key: cred.access_key.clone(), + secret_key: cred.secret_key.clone().into(), + }); + request + .headers + .insert("x-amz-security-token", cred.session_token.parse().expect("session header")); + } + request.extensions.insert(Arc::clone(&slot)); + request.extensions.insert(ReqInfo { + bucket: bucket_scoped.then(|| bucket.clone()), + ..Default::default() + }); + let mut authorization = + ListenNotificationAuthorization::capture(&request, bucket_scoped.then_some(bucket.as_str())) + .await + .expect("admit subscription"); + assert!(authorization.still_allowed().await, "initial listener {case}"); + // Fill a real stream's output channel, then revoke while its next + // send is waiting. A consumer pause cannot defer reauthorization. + let (tx, mut rx) = mpsc::channel(1); + assert!(authorization.send(&tx, Bytes::from_static(b"before")).await); + match case { + "expire" => { + tokio::time::sleep(Duration::from_secs(6)).await; + } + "detach" => { + iam.policy_db_set(&key, UserType::Reg, false, "").await.expect("detach"); + } + "disable" => { + iam.set_user_status(&key, AccountStatus::Disabled).await.expect("disable"); + } + "delete" => { + iam.delete_user(&key, false).await.expect("delete user"); + } + "rotate" => { + iam.rotate_user_secret_key(&key, "replacement-listen-secret") + .await + .expect("rotate"); + } + "recreate" => { + store + .delete_bucket(&bucket, &DeleteBucketOptions::default()) + .await + .expect("delete bucket"); + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("recreate bucket"); + } + _ => unreachable!(), + } + assert!( + !tokio::time::timeout(Duration::from_secs(8), authorization.send(&tx, Bytes::from_static(b"after"))) + .await + .expect("revocation must retire a backpressured stream"), + "{case}" + ); + assert_eq!(rx.recv().await.expect("previous event").expect("bytes"), Bytes::from_static(b"before")); + assert!(rx.try_recv().is_err(), "revoked event leaked: {case}"); + let response = build_listen_notification_response(&uri, bucket_scoped.then_some(bucket.as_str()), authorization) + .expect("stream headers"); + tokio::time::timeout(Duration::from_secs(8), http_body_util::BodyExt::collect(response.output)) + .await + .expect("an idle revoked stream must terminate") + .expect("stream ends cleanly"); + } + }); + } + #[tokio::test] async fn build_listen_notification_response_sets_event_stream_headers() { let uri: Uri = "/demo-bucket?events=s3:ObjectCreated:Put&ping=1" .parse() .expect("uri should parse"); - let resp = build_listen_notification_response(&uri, Some("demo-bucket")).expect("response should build"); + let authorization = ListenNotificationAuthorization { + request: S3Request { + input: (), + method: Method::GET, + uri: uri.clone(), + headers: HeaderMap::new(), + extensions: http::Extensions::new(), + credentials: None, + region: None, + service: None, + trailing_headers: None, + }, + action: Action::S3Action(S3Action::ListenBucketNotificationAction), + bucket_incarnation: None, + keystone_credentials: None, + refresh_at: tokio::time::Instant::now(), + }; + let resp = build_listen_notification_response(&uri, Some("demo-bucket"), authorization).expect("response should build"); assert_eq!( resp.headers.get(header::CONTENT_TYPE).and_then(|v| v.to_str().ok()), diff --git a/rustfs/src/admin/utils.rs b/rustfs/src/admin/utils.rs index 205cbbadf..5c5a37bc8 100644 --- a/rustfs/src/admin/utils.rs +++ b/rustfs/src/admin/utils.rs @@ -19,6 +19,11 @@ use s3s::header::CONTENT_TYPE; use s3s::{Body, S3Error, S3ErrorCode, S3Response, S3Result, s3_error}; use serde::Serialize; use std::collections::HashMap; +use tokio::sync::Semaphore; + +// Each compatible request can run two 64 MiB derivations. Keep both CPU and +// memory bounded without occupying runtime workers or queueing unbounded jobs. +static ADMIN_DECRYPT_PERMITS: Semaphore = Semaphore::const_new(2); /// Returns `true` if `s` contains any whitespace character. /// @@ -45,9 +50,20 @@ pub(crate) async fn read_compatible_admin_body( .map_err(|e| s3_error!(InvalidRequest, "failed to read request body: {}", e))?; if is_compat_admin_request(path) { - decrypt_stream_io(secret_key.as_bytes(), body.as_ref()) - .or_else(|_| decrypt_data(secret_key.as_bytes(), body.as_ref())) - .map_err(|e| s3_error!(InvalidRequest, "failed to decrypt MinIO admin payload: {}", e)) + let permit = ADMIN_DECRYPT_PERMITS + .try_acquire() + .map_err(|_| s3_error!(SlowDown, "Admin payload decryption is busy"))?; + let secret_key = secret_key.to_owned(); + tokio::task::spawn_blocking(move || { + // Cancellation of the request must not release admission while the + // non-cancellable derivation is still running. + let _permit = permit; + decrypt_stream_io(secret_key.as_bytes(), body.as_ref()) + .or_else(|_| decrypt_data(secret_key.as_bytes(), body.as_ref())) + .map_err(|e| s3_error!(InvalidRequest, "failed to decrypt MinIO admin payload: {}", e)) + }) + .await + .map_err(|_| s3_error!(InternalError, "Admin payload decryption failed"))? } else { Ok(body.to_vec()) } @@ -155,6 +171,7 @@ mod tests { } #[tokio::test] + #[serial_test::serial(admin_decrypt)] async fn reads_legacy_compat_payload_as_fallback() { let payload = b"{\"ok\":true}".to_vec(); let encrypted = encrypt_data(b"secret", &payload).expect("encrypt payload"); @@ -166,6 +183,71 @@ mod tests { assert_eq!(decoded, payload); } + #[tokio::test] + #[serial_test::serial(admin_decrypt)] + async fn compatible_admin_decryption_rejects_excess_work_without_blocking_native_requests() { + let permits = ADMIN_DECRYPT_PERMITS + .acquire_many(2) + .await + .expect("reserve decryption capacity"); + let error = read_compatible_admin_body(Body::from(vec![0; 64]), 1024, "/minio/admin/v3/add-user", "secret") + .await + .expect_err("excess KDF work must not queue"); + assert_eq!(error.code(), &S3ErrorCode::SlowDown); + let plain = read_compatible_admin_body(Body::from(String::from("{}")), 1024, "/rustfs/admin/v3/add-user", "secret") + .await + .expect("native admin does not use KDF capacity"); + assert_eq!(plain, b"{}"); + drop(permits); + let encrypted = encrypt_stream_io(b"secret", b"{}").expect("encrypt compatible request"); + assert_eq!( + read_compatible_admin_body(Body::from(encrypted), 1024, "/minio/admin/v3/add-user", "secret") + .await + .expect("capacity returns after work finishes"), + b"{}" + ); + } + + #[test] + #[serial_test::serial(admin_decrypt)] + fn cancelled_admin_decryption_keeps_capacity_until_blocking_work_finishes() { + let runtime = tokio::runtime::Builder::new_current_thread() + .max_blocking_threads(1) + .enable_all() + .build() + .expect("build single-worker runtime"); + runtime.block_on(async { + let (release, wait) = std::sync::mpsc::channel(); + let blocker = tokio::task::spawn_blocking(move || wait.recv().expect("release blocker")); + let request = tokio::spawn(read_compatible_admin_body( + Body::from(vec![0; 64]), + 1024, + "/minio/admin/v3/add-user", + "secret", + )); + tokio::task::yield_now().await; + assert_eq!( + ADMIN_DECRYPT_PERMITS.available_permits(), + 1, + "decryption was queued off the runtime worker" + ); + request.abort(); + assert!(request.await.expect_err("request cancelled").is_cancelled()); + assert_eq!( + ADMIN_DECRYPT_PERMITS.available_permits(), + 1, + "cancellation must not release queued work admission" + ); + release.send(()).expect("unblock worker"); + blocker.await.expect("blocking task finished"); + let all = ADMIN_DECRYPT_PERMITS + .acquire_many(2) + .await + .expect("decryption finished and released capacity"); + drop(all); + }); + } + async fn body_bytes(mut body: Body) -> Vec { body.store_all_limited(64 * 1024).await.expect("body should read").to_vec() } diff --git a/rustfs/src/app/bucket_usecase.rs b/rustfs/src/app/bucket_usecase.rs index a2248e4de..207892271 100644 --- a/rustfs/src/app/bucket_usecase.rs +++ b/rustfs/src/app/bucket_usecase.rs @@ -54,7 +54,7 @@ use super::storage_api::bucket_usecase::contract::list::{ }; use super::storage_api::bucket_usecase::error::StorageError; use super::storage_api::bucket_usecase::helper::{OperationHelper, spawn_background_with_context}; -use super::storage_api::bucket_usecase::object_utils::to_s3s_etag; +use super::storage_api::bucket_usecase::object_utils::to_s3s_xml_etag; use super::storage_api::bucket_usecase::s3_api::bucket::{ ListObjectVersionsParams, ListObjectsV2Params, build_list_buckets_output, build_list_object_versions_output, build_list_objects_output, build_list_objects_v2_output, encode_list_objects_v2_continuation_token, @@ -966,6 +966,12 @@ fn encode_list_objects_v2_value(value: &str, encoding_type: Option<&EncodingType } fn build_metadata_extension_user_metadata(user_defined: &HashMap) -> Option { + // Even builds unable to decrypt MinIO SSE-C must not expose its key-protected metadata. + if super::storage_api::bucket_usecase::sse::stored_ssec_metadata(user_defined) + || user_defined.contains_key(rustfs_utils::http::object_encryption_keys::MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER) + { + return None; + } let mut items = extract_user_defined_metadata(user_defined) .into_iter() .filter(|(key, _)| !key.is_empty()) @@ -1125,7 +1131,7 @@ fn build_list_object_versions_metadata_output( .etag .as_ref() .filter(|_| rustfs_utils::http::etag_can_be_listed(&object.user_defined)) - .map(|etag| to_s3s_etag(etag)), + .map(|etag| to_s3s_xml_etag(etag)), storage_class: Some(ObjectVersionStorageClass::from( object .storage_class @@ -1216,7 +1222,7 @@ fn build_list_objects_v2_metadata_output( .etag .as_ref() .filter(|_| rustfs_utils::http::etag_can_be_listed(&object.user_defined)) - .map(|etag| to_s3s_etag(etag)), + .map(|etag| to_s3s_xml_etag(etag)), storage_class: Some(ObjectStorageClass::from( object .storage_class @@ -1732,7 +1738,7 @@ impl DefaultBucketUsecase { Ok(S3Response::new(GetBucketLocationOutput { location_constraint })) } - #[instrument(level = "debug", skip(self))] + #[instrument(level = "debug", skip(self, req))] pub async fn execute_list_buckets(&self, req: S3Request) -> S3Result> { let Some(store) = self.object_store() else { return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); @@ -1799,7 +1805,7 @@ impl DefaultBucketUsecase { Ok(S3Response::with_status(DeleteBucketEncryptionOutput::default(), StatusCode::NO_CONTENT)) } - #[instrument(level = "debug", skip(self))] + #[instrument(level = "debug", skip(self, req))] pub async fn execute_delete_bucket_cors( &self, req: S3Request, @@ -1831,7 +1837,7 @@ impl DefaultBucketUsecase { Ok(S3Response::new(DeleteBucketCorsOutput {})) } - #[instrument(level = "debug", skip(self))] + #[instrument(level = "debug", skip(self, req))] pub async fn execute_delete_bucket_lifecycle( &self, req: S3Request, @@ -1976,7 +1982,7 @@ impl DefaultBucketUsecase { Ok(S3Response::new(DeleteBucketReplicationOutput::default())) } - #[instrument(level = "debug", skip(self))] + #[instrument(level = "debug", skip(self, req))] pub async fn execute_delete_bucket_tagging( &self, req: S3Request, @@ -2003,7 +2009,7 @@ impl DefaultBucketUsecase { Ok(S3Response::new(DeleteBucketTaggingOutput {})) } - #[instrument(level = "debug", skip(self))] + #[instrument(level = "debug", skip(self, req))] pub async fn execute_delete_public_access_block( &self, req: S3Request, @@ -2068,7 +2074,7 @@ impl DefaultBucketUsecase { })) } - #[instrument(level = "debug", skip(self))] + #[instrument(level = "debug", skip(self, req))] pub async fn execute_get_bucket_cors(&self, req: S3Request) -> S3Result> { let GetBucketCorsInput { bucket, .. } = req.input; @@ -2107,7 +2113,7 @@ impl DefaultBucketUsecase { })) } - #[instrument(level = "debug", skip(self))] + #[instrument(level = "debug", skip(self, req))] pub async fn execute_get_bucket_lifecycle_configuration( &self, req: S3Request, @@ -2348,7 +2354,7 @@ impl DefaultBucketUsecase { })) } - #[instrument(level = "debug", skip(self))] + #[instrument(level = "debug", skip(self, req))] pub async fn execute_get_bucket_tagging( &self, req: S3Request, @@ -2385,7 +2391,7 @@ impl DefaultBucketUsecase { Ok(S3Response::new(GetBucketTaggingOutput { tag_set })) } - #[instrument(level = "debug", skip(self))] + #[instrument(level = "debug", skip(self, req))] pub async fn execute_get_public_access_block( &self, req: S3Request, @@ -2419,7 +2425,7 @@ impl DefaultBucketUsecase { })) } - #[instrument(level = "debug", skip(self))] + #[instrument(level = "debug", skip(self, req))] pub async fn execute_get_bucket_versioning( &self, req: S3Request, @@ -2504,7 +2510,7 @@ impl DefaultBucketUsecase { Ok(S3Response::new(PutBucketEncryptionOutput::default())) } - #[instrument(level = "debug", skip(self))] + #[instrument(level = "debug", skip(self, req))] pub async fn execute_put_bucket_lifecycle_configuration( &self, req: S3Request, @@ -2752,7 +2758,7 @@ impl DefaultBucketUsecase { Ok(S3Response::new(PutBucketPolicyOutput {})) } - #[instrument(level = "debug", skip(self))] + #[instrument(level = "debug", skip(self, req))] pub async fn execute_put_bucket_cors(&self, req: S3Request) -> S3Result> { let expected_incarnation_id = bucket_config_mutation_incarnation(&req, &req.input.bucket)?; let request_context = req.extensions.get::().cloned(); @@ -2844,7 +2850,7 @@ impl DefaultBucketUsecase { Ok(S3Response::new(PutBucketReplicationOutput::default())) } - #[instrument(level = "debug", skip(self))] + #[instrument(level = "debug", skip(self, req))] pub async fn execute_put_public_access_block( &self, req: S3Request, @@ -2876,7 +2882,7 @@ impl DefaultBucketUsecase { Ok(S3Response::new(PutPublicAccessBlockOutput::default())) } - #[instrument(level = "debug", skip(self))] + #[instrument(level = "debug", skip(self, req))] pub async fn execute_put_bucket_tagging( &self, req: S3Request, @@ -2884,6 +2890,7 @@ impl DefaultBucketUsecase { let expected_incarnation_id = bucket_config_mutation_incarnation(&req, &req.input.bucket)?; let request_context = req.extensions.get::().cloned(); let PutBucketTaggingInput { bucket, tagging, .. } = req.input; + super::storage_api::bucket_usecase::s3_api::tagging::validate_bucket_tag_set(&tagging.tag_set)?; let Some(store) = self.object_store() else { return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); @@ -2915,7 +2922,7 @@ impl DefaultBucketUsecase { Ok(S3Response::new(PutBucketTaggingOutput::default())) } - #[instrument(level = "debug", skip(self))] + #[instrument(level = "debug", skip(self, req))] pub async fn execute_put_bucket_versioning( &self, req: S3Request, @@ -5148,7 +5155,10 @@ mod tests { assert!(listings.contents[0].object.e_tag.is_none()); assert!(listings.contents[1].object.e_tag.is_some()); assert!(listings.contents[2].object.e_tag.is_some()); - assert_eq!(listings.contents[3].object.e_tag, Some(to_s3s_etag("1b8a77dff5845270a8d81d86a28f38ea-2"))); + assert_eq!( + listings.contents[3].object.e_tag, + Some(to_s3s_xml_etag("1b8a77dff5845270a8d81d86a28f38ea-2")) + ); let xml = String::from_utf8(serialize_config(&listings).expect("serialize objects")).expect("UTF-8 objects"); assert!(!xml.contains(legacy_md5)); @@ -5178,6 +5188,159 @@ mod tests { assert!(!xml.contains(legacy_md5)); } + #[test] + #[serial_test::serial] + fn put_bucket_tagging_rejects_invalid_shapes_before_storage_mutation() { + crate::app::gating_test_env::run_large_stack_test("bucket-tag-validation", || async { + use crate::app::runtime_sources::{AppContext, ServerContextSlot}; + use crate::app::storage_api::test::ReqInfo; + use crate::app::storage_api::test::ecfs::FS; + use s3s::access::S3Access as _; + let ambient = crate::app::gating_test_env::shared_gating_ambient().await; + let store = crate::app::gating_test_env::shared_gating_ecstore().await; + let context = Arc::new(AppContext::new(store.clone(), ambient.iam(), ambient.kms())); + let slot = ServerContextSlot::new(); + assert!(slot.install(context.clone())); + let fs = FS::with_server_ctx(slot.clone()); + let usecase = DefaultBucketUsecase::with_context(Some(context)); + let bucket = format!("tag-validation-{}", uuid::Uuid::new_v4().simple()); + store.make_bucket(&bucket, &MakeBucketOptions::default()).await.unwrap(); + store + .update_bucket_metadata_config( + &bucket, + BUCKET_POLICY_CONFIG, + serde_json::to_vec( + &serde_json::json!({"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":"*"}, + "Action":"s3:PutBucketTagging","Resource":format!("arn:aws:s3:::{bucket}")}]}), + ) + .unwrap(), + ) + .await + .unwrap(); + let tag = s3s::dto::Tag { + key: Some("team".to_owned()), + value: Some("storage".to_owned()), + }; + for (tag_set, valid) in [ + (vec![tag.clone()], true), + ( + vec![s3s::dto::Tag { + key: Some("team".to_owned()), + value: None, + }], + false, + ), + (vec![tag.clone(), tag], false), + ] { + let input = PutBucketTaggingInput::builder() + .bucket(bucket.clone()) + .tagging(s3s::dto::Tagging { tag_set }) + .build() + .unwrap(); + let mut req = build_request(input, Method::PUT); + req.extensions.insert(ReqInfo::default()); + req.extensions.insert(slot.clone()); + fs.put_bucket_tagging(&mut req).await.expect("permission allows tag updates"); + let result = usecase.execute_put_bucket_tagging(req).await; + if valid { + result.expect("valid tags are persisted"); + } else { + assert_eq!(result.unwrap_err().code(), &S3ErrorCode::InvalidTag); + } + assert_eq!( + store.get_bucket_tags_for_policy(&bucket).await.unwrap(), + HashMap::from([("team".to_owned(), "storage".to_owned())]) + ); + } + }); + } + + #[test] + fn metadata_listings_hide_ssec_user_metadata_even_with_read_permission() { + let objects = [ + ("customer-key", Some(("x-amz-server-side-encryption-customer-algorithm", "AES256"))), + ( + "minio-customer-key", + Some(( + rustfs_utils::http::object_encryption_keys::MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER, + "sealed-key", + )), + ), + ("server-key", Some(("x-amz-server-side-encryption", "AES256"))), + ("plain", None), + ] + .into_iter() + .map(|(name, encryption)| { + let mut metadata = HashMap::from([("project".to_owned(), "private-project".to_owned())]); + if let Some((key, value)) = encryption { + metadata.insert(key.to_owned(), value.to_owned()); + } + ObjectInfo { + name: name.to_owned(), + user_defined: Arc::new(metadata), + ..Default::default() + } + }) + .collect::>(); + let permissions = objects + .iter() + .map(|object| { + ( + object.name.clone(), + ObjectMetadataPermissions { + metadata_allowed: true, + tags_allowed: true, + }, + ) + }) + .collect::>(); + let listings = build_list_objects_v2_metadata_output( + ListObjectsV2Info { + objects: objects.clone(), + ..Default::default() + }, + "demo-bucket", + &ListObjectsV2Params { + prefix: String::new(), + max_keys: 1000, + delimiter: None, + response_start_after: None, + start_after_for_query: None, + response_continuation_token: None, + decoded_continuation_token: None, + }, + None, + false, + &permissions, + ); + assert!(listings.contents[0].extension.user_metadata.is_none()); + assert!(listings.contents[1].extension.user_metadata.is_none()); + assert!(listings.contents[2].extension.user_metadata.is_some()); + assert!(listings.contents[3].extension.user_metadata.is_some()); + let versions = build_list_object_versions_metadata_output( + ListObjectVersionsInfo { + objects, + ..Default::default() + }, + "demo-bucket", + &ListObjectVersionsParams { + prefix: String::new(), + max_keys: 1000, + delimiter: None, + key_marker: None, + version_id_marker: None, + }, + None, + &permissions, + ); + for (index, entry) in versions.entries.iter().enumerate() { + let ListObjectVersionMetadataEntry::Version(_, extension) = entry else { + panic!("expected version"); + }; + assert_eq!(extension.user_metadata.is_some(), index >= 2); + } + } + #[test] fn build_list_object_versions_metadata_output_maps_metadata_and_preserves_entry_order() { use time::macros::datetime; diff --git a/rustfs/src/app/gating_test_env.rs b/rustfs/src/app/gating_test_env.rs index d6adec4cd..028b03b07 100644 --- a/rustfs/src/app/gating_test_env.rs +++ b/rustfs/src/app/gating_test_env.rs @@ -158,6 +158,9 @@ pub(crate) async fn durable_quota_test_bucket(prefix: &str, limit: u64) -> (Arc< .set_quota_config(&bucket, BucketQuota::new(Some(limit))) .await .expect("configure durable quota test bucket"); + super::storage_api::test::set_disk::initialize_quota_ledger_for_test(Arc::clone(&store), &bucket) + .await + .expect("bootstrap authoritative quota usage"); (store, bucket) } diff --git a/rustfs/src/app/metadata_route.rs b/rustfs/src/app/metadata_route.rs index d52e1ab0a..f15b8cd1f 100644 --- a/rustfs/src/app/metadata_route.rs +++ b/rustfs/src/app/metadata_route.rs @@ -18,13 +18,13 @@ use super::bucket_usecase::DefaultBucketUsecase; use super::object_usecase::DefaultObjectUsecase; use crate::app::runtime_sources::ServerContextSlot; use crate::auth::{check_key_valid, get_session_token}; -use crate::storage::access::{ReqInfo, authorize_request, req_info_mut}; +use crate::storage::access::{ReqInfo, authorize_bucket_listing, req_info_mut}; use crate::storage::{get_bucket_website_config_for_store, validate_website_configuration}; use async_trait::async_trait; use http::header::CONTENT_TYPE; use http::header::HOST; use http::{Extensions, HeaderMap, HeaderValue, Method, StatusCode, Uri}; -use rustfs_policy::policy::action::{Action, S3Action}; +use rustfs_policy::policy::action::S3Action; use s3s::access::S3Access; use s3s::dto::{ ETagCondition, EncodingType, GetObjectInput, GetObjectOutput, HeadObjectInput, ListObjectVersionsInput, ListObjectsV2Input, @@ -674,14 +674,14 @@ async fn check_metadata_access(req: &mut S3Request, target: BucketTarget) let req_info = req_info_mut(req)?; req_info.cred = cred; req_info.is_owner = is_owner; - req_info.bucket = Some(target.bucket); + req_info.bucket = Some(target.bucket.clone()); } let action = match target.operation { - MetadataOperation::ListObjectVersions => Action::S3Action(S3Action::ListBucketVersionsAction), - MetadataOperation::ListObjectsV2 => Action::S3Action(S3Action::ListBucketAction), + MetadataOperation::ListObjectVersions => S3Action::ListBucketVersionsAction, + MetadataOperation::ListObjectsV2 => S3Action::ListBucketAction, }; - authorize_request(req, action).await + authorize_bucket_listing(req, &target.bucket, action).await } async fn call_list_object_versions(req: S3Request, bucket: String) -> S3Result> { @@ -842,6 +842,197 @@ fn header_value(headers: &HeaderMap, name: &str) -> S3Result> { #[cfg(test)] mod tests { + #[tokio::test] + async fn metadata_route_prepares_and_applies_table_listing_isolation() { + use crate::app::runtime_sources::{AppContext, IamInterface, KmsInterface, ServerContextSlot}; + use crate::app::storage_api::bucket_usecase::contract::bucket::{BucketOperations as _, MakeBucketOptions}; + use crate::app::storage_api::object_usecase::access::TableDataPlaneListAccess; + use crate::app::storage_api::object_usecase::contract::object::ObjectIO as _; + use crate::app::storage_api::object_usecase::{StorageObjectOptions, StoragePutObjReader}; + use crate::app::table_list_isolation; + use crate::table_catalog::{ + ConfiguredTableCatalogStore, EcStoreTableCatalogObjectBackend, NamespaceEntry, TABLE_BUCKET_CATALOG_TYPE, + TABLE_BUCKET_MARKER_CONFIG, TABLE_CATALOG_ENTRY_VERSION, TableBucketEntry, TableCatalogBackingMode, + TableCatalogEntryState, TableCatalogStore, TableEntry, + }; + use std::{collections::BTreeMap, sync::Arc}; + + struct UnreadyIam; + impl IamInterface for UnreadyIam { + fn handle(&self) -> Arc> { + panic!("anonymous listing must not resolve IAM credentials") + } + fn is_ready(&self) -> bool { + false + } + } + struct TestKms; + impl KmsInterface for TestKms { + fn handle(&self) -> Arc { + Arc::new(rustfs_kms::KmsServiceManager::new()) + } + } + + let store = crate::app::gating_test_env::shared_gating_ecstore().await; + let bucket = format!("metadata-isolation-{}", uuid::Uuid::new_v4()); + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create table bucket"); + store + .update_bucket_metadata_config( + &bucket, + TABLE_BUCKET_MARKER_CONFIG, + crate::table_catalog::table_bucket_marker_json().expect("table bucket marker"), + ) + .await + .expect("enable table bucket"); + let policy = serde_json::json!({"Version": "2012-10-17", "Statement": [{ + "Effect": "Allow", "Principal": {"AWS": "*"}, + "Action": ["s3:ListBucket", "s3:ListBucketVersions"], "Resource": format!("arn:aws:s3:::{bucket}") + }]}); + store + .update_bucket_metadata_config( + &bucket, + crate::storage::storage_api::ecstore_bucket::metadata::BUCKET_POLICY_CONFIG, + serde_json::to_vec(&policy).expect("listing policy"), + ) + .await + .expect("allow anonymous object listings"); + let context = Arc::new(AppContext::new(Arc::clone(&store), Arc::new(UnreadyIam), Arc::new(TestKms))); + assert!(context.publish_action_credentials(rustfs_credentials::Credentials { + access_key: "table-list-root".to_owned(), + secret_key: "table-list-root-secret".to_owned(), + status: "on".to_owned(), + ..Default::default() + })); + let backend = + EcStoreTableCatalogObjectBackend::new_with_strong_runtime(Arc::clone(&store), context.table_catalog_strong_runtime()); + let catalog = + ConfiguredTableCatalogStore::new_for_test(backend, TableCatalogBackingMode::from_env().expect("catalog mode")); + catalog + .put_table_bucket(TableBucketEntry { + version: TABLE_CATALOG_ENTRY_VERSION, + table_bucket: bucket.clone(), + catalog_type: TABLE_BUCKET_CATALOG_TYPE.to_owned(), + warehouse_root: format!("s3://{bucket}/"), + state: TableCatalogEntryState::Active, + properties: BTreeMap::new(), + active_rename_id: None, + created_at: None, + updated_at: None, + }) + .await + .expect("seed catalog bucket"); + catalog + .create_namespace(NamespaceEntry { + version: TABLE_CATALOG_ENTRY_VERSION, + table_bucket: bucket.clone(), + namespace: "sales".to_owned(), + namespace_id: "sales".to_owned(), + state: TableCatalogEntryState::Active, + properties: BTreeMap::new(), + created_at: None, + updated_at: None, + }) + .await + .expect("seed catalog namespace"); + catalog + .create_table(TableEntry { + version: TABLE_CATALOG_ENTRY_VERSION, + table_bucket: bucket.clone(), + namespace: "sales".to_owned(), + table: "orders".to_owned(), + table_id: "table-id".to_owned(), + table_uuid: uuid::Uuid::new_v4().to_string(), + format: "ICEBERG".to_owned(), + format_version: 2, + warehouse_location: format!("s3://{bucket}/tables/table-id"), + metadata_location: format!("s3://{bucket}/tables/table-id/metadata.json"), + version_token: "v1".to_owned(), + generation: 1, + state: TableCatalogEntryState::Active, + properties: BTreeMap::new(), + created_at: None, + updated_at: None, + }) + .await + .expect("seed catalog table"); + for key in ["ordinary.txt", "tables/table-id/data.parquet", ".rustfs-table/private.json"] { + store + .put_object( + &bucket, + key, + &mut StoragePutObjReader::from_vec(b"private metadata".to_vec()), + &StorageObjectOptions::default(), + ) + .await + .expect("seed objects for listing"); + } + let slot = ServerContextSlot::new(); + assert!(slot.install(context)); + for operation in [MetadataOperation::ListObjectsV2, MetadataOperation::ListObjectVersions] { + let mut req = website_request(&format!("/{bucket}?list-type=2&metadata=true")); + req.extensions.insert(Arc::clone(&slot)); + super::check_metadata_access( + &mut req, + super::BucketTarget { + operation, + bucket: bucket.clone(), + }, + ) + .await + .expect("bucket listing is authorized"); + let access = req + .extensions + .get::() + .expect("metadata-route authorization must attach table isolation"); + assert!(access.allows_object("ordinary.txt")); + assert!(!access.allows_object("tables/table-id/data.parquet")); + assert!(!access.allows_object(".rustfs-table/private.json")); + let objects = match operation { + MetadataOperation::ListObjectsV2 => { + table_list_isolation::list_objects_v2( + Arc::clone(&store), + access, + table_list_isolation::ListObjectsV2Request { + bucket: &bucket, + prefix: "", + continuation_token: None, + delimiter: None, + max_keys: 1000, + start_after: None, + incl_deleted: false, + opaque_cursor_supported: true, + }, + ) + .await + .expect("protected object listing succeeds") + .objects + } + MetadataOperation::ListObjectVersions => { + table_list_isolation::list_object_versions( + Arc::clone(&store), + access, + table_list_isolation::ListObjectVersionsRequest { + bucket: &bucket, + prefix: "", + key_marker: None, + version_id_marker: None, + delimiter: None, + max_keys: 1000, + }, + ) + .await + .expect("protected version listing succeeds") + .objects + } + }; + let keys: Vec<_> = objects.into_iter().map(|object| object.name).collect(); + assert_eq!(keys, vec!["ordinary.txt"], "catalog and unauthorized warehouse objects must be hidden"); + } + } + #[tokio::test] async fn website_304_preserves_cache_headers_without_content() { use http_body_util::BodyExt; diff --git a/rustfs/src/app/multipart_usecase.rs b/rustfs/src/app/multipart_usecase.rs index b6be3fb7a..5b5660045 100644 --- a/rustfs/src/app/multipart_usecase.rs +++ b/rustfs/src/app/multipart_usecase.rs @@ -44,11 +44,11 @@ use super::storage_api::multipart_usecase::data_usage::{ use super::storage_api::multipart_usecase::error::{ StorageError, is_err_invalid_upload_id, is_err_object_not_found, is_err_version_not_found, }; -use super::storage_api::multipart_usecase::helper::OperationHelper; +use super::storage_api::multipart_usecase::helper::{OperationHelper, run_owned_mutation}; #[cfg(test)] use super::storage_api::multipart_usecase::io::{DecryptReader, EncryptReader, HardLimitReader, boxed_reader, wrap_reader}; use super::storage_api::multipart_usecase::io::{HashReader, WriteEncryption, WritePlan, compression_metadata_value}; -use super::storage_api::multipart_usecase::object_utils::to_s3s_etag; +use super::storage_api::multipart_usecase::object_utils::{to_s3s_etag, to_s3s_xml_etag}; use super::storage_api::multipart_usecase::options::{ copy_src_opts, extract_metadata_from_mime, get_complete_multipart_upload_opts_with_replication_authorization, get_content_sha256_with_query, get_opts, has_replication_retention_update, namespace_reserved_user_metadata, @@ -62,20 +62,20 @@ use super::storage_api::multipart_usecase::s3_api::multipart::{ use super::storage_api::multipart_usecase::set_disk::is_valid_storage_class; use super::storage_api::multipart_usecase::sse::{ DecryptionRequest, EncryptionKeyKind, EncryptionRequest, PrepareEncryptionRequest, SseKmsPrincipal, - apply_bucket_default_lock_retention, authorize_sse_kms_object_read, build_ssec_read_headers, encryption_material_to_metadata, - extract_server_side_encryption_from_headers, extract_ssec_params_from_headers, extract_ssekms_context_from_headers, - get_buffer_size_opt_in, load_bucket_object_lock_config_state, map_get_object_reader_error, mark_encrypted_multipart_metadata, - sse_decryption, sse_prepare_encryption, + apply_bucket_default_lock_retention, authorize_sse_kms_multipart_write, authorize_sse_kms_object_read, + build_ssec_read_headers, encryption_material_to_metadata, extract_server_side_encryption_from_headers, + extract_ssec_params_from_headers, extract_ssekms_context_from_headers, get_buffer_size_opt_in, + load_bucket_object_lock_config_state, map_get_object_reader_error, mark_encrypted_multipart_metadata, + multipart_completion_authorization, sse_decryption, sse_prepare_encryption, }; use super::storage_api::multipart_usecase::{ StorageObjectInfo as ObjectInfo, StorageObjectOptions as ObjectOptions, StoragePutObjReader as PutObjReader, - validate_object_key, + validate_new_object_key, }; use super::trailer_adapter::trailer_source; use crate::app::object::{ - ConcurrencyManager, ForegroundWriteAdmission, get_concurrency_manager, guard_put_object_body_read_timeout, - put_object_body_read_timeout, reject_non_wildcard_write_if_none_match, reject_oversize_single_upload, - reject_veeam_sosapi_mutation, + ConcurrencyManager, ForegroundWriteAdmission, get_concurrency_manager, put_object_body_read_timeout, + reject_non_wildcard_write_if_none_match, reject_oversize_single_upload, reject_veeam_sosapi_mutation, }; use crate::app::object_data_cache::{ ObjectDataCacheAdapter, invalidate_object_data_cache_after_complete_multipart_success, @@ -523,7 +523,7 @@ impl DefaultMultipartUsecase { get_concurrency_manager() } - #[instrument(level = "debug", skip(self))] + #[instrument(level = "debug", skip(self, req))] pub async fn execute_abort_multipart_upload( &self, req: S3Request, @@ -582,11 +582,18 @@ impl DefaultMultipartUsecase { #[instrument(level = "debug", skip(self, req))] #[hotpath::measure(impl_type = "MultipartUsecase")] pub async fn execute_complete_multipart_upload( + &self, + req: S3Request, + ) -> S3Result> { + let owner = self.clone(); + run_owned_mutation(async move { Box::pin(owner.execute_complete_multipart_upload_owned(req)).await }).await + } + + async fn execute_complete_multipart_upload_owned( &self, mut req: S3Request, ) -> S3Result> { - // Uploads created before the key was reserved must not publish it. - validate_object_key(&req.input.key, "CompleteMultipartUpload")?; + validate_new_object_key(&req.input.key, "CompleteMultipartUpload")?; reject_presigned_multipart_max_total_object_size_for_other_operation( &req.headers, req.uri.query(), @@ -617,6 +624,7 @@ impl DefaultMultipartUsecase { .. } = input; + let customer_key_provided = sse_customer_key.is_some(); reject_veeam_sosapi_mutation(|| self.object_store(), &bucket, &key).await?; reject_non_wildcard_write_if_none_match(if_none_match.as_ref(), &req.headers)?; validate_table_catalog_object_mutation(&bucket, &key).await?; @@ -702,25 +710,28 @@ impl DefaultMultipartUsecase { Err(err) if is_err_invalid_upload_id(&err) => None, Err(err) => return Err(ApiError::from(err).into()), }; - // A ciphertext-passthrough session stores encrypted parts verbatim and - // completes without the customer key (the replication client has none), - // so the SSE-C completion check must be skipped for it. + let completion_principal = SseKmsPrincipal::from_request(&req); + let completion_authorization = multipart_completion_authorization( + EncryptionRequest { + bucket: &bucket, + key: &key, + server_side_encryption: None, + ssekms_key_id: None, + ssekms_context: None, + sse_customer_algorithm: sse_customer_algorithm.clone(), + sse_customer_key: sse_customer_key.clone(), + sse_customer_key_md5: sse_customer_key_md5.clone(), + content_size: 0, + principal: completion_principal.as_ref(), + }, + replication_authorized, + ); + opts.multipart_completion_authorization = Some(completion_authorization.clone()); if let Some(info) = multipart_info.as_ref() { - if !contains_key_str(&info.user_defined, SUFFIX_REPLICATION_PRESERVE_CIPHERTEXT) { - EncryptionRequest { - bucket: &bucket, - key: &key, - server_side_encryption: None, - ssekms_key_id: None, - ssekms_context: None, - sse_customer_algorithm: sse_customer_algorithm.clone(), - sse_customer_key: sse_customer_key.clone(), - sse_customer_key_md5: sse_customer_key_md5.clone(), - content_size: 0, - principal: None, - } - .validate_complete_multipart_ssec(&info.user_defined)?; - } + completion_authorization + .authorize(&info.user_defined) + .await + .map_err(map_get_object_reader_error)?; validate_complete_multipart_checksum_type(&req.headers, &info.user_defined)?; } @@ -827,7 +838,7 @@ impl DefaultMultipartUsecase { .clone() .complete_multipart_upload(&bucket, &key, &upload_id, uploaded_parts, &opts) .await - .map_err(ApiError::from)?; + .map_err(map_get_object_reader_error)?; let _ = invalidate_object_data_cache_after_complete_multipart_success(&cache_adapter, &bucket, &key).await; // Staging reclaim on a replay can free bytes; the dirty mark is // not a usage counter. Quota, replication, lifecycle, and the @@ -880,24 +891,11 @@ impl DefaultMultipartUsecase { ) })??; - // The upload record is gone on a lost-response retry, so SSE-C is - // checked against the committed object instead of the staging metadata. - if upload_user_defined.is_none() && !contains_key_str(&obj_info.user_defined, SUFFIX_REPLICATION_PRESERVE_CIPHERTEXT) { - EncryptionRequest { - bucket: &bucket, - key: &key, - server_side_encryption: None, - ssekms_key_id: None, - ssekms_context: None, - sse_customer_algorithm, - sse_customer_key, - sse_customer_key_md5, - content_size: 0, - principal: None, - } - .validate_complete_multipart_ssec(obj_info.user_defined.as_ref())?; - } + #[cfg(test)] + super::storage_api::multipart_usecase::helper::pause_committed_mutation_for_test(&bucket).await; + // Storage has authorized the actual source or committed replay candidate + // before comparing checksums/parts, recovery publication or replay effects. let encryption_metadata = upload_user_defined.as_ref().unwrap_or(obj_info.user_defined.as_ref()); let server_side_encryption = encryption_metadata .get("x-amz-server-side-encryption") @@ -925,24 +923,55 @@ impl DefaultMultipartUsecase { // checksum: stored (decrypted) values take precedence over the request input; // additional algorithms (XXHash3/64/128, SHA-512, MD5), which have no typed // CompleteMultipartUploadOutput field, are echoed as raw response headers (#1261). - let (checksums, is_multipart) = obj_info - .decrypt_checksums(opts.part_number.unwrap_or(0), &req.headers) - .map_err(ApiError::from)?; - - let classified = crate::app::object_usecase::classify_response_checksums(checksums, is_multipart); - let checksum_crc32 = classified.crc32.or(input.checksum_crc32); - let checksum_crc32c = classified.crc32c.or(input.checksum_crc32c); - let checksum_sha1 = classified.sha1.or(input.checksum_sha1); - let checksum_sha256 = classified.sha256.or(input.checksum_sha256); - let checksum_crc64nvme = classified.crc64nvme.or(input.checksum_crc64nvme); - let checksum_type = classified.checksum_type.or(input.checksum_type); + // Keyless completion remains supported, but plaintext checksums and the + // customer-key fingerprint are returned only after verifying that key. + // Apply this equally to a new commit and a lost-response replay. + let disclose_checksums = (ssec_algorithm.is_none() && ssec_key_md5.is_none()) || customer_key_provided; + let classified = if disclose_checksums { + let (checksums, is_multipart) = obj_info + .decrypt_checksums(opts.part_number.unwrap_or(0), &req.headers) + .map_err(ApiError::from)?; + crate::app::object_usecase::classify_response_checksums(checksums, is_multipart) + } else { + Default::default() + }; + let checksum_crc32 = if disclose_checksums { + classified.crc32.or(input.checksum_crc32) + } else { + None + }; + let checksum_crc32c = if disclose_checksums { + classified.crc32c.or(input.checksum_crc32c) + } else { + None + }; + let checksum_sha1 = if disclose_checksums { + classified.sha1.or(input.checksum_sha1) + } else { + None + }; + let checksum_sha256 = if disclose_checksums { + classified.sha256.or(input.checksum_sha256) + } else { + None + }; + let checksum_crc64nvme = if disclose_checksums { + classified.crc64nvme.or(input.checksum_crc64nvme) + } else { + None + }; + let checksum_type = if disclose_checksums { + classified.checksum_type.or(input.checksum_type) + } else { + None + }; let complete_extra_checksum_headers = classified.extra; let location = build_complete_multipart_location(&req.headers, &req.uri, &bucket, &key); let output = CompleteMultipartUploadOutput { bucket: Some(bucket.clone()), key: Some(key.clone()), - e_tag: obj_info.etag.clone().map(|etag| to_s3s_etag(&etag)), + e_tag: obj_info.etag.clone().map(|etag| to_s3s_xml_etag(&etag)), location: Some(location), server_side_encryption, ssekms_key_id, @@ -970,7 +999,7 @@ impl DefaultMultipartUsecase { .headers .insert("x-amz-server-side-encryption-customer-algorithm", value); } - if let Some(key_md5) = ssec_key_md5.as_deref() { + if disclose_checksums && let Some(key_md5) = ssec_key_md5.as_deref() { let value = HeaderValue::from_str(key_md5).map_err(|_| s3_error!(InternalError, "Invalid stored SSE-C key metadata"))?; response @@ -993,12 +1022,13 @@ impl DefaultMultipartUsecase { &self, req: S3Request, ) -> S3Result> { - validate_object_key(&req.input.key, "CreateMultipartUpload")?; + validate_new_object_key(&req.input.key, "CreateMultipartUpload")?; let multipart_max_total_object_size = parse_presigned_multipart_max_total_object_size( &req.headers, req.uri.query(), req.extensions.get::().is_some(), )?; + validate_new_object_key(&req.input.key, "CreateMultipartUpload")?; reject_presigned_put_max_content_length_for_other_operation( &req.headers, req.uri.query(), @@ -1085,7 +1115,7 @@ impl DefaultMultipartUsecase { let sse_customer_key_md5 = sse_customer_key_md5.or(header_sse_customer_key_md5); // The session data key is generated here, so this is where a multipart upload is held - // to the KMS key it names. Parts and the completion reuse the resulting envelope. + // to the KMS key it names. Parts and completion also authorize their own caller. let session_principal = SseKmsPrincipal::from_request(&req); let encryption_request = PrepareEncryptionRequest { bucket: &bucket, @@ -1252,6 +1282,7 @@ impl DefaultMultipartUsecase { normalize_presigned_part_checksums(&mut req)?; let mut opts = ObjectOptions::default(); apply_bucket_generation_guard(&req, &req.input.bucket, &mut opts)?; + let part_principal = SseKmsPrincipal::from_request(&req); let input = req.input; let UploadPartInput { body, @@ -1276,7 +1307,7 @@ impl DefaultMultipartUsecase { if let Some(size) = size { reject_oversize_single_upload(size)?; } - let mut body_stream = body.ok_or_else(|| s3_error!(IncompleteBody))?; + let body_stream = body.ok_or_else(|| s3_error!(IncompleteBody))?; let Some(store) = self.object_store() else { return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); }; @@ -1284,6 +1315,7 @@ impl DefaultMultipartUsecase { .get_multipart_info(&bucket, &key, &upload_id, &opts) .await .map_err(ApiError::from)?; + authorize_sse_kms_multipart_write(part_principal.as_ref(), &fi.user_defined).await?; let max_total_object_size = multipart_max_total_object_size(&fi.user_defined)?; let mut size = require_upload_part_size(size, max_total_object_size.is_some())?; if let Some(limit) = max_total_object_size @@ -1316,23 +1348,20 @@ impl DefaultMultipartUsecase { .map(|ctx| ctx.request_id.as_str()) .unwrap_or_default(); let timeout = upload_part_body_read_timeout(put_object_body_read_timeout(), max_total_object_size.is_some()); - let raw_control = req.extensions.get::().cloned(); - let observe_read_demand = if let Some(control) = &raw_control { - control.activate( - timeout, - &bucket, - &key, - request_id, - u64::try_from(size).map_err(|_| s3_error!(UnexpectedContent))?, - ) - } else { - // Direct protocol callers have no raw HTTP body. Retain their - // existing capped-session guard without inventing a client cause. - if max_total_object_size.is_some() { - body_stream = guard_put_object_body_read_timeout(body_stream, &bucket, &key, request_id, Some(size), timeout); - } - false - }; + let (body_stream, progress_control) = + if let Some(control) = req.extensions.get::() { + control.activate_upload(timeout, &bucket, &key, request_id, size.max(0) as u64); + (body_stream, control.clone()) + } else { + super::object::request_body::guard_upload_progress( + body_stream, + timeout, + &bucket, + &key, + request_id, + size.max(0) as u64, + ) + }; let ingress_stage_start = rustfs_io_metrics::put_stage_metrics_enabled().then(std::time::Instant::now); @@ -1453,8 +1482,8 @@ impl DefaultMultipartUsecase { write_plan = write_plan.with_encryption(ssec_write); (Some(ssec_material.server_side_encryption), ssec_material.kms_key_id) } else if let Some(server_side_encryption) = server_side_encryption { - // Reuses the envelope the create-multipart-upload call was authorized for; the - // KMS key was pinned into the session metadata then and cannot change here. + // Reuse the stored envelope after authorizing this caller against its + // pinned destination key above. let managed_material = sse_decryption(DecryptionRequest { bucket: &bucket, key: &key, @@ -1478,10 +1507,11 @@ impl DefaultMultipartUsecase { }; reader = write_plan.apply(reader, actual_size).map_err(ApiError::from)?; - if observe_read_demand && let Some(control) = raw_control { + + { use rustfs_rio::HashReaderMut; let inner = reader.take_inner(); - reader.inner = rustfs_rio::boxed_reader(super::object::request_body::DemandReader::new(inner, control)); + reader.inner = rustfs_rio::boxed_reader(super::object::request_body::DemandReader::new(inner, progress_control)); } let mut reader = PutObjReader::new(reader); @@ -1796,6 +1826,7 @@ impl DefaultMultipartUsecase { .get_multipart_info(&bucket, &key, &upload_id, &dst_opts) .await .map_err(ApiError::from)?; + authorize_sse_kms_multipart_write(copy_principal.as_ref(), &mp_info.user_defined).await?; let destination_size_limit = multipart_max_total_object_size(&mp_info.user_defined)?; EncryptionRequest { bucket: &bucket, @@ -1983,8 +2014,8 @@ impl DefaultMultipartUsecase { mp_info.user_defined.clone(), ) } else if let Some(server_side_encryption) = server_side_encryption { - // Destination side of the part copy: reuses the session envelope authorized at - // create-multipart-upload time. The source side is authorized above. + // Reuse the destination envelope after authorizing this caller against + // its pinned key above. The source read is separately authorized. let managed_material = sse_decryption(DecryptionRequest { bucket: &bucket, key: &key, @@ -2049,7 +2080,7 @@ impl DefaultMultipartUsecase { checksum_sha1: checksum_value(rustfs_rio::ChecksumType::SHA1), checksum_sha256: checksum_value(rustfs_rio::ChecksumType::SHA256), checksum_crc64nvme: checksum_value(rustfs_rio::ChecksumType::CRC64_NVME), - e_tag: part_info.etag.map(|etag| to_s3s_etag(&etag)), + e_tag: part_info.etag.map(|etag| to_s3s_xml_etag(&etag)), last_modified: part_info.last_mod.map(Timestamp::from), ..Default::default() }; @@ -2712,7 +2743,15 @@ mod tests { #[tokio::test] async fn multipart_upload_rejects_reserved_directory_markers() { - for key in ["__XLDIR__", "prefix__XLDIR__/object", "object__XLDIR__"] { + for key in [ + "__XLDIR__", + "prefix__XLDIR__/object", + "object__XLDIR__", + "parent/xl.meta", + "parent/uuid/xl.meta.bkp", + "parent/uuid/delete-prepared", + "parent/uuid/delete-quorum-committed", + ] { let create = CreateMultipartUploadInput::builder() .bucket("bucket".to_owned()) .key(key.to_owned()) @@ -2976,6 +3015,117 @@ mod tests { ); } + #[test] + #[serial_test::serial] + fn ssec_completion_and_replay_require_key_before_disclosing_plaintext_checksums() { + crate::app::gating_test_env::run_large_stack_test("ssec-completion-checksums", || async { + let (_store, bucket) = + crate::app::gating_test_env::durable_quota_test_bucket("ssec-completion-checksums", 16_384).await; + let usecase = DefaultMultipartUsecase::from_global(); + let key = "customer-encrypted"; + let customer_key = "KioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKio="; + let customer_md5 = "SaaDheEjzuynJH8eW6AEpQ=="; + let plaintext_sha256 = "ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0="; + let mut create_input = CreateMultipartUploadInput::builder() + .bucket(bucket.clone()) + .key(key.to_string()) + .build() + .expect("create"); + create_input.sse_customer_algorithm = Some("AES256".to_string()); + create_input.sse_customer_key = Some(customer_key.to_string()); + create_input.sse_customer_key_md5 = Some(customer_md5.to_string()); + create_input.checksum_algorithm = Some(ChecksumAlgorithm::from_static("SHA256")); + let mut create = build_request(create_input, Method::POST); + create + .headers + .insert("x-amz-checksum-algorithm", HeaderValue::from_static("SHA256")); + create + .headers + .insert("x-amz-checksum-type", HeaderValue::from_static("COMPOSITE")); + let upload_id = usecase + .execute_create_multipart_upload(create) + .await + .expect("SSE-C create") + .output + .upload_id + .expect("upload id"); + let mut part_input = UploadPartInput::builder() + .bucket(bucket.clone()) + .key(key.to_string()) + .upload_id(upload_id.clone()) + .part_number(1) + .build() + .expect("part"); + part_input.body = Some(StreamingBlob::from(Bytes::from_static(b"abc"))); + part_input.content_length = Some(3); + part_input.sse_customer_algorithm = Some("AES256".to_string()); + part_input.sse_customer_key = Some(customer_key.to_string()); + part_input.sse_customer_key_md5 = Some(customer_md5.to_string()); + part_input.checksum_algorithm = Some(ChecksumAlgorithm::from_static("SHA256")); + part_input.checksum_sha256 = Some(plaintext_sha256.to_string()); + let mut part = build_request(part_input, Method::PUT); + part.headers + .insert("x-amz-checksum-sha256", HeaderValue::from_static(plaintext_sha256)); + let part = usecase.execute_upload_part(part).await.expect("SSE-C part").output; + let complete = CompleteMultipartUploadInput::builder() + .bucket(bucket.clone()) + .key(key.to_string()) + .upload_id(upload_id) + .multipart_upload(Some(CompletedMultipartUpload { + parts: Some(vec![CompletedPart { + part_number: Some(1), + e_tag: part.e_tag, + checksum_sha256: Some(plaintext_sha256.to_string()), + ..Default::default() + }]), + })) + .build() + .expect("complete"); + // Reject keyless completion before checksum comparison, then enforce + // the same key boundary on a successful commit and its replay. + for (attempt, keyed) in [false, true, false, true].into_iter().enumerate() { + let mut input = complete.clone(); + if keyed { + input.sse_customer_algorithm = Some("AES256".to_string()); + input.sse_customer_key = Some(customer_key.to_string()); + input.sse_customer_key_md5 = Some(customer_md5.to_string()); + } + if attempt == 2 { + let part = &mut input + .multipart_upload + .as_mut() + .expect("parts") + .parts + .as_mut() + .expect("part list")[0]; + part.e_tag = Some(ETag::Strong("guessed-etag".to_string())); + part.checksum_sha256 = Some("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=".to_string()); + } + let result = usecase + .execute_complete_multipart_upload(build_request(input, Method::POST)) + .await; + if keyed { + let response = result.expect("verified key completes and replays"); + assert!(response.output.checksum_sha256.is_some(), "verified key retains checksum response"); + assert_eq!( + response + .headers + .get("x-amz-server-side-encryption-customer-key-md5") + .expect("key MD5"), + customer_md5 + ); + } else { + assert_eq!( + result + .expect_err("keyless checksum completion must fail before disclosure") + .code(), + &S3ErrorCode::InvalidRequest + ); + } + } + }); + } + #[test] #[serial_test::serial] fn conditional_multipart_upload_if_none_match_returns_precondition_failed_when_object_exists() { @@ -3480,6 +3630,80 @@ mod tests { assert_eq!(completed.size, 4096); } + #[tokio::test] + #[serial_test::serial] + async fn cancelled_multipart_completion_emits_once_and_replay_does_not_emit() { + use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions}; + let store = crate::app::gating_test_env::shared_gating_ecstore().await; + crate::app::runtime_sources::install_test_app_context(Arc::clone(&store)).await; + let bucket = format!("mpu-owned-event-{}", Uuid::new_v4()); + let object = "object"; + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("bucket"); + let upload = store + .new_multipart_upload(&bucket, object, &ObjectOptions::default()) + .await + .expect("upload"); + let mut reader = PutObjReader::from_vec(b"part".to_vec()); + let part = store + .put_object_part(&bucket, object, &upload.upload_id, 1, &mut reader, &ObjectOptions::default()) + .await + .expect("part"); + let input = CompleteMultipartUploadInput::builder() + .bucket(bucket.clone()) + .key(object.to_owned()) + .upload_id(upload.upload_id) + .multipart_upload(Some(CompletedMultipartUpload { + parts: Some(vec![CompletedPart { + part_number: Some(1), + e_tag: part.etag.map(|etag| to_s3s_etag(&etag)), + ..Default::default() + }]), + })) + .build() + .expect("complete input"); + let probe = crate::app::storage_api::multipart_usecase::helper::MutationCompletionProbe::install(&bucket); + let (entered, resume) = probe.pause_after_commit(); + let request = |input| { + let mut request = build_request(input, Method::POST); + request.uri = format!("/{bucket}/{object}").parse().expect("complete URI"); + request + }; + let first = request(input.clone()); + let completion = tokio::spawn(async move { + DefaultMultipartUsecase::from_global() + .execute_complete_multipart_upload(first) + .await + }); + tokio::time::timeout(std::time::Duration::from_secs(10), entered.wait()) + .await + .expect("committed MPU"); + completion.abort(); + assert!(completion.await.expect_err("cancel waiter").is_cancelled()); + resume.wait().await; + probe.wait_for(1, 1).await; + assert_eq!( + store + .get_object_info(&bucket, object, &ObjectOptions::default()) + .await + .expect("committed object") + .size, + 4 + ); + DefaultMultipartUsecase::from_global() + .execute_complete_multipart_upload(request(input)) + .await + .expect("idempotent replay"); + let audits = probe.audits.lock().expect("audits"); + assert_eq!(audits.len(), 2); + assert!(audits.iter().all(|entry| entry.api.status.as_deref() == Some("success"))); + let events = probe.events.lock().expect("events"); + assert_eq!(events.len(), 1, "replay must not emit another creation event"); + assert_eq!(events[0].event_name, EventName::ObjectCreatedCompleteMultipartUpload); + } + #[tokio::test] #[serial_test::serial] async fn rejected_empty_parts_preserve_existing_object_and_staging() { @@ -4016,6 +4240,63 @@ mod tests { drop(held); } + #[tokio::test] + #[serial_test::serial] + async fn execute_upload_part_progress_timeout_releases_admission_and_leaves_no_part() { + use crate::app::object::request_body::{BodyReadControl, ObservedBody}; + use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions}; + use http_body::Frame; + use http_body_util::StreamBody; + use tokio_stream::wrappers::UnboundedReceiverStream; + let store = crate::app::gating_test_env::shared_gating_ecstore().await; + crate::app::runtime_sources::install_test_app_context(Arc::clone(&store)).await; + let ambient = crate::app::runtime_sources::current_app_context().expect("test app context"); + let context = Arc::new(AppContext::new(Arc::clone(&store), ambient.iam(), ambient.kms())); + let bucket = format!("part-progress-{}", Uuid::new_v4().simple()); + store.make_bucket(&bucket, &MakeBucketOptions::default()).await.unwrap(); + let manager = Arc::new(ConcurrencyManager::with_put_admission_for_test(true, 1, Duration::ZERO)); + let usecase = DefaultMultipartUsecase::with_context_and_concurrency_manager(Some(context), Arc::clone(&manager)); + let upload = store + .new_multipart_upload(&bucket, "object", &ObjectOptions::default()) + .await + .unwrap(); + let (sender, receiver) = tokio::sync::mpsc::unbounded_channel::, std::io::Error>>(); + let control = BodyReadControl::default(); + let body = StreamingBlob::from(s3s::Body::http_body_unsync(ObservedBody::new( + StreamBody::new(UnboundedReceiverStream::new(receiver)), + control.clone(), + ))); + let input = UploadPartInput::builder() + .bucket(bucket.clone()) + .key("object".to_owned()) + .upload_id(upload.upload_id.clone()) + .part_number(1) + .content_length(Some(65536)) + .body(Some(body)) + .build() + .unwrap(); + let mut request = build_request(input, Method::PUT); + request.extensions.insert(control.clone()); + let owner = tokio::spawn(async move { usecase.execute_upload_part(request).await }); + control.wait_until_pending_for_test().await; + assert!(matches!( + manager.admit_put_object(65536).await.unwrap(), + ForegroundWriteAdmission::Rejected + )); + control.advance_wait_for_test(Duration::from_secs(60)); + sender.send(Ok(Frame::data(Bytes::from_static(b"x")))).unwrap(); + assert_eq!(owner.await.unwrap().unwrap_err().code(), &S3ErrorCode::RequestTimeout); + assert!(matches!( + manager.admit_put_object(65536).await.unwrap(), + ForegroundWriteAdmission::Admitted(_) + )); + let parts = store + .list_object_parts(&bucket, "object", &upload.upload_id, None, 1000, &ObjectOptions::default()) + .await + .unwrap(); + assert!(parts.parts.is_empty()); + } + #[tokio::test] #[serial_test::serial] async fn execute_upload_part_rejects_when_foreground_write_admission_is_full() { diff --git a/rustfs/src/app/object/copy.rs b/rustfs/src/app/object/copy.rs index 4e856830f..5430236d6 100644 --- a/rustfs/src/app/object/copy.rs +++ b/rustfs/src/app/object/copy.rs @@ -15,6 +15,7 @@ //! CopyObject path. use super::*; +use crate::app::storage_api::object_usecase::object_utils::to_s3s_xml_etag; use futures::FutureExt; use rustfs_utils::path::fold_object_name_path_separators; use tracing::Instrument; @@ -142,7 +143,7 @@ impl DefaultObjectUsecase { Box::pin(self.execute_copy_object_body(req)) } - async fn execute_copy_object_body(&self, req: S3Request) -> S3Result> { + async fn execute_copy_object_body(&self, mut req: S3Request) -> S3Result> { reject_presigned_put_max_content_length_for_other_operation( &req.headers, req.uri.query(), @@ -238,7 +239,7 @@ impl DefaultObjectUsecase { // Validate both source and destination keys validate_object_key(&src_key, "COPY (source)")?; - validate_object_key(&key, "COPY (dest)")?; + validate_new_object_key(&key, "COPY (dest)")?; reject_veeam_sosapi_mutation(|| self.object_store(), &bucket, &key).await?; validate_table_catalog_object_mutation(&bucket, &key).await?; let replaces_metadata = match metadata_directive.as_ref().map(|directive| directive.as_str()) { @@ -481,6 +482,13 @@ impl DefaultObjectUsecase { let _source_cancellation_guard = source_cancellation.clone().drop_guard(); let mut src_info = gr.object_info.clone(); + let effective_tags = replacement_tags.unwrap_or_else(|| (*src_info.user_tags).clone()); + crate::app::storage_api::object_usecase::access::authorize_copy_object_tags( + &mut req, + &src_info.user_tags, + &effective_tags, + ) + .await?; // A copy reads the source plaintext, so it needs the source key's decrypt permission // as well as the destination key's generate permission below. The source read resolves @@ -562,7 +570,6 @@ impl DefaultObjectUsecase { // Extract user_defined from Arc for mutation; it will be re-wrapped after all edits. let mut user_defined = (*src_info.user_defined).clone(); - let effective_tags = replacement_tags.unwrap_or_else(|| (*src_info.user_tags).clone()); if !replaces_metadata { let source_expires = src_info.expires.map(Timestamp::from); insert_expires_metadata(&mut user_defined, source_expires.as_ref())?; @@ -882,7 +889,7 @@ impl DefaultObjectUsecase { } } let copy_object_result = CopyObjectResult { - e_tag: oi.etag.as_ref().map(|etag| to_s3s_etag(etag)), + e_tag: oi.etag.as_ref().map(|etag| to_s3s_xml_etag(etag)), last_modified: oi.mod_time.map(Timestamp::from), checksum_crc32: response_checksums.crc32, checksum_crc32c: response_checksums.crc32c, @@ -1510,6 +1517,71 @@ mod tests { assert_eq!(err.code(), &S3ErrorCode::InvalidRequest); } + #[tokio::test] + #[serial_test::serial] + async fn metadata_only_copy_escapes_legacy_etags_in_xml() { + use crate::app::storage_api::test::ReqInfo; + use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions}; + use s3s::xml::{SerializeContent, Serializer}; + let (store, context) = real_store_test_context().await; + let bucket = format!("copy-legacy-etag-{}", Uuid::new_v4()); + let key = "owned.txt"; + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create bucket"); + let etag = "victim&"; + let mut reader = PutObjReader::from_vec(b"legacy object".to_vec()); + store + .put_object( + &bucket, + key, + &mut reader, + &ObjectOptions { + preserve_etag: Some(etag.to_string()), + ..Default::default() + }, + ) + .await + .expect("seed legacy ETag directly in storage"); + let before = store + .get_object_info(&bucket, key, &ObjectOptions::default()) + .await + .expect("legacy object"); + assert_eq!(before.etag.as_deref(), Some(etag)); + let input = CopyObjectInput::builder() + .bucket(bucket.clone()) + .key(key.to_string()) + .copy_source(CopySource::Bucket { + bucket: bucket.clone().into(), + key: key.into(), + version_id: None, + }) + .metadata_directive(Some(s3s::dto::MetadataDirective::from_static("REPLACE"))) + .build() + .expect("self-copy request"); + let mut request = build_request(input, Method::PUT); + request.extensions.insert(ReqInfo::default()); + let result = Box::pin(DefaultObjectUsecase::with_context(Some(context)).execute_copy_object(request)) + .await + .expect("metadata-only self-copy"); + let mut bytes = Vec::new(); + result + .output + .copy_object_result + .expect("copy result") + .serialize_content(&mut Serializer::new(&mut bytes)) + .expect("serialize copy result"); + let xml = String::from_utf8(bytes).expect("XML UTF-8"); + assert!(!xml.contains(""), "{xml}"); + assert!(xml.contains("<Injected>victim</Injected>"), "{xml}"); + let stored = store + .get_object_info(&bucket, key, &ObjectOptions::default()) + .await + .expect("copied object"); + assert_eq!(stored.etag.as_deref(), Some(etag), "escaping belongs only to the wire response"); + } + #[tokio::test] #[serial_test::serial] async fn execute_copy_object_refuses_a_bucket_whose_encryption_config_is_unreadable() { diff --git a/rustfs/src/app/object/delete.rs b/rustfs/src/app/object/delete.rs index dcd08492e..7ccc1b774 100644 --- a/rustfs/src/app/object/delete.rs +++ b/rustfs/src/app/object/delete.rs @@ -54,12 +54,10 @@ async fn authorize_recursive_delete( let info = req_info_mut(req)?; info.object = Some(object.name.clone()); info.version_id = version_id.clone(); - let action = if replica { - Action::S3Action(S3Action::ReplicateDeleteAction) - } else { - delete_object_authorize_action(version_id.as_deref()) - }; - authorize_request(req, action).await?; + authorize_request(req, delete_object_authorize_action(version_id.as_deref())).await?; + if replica { + authorize_request(req, Action::S3Action(S3Action::ReplicateDeleteAction)).await?; + } if has_bypass_governance_header(&req.headers) { authorize_request(req, Action::S3Action(S3Action::BypassGovernanceRetentionAction)).await?; } @@ -524,7 +522,12 @@ fn reduce_delete_objects_result<'a>( impl DefaultObjectUsecase { #[instrument(level = "debug", skip(self, req))] - pub async fn execute_delete_objects( + pub async fn execute_delete_objects(&self, req: S3Request) -> S3Result> { + let owner = self.clone(); + run_owned_mutation(async move { Box::pin(owner.execute_delete_objects_owned(req)).await }).await + } + + async fn execute_delete_objects_owned( &self, mut req: S3Request, ) -> S3Result> { @@ -851,6 +854,8 @@ impl DefaultObjectUsecase { .delete_objects_with_tier_delete_journal_and_accounting(&bucket, object_to_delete.clone(), storage_delete_opts) .await; + #[cfg(test)] + super::super::storage_api::object_usecase::helper::pause_committed_mutation_for_test(&bucket).await; let _manager = get_concurrency_manager(); if !errs.is_empty() && errs.iter().all(|err| err.as_ref().is_some_and(is_err_bucket_not_found)) { let result = Err(S3Error::with_message(S3ErrorCode::NoSuchBucket, "Bucket not found".to_string())); @@ -1031,6 +1036,8 @@ impl DefaultObjectUsecase { .user_agent(get_request_user_agent(&req_headers)) .build(); + #[cfg(test)] + super::super::storage_api::object_usecase::helper::capture_mutation_event_for_test(&event_args); notify.notify(event_args).await; } } @@ -1050,7 +1057,15 @@ impl DefaultObjectUsecase { } #[instrument(level = "info", skip(self, req))] - pub async fn execute_delete_object(&self, mut req: S3Request) -> S3Result> { + pub async fn execute_delete_object(&self, req: S3Request) -> S3Result> { + let owner = self.clone(); + run_owned_mutation(async move { Box::pin(owner.execute_delete_object_owned(req)).await }).await + } + + async fn execute_delete_object_owned( + &self, + mut req: S3Request, + ) -> S3Result> { if let Some(context) = &self.context { let _ = context.object_store(); } @@ -1261,6 +1276,8 @@ impl DefaultObjectUsecase { } }; + #[cfg(test)] + super::super::storage_api::object_usecase::helper::pause_committed_mutation_for_test(&bucket).await; if force_delete { let _ = invalidate_object_data_cache_prefix_after_delete(&cache_adapter, &bucket, &key).await; } else { @@ -1454,6 +1471,105 @@ impl DefaultObjectUsecase { #[cfg(test)] mod tests { + #[test] + fn recursive_replica_delete_checks_each_descendant_delete_permission() { + crate::app::gating_test_env::run_large_stack_test("recursive-replica-permissions", || async { + use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions}; + let store = crate::app::gating_test_env::shared_gating_ecstore().await; + let ambient = crate::app::gating_test_env::shared_gating_ambient().await; + let slot = crate::app::runtime_sources::ServerContextSlot::new(); + assert!(slot.install(Arc::new(AppContext::new(store.clone(), ambient.iam(), ambient.kms())))); + for versioned in [false, true] { + let bucket = format!("recursive-replica-{}", Uuid::new_v4()); + store + .make_bucket( + &bucket, + &MakeBucketOptions { + versioning_enabled: versioned, + ..Default::default() + }, + ) + .await + .expect("create bucket"); + for key in ["team/public", "team/secret"] { + for _ in 0..2 { + store + .put_object( + &bucket, + key, + &mut PutObjReader::from_vec(b"protected payload".to_vec()), + &ObjectOptions { + versioned, + ..Default::default() + }, + ) + .await + .expect("seed descendant"); + } + } + let before = store + .clone() + .list_object_versions(&bucket, "team/", None, None, None, 1000) + .await + .expect("list fixture") + .objects + .len(); + for (grant_delete, deny_delete) in [(false, false), (true, true), (true, false)] { + let mut statements = vec![ + serde_json::json!({"Effect":"Allow","Principal":{"AWS":"*"},"Action":"s3:ReplicateDelete","Resource":format!("arn:aws:s3:::{bucket}/*")}), + ]; + if grant_delete { + statements.push(serde_json::json!({"Effect":"Allow","Principal":{"AWS":"*"},"Action":["s3:DeleteObject","s3:DeleteObjectVersion"],"Resource":format!("arn:aws:s3:::{bucket}/*")})); + } + if deny_delete { + statements.push(serde_json::json!({"Effect":"Deny","Principal":{"AWS":"*"},"Action":["s3:DeleteObject","s3:DeleteObjectVersion"],"Resource":format!("arn:aws:s3:::{bucket}/team/secret")})); + } + store + .update_bucket_metadata_config( + &bucket, + crate::storage::storage_api::ecstore_bucket::metadata::BUCKET_POLICY_CONFIG, + serde_json::to_vec(&serde_json::json!({"Version":"2012-10-17","Statement":statements})).unwrap(), + ) + .await + .expect("publish policy"); + for replica in [false, true] { + let mut req = build_request((), Method::DELETE); + req.extensions.insert(slot.clone()); + req.extensions.insert(crate::storage::access::ReqInfo { + bucket: Some(bucket.clone()), + object: Some("team/".to_owned()), + ..Default::default() + }); + let result = authorize_recursive_delete(&mut req, &store, &bucket, "team/", versioned, replica).await; + if grant_delete && !deny_delete { + result.expect("ordinary and replication deletes allowed"); + } else { + assert_eq!( + result.expect_err("descendant delete permission must be enforced").code(), + &S3ErrorCode::AccessDenied + ); + } + assert_eq!( + req_info_ref(&req).unwrap().object.as_deref(), + Some("team/"), + "restore original authorization context" + ); + assert_eq!( + store + .clone() + .list_object_versions(&bucket, "team/", None, None, None, 1000) + .await + .expect("versions survive preflight") + .objects + .len(), + before + ); + } + } + } + }); + } + use super::*; use http::{HeaderMap, HeaderValue, Method}; use s3s::dto::{ @@ -3104,6 +3220,92 @@ mod tests { }); } + #[test] + #[serial_test::serial] + fn cancelled_delete_requests_complete_audit_and_notifications() { + crate::app::gating_test_env::run_large_stack_test("owned-delete-events", || async { + use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions}; + let (store, context) = real_store_test_context().await; + for batch in [false, true] { + let bucket = format!("delete-owned-event-{}", Uuid::new_v4()); + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("bucket"); + for key in ["first", "second"] { + let mut reader = PutObjReader::from_vec(b"object".to_vec()); + store + .put_object(&bucket, key, &mut reader, &ObjectOptions::default()) + .await + .expect("object"); + } + let policy_json = format!( + r#"{{"Version":"2012-10-17","Statement":[{{"Effect":"Allow","Principal":{{"AWS":"*"}},"Action":["s3:DeleteObject","s3:DeleteObjectVersion"],"Resource":["arn:aws:s3:::{bucket}/*"]}}]}}"# + ); + let mut metadata = (*crate::storage::get_bucket_metadata(&bucket).await.expect("metadata")).clone(); + metadata.policy_config = Some(serde_json::from_str(&policy_json).expect("policy")); + metadata.policy_config_json = policy_json.into_bytes(); + crate::storage::storage_api::set_bucket_metadata(bucket.clone(), metadata) + .await + .expect("policy metadata"); + let probe = crate::app::storage_api::object_usecase::helper::MutationCompletionProbe::install(&bucket); + let (entered, resume) = probe.pause_after_commit(); + let usecase = DefaultObjectUsecase::with_context(Some(context.clone())); + let request_bucket = bucket.clone(); + let deletion = tokio::spawn(async move { + if batch { + let input = DeleteObjectsInput::builder() + .bucket(request_bucket.clone()) + .delete(Delete { + objects: ["first", "second"] + .into_iter() + .map(|key| ObjectIdentifier { + key: key.to_owned(), + ..Default::default() + }) + .collect(), + quiet: Some(false), + }) + .build() + .expect("batch input"); + let mut request = build_request(input, Method::POST); + request.uri = format!("/{request_bucket}").parse().expect("URI"); + request.extensions.insert(crate::storage::access::ReqInfo::default()); + usecase.execute_delete_objects(request).await.map(|_| ()) + } else { + let input = DeleteObjectInput::builder() + .bucket(request_bucket.clone()) + .key("first".to_owned()) + .build() + .expect("delete input"); + let mut request = build_request(input, Method::DELETE); + request.uri = format!("/{request_bucket}/first").parse().expect("URI"); + usecase.execute_delete_object(request).await.map(|_| ()) + } + }); + tokio::time::timeout(std::time::Duration::from_secs(10), entered.wait()) + .await + .expect("committed delete"); + deletion.abort(); + assert!(deletion.await.expect_err("cancel waiter").is_cancelled()); + resume.wait().await; + probe.wait_for(1, if batch { 2 } else { 1 }).await; + assert!( + store + .get_object_info(&bucket, "first", &ObjectOptions::default()) + .await + .is_err() + ); + let audits = probe.audits.lock().expect("audits"); + assert_eq!(audits.len(), 1); + assert_eq!(audits[0].api.status.as_deref(), Some("success")); + let events = probe.events.lock().expect("events"); + assert_eq!(events.len(), if batch { 2 } else { 1 }); + assert!(events.iter().all(|event| event.event_name == EventName::ObjectRemovedDelete)); + } + }); + } + #[test] #[serial_test::serial] fn recursive_delete_holds_writer_exclusion_after_authorization() { @@ -3791,11 +3993,16 @@ mod tests { .delete_object(&bucket, "object", ObjectOptions::default()) .await .expect("delete quota-tracked object"); + // Deleted bytes are reclaimed by the owned reconciliation job; requests + // no longer scan the bucket synchronously under the ledger lock. + crate::app::storage_api::test::set_disk::retry_quota_reconciliation_for_test(Arc::clone(&store), &bucket) + .await + .expect("reconcile the deleted object's usage through the production worker"); let mut replacement = PutObjReader::from_vec(vec![0x43; 4096]); store .put_object(&bucket, "replacement", &mut replacement, &ObjectOptions::default()) .await - .expect("deleted bytes must be reclaimed before rejecting a replacement"); + .expect("reconciled deleted bytes must admit a full-size replacement"); let mut excess = PutObjReader::from_vec(vec![0x44]); let err = store diff --git a/rustfs/src/app/object/extract.rs b/rustfs/src/app/object/extract.rs index ae14ea6b4..f07c9313c 100644 --- a/rustfs/src/app/object/extract.rs +++ b/rustfs/src/app/object/extract.rs @@ -457,7 +457,9 @@ where } let pool = get_concurrency_manager().bytes_pool(); - let mut buffer = pool.acquire_buffer(size).await; + let mut buffer = pool.try_acquire_buffer(size).ok_or_else(|| { + S3Error::with_message(S3ErrorCode::SlowDown, "upload buffer pool is full, please reduce your request rate") + })?; super::put::read_small_put_body_into(body, &mut *buffer, size).await?; Ok(ExtractStagedBody { reader: Some(super::put::PooledBufferReader::new(buffer, size)), @@ -1282,7 +1284,7 @@ fn is_empty_extract_entry_path(path: &str) -> bool { fn validate_extract_member_key(path: &str, limits: ArchiveLimits) -> Result<(), ExtractEntryError> { validate_put_object_extract_entry_path(path, limits).map_err(ExtractEntryError::Recoverable)?; - validate_object_key(path, "PUT").map_err(ExtractEntryError::Recoverable) + validate_new_object_key(path, "PUT").map_err(ExtractEntryError::Recoverable) } fn record_extract_pax_metadata_bytes( @@ -1588,6 +1590,26 @@ fn extract_member_iam_requirements( } } +// Version and replica controls require authorization even when the archive does not set a time. +// An mtime-only member can still be uploaded with server time when the caller lacks that permission. +async fn authorized_extract_member_mod_time( + req: &mut S3Request, + replication_required: bool, + archive_time: Option, +) -> S3Result> { + if !replication_required { + if archive_time.is_none() { + return Ok(None); + } + req_info_mut(req)?.suppress_denial_log = true; + } + match authorize_request(req, Action::S3Action(S3Action::ReplicateObjectAction)).await { + Ok(()) => Ok(archive_time), + Err(err) if !replication_required && err.code() == &S3ErrorCode::AccessDenied => Ok(None), + Err(err) => Err(err), + } +} + fn apply_extract_pax_overrides( overrides: &ExtractPaxOverrides, bucket: &str, @@ -2067,14 +2089,26 @@ impl DefaultObjectUsecase { true, )?; let Some(body) = body else { return Err(s3_error!(IncompleteBody)) }; - let body = guard_put_object_body_read_timeout( - body, - &bucket, - &key, - &request_context.request_id, - content_length, - put_object_body_read_timeout(), - ); + let (body, progress_control) = if let Some(control) = req.extensions.get::() { + control.activate_upload( + put_object_body_read_timeout(), + &bucket, + &key, + &request_context.request_id, + content_length.unwrap_or_default().max(0) as u64, + ); + (body, control.clone()) + } else { + super::request_body::guard_upload_progress( + body, + put_object_body_read_timeout(), + &bucket, + &key, + &request_context.request_id, + content_length.unwrap_or_default().max(0) as u64, + ) + }; + let body = super::request_body::demand_upload_body(body, progress_control.clone()); let size = match content_length { Some(c) => c, @@ -2092,7 +2126,7 @@ impl DefaultObjectUsecase { if size < 0 { return Err(s3_error!(UnexpectedContent)); } - validate_object_key(&key, "PUT")?; + validate_new_object_key(&key, "PUT")?; validate_table_catalog_object_mutation(&bucket, &key).await?; let _ = self .check_bucket_quota( @@ -2156,6 +2190,7 @@ impl DefaultObjectUsecase { ExtractArchiveEtagReader::new(archive_reader, expected_archive_length, archive_upload_state.clone()); let decoder = build_admitted_extract_archive_decoder(get_concurrency_manager(), &key, tracked_archive).await?; let decoder = ExtractDecodedLimitReader::new(decoder, extract_limits.max_decoded_size); + let decoder = super::request_body::DemandReader::new(decoder, progress_control); let mut ar = build_put_object_extract_archive(decoder, extract_limits); let mut entries = ar.entries().map_err(|e| { @@ -2357,7 +2392,10 @@ impl DefaultObjectUsecase { put_opts_with_replication_authorization( &bucket, &fpath, - outer_version_id.clone(), + // The DTO version is applied with any PAX override below, then + // authorized against this member before publication. Header + // replication controls still require the outer authorization. + None, &member_request_headers, metadata.clone(), replication_authorized, @@ -2456,15 +2494,14 @@ impl DefaultObjectUsecase { if iam_requirements.legal_hold { extract_try!(authorize_request(&mut auth_req, Action::S3Action(S3Action::PutObjectLegalHoldAction)).await); } - if iam_requirements.replication { - extract_try!(authorize_request(&mut auth_req, Action::S3Action(S3Action::ReplicateObjectAction)).await); + if let Some(mod_time) = extract_try!( + authorized_extract_member_mod_time(&mut auth_req, iam_requirements.replication, archive_entry_mod_time,).await + ) { + opts.mod_time = Some(mod_time); } let table_data_plane_publication_guards = retained_table_data_plane_publication_guards(&auth_req); drop(auth_req); extract_try!(reject_veeam_sosapi_mutation(|| Some(Arc::clone(&commit_context.store)), &bucket, &fpath).await); - if archive_entry_mod_time.is_some() { - opts.mod_time = archive_entry_mod_time; - } debug!("Extracting file: {}, size: {} bytes", fpath, size); @@ -2792,6 +2829,49 @@ mod tests { }) } + #[tokio::test(start_paused = true)] + async fn snowball_stalled_archives_release_actual_decoder_slots() { + use futures::poll; + use tokio_stream::wrappers::UnboundedReceiverStream; + let manager = ConcurrencyManager::new(); + let mut senders = Vec::new(); + let mut owners = Vec::new(); + for _ in 0..4 { + let (sender, receiver) = tokio::sync::mpsc::unbounded_channel::>(); + let (body, _control) = super::super::request_body::guard_upload_progress( + StreamingBlob::wrap(UnboundedReceiverStream::new(receiver)), + Duration::ZERO, + "bucket", + "archive.tar", + "request", + 1024, + ); + let body = StreamReader::new(body.map(|item| item.map_err(s3s_body_error_to_io))); + let mut owner = Box::pin(build_admitted_extract_archive_decoder(&manager, "archive.tar", body)); + assert!(poll!(owner.as_mut()).is_pending()); + senders.push(sender); + owners.push(owner); + } + assert!(manager.try_acquire_snowball_archive_decoder().is_none()); + for _ in 0..5 { + tokio::time::advance(Duration::from_secs(10)).await; + for (sender, owner) in senders.iter().zip(&mut owners) { + sender.send(Ok(Bytes::from_static(b"x"))).unwrap(); + assert!(poll!(owner.as_mut()).is_pending()); + } + } + tokio::time::advance(Duration::from_secs(10)).await; + for owner in owners { + let error = owner.await.err().expect("stalled archive sniff must expire"); + assert_eq!(error.code(), &S3ErrorCode::RequestTimeout); + } + let recovered: Vec<_> = (0..4) + .map(|_| manager.try_acquire_snowball_archive_decoder().expect("reusable slot")) + .collect(); + assert!(manager.try_acquire_snowball_archive_decoder().is_none()); + drop(recovered); + } + #[tokio::test] async fn snowball_archive_decoder_admission_is_global_and_lifetime_bound() { struct PanicOnRead; @@ -3585,6 +3665,107 @@ mod tests { } } + #[tokio::test] + #[serial_test::serial] + async fn snowball_member_mtime_requires_member_replication_authorization() { + use crate::app::storage_api::test::ReqInfo; + use crate::app::storage_api::test::bucket::metadata::{BUCKET_POLICY_CONFIG, BUCKET_VERSIONING_CONFIG}; + use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions}; + use http::Method; + + let (store, context) = real_store_test_context().await; + let bucket = format!("extract-time-{}", Uuid::new_v4()); + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create bucket"); + metadata_sys::update( + &bucket, + BUCKET_VERSIONING_CONFIG, + b"Enabled".to_vec(), + ) + .await + .expect("enable versioning"); + let usecase = DefaultObjectUsecase::with_context(Some(context)); + for (replicate, explicit_version, mtime) in [ + (false, false, 2_524_608_000_u64), + (false, false, 1_000_000_000), + (false, true, 2_524_608_000), + (true, false, 1_704_000_123), + (true, true, 2_524_608_000), + ] { + let actions = if replicate { + vec!["s3:PutObject", "s3:ReplicateObject"] + } else { + vec!["s3:PutObject"] + }; + let policy = serde_json::json!({"Version":"2012-10-17", "Statement":[{ + "Effect":"Allow", "Principal":"*", "Action":actions, + "Resource":format!("arn:aws:s3:::{bucket}/*") + }]}); + metadata_sys::update(&bucket, BUCKET_POLICY_CONFIG, serde_json::to_vec(&policy).expect("policy JSON")) + .await + .expect("install member permissions"); + let member = match (replicate, explicit_version) { + (true, false) => "authorized-time-only.txt", + (true, true) => "replicated.txt", + (false, true) => "denied-version.txt", + (false, false) => "ordinary.txt", + }; + let mut builder = Builder::new(Vec::new()); + let mut header = Header::new_gnu(); + header.set_size(1); + header.set_mode(0o644); + header.set_mtime(mtime); + header.set_cksum(); + builder + .append_data(&mut header, member, &b"x"[..]) + .await + .expect("append member"); + let archive = builder.into_inner().await.expect("finish archive"); + let before = OffsetDateTime::now_utc(); + let input = PutObjectInput::builder() + .bucket(bucket.clone()) + .key("archive.tar".to_string()) + .version_id(explicit_version.then(|| Uuid::new_v4().to_string())) + .content_length(Some(i64::try_from(archive.len()).expect("archive size"))) + .body(Some(StreamingBlob::from_bytes(Bytes::from(archive)))) + .build() + .expect("extract request"); + let mut request = build_request(input, Method::PUT); + request.extensions.insert(ReqInfo::default()); + let result = Box::pin(usecase.execute_put_object_extract(request)).await; + if explicit_version && !replicate { + assert_eq!( + result + .expect_err("version selection requires member replication permission") + .code(), + &S3ErrorCode::AccessDenied + ); + let missing = store + .get_object_info(&bucket, member, &ObjectOptions::default()) + .await + .expect_err("unauthorized version cannot publish a member"); + assert!(is_err_object_not_found(&missing)); + continue; + } + result.expect("extract member"); + let info = store + .get_object_info(&bucket, member, &ObjectOptions::default()) + .await + .expect("current member"); + let actual = info.mod_time.expect("stored timestamp"); + if replicate { + assert_eq!(actual.unix_timestamp(), i64::try_from(mtime).expect("timestamp")); + } else { + assert!( + actual >= before && actual <= OffsetDateTime::now_utc(), + "ordinary upload used archive time: {actual}" + ); + } + } + } + #[tokio::test] async fn archive_etag_reader_validates_sha256_before_completion() { let payload = b"archive-with-wrong-sha256".to_vec(); diff --git a/rustfs/src/app/object/get.rs b/rustfs/src/app/object/get.rs index 8b8523aad..e657b0d74 100644 --- a/rustfs/src/app/object/get.rs +++ b/rustfs/src/app/object/get.rs @@ -16,6 +16,7 @@ use super::veeam_sosapi; use super::*; +use crate::app::storage_api::object_usecase::access::replication_proxy_allowed; use crate::app::storage_api::object_usecase::object_cache::{ GetObjectReadCondition, GetObjectReadMetadata, evaluate_read_preconditions, has_read_preconditions, with_get_object_read_condition, @@ -669,48 +670,186 @@ impl GetObjectBodyLifecycle { } } -pin_project! { - // Keep the disk-read admission permit tied to the response body. This is - // intentionally conservative backpressure: a streaming GET should occupy a - // read slot until the client drains or drops the body. - struct DiskReadPermitReader { - #[pin] - inner: R, - disk_permit: Option, - } +// Bound downstream inactivity without imposing a fixed limit on large downloads. +const GET_DISK_READ_IDLE_TIMEOUT: Duration = Duration::from_secs(30); +const GET_DISK_READ_MIN_WINDOW_BYTES: usize = 64 * 1024; + +struct DiskReadLease { + reader: Option>>, + permit: Option, + pending: bool, + pending_capacity: usize, + closed: bool, + expired: bool, + last_progress: tokio::time::Instant, + window_start: tokio::time::Instant, + window_bytes: usize, + idle_timeout: Duration, + progress_window: Duration, + waker: Option, } -impl DiskReadPermitReader { - fn new(inner: R, disk_permit: GetObjectDiskPermit) -> Self { - Self { - inner, - disk_permit: Some(disk_permit), +impl DiskReadLease { + fn check_deadline(&mut self, now: tokio::time::Instant) { + if now.saturating_duration_since(self.last_progress) >= self.idle_timeout { + self.expired = true; + } + if now.saturating_duration_since(self.window_start) >= self.progress_window { + if self.window_bytes < GET_DISK_READ_MIN_WINDOW_BYTES { + self.expired = true; + } else { + self.window_start = now; + self.window_bytes = 0; + } } } + + fn next_check_delay(&self, now: tokio::time::Instant) -> Duration { + self.idle_timeout + .saturating_sub(now.saturating_duration_since(self.last_progress)) + .min( + self.progress_window + .saturating_sub(now.saturating_duration_since(self.window_start)), + ) + // Keep timer arithmetic representable even for extreme operator overrides. + .min(GET_DISK_READ_IDLE_TIMEOUT) + } } -impl AsyncRead for DiskReadPermitReader -where - R: AsyncRead, -{ +struct DiskReadPermitReader { + lease: Arc>>, + changed: Arc, +} + +impl DiskReadPermitReader { + fn new(inner: R, disk_permit: GetObjectDiskPermit) -> Self { + // Leave the configured upstream stall/reopen window ahead of this + // downstream lease, including the high-latency drive profile. + let idle_timeout = get_object_disk_read_timeout() + .saturating_mul(2) + .max(GET_DISK_READ_IDLE_TIMEOUT); + Self::with_idle_timeout(inner, disk_permit, idle_timeout) + } + + fn with_idle_timeout(inner: R, disk_permit: GetObjectDiskPermit, idle_timeout: Duration) -> Self { + let now = tokio::time::Instant::now(); + let lease = Arc::new(Mutex::new(DiskReadLease { + reader: Some(Box::pin(inner)), + permit: Some(disk_permit), + pending: false, + pending_capacity: 0, + closed: false, + expired: false, + last_progress: now, + window_start: now, + window_bytes: 0, + idle_timeout, + progress_window: idle_timeout.saturating_mul(2), + waker: None, + })); + let changed = Arc::new(tokio::sync::Notify::new()); + tokio::spawn(Self::watch(lease.clone(), changed.clone())); + Self { lease, changed } + } + + async fn watch(lease: Arc>>, changed: Arc) { + let (reader, permit, pending_capacity, waker) = loop { + let next_check = { + let mut state = lease.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + if !state.closed { + state.check_deadline(tokio::time::Instant::now()); + } + if state.closed || state.expired { + state.closed = true; + break ( + state.reader.take(), + state.permit.take(), + state.pending.then_some(state.pending_capacity), + state.waker.take(), + ); + } + state.next_check_delay(tokio::time::Instant::now()) + }; + tokio::select! { + _ = tokio::time::sleep(next_check) => {}, + _ = changed.notified() => {}, + } + }; + if let Some(waker) = waker { + waker.wake(); + } + if let Some(mut reader) = reader { + if let Some(capacity) = pending_capacity { + // A cancelled response must not release disk admission while a + // submitted read is still running. Complete just that pending + // read, discard its result, and never start another read. + let mut scratch = vec![0_u8; capacity.min(64 * 1024)]; + let mut buffer = ReadBuf::new(&mut scratch); + let _ = std::future::poll_fn(|cx| reader.as_mut().poll_read(cx, &mut buffer)).await; + } + drop(reader); + } + drop(permit); + } +} + +impl AsyncRead for DiskReadPermitReader { fn poll_read(self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { - let this = self.project(); - let had_capacity = buf.remaining() > 0; + if buf.remaining() == 0 { + return Poll::Ready(Ok(())); + } + let mut state = self.lease.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + if !state.closed { + state.check_deadline(tokio::time::Instant::now()); + } + if state.expired { + self.changed.notify_one(); + return Poll::Ready(Err(std::io::Error::new( + std::io::ErrorKind::TimedOut, + "GetObject disk-read admission expired", + ))); + } + if state.closed { + return Poll::Ready(Ok(())); + } + state.waker = Some(cx.waker().clone()); let filled_before = buf.filled().len(); - let poll = this.inner.poll_read(cx, buf); - // EOF: no more disk reads can happen through this stream, so release - // the permit instead of holding it until the client drops the body. - if had_capacity - && matches!(poll, Poll::Ready(Ok(()))) - && buf.filled().len() == filled_before - && let Some(mut disk_permit) = this.disk_permit.take() - { - disk_permit.release(); + state.pending_capacity = buf.remaining(); + let poll = match state.reader.as_mut() { + Some(reader) => reader.as_mut().poll_read(cx, buf), + None => return Poll::Ready(Ok(())), + }; + state.pending = poll.is_pending(); + if let Poll::Ready(result) = &poll { + let progress = buf.filled().len() - filled_before; + if result.is_err() || progress == 0 { + state.closed = true; + // The read completed, so normal EOF/error can release inline. + state.reader.take(); + state.permit.take(); + self.changed.notify_one(); + } else { + state.last_progress = tokio::time::Instant::now(); + state.window_bytes = state.window_bytes.saturating_add(progress); + } } poll } } +impl Drop for DiskReadPermitReader { + fn drop(&mut self) { + let mut state = self.lease.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + state.closed = true; + if !state.pending { + state.reader.take(); + state.permit.take(); + } + // The watchdog takes ownership of an outstanding read on cancellation. + self.changed.notify_one(); + } +} + pin_project! { struct GetObjectReaderStream { #[pin] @@ -2506,9 +2645,9 @@ impl DefaultObjectUsecase { ) -> S3Result> { let permit_wait_start = std::time::Instant::now(); let permit_wait_timeout = Self::disk_permit_wait_timeout(); - // Permits are held for the whole body transfer, so slow clients can pin - // all of them while disks are idle. Bound the wait on the primary pool - // and, on timeout, admit from a bounded degraded overflow lane. Total + // Active readers own admission until EOF, drop, or watchdog retirement. + // Bound the wait on the primary pool and, on timeout, admit from a + // bounded degraded overflow lane. Total // concurrent disk-active GETs are hard-capped at // `primary_cap + degraded_cap`; once that cap is reached we reject with // `SlowDown` instead of reading without any admission token. Never @@ -4328,6 +4467,7 @@ impl DefaultObjectUsecase { // their semantics) may still be served by a remote copy. if let Err(err) = &prepared_read && matches!(*err.code(), S3ErrorCode::NoSuchKey | S3ErrorCode::NoSuchVersion) + && replication_proxy_allowed(&req.extensions) { // Active-active replication lag window: proxy the GET to a // replication target (backlog#1675 P1-5). @@ -4905,20 +5045,29 @@ fn odm_get_output(head: &SourceHead, content_length: i64, content_range: Option< } } -/// Feeds the tee primary to the client through a channel: the primary owns a -/// `!Sync` boxed source while the response body must be `Sync`. Dropping the -/// body ends the pump, which drops the primary and hands the remaining -/// source bytes to the tee's drain task (`drain_on_primary_drop`). -fn odm_inline_client_body(primary: TeePrimary) -> StreamingBlob { +// These downstream budgets are independent of source idle progress. In +// particular, reading a little before each idle timeout cannot retain a pull +// slot indefinitely. Timeout drops the primary and lets the tee drain to disk. +const ODM_INLINE_CLIENT_IDLE_BUDGET: Duration = Duration::from_secs(30); +const ODM_INLINE_CLIENT_TOTAL_BUDGET: Duration = Duration::from_secs(120); + +/// Feeds the tee primary through a channel because its source is `!Sync`. +/// Dropping the body or exhausting a downstream budget drops the primary and +/// hands remaining bytes to the tee's drain task (`drain_on_primary_drop`). +fn odm_inline_client_body(primary: TeePrimary, idle_budget: Duration, total_budget: Duration) -> StreamingBlob { let (tx, rx) = tokio::sync::mpsc::channel::>(ODM_INLINE_CLIENT_CHANNEL_CHUNKS); spawn_traced(async move { - let mut chunks = tokio_util::io::ReaderStream::with_capacity(primary, ODM_SOURCE_BODY_CHUNK_BYTES); - while let Some(chunk) = chunks.next().await { - let failed = chunk.is_err(); - if tx.send(chunk).await.is_err() || failed { - break; + let pump = async move { + let mut chunks = tokio_util::io::ReaderStream::with_capacity(primary, ODM_SOURCE_BODY_CHUNK_BYTES); + while let Some(chunk) = chunks.next().await { + let failed = chunk.is_err(); + if !matches!(tokio::time::timeout(idle_budget, tx.send(chunk)).await, Ok(Ok(()))) || failed { + break; + } } - } + }; + // This task is scheduled independently of response body polling. + let _ = tokio::time::timeout(total_budget, pump).await; }); StreamingBlob::wrap(ReceiverStream::new(rx)) } @@ -5039,7 +5188,12 @@ async fn odm_get_inline( let (guarded, idle) = idle_guarded_body(source_body, Duration::from_millis(policy.source_timeout.idle_ms)); let (primary, secondary) = tee_reader_with_options(Box::pin(tokio_util::io::StreamReader::new(guarded)), ODM_INLINE_TEE_BUFFER_BYTES, options); - let output = Box::new(odm_get_output(&head, content_length, content_range, odm_inline_client_body(primary))); + let output = Box::new(odm_get_output( + &head, + content_length, + content_range, + odm_inline_client_body(primary, ODM_INLINE_CLIENT_IDLE_BUDGET, ODM_INLINE_CLIENT_TOTAL_BUDGET), + )); let commit_state = Arc::clone(state); let commit_key = key.to_string(); spawn_background_with_context(request_context, async move { @@ -5087,11 +5241,12 @@ pub(super) async fn odm_get_from_source( if head.size > policy.inline_max_bytes { return odm_get_passthrough(state, source, headers, key, None, Some(PullReason::LargeObject)).await; } - let slot = match state.acquire_pull_slot(key).await { - Ok(slot) => slot, - // The bucket state was torn down under this request: serve it - // without queueing anything on the old state. - Err(_) => return odm_get_passthrough(state, source, headers, key, None, None).await, + let permit_budget = Duration::from_millis(policy.source_timeout.first_byte_ms).min(Duration::from_secs(1)); + let slot = match tokio::time::timeout(permit_budget, state.acquire_pull_slot(key)).await { + Ok(Ok(slot)) => slot, + // A saturated or retired bucket state must not hold the response + // indefinitely. Dropping acquisition removes its singleflight entry. + Ok(Err(_)) | Err(_) => return odm_get_passthrough(state, source, headers, key, None, None).await, }; match slot { PullSlot::Leader(leader) => odm_get_inline(state, source, headers, key, leader, request_context).await, @@ -5411,6 +5566,85 @@ mod on_demand_migration_tests { assert!(!odm_get_may_consult_source(&proxy_marked, None)); } + #[tokio::test] + async fn odm_get_permit_wait_is_bounded_and_removes_the_abandoned_slot() { + let mut policy = PolicyConfig { + max_concurrent_pulls: 1, + ..Default::default() + }; + policy.source_timeout.first_byte_ms = 100; + let rt = runtime("permit-budget", policy).await; + let state = rt.state("permit-budget"); + let held = state.acquire_pull_slot("held").await.expect("occupy the only permit"); + let data = payload(1024); + let source = ScriptedSource::for_object(&data); + let reply = tokio::time::timeout( + Duration::from_secs(2), + odm_get_from_source(&state, &source, &HeaderMap::new(), KEY, None, None), + ) + .await + .expect("saturated inline permits must fall back to passthrough"); + let (mut output, backfill) = served(reply); + assert_eq!(backfill, None); + assert_eq!(collect_body(&mut output).await, data); + assert_eq!(state.inflight_keys(), 1, "timed-out waiter must not strand a singleflight entry"); + assert!(rt.write_back.puts().is_empty(), "fallback must not start another inline pull"); + drop(held); + assert_eq!(state.inflight_keys(), 0); + } + + #[tokio::test] + async fn odm_client_backpressure_and_trickle_cannot_retain_a_pull_slot() { + for trickle in [false, true] { + let rt = runtime("client-budget", PolicyConfig::default()).await; + let state = rt.state("client-budget"); + let leader = match state.acquire_pull_slot(KEY).await.expect("leader") { + PullSlot::Leader(leader) => leader, + PullSlot::Follower(_) => panic!("fresh key must lead"), + }; + let data = payload(4 * 1024 * 1024); + let head = source_head(&data); + let source: SourceBody = Box::pin(futures::stream::iter([Ok(Bytes::from(data.clone()))])); + let (guarded, idle) = idle_guarded_body(source, Duration::from_secs(5)); + let (primary, secondary) = tee_reader_with_options( + Box::pin(tokio_util::io::StreamReader::new(guarded)), + ODM_INLINE_TEE_BUFFER_BYTES, + TeeOptions { + drain_on_primary_drop: true, + max_drain_bytes: data.len(), + }, + ); + let mut client = odm_inline_client_body(primary, Duration::from_millis(100), Duration::from_millis(250)); + let commit_state = Arc::clone(&state); + let commit = tokio::spawn(async move { + let result = commit_inline(&commit_state, KEY, head, None, Box::pin(secondary.into_stream()), &idle).await; + leader.complete(result.map(|outcome| PullOutcome { + etag: outcome.etag, + size: outcome.size, + })); + }); + let received = if trickle { + let mut received = 0; + // Progress is faster than the idle budget but cannot reset the absolute budget. + while let Some(chunk) = client.next().await { + received += chunk.expect("client chunk").len(); + tokio::time::sleep(Duration::from_millis(40)).await; + } + received + } else { + 0 + }; + tokio::time::timeout(Duration::from_secs(3), commit) + .await + .expect("client pacing cannot stall write-back") + .expect("write-back task"); + assert_eq!(state.inflight_keys(), 0, "permit released while the unread response is still alive"); + assert_eq!(rt.write_back.puts()[0].1, data, "drain must preserve the entire source object"); + assert!(received < data.len(), "timed-out response must not pretend to deliver the full object"); + drop(client); + } + } + #[tokio::test] async fn odm_get_inline_streams_to_client_and_commits_the_same_bytes() { let rt = runtime("b", PolicyConfig::default()).await; @@ -9861,6 +10095,258 @@ mod tests { assert_eq!(GetObjectGuard::concurrent_count(), initial); } + struct AdmissionReadProbe { + reads: Arc, + } + + impl AsyncRead for AdmissionReadProbe { + fn poll_read(self: Pin<&mut Self>, _cx: &mut Context<'_>, buffer: &mut ReadBuf<'_>) -> Poll> { + self.reads.fetch_add(1, AtomicOrdering::SeqCst); + buffer.initialize_unfilled().fill(42); + buffer.advance(buffer.remaining()); + Poll::Ready(Ok(())) + } + } + + struct PendingAdmissionRead { + finish: tokio::sync::oneshot::Receiver<()>, + started: Arc, + completed: Arc, + submitted: bool, + } + + impl AsyncRead for PendingAdmissionRead { + fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buffer: &mut ReadBuf<'_>) -> Poll> { + if !self.submitted { + self.submitted = true; + self.started.fetch_add(1, AtomicOrdering::SeqCst); + } + match std::future::Future::poll(Pin::new(&mut self.finish), cx) { + Poll::Ready(_) => { + self.completed.fetch_add(1, AtomicOrdering::SeqCst); + buffer.put_slice(b"x"); + Poll::Ready(Ok(())) + } + Poll::Pending => Poll::Pending, + } + } + } + + #[tokio::test(start_paused = true)] + async fn disk_read_admission_expires_unpolled_and_cannot_resume_without_permit() { + use tokio::io::AsyncReadExt; + let semaphore = Arc::new(tokio::sync::Semaphore::new(1)); + let permit = semaphore.clone().acquire_owned().await.unwrap(); + let reads = Arc::new(AtomicUsize::new(0)); + let mut reader = DiskReadPermitReader::with_idle_timeout( + AdmissionReadProbe { reads: reads.clone() }, + permit.into(), + GET_DISK_READ_IDLE_TIMEOUT, + ); + tokio::time::advance(GET_DISK_READ_IDLE_TIMEOUT + Duration::from_secs(1)).await; + let replacement = tokio::time::timeout(Duration::from_secs(1), semaphore.clone().acquire_owned()) + .await + .expect("unpolled body must release admission independently") + .unwrap(); + let mut buffer = [0_u8; 1]; + for _ in 0..2 { + assert_eq!(reader.read(&mut buffer).await.unwrap_err().kind(), std::io::ErrorKind::TimedOut); + } + assert_eq!(reads.load(AtomicOrdering::SeqCst), 0, "expired readers must never touch disk"); + assert_eq!(semaphore.available_permits(), 0, "replacement still owns the only slot"); + let reopens = Arc::new(AtomicUsize::new(0)); + let count = reopens.clone(); + let reopen: GetObjectReopen> = Box::new(move |_| { + count.fetch_add(1, AtomicOrdering::SeqCst); + Box::pin(async { Err(GetObjectResumeFailure::Fatal) }) + }); + let resume = GetObjectResumeControl::new( + reopen, + RetryTimer::new( + GET_OBJECT_RESUME_MAX_ATTEMPTS, + Duration::from_millis(1), + Duration::from_millis(2), + rustfs_utils::retry::NO_JITTER, + 0, + ), + ); + let mut streaming = GetObjectStreamingReader::new( + reader, + "bucket", + "object", + "expired-admission", + None, + 1024, + Duration::from_secs(1), + GetObjectBodyLifecycle::disabled(), + Some(resume), + ); + assert_eq!(streaming.read(&mut buffer).await.unwrap_err().kind(), std::io::ErrorKind::TimedOut); + assert_eq!( + reopens.load(AtomicOrdering::SeqCst), + 0, + "expired admission is terminal for mid-stream resume" + ); + drop(streaming); + drop(replacement); + assert_eq!(semaphore.available_permits(), 1); + } + + #[tokio::test(start_paused = true)] + async fn disk_read_admission_rejects_trickle_but_allows_long_progressing_downloads() { + use tokio::io::AsyncReadExt; + let semaphore = Arc::new(tokio::sync::Semaphore::new(1)); + let permit = semaphore.clone().acquire_owned().await.unwrap(); + let reads = Arc::new(AtomicUsize::new(0)); + let mut reader = DiskReadPermitReader::with_idle_timeout( + AdmissionReadProbe { reads: reads.clone() }, + permit.into(), + GET_DISK_READ_IDLE_TIMEOUT, + ); + let mut byte = [0_u8; 1]; + for _ in 0..6 { + reader.read_exact(&mut byte).await.unwrap(); + tokio::time::advance(Duration::from_secs(10)).await; + } + let permit = tokio::time::timeout(Duration::from_secs(1), semaphore.clone().acquire_owned()) + .await + .expect("small progress cannot renew admission forever") + .unwrap(); + assert_eq!(reader.read(&mut byte).await.unwrap_err().kind(), std::io::ErrorKind::TimedOut); + assert_eq!(reads.load(AtomicOrdering::SeqCst), 6); + drop(reader); + let mut reader = DiskReadPermitReader::with_idle_timeout( + AdmissionReadProbe { reads: reads.clone() }, + permit.into(), + GET_DISK_READ_IDLE_TIMEOUT, + ); + let mut buffer = [0_u8; 16 * 1024]; + // More than 30 minutes is valid: deadlines constrain inactivity and + // rolling progress, not the total size or lifetime of a download. + for _ in 0..128 { + reader.read_exact(&mut buffer).await.unwrap(); + tokio::time::advance(Duration::from_secs(15)).await; + } + assert_eq!(semaphore.available_permits(), 0); + drop(reader); + assert_eq!(semaphore.available_permits(), 1); + } + + #[tokio::test(start_paused = true)] + async fn disk_read_admission_retains_pending_io_until_completion_after_expiry_or_drop() { + use tokio::io::AsyncReadExt; + for cancel in [false, true] { + let semaphore = Arc::new(tokio::sync::Semaphore::new(1)); + let permit = semaphore.clone().acquire_owned().await.unwrap(); + let (finish, receiver) = tokio::sync::oneshot::channel(); + let started = Arc::new(AtomicUsize::new(0)); + let completed = Arc::new(AtomicUsize::new(0)); + let mut reader = DiskReadPermitReader::with_idle_timeout( + PendingAdmissionRead { + finish: receiver, + started: started.clone(), + completed: completed.clone(), + submitted: false, + }, + permit.into(), + GET_DISK_READ_IDLE_TIMEOUT, + ); + let mut buffer = [0_u8; 1]; + let mut reading = Box::pin(reader.read(&mut buffer)); + assert!(futures::poll!(&mut reading).is_pending()); + drop(reading); + assert_eq!(started.load(AtomicOrdering::SeqCst), 1); + if cancel { + drop(reader); + } else { + tokio::time::advance(GET_DISK_READ_IDLE_TIMEOUT + Duration::from_secs(1)).await; + assert_eq!(reader.read(&mut buffer).await.unwrap_err().kind(), std::io::ErrorKind::TimedOut); + drop(reader); + } + tokio::task::yield_now().await; + assert_eq!(semaphore.available_permits(), 0, "pending disk IO still consumes admission"); + assert_eq!(completed.load(AtomicOrdering::SeqCst), 0); + finish.send(()).unwrap(); + let _replacement = tokio::time::timeout(Duration::from_secs(1), semaphore.clone().acquire_owned()) + .await + .expect("the outstanding read finishes without the response being polled") + .unwrap(); + assert_eq!(completed.load(AtomicOrdering::SeqCst), 1); + assert_eq!(started.load(AtomicOrdering::SeqCst), 1, "retirement must never submit a second read"); + } + } + + #[tokio::test(start_paused = true)] + async fn disk_read_admission_preserves_configured_slow_io_window() { + use tokio::io::AsyncReadExt; + let semaphore = Arc::new(tokio::sync::Semaphore::new(1)); + let permit = semaphore.clone().acquire_owned().await.unwrap(); + let (finish, receiver) = tokio::sync::oneshot::channel(); + let completed = Arc::new(AtomicUsize::new(0)); + let mut reader = DiskReadPermitReader::with_idle_timeout( + PendingAdmissionRead { + finish: receiver, + started: Arc::default(), + completed: completed.clone(), + submitted: false, + }, + permit.into(), + Duration::from_secs(120), + ); + let mut buffer = [0_u8; 1]; + let mut reading = Box::pin(reader.read(&mut buffer)); + assert!(futures::poll!(&mut reading).is_pending()); + tokio::time::advance(Duration::from_secs(45)).await; + assert!( + futures::poll!(&mut reading).is_pending(), + "a high-latency read must not inherit the default 30-second cutoff" + ); + assert_eq!(semaphore.available_permits(), 0); + finish.send(()).unwrap(); + assert_eq!(reading.await.unwrap(), 1); + assert_eq!(completed.load(AtomicOrdering::SeqCst), 1); + drop(reader); + assert_eq!(semaphore.available_permits(), 1); + } + + #[tokio::test] + async fn disk_read_admission_unread_file_bodies_cannot_exhaust_both_lanes() { + use tokio::io::AsyncReadExt; + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("download"); + let payload = vec![0x5a_u8; 256 * 1024]; + tokio::fs::write(&path, &payload).await.unwrap(); + let manager = ConcurrencyManager::with_disk_read_caps_for_test(1, 1); + let mut bodies = Vec::new(); + for _ in 0..2 { + let file = tokio::fs::File::open(&path).await.unwrap(); + let permit = match manager.admit_disk_read(Duration::from_millis(1)).await.unwrap() { + DiskReadAdmission::Primary(permit) | DiskReadAdmission::Degraded(permit) => permit, + other => panic!("expected an admitted read, got {other:?}"), + }; + bodies.push(DiskReadPermitReader::with_idle_timeout(file, permit.into(), GET_DISK_READ_IDLE_TIMEOUT)); + } + assert!(matches!( + manager.admit_disk_read(Duration::from_millis(1)).await.unwrap(), + DiskReadAdmission::Rejected + )); + tokio::time::pause(); + tokio::time::advance(GET_DISK_READ_IDLE_TIMEOUT + Duration::from_secs(1)).await; + let permit = match manager.admit_disk_read(Duration::from_secs(1)).await.unwrap() { + DiskReadAdmission::Primary(permit) => permit, + other => panic!("unread bodies must return the primary slot, got {other:?}"), + }; + tokio::time::resume(); + for body in &mut bodies { + assert_eq!(body.read(&mut [0_u8; 1]).await.unwrap_err().kind(), std::io::ErrorKind::TimedOut); + } + let file = tokio::fs::File::open(&path).await.unwrap(); + let mut reader = DiskReadPermitReader::with_idle_timeout(file, permit.into(), GET_DISK_READ_IDLE_TIMEOUT); + let mut actual = Vec::new(); + reader.read_to_end(&mut actual).await.unwrap(); + assert_eq!(actual, payload, "a fresh reader must read the real file after recovery"); + } + #[tokio::test] async fn disk_read_permit_reader_holds_permit_until_reader_is_dropped() { let semaphore = Arc::new(tokio::sync::Semaphore::new(1)); diff --git a/rustfs/src/app/object/head.rs b/rustfs/src/app/object/head.rs index 520437cd0..0d93fd948 100644 --- a/rustfs/src/app/object/head.rs +++ b/rustfs/src/app/object/head.rs @@ -16,10 +16,36 @@ use super::veeam_sosapi; use super::*; +use crate::app::storage_api::object_usecase::access::replication_proxy_allowed; use crate::on_demand_migration::{ BucketOdmState, HeadPolicy, OdmLookup, OdmOp, OdmOutcome, OnDemandMigrationSys, SourceClient, SourceError, SourceHead, }; +async fn head_metadata_action_allowed(req: &S3Request, action: S3Action) -> S3Result { + let Ok(mut info) = req_info_ref(req).cloned() else { + return Ok(false); + }; + info.suppress_denial_log = true; + info.suppress_denial_audit = true; + let mut probe = S3Request { + input: (), + method: req.method.clone(), + uri: req.uri.clone(), + headers: req.headers.clone(), + extensions: req.extensions.clone(), + credentials: req.credentials.clone(), + region: req.region.clone(), + service: req.service.clone(), + trailing_headers: req.trailing_headers.clone(), + }; + probe.extensions.insert(info); + match authorize_request(&mut probe, Action::S3Action(action)).await { + Ok(()) => Ok(true), + Err(err) if err.code() == &S3ErrorCode::AccessDenied => Ok(false), + Err(err) => Err(err), + } +} + /// Source HEAD seam for the on-demand migration passthrough: production goes /// through [`SourceClient`], tests script the answers. pub(super) trait OdmHeadSource { @@ -413,7 +439,9 @@ impl DefaultObjectUsecase { // Active-active replication lag window: an object missing // locally may still be served by proxying the HEAD to a // replication target (backlog#1675 P1-5). - if let Some(output) = Self::proxy_head_object_to_replication_targets(&req, &bucket, &key, &opts).await { + if replication_proxy_allowed(&req.extensions) + && let Some(output) = Self::proxy_head_object_to_replication_targets(&req, &bucket, &key, &opts).await + { let response = wrap_response_with_cors(&bucket, &req.method, &req.headers, output).await; let result = Ok(response); let _ = helper @@ -615,7 +643,12 @@ impl DefaultObjectUsecase { // Add x-amz-tagging-count header if object has tags // Per S3 API spec, this header should be present in HEAD object response when tags exist - if tag_count > 0 { + let tagging_action = if req.input.version_id.is_some() { + S3Action::GetObjectVersionTaggingAction + } else { + S3Action::GetObjectTaggingAction + }; + if tag_count > 0 && (replication_check || head_metadata_action_allowed(&req, tagging_action).await?) { let header_name = http::HeaderName::from_static(AMZ_TAG_COUNT); if let Ok(header_value) = tag_count.to_string().parse::() { response.headers.insert(header_name, header_value); @@ -623,9 +656,20 @@ impl DefaultObjectUsecase { warn!("Failed to parse x-amz-tagging-count header; skipping"); } } + let retention_allowed = replication_check + || ([ + AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER, + AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, + AMZ_OBJECT_LOCK_MODE_LOWER, + AMZ_OBJECT_LOCK_MODE, + ] + .iter() + .any(|key| metadata_map.contains_key(*key)) + && head_metadata_action_allowed(&req, S3Action::GetObjectRetentionAction).await?); if let Some(retain_date) = metadata_map .get(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER) .or_else(|| metadata_map.get(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE)) + && retention_allowed && let Ok(header_name) = http::HeaderName::from_bytes(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER.as_bytes()) && let Ok(header_value) = HeaderValue::from_str(retain_date) { @@ -634,6 +678,7 @@ impl DefaultObjectUsecase { if let Some(mode) = metadata_map .get(AMZ_OBJECT_LOCK_MODE_LOWER) .or_else(|| metadata_map.get(AMZ_OBJECT_LOCK_MODE)) + && retention_allowed && let Ok(header_name) = http::HeaderName::from_bytes(AMZ_OBJECT_LOCK_MODE_LOWER.as_bytes()) && let Ok(header_value) = HeaderValue::from_str(mode) { @@ -642,6 +687,7 @@ impl DefaultObjectUsecase { if let Some(legal_hold) = metadata_map .get(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER) .or_else(|| metadata_map.get(AMZ_OBJECT_LOCK_LEGAL_HOLD)) + && (replication_check || head_metadata_action_allowed(&req, S3Action::GetObjectLegalHoldAction).await?) && let Ok(header_name) = http::HeaderName::from_bytes(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER.as_bytes()) && let Ok(header_value) = HeaderValue::from_str(legal_hold) { @@ -1061,6 +1107,102 @@ mod tests { build_request(input, Method::HEAD) } + #[test] + #[serial_test::serial] + fn head_response_filters_lock_and_tag_metadata_by_permission() { + crate::app::gating_test_env::run_large_stack_test("head-metadata-permissions", || async { + use crate::app::runtime_sources::{AppContext, ServerContextSlot}; + use crate::app::storage_api::test::ReqInfo; + use crate::app::storage_api::test::S3Access as _; + use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions}; + use crate::app::storage_api::test::ecfs::FS; + let ambient = crate::app::gating_test_env::shared_gating_ambient().await; + let store = crate::app::gating_test_env::shared_gating_ecstore().await; + let context = Arc::new(AppContext::new(store.clone(), ambient.iam(), ambient.kms())); + let slot = ServerContextSlot::new(); + assert!(slot.install(context.clone())); + let fs = FS::with_server_ctx(slot.clone()); + let usecase = DefaultObjectUsecase::with_context(Some(context)); + let bucket = format!("head-metadata-{}", Uuid::new_v4().simple()); + store + .make_bucket( + &bucket, + &MakeBucketOptions { + versioning_enabled: true, + ..Default::default() + }, + ) + .await + .unwrap(); + let object = store + .put_object( + &bucket, + "protected", + &mut PutObjReader::from_vec(b"body".to_vec()), + &ObjectOptions { + versioned: true, + user_defined: HashMap::from([ + (AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER.to_owned(), "2099-01-01T00:00:00Z".to_owned()), + (AMZ_OBJECT_LOCK_MODE_LOWER.to_owned(), "GOVERNANCE".to_owned()), + (AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER.to_owned(), "ON".to_owned()), + (AMZ_OBJECT_TAGGING.to_owned(), "classification=secret".to_owned()), + ]), + ..Default::default() + }, + ) + .await + .unwrap(); + for (grant, explicit_deny, replica) in [ + (false, false, false), + (true, true, false), + (true, false, false), + (false, false, true), + ] { + let mut statements = vec![serde_json::json!({"Effect":"Allow","Principal":{"AWS":"*"}, + "Action":["s3:GetObject","s3:GetObjectVersion","s3:ReplicateObject"],"Resource":format!("arn:aws:s3:::{bucket}/*")})]; + for (enabled, effect) in [(grant, "Allow"), (explicit_deny, "Deny")] { + if enabled { + statements.push(serde_json::json!({"Effect":effect,"Principal":{"AWS":"*"}, + "Action":["s3:GetObjectRetention","s3:GetObjectLegalHold","s3:GetObjectTagging","s3:GetObjectVersionTagging"], + "Resource":format!("arn:aws:s3:::{bucket}/*")})); + } + } + store + .update_bucket_metadata_config( + &bucket, + crate::app::storage_api::object_usecase::bucket::metadata::BUCKET_POLICY_CONFIG, + serde_json::to_vec(&serde_json::json!({"Version":"2012-10-17","Statement":statements})).unwrap(), + ) + .await + .unwrap(); + for version in [None, object.version_id.map(|version| version.to_string())] { + let mut req = head_input(&bucket, "protected", version); + req.extensions.insert(ReqInfo::default()); + req.extensions.insert(slot.clone()); + if replica { + req.headers + .insert("x-rustfs-source-replication-check", HeaderValue::from_static("true")); + } + fs.head_object(&mut req).await.expect("ordinary HEAD read is permitted"); + let response = Box::pin(usecase.execute_head_object(req)).await.expect("HEAD succeeds"); + assert_eq!(response.output.content_length, Some(4)); + for header in [ + AMZ_TAG_COUNT, + AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER, + AMZ_OBJECT_LOCK_MODE_LOWER, + AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, + ] { + assert_eq!( + response.headers.contains_key(header), + replica || (grant && !explicit_deny), + "header {header}" + ); + } + } + } + }); + } + /// Drives `execute_head_object` against a real store with the global /// runtime configured as `head = local_only`: any HEAD that wrongly /// enters the runtime shows up as a `filtered` count, so a zero counter diff --git a/rustfs/src/app/object/internal_put.rs b/rustfs/src/app/object/internal_put.rs index c27d6e3bf..a1fe8e58d 100644 --- a/rustfs/src/app/object/internal_put.rs +++ b/rustfs/src/app/object/internal_put.rs @@ -129,7 +129,7 @@ fn internal_put_content_input(headers: &HeaderMap, tags: Option) -> PutO } async fn validate_internal_write_target(key: &str, bucket: &str, headers: &HeaderMap) -> Result<(), ApiError> { - validate_object_key(key, INTERNAL_PUT_METHOD_NAME).map_err(api_error_from_s3)?; + validate_new_object_key(key, INTERNAL_PUT_METHOD_NAME).map_err(api_error_from_s3)?; validate_table_catalog_object_mutation(bucket, key) .await .map_err(api_error_from_s3)?; diff --git a/rustfs/src/app/object/mod.rs b/rustfs/src/app/object/mod.rs index 9e27b2a66..7c27b0bf8 100644 --- a/rustfs/src/app/object/mod.rs +++ b/rustfs/src/app/object/mod.rs @@ -81,7 +81,9 @@ use super::storage_api::object_usecase::error::{ Error as EcstoreError, StorageError, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found, }; use super::storage_api::object_usecase::head_prefix::{head_prefix_not_found_message, probe_prefix_has_children}; -use super::storage_api::object_usecase::helper::{OperationHelper, build_event_resp_elements, spawn_background_with_context}; +use super::storage_api::object_usecase::helper::{ + OperationHelper, build_event_resp_elements, run_owned_mutation, spawn_background_with_context, +}; use super::storage_api::object_usecase::io::{DynReader, HashReader, WritePlan, compression_metadata_value, wrap_reader}; #[cfg(test)] use super::storage_api::object_usecase::object_cache::GetObjectBodySource; @@ -112,8 +114,9 @@ use super::storage_api::object_usecase::timeout_wrapper::{GetObjectTimeoutPolicy use super::storage_api::object_usecase::{ECStore, OldCurrentSize}; use super::storage_api::object_usecase::{ RFC1123, check_preconditions, parse_object_lock_legal_hold, parse_object_lock_retention, parse_part_number_i32_to_usize, - remove_object_lock_metadata_for_copy, strip_managed_encryption_metadata, validate_bucket_exists, validate_object_key, - validate_sse_headers_for_read, validate_sse_headers_for_write, validate_ssec_for_read, wrap_response_with_cors, + remove_object_lock_metadata_for_copy, strip_managed_encryption_metadata, validate_bucket_exists, validate_new_object_key, + validate_object_key, validate_sse_headers_for_read, validate_sse_headers_for_write, validate_ssec_for_read, + wrap_response_with_cors, }; use crate::app::runtime_sources::{ AppContext, current_app_context, current_notify_interface_for_context, current_object_data_cache_for_context, @@ -179,7 +182,7 @@ use s3s::dto::{ WebsiteRedirectLocation, }; use s3s::header::X_AMZ_RESTORE; -use s3s::stream::{ByteStream, DynByteStream, RemainingLength}; +use s3s::stream::{ByteStream, RemainingLength}; use s3s::{S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error}; fn object_s3_error(code: S3ErrorCode, message: impl Into>) -> S3Error { @@ -232,10 +235,7 @@ pub(crate) use self::on_demand_migration_put::*; #[cfg(test)] pub(crate) use self::put::PollCountingBody; use self::put::*; -pub(crate) use self::put::{ - guard_put_object_body_read_timeout, previous_current_size_from_backfill, put_object_body_read_timeout, - reject_oversize_single_upload, -}; +pub(crate) use self::put::{previous_current_size_from_backfill, put_object_body_read_timeout, reject_oversize_single_upload}; #[cfg(test)] pub(crate) use self::restore::RestoreStatusCommitBarrier; pub(crate) use self::shared::*; diff --git a/rustfs/src/app/object/put.rs b/rustfs/src/app/object/put.rs index c3912ff61..fb16761ea 100644 --- a/rustfs/src/app/object/put.rs +++ b/rustfs/src/app/object/put.rs @@ -15,10 +15,11 @@ //! PutObject write path: body admission, eager commit, zero-copy tuning. use super::*; +use futures::FutureExt; use crate::app::trailer_adapter::trailer_source; use crate::auth::{RUSTFS_MAX_CONTENT_LENGTH_QUERY, VerifiedPresignedRequest, parse_presigned_put_max_content_length}; -use crate::error::{ClientBodyReadTimeout, UploadLimitExceeded, normalize_read_quorum_error}; +use crate::error::{UploadLimitExceeded, normalize_read_quorum_error}; static PUT_FAILURE_LOGS: rustfs_utils::LogThrottle = rustfs_utils::LogThrottle::new(5_000); #[cfg(test)] @@ -92,11 +93,10 @@ static CACHED_SMALL_EAGER_PUT_MAX_SIZE_BYTES: std::sync::OnceLock = std:: const EVENT_PUT_OBJECT_STORE_INFLIGHT_SLOW: &str = "put_object_store_inflight_slow"; const EVENT_PUT_OBJECT_STORE_RETURNED: &str = "put_object_store_returned"; +const EVENT_PUT_OBJECT_REQUEST_COMPLETED: &str = "put_object_request_completed"; const EVENT_PUT_OBJECT_COMMIT_OWNER_DEADLINE: &str = "put_object_commit_owner_deadline"; -const EVENT_PUT_OBJECT_BODY_READ_STALLED: &str = "put_object_body_read_stalled"; - const PUT_OBJECT_STORE_WARN_THRESHOLD: Duration = Duration::from_secs(5); // Eager PUT bodies are fully materialized before the storage owner starts. On @@ -148,9 +148,8 @@ pub(crate) fn reject_oversize_single_upload(size: i64) -> S3Result<()> { /// Resolve the S3 request-body inter-chunk read timeout from the environment. /// -/// Returns `Duration::ZERO` when disabled (`RUSTFS_HTTP_REQUEST_BODY_READ_TIMEOUT=0`), -/// in which case [`guard_put_object_body_read_timeout`] passes the body through -/// untouched. +/// Zero disables inactivity detection; the independent minimum-progress +/// policy still bounds uploads that retain scarce resources. pub(crate) fn put_object_body_read_timeout() -> Duration { Duration::from_secs(rustfs_utils::get_env_u64( rustfs_config::ENV_HTTP_REQUEST_BODY_READ_TIMEOUT, @@ -158,36 +157,6 @@ pub(crate) fn put_object_body_read_timeout() -> Duration { )) } -/// A [`ByteStream`] decorator that aborts a request body whose peer stops -/// sending bytes without closing the connection. -/// -/// A well-behaved short body ends with EOF and is rejected promptly by the -/// eager/streaming readers. The failure this guards against is different: a -/// reverse proxy or CDN forwards a *partial* body and then goes silent while -/// holding the connection open, so the inner stream neither yields more bytes -/// nor reports EOF. Without a bound, RustFS would wait forever for bytes that -/// never arrive and the client eventually sees a hang/abort with no server-side -/// explanation (issue #3076). -/// -/// The timer resets on every chunk, so slow-but-progressing uploads are not -/// penalized; it only fires after `timeout` of complete silence. On timeout the -/// stall is logged with the received/expected byte counts and the read fails -/// with an `ErrorKind::TimedOut` error instead of hanging. -/// -/// `remaining_length` and `size_hint` are forwarded from the inner stream so -/// wrapping is transparent to length/content handling downstream. -struct RequestBodyReadTimeout { - inner: DynByteStream, - timeout: Duration, - timer: Option>>, - received: u64, - expected: Option, - bucket: String, - key: String, - request_id: String, - timed_out: bool, -} - /// Enforces a maximum size on the decoded request entity while preserving the /// streaming behavior of the underlying S3 body. struct MaxContentLengthStream { @@ -240,86 +209,8 @@ impl ByteStream for MaxContentLengthStream { } } -impl Stream for RequestBodyReadTimeout { - type Item = Result; - - fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - let this = self.get_mut(); - - // Once we have surfaced a stall error, treat the stream as terminated so - // we never poll the abandoned inner stream again. - if this.timed_out { - return Poll::Ready(None); - } - - match Pin::new(&mut this.inner).poll_next(cx) { - Poll::Ready(Some(Ok(chunk))) => { - this.timer = None; - this.received = this.received.saturating_add(chunk.len() as u64); - Poll::Ready(Some(Ok(chunk))) - } - Poll::Ready(other) => { - this.timer = None; - Poll::Ready(other) - } - Poll::Pending => { - if this.timeout.is_zero() { - return Poll::Pending; - } - - if this.timer.is_none() { - this.timer = Some(Box::pin(tokio::time::sleep(this.timeout))); - } - - if let Some(timer) = this.timer.as_mut() - && std::future::Future::poll(timer.as_mut(), cx).is_ready() - { - this.timer = None; - this.timed_out = true; - let expected_display = this.expected.map(|v| v.to_string()).unwrap_or_else(|| "unknown".to_string()); - warn!( - target: "rustfs::app::object_usecase", - event = EVENT_PUT_OBJECT_BODY_READ_STALLED, - component = LOG_COMPONENT_APP, - subsystem = LOG_SUBSYSTEM_OBJECT, - request_id = %this.request_id, - bucket = %this.bucket, - key = %this.key, - received_bytes = this.received, - expected_bytes = %expected_display, - timeout_secs = this.timeout.as_secs(), - state = "stall_timeout", - "PutObject request body read stalled; aborting. A proxy/CDN likely forwarded a partial body without closing the connection." - ); - return Poll::Ready(Some(Err(Box::new(std::io::Error::new( - std::io::ErrorKind::TimedOut, - ClientBodyReadTimeout { - timeout: this.timeout, - raw_bytes_received: this.received, - }, - )) as StdError))); - } - - Poll::Pending - } - } - } - - fn size_hint(&self) -> (usize, Option) { - self.inner.size_hint() - } -} - -impl ByteStream for RequestBodyReadTimeout { - fn remaining_length(&self) -> RemainingLength { - self.inner.remaining_length() - } -} - -/// Wrap an incoming request body with [`RequestBodyReadTimeout`] unless the -/// feature is disabled (`timeout == 0`), in which case the body is returned -/// untouched. `remaining_length` is preserved via [`StreamingBlob::new`]. -pub(crate) fn guard_put_object_body_read_timeout( +#[cfg(test)] +pub(super) fn guard_put_object_body_read_timeout( body: StreamingBlob, bucket: &str, key: &str, @@ -327,21 +218,8 @@ pub(crate) fn guard_put_object_body_read_timeout( expected: Option, timeout: Duration, ) -> StreamingBlob { - if timeout.is_zero() { - return body; - } - - StreamingBlob::new(RequestBodyReadTimeout { - inner: body.into(), - timeout, - timer: None, - received: 0, - expected: expected.and_then(|v| u64::try_from(v).ok()), - bucket: bucket.to_string(), - key: key.to_string(), - request_id: request_id.to_string(), - timed_out: false, - }) + super::request_body::guard_upload_progress(body, timeout, bucket, key, request_id, expected.unwrap_or_default().max(0) as u64) + .0 } pub(super) struct PooledBufferReader { @@ -700,7 +578,9 @@ async fn read_small_put_body_exact_pooled(mut body: R, size: usize, pool: &By where R: AsyncRead + Unpin, { - let mut buf = pool.acquire_buffer(size).await; + let mut buf = pool.try_acquire_buffer(size).ok_or_else(|| { + S3Error::with_message(S3ErrorCode::SlowDown, "upload buffer pool is full, please reduce your request rate") + })?; read_small_put_body_into(&mut body, &mut *buf, size).await?; Ok(buf) } @@ -1139,57 +1019,53 @@ pub(super) struct PutObjectCommitted { pub(super) sse_customer_algorithm: Option, pub(super) sse_customer_key_md5: Option, pub(super) put_extra_checksum_headers: Vec<(&'static str, String)>, - completion: PutObjectCompletion, + completion: Option, put_request_guard: PutObjectGuard, bucket: String, key: String, + concurrent_put_requests: usize, + buffer_size: usize, start_time: Instant, size: i64, use_zero_copy_eager_put_path: bool, - concurrent_put_requests: usize, - buffer_size: usize, } impl PutObjectCommitted { /// Publish the audit entry / creation event for `result`, record the /// request-level PutObject metrics and release the request guard. - pub(super) fn finish(self, result: &S3Result>) { - let Self { - completion, - mut put_request_guard, - bucket, - key, - start_time, - size, - use_zero_copy_eager_put_path, - concurrent_put_requests, - buffer_size, - .. - } = self; - let _ = completion.complete(result); - - // Record PutObject metrics via zero-copy-metrics - { - let duration_ms = start_time.elapsed().as_millis() as f64; - rustfs_io_metrics::record_put_object( - duration_ms, - size, - use_zero_copy_eager_put_path, // Track if zero-copy was enabled - ); + pub(super) fn finish(mut self, result: &S3Result>) { + if let Some(completion) = self.completion.take() { + let _ = completion.complete(result); } + } +} +impl Drop for PutObjectCommitted { + fn drop(&mut self) { + // This value exists only after the owned storage operation succeeds. + // A lost HTTP receiver must not discard the committed write's event. + if let Some(completion) = self.completion.take() { + let result = Ok(S3Response::new(())); + let _ = completion.complete(&result); + } + rustfs_io_metrics::record_put_object( + self.start_time.elapsed().as_millis() as f64, + self.size, + self.use_zero_copy_eager_put_path, + ); debug!( target: "rustfs::app::object_usecase", - component = "app", - subsystem = "object", - bucket = %bucket, - key = %key, - concurrent_put_requests, - buffer_size, + event = EVENT_PUT_OBJECT_REQUEST_COMPLETED, + component = LOG_COMPONENT_APP, + subsystem = LOG_SUBSYSTEM_OBJECT, + result = "success", + bucket = %self.bucket, + key = %self.key, + concurrent_put_requests = self.concurrent_put_requests, + buffer_size = self.buffer_size, "PutObject request completed" ); - - put_request_guard.finish_ok(); + self.put_request_guard.finish_ok(); } } @@ -1322,7 +1198,7 @@ impl DefaultObjectUsecase { let server_side_encryption = server_side_encryption.or(extract_server_side_encryption_from_headers(&req.headers)?); // Validate object key - validate_object_key(&key, request_method_name)?; + validate_new_object_key(&key, request_method_name)?; validate_table_catalog_object_mutation(&bucket, &key).await?; // Validate archive content encoding (reject when strict mode is enabled) @@ -1335,18 +1211,6 @@ impl DefaultObjectUsecase { let Some(body) = body else { return Err(s3_error!(IncompleteBody)) }; - // Guard against a proxy/CDN that forwards a partial body then goes silent - // without closing the connection: bound the inter-chunk wait so the read - // fails (with a diagnostic log) instead of hanging forever (issue #3076). - let body = { - let request_id = req - .extensions - .get::() - .map(|ctx| ctx.request_id.clone()) - .unwrap_or_default(); - guard_put_object_body_read_timeout(body, &bucket, &key, &request_id, content_length, put_object_body_read_timeout()) - }; - let body = match max_content_length { Some(limit) => StreamingBlob::new(MaxContentLengthStream { inner: body, @@ -1509,6 +1373,33 @@ impl DefaultObjectUsecase { sse_customer_key_md5, } = sse; + // The same network budget covers eager and streaming reads. Internal + // server-generated bodies do not represent a client upload. + let (body, progress_control) = match &origin { + PutObjectOrigin::S3 { req, .. } => { + let request_id = req + .extensions + .get::() + .map(|context| context.request_id.as_str()) + .unwrap_or_default(); + let (body, control) = if let Some(control) = req.extensions.get::() { + control.activate_upload(put_object_body_read_timeout(), &bucket, &key, request_id, size.max(0) as u64); + (body, control.clone()) + } else { + super::request_body::guard_upload_progress( + body, + put_object_body_read_timeout(), + &bucket, + &key, + request_id, + size.max(0) as u64, + ) + }; + (super::request_body::demand_upload_body(body, control.clone()), Some(control)) + } + PutObjectOrigin::Internal { .. } => (body, None), + }; + reject_veeam_sosapi_mutation(|| self.object_store(), bucket.as_str(), key.as_str()).await?; // The app check preserves the existing S3 error contract; the storage @@ -2006,6 +1897,11 @@ impl DefaultObjectUsecase { } reader = write_plan.apply(reader, actual_size).map_err(ApiError::from)?; + if let Some(control) = progress_control { + use rustfs_rio::HashReaderMut; + let inner = reader.take_inner(); + reader.inner = rustfs_rio::boxed_reader(super::request_body::DemandReader::new(inner, control)); + } rustfs_io_metrics::record_put_object_stage_duration_from("app_encryption_prepare", encryption_stage_start); let reader = PutObjReader::new(reader); @@ -2269,54 +2165,55 @@ impl DefaultObjectUsecase { rustfs_io_metrics::s3_http_metrics::current_s3_http_operation_context(), put_commit_future, ); - let put_commit = spawn_traced_join(put_commit_future); - let commit_stage_start = put_stage_metrics_enabled.then(Instant::now); - let put_commit_result = if let Some(cancellation) = eager_put_commit_cancellation { + let put_commit = spawn_traced_join(async move { + let commit_stage_start = put_stage_metrics_enabled.then(Instant::now); + let result = std::panic::AssertUnwindSafe(put_commit_future).catch_unwind().await; + rustfs_io_metrics::record_put_object_stage_duration_from("store_commit", commit_stage_start); + let PutObjectCommitResult { obj_info, put_versioned } = match result { + Ok(Ok(result)) => result, + Ok(Err(err)) => { + put_request_guard.finish_err(); + return Err(fail_put_object(completion, err)); + } + Err(_) => { + put_request_guard.finish_err(); + return Err(fail_put_object( + completion, + s3_error!(InternalError, "put object commit owner task panicked"), + )); + } + }; + completion = completion.object(obj_info.clone()); + if let Some(version_id) = obj_info.version_id { + completion = completion.version_id(version_id.to_string()); + } + Ok(PutObjectCommitted { + obj_info, + put_versioned, + effective_sse, + effective_kms_key_id, + sse_customer_algorithm, + sse_customer_key_md5, + put_extra_checksum_headers, + completion: Some(completion), + put_request_guard, + bucket, + key, + concurrent_put_requests, + buffer_size, + start_time, + size, + use_zero_copy_eager_put_path, + }) + }); + let result = if let Some(cancellation) = eager_put_commit_cancellation { EagerPutCommitOwner::new(put_commit, cancellation, EAGER_PUT_COMMIT_CANCELLATION_GRACE) .join() .await } else { put_commit.await }; - rustfs_io_metrics::record_put_object_stage_duration_from("store_commit", commit_stage_start); - let PutObjectCommitResult { obj_info, put_versioned } = match put_commit_result { - Ok(Ok(result)) => result, - Ok(Err(err)) => { - put_request_guard.finish_err(); - return Err(fail_put_object(completion, err)); - } - Err(err) => { - put_request_guard.finish_err(); - return Err(fail_put_object( - completion, - S3Error::with_message(S3ErrorCode::InternalError, format!("put object commit owner task failed: {err}")), - )); - } - }; - - completion = completion.object(obj_info.clone()); - if let Some(version_id) = obj_info.version_id { - completion = completion.version_id(version_id.to_string()); - } - - Ok(PutObjectCommitted { - obj_info, - put_versioned, - effective_sse, - effective_kms_key_id, - sse_customer_algorithm, - sse_customer_key_md5, - put_extra_checksum_headers, - completion, - put_request_guard, - bucket, - key, - start_time, - size, - use_zero_copy_eager_put_path, - concurrent_put_requests, - buffer_size, - }) + result.map_err(|_| s3_error!(InternalError, "put object commit owner task failed"))? } } @@ -2441,7 +2338,9 @@ mod tests { .content_length(Some(12)) .build() .expect("quorum probe input must build"); + let probe = crate::app::storage_api::object_usecase::helper::MutationCompletionProbe::install(&bucket); let mut req = build_request(input, method.clone()); + req.uri = format!("/{bucket}/{key}").parse().expect("PUT URI"); if method == Method::POST { req.extensions.insert(PostObjectRequestMarker); } @@ -2452,6 +2351,14 @@ mod tests { ) .await .expect_err("read quorum failure must refuse the upload"); + if stage == PutStorageTestStage::Commit { + probe.wait_for(1, 0).await; + let audits = probe.audits.lock().expect("audits"); + assert_eq!(audits.len(), 1); + assert_eq!(audits[0].api.status.as_deref(), Some("failure")); + assert_eq!(audits[0].api.status_code, Some(503)); + assert!(probe.events.lock().expect("events").is_empty()); + } assert_eq!(err.code(), &S3ErrorCode::ServiceUnavailable, "stage {index}, method {method}"); assert_eq!(err.status_code(), Some(http::StatusCode::SERVICE_UNAVAILABLE)); assert_eq!(err.message(), Some("The service is unavailable. Please retry.")); @@ -2842,6 +2749,7 @@ mod tests { let (store, context) = real_cold_fill_test_context().await; let bucket = format!("put-owner-tail-{}", Uuid::new_v4()); let object = "object.bin"; + let probe = crate::app::storage_api::object_usecase::helper::MutationCompletionProbe::install(&bucket); store .make_bucket(&bucket, &MakeBucketOptions::default()) .await @@ -2865,11 +2773,9 @@ mod tests { .build() .expect("PUT input must build"); let put_usecase = DefaultObjectUsecase::with_context(Some(Arc::clone(&context))); - let put = tokio::spawn(async move { - put_usecase - .execute_put_object(&FS::new(), build_request(put_input, Method::PUT)) - .await - }); + let mut request = build_request(put_input, Method::PUT); + request.uri = format!("/{bucket}/{object}").parse().expect("PUT URI"); + let put = tokio::spawn(async move { put_usecase.execute_put_object(&FS::new(), request).await }); tokio::time::timeout(Duration::from_secs(10), post_store_entered.wait()) .await @@ -2902,6 +2808,27 @@ mod tests { .await .expect("cancelled request's owned commit must still publish the object"); assert_eq!(recovered.size, i64::try_from(payload.len()).expect("test payload length must fit i64")); + probe.wait_for(1, 1).await; + let normal_input = PutObjectInput::builder() + .bucket(bucket.clone()) + .key("normal.bin".to_owned()) + .body(Some(StreamingBlob::from(s3s::Body::from(Bytes::from_static(b"normal"))))) + .content_length(Some(6)) + .build() + .expect("normal PUT"); + let mut normal_request = build_request(normal_input, Method::PUT); + normal_request.uri = format!("/{bucket}/normal.bin").parse().expect("URI"); + DefaultObjectUsecase::with_context(Some(context)) + .execute_put_object(&FS::new(), normal_request) + .await + .expect("normal response completion"); + let audits = probe.audits.lock().expect("audits"); + assert_eq!(audits.len(), 2, "both cancelled and returned commits audit exactly once"); + assert_eq!(audits[0].api.status.as_deref(), Some("success")); + let events = probe.events.lock().expect("events"); + assert_eq!(events.len(), 2, "explicit completion and Drop must not double publish"); + assert_eq!(events[0].event_name, EventName::ObjectCreatedPut); + assert_eq!(events[0].object.name, object); } #[tokio::test] @@ -3399,6 +3326,160 @@ mod tests { } } + #[tokio::test] + #[serial_test::serial] + async fn execute_put_object_expires_stalled_body_without_creating_object() { + use super::super::request_body::{BodyReadControl, ObservedBody}; + use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions}; + use http_body::Frame; + use http_body_util::StreamBody; + use tokio_stream::wrappers::UnboundedReceiverStream; + let (store, context) = real_store_test_context().await; + let bucket = format!("put-progress-{}", Uuid::new_v4().simple()); + store.make_bucket(&bucket, &MakeBucketOptions::default()).await.unwrap(); + for size in [8192, 2 * 1024 * 1024, 32 * 1024 * 1024] { + let (sender, receiver) = tokio::sync::mpsc::unbounded_channel::, std::io::Error>>(); + let control = BodyReadControl::default(); + let body = StreamingBlob::from(s3s::Body::http_body_unsync(ObservedBody::new( + StreamBody::new(UnboundedReceiverStream::new(receiver)), + control.clone(), + ))); + let key = format!("object-{size}.bin"); + let input = PutObjectInput::builder() + .bucket(bucket.clone()) + .key(key.clone()) + .body(Some(body)) + .content_length(Some(size)) + .build() + .unwrap(); + let mut request = build_request(input, Method::PUT); + request.extensions.insert(control.clone()); + let usecase = DefaultObjectUsecase::with_context(Some(Arc::clone(&context))); + let owner = tokio::spawn(async move { usecase.execute_put_object(&FS::new(), request).await }); + control.wait_until_pending_for_test().await; + control.advance_wait_for_test(Duration::from_secs(60)); + sender.send(Ok(Frame::data(Bytes::new()))).unwrap(); + let error = owner.await.unwrap().expect_err("stalled eager/zero-copy/streaming body"); + assert_eq!(error.code(), &S3ErrorCode::RequestTimeout); + assert!(is_err_object_not_found( + &store + .get_object_info(&bucket, &key, &ObjectOptions::default()) + .await + .unwrap_err() + )); + } + } + + #[tokio::test] + #[serial_test::serial] + async fn pooled_upload_keeps_admission_and_buffer_during_real_commit_after_caller_cancel() { + use crate::app::storage_api::test::set_disk::{PutObjectCommitBarrier, PutObjectCommitPause}; + use rustfs_io_core::BytesPoolConfig; + let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("upload-commit-owner", 16384).await; + let manager = ConcurrencyManager::with_put_admission_for_test(true, 1, Duration::ZERO); + let admission = manager.admit_put_object(8192).await.unwrap(); + let permit = match admission { + ForegroundWriteAdmission::Admitted(permit) => permit, + _ => panic!("initial permit"), + }; + let pool = Arc::new(BytesPool::with_config(BytesPoolConfig { + small_max: 1, + ..Default::default() + })); + let (body, control) = super::super::request_body::guard_upload_progress( + StreamingBlob::from(s3s::Body::from(Bytes::from(vec![7; 8192]))), + Duration::ZERO, + &bucket, + "object", + "request", + 8192, + ); + let buffer = + read_small_put_body_exact_pooled(StreamReader::new(body.map(|item| item.map_err(s3s_body_error_to_io))), 8192, &pool) + .await + .unwrap(); + let reader = super::super::request_body::DemandReader::new(PooledBufferReader::new(buffer, 8192), control); + let mut reader = PutObjReader::new(HashReader::from_stream(reader, 8192, 8192, None, None, false).unwrap()); + let barrier = PutObjectCommitBarrier::install(&bucket, "object", PutObjectCommitPause::BeforeQuotaRename); + let write_store = Arc::clone(&store); + let write_bucket = bucket.clone(); + let (finished, completion) = tokio::sync::oneshot::channel(); + let owner = tokio::spawn(async move { + let permit_guard = permit; + let result = write_store + .put_object(&write_bucket, "object", &mut reader, &ObjectOptions::default()) + .await; + drop(reader); + drop(permit_guard); + finished.send(result).unwrap(); + }); + barrier.wait_until_paused().await; + drop(owner); + assert!(pool.try_acquire_buffer(8192).is_none()); + assert!(matches!( + manager.admit_put_object(8192).await.unwrap(), + ForegroundWriteAdmission::Rejected + )); + barrier.release(); + completion.await.unwrap().unwrap(); + assert!(pool.try_acquire_buffer(8192).is_some()); + assert!(matches!( + manager.admit_put_object(8192).await.unwrap(), + ForegroundWriteAdmission::Admitted(_) + )); + assert_eq!( + store + .get_object_info(&bucket, "object", &ObjectOptions::default()) + .await + .unwrap() + .size, + 8192 + ); + } + + #[tokio::test(start_paused = true)] + async fn pooled_upload_exhaustion_rejects_without_allocating_and_recovers_after_stall() { + use futures::poll; + use rustfs_io_core::BytesPoolConfig; + use tokio_stream::wrappers::UnboundedReceiverStream; + let pool = BytesPool::with_config(BytesPoolConfig { + small_max: 1, + ..Default::default() + }); + let (sender, receiver) = tokio::sync::mpsc::unbounded_channel::>(); + let (body, _control) = super::super::request_body::guard_upload_progress( + StreamingBlob::wrap(UnboundedReceiverStream::new(receiver)), + Duration::ZERO, + "bucket", + "object", + "request", + 8192, + ); + let mut owner = Box::pin(read_small_put_body_exact_pooled( + StreamReader::new(body.map(|item| item.map_err(s3s_body_error_to_io))), + 8192, + &pool, + )); + assert!(poll!(owner.as_mut()).is_pending()); + let denied = read_small_put_body_exact_pooled(std::io::Cursor::new(vec![0; 8192]), 8192, &pool).await; + assert_eq!(denied.err().expect("full pool must reject immediately").code(), &S3ErrorCode::SlowDown); + for _ in 0..5 { + tokio::time::advance(Duration::from_secs(10)).await; + sender.send(Ok(Bytes::from_static(b"x"))).unwrap(); + assert!(poll!(owner.as_mut()).is_pending()); + assert!(pool.try_acquire_buffer(8192).is_none()); + } + tokio::time::advance(Duration::from_secs(10)).await; + assert_eq!(owner.await.err().expect("trickle must expire").code(), &S3ErrorCode::RequestTimeout); + let healthy = read_small_put_body_exact_pooled(std::io::Cursor::new(vec![1; 8192]), 8192, &pool) + .await + .expect("timed-out buffer must be reusable"); + assert_eq!(&healthy[..], &[1; 8192]); + assert!(pool.try_acquire_buffer(8192).is_none(), "commit reader still owns capacity"); + drop(healthy); + assert!(pool.try_acquire_buffer(8192).is_some()); + } + #[tokio::test] async fn read_small_put_body_exact_pooled_reads_exact_bytes_without_prefill() { let pool = get_concurrency_manager().bytes_pool(); @@ -4279,6 +4360,9 @@ mod tests { .set_quota_config(&bucket, BucketQuota::new(Some(LIMIT))) .await .expect("configure durable quota"); + crate::app::storage_api::test::set_disk::initialize_quota_ledger_for_test(Arc::clone(&store), &bucket) + .await + .expect("bootstrap authoritative quota usage"); let payload = Bytes::from(vec![0x42; 1024]); let input = PutObjectInput::builder() @@ -4462,7 +4546,7 @@ mod tests { #[tokio::test] #[serial_test::serial] - async fn cancelled_put_after_commit_marker_is_reconciled() { + async fn cancelled_put_before_rename_releases_marked_quota_reservation() { use crate::app::storage_api::test::set_disk::{PutObjectCommitBarrier, PutObjectCommitPause}; let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("cancelled-spawned-put-quota", 4096).await; @@ -4476,6 +4560,12 @@ mod tests { .await }); commit_barrier.wait_until_paused().await; + let mut denied = PutObjReader::from_vec(vec![0x54; 4096]); + let error = store + .put_object(&bucket, "replacement", &mut denied, &ObjectOptions::default()) + .await + .expect_err("the live commit marker must still reserve the entire quota"); + assert!(matches!(error, StorageError::QuotaExceeded { .. })); first.abort(); assert!(first.await.is_err(), "the outer request task must be cancelled"); drop(commit_barrier); @@ -4484,11 +4574,25 @@ mod tests { .get_object_info(&bucket, "object", &ObjectOptions::default()) .await .expect_err("cancelling before rename must not commit the object"); + // The owned pre-rename cancellation path proves no publication occurred, + // releases the mutation fences, and aborts the reservation before unlock. let mut replacement = PutObjReader::from_vec(vec![0x54; 4096]); store .put_object(&bucket, "replacement", &mut replacement, &ObjectOptions::default()) .await - .expect("the next admission must reap the abandoned commit marker"); + .expect("safe pre-rename rollback must release the marked reservation without a scan"); + let mut excess = PutObjReader::from_vec(vec![0x55]); + let error = store + .put_object(&bucket, "excess", &mut excess, &ObjectOptions::default()) + .await + .expect_err("the replacement must consume the reclaimed quota exactly once"); + assert!(matches!( + error, + StorageError::QuotaExceeded { + current: 4096, + limit: 4096 + } + )); } #[tokio::test] @@ -4586,6 +4690,9 @@ mod tests { .set_quota_config(&bucket, BucketQuota::new(Some(4096))) .await .expect("lower bucket quota"); + crate::app::storage_api::test::set_disk::initialize_quota_ledger_for_test(Arc::clone(&store), &bucket) + .await + .expect("bootstrap authoritative quota usage"); let mut excess = PutObjReader::from_vec(vec![0x72]); let err = store .put_object(&bucket, "excess", &mut excess, &ObjectOptions::default()) @@ -4646,6 +4753,9 @@ mod tests { .await .expect("quota update task should not panic") .expect("quota update should proceed after commit"); + crate::app::storage_api::test::set_disk::initialize_quota_ledger_for_test(Arc::clone(&store), &bucket) + .await + .expect("bootstrap authoritative quota usage"); let mut excess = PutObjReader::from_vec(vec![0x74]); let err = store diff --git a/rustfs/src/app/object/request_body.rs b/rustfs/src/app/object/request_body.rs index c59d2b86c..ffc2393d8 100644 --- a/rustfs/src/app/object/request_body.rs +++ b/rustfs/src/app/object/request_body.rs @@ -17,8 +17,10 @@ //! after its input returned Pending, so the transport cannot infer read demand. use super::{LOG_COMPONENT_APP, LOG_SUBSYSTEM_OBJECT}; +use crate::app::storage_api::request_body::{ByteStream, DynByteStream, RemainingLength, StdError, StreamingBlob}; use crate::error::ClientBodyReadTimeout; use bytes::Bytes; +use futures::{Stream, StreamExt}; use http_body::{Body, Frame, SizeHint}; use parking_lot::Mutex; use rustfs_rio::{DynReader, EtagResolvable, HashReaderDetector, HashReaderMut, Index, TryGetIndex}; @@ -33,8 +35,12 @@ use std::time::Duration; use tokio::io::{AsyncRead, ReadBuf}; use tokio::time::{Instant, Sleep}; +const EVENT_UPLOAD_BODY_PROGRESS_STALLED: &str = "upload_body_progress_stalled"; const EVENT_UPLOAD_PART_BODY_READ_STALLED: &str = "upload_part_body_read_stalled"; +pub(crate) const UPLOAD_PROGRESS_WINDOW: Duration = Duration::from_secs(60); +pub(crate) const UPLOAD_PROGRESS_BYTES: u64 = 64 * 1024; + struct ReadPolicy { timeout: Duration, bucket: String, @@ -50,13 +56,18 @@ struct ReadBudget { waiting_since: Option, waited: Duration, raw_bytes_received: u64, + progress_floor: bool, + progress_bytes: u64, + progress_waited: Duration, finished: bool, } impl ReadBudget { fn pause(&mut self) { if let Some(start) = self.waiting_since.take() { - self.waited = self.waited.saturating_add(start.elapsed()); + let elapsed = start.elapsed(); + self.waited = self.waited.saturating_add(elapsed); + self.progress_waited = self.progress_waited.saturating_add(elapsed); } self.demand = false; } @@ -73,6 +84,7 @@ struct SharedBudget { } impl BodyReadControl { + #[cfg(test)] pub(crate) fn activate( &self, timeout: Duration, @@ -99,6 +111,22 @@ impl BodyReadControl { true } + pub(crate) fn activate_upload(&self, timeout: Duration, bucket: &str, key: &str, request_id: &str, expected: u64) { + let mut state = self.0.budget.lock(); + if state.finished { + return; + } + state.policy = Some(ReadPolicy { + timeout, + bucket: bucket.to_owned(), + key: key.to_owned(), + request_id: request_id.to_owned(), + expected_decoded_bytes: expected, + }); + state.progress_floor = true; + self.0.active.store(true, Ordering::Release); + } + fn begin_read(&self) { let mut state = self.0.budget.lock(); if !state.finished { @@ -111,7 +139,7 @@ impl BodyReadControl { } fn progress(&self, bytes: usize) { - // Other HTTP operations never activate this UploadPart policy. + // Other HTTP operations never activate this upload policy. if !self.0.active.load(Ordering::Acquire) { return; } @@ -119,8 +147,17 @@ impl BodyReadControl { state.raw_bytes_received = state .raw_bytes_received .saturating_add(u64::try_from(bytes).unwrap_or(u64::MAX)); + // Charge the pending interval before resetting inactivity. Small + // frames do not reset the minimum-progress window or bank credit. + if let Some(start) = state.waiting_since.take() { + state.progress_waited = state.progress_waited.saturating_add(start.elapsed()); + } + state.progress_bytes = state.progress_bytes.saturating_add(bytes as u64); + if state.progress_bytes >= UPLOAD_PROGRESS_BYTES { + state.progress_bytes = 0; + state.progress_waited = Duration::ZERO; + } state.waited = Duration::ZERO; - state.waiting_since = None; } fn finish(&self) { @@ -141,38 +178,54 @@ impl BodyReadControl { if !state.demand || state.finished { return None; } - let remaining = timeout.saturating_sub(state.waited); + let inactivity_remaining = if timeout.is_zero() { + Duration::MAX + } else { + timeout.saturating_sub(state.waited) + }; + let remaining = if state.progress_floor { + inactivity_remaining.min(UPLOAD_PROGRESS_WINDOW.saturating_sub(state.progress_waited)) + } else { + inactivity_remaining + }; let start = *state.waiting_since.get_or_insert_with(Instant::now); // A timeout beyond the clock's representable range cannot elapse. start.checked_add(remaining) } fn expire(&self) -> Option { - let (policy, raw_bytes_received) = { + let (policy, raw_bytes_received, progress_floor) = { let mut state = self.0.budget.lock(); let timeout = state.policy.as_ref()?.timeout; - if !state.demand || state.waited.saturating_add(state.waiting_since?.elapsed()) < timeout { + let elapsed = state.waiting_since?.elapsed(); + let inactivity_expired = !timeout.is_zero() && state.waited.saturating_add(elapsed) >= timeout; + let progress_expired = + state.progress_floor && state.progress_waited.saturating_add(elapsed) >= UPLOAD_PROGRESS_WINDOW; + if !state.demand || (!inactivity_expired && !progress_expired) { return None; } + if progress_expired { + state.policy.as_mut()?.timeout = UPLOAD_PROGRESS_WINDOW; + } state.finished = true; self.0.active.store(false, Ordering::Release); state.demand = false; state.waiting_since = None; - (state.policy.take()?, state.raw_bytes_received) + (state.policy.take()?, state.raw_bytes_received, state.progress_floor) }; tracing::error!( - event = EVENT_UPLOAD_PART_BODY_READ_STALLED, + event = if progress_floor { EVENT_UPLOAD_BODY_PROGRESS_STALLED } else { EVENT_UPLOAD_PART_BODY_READ_STALLED }, component = LOG_COMPONENT_APP, subsystem = LOG_SUBSYSTEM_OBJECT, state = "stall_timeout", - operation = "UploadPart", + operation = if progress_floor { "Upload" } else { "UploadPart" }, request_id = %policy.request_id, bucket = %policy.bucket, key = %policy.key, raw_bytes_received, expected_decoded_bytes = policy.expected_decoded_bytes, timeout_secs = policy.timeout.as_secs(), - "UploadPart request body read stalled" + "Upload request body read budget exhausted" ); Some(ClientBodyReadTimeout { timeout: policy.timeout, @@ -245,6 +298,11 @@ impl + Unpin> Body for ObservedBody { if self.ended { return Poll::Ready(None); } + let already_waiting = self.control.0.budget.lock().waiting_since.is_some(); + if already_waiting && let Some(error) = self.poll_timeout(cx) { + self.ended = true; + return Poll::Ready(Some(Err(ObservedBodyError::Inactivity(error)))); + } // Ignore empty data without resetting the budget. Bound work per poll // even if a body repeatedly returns immediately-ready empty frames. for _ in 0..32 { @@ -308,18 +366,18 @@ impl Drop for ObservedBody { /// Current erasure readers are owned by their read future/producer: canceling /// that owner drops this reader. A future retained-reader cancellation path /// must explicitly pause its read demand before retaining the reader. -pub(crate) struct DemandReader { - inner: DynReader, +pub(crate) struct DemandReader { + inner: R, control: BodyReadControl, } -impl DemandReader { - pub(crate) fn new(inner: DynReader, control: BodyReadControl) -> Self { +impl DemandReader { + pub(crate) fn new(inner: R, control: BodyReadControl) -> Self { Self { inner, control } } } -impl AsyncRead for DemandReader { +impl AsyncRead for DemandReader { fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { self.control.begin_read(); let result = Pin::new(&mut self.inner).poll_read(cx, buf); @@ -330,13 +388,13 @@ impl AsyncRead for DemandReader { } } -impl Drop for DemandReader { +impl Drop for DemandReader { fn drop(&mut self) { self.control.finish(); } } -impl EtagResolvable for DemandReader { +impl EtagResolvable for DemandReader { fn is_etag_reader(&self) -> bool { self.inner.is_etag_reader() } @@ -345,7 +403,7 @@ impl EtagResolvable for DemandReader { } } -impl HashReaderDetector for DemandReader { +impl HashReaderDetector for DemandReader { fn is_hash_reader(&self) -> bool { self.inner.is_hash_reader() } @@ -354,11 +412,145 @@ impl HashReaderDetector for DemandReader { } } -impl TryGetIndex for DemandReader { +impl TryGetIndex for DemandReader { fn try_get_index(&self) -> Option<&Index> { self.inner.try_get_index() } } +/// Establish demand for eager reads and archive sniffing before the final +/// transformed reader exists. Pending demand is subsequently paused by the +/// outer DemandReader when a transform returns buffered output. +pub(crate) fn demand_upload_body(body: StreamingBlob, control: BodyReadControl) -> StreamingBlob { + StreamingBlob::new(DemandStream { + inner: body.into(), + control, + }) +} +struct DemandStream { + inner: DynByteStream, + control: BodyReadControl, +} +impl Stream for DemandStream { + type Item = Result; + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + self.control.begin_read(); + let result = self.inner.poll_next_unpin(cx); + if result.is_ready() { + self.control.pause_read(); + } + result + } + fn size_hint(&self) -> (usize, Option) { + self.inner.size_hint() + } +} +impl ByteStream for DemandStream { + fn remaining_length(&self) -> RemainingLength { + self.inner.remaining_length() + } +} + +/// Fallback for direct protocol callers without the raw HTTP observation. +/// HTTP uses ObservedBody because a signed chunk may not yield decoded output +/// until its full payload has arrived and its signature has been verified. +/// The final DemandReader pauses pending reads when transforms return output. +pub(crate) fn guard_upload_progress( + body: StreamingBlob, + timeout: Duration, + bucket: &str, + key: &str, + request_id: &str, + expected: u64, +) -> (StreamingBlob, BodyReadControl) { + let control = BodyReadControl::default(); + control.activate_upload(timeout, bucket, key, request_id, expected); + let stream = UploadProgressStream { + inner: body.into(), + control: control.clone(), + timer: None, + ended: false, + }; + (StreamingBlob::new(stream), control) +} + +struct UploadProgressStream { + inner: DynByteStream, + control: BodyReadControl, + timer: Option>>, + ended: bool, +} + +impl UploadProgressStream { + fn poll_timeout(&mut self, cx: &mut Context<'_>) -> Option { + let deadline = self.control.waiting_deadline()?; + let timer = self.timer.get_or_insert_with(|| Box::pin(tokio::time::sleep_until(deadline))); + if timer.deadline() != deadline { + timer.as_mut().reset(deadline); + } + timer.as_mut().poll(cx).is_ready().then(|| self.control.expire()).flatten() + } +} + +impl Stream for UploadProgressStream { + type Item = Result; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + if self.ended { + return Poll::Ready(None); + } + self.control.begin_read(); + // An overdue read cannot be revived by a late frame (including EOF). + // Do not start a wait until the source actually stalls. + let already_waiting = self.control.0.budget.lock().waiting_since.is_some(); + if already_waiting && let Some(error) = self.poll_timeout(cx) { + self.ended = true; + return Poll::Ready(Some(Err(Box::new(std::io::Error::new(std::io::ErrorKind::TimedOut, error))))); + } + let mut empty_frames_only = true; + for _ in 0..32 { + match self.inner.poll_next_unpin(cx) { + Poll::Ready(Some(Ok(bytes))) if !bytes.is_empty() => { + self.control.progress(bytes.len()); + self.control.pause_read(); + return Poll::Ready(Some(Ok(bytes))); + } + Poll::Ready(Some(Ok(_))) => {} + Poll::Ready(other) => { + self.ended = true; + self.control.finish(); + return Poll::Ready(other); + } + Poll::Pending => { + empty_frames_only = false; + break; + } + } + } + if let Some(error) = self.poll_timeout(cx) { + self.ended = true; + return Poll::Ready(Some(Err(Box::new(std::io::Error::new(std::io::ErrorKind::TimedOut, error))))); + } + // Also bound an always-ready source of empty frames. + if empty_frames_only { + cx.waker().wake_by_ref(); + } + Poll::Pending + } + fn size_hint(&self) -> (usize, Option) { + self.inner.size_hint() + } +} +impl ByteStream for UploadProgressStream { + fn remaining_length(&self) -> RemainingLength { + self.inner.remaining_length() + } +} +impl Drop for UploadProgressStream { + fn drop(&mut self) { + self.control.finish(); + } +} + #[cfg(test)] mod tests; diff --git a/rustfs/src/app/object/request_body/tests.rs b/rustfs/src/app/object/request_body/tests.rs index 8cb2cefab..9ea4f53e7 100644 --- a/rustfs/src/app/object/request_body/tests.rs +++ b/rustfs/src/app/object/request_body/tests.rs @@ -28,6 +28,19 @@ mod protocol; type FrameSender = mpsc::UnboundedSender, io::Error>>; impl BodyReadControl { + pub(crate) async fn wait_until_pending_for_test(&self) { + tokio::time::timeout(Duration::from_secs(30), async { + loop { + if self.0.budget.lock().waiting_since.is_some() { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("body must reach a pending network read"); + } + /// Elapse this body's active wait without advancing storage timers. /// The fixture must wake the raw body afterward so it polls the deadline. pub(crate) fn advance_wait_for_test(&self, elapsed: Duration) { @@ -37,6 +50,7 @@ impl BodyReadControl { "body must already be waiting for raw input" ); state.waited = state.waited.saturating_add(elapsed); + state.progress_waited = state.progress_waited.saturating_add(elapsed); } } @@ -277,3 +291,184 @@ async fn native_body_errors_preserve_their_original_source() { let error = reader.read_to_end(&mut Vec::new()).await.expect_err("native error"); assert_eq!(ApiError::from(error).code, S3ErrorCode::InternalError); } + +fn progress_reader() -> (mpsc::UnboundedSender>, impl AsyncRead + Unpin) { + let (sender, receiver) = mpsc::unbounded_channel(); + let (stream, control) = guard_upload_progress( + StreamingBlob::wrap(UnboundedReceiverStream::new(receiver)), + Duration::ZERO, + "bucket", + "object", + "request", + 1024 * 1024, + ); + let inner = StreamReader::new(stream.map(|item| item.map_err(io::Error::other))); + (sender, DemandReader::new(inner, control)) +} + +#[tokio::test(start_paused = true)] +async fn upload_progress_rejects_trickle_empty_frames_and_late_revival() { + for empty in [false, true] { + let (sender, mut reader) = progress_reader(); + let mut output = Vec::new(); + let mut read = Box::pin(reader.read_to_end(&mut output)); + assert!(poll!(read.as_mut()).is_pending()); + for _ in 0..5 { + tokio::time::advance(Duration::from_secs(10)).await; + sender + .send(Ok(if empty { Bytes::new() } else { Bytes::from_static(b"x") })) + .unwrap(); + assert!(poll!(read.as_mut()).is_pending()); + } + tokio::time::advance(Duration::from_secs(10)).await; + // Even a full qualifying chunk arriving at the deadline is too late. + sender.send(Ok(Bytes::from(vec![0; UPLOAD_PROGRESS_BYTES as usize]))).unwrap(); + let error = read.await.expect_err("small or empty frames must not renew the window"); + assert_eq!(ApiError::from(error).code, S3ErrorCode::RequestTimeout); + let mut byte = [0]; + assert_eq!(reader.read(&mut byte).await.unwrap(), 0, "expired reader cannot resume"); + } +} + +#[tokio::test(start_paused = true)] +async fn upload_progress_allows_long_uploads_without_banking_burst_credit() { + let (sender, mut reader) = progress_reader(); + let mut output = Vec::new(); + let mut read = Box::pin(reader.read_to_end(&mut output)); + assert!(poll!(read.as_mut()).is_pending()); + for _ in 0..80 { + tokio::time::advance(Duration::from_secs(59)).await; + sender.send(Ok(Bytes::from(vec![7; UPLOAD_PROGRESS_BYTES as usize]))).unwrap(); + assert!(poll!(read.as_mut()).is_pending()); + } + drop(sender); + assert_eq!( + read.await.unwrap(), + usize::try_from(80 * UPLOAD_PROGRESS_BYTES).expect("bounded upload fixture size") + ); + + let (sender, mut reader) = progress_reader(); + let mut output = Vec::new(); + let mut read = Box::pin(reader.read_to_end(&mut output)); + sender + .send(Ok(Bytes::from(vec![7; 4 * UPLOAD_PROGRESS_BYTES as usize]))) + .unwrap(); + assert!(poll!(read.as_mut()).is_pending()); + tokio::time::advance(UPLOAD_PROGRESS_WINDOW).await; + assert_eq!(ApiError::from(read.await.unwrap_err()).code, S3ErrorCode::RequestTimeout); +} + +#[tokio::test(start_paused = true)] +async fn upload_progress_excludes_storage_and_buffered_transform_backpressure() { + // A transform may return buffered output after polling its input Pending. + struct BufferedOutput { + inner: R, + first: bool, + } + impl AsyncRead for BufferedOutput { + fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { + let result = Pin::new(&mut self.inner).poll_read(cx, buf); + if result.is_pending() && self.first { + self.first = false; + buf.put_slice(b"buffered"); + return Poll::Ready(Ok(())); + } + result + } + } + let (sender, receiver) = mpsc::unbounded_channel::>(); + let (stream, control) = guard_upload_progress( + StreamingBlob::wrap(UnboundedReceiverStream::new(receiver)), + Duration::from_secs(300), + "bucket", + "object", + "request", + 8, + ); + let inner = StreamReader::new(stream.map(|item| item.map_err(io::Error::other))); + let mut reader = DemandReader::new(BufferedOutput { inner, first: true }, control); + let mut buffer = [0; 8]; + assert_eq!(reader.read(&mut buffer).await.unwrap(), 8); + tokio::time::advance(Duration::from_secs(3600)).await; + sender.send(Ok(Bytes::from_static(b"uploaded"))).unwrap(); + assert_eq!(reader.read(&mut buffer).await.unwrap(), 8); + assert_eq!(&buffer, b"uploaded"); + drop(sender); + assert_eq!(reader.read(&mut buffer).await.unwrap(), 0); +} + +#[tokio::test(start_paused = true)] +async fn upload_progress_reclaims_foreground_and_multipart_admission_after_trickle() { + use crate::app::storage_api::object_usecase::concurrency::{ConcurrencyManager, ForegroundWriteAdmission}; + let manager = ConcurrencyManager::with_put_admission_for_test(true, 1, Duration::ZERO); + for multipart in [false, true] { + let admission = if multipart { + manager.admit_multipart_part(65536).await.unwrap() + } else { + manager.admit_put_object(65536).await.unwrap() + }; + let permit = match admission { + ForegroundWriteAdmission::Admitted(permit) => permit, + _ => panic!("initial admission"), + }; + let (sender, mut reader) = progress_reader(); + let mut owner = Box::pin(async move { + let _permit = permit; + let mut output = Vec::new(); + reader.read_to_end(&mut output).await + }); + assert!(poll!(owner.as_mut()).is_pending()); + assert!(matches!( + manager.admit_put_object(65536).await.unwrap(), + ForegroundWriteAdmission::Rejected + )); + for _ in 0..5 { + tokio::time::advance(Duration::from_secs(10)).await; + sender.send(Ok(Bytes::from_static(b"x"))).unwrap(); + assert!(poll!(owner.as_mut()).is_pending()); + } + tokio::time::advance(Duration::from_secs(10)).await; + assert!(owner.await.is_err()); + assert!(matches!( + manager.admit_multipart_part(65536).await.unwrap(), + ForegroundWriteAdmission::Admitted(_) + )); + } +} + +#[tokio::test(start_paused = true)] +async fn upload_progress_does_not_release_an_owned_commit_during_storage_io_or_cancellation() { + use crate::app::storage_api::object_usecase::concurrency::{ConcurrencyManager, ForegroundWriteAdmission}; + let manager = ConcurrencyManager::with_put_admission_for_test(true, 1, Duration::ZERO); + let permit = match manager.admit_put_object(65536).await.unwrap() { + ForegroundWriteAdmission::Admitted(permit) => permit, + _ => panic!("initial admission"), + }; + let (sender, mut reader) = progress_reader(); + sender.send(Ok(Bytes::from_static(b"x"))).unwrap(); + let (entered, at_io) = tokio::sync::oneshot::channel(); + let (release, released) = tokio::sync::oneshot::channel(); + let owner = tokio::spawn(async move { + let _permit = permit; + let mut byte = [0]; + reader.read_exact(&mut byte).await.unwrap(); + entered.send(()).unwrap(); + released.await.unwrap(); + reader.read_exact(&mut byte).await + }); + at_io.await.unwrap(); + // Dropping the request's join handle must leave the commit owner intact. + drop(owner); + tokio::time::advance(Duration::from_secs(3600)).await; + assert!(matches!( + manager.admit_put_object(65536).await.unwrap(), + ForegroundWriteAdmission::Rejected + )); + sender.send(Ok(Bytes::from_static(b"y"))).unwrap(); + release.send(()).unwrap(); + tokio::task::yield_now().await; + assert!(matches!( + manager.admit_put_object(65536).await.unwrap(), + ForegroundWriteAdmission::Admitted(_) + )); +} diff --git a/rustfs/src/app/object/request_body/tests/protocol.rs b/rustfs/src/app/object/request_body/tests/protocol.rs index 431a53fd9..93fb4c0ed 100644 --- a/rustfs/src/app/object/request_body/tests/protocol.rs +++ b/rustfs/src/app/object/request_body/tests/protocol.rs @@ -38,7 +38,7 @@ impl S3 for Consumer { .get::() .expect("HTTP control extension") .clone(); - control.activate(Duration::from_secs(300), "test-bucket", "test-key", "request", expected as u64); + control.activate_upload(Duration::from_secs(300), "test-bucket", "test-key", "request", expected as u64); let stream = req.input.body.expect("body"); let inner = rustfs_rio::wrap_reader(StreamReader::new(stream.map(|item| item.map_err(io::Error::other)))); let mut reader = @@ -207,7 +207,7 @@ fn signed_request(payload: &[u8], unsigned_trailer: bool) -> SignedRequest { #[tokio::test(start_paused = true)] async fn signed_chunk_with_raw_progress_survives_eight_minutes_without_decoded_output() { - let payload = vec![7; 65536]; + let payload = vec![7; 1024 * 1024]; let SignedRequest { request, sender, @@ -220,13 +220,13 @@ async fn signed_chunk_with_raw_progress_survives_eight_minutes_without_decoded_o sender.send(Ok(Frame::data(prefix))).expect("body receiver"); assert!(poll!(call.as_mut()).is_pending()); let start = Instant::now(); - for chunk in payload.chunks(8192) { + for chunk in payload.chunks(65536) { assert_eq!( consumer.received.load(Ordering::Relaxed), 0, "incomplete chunks must not escape signature validation" ); - tokio::time::advance(Duration::from_secs(60)).await; + tokio::time::advance(Duration::from_secs(30)).await; sender .send(Ok(Frame::data(Bytes::copy_from_slice(chunk)))) .expect("body receiver"); @@ -356,3 +356,27 @@ async fn trailer_adapter_is_pending_until_the_decoded_body_ends() { assert_eq!(trailer.lookup("x-amz-checksum-sha256"), TrailerValue::Missing); assert_eq!(*consumer.committed.lock(), Some(payload.to_vec())); } + +#[tokio::test(start_paused = true)] +async fn signed_chunk_trickle_cannot_renew_upload_progress_window() { + let SignedRequest { + request, sender, prefix, .. + } = signed_request(&[7; 65536], false); + let consumer = Consumer::default(); + let service = service(consumer.clone()); + sender.send(Ok(Frame::data(prefix))).unwrap(); + let mut call = Box::pin(service.call(request)); + assert!(poll!(call.as_mut()).is_pending()); + for _ in 0..5 { + tokio::time::advance(Duration::from_secs(10)).await; + sender.send(Ok(Frame::data(Bytes::from_static(&[7])))).unwrap(); + assert!(poll!(call.as_mut()).is_pending()); + assert_eq!(consumer.received.load(Ordering::Relaxed), 0); + } + tokio::time::advance(Duration::from_secs(10)).await; + let response = call.await.expect("S3 response"); + assert_eq!(response.status(), http::StatusCode::BAD_REQUEST); + let body = BodyExt::collect(response.into_body()).await.unwrap().to_bytes(); + assert!(String::from_utf8_lossy(&body).contains("RequestTimeout")); + assert!(consumer.committed.lock().is_none()); +} diff --git a/rustfs/src/app/object/shared.rs b/rustfs/src/app/object/shared.rs index 3e36bd7ca..fe02ac4d8 100644 --- a/rustfs/src/app/object/shared.rs +++ b/rustfs/src/app/object/shared.rs @@ -700,7 +700,8 @@ fn build_put_object_expiration_header(event: &lifecycle::Event) -> Option ObjectOptions { @@ -1955,6 +1956,27 @@ mod tests { assert!(build_put_object_expiration_header(&event).is_none()); } + #[test] + fn expiration_header_ignores_unsafe_legacy_rule_ids() { + let mut event = lifecycle::Event { + action: lifecycle::IlmAction::DeleteAction, + rule_id: String::new(), + due: Some(OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid expiry")), + noncurrent_days: 0, + newer_noncurrent_versions: 0, + storage_class: String::new(), + expiry_policy: None, + }; + for id in ["rule\nname", "rule\rname", "rule\0name", "rule\u{7f}name"] { + event.rule_id = id.to_owned(); + assert!(build_put_object_expiration_header(&event).is_none(), "{id:?}"); + } + event.rule_id = "保留策略".to_owned(); + let header = build_put_object_expiration_header(&event).expect("valid UTF-8 IDs remain supported"); + assert!(http::HeaderValue::from_str(&header).is_ok()); + assert!(header.contains("保留策略")); + } + #[test] fn build_put_object_expiration_header_formats_expected_value() { let expire_time = OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(); diff --git a/rustfs/src/app/object/veeam_sosapi/tests.rs b/rustfs/src/app/object/veeam_sosapi/tests.rs index e7754ddfb..a27ec6fa9 100644 --- a/rustfs/src/app/object/veeam_sosapi/tests.rs +++ b/rustfs/src/app/object/veeam_sosapi/tests.rs @@ -877,6 +877,7 @@ fn veeam_sosapi_http_contract() { ) .await .expect("set quota on fresh bucket"); + crate::app::storage_api::test::set_disk::initialize_quota_ledger_for_test(Arc::clone(&store), "a_bucket).await.expect("bootstrap quota usage"); snapshot.last_update = Some(std::time::SystemTime::now() + Duration::from_secs(2)); snapshot.buckets_count = 2; let quota_incarnation = store.bucket_incarnation_id_from_disk("a_bucket).await.expect("quota bucket incarnation"); diff --git a/rustfs/src/app/runtime_sources.rs b/rustfs/src/app/runtime_sources.rs index 389d97517..221db837a 100644 --- a/rustfs/src/app/runtime_sources.rs +++ b/rustfs/src/app/runtime_sources.rs @@ -22,6 +22,8 @@ pub(crate) use crate::runtime_sources::{ current_notification_system_for_context, current_object_data_cache_handle_for_context, current_object_store_handle_for_context, }; +#[cfg(test)] +pub(crate) use crate::runtime_sources::{IamInterface, KmsInterface}; use rustfs_s3select_api::{QueryResult, server::dbms::DatabaseManagerSystem}; use s3s::dto::SelectObjectContentInput; use std::sync::Arc; diff --git a/rustfs/src/app/select_object.rs b/rustfs/src/app/select_object.rs index 3ac691a26..fdbb0b790 100644 --- a/rustfs/src/app/select_object.rs +++ b/rustfs/src/app/select_object.rs @@ -1781,6 +1781,81 @@ mod tests { assert!(error.source().is_none()); } + #[test] + #[serial_test::serial] + fn unread_select_response_does_not_block_real_overwrite() { + crate::app::gating_test_env::run_large_stack_test("select-unread-writer", || async { + use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions}; + use crate::app::storage_api::test::contract::object::ObjectIO as _; + let store = crate::app::gating_test_env::shared_gating_ecstore().await; + crate::app::runtime_sources::install_test_app_context(Arc::clone(&store)).await; + let bucket = format!("select-unread-{}", uuid::Uuid::new_v4().simple()); + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("bucket"); + let payload = b"original,123456789012345678901234567890\n".repeat(100_000); + let mut initial = crate::app::storage_api::object_usecase::StoragePutObjReader::from_vec(payload); + store + .put_object(&bucket, "input.csv", &mut initial, &Default::default()) + .await + .expect("CSV fixture"); + let snapshot = store + .prepare_select_object_snapshot(&bucket, "input.csv", &HeaderMap::new(), &Default::default()) + .await + .expect("snapshot for terminal consistency fence"); + let snapshot = Arc::new(snapshot); + let mut input = base_input(); + input.bucket = bucket.clone(); + input.key = "input.csv".to_owned(); + let request = S3Request { + input, + method: http::Method::POST, + uri: format!("/{bucket}/input.csv?select&select-type=2").parse().expect("URI"), + headers: HeaderMap::new(), + extensions: http::Extensions::new(), + credentials: None, + region: None, + service: None, + trailing_headers: None, + }; + let response = execute_select_object_content(request) + .await + .expect("start actual SELECT response"); + // Keep the response alive without polling its event stream. The + // producer may fill its bounded channel, but cannot retain object locks. + let mut replacement = + crate::app::storage_api::object_usecase::StoragePutObjReader::from_vec(b"replacement,2\n".to_vec()); + tokio::time::timeout( + Duration::from_secs(10), + store.put_object(&bucket, "input.csv", &mut replacement, &Default::default()), + ) + .await + .expect("unread SELECT must not block writer") + .expect("normal overwrite"); + drop(response); + assert!(snapshot.open_reader(None).await.is_err(), "first range must reject the real overwrite"); + let schema = Arc::new(Schema::new(vec![Field::new("value", DataType::Utf8, false)])); + let output = Box::pin(RecordBatchStreamAdapter::new( + schema, + futures::stream::empty::>(), + )); + let (mut events, mut rx) = test_event_channel(4); + let outcome = send_select_events( + output, + &mut events, + csv_validation(), + Arc::new(SelectInputMetrics::default()), + Instant::now() + Duration::from_secs(30), + 30, + &snapshot, + ) + .await; + assert!(matches!(outcome, SelectProducerOutcome::Terminal(Err(_)))); + assert!(rx.try_recv().is_err(), "a real changed-generation fence must never emit Stats or End"); + }); + } + fn base_input() -> SelectObjectContentInput { SelectObjectContentInput { bucket: "bucket".to_string(), diff --git a/rustfs/src/app/storage_api.rs b/rustfs/src/app/storage_api.rs index af50ccd5c..bc72eed38 100644 --- a/rustfs/src/app/storage_api.rs +++ b/rustfs/src/app/storage_api.rs @@ -24,6 +24,13 @@ pub(crate) fn EndpointServerPools( crate::storage::storage_api::EndpointServerPools::from(pools) } +/// Streaming body types the request-body adapters implement against. +pub(crate) mod request_body { + pub(crate) use s3s::StdError; + pub(crate) use s3s::dto::StreamingBlob; + pub(crate) use s3s::stream::{ByteStream, DynByteStream, RemainingLength}; +} + #[cfg(feature = "diagnostic-read-version-caller")] pub(crate) mod diagnostic { pub(crate) use crate::storage::storage_api::ecstore_diagnostic::{ @@ -303,11 +310,11 @@ pub(crate) mod access { pub(crate) use crate::storage::storage_api::access_consumer::{ PostObjectRequestMarker, TABLE_DATA_PLANE_LIST_CURSOR_PREFIX, TableDataPlaneListAccess, TableDataPlaneListCursorPosition, TableDataPlanePublicationGuards, apply_bucket_generation_guard, apply_copy_source_bucket_generation_guard, - authorize_request, bucket_config_mutation_incarnation, delete_object_authorize_action, has_bypass_governance_header, - load_bucket_generation_from_store, log_list_buckets_iam_implicit_deny, odm_read_generation, + authorize_copy_object_tags, authorize_request, bucket_config_mutation_incarnation, delete_object_authorize_action, + has_bypass_governance_header, load_bucket_generation_from_store, log_list_buckets_iam_implicit_deny, odm_read_generation, prepare_list_buckets_iam_authorization, prepare_odm_read_generation, recursive_force_delete_has_authenticated_caller, - replication_request_authorized, req_info_mut, req_info_ref, retained_table_data_plane_publication_guards, - set_requested_object_lock_retain_until, + replication_proxy_allowed, replication_request_authorized, req_info_mut, req_info_ref, + retained_table_data_plane_publication_guards, set_requested_object_lock_retain_until, }; } @@ -1130,13 +1137,17 @@ pub(crate) mod head_prefix { } pub(crate) mod helper { + #[cfg(test)] pub(crate) use crate::storage::storage_api::helper_consumer::{ - OperationHelper, build_event_resp_elements, spawn_background_with_context, + MutationCompletionProbe, capture_mutation_event_for_test, pause_committed_mutation_for_test, + }; + pub(crate) use crate::storage::storage_api::helper_consumer::{ + OperationHelper, build_event_resp_elements, run_owned_mutation, spawn_background_with_context, }; } pub(crate) mod object_utils { - pub(crate) use crate::storage::storage_api::to_s3s_etag; + pub(crate) use crate::storage::storage_api::{to_s3s_etag, to_s3s_xml_etag}; } pub(crate) mod io { @@ -1167,14 +1178,15 @@ pub(crate) mod request_context { pub(crate) mod sse { pub(crate) use crate::storage::storage_api::sse_consumer::{ DecryptionRequest, EncryptionRequest, PrepareEncryptionRequest, SseKmsPrincipal, apply_bucket_default_lock_retention, - authorize_sse_kms_object_read, classify_sse_read_response, extract_server_side_encryption_from_headers, - get_buffer_size_opt_in, load_bucket_object_lock_config_state, project_sse_read_response_headers, sse_decryption, - sse_encryption, sse_prepare_encryption, validate_bucket_object_lock_enabled_state, + authorize_sse_kms_multipart_write, authorize_sse_kms_object_read, classify_sse_read_response, + extract_server_side_encryption_from_headers, get_buffer_size_opt_in, load_bucket_object_lock_config_state, + project_sse_read_response_headers, sse_decryption, sse_encryption, sse_prepare_encryption, + validate_bucket_object_lock_enabled_state, }; pub(crate) use crate::storage::storage_api::sse_consumer::{ EncryptionKeyKind, bucket_default_write_sse, build_ssec_read_headers, encryption_material_to_metadata, extract_ssec_params_from_headers, extract_ssekms_context_from_headers, map_get_object_reader_error, - mark_encrypted_multipart_metadata, + mark_encrypted_multipart_metadata, multipart_completion_authorization, stored_ssec_metadata, }; } @@ -1213,7 +1225,7 @@ pub(crate) mod s3_api { pub(crate) mod tagging { pub(crate) use crate::storage::storage_api::s3_api_consumer::tagging::{ - parse_copy_object_tags, resolve_copy_object_tags, + parse_copy_object_tags, resolve_copy_object_tags, validate_bucket_tag_set, }; } } @@ -1248,7 +1260,7 @@ pub(crate) mod bucket_usecase { } } - pub(crate) use super::{access, bucket, error, helper, object_utils, request_context, s3, s3_api}; + pub(crate) use super::{access, bucket, error, helper, object_utils, request_context, s3, s3_api, sse}; pub(crate) use crate::storage::storage_api::{ ECStore, StorageObjectInfo, StorageObjectOptions, get_validated_store, process_lambda_configurations, process_queue_configurations, process_topic_configurations, validate_list_object_unordered_with_delimiter, @@ -1298,8 +1310,9 @@ pub(crate) mod object_usecase { ECStore, GetObjectReader, OldCurrentSize, RFC1123, StorageDeletedObject, StorageObjectInfo, StorageObjectLockDeleteOptions, StorageObjectOptions, StorageObjectToDelete, StoragePutObjReader, check_preconditions, parse_object_lock_legal_hold, parse_object_lock_retention, parse_part_number_i32_to_usize, - remove_object_lock_metadata_for_copy, strip_managed_encryption_metadata, validate_bucket_exists, validate_object_key, - validate_sse_headers_for_read, validate_sse_headers_for_write, validate_ssec_for_read, wrap_response_with_cors, + remove_object_lock_metadata_for_copy, strip_managed_encryption_metadata, validate_bucket_exists, validate_new_object_key, + validate_object_key, validate_sse_headers_for_read, validate_sse_headers_for_write, validate_ssec_for_read, + wrap_response_with_cors, }; } @@ -1332,7 +1345,7 @@ pub(crate) mod multipart_usecase { access, bucket, compression, data_usage, error, helper, io, object_utils, options, request_context, s3_api, set_disk, sse, }; pub(crate) use crate::storage::storage_api::{ - ECStore, StorageObjectInfo, StorageObjectOptions, StoragePutObjReader, validate_object_key, + ECStore, StorageObjectInfo, StorageObjectOptions, StoragePutObjReader, validate_new_object_key, }; } @@ -1359,6 +1372,7 @@ pub(crate) mod context { #[cfg(test)] pub(crate) mod test { pub(crate) use super::EndpointServerPools; + pub(crate) use s3s::access::S3Access; pub(crate) mod contract { pub(crate) mod bucket { pub(crate) use super::super::super::storage_contracts::{ @@ -1402,7 +1416,8 @@ pub(crate) mod test { pub(crate) mod set_disk { pub(crate) use crate::storage::storage_api::ecstore_set_disk::{ ConditionalReadBenchmarkMetadataGuard, MultipartCommitBarrier, MultipartCommitPause, PutObjectCommitBarrier, - PutObjectCommitPause, fail_next_quota_ledger_save_for_test, + PutObjectCommitPause, fail_next_quota_ledger_save_for_test, initialize_quota_ledger_for_test, + retry_quota_reconciliation_for_test, }; } diff --git a/rustfs/src/auth.rs b/rustfs/src/auth.rs index ed1effd36..efe52a658 100644 --- a/rustfs/src/auth.rs +++ b/rustfs/src/auth.rs @@ -194,20 +194,8 @@ impl S3Auth for IAMAuth { return Err(s3_error!(UnauthorizedAccess, "Your account is not signed up")); } - // Check if this is a Keystone access key (from mixed auth scenario) - // Keystone credentials use token authentication, not signature verification if access_key.starts_with("keystone:") { - debug!( - event = EVENT_KEYSTONE_CREDENTIALS_DETECTED, - component = LOG_COMPONENT_AUTH, - subsystem = LOG_SUBSYSTEM_KEYSTONE, - access_key = %MaskedAccessKey(access_key), - result = "token_auth", - "Keystone token-auth access key detected" - ); - // Return empty secret key - Keystone uses token validation, not AWS signatures - // The actual credentials are stored in task-local storage by KeystoneAuthMiddleware - return Ok(SecretKey::from(String::new())); + return Err(s3_error!(InvalidAccessKeyId, "Keystone authentication requires X-Auth-Token header")); } if access_key == self.access_key { @@ -1590,6 +1578,39 @@ mod tests { assert!(!has_root_access(&sys_cred, &oidc_cred)); } + #[tokio::test] + async fn keystone_secret_lookup_requires_validated_token_context() { + use rustfs_keystone::KEYSTONE_CREDENTIALS; + let auth = IAMAuth::new("root", "root-secret"); + for key in ["keystone:x", "keystone:admin"] { + assert_eq!(auth.get_secret_key(key).await.unwrap_err().code(), &S3ErrorCode::InvalidAccessKeyId); + KEYSTONE_CREDENTIALS + .scope(None, async { + assert_eq!(auth.get_secret_key(key).await.unwrap_err().code(), &S3ErrorCode::InvalidAccessKeyId); + }) + .await; + } + let credentials = Credentials { + access_key: "keystone:validated".to_owned(), + ..Default::default() + }; + KEYSTONE_CREDENTIALS + .scope(Some(credentials), async { + assert_eq!( + auth.get_secret_key("keystone:validated") + .await + .expect("validated Keystone token") + .expose(), + "" + ); + }) + .await; + assert_eq!( + auth.get_secret_key("root").await.expect("regular root credential").expose(), + "root-secret" + ); + } + #[test] fn test_iam_auth_creation() { let access_key = "test-access-key"; diff --git a/rustfs/src/server/http.rs b/rustfs/src/server/http.rs index 22c803018..cae1c6642 100644 --- a/rustfs/src/server/http.rs +++ b/rustfs/src/server/http.rs @@ -1429,6 +1429,329 @@ where } } +// Validate control-plane digests before s3s discards the serialized body. +// Object uploads and multipart/form-data must retain their streaming readers. +struct ControlRequestBody(s3s::Body); + +impl http_body::Body for ControlRequestBody { + type Data = Bytes; + type Error = Error; + + fn poll_frame( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll, Self::Error>>> { + Pin::new(&mut self.0) + .poll_frame(cx) + .map(|frame| frame.map(|result| result.map_err(Error::other))) + } + + fn is_end_stream(&self) -> bool { + http_body::Body::is_end_stream(&self.0) + } + + fn size_hint(&self) -> http_body::SizeHint { + http_body::Body::size_hint(&self.0) + } +} + +fn control_request_body(body: s3s::Body) -> ControlRequestBody { + ControlRequestBody(body) +} + +#[derive(Clone)] +struct ControlBodyDigestService { + inner: S, + idle_timeout: Duration, + total_timeout: Duration, + max_body_size: usize, + virtual_host: Option>, +} + +impl ControlBodyDigestService { + fn new(inner: S, idle_timeout: Duration) -> Self { + Self { + inner, + idle_timeout, + total_timeout: Duration::from_secs(30), + max_body_size: rustfs_s3_config().xml_max_body_size, + virtual_host: None, + } + } + + fn with_virtual_host_domains(mut self, domains: Arc<[String]>) -> Self { + if !domains.is_empty() { + // Startup already validated this same domain set for S3Service. + self.virtual_host = Some(Arc::new(MultiDomain::new(domains.iter()).expect("validated S3 host domains"))); + } + self + } +} + +fn control_body_digest_required(request: &HttpRequest, virtual_host: Option<&MultiDomain>) -> Option { + let query = url::form_urlencoded::parse(request.uri().query().unwrap_or_default().as_bytes()); + // Match s3s' multipart POST precedence, including its unique-header rule. + if request.method() == Method::POST + && request.headers().get_all("content-type").iter().count() == 1 + && request + .headers() + .get("content-type") + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.parse::().ok()) + .is_some_and(|mime| mime.type_() == mime_guess::mime::MULTIPART && mime.subtype() == mime_guess::mime::FORM_DATA) + { + return None; + } + let mut operation_ids = query.filter(|(key, _)| key == "x-id"); + let operation_id = operation_ids.next().map(|(_, value)| value); + // s3s rejects duplicate declarations and treats a unique x-id as authoritative. + if operation_ids.next().is_some() { + return None; + } + if request.method() == Method::POST { + let decoded_path = urlencoding::decode(request.uri().path()).ok()?; + use s3s::host::S3Host; + // Use s3s' host parser, including CNAME fallback, and its Host-first + // precedence. A rate-limit host hint is intentionally only best-effort. + let mut host_values = request.headers().get_all("host").iter(); + let host = host_values.next().and_then(|value| value.to_str().ok()).or_else(|| { + matches!(request.version(), Version::HTTP_2 | Version::HTTP_3) + .then(|| request.uri().authority().map(|value| value.as_str())) + .flatten() + }); + if host_values.next().is_some() { + return None; + } + let host = match (virtual_host, host) { + (Some(parser), Some(host)) if host.parse::().is_err() && host.parse::().is_err() => { + Some(parser.parse_host_header(host).ok()?) + } + _ => None, + }; + let path = s3s::path::parse_virtual_hosted_style(host.as_ref().and_then(|host| host.bucket()), &decoded_path).ok()?; + if !matches!(path, s3s::path::S3Path::Bucket { .. }) { + return None; + } + let delete_objects = match operation_id.as_deref() { + Some("DeleteObjects") => true, + Some(_) => false, + None => { + // These bucket POST routes precede DeleteObjects in s3s' router. + query.clone().any(|(key, _)| key == "delete") + && !query + .clone() + .any(|(key, _)| matches!(key.as_ref(), "metadataConfiguration" | "metadataTable")) + } + }; + return delete_objects.then_some(true); + } + if request.method() == Method::PUT + && operation_id + .as_deref() + .is_some_and(|operation| matches!(operation, "PutObject" | "UploadPart")) + { + return None; + } + // Other configuration APIs keep their existing optional-MD5 contract, but + // an MD5 that is supplied (and signed by SigV2) must actually match. + if request.method() == Method::PUT + && request.headers().contains_key("content-md5") + && query.clone().any(|(key, value)| { + (key == "x-id" && value.starts_with("Put") && value != "PutObject") + || matches!( + key.as_ref(), + "acl" + | "cors" + | "lifecycle" + | "logging" + | "notification" + | "policy" + | "replication" + | "requestPayment" + | "tagging" + | "versioning" + | "website" + | "object-lock" + | "retention" + | "legal-hold" + | "ownershipControls" + | "encryption" + | "accelerate" + | "intelligent-tiering" + | "analytics" + | "metrics" + | "inventory" + | "publicAccessBlock" + ) + }) + { + return Some(false); + } + None +} + +fn control_body_digest_error(code: &'static str) -> Response { + // Only static protocol error codes reach this function; never reflect headers or bodies. + let xml = format!("{code}{code}"); + let length = xml.len(); + let mut response = Response::new(s3s::Body::from(xml)); + *response.status_mut() = http::StatusCode::BAD_REQUEST; + response + .headers_mut() + .insert("content-type", HeaderValue::from_static("application/xml")); + response.headers_mut().insert("content-length", HeaderValue::from(length)); + response +} + +const CONTROL_BODY_CHECKSUM_HEADERS: [&str; 10] = [ + "x-amz-checksum-crc32", + "x-amz-checksum-crc32c", + "x-amz-checksum-crc64nvme", + "x-amz-checksum-sha1", + "x-amz-checksum-sha256", + "x-amz-checksum-sha512", + "x-amz-checksum-xxhash64", + "x-amz-checksum-xxhash3", + "x-amz-checksum-xxhash128", + "x-amz-checksum-md5", +]; + +fn verify_control_body_digest(headers: &HeaderMap, bytes: &[u8]) -> std::result::Result<(), &'static str> { + use md5::Digest; + if let Some(expected) = headers.get("content-md5") { + if headers.get_all("content-md5").iter().count() != 1 { + return Err("InvalidDigest"); + } + let expected = base64_simd::STANDARD + .decode_to_vec(expected.as_bytes()) + .map_err(|_| "InvalidDigest")?; + if expected.len() != 16 { + return Err("InvalidDigest"); + } + if md5::Md5::digest(bytes).as_slice() != expected.as_slice() { + return Err("BadDigest"); + } + } + let mut checksums = s3s::checksum::ChecksumHasher { + crc32: headers.contains_key("x-amz-checksum-crc32").then(Default::default), + crc32c: headers.contains_key("x-amz-checksum-crc32c").then(Default::default), + crc64nvme: headers.contains_key("x-amz-checksum-crc64nvme").then(Default::default), + sha1: headers.contains_key("x-amz-checksum-sha1").then(Default::default), + sha256: headers.contains_key("x-amz-checksum-sha256").then(Default::default), + sha512: headers.contains_key("x-amz-checksum-sha512").then(Default::default), + xxhash64: headers.contains_key("x-amz-checksum-xxhash64").then(Default::default), + xxhash3: headers.contains_key("x-amz-checksum-xxhash3").then(Default::default), + xxhash128: headers.contains_key("x-amz-checksum-xxhash128").then(Default::default), + md5: headers.contains_key("x-amz-checksum-md5").then(Default::default), + }; + checksums.update(bytes); + let checksums = checksums.finalize(); + let computed = [ + checksums.checksum_crc32, + checksums.checksum_crc32c, + checksums.checksum_crc64nvme, + checksums.checksum_sha1, + checksums.checksum_sha256, + checksums.checksum_sha512, + checksums.checksum_xxhash64, + checksums.checksum_xxhash3, + checksums.checksum_xxhash128, + checksums.checksum_md5, + ]; + for (name, computed) in CONTROL_BODY_CHECKSUM_HEADERS.into_iter().zip(computed) { + if let Some(computed) = computed + && (headers.get_all(name).iter().count() != 1 + || headers.get(name).map(HeaderValue::as_bytes) != Some(computed.as_bytes())) + { + return Err("BadDigest"); + } + } + Ok(()) +} + +impl Service> for ControlBodyDigestService +where + S: Service, Response = Response> + Clone + Send + 'static, + S::Future: Send + 'static, + S::Error: Send + 'static, + B: http_body::Body + Send + 'static, + B::Error: Send + Sync + 'static, + Box: From, +{ + type Response = Response; + type Error = S::Error; + type Future = Pin> + Send>>; + + fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { + self.inner.poll_ready(cx) + } + + fn call(&mut self, request: HttpRequest) -> Self::Future { + use http_body_util::BodyExt; + let clone = self.inner.clone(); + let mut inner = std::mem::replace(&mut self.inner, clone); + let required = control_body_digest_required(&request, self.virtual_host.as_deref()); + let idle_timeout = self.idle_timeout; + let total_timeout = self.total_timeout; + let limit = self.max_body_size; + Box::pin(async move { + let mut request = request.map(s3s::Body::http_body_unsync); + let Some(required) = required else { + return inner.call(request.map(control_request_body)).await; + }; + let headers = request.headers(); + let sig_v2 = headers + .get("authorization") + .is_some_and(|value| value.as_bytes().starts_with(b"AWS ")) + || url::form_urlencoded::parse(request.uri().query().unwrap_or_default().as_bytes()) + .any(|(key, _)| key == "AWSAccessKeyId"); + let has_checksum = CONTROL_BODY_CHECKSUM_HEADERS.iter().any(|name| headers.contains_key(*name)); + if required && !headers.contains_key("content-md5") && (sig_v2 || !has_checksum) { + return Ok(control_body_digest_error("MissingContentMD5")); + } + if headers + .get("content-length") + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.parse::().ok()) + .is_some_and(|length| length > limit as u64) + { + return Ok(control_body_digest_error("EntityTooLarge")); + } + let deadline = tokio::time::Instant::now() + total_timeout; + let collected = tokio::time::timeout_at(deadline, async { + let mut bytes = bytes::BytesMut::new(); + while let Some(frame) = tokio::time::timeout(idle_timeout, request.body_mut().frame()) + .await + .map_err(|_| "RequestTimeout")? + { + if tokio::time::Instant::now() >= deadline { + return Err("RequestTimeout"); + } + let frame = frame.map_err(|_| "IncompleteBody")?; + if let Ok(data) = frame.into_data() { + if data.len() > limit.saturating_sub(bytes.len()) { + return Err("EntityTooLarge"); + } + bytes.extend_from_slice(&data); + } + } + Ok(bytes.freeze()) + }) + .await + .unwrap_or(Err("RequestTimeout")); + let bytes = match collected { + Ok(bytes) => bytes, + Err(code) => return Ok(control_body_digest_error(code)), + }; + if let Err(code) = verify_control_body_digest(request.headers(), &bytes) { + return Ok(control_body_digest_error(code)); + } + *request.body_mut() = s3s::Body::from(bytes); + inner.call(request.map(control_request_body)).await + }) + } +} + fn trace_on_response(response: &Response, latency: Duration, span: &Span) { span.record("status_code", tracing::field::display(response.status())); let _enter = span.enter(); @@ -2139,10 +2462,10 @@ async fn start_http_server_inner( let make_service = move |peer_addr: SocketAddr| -> Http3ExternalService { let remote_addr = RemoteAddr(peer_addr); + let service = ControlBodyDigestService::new(h3_s3_service.clone(), http_request_body_read_timeout) + .with_virtual_host_domains(h3_vh_domain_hosts.clone()); #[cfg(feature = "swift")] - let service = SwiftService::new(true, None, h3_s3_service.clone()); - #[cfg(not(feature = "swift"))] - let service = h3_s3_service.clone(); + let service = SwiftService::new(true, None, service); let service = EarlyResponseBodyService::new(service, http_request_body_read_timeout); let service = ServiceBuilder::new() .layer(tower::util::MapResponseLayer::new(|response: Response| { @@ -2794,10 +3117,10 @@ fn process_connection( .prepare(), ); + let http_service = ControlBodyDigestService::new(s3_service, request_body_idle_timeout) + .with_virtual_host_domains(vh_domain_hosts.clone()); #[cfg(feature = "swift")] - let http_service = SwiftService::new(true, None, s3_service); - #[cfg(not(feature = "swift"))] - let http_service = s3_service; + let http_service = SwiftService::new(true, None, http_service); let http_service = EarlyResponseBodyService::new(http_service, request_body_idle_timeout).with_task_owner(task_owner.clone()); let http_service = InternodeRpcService::new(http_service); @@ -3350,6 +3673,552 @@ mod tests { use tokio::sync::{Notify, mpsc}; use tower::{Layer, Service, ServiceBuilder}; + const DIGEST_DELETE_XML: &str = "safe-object"; + + struct DigestCheckedS3(Arc); + + #[async_trait::async_trait] + impl s3s::S3 for DigestCheckedS3 { + async fn delete_objects( + &self, + request: s3s::S3Request, + ) -> s3s::S3Result> { + assert_eq!(request.credentials.expect("authenticated request").access_key, "digest-key"); + assert_eq!(request.input.delete.objects[0].key, "safe-object"); + self.0.fetch_add(1, Ordering::SeqCst); + Ok(s3s::S3Response::new(s3s::dto::DeleteObjectsOutput::default())) + } + } + + struct DigestMultipartS3(Arc); + + #[async_trait::async_trait] + impl s3s::S3 for DigestMultipartS3 { + async fn create_multipart_upload( + &self, + request: s3s::S3Request, + ) -> s3s::S3Result> { + assert_eq!(request.credentials.expect("authenticated request").access_key, "digest-key"); + assert_eq!(request.input.bucket, "bucket"); + assert_eq!(request.input.key, "object"); + self.0.fetch_add(1, Ordering::SeqCst); + Ok(s3s::S3Response::new(s3s::dto::CreateMultipartUploadOutput::default())) + } + + async fn complete_multipart_upload( + &self, + request: s3s::S3Request, + ) -> s3s::S3Result> { + assert_eq!(request.credentials.expect("authenticated request").access_key, "digest-key"); + assert_eq!(request.input.bucket, "bucket"); + assert_eq!(request.input.key, "object"); + self.0.fetch_add(1, Ordering::SeqCst); + Ok(s3s::S3Response::new(s3s::dto::CompleteMultipartUploadOutput::default())) + } + } + + fn signed_digest_route_request(uri: &str, body: &'static str, with_md5: bool, virtual_host: bool) -> HttpRequest { + use md5::Digest; + let uri: Uri = uri.parse().unwrap(); + let mut request = HttpRequest::builder() + .method("POST") + .header("date", chrono::Utc::now().format("%a, %d %b %Y %H:%M:%S GMT").to_string()) + .header("host", uri.authority().unwrap().as_str()) + .uri(uri) + .header("content-length", body.len()) + .header("content-type", "application/xml") + .body(s3s::Body::from(body.to_string())) + .unwrap(); + if with_md5 { + request.headers_mut().insert( + "content-md5", + base64_simd::STANDARD + .encode_to_string(md5::Md5::digest(body.as_bytes())) + .parse() + .unwrap(), + ); + } + let wire_uri = request.uri().clone(); + if virtual_host { + // The repository V2 signer currently ignores its virtual_host flag. + // Sign the canonical bucket-prefixed resource, then restore the wire URI. + let bucket = wire_uri.host().unwrap().split('.').next().unwrap(); + let mut parts = wire_uri.clone().into_parts(); + parts.path_and_query = Some(format!("/{bucket}{}", wire_uri.path_and_query().unwrap()).parse().unwrap()); + *request.uri_mut() = Uri::from_parts(parts).unwrap(); + } + let mut request = rustfs_signer::try_sign_v2(request, body.len() as i64, "digest-key", "digest-secret", false).unwrap(); + *request.uri_mut() = wire_uri; + request + } + + #[tokio::test] + async fn control_body_digest_follows_s3_multipart_routing_with_delete_query() { + use tower::ServiceExt; + let calls = Arc::new(AtomicUsize::new(0)); + let mut builder = S3ServiceBuilder::new(DigestMultipartS3(calls.clone())); + builder.set_auth(s3s::auth::SimpleAuth::from_single("digest-key", "digest-secret")); + builder.set_config(Arc::new(StaticConfigProvider::new(Arc::new(rustfs_s3_config())))); + builder.set_host(MultiDomain::new(["example.com"]).unwrap()); + let service = ControlBodyDigestService::new(builder.build(), Duration::from_secs(2)) + .with_virtual_host_domains(vec!["example.com".to_owned()].into()); + for virtual_host in [false, true] { + let base = if virtual_host { + "http://bucket.example.com/object" + } else { + "http://example.com/bucket/object" + }; + for (query, body) in [ + ("x-id=CreateMultipartUpload&uploads&delete", ""), + ("uploads&delete", ""), + ( + "x-id=CompleteMultipartUpload&uploadId=id&delete", + "etag1", + ), + ( + "uploadId=id&delete", + "etag1", + ), + ] { + let uri = format!("{base}?{query}"); + let response = service + .clone() + .oneshot(signed_digest_route_request(&uri, body, false, virtual_host)) + .await + .unwrap(); + let status = response.status(); + let bytes = response.into_body().collect().await.unwrap().to_bytes(); + assert_eq!(status, StatusCode::OK, "{uri}: {}", String::from_utf8_lossy(&bytes)); + } + } + assert_eq!(calls.load(Ordering::SeqCst), 8); + } + + #[tokio::test] + async fn control_body_digest_keeps_bucket_deletes_guarded_with_multipart_query() { + use tower::ServiceExt; + let calls = Arc::new(AtomicUsize::new(0)); + let mut builder = S3ServiceBuilder::new(DigestCheckedS3(calls.clone())); + builder.set_auth(s3s::auth::SimpleAuth::from_single("digest-key", "digest-secret")); + builder.set_config(Arc::new(StaticConfigProvider::new(Arc::new(rustfs_s3_config())))); + builder.set_host(MultiDomain::new(["example.com"]).unwrap()); + let service = ControlBodyDigestService::new(builder.build(), Duration::from_secs(2)) + .with_virtual_host_domains(vec!["example.com".to_owned()].into()); + for virtual_host in [false, true] { + let base = if virtual_host { + "http://bucket.example.com/" + } else { + "http://example.com/bucket" + }; + for query in ["delete&uploads", "delete&uploadId=id", "x-id=DeleteObjects&uploads"] { + let uri = format!("{base}?{query}"); + digest_error( + service + .clone() + .oneshot(signed_digest_route_request(&uri, DIGEST_DELETE_XML, false, virtual_host)) + .await + .unwrap(), + "MissingContentMD5", + ) + .await; + let response = service + .clone() + .oneshot(signed_digest_route_request(&uri, DIGEST_DELETE_XML, true, virtual_host)) + .await + .unwrap(); + let status = response.status(); + let bytes = response.into_body().collect().await.unwrap().to_bytes(); + assert_eq!(status, StatusCode::OK, "{uri}: {}", String::from_utf8_lossy(&bytes)); + let mut tampered = signed_digest_route_request(&uri, DIGEST_DELETE_XML, true, virtual_host); + *tampered.body_mut() = s3s::Body::from(DIGEST_DELETE_XML.replace("safe-object", "evil-object")); + digest_error(service.clone().oneshot(tampered).await.unwrap(), "BadDigest").await; + } + } + assert_eq!(calls.load(Ordering::SeqCst), 6); + } + + #[tokio::test] + async fn control_body_digest_guards_cname_and_host_header_precedence() { + use tower::ServiceExt; + let calls = Arc::new(AtomicUsize::new(0)); + let mut builder = S3ServiceBuilder::new(DigestCheckedS3(calls.clone())); + builder.set_auth(s3s::auth::SimpleAuth::from_single("digest-key", "digest-secret")); + builder.set_config(Arc::new(StaticConfigProvider::new(Arc::new(rustfs_s3_config())))); + builder.set_host(MultiDomain::new(["example.com"]).unwrap()); + let service = ControlBodyDigestService::new(builder.build(), Duration::from_secs(2)) + .with_virtual_host_domains(vec!["example.com".to_owned()].into()); + for different_authority in [false, true] { + for md5 in [false, true] { + // SigV2 canonicalizes a CNAME request as //. + let mut request = signed_digest_route_request( + "http://alias.example.net/alias.example.net/?delete", + DIGEST_DELETE_XML, + md5, + false, + ); + *request.uri_mut() = if different_authority { + "http://example.com/?delete" + } else { + "http://alias.example.net/?delete" + } + .parse() + .unwrap(); + let response = service.clone().oneshot(request).await.unwrap(); + if md5 { + let status = response.status(); + let body = response.into_body().collect().await.unwrap().to_bytes(); + assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&body)); + } else { + digest_error(response, "MissingContentMD5").await; + } + } + } + assert_eq!(calls.load(Ordering::SeqCst), 2); + } + + fn digest_checked_s3(calls: Arc) -> ControlBodyDigestService { + let mut builder = S3ServiceBuilder::new(DigestCheckedS3(calls)); + builder.set_auth(s3s::auth::SimpleAuth::from_single("digest-key", "digest-secret")); + builder.set_config(Arc::new(StaticConfigProvider::new(Arc::new(rustfs_s3_config())))); + ControlBodyDigestService::new(builder.build(), Duration::from_secs(2)) + } + + fn digest_v2_request(presigned: bool, md5: bool, secret: &str) -> HttpRequest { + digest_v2_request_at(presigned, md5, secret, "http://localhost/bucket?delete") + } + + fn digest_v2_request_at(presigned: bool, md5: bool, secret: &str, uri: &str) -> HttpRequest { + use md5::Digest; + let signing_uri = uri.replace("%64elete", "delete"); + let mut request = HttpRequest::builder() + .method("POST") + .header("date", chrono::Utc::now().format("%a, %d %b %Y %H:%M:%S GMT").to_string()) + .uri(signing_uri) + .header("host", "localhost") + .header("content-type", "application/xml") + .header("content-length", DIGEST_DELETE_XML.len()) + .body(s3s::Body::from(DIGEST_DELETE_XML.to_string())) + .unwrap(); + if md5 { + request.headers_mut().insert( + "content-md5", + base64_simd::STANDARD + .encode_to_string(md5::Md5::digest(DIGEST_DELETE_XML.as_bytes())) + .parse() + .unwrap(), + ); + } + let mut request = if presigned { + let mut request = rustfs_signer::try_pre_sign_v2(request, "digest-key", secret, 60, false).unwrap(); + // The signer leaves the final base64 query value unescaped. + let uri = request.uri().to_string(); + let (prefix, signature) = uri.rsplit_once("&Signature=").unwrap(); + let encoded = url::form_urlencoded::Serializer::new(String::new()) + .append_pair("Signature", signature) + .finish(); + *request.uri_mut() = format!("{prefix}&{encoded}").parse().unwrap(); + request + } else { + rustfs_signer::try_sign_v2(request, DIGEST_DELETE_XML.len() as i64, "digest-key", secret, false).unwrap() + }; + if uri.contains("%64elete") { + *request.uri_mut() = request + .uri() + .to_string() + .replace("?delete", "?%64elete") + .replace("&delete", "&%64elete") + .parse() + .unwrap(); + } + request + } + + async fn digest_error(response: Response, code: &str) { + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + let bytes = response.into_body().collect().await.unwrap().to_bytes(); + assert!(String::from_utf8_lossy(&bytes).contains(&format!("{code}"))); + } + + #[tokio::test] + async fn delete_objects_digest_binds_sigv2_header_and_query_bodies() { + use tower::ServiceExt; + for presigned in [false, true] { + let calls = Arc::new(AtomicUsize::new(0)); + let service = digest_checked_s3(calls.clone()); + let response = service + .clone() + .oneshot(digest_v2_request(presigned, true, "digest-secret")) + .await + .unwrap(); + let status = response.status(); + let body = response.into_body().collect().await.unwrap().to_bytes(); + assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&body)); + assert_eq!(calls.load(Ordering::SeqCst), 1); + + let mut tampered = digest_v2_request(presigned, true, "digest-secret"); + *tampered.body_mut() = s3s::Body::from(DIGEST_DELETE_XML.replace("safe-object", "evil-object")); + digest_error(service.clone().oneshot(tampered).await.unwrap(), "BadDigest").await; + digest_error( + service + .clone() + .oneshot(digest_v2_request(presigned, false, "digest-secret")) + .await + .unwrap(), + "MissingContentMD5", + ) + .await; + let mut checksum_only = digest_v2_request(presigned, false, "digest-secret"); + checksum_only + .headers_mut() + .insert("x-amz-checksum-sha256", HeaderValue::from_static("placeholder")); + digest_error(service.clone().oneshot(checksum_only).await.unwrap(), "MissingContentMD5").await; + + let mut malformed = digest_v2_request(presigned, true, "digest-secret"); + malformed + .headers_mut() + .insert("content-md5", HeaderValue::from_static("invalid")); + digest_error(service.clone().oneshot(malformed).await.unwrap(), "InvalidDigest").await; + let response = service + .oneshot(digest_v2_request(presigned, true, "wrong-secret")) + .await + .unwrap(); + assert_eq!( + response.status(), + StatusCode::FORBIDDEN, + "digest verification must not replace authentication" + ); + assert_eq!(calls.load(Ordering::SeqCst), 1, "only the original authenticated XML may dispatch"); + } + } + + #[tokio::test] + async fn delete_objects_digest_covers_operation_id_and_encoded_query_routes() { + use tower::ServiceExt; + for uri in [ + "http://localhost/bucket?x-id=DeleteObjects", + "http://localhost/bucket?%64elete", + ] { + for presigned in [false, true] { + let calls = Arc::new(AtomicUsize::new(0)); + let service = digest_checked_s3(calls.clone()); + let request = digest_v2_request_at(presigned, true, "digest-secret", uri); + let response = service.clone().oneshot(request).await.unwrap(); + let status = response.status(); + let body = response.into_body().collect().await.unwrap().to_bytes(); + assert_eq!(status, StatusCode::OK, "{uri}: {}", String::from_utf8_lossy(&body)); + let mut request = digest_v2_request_at(presigned, true, "digest-secret", uri); + *request.body_mut() = s3s::Body::from(DIGEST_DELETE_XML.replace("safe-object", "evil-object")); + digest_error(service.clone().oneshot(request).await.unwrap(), "BadDigest").await; + digest_error( + service + .oneshot(digest_v2_request_at(presigned, false, "digest-secret", uri)) + .await + .unwrap(), + "MissingContentMD5", + ) + .await; + assert_eq!(calls.load(Ordering::SeqCst), 1); + } + } + } + + fn digest_v4_request(checksum: &str) -> HttpRequest { + use s3s_sigv4::{AmzDate, Payload, calculate_signature, create_canonical_request, create_string_to_sign}; + let timestamp = chrono::Utc::now().format("%Y%m%dT%H%M%SZ").to_string(); + let date = AmzDate::parse(×tamp).unwrap(); + let canonical = create_canonical_request( + "POST", + "/bucket", + &[("delete", "")], + [ + ("host", "localhost"), + ("x-amz-checksum-sha256", checksum), + ("x-amz-content-sha256", "UNSIGNED-PAYLOAD"), + ("x-amz-date", timestamp.as_str()), + ], + Payload::Unsigned, + ); + let to_sign = create_string_to_sign(&canonical, &date, "us-east-1", "s3"); + let signature = calculate_signature(&to_sign, "digest-secret", &date, "us-east-1", "s3"); + HttpRequest::builder().method("POST").uri("http://localhost/bucket?delete") + .header("host", "localhost").header("content-length", DIGEST_DELETE_XML.len()) + .header("x-amz-checksum-sha256", checksum).header("x-amz-content-sha256", "UNSIGNED-PAYLOAD") + .header("x-amz-date", ×tamp) + .header("authorization", format!("AWS4-HMAC-SHA256 Credential=digest-key/{}/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-checksum-sha256;x-amz-content-sha256;x-amz-date, Signature={signature}", date.fmt_date())) + .body(s3s::Body::from(DIGEST_DELETE_XML.to_string())).unwrap() + } + + #[tokio::test] + async fn delete_objects_digest_preserves_sigv4_checksum_requests() { + use sha2::Digest; + use tower::ServiceExt; + let checksum = base64_simd::STANDARD.encode_to_string(sha2::Sha256::digest(DIGEST_DELETE_XML.as_bytes())); + let calls = Arc::new(AtomicUsize::new(0)); + let service = digest_checked_s3(calls.clone()); + let response = service.clone().oneshot(digest_v4_request(&checksum)).await.unwrap(); + let status = response.status(); + let body = response.into_body().collect().await.unwrap().to_bytes(); + assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&body)); + let mut tampered = digest_v4_request(&checksum); + *tampered.body_mut() = s3s::Body::from(DIGEST_DELETE_XML.replace("safe-object", "evil-object")); + digest_error(service.oneshot(tampered).await.unwrap(), "BadDigest").await; + assert_eq!(calls.load(Ordering::SeqCst), 1); + } + + #[tokio::test] + async fn control_body_digest_bounds_declared_and_undeclared_bodies() { + use tower::ServiceExt; + let calls = Arc::new(AtomicUsize::new(0)); + let mut service = digest_checked_s3(calls.clone()); + service.max_body_size = DIGEST_DELETE_XML.len(); + for declared in [false, true] { + let mut request = digest_v2_request(false, true, "digest-secret"); + let (sender, receiver) = mpsc::unbounded_channel(); + let polled = Arc::new(AtomicUsize::new(0)); + let dropped = Arc::new(AtomicBool::new(false)); + sender.send(Bytes::from_static(DIGEST_DELETE_XML.as_bytes())).unwrap(); + sender.send(Bytes::from_static(b" ")).unwrap(); + if declared { + request + .headers_mut() + .insert(CONTENT_LENGTH, HeaderValue::from(DIGEST_DELETE_XML.len() + 1)); + } else { + request.headers_mut().remove(CONTENT_LENGTH); + } + let request = request.map(|_| TrackedRequestBody { + receiver, + bytes_polled: polled.clone(), + dropped: dropped.clone(), + }); + digest_error(service.clone().oneshot(request).await.unwrap(), "EntityTooLarge").await; + assert!(dropped.load(Ordering::SeqCst)); + assert_eq!(polled.load(Ordering::SeqCst), if declared { 0 } else { DIGEST_DELETE_XML.len() + 1 }); + } + let mut request = digest_v2_request(false, true, "digest-secret"); + request.headers_mut().remove(CONTENT_LENGTH); + let (sender, receiver) = mpsc::unbounded_channel(); + sender.send(Bytes::from_static(DIGEST_DELETE_XML.as_bytes())).unwrap(); + drop(sender); + let response = service + .oneshot(request.map(|_| TrackedRequestBody { + receiver, + bytes_polled: Arc::default(), + dropped: Arc::default(), + })) + .await + .unwrap(); + let status = response.status(); + let body = response.into_body().collect().await.unwrap().to_bytes(); + assert_eq!(status, StatusCode::LENGTH_REQUIRED, "{}", String::from_utf8_lossy(&body)); + assert_eq!( + calls.load(Ordering::SeqCst), + 0, + "preserve s3s' missing-length rejection after bounded collection" + ); + } + + #[tokio::test(start_paused = true)] + async fn control_body_digest_idle_total_deadlines_and_cancellation_release_body() { + use tower::ServiceExt; + for trickle in [false, true] { + let calls = Arc::new(AtomicUsize::new(0)); + let mut service = digest_checked_s3(calls.clone()); + service.total_timeout = Duration::from_secs(5); + let (sender, receiver) = mpsc::unbounded_channel(); + let dropped = Arc::new(AtomicBool::new(false)); + let request = digest_v2_request(false, true, "digest-secret").map(|_| TrackedRequestBody { + receiver, + bytes_polled: Arc::default(), + dropped: dropped.clone(), + }); + let task = tokio::spawn(service.oneshot(request)); + for _ in 0..6 { + if trickle { + let _ = sender.send(Bytes::from_static(b" ")); + } + tokio::task::yield_now().await; + tokio::time::advance(Duration::from_secs(1)).await; + } + digest_error(task.await.unwrap().unwrap(), "RequestTimeout").await; + assert!(dropped.load(Ordering::SeqCst)); + assert_eq!(calls.load(Ordering::SeqCst), 0); + } + let calls = Arc::new(AtomicUsize::new(0)); + let (_sender, receiver) = mpsc::unbounded_channel(); + let dropped = Arc::new(AtomicBool::new(false)); + let task = tokio::spawn( + digest_checked_s3(calls.clone()).oneshot(digest_v2_request(false, true, "digest-secret").map(|_| { + TrackedRequestBody { + receiver, + bytes_polled: Arc::default(), + dropped: dropped.clone(), + } + })), + ); + tokio::task::yield_now().await; + task.abort(); + assert!(task.await.unwrap_err().is_cancelled()); + assert!(dropped.load(Ordering::SeqCst)); + assert_eq!(calls.load(Ordering::SeqCst), 0); + } + + #[tokio::test] + async fn control_body_digest_preserves_streaming_uploads_and_checks_config_md5() { + use tower::ServiceExt; + for (method, uri) in [ + ("PUT", "/bucket/key"), + ("PUT", "/bucket/key?partNumber=1&uploadId=id"), + ("POST", "/bucket?uploads"), + ("POST", "/bucket"), + ("POST", "/bucket/key?uploadId=id"), + ("POST", "/bucket?delete"), + ("PUT", "/bucket/key?x-id=PutObject&tagging"), + ] { + let (_sender, receiver) = mpsc::unbounded_channel(); + let polled = Arc::new(AtomicUsize::new(0)); + let inner_polled = polled.clone(); + let inner = tower::service_fn(move |_: HttpRequest| { + assert_eq!(inner_polled.load(Ordering::SeqCst), 0, "uploads must reach dispatch without buffering"); + async { Ok::<_, Infallible>(Response::new(s3s::Body::empty())) } + }); + let request = HttpRequest::builder() + .method(method) + .uri(uri) + .header("content-md5", "invalid") + .header("content-type", "multipart/form-data; boundary=upload") + .body(TrackedRequestBody { + receiver, + bytes_polled: polled, + dropped: Arc::default(), + }) + .unwrap(); + let response = ControlBodyDigestService::new(inner, Duration::from_secs(1)) + .oneshot(request) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + } + let calls = Arc::new(AtomicUsize::new(0)); + let inner_calls = calls.clone(); + let inner = tower::service_fn(move |_: HttpRequest| { + inner_calls.fetch_add(1, Ordering::SeqCst); + async { Ok::<_, Infallible>(Response::new(s3s::Body::empty())) } + }); + let service = ControlBodyDigestService::new(inner, Duration::from_secs(1)); + let mut request = digest_v2_request(false, true, "digest-secret"); + *request.method_mut() = Method::PUT; + *request.uri_mut() = "/bucket?versioning".parse().unwrap(); + assert_eq!(service.clone().oneshot(request).await.unwrap().status(), StatusCode::OK); + let request = HttpRequest::builder() + .method("PUT") + .uri("/bucket?versioning") + .header("content-md5", "1B2M2Y8AsgTpgAmY7PhCfg==") + .body(s3s::Body::from(String::from(""))) + .unwrap(); + digest_error(service.oneshot(request).await.unwrap(), "BadDigest").await; + assert_eq!(calls.load(Ordering::SeqCst), 1); + } + type RateLimitTestCore = tower::util::BoxCloneService< HttpRequest, Response>>, diff --git a/rustfs/src/server/layer.rs b/rustfs/src/server/layer.rs index b2b11e6a7..e4df81293 100644 --- a/rustfs/src/server/layer.rs +++ b/rustfs/src/server/layer.rs @@ -90,10 +90,6 @@ const SWIFT_API_PATH_PREFIX: &str = "/v1/"; pub(crate) fn redact_sensitive_uri_query(uri: &http::Uri) -> String { let path = uri.path(); - if !is_object_zip_download_path(path) { - return uri.to_string(); - } - let Some(query) = uri.query() else { return uri.to_string(); }; @@ -101,7 +97,11 @@ pub(crate) fn redact_sensitive_uri_query(uri: &http::Uri) -> String { let mut redacted_token = false; let mut serializer = form_urlencoded::Serializer::new(String::new()); for (key, value) in form_urlencoded::parse(query.as_bytes()) { - if key == "token" { + if ["X-Amz-Signature", "X-Amz-Security-Token", "Signature"] + .iter() + .any(|sensitive| key.eq_ignore_ascii_case(sensitive)) + || (key == "token" && is_object_zip_download_path(path)) + { redacted_token = true; serializer.append_pair(&key, REDACTED_QUERY_VALUE); } else { @@ -1045,6 +1045,9 @@ where let (parts, body) = response.into_parts(); let response = match body { + HybridBody::Rest { rest_body } if !is_xml_compat_body(&rest_body) => { + Response::from_parts(parts, HybridBody::Rest { rest_body }) + } HybridBody::Rest { rest_body } => { let (rest_body, changed) = fix_s3_error_message_in_xml(rest_body).await.map_err(Into::into)?; let mut parts = parts; @@ -1197,6 +1200,9 @@ where let (parts, body) = response.into_parts(); let response = match body { + HybridBody::Rest { rest_body } if !is_xml_compat_body(&rest_body) => { + Response::from_parts(parts, HybridBody::Rest { rest_body }) + } HybridBody::Rest { rest_body } => { let (rest_body, converted_status) = convert_iceberg_error_in_xml( rest_body, @@ -1371,6 +1377,9 @@ where let (parts, body) = response.into_parts(); let response = match body { + HybridBody::Rest { rest_body } if !is_xml_compat_body(&rest_body) => { + Response::from_parts(parts, HybridBody::Rest { rest_body }) + } HybridBody::Rest { rest_body } => { let rest_body = fix_object_attributes_etag_in_xml(rest_body).await.map_err(Into::into)?; @@ -2009,6 +2018,14 @@ fn is_bodyless_status(status: StatusCode) -> bool { || status == StatusCode::NOT_MODIFIED } +const MAX_XML_COMPAT_BODY_BYTES: u64 = 1024 * 1024; + +fn is_xml_compat_body(body: &B) -> bool { + body.size_hint() + .upper() + .is_some_and(|length| length <= MAX_XML_COMPAT_BODY_BYTES) +} + fn is_xml_response(headers: &HeaderMap) -> bool { let is_xml = headers .get(http::header::CONTENT_TYPE) @@ -5226,6 +5243,77 @@ mod tests { ) } + #[tokio::test] + async fn xml_compat_layers_do_not_poll_large_or_unbounded_bodies() { + struct UnpolledXmlBody(Option); + impl From for UnpolledXmlBody { + fn from(_: Bytes) -> Self { + panic!("streaming bodies must not be rewritten") + } + } + impl Body for UnpolledXmlBody { + type Data = Bytes; + type Error = Infallible; + fn poll_frame( + self: Pin<&mut Self>, + _: &mut Context<'_>, + ) -> Poll, Infallible>>> { + panic!("compatibility layer must not consume the object stream") + } + fn size_hint(&self) -> http_body::SizeHint { + let mut hint = http_body::SizeHint::new(); + if let Some(length) = self.0 { + hint.set_exact(length); + } + hint + } + } + #[derive(Clone)] + struct StreamingXmlResponse { + length: Option, + status: StatusCode, + } + impl Service> for StreamingXmlResponse { + type Response = Response>>; + type Error = Infallible; + type Future = Ready>; + fn poll_ready(&mut self, _: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + fn call(&mut self, _: Request<()>) -> Self::Future { + ready(Ok(Response::builder() + .status(self.status) + .header(http::header::CONTENT_TYPE, "application/xml") + .body(HybridBody::Rest { + rest_body: UnpolledXmlBody(self.length), + }) + .expect("streaming response"))) + } + } + for length in [None, Some(MAX_XML_COMPAT_BODY_BYTES + 1)] { + for (layer, path, status) in [ + (0, "/bucket/object", StatusCode::FORBIDDEN), + (1, "/iceberg/v1/warehouse/namespaces/ns/tables/table", StatusCode::FORBIDDEN), + (2, "/bucket/object?attributes", StatusCode::OK), + ] { + let inner = StreamingXmlResponse { length, status }; + let request = Request::builder() + .method(Method::GET) + .uri(path) + .body(()) + .expect("XML request"); + let response = match layer { + 0 => S3ErrorMessageCompatLayer.layer(inner).call(request).await, + 1 => IcebergRestErrorCompatLayer.layer(inner).call(request).await, + _ => ObjectAttributesEtagFixLayer.layer(inner).call(request).await, + } + .expect("stream passed through"); + assert_eq!(response.status(), status); + assert_eq!(response.headers()[http::header::CONTENT_TYPE], "application/xml"); + } + } + } + #[tokio::test] async fn s3_error_message_compat_fixes_regular_forbidden_xml() { let body = Bytes::from_static(b"SignatureDoesNotMatch"); @@ -5875,6 +5963,78 @@ mod tests { } } + #[test] + #[serial_test::serial] + fn conditional_cors_uses_the_host_bucket_with_real_distinct_bucket_configs() { + crate::app::gating_test_env::run_large_stack_test("cors-bucket-authority", || async { + use crate::app::storage_api::bucket::{metadata::BUCKET_CORS_CONFIG, metadata_sys}; + use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions}; + let store = crate::app::gating_test_env::shared_gating_ecstore().await; + let suffix = uuid::Uuid::new_v4().simple().to_string(); + let owner = format!("cors-owner-{suffix}"); + let other = format!("cors-other-{suffix}"); + for (bucket, origin) in [(&owner, "https://owner.example"), (&other, "https://other.example")] { + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("CORS bucket"); + let xml = format!( + "{origin}GET" + ); + metadata_sys::update(bucket, BUCKET_CORS_CONFIG, xml.into_bytes()) + .await + .expect("persist distinct bucket CORS rules"); + } + let layer = ConditionalCorsLayer { + cors_origins: Some("*".to_owned()), + vh_domain_hosts: Arc::from(["s3.example".to_owned(), "website.example".to_owned()]), + }; + let mut service = layer.layer(CorsOkService); + for (uri, host) in [ + (format!("/{other}/q3.csv"), Some(format!("{owner}.s3.example"))), + (format!("/{other}/q3.csv"), Some(format!("{owner}.website.example"))), + (format!("http://{owner}.s3.example/{other}/q3.csv"), None), + ("/".to_owned(), Some(format!("{owner}.s3.example"))), + (format!("/{owner}/{other}/q3.csv"), Some("s3.example".to_owned())), + ] { + for method in [Method::GET, Method::OPTIONS] { + for (origin, allowed) in [("https://owner.example", true), ("https://other.example", false)] { + let mut request = Request::builder() + .method(method.clone()) + .uri(&uri) + .header(cors::standard::ORIGIN, origin); + if let Some(host) = &host { + request = request.header(http::header::HOST, host); + } + if method == Method::OPTIONS { + request = request.header(cors::request::ACCESS_CONTROL_REQUEST_METHOD, "GET"); + } + let response = service + .call(request.body(()).expect("CORS request")) + .await + .expect("CORS service"); + assert_eq!( + response + .headers() + .get(cors::response::ACCESS_CONTROL_ALLOW_ORIGIN) + .and_then(|value| value.to_str().ok()), + allowed.then_some(origin), + "{method} {uri}: the object owner's CORS rule must be authoritative" + ); + assert_eq!( + response.status(), + if method == Method::OPTIONS && !allowed { + StatusCode::FORBIDDEN + } else { + StatusCode::OK + } + ); + } + } + } + }); + } + #[tokio::test] async fn conditional_cors_passthrough_without_origin() { let layer = ConditionalCorsLayer { @@ -6472,6 +6632,36 @@ mod tests { assert_eq!(redact_sensitive_uri_query(&uri), "/rustfs/admin/v3/users?token=not-a-download-token"); } + #[test] + fn redact_sensitive_uri_query_removes_presigned_credentials() { + for path in ["/bucket/object", "/prefix/bucket/object", "/"] { + let uri: http::Uri = format!( + "{path}?X-Amz-Signature=secret-signature&x-amz-security-token=secret-session&Signature=secret-v2&partNumber=1" + ) + .parse() + .expect("presigned URI"); + let logged = redact_sensitive_uri_query(&uri); + assert_eq!( + logged, + format!("{path}?X-Amz-Signature=redacted&x-amz-security-token=redacted&Signature=redacted&partNumber=1") + ); + assert!(!logged.contains("secret")); + } + let uri: http::Uri = "/bucket/object?%58-Amz-Signature=secret&X-Amz-Signature=another&signature=v2" + .parse() + .expect("URI"); + assert_eq!( + redact_sensitive_uri_query(&uri), + "/bucket/object?X-Amz-Signature=redacted&X-Amz-Signature=redacted&signature=redacted" + ); + } + + #[test] + fn redact_sensitive_uri_query_preserves_ordinary_query_encoding() { + let uri: http::Uri = "/bucket?prefix=one%20two&delimiter=%2f".parse().expect("URI"); + assert_eq!(redact_sensitive_uri_query(&uri), uri.to_string()); + } + #[tokio::test(start_paused = true)] async fn request_logging_watchdog_cancels_dropped_futures() { for poll_request in [false, true] { diff --git a/rustfs/src/storage/access.rs b/rustfs/src/storage/access.rs index ec5a4b987..f8c3c9cfd 100644 --- a/rustfs/src/storage/access.rs +++ b/rustfs/src/storage/access.rs @@ -168,6 +168,10 @@ struct RequestedObjectTags(Tagging); #[derive(Clone, Debug)] struct PostObjectPolicyHeaders(http::HeaderMap); +/// Copy source as the handler resolves it (percent-decoded, versionId canonical). +#[derive(Clone)] +struct RequestedCopySource(String); + /// These operations stage upload state without publishing an object. Keep this /// separate from the marker controlling multipart publication fences. #[derive(Clone, Copy, Debug)] @@ -754,6 +758,12 @@ impl ObjectTagConditions { } } +/// Local tag evidence cannot authorize an object that only exists on a replica. +/// Keep the local miss whenever authorization needed existing-object tags. +pub(crate) fn replication_proxy_allowed(extensions: &http::Extensions) -> bool { + extensions.get::().is_none() +} + const AMZ_WRITE_OFFSET_BYTES_HEADER: &str = "x-amz-write-offset-bytes"; fn has_write_offset_bytes_header(headers: &http::HeaderMap) -> bool { @@ -785,13 +795,13 @@ fn classify_bucket_policy_raw_load_error(err: &StorageError) -> BucketPolicyRawL } } -/// Load and parse bucket policy once for ExistingObjectTag hint checks. +/// Reuse the parsed bucket metadata policy for ExistingObjectTag hint checks. async fn load_bucket_policy_existing_object_tag_hint( store: &ECStore, bucket: &str, action: Action, ) -> BucketPolicyExistingObjectTagHint { - let (policy_str, _) = match store.get_bucket_policy_raw(bucket).await { + let (policy, _) = match store.get_bucket_policy(bucket).await { Ok(v) => v, Err(err) => match classify_bucket_policy_raw_load_error(&err) { BucketPolicyRawLoadErrorKind::PolicyMissing => { @@ -822,23 +832,10 @@ async fn load_bucket_policy_existing_object_tag_hint( } }, }; - match serde_json::from_str::(policy_str.as_str()) { - Ok(policy) => { - if bucket_policy_uses_existing_object_tag_conditions(&policy) { - BucketPolicyExistingObjectTagHint::Parsed(policy) - } else { - BucketPolicyExistingObjectTagHint::NoTagRequirement - } - } - Err(err) => { - tracing::warn!( - bucket = %bucket, - ?action, - error = %err, - "malformed bucket policy while checking ExistingObjectTag hint; conservatively enabling tag fetch" - ); - BucketPolicyExistingObjectTagHint::ConservativeTagRequired - } + if bucket_policy_uses_existing_object_tag_conditions(&policy) { + BucketPolicyExistingObjectTagHint::Parsed(policy) + } else { + BucketPolicyExistingObjectTagHint::NoTagRequirement } } @@ -877,28 +874,34 @@ fn action_tag_metric_label(action: &Action) -> &'static str { } } -fn merge_list_bucket_query_conditions(action: Action, query: Option<&str>, conditions: &mut HashMap>) { +fn merge_list_bucket_query_conditions( + action: Action, + query: Option<&str>, + conditions: &mut HashMap>, +) -> S3Result<()> { if !matches!( action, Action::S3Action( S3Action::ListBucketAction | S3Action::ListBucketVersionsAction | S3Action::ListBucketMultipartUploadsAction ) ) { - return; + return Ok(()); + } + for key in ["prefix", "delimiter", "max-keys"] { + conditions.remove(key); } - let Some(query) = query else { - return; + return Ok(()); }; for (key, value) in form_urlencoded::parse(query.as_bytes()) { - match key.as_ref() { - "prefix" | "delimiter" | "max-keys" => { - conditions.entry(key.into_owned()).or_default().push(value.into_owned()); - } - _ => {} + if matches!(key.as_ref(), "prefix" | "delimiter" | "max-keys") + && conditions.insert(key.to_string(), vec![value.into_owned()]).is_some() + { + return Err(s3_error!(InvalidRequest, "duplicate query: {key}")); } } + Ok(()) } fn merge_request_object_tag_conditions( @@ -969,8 +972,11 @@ fn authorization_conditions( { conditions.insert("LocationConstraint".to_string(), vec![location]); } - merge_list_bucket_query_conditions(action, req.uri.query(), &mut conditions); + merge_list_bucket_query_conditions(action, req.uri.query(), &mut conditions)?; merge_request_object_tag_conditions(action, headers, &mut conditions)?; + if let Some(RequestedCopySource(source)) = req.extensions.get::() { + conditions.insert("x-amz-copy-source".to_string(), vec![source.clone()]); + } if let Some(RequestedObjectLegalHold(legal_hold)) = req.extensions.get::() { // Standalone legal hold writes apply the XML body, so headers must not // supply a different value to the policy decision. @@ -1087,6 +1093,7 @@ fn set_post_object_policy_headers(req: &mut S3Request) -> S3Resu ("x-amz-server-side-encryption-customer-algorithm", input.sse_customer_algorithm.as_deref()), ("x-amz-grant-full-control", input.grant_full_control.as_deref()), ("x-amz-grant-read", input.grant_read.as_deref()), + ("x-amz-grant-write", None), ("x-amz-grant-read-acp", input.grant_read_acp.as_deref()), ("x-amz-grant-write-acp", input.grant_write_acp.as_deref()), ] { @@ -1098,6 +1105,17 @@ fn set_post_object_policy_headers(req: &mut S3Request) -> S3Resu headers.insert(key, value); } } + for (key, value) in [("if-match", &input.if_match), ("if-none-match", &input.if_none_match)] { + headers.remove(key); + if let Some(value) = value { + headers.insert( + key, + value + .to_http_header() + .map_err(|_| S3Error::with_message(S3ErrorCode::InvalidArgument, "Invalid form field"))?, + ); + } + } req.extensions.insert(PostObjectPolicyHeaders(headers)); Ok(()) } @@ -1513,6 +1531,72 @@ pub async fn authorize_request(req: &mut S3Request, action: Action) -> S3R authorize_request_with_location_constraint(req, action, None).await } +/// Authorize the exact tag set selected by CopyObject's source reader before publication. +/// The access hook also calls this with a preliminary snapshot, so tag-dependent +/// destination Allow statements can be evaluated before opening either object. +pub(crate) async fn authorize_copy_object_tags( + req: &mut S3Request, + source_tags: &str, + effective_tags: &str, +) -> S3Result<()> { + // Direct internal use-case calls with no tags retain their existing contract. + // Public CopyObject requests always carry the access hook's requested-tag snapshot. + if effective_tags.is_empty() && req.extensions.get::().is_none() { + return Ok(()); + } + let tag_set = crate::storage::s3_api::tagging::parse_object_tag_header(effective_tags)?; + let copies_tags = req.input.tagging_directive.as_ref().map(TaggingDirective::as_str) != Some(TaggingDirective::REPLACE); + if copies_tags && !tag_set.is_empty() { + let CopySource::Bucket { bucket, key, version_id } = &req.input.copy_source else { + return Err(s3_error!(NotImplemented)); + }; + let (bucket, key, version_id) = (bucket.to_string(), key.to_string(), version_id.as_ref().map(ToString::to_string)); + let destination = req_info_ref(req)?.clone(); + let destination_tags = req.extensions.remove::(); + req.extensions.remove::(); + // Reuse the reader's metadata instead of reopening the source under its + // read guard (or the self-copy write guard). These are the tags being copied. + let source_conditions = crate::storage::s3_api::tagging::parse_object_tag_header(source_tags)? + .into_iter() + .filter_map(|tag| Some((format!("ExistingObjectTag/{}", tag.key?), vec![tag.value?]))) + .collect(); + req.extensions + .insert(ObjectTagConditions::new(&bucket, &key, version_id.as_deref(), source_conditions)); + let info = req_info_mut(req)?; + info.bucket = Some(bucket); + info.object = Some(key); + info.version_id = version_id.clone(); + let result = + authorize_request(req, versioned_tagging_action(S3Action::GetObjectTaggingAction, version_id.as_deref())).await; + req.extensions.insert(destination); + req.extensions.remove::(); + if let Some(tags) = destination_tags { + req.extensions.insert(tags); + } + result?; + } + req.extensions.insert(RequestedObjectTags(Tagging { tag_set })); + authorize_request(req, Action::S3Action(S3Action::PutObjectAction)).await +} + +/// A specific destination ETag reveals stored content and requires read access. +/// Wildcard existence conditions retain the existing write-only contract. +async fn authorize_write_etag_condition(req: &mut S3Request, specific_dto_etag: bool) -> S3Result<()> { + let headers = req + .extensions + .get::() + .map_or(&req.headers, |form| &form.0); + let specific_header_etag = headers + .get_all(http::header::IF_MATCH) + .iter() + .any(|value| value.to_str().map_or(true, |value| value.trim() != "*")); + if specific_dto_etag || specific_header_etag { + let action = versioned_read_action(req_info_ref(req)?.version_id.as_deref()); + authorize_request(req, action).await?; + } + Ok(()) +} + async fn authorize_request_with_location_constraint( req: &mut S3Request, action: Action, @@ -2127,6 +2211,16 @@ fn table_list_catalog_error(err: crate::table_catalog::TableCatalogStoreError) - } } +pub(crate) async fn authorize_bucket_listing(req: &mut S3Request, bucket: &str, action: S3Action) -> S3Result<()> { + authorize_request(req, Action::S3Action(action)).await?; + if action == S3Action::ListBucketAction + && get_header(&req.headers, rustfs_utils::http::SUFFIX_INCLUDE_DELETED).as_deref() == Some("true") + { + authorize_request(req, Action::S3Action(S3Action::ListBucketVersionsAction)).await?; + } + prepare_table_data_plane_list_access(req, bucket, action).await +} + async fn prepare_table_data_plane_list_access(req: &mut S3Request, bucket: &str, action: S3Action) -> S3Result<()> { if !table_bucket_enabled_for_data_plane(req, bucket).await? { return Ok(()); @@ -2541,6 +2635,8 @@ impl S3Access for FS { req_info.object = Some(req.input.key.clone()); authorize_request(req, complete_multipart_upload_authorize_action()).await?; + let specific_etag = req.input.if_match.as_ref().is_some_and(|etag| !etag.is_any()); + authorize_write_etag_condition(req, specific_etag).await?; authorize_replication_only_put_headers(req).await?; req.extensions.insert(bucket_generation?); Ok(()) @@ -2559,6 +2655,11 @@ impl S3Access for FS { (bucket.to_string(), key.to_string(), version_id.as_ref().map(|v| v.to_string())) } }; + let source = match version_id.as_deref() { + Some(version) => format!("{src_bucket}/{src_key}?versionId={}", crate::auth::canonical_policy_version_id(version)), + None => format!("{src_bucket}/{src_key}"), + }; + req.extensions.insert(RequestedCopySource(source)); let source_bucket_generation = load_copy_source_bucket_generation(self, &src_bucket).await; { @@ -2577,11 +2678,60 @@ impl S3Access for FS { req_info.bucket = Some(req.input.bucket.clone()); req_info.object = Some(req.input.key.clone()); - req_info.version_id = req.input.version_id.clone(); + req_info.version_id = super::options::destination_version_id(req.input.version_id.as_deref(), &req.headers); let requested_retain_until = req.input.object_lock_retain_until_date.clone(); set_requested_object_lock_retain_until(req, requested_retain_until.as_ref()); - authorize_request(req, Action::S3Action(S3Action::PutObjectAction)).await?; + let replacement = crate::storage::s3_api::tagging::resolve_copy_object_tags( + req.input.tagging.as_deref(), + req.input.tagging_directive.as_ref(), + )?; + let store = request_object_store(req)?; + let source_tags = if replacement.is_none() { + use crate::storage::storage_api::contract::object::ObjectOperations as _; + match store + .get_object_tags( + &src_bucket, + &src_key, + &ObjectOptions { + // Keep the explicit version in the policy context, but + // address its storage representation just like COPY's reader. + version_id: version_id.as_deref().map(|version| { + let version = version.trim(); + if version.eq_ignore_ascii_case("null") { + uuid::Uuid::nil().to_string() + } else { + version.to_owned() + } + }), + ..Default::default() + }, + ) + .await + { + Ok(tags) => tags, + Err(error) if super::is_err_object_not_found(&error) || super::is_err_version_not_found(&error) => String::new(), + Err(error) => return Err(ApiError::from(error).into()), + } + } else { + String::new() + }; + // Self-copy later holds the destination write lock. Capture its existing + // tags now, including when only the final requested tags activate a policy. + if src_bucket == req.input.bucket && src_key == req.input.key { + let destination_version = req_info_ref(req)?.version_id.clone(); + get_or_fetch_object_tag_conditions( + req, + store.as_ref(), + &src_bucket, + &src_key, + destination_version.as_deref(), + Action::S3Action(S3Action::PutObjectAction), + ) + .await?; + } + req.extensions.insert(RequestedObjectTags(Tagging::default())); + authorize_copy_object_tags(req, &source_tags, replacement.as_deref().unwrap_or(&source_tags)).await?; authorize_replication_only_put_headers(req).await?; @@ -2604,6 +2754,7 @@ impl S3Access for FS { let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); req_info.object = Some(req.input.key.clone()); + req_info.version_id = super::options::destination_version_id(req.input.version_id.as_deref(), &req.headers); let requested_retain_until = req.input.object_lock_retain_until_date.clone(); set_requested_object_lock_retain_until(req, requested_retain_until.as_ref()); @@ -2748,7 +2899,12 @@ impl S3Access for FS { let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); - authorize_bucket_config_mutation(self, req, Action::S3Action(S3Action::PutBucketTaggingAction)).await + rustfs_iam::sys::with_bucket_tag_deletion(authorize_bucket_config_mutation( + self, + req, + Action::S3Action(S3Action::PutBucketTaggingAction), + )) + .await } /// Checks whether the DeleteBucketWebsite request has accesses to the resources. @@ -3204,18 +3360,14 @@ impl S3Access for FS { req_info.object = Some(req.input.key.clone()); req_info.version_id = req.input.version_id.clone(); - // A replication convergence check HEADs the replica to compare - // etag/size/mtime. For SSE-C replicas the worker holds no customer key, - // so authorize it as a replication action and let the handler skip the - // SSE-C read validation. + authorize_request(req, versioned_read_action(req.input.version_id.as_deref())).await?; + // Replication permission additionally unlocks the SSE-C convergence + // check; it does not replace permission to read this version's metadata. if get_header(&req.headers, SUFFIX_SOURCE_REPLICATION_CHECK).as_deref() == Some("true") { authorize_request(req, Action::S3Action(S3Action::ReplicateObjectAction)).await?; req_info_mut(req)?.replication_request_authorized = true; - req.extensions.insert(source_generation); - return Ok(()); } - authorize_request(req, versioned_read_action(req.input.version_id.as_deref())).await?; req.extensions.insert(source_generation); Ok(()) } @@ -3291,8 +3443,7 @@ impl S3Access for FS { let bucket = req.input.bucket.clone(); let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); - authorize_request(req, Action::S3Action(S3Action::ListBucketVersionsAction)).await?; - prepare_table_data_plane_list_access(req, &bucket, S3Action::ListBucketVersionsAction).await + authorize_bucket_listing(req, &bucket, S3Action::ListBucketVersionsAction).await } /// Checks whether the ListObjects request has accesses to the resources. @@ -3303,8 +3454,7 @@ impl S3Access for FS { let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); - authorize_request(req, Action::S3Action(S3Action::ListBucketAction)).await?; - prepare_table_data_plane_list_access(req, &bucket, S3Action::ListBucketAction).await + authorize_bucket_listing(req, &bucket, S3Action::ListBucketAction).await } /// Checks whether the ListObjectsV2 request has accesses to the resources. @@ -3316,8 +3466,7 @@ impl S3Access for FS { let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); - authorize_request(req, Action::S3Action(S3Action::ListBucketAction)).await?; - prepare_table_data_plane_list_access(req, &bucket, S3Action::ListBucketAction).await?; + authorize_bucket_listing(req, &bucket, S3Action::ListBucketAction).await?; req.extensions.insert(source_generation); Ok(()) } @@ -3539,7 +3688,7 @@ impl S3Access for FS { let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); req_info.object = Some(req.input.key.clone()); - req_info.version_id = req.input.version_id.clone(); + req_info.version_id = super::options::destination_version_id(req.input.version_id.as_deref(), &req.headers); if has_write_offset_bytes_header(&req.headers) { return Err(S3Error::with_message( @@ -3562,6 +3711,8 @@ impl S3Access for FS { set_requested_object_lock_retain_until(req, requested_retain_until.as_ref()); set_post_object_policy_headers(req)?; authorize_request(req, Action::S3Action(S3Action::PutObjectAction)).await?; + let specific_etag = req.input.if_match.as_ref().is_some_and(|etag| !etag.is_any()); + authorize_write_etag_condition(req, specific_etag).await?; req.extensions.insert(bucket_generation?); if req.method != http::Method::POST { @@ -3727,6 +3878,11 @@ impl S3Access for FS { (bucket.to_string(), key.to_string(), version_id.as_ref().map(|v| v.to_string())) } }; + let source = match version_id.as_deref() { + Some(version) => format!("{src_bucket}/{src_key}?versionId={}", crate::auth::canonical_policy_version_id(version)), + None => format!("{src_bucket}/{src_key}"), + }; + req.extensions.insert(RequestedCopySource(source)); let source_bucket_generation = load_copy_source_bucket_generation(self, &src_bucket).await; { @@ -3800,6 +3956,198 @@ mod tests { use std::{collections::HashMap, sync::Arc}; use time::OffsetDateTime; + #[test] + fn replication_head_checks_read_permissions_before_replication_permissions() { + crate::app::gating_test_env::run_large_stack_test("replication-head-permissions", || async { + let store = crate::app::gating_test_env::shared_gating_ecstore().await; + let bucket = format!("replica-head-{}", uuid::Uuid::new_v4()); + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create bucket"); + let slot = ServerContextSlot::new(); + assert!(slot.install(Arc::new(AppContext::new(store.clone(), Arc::new(UnreadyIam), Arc::new(TestKms))))); + let fs = FS::with_server_ctx(slot.clone()); + for (grant_read, deny_read) in [(false, false), (true, true), (true, false)] { + let mut statements = vec![ + serde_json::json!({"Effect":"Allow","Principal":{"AWS":"*"},"Action":"s3:ReplicateObject","Resource":format!("arn:aws:s3:::{bucket}/*")}), + ]; + for (enabled, effect) in [(grant_read, "Allow"), (deny_read, "Deny")] { + if enabled { + statements.push(serde_json::json!({"Effect":effect,"Principal":{"AWS":"*"},"Action":["s3:GetObject","s3:GetObjectVersion"],"Resource":format!("arn:aws:s3:::{bucket}/*")})); + } + } + store + .update_bucket_metadata_config( + &bucket, + crate::storage::storage_api::ecstore_bucket::metadata::BUCKET_POLICY_CONFIG, + serde_json::to_vec(&serde_json::json!({"Version":"2012-10-17","Statement":statements})).unwrap(), + ) + .await + .expect("publish policy"); + for version in [None, Some("null"), Some("12345678-1234-1234-1234-123456789abc")] { + for header in ["x-rustfs-source-replication-check", "x-minio-source-replication-check"] { + let mut req = build_request( + HeadObjectInput { + bucket: bucket.clone(), + key: "object".to_owned(), + version_id: version.map(str::to_owned), + ..Default::default() + }, + Method::HEAD, + ); + ensure_req_info(&mut req); + req.extensions.insert(slot.clone()); + req.headers.insert(header, HeaderValue::from_static("true")); + let result = fs.head_object(&mut req).await; + if grant_read && !deny_read { + result.expect("read and replication permissions allow metadata check"); + assert!(super::replication_request_authorized(&req)); + } else { + assert_eq!( + result.expect_err("replication cannot replace read permission").code(), + &S3ErrorCode::AccessDenied + ); + assert!(!super::replication_request_authorized(&req)); + } + } + } + } + }); + } + + #[test] + fn replication_write_hooks_authorize_the_destination_version_and_tags() { + crate::app::gating_test_env::run_large_stack_test("replication-version-permissions", || async { + let store = crate::app::gating_test_env::shared_gating_ecstore().await; + let bucket = format!("replica-write-{}", uuid::Uuid::new_v4()); + store + .make_bucket( + &bucket, + &MakeBucketOptions { + versioning_enabled: true, + ..Default::default() + }, + ) + .await + .expect("create versioned bucket"); + let slot = ServerContextSlot::new(); + assert!(slot.install(Arc::new(AppContext::new(store.clone(), Arc::new(UnreadyIam), Arc::new(TestKms))))); + let fs = FS::with_server_ctx(slot.clone()); + let mut versions = Vec::new(); + for (body, tags) in [ + (b"secret".to_vec(), "classification=secret"), + (b"public".to_vec(), "classification=public"), + ] { + let object = store + .put_object( + &bucket, + "target", + &mut crate::storage::PutObjReader::from_vec(body), + &crate::storage::ObjectOptions { + versioned: true, + ..Default::default() + }, + ) + .await + .expect("seed version"); + let version = object.version_id.expect("UUID version").to_string(); + store + .put_object_tags( + &bucket, + "target", + tags, + &crate::storage::ObjectOptions { + version_id: Some(version.clone()), + versioned: true, + ..Default::default() + }, + ) + .await + .expect("tag version"); + versions.push(version); + } + for condition in [ + serde_json::json!({"s3:versionid":versions[0]}), + serde_json::json!({"s3:ExistingObjectTag/classification":"secret"}), + ] { + let policy = serde_json::json!({"Version":"2012-10-17","Statement":[ + {"Effect":"Allow","Principal":{"AWS":"*"},"Action":"s3:*","Resource":format!("arn:aws:s3:::{bucket}/*")}, + {"Effect":"Deny","Principal":{"AWS":"*"},"Action":"s3:PutObject","Resource":format!("arn:aws:s3:::{bucket}/target"),"Condition":{"StringEquals":condition}} + ]}); + store + .update_bucket_metadata_config( + &bucket, + crate::storage::storage_api::ecstore_bucket::metadata::BUCKET_POLICY_CONFIG, + serde_json::to_vec(&policy).unwrap(), + ) + .await + .expect("publish version policy"); + for (index, version) in versions.iter().enumerate() { + for header in [None, Some("x-rustfs-source-version-id"), Some("x-minio-source-version-id")] { + for operation in ["put", "copy", "multipart"] { + let query_version = header.is_none().then(|| version.clone()); + let mut base = build_request((), Method::PUT); + ensure_req_info(&mut base); + base.extensions.insert(slot.clone()); + // A query version alone also requires replication authorization. + base.headers + .insert("x-rustfs-source-replication-request", HeaderValue::from_static("true")); + if let Some(header) = header { + base.headers.insert(header, HeaderValue::from_str(version).unwrap()); + } + let (result, info) = match operation { + "put" => { + let mut req = base.map_input(|_| PutObjectInput { + bucket: bucket.clone(), + key: "target".to_owned(), + version_id: query_version, + ..Default::default() + }); + let result = fs.put_object(&mut req).await; + (result, req.extensions.get::().unwrap().clone()) + } + "copy" => { + let mut req = base.map_input(|_| { + CopyObjectInput::builder() + .bucket(bucket.clone()) + .key("target".to_owned()) + .version_id(query_version) + .copy_source(CopySource::parse(&format!("/{bucket}/source")).expect("copy source")) + .build() + .expect("copy input") + }); + let result = fs.copy_object(&mut req).await; + (result, req.extensions.get::().unwrap().clone()) + } + _ => { + let mut req = base.map_input(|_| CreateMultipartUploadInput { + bucket: bucket.clone(), + key: "target".to_owned(), + version_id: query_version, + ..Default::default() + }); + let result = fs.create_multipart_upload(&mut req).await; + (result, req.extensions.get::().unwrap().clone()) + } + }; + assert_eq!(info.version_id.as_deref(), Some(version.as_str()), "{operation}"); + if index == 0 { + assert_eq!( + result.expect_err("protected destination version must be denied").code(), + &S3ErrorCode::AccessDenied, + "{operation}" + ); + } else { + result.expect("unprotected destination version remains authorized"); + } + } + } + } + } + }); + } + #[test] fn terminal_iam_denial_is_access_denied_and_backend_failure_stays_internal() { let denied = s3s::S3Error::from(super::iam_authorization_error(rustfs_iam::error::Error::IAMActionNotAllowed)); @@ -4574,7 +4922,8 @@ mod tests { Action::S3Action(S3Action::ListBucketAction), Some("prefix=photos%2F2024%2F&delimiter=%2F&max-keys=10&encoding-type=url"), &mut conditions, - ); + ) + .expect("valid listing query"); assert_eq!(conditions.get("prefix"), Some(&vec!["photos/2024/".to_string()])); assert_eq!(conditions.get("delimiter"), Some(&vec!["/".to_string()])); @@ -4589,7 +4938,8 @@ mod tests { Action::S3Action(S3Action::ListBucketVersionsAction), Some("prefix=&delimiter=%2F"), &mut conditions, - ); + ) + .expect("valid listing query"); assert_eq!(conditions.get("prefix"), Some(&vec![String::new()])); assert_eq!(conditions.get("delimiter"), Some(&vec!["/".to_string()])); @@ -4602,11 +4952,36 @@ mod tests { Action::S3Action(S3Action::GetObjectAction), Some("prefix=photos%2F2024%2F&delimiter=%2F&max-keys=10"), &mut conditions, - ); + ) + .expect("non-list query ignored"); assert!(conditions.is_empty()); } + #[test] + fn listing_conditions_reject_repeated_query_and_ignore_headers() { + for action in [ + S3Action::ListBucketAction, + S3Action::ListBucketVersionsAction, + S3Action::ListBucketMultipartUploadsAction, + ] { + for key in ["prefix", "delimiter", "max-keys"] { + let mut req = build_request((), Method::GET); + req.headers.insert(key, HeaderValue::from_static("allowed")); + let action = Action::S3Action(action); + let credentials = rustfs_credentials::Credentials::default(); + let conditions = authorization_conditions(&req, &credentials, None, None, None, None, action).unwrap(); + assert!(!conditions.contains_key(key), "{key} must come from the applied query"); + req.uri = format!("/bucket?{key}=actual").parse().unwrap(); + let conditions = authorization_conditions(&req, &credentials, None, None, None, None, action).unwrap(); + assert_eq!(conditions.get(key), Some(&vec!["actual".to_string()])); + req.uri = format!("/bucket?{key}=actual&{key}=allowed").parse().unwrap(); + let error = authorization_conditions(&req, &credentials, None, None, None, None, action).unwrap_err(); + assert_eq!(error.code(), &S3ErrorCode::InvalidRequest); + } + } + } + #[test] fn test_merge_request_object_tag_conditions_applies_only_to_put_object() { let mut headers = HeaderMap::new(); @@ -5013,6 +5388,252 @@ mod tests { assert_eq!(conditions.get("object-lock-legal-hold"), Some(&vec!["ON".to_string()])); } + #[tokio::test] + async fn copy_source_conditions_use_the_decoded_source_for_both_copy_operations() { + use rustfs_policy::policy::{Args, BucketPolicy, BucketPolicyArgs, Policy}; + let mut document = serde_json::json!({"Version":"2012-10-17","Statement":[ + {"Effect":"Allow","Action":"s3:PutObject","Resource":"arn:aws:s3:::bucket/*"}, + {"Effect":"Deny","Action":"s3:PutObject","Resource":"arn:aws:s3:::bucket/*", + "Condition":{"StringLike":{"s3:x-amz-copy-source":"secret/*"}}} + ]}); + let identity = + Policy::parse_config(&serde_json::to_vec(&document).expect("encode identity policy")).expect("parse policy"); + for statement in document["Statement"].as_array_mut().expect("statements") { + statement["Principal"] = serde_json::json!("*"); + } + let bucket: BucketPolicy = + serde_json::from_slice(&serde_json::to_vec(&document).expect("encode bucket policy")).expect("parse bucket policy"); + for spelling in [ + "secret/report.txt", + "%73ecret/report.txt", + "secret%2Freport.txt", + "/secret/report.txt", + "%2Fsecret%2Freport.txt?versionId=00000000-0000-0000-0000-000000000000", + "public/report.txt", + "/public%2Freport.txt", + ] { + for multipart in [false, true] { + let source = CopySource::parse(spelling).expect("parse source exactly as the request parser does"); + let mut req = if multipart { + let input = UploadPartCopyInput::builder() + .bucket("bucket".to_string()) + .key("copy".to_string()) + .copy_source(source) + .upload_id("unused".to_string()) + .part_number(1) + .build() + .expect("part copy input"); + let mut req = build_request(input, Method::PUT); + ensure_req_info(&mut req); + let _ = FS::new().upload_part_copy(&mut req).await; + req.map_input(|_| ()) + } else { + let input = CopyObjectInput::builder() + .bucket("bucket".to_string()) + .key("copy".to_string()) + .copy_source(source) + .build() + .expect("copy input"); + let mut req = build_request(input, Method::PUT); + ensure_req_info(&mut req); + let _ = FS::new().copy_object(&mut req).await; + req.map_input(|_| ()) + }; + req.headers + .insert("x-amz-copy-source", HeaderValue::from_str(spelling).expect("source header")); + let action = Action::S3Action(S3Action::PutObjectAction); + let conditions = remaining_retention_days_conditions(&req, action); + let canonical = &conditions["x-amz-copy-source"]; + assert_eq!(canonical.len(), 1, "one semantic source, independent of header spelling"); + let expected = if spelling.contains("public") { + "public/report.txt" + } else if spelling.contains('?') { + "secret/report.txt?versionId=null" + } else { + "secret/report.txt" + }; + assert_eq!(canonical[0], expected, "{spelling}, multipart={multipart}"); + let allowed = expected.starts_with("public/"); + assert_eq!( + identity + .is_allowed(&Args { + account: "user", + groups: &None, + action, + bucket: "bucket", + object: "copy", + conditions: &conditions, + claims: &HashMap::new(), + is_owner: false, + deny_only: false + }) + .await, + allowed + ); + assert_eq!( + bucket + .is_allowed(&BucketPolicyArgs { + account: "user", + groups: &None, + action, + bucket: "bucket", + object: "copy", + conditions: &conditions, + is_owner: false + }) + .await, + allowed + ); + } + } + } + + #[tokio::test] + async fn legal_hold_policy_checks_xml_body() { + let action = Action::S3Action(S3Action::PutObjectLegalHoldAction); + for header in [None, Some("ON")] { + let mut req = build_request( + PutObjectLegalHoldInput { + bucket: "bucket".to_string(), + key: "object".to_string(), + legal_hold: Some(ObjectLockLegalHold { + status: Some(ObjectLockLegalHoldStatus::from_static("OFF")), + }), + ..Default::default() + }, + Method::PUT, + ); + ensure_req_info(&mut req); + if let Some(header) = header { + req.headers + .insert("x-amz-object-lock-legal-hold", HeaderValue::from_static(header)); + } + let _ = FS::new().put_object_legal_hold(&mut req).await; + let conditions = remaining_retention_days_conditions(&req, action); + assert_eq!(conditions.get("object-lock-legal-hold"), Some(&vec!["OFF".to_string()])); + assert_requested_value_denied(action, "s3:object-lock-legal-hold", "OFF", &conditions).await; + } + } + + #[tokio::test] + async fn tagging_policy_checks_xml_body() { + for version in [None, Some("version")] { + let mut req = build_request( + PutObjectTaggingInput { + bucket: "bucket".to_string(), + key: "object".to_string(), + version_id: version.map(str::to_string), + tagging: Tagging { + tag_set: vec![Tag { + key: Some("classification".to_string()), + value: Some("public".to_string()), + }], + }, + checksum_algorithm: None, + content_md5: None, + expected_bucket_owner: None, + request_payer: None, + }, + Method::PUT, + ); + ensure_req_info(&mut req); + req.headers + .insert("x-amz-tagging", HeaderValue::from_static("classification=secret&spoofed=yes")); + let _ = FS::new().put_object_tagging(&mut req).await; + let action = versioned_tagging_action(S3Action::PutObjectTaggingAction, version); + let conditions = remaining_retention_days_conditions(&req, action); + assert_eq!(conditions.get("RequestObjectTag/classification"), Some(&vec!["public".to_string()])); + assert_eq!(conditions.get("RequestObjectTagKeys"), Some(&vec!["classification".to_string()])); + assert!(!conditions.contains_key("RequestObjectTag/spoofed")); + assert_requested_value_denied(action, "s3:RequestObjectTag/classification", "public", &conditions).await; + } + } + + async fn assert_requested_value_denied(action: Action, key: &str, value: &str, conditions: &HashMap>) { + let document = serde_json::json!({ + "Version": "2012-10-17", + "Statement": [ + {"Effect": "Allow", "Action": action, "Resource": "arn:aws:s3:::bucket/*"}, + {"Effect": "Deny", "Action": action, "Resource": "arn:aws:s3:::bucket/*", + "Condition": {"StringEquals": {key: value}}} + ] + }); + let policy = rustfs_policy::policy::Policy::parse_config(&serde_json::to_vec(&document).unwrap()).unwrap(); + let claims = HashMap::new(); + let args = rustfs_policy::policy::Args { + account: "writer", + groups: &None, + action, + bucket: "bucket", + conditions, + is_owner: false, + object: "object", + claims: &claims, + deny_only: false, + }; + assert!(!policy.is_allowed(&args).await, "the applied request value must trigger Deny"); + } + + #[tokio::test] + async fn post_object_policy_checks_form_fields() { + let mut req = build_request( + PutObjectInput { + bucket: "bucket".to_string(), + key: "object".to_string(), + object_lock_mode: Some(ObjectLockMode::from_static("COMPLIANCE")), + object_lock_legal_hold_status: Some(ObjectLockLegalHoldStatus::from_static("ON")), + object_lock_retain_until_date: Some(Timestamp::parse(TimestampFormat::DateTime, "2035-01-01T00:00:00Z").unwrap()), + storage_class: Some(StorageClass::from_static("STANDARD_IA")), + tagging: Some("classification=public".to_string()), + ..Default::default() + }, + Method::POST, + ); + ensure_req_info(&mut req); + for (key, value) in [ + ("x-amz-object-lock-mode", "GOVERNANCE"), + ("x-amz-object-lock-legal-hold", "OFF"), + ("x-amz-object-lock-retain-until-date", "2029-01-01T00:00:00Z"), + ("x-amz-storage-class", "STANDARD"), + ("x-amz-tagging", "classification=secret&spoofed=yes"), + ("x-amz-acl", "public-read"), + ] { + req.headers.insert(key, HeaderValue::from_static(value)); + } + let _ = FS::new().put_object(&mut req).await; + let action = Action::S3Action(S3Action::PutObjectAction); + let conditions = remaining_retention_days_conditions(&req, action); + for (key, value) in [ + ("object-lock-mode", "COMPLIANCE"), + ("object-lock-legal-hold", "ON"), + ("object-lock-retain-until-date", "2035-01-01T00:00:00Z"), + ("x-amz-storage-class", "STANDARD_IA"), + ("RequestObjectTag/classification", "public"), + ] { + assert_eq!(conditions.get(key), Some(&vec![value.to_string()])); + } + assert!(!conditions.contains_key("x-amz-acl")); + assert!(!conditions.contains_key("RequestObjectTag/spoofed")); + assert_requested_value_denied(action, "s3:object-lock-mode", "COMPLIANCE", &conditions).await; + assert_requested_value_denied(action, "s3:RequestObjectTag/classification", "public", &conditions).await; + let post_req = req.map_input(|input| PostObjectInput { + bucket: input.bucket, + key: input.key, + object_lock_mode: input.object_lock_mode, + object_lock_legal_hold_status: input.object_lock_legal_hold_status, + object_lock_retain_until_date: input.object_lock_retain_until_date, + storage_class: input.storage_class, + tagging: input.tagging, + ..Default::default() + }); + let post_conditions = remaining_retention_days_conditions(&post_req, action); + assert_eq!(post_conditions.get("object-lock-mode"), conditions.get("object-lock-mode")); + assert_eq!( + post_conditions.get("RequestObjectTag/classification"), + conditions.get("RequestObjectTag/classification") + ); + } + #[tokio::test] async fn retention_policy_checks_body_instead_of_conflicting_headers() { use rustfs_policy::policy::{Args, Policy}; @@ -5258,6 +5879,23 @@ mod tests { assert!(!conditions.contains_key("RequestObjectTagKeys")); } + #[test] + fn replication_proxy_requires_authorization_independent_of_local_tags() { + let mut extensions = http::Extensions::new(); + assert!(super::replication_proxy_allowed(&extensions)); + // A local miss produces an empty tag map, but still establishes that the + // policy needs tags. Neither anonymous nor authenticated callers may proxy. + extensions.insert(ObjectTagConditions::new("bucket", "missing", None, HashMap::new())); + assert!(!super::replication_proxy_allowed(&extensions)); + extensions.insert(ObjectTagConditions::new( + "bucket", + "missing", + Some("version"), + HashMap::from([("ExistingObjectTag/protected".to_string(), vec!["yes".to_string()])]), + )); + assert!(!super::replication_proxy_allowed(&extensions)); + } + #[tokio::test] async fn test_required_existing_object_tags_are_merged_into_authorization_conditions() { let (_temp_dir, _disk_paths, store) = crate::app::gating_test_env::isolated_multi_pool_ecstore().await; @@ -5290,6 +5928,7 @@ mod tests { .expect("merge required existing object tags into authorization conditions"); assert_eq!(conditions.get("ExistingObjectTag/security"), Some(&vec!["restricted".to_string()])); + assert!(!super::replication_proxy_allowed(&request.extensions)); assert_eq!(conditions.get("ExistingObjectTag/project"), Some(&vec!["webapp".to_string()])); assert_eq!(conditions.get("delimiter"), Some(&vec!["/".to_string()])); } @@ -5348,6 +5987,42 @@ mod tests { assert!(conservative_with_tag_keys); } + #[tokio::test] + #[serial] + async fn existing_tag_hint_reuses_parsed_metadata_and_fails_closed_without_it() { + let store = crate::app::gating_test_env::shared_gating_ecstore().await; + let bucket = format!("cached-policy-hint-{}", uuid::Uuid::new_v4()); + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create hint bucket"); + let document = serde_json::json!({"Version":"2012-10-17","Statement":[{ + "Effect":"Deny","Principal":"*","Action":"s3:GetObject","Resource":format!("arn:aws:s3:::{bucket}/*"), + "Condition":{"StringEquals":{"s3:ExistingObjectTag/classification":"secret"}} + }]}); + let mut metadata = (*crate::storage::get_bucket_metadata(&bucket).await.expect("bucket metadata")).clone(); + metadata.policy_config = + Some(serde_json::from_slice(&serde_json::to_vec(&document).expect("policy JSON")).expect("parsed policy")); + // A cached parsed snapshot is the authority used by actual policy + // evaluation; the tag hint must consult that same snapshot. + metadata.policy_config_json = b"invalid raw JSON".to_vec(); + crate::storage::storage_api::set_bucket_metadata(bucket.clone(), metadata.clone()) + .await + .expect("publish parsed snapshot"); + assert!(matches!( + load_bucket_policy_existing_object_tag_hint(&store, &bucket, Action::S3Action(S3Action::GetObjectAction)).await, + super::BucketPolicyExistingObjectTagHint::Parsed(_) + )); + metadata.policy_config = None; + crate::storage::storage_api::set_bucket_metadata(bucket.clone(), metadata) + .await + .expect("publish unreadable snapshot"); + assert!(matches!( + load_bucket_policy_existing_object_tag_hint(&store, &bucket, Action::S3Action(S3Action::GetObjectAction)).await, + super::BucketPolicyExistingObjectTagHint::ConservativeTagRequired + )); + } + #[test] fn test_classify_bucket_policy_raw_load_error() { assert_eq!( @@ -6131,6 +6806,179 @@ mod tests { } } + #[tokio::test] + #[serial] + async fn tag_dependent_access_hooks_preserve_local_miss_for_replication_proxy() { + let store = crate::app::gating_test_env::shared_gating_ecstore().await; + let server_ctx = ServerContextSlot::new(); + assert!(server_ctx.install(Arc::new(AppContext::new(Arc::clone(&store), Arc::new(UnreadyIam), Arc::new(TestKms))))); + let fs = FS::with_server_ctx(server_ctx); + let bucket = format!("tag-proxy-boundary-{}", uuid::Uuid::new_v4()); + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create bucket"); + let policy_json = format!( + r#"{{"Version":"2012-10-17","Statement":[ + {{"Effect":"Allow","Principal":{{"AWS":"*"}},"Action":["s3:GetObject","s3:GetObjectTagging","s3:PutObjectTagging","s3:DeleteObjectTagging"],"Resource":["arn:aws:s3:::{bucket}/*"]}}, + {{"Effect":"Deny","Principal":{{"AWS":"*"}},"Action":["s3:GetObject","s3:GetObjectTagging","s3:PutObjectTagging","s3:DeleteObjectTagging"],"Resource":["arn:aws:s3:::{bucket}/*"],"Condition":{{"StringEquals":{{"s3:ExistingObjectTag/protected":"yes"}}}}}} + ]}}"# + ); + let policy_bytes = policy_json.into_bytes(); + store + .update_bucket_metadata_config( + &bucket, + crate::storage::storage_api::ecstore_bucket::metadata::BUCKET_POLICY_CONFIG, + policy_bytes, + ) + .await + .expect("publish policy"); + macro_rules! check_hook { + ($input:expr, $method:expr, $hook:ident) => {{ + let mut req = build_request($input, $method); + ensure_req_info(&mut req); + req.extensions.insert(fs.server_ctx().clone()); + fs.$hook(&mut req) + .await + .expect("local miss does not match the tag deny"); + let evidence = req + .extensions + .get::() + .expect("authorization must retain tag dependency"); + assert!(evidence.values.is_empty(), "missing local object has no tags"); + assert!( + !super::replication_proxy_allowed(&req.extensions), + "local absence cannot authorize a tagged replica" + ); + }}; + } + check_hook!( + GetObjectInput::builder() + .bucket(bucket.clone()) + .key("missing".to_string()) + .build() + .expect("GET"), + Method::GET, + get_object + ); + check_hook!( + HeadObjectInput::builder() + .bucket(bucket.clone()) + .key("missing".to_string()) + .build() + .expect("HEAD"), + Method::HEAD, + head_object + ); + check_hook!( + GetObjectTaggingInput::builder() + .bucket(bucket.clone()) + .key("missing".to_string()) + .build() + .expect("get tags"), + Method::GET, + get_object_tagging + ); + check_hook!( + DeleteObjectTaggingInput::builder() + .bucket(bucket.clone()) + .key("missing".to_string()) + .build() + .expect("delete tags"), + Method::DELETE, + delete_object_tagging + ); + check_hook!( + PutObjectTaggingInput::builder() + .bucket(bucket.clone()) + .key("missing".to_string()) + .tagging(s3s::dto::Tagging { tag_set: Vec::new() }) + .build() + .expect("put tags"), + Method::PUT, + put_object_tagging + ); + } + + #[tokio::test] + #[serial] + async fn conditional_writes_require_read_permission_only_for_specific_etags() { + let store = crate::app::gating_test_env::shared_gating_ecstore().await; + let server_ctx = ServerContextSlot::new(); + assert!(server_ctx.install(Arc::new(AppContext::new(Arc::clone(&store), Arc::new(UnreadyIam), Arc::new(TestKms))))); + let fs = FS::with_server_ctx(server_ctx); + for readable in [false, true] { + let bucket = format!("etag-oracle-{}", uuid::Uuid::new_v4()); + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create bucket"); + let read_effect = if readable { "Allow" } else { "Deny" }; + let policy_json = format!( + r#"{{"Version":"2012-10-17","Statement":[ + {{"Effect":"Allow","Principal":{{"AWS":"*"}},"Action":["s3:PutObject"],"Resource":["arn:aws:s3:::{bucket}/*"]}}, + {{"Effect":"{read_effect}","Principal":{{"AWS":"*"}},"Action":["s3:GetObject","s3:GetObjectVersion"],"Resource":["arn:aws:s3:::{bucket}/*"]}} + ]}}"# + ); + let policy_bytes = policy_json.into_bytes(); + store + .update_bucket_metadata_config( + &bucket, + crate::storage::storage_api::ecstore_bucket::metadata::BUCKET_POLICY_CONFIG, + policy_bytes, + ) + .await + .expect("publish policy"); + for condition in ["absent", "wildcard", "dto", "header"] { + let specific = matches!(condition, "dto" | "header"); + let dto = match condition { + "dto" => Some(s3s::dto::ETagCondition::ETag(s3s::dto::ETag::Strong("guessed-etag".to_string()))), + "wildcard" => Some(s3s::dto::ETagCondition::Any), + _ => None, + }; + let mut put_input = PutObjectInput::builder() + .bucket(bucket.clone()) + .key("object".to_string()) + .build() + .expect("put"); + put_input.if_match = dto.clone(); + let mut put = build_request(put_input, Method::PUT); + ensure_req_info(&mut put); + put.extensions.insert(fs.server_ctx().clone()); + let mut complete_input = CompleteMultipartUploadInput::builder() + .bucket(bucket.clone()) + .key("object".to_string()) + .upload_id("upload".to_string()) + .build() + .expect("complete"); + complete_input.if_match = dto; + let mut complete = build_request(complete_input, Method::POST); + ensure_req_info(&mut complete); + complete.extensions.insert(fs.server_ctx().clone()); + if condition == "header" { + for headers in [&mut put.headers, &mut complete.headers] { + headers.insert(http::header::IF_MATCH, HeaderValue::from_static("\"guessed-etag\"")); + } + } + for result in [ + fs.put_object(&mut put).await, + fs.complete_multipart_upload(&mut complete).await, + ] { + if specific && !readable { + assert_eq!( + result + .expect_err("a write-only caller must not compare content fingerprints") + .code(), + &S3ErrorCode::AccessDenied + ); + } else { + result.expect("readable or wildcard/ordinary writes remain authorized"); + } + } + } + } + } + #[tokio::test] #[serial] async fn put_object_access_captures_authorized_bucket_incarnation() { @@ -6182,6 +7030,107 @@ mod tests { ); } + #[tokio::test] + #[serial] + async fn include_deleted_listing_enforces_version_permissions() { + use rustfs_iam::store::Store as _; + use rustfs_madmin::{AccountStatus, AddOrUpdateUserReq}; + struct ReadyIam(Arc>); + impl IamInterface for ReadyIam { + fn handle(&self) -> Arc> { + self.0.clone() + } + fn is_ready(&self) -> bool { + true + } + } + let (_directory, _paths, store) = crate::app::gating_test_env::isolated_multi_pool_ecstore().await; + ObjectStore::new(store.clone()) + .save_iam_config( + serde_json::json!({"version":1}), + format!("{}/format.json", *rustfs_iam::store::object::IAM_CONFIG_PREFIX), + ) + .await + .expect("seed IAM format"); + let iam = rustfs_iam::build_iam_sys(store.clone()).await.expect("initialize IAM"); + iam.create_user( + "list-reader", + &AddOrUpdateUserReq { + secret_key: "list-reader-secret".to_string(), + policy: None, + status: AccountStatus::Enabled, + }, + ) + .await + .expect("create reader"); + let user = iam.get_user("list-reader").await.expect("reader identity").credentials; + iam.set_policy( + "list-only", + rustfs_policy::policy::Policy::parse_config( + br#"{"Version":"2012-10-17","Statement":[ + {"Effect":"Allow","Action":"s3:ListBucket","Resource":"arn:aws:s3:::*"}]}"#, + ) + .expect("list policy"), + ) + .await + .expect("store identity policy"); + iam.policy_db_set("list-reader", rustfs_iam::store::UserType::Reg, false, "list-only") + .await + .expect("attach list policy"); + let bucket = format!("deleted-list-auth-{}", uuid::Uuid::new_v4()); + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create bucket"); + let slot = ServerContextSlot::new(); + assert!(slot.install(Arc::new(AppContext::new(Arc::clone(&store), Arc::new(ReadyIam(iam)), Arc::new(TestKms))))); + for (grant_versions, deny_versions) in [(false, false), (true, false), (true, true)] { + let mut statements = vec![serde_json::json!({ + "Effect": "Allow", "Principal": {"AWS": "*"}, "Action": ["s3:ListBucket"], "Resource": format!("arn:aws:s3:::{bucket}") + })]; + if grant_versions { + statements.push(serde_json::json!({ + "Effect": "Allow", "Principal": {"AWS": "*"}, "Action": ["s3:ListBucketVersions"], "Resource": format!("arn:aws:s3:::{bucket}") + })); + } + if deny_versions { + statements.push(serde_json::json!({ + "Effect": "Deny", "Principal": {"AWS": "*"}, "Action": ["s3:ListBucketVersions"], "Resource": format!("arn:aws:s3:::{bucket}") + })); + } + let policy = serde_json::json!({"Version": "2012-10-17", "Statement": statements}); + store + .update_bucket_metadata_config( + &bucket, + crate::storage::storage_api::ecstore_bucket::metadata::BUCKET_POLICY_CONFIG, + serde_json::to_vec(&policy).unwrap(), + ) + .await + .expect("publish listing policy"); + for header in ["x-rustfs-include-deleted", "x-minio-include-deleted"] { + for (value, authenticated) in [("true", true), ("false", true), ("true", false), ("false", false)] { + let mut req = build_request((), Method::GET); + req.extensions.insert(slot.clone()); + req.extensions.insert(ReqInfo { + bucket: Some(bucket.clone()), + cred: authenticated.then(|| user.clone()), + ..Default::default() + }); + req.headers.insert(header, HeaderValue::from_static(value)); + let result = super::authorize_bucket_listing(&mut req, &bucket, S3Action::ListBucketAction).await; + if value == "true" && (deny_versions || (authenticated && !grant_versions)) { + assert_eq!( + result.expect_err("version visibility requires permission").code(), + &S3ErrorCode::AccessDenied + ); + } else { + result.expect("authorized listings remain available"); + } + } + } + } + } + #[tokio::test] #[serial] async fn request_slot_keeps_bucket_policy_bound_to_its_store() { @@ -6454,6 +7403,285 @@ mod tests { .expect("the new-generation object must survive the stale request"); } + #[test] + fn copy_tags_authorize_source_and_effective_destination_tags() { + crate::app::gating_test_env::run_large_stack_test("copy-tag-authorization", || async { + let store = crate::app::gating_test_env::shared_gating_ecstore().await; + let slot = ServerContextSlot::new(); + let context = Arc::new(AppContext::new(store.clone(), Arc::new(UnreadyIam), Arc::new(TestKms))); + assert!(slot.install(context.clone())); + let fs = FS::with_server_ctx(slot.clone()); + let bucket = format!("copy-tags-{}", uuid::Uuid::new_v4()); + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("bucket"); + let mut reader = crate::storage::PutObjReader::from_vec(b"confidential source".to_vec()); + let source = store + .put_object( + &bucket, + "source", + &mut reader, + &crate::storage::ObjectOptions { + versioned: true, + user_defined: HashMap::from([( + rustfs_utils::http::AMZ_OBJECT_TAGGING.to_string(), + "visibility=private".to_string(), + )]), + ..Default::default() + }, + ) + .await + .expect("tagged historical source"); + let historical = source.version_id.expect("versioned source").to_string(); + let mut reader = crate::storage::PutObjReader::from_vec(b"current source".to_vec()); + store + .put_object( + &bucket, + "source", + &mut reader, + &crate::storage::ObjectOptions { + versioned: true, + user_defined: HashMap::from([( + rustfs_utils::http::AMZ_OBJECT_TAGGING.to_string(), + "visibility=private".to_string(), + )]), + ..Default::default() + }, + ) + .await + .expect("current tagged source"); + for (key, tags) in [("source-null", "visibility=private"), ("source-empty", "")] { + let mut reader = crate::storage::PutObjReader::from_vec(b"unversioned source".to_vec()); + store + .put_object( + &bucket, + key, + &mut reader, + &crate::storage::ObjectOptions { + user_defined: HashMap::from([(rustfs_utils::http::AMZ_OBJECT_TAGGING.to_string(), tags.to_string())]), + ..Default::default() + }, + ) + .await + .expect("unversioned source"); + } + for (index, (directive, replacement, explicit_version, tag_permission, deny_public, expected)) in [ + (None, None, None, "missing", false, false), + (Some("COPY"), None, None, "deny", false, false), + (None, None, None, "current", false, true), + (Some("COPY"), None, Some("historical"), "current", false, false), + (Some("COPY"), None, Some("historical"), "version", false, true), + (Some("COPY"), None, Some("null"), "current", false, false), + (Some("COPY"), None, Some("null"), "version", false, true), + (None, None, None, "empty", false, true), + (Some("REPLACE"), Some("visibility=private"), None, "deny", false, true), + (Some("REPLACE"), Some(""), None, "deny", false, true), + (Some("REPLACE"), Some("visibility=public"), None, "deny", true, false), + ] + .into_iter() + .enumerate() + { + let mut statements = vec![ + serde_json::json!({"Effect":"Allow","Principal":{"AWS":"*"},"Action":["s3:GetObject","s3:GetObjectVersion"],"Resource":format!("arn:aws:s3:::{bucket}/source*")}), + serde_json::json!({"Effect":"Allow","Principal":{"AWS":"*"},"Action":"s3:PutObject","Resource":format!("arn:aws:s3:::{bucket}/copied-*")}), + ]; + if !matches!(tag_permission, "missing" | "empty") { + let action = if tag_permission == "version" { + "s3:GetObjectVersionTagging" + } else { + "s3:GetObjectTagging" + }; + statements.push(serde_json::json!({"Effect":if tag_permission == "deny" { "Deny" } else { "Allow" },"Principal":{"AWS":"*"},"Action":action,"Resource":format!("arn:aws:s3:::{bucket}/source*")})); + } + if deny_public { + statements.push(serde_json::json!({"Effect":"Deny","Principal":{"AWS":"*"},"Action":"s3:PutObject","Resource":format!("arn:aws:s3:::{bucket}/*"),"Condition":{"StringEquals":{"s3:RequestObjectTag/visibility":"public"}}})); + } + let policy = + serde_json::to_vec(&serde_json::json!({"Version":"2012-10-17","Statement":statements})).expect("policy"); + store + .update_bucket_metadata_config( + &bucket, + crate::storage::storage_api::ecstore_bucket::metadata::BUCKET_POLICY_CONFIG, + policy, + ) + .await + .expect("policy publication"); + let destination = format!("copied-{index}"); + let input = CopyObjectInput::builder() + .bucket(bucket.clone()) + .key(destination.clone()) + .copy_source(CopySource::Bucket { + bucket: bucket.clone().into(), + key: (if tag_permission == "empty" { + "source-empty" + } else if explicit_version == Some("null") { + "source-null" + } else { + "source" + }) + .into(), + version_id: explicit_version.map(|version| { + if version == "historical" { + historical.clone().into() + } else { + "null".into() + } + }), + }) + .tagging_directive(directive.map(|value| TaggingDirective::from(value.to_string()))) + .tagging(replacement.map(str::to_string)) + .build() + .expect("copy input"); + let mut request = build_request(input, Method::PUT); + ensure_req_info(&mut request); + request.extensions.insert(slot.clone()); + let access = fs.copy_object(&mut request).await; + if !expected { + assert_eq!( + access.expect_err("copy must enforce tagging and destination policy").code(), + &S3ErrorCode::AccessDenied, + "case {index}" + ); + assert!( + store + .get_object_info(&bucket, &destination, &crate::storage::ObjectOptions::default()) + .await + .is_err() + ); + continue; + } + access.expect("authorized copy"); + crate::app::object_usecase::DefaultObjectUsecase::with_context(Some(context.clone())) + .execute_copy_object(request) + .await + .expect("copy preserves allowed tags"); + assert_eq!( + store + .get_object_tags(&bucket, &destination, &crate::storage::ObjectOptions::default()) + .await + .expect("destination tags"), + replacement.unwrap_or(if tag_permission == "empty" { "" } else { "visibility=private" }) + ); + } + }); + } + + #[test] + fn copy_tags_rechecks_actual_source_after_access_snapshot() { + crate::app::gating_test_env::run_large_stack_test("copy-tag-snapshot", || async { + let store = crate::app::gating_test_env::shared_gating_ecstore().await; + let slot = ServerContextSlot::new(); + let context = Arc::new(AppContext::new(store.clone(), Arc::new(UnreadyIam), Arc::new(TestKms))); + assert!(slot.install(context.clone())); + let fs = FS::with_server_ctx(slot.clone()); + for mode in ["tag-change", "new-tags", "allowed-tags", "source-tag-change", "self-copy"] { + let bucket = format!("copy-snapshot-{}", uuid::Uuid::new_v4()); + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("bucket"); + let destination = if mode == "self-copy" { "source" } else { "destination" }; + let initial = if mode == "new-tags" { "" } else { "visibility=private" }; + let mut reader = crate::storage::PutObjReader::from_vec(b"source".to_vec()); + store + .put_object( + &bucket, + "source", + &mut reader, + &crate::storage::ObjectOptions { + user_defined: HashMap::from([( + rustfs_utils::http::AMZ_OBJECT_TAGGING.to_string(), + initial.to_string(), + )]), + ..Default::default() + }, + ) + .await + .expect("source"); + let mut statements = vec![ + serde_json::json!({"Effect":"Allow","Principal":{"AWS":"*"},"Action":"s3:GetObject","Resource":format!("arn:aws:s3:::{bucket}/source")}), + serde_json::json!({"Effect":"Allow","Principal":{"AWS":"*"},"Action":"s3:PutObject","Resource":format!("arn:aws:s3:::{bucket}/{destination}")}), + serde_json::json!({"Effect":"Deny","Principal":{"AWS":"*"},"Action":"s3:PutObject","Resource":format!("arn:aws:s3:::{bucket}/*"),"Condition":{"StringEquals":{"s3:RequestObjectTag/visibility":"public"}}}), + ]; + if mode != "new-tags" { + statements.push(serde_json::json!({"Effect":"Allow","Principal":{"AWS":"*"},"Action":"s3:GetObjectTagging","Resource":format!("arn:aws:s3:::{bucket}/source")})); + // A conditional Allow must work at the preliminary hook as well. + statements[1]["Condition"] = serde_json::json!({"StringEquals":{"s3:RequestObjectTag/visibility":"private"}}); + if mode == "source-tag-change" { + statements[1].as_object_mut().expect("statement").remove("Condition"); + statements.remove(2); + statements.last_mut().expect("tag read statement")["Condition"] = + serde_json::json!({"StringEquals":{"s3:ExistingObjectTag/visibility":"private"}}); + } else if mode == "self-copy" { + statements[1]["Condition"]["StringEquals"]["s3:ExistingObjectTag/visibility"] = + serde_json::json!("private"); + statements.last_mut().expect("tag read statement")["Condition"] = + serde_json::json!({"StringEquals":{"s3:ExistingObjectTag/visibility":"private"}}); + } + } + store + .update_bucket_metadata_config( + &bucket, + crate::storage::storage_api::ecstore_bucket::metadata::BUCKET_POLICY_CONFIG, + serde_json::to_vec(&serde_json::json!({"Version":"2012-10-17","Statement":statements})).expect("policy"), + ) + .await + .expect("publish policy"); + let input = CopyObjectInput::builder() + .bucket(bucket.clone()) + .key(destination.to_string()) + .copy_source(CopySource::Bucket { + bucket: bucket.clone().into(), + key: "source".into(), + version_id: None, + }) + .metadata_directive((mode == "self-copy").then(|| MetadataDirective::from_static(MetadataDirective::REPLACE))) + .build() + .expect("copy"); + let mut request = build_request(input, Method::PUT); + ensure_req_info(&mut request); + request.extensions.insert(slot.clone()); + fs.copy_object(&mut request).await.expect("preliminary snapshot permits copy"); + if !matches!(mode, "allowed-tags" | "self-copy") { + store + .put_object_tags(&bucket, "source", "visibility=public", &crate::storage::ObjectOptions::default()) + .await + .expect("change source after access hook"); + } + let result = tokio::time::timeout( + std::time::Duration::from_secs(10), + crate::app::object_usecase::DefaultObjectUsecase::with_context(Some(context.clone())) + .execute_copy_object(request), + ) + .await + .expect("copy authorization must not reacquire its own object lock"); + if matches!(mode, "allowed-tags" | "self-copy") { + result.expect("unchanged allowed tags complete"); + assert_eq!( + store + .get_object_tags(&bucket, destination, &crate::storage::ObjectOptions::default()) + .await + .expect("copied tags"), + "visibility=private" + ); + } else { + assert_eq!( + result.expect_err("actual source tags require fresh authorization").code(), + &S3ErrorCode::AccessDenied + ); + assert!( + store + .get_object_info(&bucket, "destination", &crate::storage::ObjectOptions::default()) + .await + .is_err(), + "denial must precede destination publication" + ); + } + } + }); + } + #[tokio::test] #[serial] async fn copy_operations_reject_recreated_source_bucket_after_authorization() { diff --git a/rustfs/src/storage/ecfs.rs b/rustfs/src/storage/ecfs.rs index d0afd65c9..97b88aed4 100644 --- a/rustfs/src/storage/ecfs.rs +++ b/rustfs/src/storage/ecfs.rs @@ -23,7 +23,9 @@ use super::{ }; use crate::admin::handlers::site_replication::site_replication_bucket_meta_hook; use crate::error::{ApiError, ApiErrorDiagnostic}; -use crate::storage::access::{apply_bucket_generation_guard, bucket_config_mutation_incarnation, has_bypass_governance_header}; +use crate::storage::access::{ + apply_bucket_generation_guard, bucket_config_mutation_incarnation, has_bypass_governance_header, replication_proxy_allowed, +}; use crate::storage::helper::{OperationHelper, audit_only_operation}; use crate::storage::options::get_opts; use crate::storage::s3_api::{self, acl}; @@ -722,7 +724,7 @@ impl FS { #[async_trait::async_trait] impl S3 for FS { - #[instrument(level = "debug", skip(self))] + #[instrument(level = "debug", skip(self, req))] async fn abort_multipart_upload( &self, req: S3Request, @@ -775,7 +777,7 @@ impl S3 for FS { usecase.execute_delete_bucket(req).await } - #[instrument(level = "debug", skip(self))] + #[instrument(level = "debug", skip(self, req))] async fn delete_bucket_cors(&self, req: S3Request) -> S3Result> { let usecase = s3_api::bucket_usecase_for(self); audit_only_operation(req, S3Operation::DeleteBucketCors, |req| usecase.execute_delete_bucket_cors(req)).await @@ -792,7 +794,7 @@ impl S3 for FS { .await } - #[instrument(level = "debug", skip(self))] + #[instrument(level = "debug", skip(self, req))] async fn delete_bucket_lifecycle( &self, req: S3Request, @@ -819,7 +821,7 @@ impl S3 for FS { audit_only_operation(req, S3Operation::DeleteBucketReplication, |req| self.delete_bucket_replication_inner(req)).await } - #[instrument(level = "debug", skip(self))] + #[instrument(level = "debug", skip(self, req))] async fn delete_bucket_tagging( &self, req: S3Request, @@ -849,7 +851,7 @@ impl S3 for FS { Ok(S3Response::new(DeleteBucketWebsiteOutput::default())) } - #[instrument(level = "debug", skip(self))] + #[instrument(level = "debug", skip(self, req))] async fn delete_public_access_block( &self, req: S3Request, @@ -877,7 +879,7 @@ impl S3 for FS { result } - #[instrument(level = "debug", skip(self))] + #[instrument(level = "debug", skip(self, req))] async fn delete_object_tagging( &self, req: S3Request, @@ -916,6 +918,7 @@ impl S3 for FS { // (backlog#1675 P1-5). No local object exists, so no bucket // notification event is emitted for the proxied write. if (is_err_object_not_found(&e) || is_err_version_not_found(&e)) + && replication_proxy_allowed(&req.extensions) && Self::proxy_delete_object_tagging(&bucket, &object, version_id.clone(), &req.headers) .await .is_some() @@ -1045,7 +1048,7 @@ impl S3 for FS { } } - #[instrument(level = "debug", skip(self))] + #[instrument(level = "debug", skip(self, req))] async fn get_bucket_cors(&self, req: S3Request) -> S3Result> { record_s3_op(S3Operation::GetBucketCors); let usecase = s3_api::bucket_usecase_for(self); @@ -1061,7 +1064,7 @@ impl S3 for FS { usecase.execute_get_bucket_encryption(req).await } - #[instrument(level = "debug", skip(self))] + #[instrument(level = "debug", skip(self, req))] async fn get_bucket_lifecycle_configuration( &self, req: S3Request, @@ -1136,14 +1139,14 @@ impl S3 for FS { } } - #[instrument(level = "debug", skip(self))] + #[instrument(level = "debug", skip(self, req))] async fn get_bucket_tagging(&self, req: S3Request) -> S3Result> { record_s3_op(S3Operation::GetBucketTagging); let usecase = s3_api::bucket_usecase_for(self); usecase.execute_get_bucket_tagging(req).await } - #[instrument(level = "debug", skip(self))] + #[instrument(level = "debug", skip(self, req))] async fn get_public_access_block( &self, req: S3Request, @@ -1153,7 +1156,7 @@ impl S3 for FS { usecase.execute_get_public_access_block(req).await } - #[instrument(level = "debug", skip(self))] + #[instrument(level = "debug", skip(self, req))] async fn get_bucket_versioning( &self, req: S3Request, @@ -1296,7 +1299,7 @@ impl S3 for FS { result } - #[instrument(level = "debug", skip(self))] + #[instrument(level = "debug", skip(self, req))] async fn get_object_lock_configuration( &self, req: S3Request, @@ -1388,7 +1391,7 @@ impl S3 for FS { result } - #[instrument(level = "debug", skip(self))] + #[instrument(level = "debug", skip(self, req))] async fn get_object_tagging(&self, req: S3Request) -> S3Result> { use crate::storage::storage_api::ecstore_bucket::versioning::VersioningApi as _; @@ -1436,6 +1439,7 @@ impl S3 for FS { // replication target even though it is missing locally — // proxy the tagging read there (backlog#1675 P1-5). if (is_err_object_not_found(&e) || is_err_version_not_found(&e)) + && replication_proxy_allowed(&req.extensions) && let Some(output) = Self::proxy_get_object_tagging(bucket, object, req.input.version_id.clone(), &req.headers).await { @@ -1516,7 +1520,7 @@ impl S3 for FS { usecase.execute_head_object(req).await } - #[instrument(level = "debug", skip(self))] + #[instrument(level = "debug", skip(self, req))] async fn list_buckets(&self, req: S3Request) -> S3Result> { // List buckets not associated with a bucket, give it bucket label "*" to denote "all". record_s3_op(S3Operation::ListBuckets); @@ -1604,7 +1608,7 @@ impl S3 for FS { Ok(S3Response::new(PutBucketAccelerateConfigurationOutput::default())) } - #[instrument(level = "debug", skip(self))] + #[instrument(level = "debug", skip(self, req))] async fn put_bucket_cors(&self, req: S3Request) -> S3Result> { let usecase = s3_api::bucket_usecase_for(self); audit_only_operation(req, S3Operation::PutBucketCors, |req| usecase.execute_put_bucket_cors(req)).await @@ -1641,7 +1645,7 @@ impl S3 for FS { audit_only_operation(req, S3Operation::PutBucketEncryption, |req| usecase.execute_put_bucket_encryption(req)).await } - #[instrument(level = "debug", skip(self))] + #[instrument(level = "debug", skip(self, req))] async fn put_bucket_lifecycle_configuration( &self, req: S3Request, @@ -1703,7 +1707,7 @@ impl S3 for FS { Ok(S3Response::new(PutBucketRequestPaymentOutput::default())) } - #[instrument(level = "debug", skip(self))] + #[instrument(level = "debug", skip(self, req))] async fn put_public_access_block( &self, req: S3Request, @@ -1712,13 +1716,13 @@ impl S3 for FS { audit_only_operation(req, S3Operation::PutPublicAccessBlock, |req| usecase.execute_put_public_access_block(req)).await } - #[instrument(level = "debug", skip(self))] + #[instrument(level = "debug", skip(self, req))] async fn put_bucket_tagging(&self, req: S3Request) -> S3Result> { let usecase = s3_api::bucket_usecase_for(self); audit_only_operation(req, S3Operation::PutBucketTagging, |req| usecase.execute_put_bucket_tagging(req)).await } - #[instrument(level = "debug", skip(self))] + #[instrument(level = "debug", skip(self, req))] async fn put_bucket_versioning( &self, req: S3Request, @@ -1894,7 +1898,7 @@ impl S3 for FS { result } - #[instrument(level = "debug", skip(self))] + #[instrument(level = "debug", skip(self, req))] async fn put_object_lock_configuration( &self, req: S3Request, @@ -2061,6 +2065,7 @@ impl S3 for FS { // (backlog#1675 P1-5). No local object exists, so no bucket // notification event is emitted for the proxied write. if (is_err_object_not_found(&e) || is_err_version_not_found(&e)) + && replication_proxy_allowed(&req.extensions) && Self::proxy_put_object_tagging(&bucket, &object, version_id.clone(), &req.headers, &tagging.tag_set) .await .is_some() diff --git a/rustfs/src/storage/ecfs_extend.rs b/rustfs/src/storage/ecfs_extend.rs index 7e2e4c143..d0aea36f6 100644 --- a/rustfs/src/storage/ecfs_extend.rs +++ b/rustfs/src/storage/ecfs_extend.rs @@ -322,11 +322,36 @@ pub(crate) fn get_buffer_size_opt_in(file_size: i64) -> usize { buffer_size } -/// Validate object key for control characters and log special characters +/// Reject XML-incompatible and storage-sidecar keys at new object boundaries. +pub(crate) fn validate_new_object_key(key: &str, operation: &str) -> S3Result<()> { + validate_object_key(key, operation)?; + // These persistent sidecar filenames share directories with client objects. + // Case-insensitive checks also protect case-insensitive disk deployments. + if key.split('/').any(|segment| { + ["xl.meta", "xl.meta.bkp", "delete-prepared", "delete-quorum-committed"] + .iter() + .any(|reserved| segment.eq_ignore_ascii_case(reserved)) + }) { + return Err(S3Error::with_message( + S3ErrorCode::InvalidArgument, + "Object key contains a reserved storage filename", + )); + } + if crate::storage::storage_api::ecstore_bucket::utils::has_xml_invalid_control_characters(key) { + return Err(S3Error::with_message( + S3ErrorCode::InvalidArgument, + "Object key contains characters unsupported by XML listings", + )); + } + Ok(()) +} + +/// Validate client object keys and log special characters /// /// This function: /// 1. Rejects keys containing control characters (null bytes, newlines, carriage returns) -/// 2. Logs debug information for keys containing spaces, plus signs, or percent signs +/// 2. Rejects reserved directory markers and invalid path segments +/// 3. Logs debug information for keys containing spaces, plus signs, or percent signs /// /// The s3s library handles URL decoding, so keys are already decoded when they reach this function. /// This validation ensures that invalid characters that could cause issues are rejected early. @@ -1155,7 +1180,39 @@ pub(crate) fn parse_part_number_i32_to_usize(part_number: Option, op: &'sta #[cfg(test)] mod object_key_admission_tests { - use super::validate_object_key; + use super::{validate_new_object_key, validate_object_key}; + + #[test] + fn storage_sidecar_keys_are_rejected_only_for_new_writes() { + for reserved in ["xl.meta", "xl.meta.bkp", "delete-prepared", "delete-quorum-committed"] { + for key in [ + reserved.to_string(), + format!("parent/uuid/{reserved}"), + format!("parent/{reserved}/child"), + reserved.to_uppercase(), + ] { + for operation in ["PutObject", "COPY (dest)", "CreateMultipartUpload", "CompleteMultipartUpload"] { + assert!(validate_new_object_key(&key, operation).is_err(), "{operation}: {key}"); + } + assert!(validate_object_key(&key, "GetObject").is_ok(), "legacy read: {key}"); + assert!(validate_object_key(&key, "DeleteObject").is_ok(), "legacy cleanup: {key}"); + } + assert!(validate_new_object_key(&format!("ordinary/{reserved}-report"), "PutObject").is_ok()); + } + } + + #[test] + fn xml_invalid_keys_are_rejected_only_for_new_writes() { + for byte in (1u8..32).filter(|byte| !matches!(byte, 9 | 10 | 13)) { + let key = format!("legacy-{}", char::from(byte)); + assert!(validate_new_object_key(&key, "PutObject").is_err()); + assert!(validate_object_key(&key, "GetObject").is_ok()); + assert!(validate_object_key(&key, "DeleteObject").is_ok()); + } + for key in ["ordinary/key", "tab\tkey", "unicode-中文", "directory/"] { + assert!(validate_new_object_key(key, "PutObject").is_ok()); + } + } #[test] fn reserved_directory_marker_keys_are_rejected() { diff --git a/rustfs/src/storage/ecfs_test.rs b/rustfs/src/storage/ecfs_test.rs index 22e870cbf..c7e0cf3e8 100644 --- a/rustfs/src/storage/ecfs_test.rs +++ b/rustfs/src/storage/ecfs_test.rs @@ -14,6 +14,26 @@ #[cfg(test)] mod tests { + #[test] + fn object_key_rejects_reserved_directory_markers() { + for key in [ + "__XLDIR__", + "prefix/__XLDIR__", + "stash__XLDIR__/blob", + "file__XLDIR__", + "file__XLDIR__/", + "a__XLDIR__b", + ] { + let error = crate::storage::ecfs_extend::validate_object_key(key, "PUT") + .expect_err("client keys must not alias internal directory markers"); + assert_eq!(error.code(), &s3s::S3ErrorCode::InvalidArgument, "{key}"); + } + for key in ["file", "folder/", "prefix/folder/", "__xldir__", "percent%20name"] { + crate::storage::ecfs_extend::validate_object_key(key, "PUT") + .expect("ordinary objects and directory objects remain valid"); + } + } + use crate::config::WorkloadProfile; use crate::server::cors; use crate::storage::StorageError; diff --git a/rustfs/src/storage/helper.rs b/rustfs/src/storage/helper.rs index 1a8b62601..93f715422 100644 --- a/rustfs/src/storage/helper.rs +++ b/rustfs/src/storage/helper.rs @@ -94,6 +94,31 @@ where } } +/// Own a parsed mutation through its storage and event/audit completion. +/// Dropping the HTTP waiter does not cancel the operation. Streaming bodies +/// must use their bounded upload owner instead of this entry point. +pub(crate) async fn run_owned_mutation(operation: F) -> S3Result> +where + T: Send + 'static, + F: Future>> + Send + 'static, +{ + let credentials = rustfs_keystone::KEYSTONE_CREDENTIALS.try_with(Clone::clone).ok().flatten(); + let operation = rustfs_keystone::KEYSTONE_CREDENTIALS.scope(credentials, operation); + #[cfg(feature = "diagnostic-read-version-caller")] + let operation = ecstore_diagnostic::with_captured_read_version_context( + ecstore_diagnostic::ReadVersionContextBoundary::BackgroundTask, + operation, + ); + #[cfg(not(feature = "diagnostic-read-version-caller"))] + let operation = rustfs_io_metrics::s3_http_metrics::with_s3_http_operation_context( + rustfs_io_metrics::s3_http_metrics::current_s3_http_operation_context(), + operation, + ); + super::request_context::spawn_traced_join(operation) + .await + .map_err(|_| s3s::s3_error!(InternalError, "object mutation owner task failed"))? +} + /// Builds S3-compatible notification response elements with the canonical request ID. pub(crate) fn build_event_resp_elements(response: &S3Response, request_id: &str) -> HashMap { let mut resp_elements = extract_resp_elements(response); @@ -137,6 +162,84 @@ fn request_audit_builder( builder } +#[cfg(test)] +type MutationCompletionPause = (std::sync::Arc, std::sync::Arc); +#[cfg(test)] +type MutationCompletionProbes = HashMap>; + +#[cfg(test)] +#[derive(Default)] +pub(crate) struct MutationCompletionProbe { + pub(crate) audits: std::sync::Mutex>, + pub(crate) events: std::sync::Mutex>, + changed: tokio::sync::Notify, + pause: std::sync::Mutex>, +} + +#[cfg(test)] +fn completion_probes() -> &'static std::sync::Mutex { + static PROBES: std::sync::OnceLock> = std::sync::OnceLock::new(); + PROBES.get_or_init(Default::default) +} + +#[cfg(test)] +fn completion_probe(bucket: &str) -> Option> { + completion_probes() + .lock() + .expect("completion probes") + .get(bucket) + .and_then(std::sync::Weak::upgrade) +} + +#[cfg(test)] +impl MutationCompletionProbe { + pub(crate) fn install(bucket: &str) -> std::sync::Arc { + let probe = std::sync::Arc::new(Self::default()); + let mut probes = completion_probes().lock().expect("completion probes"); + probes.retain(|_, value| value.strong_count() > 0); + probes.insert(bucket.to_owned(), std::sync::Arc::downgrade(&probe)); + probe + } + + pub(crate) fn pause_after_commit(&self) -> MutationCompletionPause { + let entered = std::sync::Arc::new(tokio::sync::Barrier::new(2)); + let resume = std::sync::Arc::new(tokio::sync::Barrier::new(2)); + *self.pause.lock().expect("completion pause") = Some((entered.clone(), resume.clone())); + (entered, resume) + } + + pub(crate) async fn wait_for(&self, audits: usize, events: usize) { + tokio::time::timeout(std::time::Duration::from_secs(10), async { + loop { + let changed = self.changed.notified(); + if self.audits.lock().expect("audits").len() >= audits && self.events.lock().expect("events").len() >= events { + break; + } + changed.await; + } + }) + .await + .expect("committed mutation must dispatch its audit and events"); + } +} + +#[cfg(test)] +pub(crate) async fn pause_committed_mutation_for_test(bucket: &str) { + let pause = completion_probe(bucket).and_then(|probe| probe.pause.lock().expect("completion pause").take()); + if let Some((entered, resume)) = pause { + entered.wait().await; + resume.wait().await; + } +} + +#[cfg(test)] +pub(crate) fn capture_mutation_event_for_test(event: &rustfs_notify::EventArgs) { + if let Some(probe) = completion_probe(&event.bucket_name) { + probe.events.lock().expect("events").push(event.clone()); + probe.changed.notify_one(); + } +} + /// A unified helper structure for building and distributing audit logs and event notifications via RAII mode at the end of an S3 operation scope. pub enum OperationHelper { Disabled, @@ -191,8 +294,20 @@ impl OperationHelper { ); } + #[cfg(test)] + let probe_enabled = completion_probe( + req.extensions + .get::() + .and_then(|info| info.bucket.as_deref()) + .unwrap_or_else(|| req.uri.path().trim_start_matches('/').split('/').next().unwrap_or_default()), + ) + .is_some(); let audit_enabled = is_audit_module_enabled(); let notify_enabled = !audit_only && should_build_notification_event(is_notify_module_enabled()); + #[cfg(test)] + let audit_enabled = audit_enabled || probe_enabled; + #[cfg(test)] + let notify_enabled = notify_enabled || (!audit_only && probe_enabled); if audit_only && !audit_enabled { // Nothing to record and no metric to bump: skip the path parsing below. return Self::Disabled; @@ -547,13 +662,39 @@ impl Drop for OperationHelper { return; }; + // An early error or panic must have an outcome, but is never proof + // of a successful mutation. Committed owners complete explicitly. + if state.api_builder.0.status.is_none() { + let elapsed = state.start_time.elapsed(); + state.api_builder = state + .api_builder + .clone() + .status("failure".to_string()) + .status_code(StatusCode::INTERNAL_SERVER_ERROR.as_u16() as i32) + .time_to_response(format!("{elapsed:.2?}")) + .time_to_response_in_ns(elapsed.as_nanos().to_string()); + if let Some(builder) = state.audit_builder.take() { + state.audit_builder = Some( + builder + .api(state.api_builder.clone().build()) + .error("Operation ended before reporting its outcome".to_string()), + ); + } + } + // Distribute audit logs if state.audit_enabled && let Some(builder) = state.audit_builder.take() { + let entry = builder.build(); + #[cfg(test)] + if let Some(probe) = entry.api.bucket.as_deref().and_then(completion_probe) { + probe.audits.lock().expect("audits").push(entry.clone()); + probe.changed.notify_one(); + } let ctx = Some(state.request_context.clone()); spawn_background_with_context(ctx, async move { - AuditLogger::log(builder.build()).await; + AuditLogger::log(entry).await; }); } @@ -565,6 +706,8 @@ impl Drop for OperationHelper { let event_args = builder.build(); // Avoid generating notifications for copy requests if !event_args.is_replication_request() { + #[cfg(test)] + capture_mutation_event_for_test(&event_args); let ctx = Some(state.request_context.clone()); spawn_background_with_context(ctx, async move { runtime_sources::current_notify_interface().notify(event_args).await; @@ -646,6 +789,48 @@ mod tests { assert_eq!(rx.await.expect("unscoped task completes"), (None, None)); } + #[tokio::test] + async fn owned_mutation_keeps_keystone_context_and_audits_panics_as_failure() { + let bucket = "owned-mutation-panic"; + let probe = super::MutationCompletionProbe::install(bucket); + let credentials = Credentials { + access_key: "keystone:test".to_owned(), + ..Default::default() + }; + let result: S3Result> = rustfs_keystone::KEYSTONE_CREDENTIALS + .scope(Some(credentials), async { + super::run_owned_mutation(async { + assert_eq!( + rustfs_keystone::KEYSTONE_CREDENTIALS + .with(|credentials| credentials.as_ref().map(|value| value.access_key.clone())), + Some("keystone:test".to_owned()) + ); + let req = S3Request { + input: (), + method: Method::DELETE, + uri: Uri::from_static("/owned-mutation-panic/object"), + headers: HeaderMap::new(), + extensions: Extensions::new(), + credentials: None, + region: None, + service: None, + trailing_headers: None, + }; + let _helper = OperationHelper::new(&req, EventName::ObjectRemovedDelete, S3Operation::DeleteObject); + panic!("injected mutation panic before reporting a commit"); + }) + .await + }) + .await; + assert_eq!(result.expect_err("join failure").code(), &S3ErrorCode::InternalError); + probe.wait_for(1, 0).await; + let audits = probe.audits.lock().expect("audits"); + assert_eq!(audits.len(), 1); + assert_eq!(audits[0].api.status.as_deref(), Some("failure")); + assert_eq!(audits[0].api.status_code, Some(500)); + assert!(probe.events.lock().expect("events").is_empty()); + } + #[cfg(feature = "diagnostic-read-version-caller")] #[test] fn diagnostic_background_tasks_keep_existing_entry_on_fallback_thread() { diff --git a/rustfs/src/storage/options.rs b/rustfs/src/storage/options.rs index 9378b7fd7..f53c7b811 100644 --- a/rustfs/src/storage/options.rs +++ b/rustfs/src/storage/options.rs @@ -360,6 +360,21 @@ pub async fn put_opts( put_opts_with_replication_authorization(bucket, object, vid, headers, metadata, false).await } +/// Resolve the version addressed by a write before evaluating version-scoped policy. +/// Query parameters take precedence over compatible replication headers. +pub(crate) fn destination_version_id(vid: Option<&str>, headers: &HeaderMap) -> Option { + let header = get_header(headers, SUFFIX_SOURCE_VERSION_ID); + let value = vid.or(header.as_deref())?.trim(); + if value.eq_ignore_ascii_case("null") { + return Some("null".to_owned()); + } + Some(match Uuid::parse_str(value) { + Ok(id) if id.is_nil() => "null".to_owned(), + Ok(id) => id.to_string(), + Err(_) => value.to_owned(), + }) +} + pub async fn put_opts_with_replication_authorization( bucket: &str, object: &str, @@ -379,8 +394,8 @@ pub async fn put_opts_with_replication_authorization( let versioned = versioning_cfg.prefix_enabled(object); let version_suspended = versioning_cfg.prefix_suspended(object); - let vid = if vid.is_none() && replication_request_authorized { - get_header(headers, SUFFIX_SOURCE_VERSION_ID).map(|s| s.into_owned()) + let vid = if replication_request_authorized { + destination_version_id(vid.as_deref(), headers) } else { vid }; @@ -439,7 +454,7 @@ pub fn get_complete_multipart_upload_opts_with_replication_authorization( if replication_request_authorized && get_header(headers, SUFFIX_SOURCE_REPLICATION_REQUEST).as_deref() == Some("true") { replication_request = true; mod_time = replication_source_mtime(headers); - preserve_etag = replication_source_etag(headers); + preserve_etag = replication_source_etag(headers)?; if let Some(actual_size_str) = get_header(headers, SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE) { rustfs_utils::http::insert_str( &mut user_defined, @@ -518,25 +533,47 @@ pub fn put_opts_from_headers_with_replication_authorization( if replication_request_authorized && get_header(headers, SUFFIX_SOURCE_REPLICATION_REQUEST).as_deref() == Some("true") { opts.replication_request = true; opts.mod_time = replication_source_mtime(headers); - opts.preserve_etag = replication_source_etag(headers); + opts.preserve_etag = replication_source_etag(headers)?; // SSE-C ciphertext passthrough: restore the stored encryption metadata // from the transport headers and mark the body as already encrypted so // the write path stores it verbatim. if let Some(restored) = rustfs_utils::http::ssec_transport_to_stored_metadata(headers) { + if restored + .get(rustfs_utils::http::object_encryption_keys::SSEC_ORIGINAL_SIZE_HEADER) + .is_some_and(|value| !value.parse::().is_ok_and(|size| size >= 0)) + { + return Err(StorageError::InvalidArgument( + String::new(), + String::new(), + "Invalid replicated SSE-C original size".to_string(), + )); + } opts.user_defined.extend(restored); opts.preserve_ciphertext = true; // A compressed passthrough object restores its compression // layout as well, so the replica decompresses after decrypting - // (backlog#2363). The value is validated when the object is read. + // (backlog#2363). Reject unreadable layouts before persisting them. if let Some(scheme) = get_header(headers, rustfs_utils::http::SUFFIX_REPLICATION_COMPRESSION) { + super::storage_api::options_consumer::validate_compression_scheme(&scheme).map_err(|_| { + StorageError::InvalidArgument( + String::new(), + String::new(), + "Unsupported replicated compression scheme".to_string(), + ) + })?; rustfs_utils::http::insert_str( &mut opts.user_defined, rustfs_utils::http::SUFFIX_COMPRESSION, scheme.into_owned(), ); - if let Some(actual_size) = get_header(headers, rustfs_utils::http::SUFFIX_REPLICATION_COMPRESSION_ACTUAL_SIZE) - && actual_size.parse::().is_ok_and(|size| size >= 0) - { + if let Some(actual_size) = get_header(headers, rustfs_utils::http::SUFFIX_REPLICATION_COMPRESSION_ACTUAL_SIZE) { + if !actual_size.parse::().is_ok_and(|size| size >= 0) { + return Err(StorageError::InvalidArgument( + String::new(), + String::new(), + "Invalid replicated compression size".to_string(), + )); + } rustfs_utils::http::insert_str( &mut opts.user_defined, rustfs_utils::http::SUFFIX_ACTUAL_SIZE, @@ -556,17 +593,30 @@ pub fn put_opts_from_headers_with_replication_authorization( /// Replicas must keep the source object's ETag: managed-SSE replication /// re-encrypts on the target, so a recomputed ETag would differ from the /// source and every HEAD comparison would re-schedule the object forever. -fn replication_source_etag(headers: &HeaderMap) -> Option { - let value = get_header(headers, SUFFIX_SOURCE_ETAG)?; - let value = value.trim().trim_matches('"'); - (!value.is_empty()).then(|| value.to_string()) +fn replication_source_etag(headers: &HeaderMap) -> std::io::Result> { + let Some(value) = get_header(headers, SUFFIX_SOURCE_ETAG) else { + return Ok(None); + }; + let value = value.trim(); + let value = value + .strip_prefix('"') + .and_then(|value| value.strip_suffix('"')) + .unwrap_or(value); + if value + .bytes() + .any(|byte| !byte.is_ascii_graphic() || matches!(byte, b'"' | b'<' | b'>' | b'&')) + { + return Err(std::io::Error::new(std::io::ErrorKind::InvalidInput, "Invalid replication source ETag")); + } + Ok((!value.is_empty()).then(|| value.to_string())) } fn replication_source_mtime(headers: &HeaderMap) -> Option { let value = get_header(headers, SUFFIX_SOURCE_MTIME)?; let value = value.trim(); match time::OffsetDateTime::parse(value, &time::format_description::well_known::Rfc3339) { - Ok(mtime) => Some(mtime), + Ok(mtime) if mtime.unix_timestamp_nanos() != 0 && i64::try_from(mtime.unix_timestamp_nanos()).is_ok() => Some(mtime), + Ok(_) => None, Err(err) => { tracing::warn!("Invalid source-mtime value '{}' (replication request=true): {}", value, err); None @@ -1207,6 +1257,45 @@ mod tests { use std::collections::HashMap; use uuid::Uuid; + #[test] + fn replication_etags_reject_xml_and_header_delimiters_before_persistence() { + for etag in ["victim", "a&b", "a\"b", "a\tb"] { + let mut headers = HeaderMap::new(); + insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_REQUEST, "true"); + insert_header(&mut headers, SUFFIX_SOURCE_ETAG, etag); + assert!( + put_opts_from_headers_with_replication_authorization(&headers, HashMap::new(), true).is_err(), + "{etag:?}" + ); + assert!( + get_complete_multipart_upload_opts_with_replication_authorization(&headers, true).is_err(), + "{etag:?}" + ); + assert!( + put_opts_from_headers_with_replication_authorization(&headers, HashMap::new(), false) + .expect("untrusted header is inert") + .preserve_etag + .is_none() + ); + } + for etag in [ + "0123456789abcdef0123456789abcdef", + "\"0123456789abcdef0123456789abcdef-3\"", + "opaque-provider-etag", + ] { + let mut headers = HeaderMap::new(); + insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_REQUEST, "true"); + insert_header(&mut headers, SUFFIX_SOURCE_ETAG, etag); + assert_eq!( + put_opts_from_headers_with_replication_authorization(&headers, HashMap::new(), true) + .expect("valid replica ETag") + .preserve_etag + .as_deref(), + Some(etag.trim_matches('"')) + ); + } + } + fn create_test_headers() -> HeaderMap { let mut headers = HeaderMap::new(); headers.insert("content-type", HeaderValue::from_static("application/json")); @@ -1562,6 +1651,25 @@ mod tests { } } + #[test] + fn destination_version_resolution_preserves_query_precedence_and_normalizes_aliases() { + for header in ["x-rustfs-source-version-id", "x-minio-source-version-id"] { + let mut headers = HeaderMap::new(); + headers.insert(header, HeaderValue::from_static(" ABCDEF12-1234-1234-1234-123456789ABC ")); + assert_eq!( + super::destination_version_id(None, &headers).as_deref(), + Some("abcdef12-1234-1234-1234-123456789abc") + ); + for query in ["null", "NULL", " null ", "00000000-0000-0000-0000-000000000000"] { + assert_eq!(super::destination_version_id(Some(query), &headers).as_deref(), Some("null")); + } + headers.insert(header, HeaderValue::from_static("null")); + assert_eq!(super::destination_version_id(None, &headers).as_deref(), Some("null")); + assert_eq!(super::destination_version_id(Some("invalid"), &headers).as_deref(), Some("invalid")); + } + assert_eq!(super::destination_version_id(None, &HeaderMap::new()), None); + } + #[tokio::test] async fn test_put_opts_normalizes_null_version_id() { // MinIO-compatible replication senders (including RustFS itself since @@ -1684,9 +1792,10 @@ mod tests { get_str, }; + let scheme = crate::storage::storage_api::compression_metadata_value(rustfs_utils::CompressionAlgorithm::Lz4); let mut headers = HeaderMap::new(); insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_REQUEST, "true"); - insert_header(&mut headers, SUFFIX_REPLICATION_COMPRESSION, "klauspost/compress/s2"); + insert_header(&mut headers, SUFFIX_REPLICATION_COMPRESSION, &scheme); insert_header(&mut headers, SUFFIX_REPLICATION_COMPRESSION_ACTUAL_SIZE, "6295552"); // Without SSE-C transport headers the request is not a passthrough: @@ -1714,19 +1823,49 @@ mod tests { let trusted = put_opts_from_headers_with_replication_authorization(&headers, HashMap::new(), true) .expect("authorized replication request should parse"); assert!(trusted.preserve_ciphertext); - assert_eq!( - get_str(&trusted.user_defined, SUFFIX_COMPRESSION).as_deref(), - Some("klauspost/compress/s2") - ); + assert_eq!(get_str(&trusted.user_defined, SUFFIX_COMPRESSION).as_deref(), Some(scheme.as_str())); assert_eq!(get_str(&trusted.user_defined, SUFFIX_ACTUAL_SIZE).as_deref(), Some("6295552")); - // A malformed plaintext size is dropped; the scheme alone still lets - // the read path derive the size from the parts. insert_header(&mut headers, SUFFIX_REPLICATION_COMPRESSION_ACTUAL_SIZE, "-5"); - let malformed = put_opts_from_headers_with_replication_authorization(&headers, HashMap::new(), true) - .expect("authorized replication request should parse"); - assert!(get_str(&malformed.user_defined, SUFFIX_COMPRESSION).is_some()); - assert!(get_str(&malformed.user_defined, SUFFIX_ACTUAL_SIZE).is_none()); + assert!( + put_opts_from_headers_with_replication_authorization(&headers, HashMap::new(), true).is_err(), + "invalid replicated sizes must not reach persisted metadata" + ); + } + + #[test] + fn replicated_encryption_metadata_rejects_invalid_sizes_and_compression() { + use rustfs_utils::http::object_encryption_keys::REPLICATION_SSEC_ORIGINAL_SIZE_HEADER; + use rustfs_utils::http::{SUFFIX_REPLICATION_COMPRESSION, SUFFIX_REPLICATION_COMPRESSION_ACTUAL_SIZE}; + for (header, value) in [ + (REPLICATION_SSEC_ORIGINAL_SIZE_HEADER, "abc"), + (REPLICATION_SSEC_ORIGINAL_SIZE_HEADER, "-1"), + (REPLICATION_SSEC_ORIGINAL_SIZE_HEADER, "9223372036854775808"), + ] { + let mut headers = HeaderMap::new(); + insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_REQUEST, "true"); + headers.insert( + http::HeaderName::from_bytes(header.as_bytes()).expect("header name"), + HeaderValue::from_static(value), + ); + assert!(put_opts_from_headers_with_replication_authorization(&headers, HashMap::new(), true).is_err()); + assert!( + put_opts_from_headers_with_replication_authorization(&headers, HashMap::new(), false).is_ok(), + "untrusted transport headers stay inert" + ); + } + for size in ["0", "5", "9223372036854775807"] { + let mut headers = HeaderMap::new(); + insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_REQUEST, "true"); + headers.insert( + http::HeaderName::from_bytes(REPLICATION_SSEC_ORIGINAL_SIZE_HEADER.as_bytes()).expect("size header"), + HeaderValue::from_static(size), + ); + assert!(put_opts_from_headers_with_replication_authorization(&headers, HashMap::new(), true).is_ok()); + insert_header(&mut headers, SUFFIX_REPLICATION_COMPRESSION, "not-a-compression-format"); + insert_header(&mut headers, SUFFIX_REPLICATION_COMPRESSION_ACTUAL_SIZE, "5"); + assert!(put_opts_from_headers_with_replication_authorization(&headers, HashMap::new(), true).is_err()); + } } #[test] @@ -1808,6 +1947,26 @@ mod tests { assert!(opts_invalid.mod_time.is_none()); } + #[test] + fn replication_source_time_must_survive_the_metadata_codec() { + for timestamp in ["1970-01-01T00:00:00Z", "9999-12-31T23:59:59Z", "0001-01-01T00:00:00Z"] { + let mut headers = HeaderMap::new(); + insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_REQUEST, "true"); + insert_header(&mut headers, SUFFIX_SOURCE_MTIME, timestamp); + let opts = put_opts_from_headers_with_replication_authorization(&headers, HashMap::new(), true) + .expect("invalid source time falls back to server commit time"); + assert!(opts.mod_time.is_none(), "{timestamp}"); + } + for timestamp in ["1969-12-31T23:59:59Z", "1970-01-01T00:00:00.000000001Z"] { + let mut headers = HeaderMap::new(); + insert_header(&mut headers, SUFFIX_SOURCE_MTIME, timestamp); + assert!( + super::replication_source_mtime(&headers).is_some(), + "representable time {timestamp} remains readable" + ); + } + } + /// A client PUT must not materialize the replication transport names as /// bare stored user-metadata keys: the outbound replication header /// builder forwards user metadata verbatim on a server-authorized diff --git a/rustfs/src/storage/s3_api/bucket.rs b/rustfs/src/storage/s3_api/bucket.rs index 4145566ac..f93a7bb60 100644 --- a/rustfs/src/storage/s3_api/bucket.rs +++ b/rustfs/src/storage/s3_api/bucket.rs @@ -17,7 +17,7 @@ use crate::storage::storage_api::s3_api_consumer::bucket::contract::{ bucket::BucketInfo, list::{ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsV2Info as StorageListObjectsV2Info}, }; -use crate::storage::storage_api::s3_api_consumer::bucket::to_s3s_etag; +use crate::storage::storage_api::s3_api_consumer::bucket::to_s3s_xml_etag; use percent_encoding::percent_decode_str; use s3s::dto::{ Bucket, CommonPrefix, DeleteMarkerEntry, EncodingType, ListBucketsOutput, ListObjectVersionsOutput, ListObjectsOutput, @@ -245,7 +245,7 @@ pub(crate) fn build_list_object_versions_output( .etag .as_ref() .filter(|_| rustfs_utils::http::etag_can_be_listed(&v.user_defined)) - .map(|etag| to_s3s_etag(etag)), + .map(|etag| to_s3s_xml_etag(etag)), storage_class: v.storage_class.clone().map(ObjectVersionStorageClass::from), ..Default::default() }) @@ -334,7 +334,7 @@ pub(crate) fn build_list_objects_v2_output( .etag .as_ref() .filter(|_| rustfs_utils::http::etag_can_be_listed(&v.user_defined)) - .map(|etag| to_s3s_etag(etag)), + .map(|etag| to_s3s_xml_etag(etag)), storage_class: v.storage_class.clone().map(ObjectStorageClass::from), ..Default::default() }; @@ -452,7 +452,8 @@ mod tests { use super::{ ListObjectVersionsInfo, ListObjectVersionsParams, ListObjectsV2Info, build_list_buckets_output, build_list_object_versions_output, build_list_objects_output, build_list_objects_v2_output, - encode_list_objects_v2_continuation_token, parse_list_object_versions_params, parse_list_objects_v2_params, to_s3s_etag, + encode_list_objects_v2_continuation_token, parse_list_object_versions_params, parse_list_objects_v2_params, + to_s3s_xml_etag, }; use crate::storage::s3_api::common::rustfs_owner; use crate::storage::storage_api::s3_api_consumer::bucket::StorageObjectInfo as ObjectInfo; @@ -464,6 +465,66 @@ mod tests { use time::OffsetDateTime; use uuid::Uuid; + #[test] + fn listings_escape_legacy_etags_without_creating_xml_entries() { + use s3s::xml::{Serialize, Serializer}; + fn xml(value: &T) -> String { + let mut bytes = Vec::new(); + value.serialize(&mut Serializer::new(&mut bytes)).expect("serialize listing"); + String::from_utf8(bytes).expect("UTF-8 XML") + } + let malicious = "victim&"; + let object = ObjectInfo { + name: "owned/key".into(), + etag: Some(malicious.into()), + ..Default::default() + }; + let v2 = build_list_objects_v2_output( + ListObjectsV2Info { + objects: vec![object.clone()], + ..Default::default() + }, + false, + 1000, + "bucket".into(), + String::new(), + None, + None, + None, + None, + ); + let v1 = build_list_objects_output(v2.clone(), None); + let versions = build_list_object_versions_output( + ListObjectVersionsInfo { + objects: vec![object], + ..Default::default() + }, + "bucket".into(), + &ListObjectVersionsParams { + prefix: String::new(), + delimiter: None, + key_marker: None, + version_id_marker: None, + max_keys: 1000, + }, + None, + ); + for output in [xml(&v1), xml(&v2), xml(&versions)] { + let mut reader = quick_xml::Reader::from_str(&output); + let mut keys = 0; + loop { + match reader.read_event().expect("legacy ETag must leave well-formed XML") { + quick_xml::events::Event::Start(event) if event.name().as_ref() == "Key" => keys += 1, + quick_xml::events::Event::Eof => break, + _ => {} + } + } + assert_eq!(keys, 1, "{output}"); + assert!(output.contains("</ETag><Contents>"), "{output}"); + assert!(output.contains("&amp;"), "existing entities must remain literal: {output}"); + } + } + #[test] fn test_build_list_buckets_output_maps_bucket_infos_and_owner() { let bucket_infos = vec![ @@ -771,7 +832,7 @@ mod tests { let entries = listing.as_ref().expect("listed objects"); assert_eq!(entries.len(), 5); for (index, entry) in entries.iter().enumerate() { - assert_eq!(entry.e_tag, (index >= 2).then(|| to_s3s_etag("0123456789abcdef0123456789abcdef"))); + assert_eq!(entry.e_tag, (index >= 2).then(|| to_s3s_xml_etag("0123456789abcdef0123456789abcdef"))); assert_eq!(entry.key.as_deref(), Some(format!("object-{index}").as_str())); assert_eq!(entry.size, Some(42)); } @@ -794,7 +855,7 @@ mod tests { let versions = versions.versions.expect("listed versions"); assert_eq!(versions.len(), 5); for (index, entry) in versions.iter().enumerate() { - assert_eq!(entry.e_tag, (index >= 2).then(|| to_s3s_etag("0123456789abcdef0123456789abcdef"))); + assert_eq!(entry.e_tag, (index >= 2).then(|| to_s3s_xml_etag("0123456789abcdef0123456789abcdef"))); assert_eq!(entry.size, Some(42)); } } diff --git a/rustfs/src/storage/s3_api/multipart.rs b/rustfs/src/storage/s3_api/multipart.rs index afaf78d53..97360a882 100644 --- a/rustfs/src/storage/s3_api/multipart.rs +++ b/rustfs/src/storage/s3_api/multipart.rs @@ -18,7 +18,7 @@ use crate::storage::storage_api::effective_storage_class; use crate::storage::storage_api::s3_api_consumer::multipart::contract::multipart::{ ListMultipartsInfo, ListPartsInfo, MAX_MULTIPART_PART_NUMBER, }; -use crate::storage::storage_api::s3_api_consumer::multipart::to_s3s_etag; +use crate::storage::storage_api::s3_api_consumer::multipart::to_s3s_xml_etag; use s3s::dto::{CommonPrefix, EncodingType, ListMultipartUploadsOutput, ListPartsOutput, MultipartUpload, Part, Timestamp}; use s3s::{S3Error, S3ErrorCode}; @@ -54,7 +54,7 @@ pub(crate) fn build_list_parts_output(res: ListPartsInfo) -> ListPartsOutput { res.parts .into_iter() .map(|p| Part { - e_tag: p.etag.map(|etag| to_s3s_etag(&etag)), + e_tag: p.etag.map(|etag| to_s3s_xml_etag(&etag)), last_modified: p.last_mod.map(Timestamp::from), part_number: p.part_num.try_into().ok(), // Compressed parts store fewer bytes than the client sent; S3 @@ -222,7 +222,7 @@ mod tests { use crate::storage::storage_api::s3_api_consumer::multipart::contract::multipart::{ ListMultipartsInfo, ListPartsInfo, MultipartInfo, PartInfo, }; - use crate::storage::storage_api::s3_api_consumer::multipart::to_s3s_etag; + use crate::storage::storage_api::s3_api_consumer::multipart::to_s3s_xml_etag; use s3s::S3ErrorCode; use s3s::dto::Timestamp; use time::OffsetDateTime; @@ -261,7 +261,7 @@ mod tests { assert_eq!(parts.len(), 1); assert_eq!(parts[0].part_number, Some(1)); assert_eq!(parts[0].size, Some(11)); - assert_eq!(parts[0].e_tag, Some(to_s3s_etag("etag-1"))); + assert_eq!(parts[0].e_tag, Some(to_s3s_xml_etag("etag-1"))); assert_eq!(output.owner, Some(rustfs_owner())); assert_eq!(output.initiator, Some(rustfs_initiator())); } diff --git a/rustfs/src/storage/s3_api/tagging.rs b/rustfs/src/storage/s3_api/tagging.rs index 1b22a3f88..99b7538b3 100644 --- a/rustfs/src/storage/s3_api/tagging.rs +++ b/rustfs/src/storage/s3_api/tagging.rs @@ -15,8 +15,6 @@ use percent_encoding::percent_decode_str; use s3s::dto::{Tag, TaggingDirective}; use s3s::{S3Error, S3ErrorCode, S3Result}; -use std::collections::HashSet; -use std::sync::OnceLock; use url::form_urlencoded; use crate::storage::storage_api::encode_tags; @@ -25,46 +23,18 @@ fn invalid_tag_error(message: &str) -> S3Error { S3Error::with_message(S3ErrorCode::InvalidTag, message.to_string()) } -fn is_valid_tag_text(value: &str) -> bool { - static VALID_OBJECT_TAG_TEXT: OnceLock> = OnceLock::new(); - VALID_OBJECT_TAG_TEXT - .get_or_init(|| regex::Regex::new(r"^[\p{L}\p{N}\p{Z}+\-=._:/@]*$")) - .as_ref() - .is_ok_and(|pattern| pattern.is_match(value)) -} - pub(crate) fn validate_object_tag_set(tag_set: &[Tag]) -> S3Result<()> { if tag_set.len() > 10 { return Err(invalid_tag_error("Cannot have more than 10 tags per object")); } + crate::storage::storage_api::ecstore_bucket::metadata::validate_tag_set_entries(tag_set).map_err(invalid_tag_error) +} - let mut tag_keys = HashSet::with_capacity(tag_set.len()); - for tag in tag_set { - let key = tag - .key - .as_deref() - .filter(|key| !key.is_empty()) - .ok_or_else(|| invalid_tag_error("Tag key cannot be empty"))?; - - if key.encode_utf16().count() > 128 || !is_valid_tag_text(key) { - return Err(invalid_tag_error("The TagKey you have provided is invalid")); - } - - let value = tag - .value - .as_deref() - .ok_or_else(|| invalid_tag_error("Tag value cannot be null"))?; - - if value.encode_utf16().count() > 256 || !is_valid_tag_text(value) { - return Err(invalid_tag_error("The TagValue you have provided is invalid")); - } - - if !tag_keys.insert(key) { - return Err(invalid_tag_error("Cannot provide multiple Tags with the same key")); - } +pub(crate) fn validate_bucket_tag_set(tag_set: &[Tag]) -> S3Result<()> { + if tag_set.len() > crate::storage::storage_api::ecstore_bucket::metadata::MAX_BUCKET_TAGS { + return Err(invalid_tag_error("Cannot have more than 50 tags per bucket")); } - - Ok(()) + crate::storage::storage_api::ecstore_bucket::metadata::validate_tag_set_entries(tag_set).map_err(invalid_tag_error) } fn validate_tag_component_encoding(component: &str) -> S3Result<()> { diff --git a/rustfs/src/storage/sse.rs b/rustfs/src/storage/sse.rs index ee081e25b..8b28e695f 100644 --- a/rustfs/src/storage/sse.rs +++ b/rustfs/src/storage/sse.rs @@ -476,6 +476,11 @@ impl EncryptionRequest<'_> { .decode_to_vec(key_md5) .is_ok_and(|decoded| decoded.len() == 16) => { + if rustfs_utils::http::contains_key_str(user_defined, rustfs_utils::http::SUFFIX_PLAINTEXT_CHECKSUM) { + return Err(ssec_invalid_request( + "SSE-C multipart uploads with additional checksums require the customer key for completion.", + )); + } Ok(()) } _ => Err(ssec_invalid_request("The multipart upload contains invalid SSE-C metadata.")), @@ -901,7 +906,7 @@ impl SseKmsPrincipal { conditions: cred.map(|cred| resource_free_condition_values(req, cred)).unwrap_or_default(), request_audit: request_context_from_req(req).and_then(|context| kms_request_audit(&context.request_id)), #[cfg(test)] - test_hooks: None, + test_hooks: req.extensions.get::().cloned(), }) } @@ -1128,6 +1133,87 @@ pub async fn authorize_sse_kms_object_read( result } +/// Authorize the current request to write under the upload's stored KMS key. +/// Upload IDs do not confer the creator's KMS permissions on another caller. +pub async fn authorize_sse_kms_multipart_write( + principal: Option<&SseKmsPrincipal>, + metadata: &HashMap, +) -> Result<(), ApiError> { + let Some((sse_type, key_id)) = stored_managed_encryption_key(metadata) else { + return Ok(()); + }; + let result = authorize_sse_kms_key(principal, sse_type, KmsAction::GenerateDataKeyAction, &key_id).await; + if let Err(error) = &result { + record_managed_kms_outcome( + principal, + sse_type, + Some(&key_id), + || stored_envelope_master_key_version(metadata), + Err(error), + ); + } + result +} + +/// Capture the request's encryption gate without exposing customer keys in +/// ObjectOptions Debug output. Storage applies it to the source selected under +/// completion locks, including recovered uploads and historical replay versions. +pub(crate) fn multipart_completion_authorization( + request: EncryptionRequest<'_>, + replication_authorized: bool, +) -> Arc> { + Arc::new(SseMultipartCompletionAuthorization { + bucket: request.bucket.to_string(), + key: request.key.to_string(), + principal: request.principal.cloned(), + algorithm: request.sse_customer_algorithm, + customer_key: request.sse_customer_key, + customer_md5: request.sse_customer_key_md5, + replication_authorized, + }) +} + +struct SseMultipartCompletionAuthorization { + bucket: String, + key: String, + principal: Option, + algorithm: Option, + customer_key: Option, + customer_md5: Option, + replication_authorized: bool, +} + +#[async_trait::async_trait] +impl super::storage_api::contract::multipart::MultipartCompletionAuthorization for SseMultipartCompletionAuthorization { + type Error = StorageError; + + async fn authorize(&self, metadata: &HashMap) -> Result<(), StorageError> { + authorize_sse_kms_multipart_write(self.principal.as_ref(), metadata) + .await + .map_err(|error| StorageError::other(map_encryption_resolution_error(error)))?; + if self.customer_key.is_some() + || !self.replication_authorized + || !rustfs_utils::http::contains_key_str(metadata, rustfs_utils::http::SUFFIX_REPLICATION_PRESERVE_CIPHERTEXT) + { + EncryptionRequest { + bucket: &self.bucket, + key: &self.key, + server_side_encryption: None, + ssekms_key_id: None, + ssekms_context: None, + sse_customer_algorithm: self.algorithm.clone(), + sse_customer_key: self.customer_key.clone(), + sse_customer_key_md5: self.customer_md5.clone(), + content_size: 0, + principal: self.principal.as_ref(), + } + .validate_complete_multipart_ssec(metadata) + .map_err(|error| StorageError::other(map_encryption_resolution_error(error)))?; + } + Ok(()) + } +} + // ============================================================================ // Data-plane KMS audit attachment (SSE-S3 / SSE-KMS) // ============================================================================ @@ -4185,7 +4271,7 @@ fn is_legacy_rustfs_managed_metadata(metadata: &HashMap) -> bool /// detection only in builds that can actually read it — default builds keep /// the resolver's fail-closed diagnosis instead of demanding customer keys /// they could not use. -fn stored_ssec_metadata(metadata: &HashMap) -> bool { +pub(crate) fn stored_ssec_metadata(metadata: &HashMap) -> bool { metadata.contains_key("x-amz-server-side-encryption-customer-algorithm") || minio_sealed_ssec_metadata(metadata) } @@ -6875,6 +6961,35 @@ mod tests { } } + #[test] + fn ssec_checksum_completion_requires_key_for_both_metadata_aliases() { + let no_key = EncryptionRequest { + sse_customer_algorithm: None, + sse_customer_key: None, + sse_customer_key_md5: None, + ..multipart_ssec_request(42) + }; + for prefix in ["x-rustfs-internal-", "x-minio-internal-"] { + let mut metadata = multipart_ssec_metadata(42); + metadata.insert(format!("{prefix}plaintext-checksum"), "true".to_string()); + assert_eq!( + no_key + .validate_complete_multipart_ssec(&metadata) + .expect_err("no key cannot compare or reveal plaintext checksums") + .code, + S3ErrorCode::InvalidRequest + ); + multipart_ssec_request(42) + .validate_complete_multipart_ssec(&metadata) + .expect("correct key"); + assert!( + multipart_ssec_request(43) + .validate_complete_multipart_ssec(&metadata) + .is_err() + ); + } + } + #[test] fn test_validate_complete_multipart_ssec_still_validates_present_parameters() { let partial = EncryptionRequest { @@ -8154,6 +8269,234 @@ mod tests { ); } + #[test] + #[serial_test::serial] + fn multipart_handlers_reject_another_callers_denied_kms_upload() { + crate::app::gating_test_env::run_large_stack_test("mpu-kms-current-caller", || async { + use crate::app::multipart_usecase::DefaultMultipartUsecase; + use crate::storage::storage_api::contract::multipart::MultipartOperations as _; + use s3s::dto::{ + CompleteMultipartUploadInput, CompletedMultipartUpload, CompletedPart, CopySource, StreamingBlob, + UploadPartCopyInput, UploadPartInput, + }; + let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("mpu-kms-current-caller", 16_384).await; + let usecase = DefaultMultipartUsecase::from_global(); + let key = "kms-owned-upload"; + let metadata = sse_kms_object_metadata(); + let upload = store + .new_multipart_upload( + &bucket, + key, + &crate::storage::ObjectOptions { + user_defined: metadata, + ..Default::default() + }, + ) + .await + .expect("seed authorized upload envelope"); + let authorizer = RecordingKmsKeyAuthorizer::new(false); + fn denied_request(input: T, authorizer: Arc) -> s3s::S3Request { + let mut req = s3s::S3Request { + input, + method: http::Method::PUT, + uri: http::Uri::from_static("/"), + headers: http::HeaderMap::new(), + extensions: http::Extensions::new(), + credentials: None, + region: None, + service: None, + trailing_headers: None, + }; + req.extensions.insert(super::ReqInfo { + cred: Some(rustfs_credentials::Credentials { + access_key: "denied-writer".to_string(), + ..Default::default() + }), + ..Default::default() + }); + req.extensions.insert(super::TestAuthorizationHooks { + enforced: true, + authorizer, + }); + req + } + let mut part = UploadPartInput::builder() + .bucket(bucket.clone()) + .key(key.to_string()) + .upload_id(upload.upload_id.clone()) + .part_number(1) + .build() + .expect("part"); + part.body = Some(StreamingBlob::from(bytes::Bytes::from_static(b"abc"))); + part.content_length = Some(3); + // Keep typed request construction separate: each real handler must + // consult the destination envelope before encryption or commit. + let error = usecase + .execute_upload_part(denied_request(part, authorizer.clone())) + .await + .expect_err("denied part"); + assert_eq!(error.code(), &S3ErrorCode::AccessDenied); + let copy = UploadPartCopyInput::builder() + .bucket(bucket.clone()) + .key(key.to_string()) + .upload_id(upload.upload_id.clone()) + .part_number(1) + .copy_source(CopySource::Bucket { + bucket: bucket.clone().into(), + key: "source".into(), + version_id: None, + }) + .build() + .expect("copy"); + let copy_req = denied_request(copy, authorizer.clone()); + let error = usecase + .execute_upload_part_copy(copy_req) + .await + .expect_err("denied destination copy"); + assert_eq!(error.code(), &S3ErrorCode::AccessDenied); + let complete = CompleteMultipartUploadInput::builder() + .bucket(bucket.clone()) + .key(key.to_string()) + .upload_id(upload.upload_id.clone()) + .multipart_upload(Some(CompletedMultipartUpload { + parts: Some(vec![CompletedPart { + part_number: Some(1), + e_tag: Some(s3s::dto::ETag::Strong("etag".to_string())), + ..Default::default() + }]), + })) + .build() + .expect("complete"); + let complete_req = denied_request(complete, authorizer.clone()); + let error = usecase + .execute_complete_multipart_upload(complete_req) + .await + .expect_err("denied completion"); + assert_eq!(error.code(), &S3ErrorCode::AccessDenied); + assert_eq!(authorizer.calls(), vec![(KmsAction::GenerateDataKeyAction, "finance-key".to_string()); 3]); + assert!( + store + .get_multipart_info(&bucket, key, &upload.upload_id, &crate::storage::ObjectOptions::default()) + .await + .is_ok(), + "denials must retain the original upload" + ); + let mut reader = crate::storage::PutObjReader::from_vec(b"abc".to_vec()); + let uploaded_part = store + .put_object_part(&bucket, key, &upload.upload_id, 1, &mut reader, &crate::storage::ObjectOptions::default()) + .await + .expect("trusted owner stages part"); + store + .clone() + .complete_multipart_upload( + &bucket, + key, + &upload.upload_id, + vec![crate::storage::storage_api::contract::multipart::CompletePart { + part_num: 1, + etag: uploaded_part.etag, + ..Default::default() + }], + &crate::storage::ObjectOptions::default(), + ) + .await + .expect("trusted owner completes upload"); + let replay = CompleteMultipartUploadInput::builder() + .bucket(bucket.clone()) + .key(key.to_string()) + .upload_id(upload.upload_id) + .multipart_upload(Some(CompletedMultipartUpload { + parts: Some(vec![CompletedPart { + part_number: Some(1), + e_tag: Some(s3s::dto::ETag::Strong("guessed-etag".to_string())), + ..Default::default() + }]), + })) + .build() + .expect("replay"); + let error = usecase + .execute_complete_multipart_upload(denied_request(replay, authorizer.clone())) + .await + .expect_err("replay must authorize before comparing guessed part metadata"); + assert_eq!(error.code(), &S3ErrorCode::AccessDenied); + assert_eq!(authorizer.calls().len(), 4); + }); + } + + #[tokio::test] + async fn completion_authorization_preserves_typed_denials_and_redacts_customer_keys() { + let key = multipart_ssec_request(42).sse_customer_key.expect("test key"); + let mut metadata = multipart_ssec_metadata(42); + rustfs_utils::http::insert_str(&mut metadata, rustfs_utils::http::SUFFIX_PLAINTEXT_CHECKSUM, "true".to_string()); + let no_key = super::multipart_completion_authorization( + EncryptionRequest { + sse_customer_algorithm: None, + sse_customer_key: None, + sse_customer_key_md5: None, + ..multipart_ssec_request(42) + }, + false, + ); + let error = no_key.authorize(&metadata).await.expect_err("customer key required"); + assert_eq!(map_get_object_reader_error(error).code, S3ErrorCode::InvalidRequest); + let (denied, _) = enforcing_principal(false); + let denied = super::multipart_completion_authorization( + EncryptionRequest { + principal: Some(&denied), + ..multipart_ssec_request(42) + }, + false, + ); + let error = denied.authorize(&sse_kms_object_metadata()).await.expect_err("KMS denied"); + assert_eq!(map_get_object_reader_error(error).code, S3ErrorCode::AccessDenied); + let valid = super::multipart_completion_authorization(multipart_ssec_request(42), false); + valid.authorize(&metadata).await.expect("correct customer key"); + let opts = crate::storage::ObjectOptions { + multipart_completion_authorization: Some(valid), + ..Default::default() + }; + assert!( + !format!("{opts:?}").contains(&key), + "ObjectOptions Debug must never expose the captured customer key" + ); + } + + #[tokio::test] + async fn multipart_write_authorization_uses_stored_key_and_current_principal() { + let (denied, denied_authorizer) = enforcing_principal(false); + let (allowed, allowed_authorizer) = enforcing_principal(true); + let metadata = sse_kms_object_metadata(); + let error = super::authorize_sse_kms_multipart_write(Some(&denied), &metadata) + .await + .expect_err("upload ID must not grant its creator's KMS permission"); + assert_eq!(error.code, S3ErrorCode::AccessDenied); + assert_eq!( + denied_authorizer.calls(), + vec![(KmsAction::GenerateDataKeyAction, "finance-key".to_string())] + ); + super::authorize_sse_kms_multipart_write(Some(&allowed), &metadata) + .await + .expect("authorized writer"); + assert_eq!( + allowed_authorizer.calls(), + vec![(KmsAction::GenerateDataKeyAction, "finance-key".to_string())] + ); + for metadata in [HashMap::new(), sse_s3_object_metadata(), multipart_ssec_metadata(42)] { + super::authorize_sse_kms_multipart_write(Some(&denied), &metadata) + .await + .expect("non-KMS upload"); + } + assert_eq!(denied_authorizer.calls().len(), 1); + let (off, off_authorizer) = permissive_principal_with_enforcement_off(); + super::authorize_sse_kms_multipart_write(Some(&off), &metadata) + .await + .expect("enforcement off"); + super::authorize_sse_kms_multipart_write(None, &metadata) + .await + .expect("trusted internal writer"); + assert!(off_authorizer.calls().is_empty()); + } + #[tokio::test] async fn copy_source_read_authorization_targets_the_source_key() { let (denied, denied_authorizer) = enforcing_principal(false); diff --git a/rustfs/src/storage/storage_api.rs b/rustfs/src/storage/storage_api.rs index d99c146a2..29b5a715a 100644 --- a/rustfs/src/storage/storage_api.rs +++ b/rustfs/src/storage/storage_api.rs @@ -97,10 +97,11 @@ pub(crate) mod contract { } pub(crate) mod multipart { + pub(crate) use super::super::storage_contracts::MultipartCompletionAuthorization; pub(crate) const MAX_MULTIPART_PART_NUMBER: i32 = 10000; - pub(crate) use super::super::storage_contracts::{ListMultipartsInfo, ListPartsInfo}; #[cfg(test)] - pub(crate) use super::super::storage_contracts::{MultipartInfo, MultipartOperations, PartInfo}; + pub(crate) use super::super::storage_contracts::{CompletePart, MultipartInfo, MultipartOperations, PartInfo}; + pub(crate) use super::super::storage_contracts::{ListMultipartsInfo, ListPartsInfo}; } pub(crate) mod object { @@ -133,26 +134,27 @@ pub(crate) use super::ecfs_extend::{ get_buffer_size_opt_in, get_validated_store, has_read_preconditions, load_bucket_object_lock_config_state, parse_object_lock_legal_hold, parse_object_lock_retention, parse_part_number_i32_to_usize, process_lambda_configurations, process_queue_configurations, process_topic_configurations, remove_object_lock_metadata_for_copy, validate_bucket_exists, - validate_bucket_object_lock_enabled_state, validate_list_object_unordered_with_delimiter, validate_object_key, - wrap_response_with_cors, + validate_bucket_object_lock_enabled_state, validate_list_object_unordered_with_delimiter, validate_new_object_key, + validate_object_key, wrap_response_with_cors, }; pub(crate) use super::sse::{ DecryptionRequest, EncryptionRequest, ObjectDekRewrapOutcome, PrepareEncryptionRequest, SseKmsPrincipal, - authorize_sse_kms_object_read, classify_sse_read_response, extract_server_side_encryption_from_headers, - project_sse_read_response_headers, rewrap_object_encryption_metadata, sse_decryption, sse_encryption, sse_prepare_encryption, - strip_managed_encryption_metadata, validate_sse_headers_for_read, validate_sse_headers_for_write, validate_ssec_for_read, + authorize_sse_kms_multipart_write, authorize_sse_kms_object_read, classify_sse_read_response, + extract_server_side_encryption_from_headers, project_sse_read_response_headers, rewrap_object_encryption_metadata, + sse_decryption, sse_encryption, sse_prepare_encryption, strip_managed_encryption_metadata, validate_sse_headers_for_read, + validate_sse_headers_for_write, validate_ssec_for_read, }; pub(crate) mod access_consumer { pub(crate) use super::super::access::{ MultipartStagingOperation, PostObjectRequestMarker, ReqInfo, TABLE_DATA_PLANE_LIST_CURSOR_PREFIX, TableDataPlaneListAccess, TableDataPlaneListCursorPosition, TableDataPlanePublicationGuards, - apply_bucket_generation_guard, apply_copy_source_bucket_generation_guard, authorize_internal_object_request, - authorize_request, bucket_config_mutation_incarnation, delete_object_authorize_action, has_bypass_governance_header, - load_bucket_generation_from_store, log_list_buckets_iam_implicit_deny, odm_read_generation, + apply_bucket_generation_guard, apply_copy_source_bucket_generation_guard, authorize_copy_object_tags, + authorize_internal_object_request, authorize_request, bucket_config_mutation_incarnation, delete_object_authorize_action, + has_bypass_governance_header, load_bucket_generation_from_store, log_list_buckets_iam_implicit_deny, odm_read_generation, prepare_list_buckets_iam_authorization, prepare_odm_read_generation, recursive_force_delete_has_authenticated_caller, - replication_request_authorized, req_info_mut, req_info_ref, retained_table_data_plane_publication_guards, - set_requested_object_lock_retain_until, + replication_proxy_allowed, replication_request_authorized, req_info_mut, req_info_ref, + retained_table_data_plane_publication_guards, set_requested_object_lock_retain_until, }; } @@ -217,7 +219,13 @@ pub(crate) mod head_prefix_consumer { } pub(crate) mod helper_consumer { - pub(crate) use super::super::helper::{OperationHelper, build_event_resp_elements, spawn_background_with_context}; + #[cfg(test)] + pub(crate) use super::super::helper::{ + MutationCompletionProbe, capture_mutation_event_for_test, pause_committed_mutation_for_test, + }; + pub(crate) use super::super::helper::{ + OperationHelper, build_event_resp_elements, run_owned_mutation, spawn_background_with_context, + }; pub(crate) type StorageObjectInfo = super::StorageObjectInfo; } @@ -233,6 +241,7 @@ pub(crate) mod options_consumer { parse_copy_source_range, preserve_unclassified_user_metadata, put_opts_with_replication_authorization, validate_archive_content_encoding, }; + pub(crate) use rustfs_ecstore::api::rio::validate_compression_scheme; pub(crate) mod contract { pub(crate) mod object { @@ -358,9 +367,7 @@ pub(crate) mod s3_api_consumer { pub(crate) type StorageObjectInfo = super::super::StorageObjectInfo; - pub(crate) fn to_s3s_etag(etag: &str) -> s3s::dto::ETag { - super::super::to_s3s_etag(etag) - } + pub(crate) use super::super::to_s3s_xml_etag; } pub(crate) mod common { @@ -384,13 +391,13 @@ pub(crate) mod s3_api_consumer { } } - pub(crate) fn to_s3s_etag(etag: &str) -> s3s::dto::ETag { - super::super::to_s3s_etag(etag) - } + pub(crate) use super::super::to_s3s_xml_etag; } pub(crate) mod tagging { - pub(crate) use super::super::super::s3_api::tagging::{parse_copy_object_tags, resolve_copy_object_tags}; + pub(crate) use super::super::super::s3_api::tagging::{ + parse_copy_object_tags, resolve_copy_object_tags, validate_bucket_tag_set, + }; } } @@ -398,13 +405,14 @@ pub(crate) mod sse_consumer { pub(crate) use super::super::sse::{ EncryptionKeyKind, bucket_default_write_sse, build_ssec_read_headers, encryption_material_to_metadata, extract_ssec_params_from_headers, extract_ssekms_context_from_headers, log_sse_kms_key_policy_mode, - map_get_object_reader_error, mark_encrypted_multipart_metadata, + map_get_object_reader_error, mark_encrypted_multipart_metadata, multipart_completion_authorization, stored_ssec_metadata, }; pub(crate) use super::{ DecryptionRequest, EncryptionRequest, PrepareEncryptionRequest, SseKmsPrincipal, apply_bucket_default_lock_retention, - authorize_sse_kms_object_read, classify_sse_read_response, extract_server_side_encryption_from_headers, - get_buffer_size_opt_in, load_bucket_object_lock_config_state, project_sse_read_response_headers, sse_decryption, - sse_encryption, sse_prepare_encryption, validate_bucket_object_lock_enabled_state, + authorize_sse_kms_multipart_write, authorize_sse_kms_object_read, classify_sse_read_response, + extract_server_side_encryption_from_headers, get_buffer_size_opt_in, load_bucket_object_lock_config_state, + project_sse_read_response_headers, sse_decryption, sse_encryption, sse_prepare_encryption, + validate_bucket_object_lock_enabled_state, }; } @@ -649,7 +657,8 @@ pub(crate) mod ecstore_set_disk { #[cfg(test)] pub(crate) use rustfs_ecstore::api::set_disk::test_util::{ ConditionalReadBenchmarkMetadataGuard, MultipartCommitBarrier, MultipartCommitPause, PutObjectCommitBarrier, - PutObjectCommitPause, fail_next_quota_ledger_save_for_test, + PutObjectCommitPause, fail_next_quota_ledger_save_for_test, initialize_quota_ledger_for_test, + retry_quota_reconciliation_for_test, }; pub(crate) use rustfs_ecstore::api::set_disk::{ DEFAULT_READ_BUFFER_SIZE, file_info_quorum_hash, get_lock_acquire_timeout, is_valid_storage_class, @@ -2447,6 +2456,16 @@ pub(crate) fn to_s3s_etag(etag: &str) -> s3s::dto::ETag { ecstore_client::object_api_utils::to_s3s_etag(etag) } +/// The pinned s3s XML serializer writes ETag values as raw text. Escape only +/// at XML response boundaries; HTTP headers and conditional comparisons keep +/// the original stored value. Existing unsafe ETags must remain listable. +pub(crate) fn to_s3s_xml_etag(etag: &str) -> s3s::dto::ETag { + let mut etag = to_s3s_etag(etag); + let (s3s::dto::ETag::Strong(value) | s3s::dto::ETag::Weak(value)) = &mut etag; + *value = quick_xml::escape::escape(value.as_str()).into_owned(); + etag +} + pub(crate) fn table_catalog_path_hash(value: &str) -> String { ecstore_bucket::metadata::table_catalog_path_hash(value) } diff --git a/rustfs/src/table_catalog/tests.rs b/rustfs/src/table_catalog/tests.rs index 3d2b7c6b9..5d9895cef 100644 --- a/rustfs/src/table_catalog/tests.rs +++ b/rustfs/src/table_catalog/tests.rs @@ -22111,10 +22111,10 @@ fn object_mutation_entrypoints_call_reserved_prefix_guard() { .expect("delete object entrypoints should remain in app/object/delete.rs"); for expected in [ - "validate_object_key(&key, request_method_name)?;\n validate_table_catalog_object_mutation(&bucket, &key).await?;", + "validate_new_object_key(&key, request_method_name)?;\n validate_table_catalog_object_mutation(&bucket, &key).await?;", "if let Err(err) = validate_table_catalog_object_mutation(&bucket, &obj_id.key).await", "validate_table_catalog_object_mutation(&bucket, &object).await?;", - "validate_object_key(&key, \"PUT\")?;\n validate_table_catalog_object_mutation(&bucket, &key).await?;", + "validate_new_object_key(&key, \"PUT\")?;\n validate_table_catalog_object_mutation(&bucket, &key).await?;", "extract_try!(validate_table_catalog_object_mutation(&bucket, &fpath).await);", ] { assert!(source.contains(expected), "missing object mutation guard: {expected}"); @@ -22124,7 +22124,7 @@ fn object_mutation_entrypoints_call_reserved_prefix_guard() { .map(|(_, body)| body.split_once("\n#[cfg(test)]").map_or(body, |(entrypoint, _)| entrypoint)) .expect("copy object entrypoint should remain in app/object/copy.rs"); let copy_key = copy_object - .find("validate_object_key(&key, \"COPY (dest)\")?;") + .find("validate_new_object_key(&key, \"COPY (dest)\")?;") .expect("copy destination key validation"); let copy_guard = copy_object .find("validate_table_catalog_object_mutation(&bucket, &key).await?;")