diff --git a/Cargo.lock b/Cargo.lock index ce0aea077..5298036bb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11233,8 +11233,7 @@ checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" [[package]] name = "s3s" version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "030a43692a1fd97bbd2c594eeaad8f5c1e452a9d29ad6a3eebce1650937c4a77" +source = "git+https://github.com/s3s-project/s3s?rev=5761ddfe4c509fcac87e3b99fd5bd9e176b442c0#5761ddfe4c509fcac87e3b99fd5bd9e176b442c0" dependencies = [ "arc-swap", "arrayvec", @@ -11244,7 +11243,6 @@ dependencies = [ "bytes", "bytestring", "cfg-if", - "chrono", "crc-fast", "futures", "hex-simd", @@ -11292,8 +11290,7 @@ dependencies = [ [[package]] name = "s3s-multipart" version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a543079d42c20c81781b7e31704ecacd64b0e3819b2384db5bf4869abf6c6f01" +source = "git+https://github.com/s3s-project/s3s?rev=5761ddfe4c509fcac87e3b99fd5bd9e176b442c0#5761ddfe4c509fcac87e3b99fd5bd9e176b442c0" dependencies = [ "arrayvec", "bytes", @@ -11306,8 +11303,7 @@ dependencies = [ [[package]] name = "s3s-rfc2047" version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "918b207c39474baef4774355100e712dfea46020e3289ddd5b4bc6e21927b826" +source = "git+https://github.com/s3s-project/s3s?rev=5761ddfe4c509fcac87e3b99fd5bd9e176b442c0#5761ddfe4c509fcac87e3b99fd5bd9e176b442c0" dependencies = [ "base64-simd", "thiserror 2.0.21", @@ -11316,8 +11312,7 @@ dependencies = [ [[package]] name = "s3s-sigv2" version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0979baf8cdf30e38bdfba8e58ee2776d78e55bb3bd3d1f3502f917829da79933" +source = "git+https://github.com/s3s-project/s3s?rev=5761ddfe4c509fcac87e3b99fd5bd9e176b442c0#5761ddfe4c509fcac87e3b99fd5bd9e176b442c0" dependencies = [ "base64-simd", "hmac 0.13.0", @@ -11330,8 +11325,7 @@ dependencies = [ [[package]] name = "s3s-sigv4" version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "330996b9a24804b3df4ad37e62e7817968857d99cf6029c21512b6b64b97f645" +source = "git+https://github.com/s3s-project/s3s?rev=5761ddfe4c509fcac87e3b99fd5bd9e176b442c0#5761ddfe4c509fcac87e3b99fd5bd9e176b442c0" dependencies = [ "arrayvec", "base64-simd", @@ -11343,6 +11337,7 @@ dependencies = [ "sha2 0.11.0", "smallvec", "std-next", + "subtle", "thiserror 2.0.21", "zeroize", ] diff --git a/Cargo.toml b/Cargo.toml index 8622f2750..25d8cac2c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -314,8 +314,8 @@ rustify = { version = "0.7", default-features = false } rustix = { version = "1.1.5" } rust-embed = { version = "8.12.0" } rustc-hash = { version = "2.1.3" } -s3s = { version = "0.17.0", features = ["minio"] } -s3s-sigv4 = "0.17.0" +s3s = { git = "https://github.com/s3s-project/s3s", rev = "5761ddfe4c509fcac87e3b99fd5bd9e176b442c0", features = ["minio"] } +s3s-sigv4 = { git = "https://github.com/s3s-project/s3s", rev = "5761ddfe4c509fcac87e3b99fd5bd9e176b442c0" } serial_test = "4.0.1" shadow-rs = { default-features = false, version = "2.0.0" } siphasher = "1.0.4" diff --git a/deny.toml b/deny.toml index 7c22ab128..b810318e7 100644 --- a/deny.toml +++ b/deny.toml @@ -36,10 +36,9 @@ unknown-registry = "deny" unknown-git = "deny" allow-registry = ["https://github.com/rust-lang/crates.io-index"] allow-git = [ - # Official s3s repository; the application now consumes the 0.17.0 - # crates.io releases for s3s and s3s-sigv4. - # owner: marshawcoco review: 2026-10 - # "https://github.com/s3s-project/s3s.git", + # Official s3s repository pinned for Iceberg SigV4 path encoding support. + # owner: GatewayJ review: 2026-10 + "https://github.com/s3s-project/s3s", # RustFS io_uring backend is now consumed from crates.io at 0.3.0. # owner: rustfs-maintainers review: 2026-10 # "https://github.com/rustfs/uring", diff --git a/rustfs/src/admin/router.rs b/rustfs/src/admin/router.rs index 328395f17..3f454dff8 100644 --- a/rustfs/src/admin/router.rs +++ b/rustfs/src/admin/router.rs @@ -39,7 +39,7 @@ use crate::error::ApiError; use crate::license::license_check; use crate::server::{ ADMIN_PREFIX, HEALTH_PREFIX, HEALTH_READY_PATH, MINIO_ADMIN_PREFIX, PROFILE_CPU_PATH, PROFILE_MEMORY_PATH, is_admin_path, - is_sts_query_request, + is_sts_query_request, is_table_catalog_path, }; use crate::storage::storage_api::lock_bucket_targets_metadata; use aws_sdk_s3::primitives::ByteStream as AwsByteStream; @@ -85,6 +85,7 @@ use s3s::S3Request; use s3s::S3Response; use s3s::S3Result; use s3s::StdError; +use s3s::auth::SigV4PathEncoding; use s3s::dto::{GetObjectInput, GetObjectOutput, IfMatch, IfNoneMatch, Range, StreamingBlob, Timestamp, TimestampFormat}; use s3s::header; use s3s::route::S3Route; @@ -3242,12 +3243,16 @@ impl S3Route for S3Router where T: Operation, { - fn is_match(&self, method: &Method, uri: &Uri, headers: &HeaderMap, _: &mut Extensions) -> bool { + fn is_match(&self, method: &Method, uri: &Uri, headers: &HeaderMap, extensions: &mut Extensions) -> bool { if parse_replication_extension_request(method, uri).is_some() || parse_misc_extension_request(method, uri).is_some() { return true; } let path = uri.path(); + if is_table_catalog_path(path) { + extensions.insert(SigV4PathEncoding::DoubleEncoded); + return true; + } // Profiling endpoints if method == Method::GET && (path == PROFILE_CPU_PATH || path == PROFILE_MEMORY_PATH) { @@ -3574,6 +3579,86 @@ mod tests { assert!(!is_admin_path("/minio/adminx/object")); } + #[test] + fn iceberg_routes_select_double_encoded_sigv4() { + let router: S3Router = S3Router::new(false); + for path in [ + "/iceberg/v1", + "/iceberg/v1/warehouse/namespaces/ods%1Forders/tables/files", + "/_iceberg/v1", + "/_iceberg/v1/warehouse/namespaces/ods%1Forders/tables/files", + ] { + let mut extensions = Extensions::new(); + assert!(router.is_match(&Method::GET, &path.parse().unwrap(), &HeaderMap::new(), &mut extensions)); + assert_eq!( + extensions.get::(), + Some(&s3s::auth::SigV4PathEncoding::DoubleEncoded), + "{path}" + ); + } + for path in [ + "/bucket/name%20space", + "/rustfs/admin/v3/info", + "/minio/admin/v3/info", + "/health", + "/iceberg/v10/config", + "/_iceberg/v10/config", + "/iceberg/v1suffix/config", + ] { + let mut extensions = Extensions::new(); + router.is_match(&Method::GET, &path.parse().unwrap(), &HeaderMap::new(), &mut extensions); + assert!(extensions.get::().is_none(), "{path}"); + } + } + + #[tokio::test] + async fn iceberg_metadata_probe_passes_sigv4_verification() { + use s3s::auth::SimpleAuth; + use s3s::config::{S3Config, StaticConfigProvider}; + use s3s::service::S3ServiceBuilder; + + struct TestS3; + #[async_trait::async_trait] + impl s3s::S3 for TestS3 {} + + let mut router = S3Router::new(false); + router + .insert( + Method::GET, + "/iceberg/v1/warehouse/namespaces/{namespace}/tables/{table}", + StatusOperation(StatusCode::NOT_FOUND), + ) + .unwrap(); + let mut builder = S3ServiceBuilder::new(TestS3); + builder.set_route(router); + builder.set_auth(SimpleAuth::from_single("catalog-test-access", "catalog-test-secret")); + let mut config = S3Config::default(); + config.presigned_url_max_skew_time_secs = u32::MAX; + builder.set_config(Arc::new(StaticConfigProvider::new(Arc::new(config)))); + let service = builder.build(); + + // Captured from botocore SigV4Auth, independently of the server signer. + let request = http::Request::builder() + .method(Method::GET) + .uri("/iceberg/v1/warehouse/namespaces/ods%1Fkfk_log_order/tables/files") + .header("host", "catalog.example:9000") + .header("content-type", "application/json") + .header("x-amz-date", "20200101T000000Z") + .header("x-amz-content-sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855") + .header( + "authorization", + concat!( + "AWS4-HMAC-SHA256 Credential=catalog-test-access/20200101/us-east-1/s3/aws4_request, ", + "SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date, ", + "Signature=38357111b8355d0efdc6746cc5e94930c455d2f6741ea38c65e6556fe45d882d" + ), + ) + .body(Body::empty()) + .unwrap(); + let response = service.call(request).await.unwrap(); + assert_eq!(response.status(), StatusCode::NOT_FOUND); + } + #[test] fn parse_replication_extension_request_matches_metrics_and_check() { let metrics: Uri = "/demo-bucket?replication-metrics".parse().expect("uri should parse"); diff --git a/scripts/table-catalog/pyiceberg_smoke.py b/scripts/table-catalog/pyiceberg_smoke.py index 02e766c82..ebf7b5de2 100755 --- a/scripts/table-catalog/pyiceberg_smoke.py +++ b/scripts/table-catalog/pyiceberg_smoke.py @@ -266,7 +266,6 @@ class RuntimeDeps: botocore_config: Any botocore_credentials: Any botocore_auth: Any - botocore_s3_auth: Any botocore_awsrequest: Any pyarrow: Any load_catalog: Any @@ -472,13 +471,13 @@ def load_runtime_deps() -> RuntimeDeps: pyarrow = None missing.append("pyarrow") try: - from botocore.auth import S3SigV4Auth, SigV4Auth + from botocore.auth import SigV4Auth from botocore.awsrequest import AWSRequest from botocore.exceptions import ClientError from botocore.config import Config from botocore.credentials import Credentials except ModuleNotFoundError: - ClientError = Config = Credentials = S3SigV4Auth = SigV4Auth = AWSRequest = None + ClientError = Config = Credentials = SigV4Auth = AWSRequest = None missing.append("botocore") try: from pyiceberg.catalog import load_catalog @@ -499,7 +498,6 @@ def load_runtime_deps() -> RuntimeDeps: botocore_config=Config, botocore_credentials=Credentials, botocore_auth=SigV4Auth, - botocore_s3_auth=S3SigV4Auth, botocore_awsrequest=AWSRequest, pyarrow=pyarrow, load_catalog=load_catalog, @@ -535,8 +533,7 @@ def unsigned_ssl_context(insecure: bool) -> ssl.SSLContext | None: def sign_rest_request(args: argparse.Namespace, deps: RuntimeDeps, request: Any) -> None: credentials = deps.botocore_credentials(args.access_key, args.secret_key) - signer = deps.botocore_s3_auth if args.profile in RUSTFS_PROFILES else deps.botocore_auth - signer(credentials, args.rest_signing_name, args.region).add_auth(request) + deps.botocore_auth(credentials, args.rest_signing_name, args.region).add_auth(request) def signed_rest_request(args: argparse.Namespace, deps: RuntimeDeps, method: str, path: str, body: dict[str, Any] | None = None) -> dict[str, Any]: diff --git a/scripts/table-catalog/test_pyiceberg_signing.py b/scripts/table-catalog/test_pyiceberg_signing.py index 2ccb4f56b..c326ea636 100644 --- a/scripts/table-catalog/test_pyiceberg_signing.py +++ b/scripts/table-catalog/test_pyiceberg_signing.py @@ -13,7 +13,7 @@ from datetime import datetime, timezone from types import SimpleNamespace from unittest import mock -from botocore.auth import S3SigV4Auth, SigV4Auth +from botocore.auth import SigV4Auth from botocore.awsrequest import AWSRequest from botocore.credentials import Credentials from requests import Request, Response, Session @@ -29,7 +29,6 @@ class PyIcebergSigningTest(unittest.TestCase): def setUp(self) -> None: self.deps = SimpleNamespace( botocore_auth=SigV4Auth, - botocore_s3_auth=S3SigV4Auth, botocore_credentials=Credentials, botocore_awsrequest=AWSRequest, ) @@ -43,9 +42,13 @@ class PyIcebergSigningTest(unittest.TestCase): ): return pyiceberg_smoke.parse_args() - def expected_signature(self, request: object, args: object, signer: object = S3SigV4Auth) -> str: - headers = {key: value for key, value in request.headers.items() if key.lower() != "authorization"} - expected = AWSRequest(method=request.method, url=request.url, data=request.body, headers=headers) + def expected_signature(self, request: object, args: object, signer: object = SigV4Auth) -> str: + headers = {key: value for key, value in request.headers.items() if key.lower() not in {"authorization", "x-amz-content-sha256"}} + body = request.body or b"" + if isinstance(body, str): + body = body.encode("utf-8") + headers["x-amz-content-sha256"] = hashlib.sha256(body).hexdigest() + expected = AWSRequest(method=request.method, url=request.url, data=body, headers=headers) signer(Credentials(args.access_key, args.secret_key), args.rest_signing_name, args.region).add_auth(expected) return expected.headers["Authorization"]