gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-10-13 08:02:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(table-catalog): verify Iceberg metadata path signatures (#8291)

Browse Source 
* fix(table-catalog): verify Iceberg metadata path signatures

* fix(deps): allow pinned s3s Git source
This commit is contained in:
GatewayJ
2026-10-05 17:03:41 +08:00
committed by GitHub
parent c71e4f72ea
commit 051a54f129
6 changed files with 109 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+6 -11
View File
@@ -11233,8 +11233,7 @@ checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
[[package]]
name = "s3s"
version = "0.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "030a43692a1fd97bbd2c594eeaad8f5c1e452a9d29ad6a3eebce1650937c4a77"
source = "git+https://github.com/s3s-project/s3s?rev=5761ddfe4c509fcac87e3b99fd5bd9e176b442c0#5761ddfe4c509fcac87e3b99fd5bd9e176b442c0"
dependencies = [
"arc-swap",
"arrayvec",
@@ -11244,7 +11243,6 @@ dependencies = [
"bytes",
"bytestring",
"cfg-if",
"chrono",
"crc-fast",
"futures",
"hex-simd",
@@ -11292,8 +11290,7 @@ dependencies = [
[[package]]
name = "s3s-multipart"
version = "0.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a543079d42c20c81781b7e31704ecacd64b0e3819b2384db5bf4869abf6c6f01"
source = "git+https://github.com/s3s-project/s3s?rev=5761ddfe4c509fcac87e3b99fd5bd9e176b442c0#5761ddfe4c509fcac87e3b99fd5bd9e176b442c0"
dependencies = [
"arrayvec",
"bytes",
@@ -11306,8 +11303,7 @@ dependencies = [
[[package]]
name = "s3s-rfc2047"
version = "0.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "918b207c39474baef4774355100e712dfea46020e3289ddd5b4bc6e21927b826"
source = "git+https://github.com/s3s-project/s3s?rev=5761ddfe4c509fcac87e3b99fd5bd9e176b442c0#5761ddfe4c509fcac87e3b99fd5bd9e176b442c0"
dependencies = [
"base64-simd",
"thiserror 2.0.21",
@@ -11316,8 +11312,7 @@ dependencies = [
[[package]]
name = "s3s-sigv2"
version = "0.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0979baf8cdf30e38bdfba8e58ee2776d78e55bb3bd3d1f3502f917829da79933"
source = "git+https://github.com/s3s-project/s3s?rev=5761ddfe4c509fcac87e3b99fd5bd9e176b442c0#5761ddfe4c509fcac87e3b99fd5bd9e176b442c0"
dependencies = [
"base64-simd",
"hmac 0.13.0",
@@ -11330,8 +11325,7 @@ dependencies = [
[[package]]
name = "s3s-sigv4"
version = "0.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "330996b9a24804b3df4ad37e62e7817968857d99cf6029c21512b6b64b97f645"
source = "git+https://github.com/s3s-project/s3s?rev=5761ddfe4c509fcac87e3b99fd5bd9e176b442c0#5761ddfe4c509fcac87e3b99fd5bd9e176b442c0"
dependencies = [
"arrayvec",
"base64-simd",
@@ -11343,6 +11337,7 @@ dependencies = [
"sha2 0.11.0",
"smallvec",
"std-next",
"subtle",
"thiserror 2.0.21",
"zeroize",
]
Cargo.toml
+2 -2
View File
@@ -314,8 +314,8 @@ rustify = { version = "0.7", default-features = false }
rustix = { version = "1.1.5" }
rust-embed = { version = "8.12.0" }
rustc-hash = { version = "2.1.3" }
s3s = { version = "0.17.0", features = ["minio"] }
s3s-sigv4 = "0.17.0"
s3s = { git = "https://github.com/s3s-project/s3s", rev = "5761ddfe4c509fcac87e3b99fd5bd9e176b442c0", features = ["minio"] }
s3s-sigv4 = { git = "https://github.com/s3s-project/s3s", rev = "5761ddfe4c509fcac87e3b99fd5bd9e176b442c0" }
serial_test = "4.0.1"
shadow-rs = { default-features = false, version = "2.0.0" }
siphasher = "1.0.4"
deny.toml
+3 -4
View File
@@ -36,10 +36,9 @@ unknown-registry = "deny"
unknown-git = "deny"
allow-registry = ["https://github.com/rust-lang/crates.io-index"]
allow-git = [
# Official s3s repository; the application now consumes the 0.17.0
# crates.io releases for s3s and s3s-sigv4.
# owner: marshawcoco review: 2026-10
# "https://github.com/s3s-project/s3s.git",
# Official s3s repository pinned for Iceberg SigV4 path encoding support.
# owner: GatewayJ review: 2026-10
"https://github.com/s3s-project/s3s",
# RustFS io_uring backend is now consumed from crates.io at 0.3.0.
# owner: rustfs-maintainers review: 2026-10
# "https://github.com/rustfs/uring",
rustfs/src/admin/router.rs
+87 -2
View File
@@ -39,7 +39,7 @@ use crate::error::ApiError;
use crate::license::license_check;
use crate::server::{
ADMIN_PREFIX, HEALTH_PREFIX, HEALTH_READY_PATH, MINIO_ADMIN_PREFIX, PROFILE_CPU_PATH, PROFILE_MEMORY_PATH, is_admin_path,
is_sts_query_request,
is_sts_query_request, is_table_catalog_path,
};
use crate::storage::storage_api::lock_bucket_targets_metadata;
use aws_sdk_s3::primitives::ByteStream as AwsByteStream;
@@ -85,6 +85,7 @@ use s3s::S3Request;
use s3s::S3Response;
use s3s::S3Result;
use s3s::StdError;
use s3s::auth::SigV4PathEncoding;
use s3s::dto::{GetObjectInput, GetObjectOutput, IfMatch, IfNoneMatch, Range, StreamingBlob, Timestamp, TimestampFormat};
use s3s::header;
use s3s::route::S3Route;
@@ -3242,12 +3243,16 @@ impl<T> S3Route for S3Router<T>
where
T: Operation,
{
fn is_match(&self, method: &Method, uri: &Uri, headers: &HeaderMap, _: &mut Extensions) -> bool {
fn is_match(&self, method: &Method, uri: &Uri, headers: &HeaderMap, extensions: &mut Extensions) -> bool {
if parse_replication_extension_request(method, uri).is_some() || parse_misc_extension_request(method, uri).is_some() {
return true;
}
let path = uri.path();
if is_table_catalog_path(path) {
extensions.insert(SigV4PathEncoding::DoubleEncoded);
return true;
}
// Profiling endpoints
if method == Method::GET && (path == PROFILE_CPU_PATH || path == PROFILE_MEMORY_PATH) {
@@ -3574,6 +3579,86 @@ mod tests {
assert!(!is_admin_path("/minio/adminx/object"));
}
#[test]
fn iceberg_routes_select_double_encoded_sigv4() {
let router: S3Router<StatusOperation> = S3Router::new(false);
for path in [
"/iceberg/v1",
"/iceberg/v1/warehouse/namespaces/ods%1Forders/tables/files",
"/_iceberg/v1",
"/_iceberg/v1/warehouse/namespaces/ods%1Forders/tables/files",
] {
let mut extensions = Extensions::new();
assert!(router.is_match(&Method::GET, &path.parse().unwrap(), &HeaderMap::new(), &mut extensions));
assert_eq!(
extensions.get::<s3s::auth::SigV4PathEncoding>(),
Some(&s3s::auth::SigV4PathEncoding::DoubleEncoded),
"{path}"
);
}
for path in [
"/bucket/name%20space",
"/rustfs/admin/v3/info",
"/minio/admin/v3/info",
"/health",
"/iceberg/v10/config",
"/_iceberg/v10/config",
"/iceberg/v1suffix/config",
] {
let mut extensions = Extensions::new();
router.is_match(&Method::GET, &path.parse().unwrap(), &HeaderMap::new(), &mut extensions);
assert!(extensions.get::<s3s::auth::SigV4PathEncoding>().is_none(), "{path}");
}
}
#[tokio::test]
async fn iceberg_metadata_probe_passes_sigv4_verification() {
use s3s::auth::SimpleAuth;
use s3s::config::{S3Config, StaticConfigProvider};
use s3s::service::S3ServiceBuilder;
struct TestS3;
#[async_trait::async_trait]
impl s3s::S3 for TestS3 {}
let mut router = S3Router::new(false);
router
.insert(
Method::GET,
"/iceberg/v1/warehouse/namespaces/{namespace}/tables/{table}",
StatusOperation(StatusCode::NOT_FOUND),
)
.unwrap();
let mut builder = S3ServiceBuilder::new(TestS3);
builder.set_route(router);
builder.set_auth(SimpleAuth::from_single("catalog-test-access", "catalog-test-secret"));
let mut config = S3Config::default();
config.presigned_url_max_skew_time_secs = u32::MAX;
builder.set_config(Arc::new(StaticConfigProvider::new(Arc::new(config))));
let service = builder.build();
// Captured from botocore SigV4Auth, independently of the server signer.
let request = http::Request::builder()
.method(Method::GET)
.uri("/iceberg/v1/warehouse/namespaces/ods%1Fkfk_log_order/tables/files")
.header("host", "catalog.example:9000")
.header("content-type", "application/json")
.header("x-amz-date", "20200101T000000Z")
.header("x-amz-content-sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
.header(
"authorization",
concat!(
"AWS4-HMAC-SHA256 Credential=catalog-test-access/20200101/us-east-1/s3/aws4_request, ",
"SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date, ",
"Signature=38357111b8355d0efdc6746cc5e94930c455d2f6741ea38c65e6556fe45d882d"
),
)
.body(Body::empty())
.unwrap();
let response = service.call(request).await.unwrap();
assert_eq!(response.status(), StatusCode::NOT_FOUND);
}
#[test]
fn parse_replication_extension_request_matches_metrics_and_check() {
let metrics: Uri = "/demo-bucket?replication-metrics".parse().expect("uri should parse");
scripts/table-catalog/pyiceberg_smoke.py
+3 -6
View File
@@ -266,7 +266,6 @@ class RuntimeDeps:
botocore_config: Any
botocore_credentials: Any
botocore_auth: Any
botocore_s3_auth: Any
botocore_awsrequest: Any
pyarrow: Any
load_catalog: Any
@@ -472,13 +471,13 @@ def load_runtime_deps() -> RuntimeDeps:
pyarrow = None
missing.append("pyarrow")
try:
from botocore.auth import S3SigV4Auth, SigV4Auth
from botocore.auth import SigV4Auth
from botocore.awsrequest import AWSRequest
from botocore.exceptions import ClientError
from botocore.config import Config
from botocore.credentials import Credentials
except ModuleNotFoundError:
ClientError = Config = Credentials = S3SigV4Auth = SigV4Auth = AWSRequest = None
ClientError = Config = Credentials = SigV4Auth = AWSRequest = None
missing.append("botocore")
try:
from pyiceberg.catalog import load_catalog
@@ -499,7 +498,6 @@ def load_runtime_deps() -> RuntimeDeps:
botocore_config=Config,
botocore_credentials=Credentials,
botocore_auth=SigV4Auth,
botocore_s3_auth=S3SigV4Auth,
botocore_awsrequest=AWSRequest,
pyarrow=pyarrow,
load_catalog=load_catalog,
@@ -535,8 +533,7 @@ def unsigned_ssl_context(insecure: bool) -> ssl.SSLContext | None:
def sign_rest_request(args: argparse.Namespace, deps: RuntimeDeps, request: Any) -> None:
credentials = deps.botocore_credentials(args.access_key, args.secret_key)
signer = deps.botocore_s3_auth if args.profile in RUSTFS_PROFILES else deps.botocore_auth
signer(credentials, args.rest_signing_name, args.region).add_auth(request)
deps.botocore_auth(credentials, args.rest_signing_name, args.region).add_auth(request)
def signed_rest_request(args: argparse.Namespace, deps: RuntimeDeps, method: str, path: str, body: dict[str, Any] | None = None) -> dict[str, Any]:
scripts/table-catalog/test_pyiceberg_signing.py
+8 -5
View File
@@ -13,7 +13,7 @@ from datetime import datetime, timezone
from types import SimpleNamespace
from unittest import mock
from botocore.auth import S3SigV4Auth, SigV4Auth
from botocore.auth import SigV4Auth
from botocore.awsrequest import AWSRequest
from botocore.credentials import Credentials
from requests import Request, Response, Session
@@ -29,7 +29,6 @@ class PyIcebergSigningTest(unittest.TestCase):
def setUp(self) -> None:
self.deps = SimpleNamespace(
botocore_auth=SigV4Auth,
botocore_s3_auth=S3SigV4Auth,
botocore_credentials=Credentials,
botocore_awsrequest=AWSRequest,
)
@@ -43,9 +42,13 @@ class PyIcebergSigningTest(unittest.TestCase):
):
return pyiceberg_smoke.parse_args()
def expected_signature(self, request: object, args: object, signer: object = S3SigV4Auth) -> str:
headers = {key: value for key, value in request.headers.items() if key.lower() != "authorization"}
expected = AWSRequest(method=request.method, url=request.url, data=request.body, headers=headers)
def expected_signature(self, request: object, args: object, signer: object = SigV4Auth) -> str:
headers = {key: value for key, value in request.headers.items() if key.lower() not in {"authorization", "x-amz-content-sha256"}}
body = request.body or b""
if isinstance(body, str):
body = body.encode("utf-8")
headers["x-amz-content-sha256"] = hashlib.sha256(body).hexdigest()
expected = AWSRequest(method=request.method, url=request.url, data=body, headers=headers)
signer(Credentials(args.access_key, args.secret_key), args.rest_signing_name, args.region).add_auth(expected)
return expected.headers["Authorization"]