gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-07-26 08:18:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(table-catalog): bridge table data-plane policy (#3436)

Browse Source 
* feat(table-catalog): bridge table data-plane policy

* test(table-catalog): harden vended credential smoke

* test(table-catalog): cover data-plane policy denials

* fix(table-catalog): protect relocated warehouse scope

* fix(table-catalog): skip invalid warehouse entries

---------

Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
This commit is contained in:
Henry Guo
2026-06-14 18:13:02 +08:00
committed by GitHub
parent e29c136ad5
commit 9372ee7032
6 changed files with 651 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/table-catalog/README.md
+8 -5
View File
@@ -94,12 +94,15 @@ table-scoped S3 access key, secret key, and session token before append, reload,
and scan operations.
Before the PyIceberg append, the profile also checks that the returned
credential prefix exactly matches the created table warehouse location, then
runs a direct S3 data-plane scope probe with the returned temporary credentials:
credential prefix exactly matches the created table warehouse location after
canonical S3 URI normalization, including percent-decoding equivalent path
encodings. It then runs a direct S3 data-plane scope probe with the returned
temporary credentials:
- `PutObject`, `HeadObject`, and `DeleteObject` must work inside the returned
table warehouse prefix.
- `PutObject` to the same bucket outside that prefix must be rejected.
- `PutObject`, `HeadObject`, `GetObject`, and `DeleteObject` must work inside
the returned table warehouse prefix.
- `PutObject` and `GetObject` to the same bucket outside that prefix must be
rejected.
## Machine-Readable Inventories
scripts/table-catalog/pyiceberg_smoke.py
+35 -10
View File
@@ -431,12 +431,15 @@ def s3_scope_from_uri(s3_uri: str, description: str) -> tuple[str, str]:
parsed = urllib.parse.urlparse(s3_uri)
if parsed.scheme != "s3" or not parsed.netloc:
raise RuntimeError(f"{description} must be an s3 URI")
object_prefix = parsed.path.lstrip("/")
bucket = urllib.parse.unquote(parsed.netloc)
object_prefix = urllib.parse.unquote(parsed.path.lstrip("/")).rstrip("/")
if not object_prefix:
raise RuntimeError(f"{description} must include an object prefix")
if not object_prefix.endswith("/"):
object_prefix = f"{object_prefix}/"
return parsed.netloc, object_prefix
if "\\" in object_prefix:
raise RuntimeError(f"{description} contains an invalid path separator")
if any(segment in {"", ".", ".."} for segment in object_prefix.split("/")):
raise RuntimeError(f"{description} contains an invalid path segment")
return bucket, f"{object_prefix}/"
def s3_scope_from_credential(storage_credential: StorageCredential) -> tuple[str, str]:
@@ -520,22 +523,44 @@ def verify_vended_credential_data_plane_scope(
client.put_object(Bucket=bucket, Key=inside_key, Body=b"rustfs table credential scope probe\n")
put_inside = True
client.head_object(Bucket=bucket, Key=inside_key)
client.get_object(Bucket=bucket, Key=inside_key)
finally:
if put_inside:
client.delete_object(Bucket=bucket, Key=inside_key)
put_denied = False
try:
client.put_object(Bucket=bucket, Key=denied_key, Body=b"rustfs denied table credential scope probe\n")
except deps.botocore_client_error as error:
if is_access_denied_error(error):
return
raise RuntimeError(f"scope-denied data-plane probe failed with unexpected S3 error: {error}") from error
put_denied = True
else:
raise RuntimeError(f"scope-denied write probe failed with unexpected S3 error: {error}") from error
if not put_denied:
try:
configured_s3_client(args, deps).delete_object(Bucket=bucket, Key=denied_key)
except Exception as cleanup_error:
print(f"warning: failed to clean unexpected denied-scope probe object {denied_key}: {cleanup_error}", file=sys.stderr)
raise RuntimeError("vended table credentials unexpectedly wrote outside the table warehouse prefix")
admin_client = configured_s3_client(args, deps)
seeded_denied_key = False
try:
configured_s3_client(args, deps).delete_object(Bucket=bucket, Key=denied_key)
except Exception as cleanup_error:
print(f"warning: failed to clean unexpected denied-scope probe object {denied_key}: {cleanup_error}", file=sys.stderr)
raise RuntimeError("vended table credentials unexpectedly wrote outside the table warehouse prefix")
admin_client.put_object(Bucket=bucket, Key=denied_key, Body=b"rustfs denied read table credential scope probe\n")
seeded_denied_key = True
try:
client.get_object(Bucket=bucket, Key=denied_key)
except deps.botocore_client_error as error:
if is_access_denied_error(error):
return
raise RuntimeError(f"scope-denied read probe failed with unexpected S3 error: {error}") from error
raise RuntimeError("vended table credentials unexpectedly read outside the table warehouse prefix")
finally:
if seeded_denied_key:
try:
admin_client.delete_object(Bucket=bucket, Key=denied_key)
except Exception as cleanup_error:
print(f"warning: failed to clean denied-scope read probe object {denied_key}: {cleanup_error}", file=sys.stderr)
def catalog_properties(args: argparse.Namespace, storage_credential: StorageCredential | None = None) -> dict[str, str]:
scripts/table-catalog/test_pyiceberg_smoke.py
+38 -2
View File
@@ -149,6 +149,15 @@ class PyIcebergSmokeConfigTest(unittest.TestCase):
self.assertEqual(bucket, "lake")
self.assertEqual(key_prefix, "tables/table-id/")
def test_s3_scope_from_uri_decodes_equivalent_prefix_encoding(self) -> None:
bucket, key_prefix = pyiceberg_smoke.s3_scope_from_uri(
"s3://lake/tables/table%2Did/",
"storage credential prefix",
)
self.assertEqual(bucket, "lake")
self.assertEqual(key_prefix, "tables/table-id/")
def test_storage_credential_prefix_rejects_bucket_scope(self) -> None:
credential = pyiceberg_smoke.StorageCredential(
prefix="s3://lake",
@@ -220,6 +229,22 @@ class PyIcebergSmokeConfigTest(unittest.TestCase):
def head_object(self, *, Bucket: str, Key: str) -> None:
self.calls.append(("head", Key))
def get_object(self, *, Bucket: str, Key: str) -> dict[str, bytes]:
self.calls.append(("get", Key))
if Key == "outside/probe":
raise FakeClientError(403, "AccessDenied")
return {"Body": b"rustfs table credential scope probe\n"}
def delete_object(self, *, Bucket: str, Key: str) -> None:
self.calls.append(("delete", Key))
class FakeAdminS3Client:
def __init__(self) -> None:
self.calls: list[tuple[str, str]] = []
def put_object(self, *, Bucket: str, Key: str, Body: bytes) -> None:
self.calls.append(("put", Key))
def delete_object(self, *, Bucket: str, Key: str) -> None:
self.calls.append(("delete", Key))
@@ -233,19 +258,30 @@ class PyIcebergSmokeConfigTest(unittest.TestCase):
},
)
fake_client = FakeS3Client()
fake_admin_client = FakeAdminS3Client()
deps = mock.Mock(botocore_client_error=FakeClientError)
with mock.patch.object(pyiceberg_smoke, "vended_s3_client", return_value=fake_client):
with mock.patch.object(pyiceberg_smoke, "scope_probe_keys", return_value=("tables/table-id/probe", "outside/probe")):
pyiceberg_smoke.verify_vended_credential_data_plane_scope(args, deps, credential, "s3://lake/tables/table-id")
with mock.patch.object(pyiceberg_smoke, "configured_s3_client", return_value=fake_admin_client):
with mock.patch.object(pyiceberg_smoke, "scope_probe_keys", return_value=("tables/table-id/probe", "outside/probe")):
pyiceberg_smoke.verify_vended_credential_data_plane_scope(args, deps, credential, "s3://lake/tables/table-id")
self.assertEqual(
fake_client.calls,
[
("put", "tables/table-id/probe"),
("head", "tables/table-id/probe"),
("get", "tables/table-id/probe"),
("delete", "tables/table-id/probe"),
("put", "outside/probe"),
("get", "outside/probe"),
],
)
self.assertEqual(
fake_admin_client.calls,
[
("put", "outside/probe"),
("delete", "outside/probe"),
],
)