mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-26 08:18:18 +00:00
test(table-catalog): add vendor compatibility audit (#4299)
* test(table-catalog): add vendor compatibility audit * fix(table-catalog): include OSS data-plane endpoint --------- Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
This commit is contained in:
@@ -34,7 +34,7 @@ catalog extension.
|
||||
| AWS S3 Tables endpoint shape | Profile generator | Generates the AWS catalog URI and S3 Tables warehouse ARN shape for migration docs. Full AWS S3 Tables API parity is not claimed. |
|
||||
| MinIO AIStor Tables profile | Profile generator plus RustFS alias smoke | RustFS exposes the alias shape, but does not claim all AIStor private extensions. |
|
||||
| Cloudflare R2 Data Catalog profile | Profile generator | Generates the catalog URI and warehouse-name shape for migration docs. Live RustFS interoperability is not claimed. |
|
||||
| Alibaba OSS Tables profile | Profile generator | Generates provider endpoint and warehouse shapes for migration docs. Live RustFS interoperability is not claimed. |
|
||||
| Alibaba OSS Tables profile | Profile generator | Generates provider endpoint, `acs:osstables` warehouse ARN, `osstables` signing-name, and `https://oss-{region}.aliyuncs.com` S3FileIO endpoint shapes for migration docs. Live RustFS interoperability is not claimed. |
|
||||
|
||||
## Client And Engine Matrix
|
||||
|
||||
@@ -54,6 +54,7 @@ catalog extension.
|
||||
|---|---|---|
|
||||
| Live conformance evidence template | Generated harness | `engine_compatibility.py --print-live-conformance` records required run metadata, per-client result rows, and claim promotion rules before a manual/live result can expand compatibility wording. |
|
||||
| Production operations guide | Generated harness | `engine_compatibility.py --print-operations-guide` records command, evidence, pass criteria, and fail-closed signals for live conformance, durable backing cutover, maintenance, recovery, permissions, credential vending, and unsupported-claim governance. |
|
||||
| Vendor compatibility gap audit | Generated harness | `engine_compatibility.py --print-vendor-audit` records provider source URLs, catalog path and warehouse shapes, signing/auth models, error/permission/maintenance validation categories, and not-claimed boundaries for AWS S3 Tables, MinIO AIStor Tables, Cloudflare R2 Data Catalog, and Alibaba OSS Tables. |
|
||||
| Client claim promotion | Documented, not automated | PyIceberg remains the automated claim. Spark can be promoted only with recorded manual/live evidence; Trino and DuckDB read probes do not promote write compatibility; Snowflake and vendor profiles remain reference-only without repeatable live evidence. |
|
||||
|
||||
## Catalog API Matrix
|
||||
@@ -196,6 +197,7 @@ python3 scripts/table-catalog/pyiceberg_smoke.py --print-engine-compatibility
|
||||
python3 scripts/table-catalog/pyiceberg_smoke.py --print-production-failure-coverage
|
||||
python3 scripts/table-catalog/pyiceberg_smoke.py --print-vendor-profiles
|
||||
python3 scripts/table-catalog/pyiceberg_smoke.py --print-production-readiness
|
||||
python3 scripts/table-catalog/engine_compatibility.py --print-vendor-audit
|
||||
python3 scripts/table-catalog/engine_compatibility.py --print-spark-config
|
||||
python3 scripts/table-catalog/engine_compatibility.py \
|
||||
--profile aws-s3tables \
|
||||
|
||||
@@ -150,11 +150,20 @@ the `selected_vendor_profile` object renders the active profile with the
|
||||
provided endpoint, account, bucket, and warehouse arguments.
|
||||
|
||||
Spark vendor config generation only emits S3-compatible data-plane endpoint,
|
||||
path-style, and static S3 credential properties for profiles that explicitly use
|
||||
the supplied endpoint as object storage, such as RustFS and MinIO AIStor. AWS S3
|
||||
Tables, Cloudflare R2 Data Catalog, and OSS Tables reference profiles leave
|
||||
provider object I/O endpoint and credential selection to the provider-specific
|
||||
Spark/Iceberg runtime configuration instead of inheriting RustFS local defaults.
|
||||
path-style, and static S3 credential properties for profiles that require them.
|
||||
RustFS and MinIO AIStor use the supplied endpoint as object storage. Alibaba
|
||||
OSS Tables uses the documented `https://oss-{region}.aliyuncs.com` public
|
||||
S3FileIO endpoint by default. AWS S3 Tables and Cloudflare R2 Data Catalog
|
||||
reference profiles leave provider object I/O endpoint and credential selection
|
||||
to the provider-specific Spark/Iceberg runtime configuration instead of
|
||||
inheriting RustFS local defaults.
|
||||
|
||||
The standalone engine helper also prints a vendor compatibility audit. This
|
||||
records the public reference source, catalog path, warehouse shape, signing
|
||||
name, auth model, required validation categories, and explicit not-claimed
|
||||
boundaries for AWS S3 Tables, MinIO AIStor Tables, Cloudflare R2 Data Catalog,
|
||||
and Alibaba OSS Tables. Use it to plan compatibility work; do not treat it as
|
||||
live interoperability evidence.
|
||||
|
||||
Generate an AWS S3 Tables-style reference profile:
|
||||
|
||||
@@ -177,11 +186,24 @@ python3 scripts/table-catalog/pyiceberg_smoke.py \
|
||||
--print-vendor-profiles
|
||||
```
|
||||
|
||||
Generate an Alibaba OSS Tables-style reference profile:
|
||||
|
||||
```bash
|
||||
python3 scripts/table-catalog/pyiceberg_smoke.py \
|
||||
--profile oss-tables \
|
||||
--endpoint https://cn-hangzhou.oss-tables.aliyuncs.com \
|
||||
--region cn-hangzhou \
|
||||
--account-id 123456789012 \
|
||||
--table-bucket analytics \
|
||||
--print-vendor-profiles
|
||||
```
|
||||
|
||||
The standalone engine helper prints the same compatibility matrix and can also
|
||||
generate Spark REST catalog input without importing PyIceberg:
|
||||
|
||||
```bash
|
||||
python3 scripts/table-catalog/engine_compatibility.py --print-engine-matrix
|
||||
python3 scripts/table-catalog/engine_compatibility.py --print-vendor-audit
|
||||
python3 scripts/table-catalog/engine_compatibility.py --print-spark-config
|
||||
python3 scripts/table-catalog/engine_compatibility.py \
|
||||
--profile aws-s3tables \
|
||||
@@ -352,7 +374,7 @@ expected status, observed status, and metadata location.
|
||||
| `aws-s3tables` | `https://s3tables.{region}.amazonaws.com/iceberg` | `arn:aws:s3tables:{region}:{account_id}:bucket/{table_bucket}` | `s3tables` | profile generator only; full AWS S3 Tables API parity is not claimed |
|
||||
| `minio-aistor` | `{endpoint}/_iceberg` | `{warehouse}` | `s3tables` | profile generator plus RustFS alias smoke; full AIStor extension parity is not claimed |
|
||||
| `cloudflare-r2-data-catalog` | catalog URI returned by R2 | `{warehouse_name}` | `s3` | profile generator only; live RustFS interoperability is not claimed |
|
||||
| `oss-tables` | provider REST endpoint | `{warehouse}` | `s3` | profile generator only; live RustFS interoperability is not claimed |
|
||||
| `oss-tables` | provider REST endpoint | `acs:osstables:{region}:{account_id}:bucket/{table_bucket}` | `osstables` | profile generator only; live RustFS interoperability is not claimed |
|
||||
|
||||
## Unsupported Inventory
|
||||
|
||||
|
||||
@@ -57,8 +57,9 @@ VENDOR_SPARK_PROFILES: dict[str, dict[str, str]] = {
|
||||
},
|
||||
"oss-tables": {
|
||||
"catalog_uri": "{endpoint}/iceberg",
|
||||
"warehouse": "{warehouse}",
|
||||
"rest_signing_name": "s3",
|
||||
"warehouse": "acs:osstables:{region}:{account_id}:bucket/{table_bucket}",
|
||||
"rest_signing_name": "osstables",
|
||||
"s3_endpoint": "https://oss-{region}.aliyuncs.com",
|
||||
},
|
||||
}
|
||||
|
||||
@@ -946,6 +947,231 @@ def production_operations_guide(
|
||||
)
|
||||
|
||||
|
||||
def audit_provider(
|
||||
*,
|
||||
profile: str,
|
||||
display_name: str,
|
||||
source: str,
|
||||
catalog_base_path: str,
|
||||
warehouse_shape: str,
|
||||
signing_name: str,
|
||||
auth_model: str,
|
||||
rustfs_claim: str,
|
||||
not_claimed: list[str],
|
||||
notes: list[str],
|
||||
) -> OrderedDict[str, Any]:
|
||||
return OrderedDict(
|
||||
[
|
||||
("profile", profile),
|
||||
("display_name", display_name),
|
||||
("source", source),
|
||||
("catalog_base_path", catalog_base_path),
|
||||
("warehouse_shape", warehouse_shape),
|
||||
("signing_name", signing_name),
|
||||
("auth_model", auth_model),
|
||||
("rustfs_claim", rustfs_claim),
|
||||
(
|
||||
"audit_categories",
|
||||
[
|
||||
"iceberg-rest-routes",
|
||||
"request-response-fields",
|
||||
"error-shapes",
|
||||
"permission-actions",
|
||||
"maintenance-semantics",
|
||||
"credential-vending",
|
||||
"client-live-run",
|
||||
],
|
||||
),
|
||||
("not_claimed", not_claimed),
|
||||
("notes", notes),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def validation_step(name: str, evidence: list[str]) -> OrderedDict[str, Any]:
|
||||
return OrderedDict(
|
||||
[
|
||||
("name", name),
|
||||
("status", "manual-audit-required"),
|
||||
("evidence", evidence),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def vendor_compatibility_audit() -> OrderedDict[str, Any]:
|
||||
return OrderedDict(
|
||||
[
|
||||
("claim_boundary", "reference profiles and gap audit"),
|
||||
("source_last_checked", "2026-07-05"),
|
||||
("promotion_policy", "live evidence required before compatibility claims expand"),
|
||||
(
|
||||
"providers",
|
||||
[
|
||||
audit_provider(
|
||||
profile="aws-s3tables",
|
||||
display_name="Amazon S3 Tables Iceberg REST endpoint",
|
||||
source="https://docs.aws.amazon.com/AmazonS3/latest/userguide/s3-tables-integrating-open-source.html",
|
||||
catalog_base_path="/iceberg",
|
||||
warehouse_shape="arn:aws:s3tables:{region}:{account_id}:bucket/{table_bucket}",
|
||||
signing_name="s3tables",
|
||||
auth_model="sigv4-s3tables-iam",
|
||||
rustfs_claim="profile-generator-only",
|
||||
not_claimed=[
|
||||
"full AWS S3 Tables compatibility",
|
||||
"AWS Glue Iceberg REST endpoint parity",
|
||||
"Lake Formation governance parity",
|
||||
"CloudTrail event parity",
|
||||
],
|
||||
notes=[
|
||||
"S3 Tables uses a table bucket ARN as the Iceberg REST prefix.",
|
||||
"S3 Tables only supports single-level namespaces.",
|
||||
"REST operations map to s3tables IAM actions and CloudTrail event names.",
|
||||
],
|
||||
),
|
||||
audit_provider(
|
||||
profile="minio-aistor",
|
||||
display_name="MinIO AIStor Tables",
|
||||
source="https://docs.min.io/aistor/developers/aistor-tables/aistor-tables-api/",
|
||||
catalog_base_path="/_iceberg",
|
||||
warehouse_shape="{warehouse}",
|
||||
signing_name="s3tables",
|
||||
auth_model="sigv4-s3tables-pbac",
|
||||
rustfs_claim="alias-smoke-plus-profile-generator",
|
||||
not_claimed=[
|
||||
"private-extension-parity",
|
||||
"full AIStor PBAC action parity",
|
||||
"AIStor engine compatibility matrix parity",
|
||||
],
|
||||
notes=[
|
||||
"AIStor serves the Iceberg REST catalog from the S3 endpoint under /_iceberg.",
|
||||
"Catalog authentication uses SigV4 with service name s3tables.",
|
||||
"The RustFS /_iceberg alias is a compatibility surface, not a claim of all AIStor extensions.",
|
||||
],
|
||||
),
|
||||
audit_provider(
|
||||
profile="cloudflare-r2-data-catalog",
|
||||
display_name="Cloudflare R2 Data Catalog",
|
||||
source="https://developers.cloudflare.com/r2/data-catalog/manage-catalogs/",
|
||||
catalog_base_path="provider-catalog-uri",
|
||||
warehouse_shape="{warehouse_name}",
|
||||
signing_name="s3",
|
||||
auth_model="api-token",
|
||||
rustfs_claim="profile-generator-only",
|
||||
not_claimed=[
|
||||
"full Cloudflare R2 Data Catalog interoperability",
|
||||
"managed-maintenance-parity",
|
||||
"Cloudflare API token policy parity",
|
||||
],
|
||||
notes=[
|
||||
"Enabling a Cloudflare catalog returns a Catalog URI and Warehouse name.",
|
||||
"Cloudflare documents optional compaction and snapshot expiration management.",
|
||||
"RustFS generates reference connection shapes but does not claim live Cloudflare interoperability.",
|
||||
],
|
||||
),
|
||||
audit_provider(
|
||||
profile="oss-tables",
|
||||
display_name="Alibaba Cloud OSS Tables",
|
||||
source="https://www.alibabacloud.com/help/en/oss/user-guide/spark-access-oss-tables",
|
||||
catalog_base_path="/iceberg",
|
||||
warehouse_shape="acs:osstables:{region}:{account_id}:bucket/{table_bucket}",
|
||||
signing_name="osstables",
|
||||
auth_model="sigv4-osstables",
|
||||
rustfs_claim="profile-generator-only",
|
||||
not_claimed=[
|
||||
"full Alibaba OSS Tables interoperability",
|
||||
"provider-error-code-parity",
|
||||
"Alibaba RAM policy parity",
|
||||
"internal-network-endpoint parity",
|
||||
],
|
||||
notes=[
|
||||
"OSS Tables documents public and internal Iceberg REST endpoints.",
|
||||
"Spark configuration uses an acs:osstables warehouse ARN and rest.signing-name=osstables.",
|
||||
"RustFS reference profiles must not imply live OSS Tables interoperability without a recorded run.",
|
||||
],
|
||||
),
|
||||
],
|
||||
),
|
||||
(
|
||||
"required_validation_steps",
|
||||
[
|
||||
validation_step(
|
||||
"route-and-field-shape",
|
||||
[
|
||||
"request path",
|
||||
"request fields",
|
||||
"response fields",
|
||||
"pagination fields",
|
||||
"unsupported route response",
|
||||
],
|
||||
),
|
||||
validation_step(
|
||||
"error-response-shape",
|
||||
[
|
||||
"HTTP status",
|
||||
"error code",
|
||||
"error message",
|
||||
"conflict response body",
|
||||
"not found response body",
|
||||
],
|
||||
),
|
||||
validation_step(
|
||||
"permission-action-mapping",
|
||||
[
|
||||
"catalog action",
|
||||
"table bucket resource",
|
||||
"namespace resource",
|
||||
"table resource",
|
||||
"ordinary S3 data-plane action",
|
||||
],
|
||||
),
|
||||
validation_step(
|
||||
"maintenance-behavior",
|
||||
[
|
||||
"compaction support",
|
||||
"snapshot expiration support",
|
||||
"orphan cleanup support",
|
||||
"scheduler behavior",
|
||||
"manual-review boundary",
|
||||
],
|
||||
),
|
||||
validation_step(
|
||||
"credential-model",
|
||||
[
|
||||
"catalog authentication",
|
||||
"data-plane authentication",
|
||||
"credential vending response",
|
||||
"temporary credential scope",
|
||||
"token expiration",
|
||||
],
|
||||
),
|
||||
validation_step(
|
||||
"client-live-run",
|
||||
[
|
||||
"client name",
|
||||
"client version",
|
||||
"RustFS build",
|
||||
"catalog backing",
|
||||
"expected status",
|
||||
"observed status",
|
||||
],
|
||||
),
|
||||
],
|
||||
),
|
||||
(
|
||||
"global_not_claimed",
|
||||
[
|
||||
"full AWS S3 Tables compatibility",
|
||||
"full MinIO AIStor private extension parity",
|
||||
"full Cloudflare R2 Data Catalog interoperability",
|
||||
"full Alibaba OSS Tables interoperability",
|
||||
"vendor-managed governance parity",
|
||||
"vendor-managed maintenance parity",
|
||||
],
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def live_conformance_harness(
|
||||
*,
|
||||
endpoint: str,
|
||||
@@ -1199,6 +1425,7 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
||||
parser.add_argument("--metadata-location")
|
||||
parser.add_argument("--cleanup", action="store_true")
|
||||
parser.add_argument("--print-engine-matrix", action="store_true")
|
||||
parser.add_argument("--print-vendor-audit", action="store_true")
|
||||
parser.add_argument("--print-live-conformance", action="store_true")
|
||||
parser.add_argument("--print-operations-guide", action="store_true")
|
||||
parser.add_argument("--print-spark-config", action="store_true")
|
||||
@@ -1225,6 +1452,9 @@ def run(args: argparse.Namespace, output: StringIO | None = None) -> None:
|
||||
if args.print_engine_matrix:
|
||||
print_json({"engine_compatibility": engine_compatibility_matrix()}, output)
|
||||
printed = True
|
||||
if args.print_vendor_audit:
|
||||
print_json({"vendor_compatibility_audit": vendor_compatibility_audit()}, output)
|
||||
printed = True
|
||||
if args.print_spark_config:
|
||||
print_json(
|
||||
{
|
||||
|
||||
@@ -118,16 +118,20 @@ PROFILE_DEFAULTS: dict[str, dict[str, Any]] = {
|
||||
"oss-tables": {
|
||||
"catalog_uri": "{endpoint}/iceberg",
|
||||
"rest_path": "/iceberg",
|
||||
"rest_signing_name": "s3",
|
||||
"warehouse": "{warehouse}",
|
||||
"rest_signing_name": "osstables",
|
||||
"warehouse": "acs:osstables:{region}:{account_id}:bucket/{table_bucket}",
|
||||
"credential_mode": "sigv4-s3fileio-credentials",
|
||||
"status": "reference-only",
|
||||
"compatibility_stage": "reference-only",
|
||||
"catalog_uri_shape": "{endpoint}/iceberg",
|
||||
"warehouse_shape": "{warehouse}",
|
||||
"warehouse_shape": "acs:osstables:{region}:{account_id}:bucket/{table_bucket}",
|
||||
"namespace_model": "provider-defined",
|
||||
"pagination_model": "vendor-specific",
|
||||
"not_claimed": ["live RustFS interoperability", "Alibaba OSS Tables API parity"],
|
||||
"not_claimed": [
|
||||
"live RustFS interoperability",
|
||||
"Alibaba OSS Tables API parity",
|
||||
"provider-error-code parity",
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@@ -164,6 +164,73 @@ class EngineCompatibilityTest(unittest.TestCase):
|
||||
self.assertIn("engine_compatibility", document)
|
||||
self.assertTrue(document["engine_compatibility"])
|
||||
|
||||
def test_vendor_compatibility_audit_records_provider_gaps(self) -> None:
|
||||
audit = engine_compatibility.vendor_compatibility_audit()
|
||||
by_profile = {entry["profile"]: entry for entry in audit["providers"]}
|
||||
|
||||
self.assertEqual(audit["claim_boundary"], "reference profiles and gap audit")
|
||||
self.assertEqual(audit["promotion_policy"], "live evidence required before compatibility claims expand")
|
||||
self.assertIn("aws-s3tables", by_profile)
|
||||
self.assertIn("minio-aistor", by_profile)
|
||||
self.assertIn("cloudflare-r2-data-catalog", by_profile)
|
||||
self.assertIn("oss-tables", by_profile)
|
||||
|
||||
aws = by_profile["aws-s3tables"]
|
||||
self.assertEqual(aws["signing_name"], "s3tables")
|
||||
self.assertEqual(aws["warehouse_shape"], "arn:aws:s3tables:{region}:{account_id}:bucket/{table_bucket}")
|
||||
self.assertIn("iceberg-rest-routes", aws["audit_categories"])
|
||||
self.assertIn("error-shapes", aws["audit_categories"])
|
||||
self.assertIn("permission-actions", aws["audit_categories"])
|
||||
self.assertIn("maintenance-semantics", aws["audit_categories"])
|
||||
self.assertIn("credential-vending", aws["audit_categories"])
|
||||
self.assertEqual(aws["rustfs_claim"], "profile-generator-only")
|
||||
|
||||
minio = by_profile["minio-aistor"]
|
||||
self.assertEqual(minio["catalog_base_path"], "/_iceberg")
|
||||
self.assertEqual(minio["signing_name"], "s3tables")
|
||||
self.assertEqual(minio["rustfs_claim"], "alias-smoke-plus-profile-generator")
|
||||
self.assertIn("private-extension-parity", minio["not_claimed"])
|
||||
|
||||
cloudflare = by_profile["cloudflare-r2-data-catalog"]
|
||||
self.assertEqual(cloudflare["auth_model"], "api-token")
|
||||
self.assertIn("managed-maintenance-parity", cloudflare["not_claimed"])
|
||||
self.assertEqual(cloudflare["rustfs_claim"], "profile-generator-only")
|
||||
|
||||
oss = by_profile["oss-tables"]
|
||||
self.assertEqual(oss["signing_name"], "osstables")
|
||||
self.assertEqual(oss["warehouse_shape"], "acs:osstables:{region}:{account_id}:bucket/{table_bucket}")
|
||||
self.assertIn("provider-error-code-parity", oss["not_claimed"])
|
||||
|
||||
self.assertIn("full AWS S3 Tables compatibility", audit["global_not_claimed"])
|
||||
self.assertIn("full Cloudflare R2 Data Catalog interoperability", audit["global_not_claimed"])
|
||||
|
||||
def test_vendor_compatibility_audit_has_stable_validation_steps(self) -> None:
|
||||
audit = engine_compatibility.vendor_compatibility_audit()
|
||||
step_names = {step["name"] for step in audit["required_validation_steps"]}
|
||||
|
||||
self.assertIn("route-and-field-shape", step_names)
|
||||
self.assertIn("error-response-shape", step_names)
|
||||
self.assertIn("permission-action-mapping", step_names)
|
||||
self.assertIn("maintenance-behavior", step_names)
|
||||
self.assertIn("credential-model", step_names)
|
||||
self.assertIn("client-live-run", step_names)
|
||||
|
||||
route_step = next(step for step in audit["required_validation_steps"] if step["name"] == "route-and-field-shape")
|
||||
self.assertIn("request path", route_step["evidence"])
|
||||
self.assertIn("response fields", route_step["evidence"])
|
||||
self.assertEqual(route_step["status"], "manual-audit-required")
|
||||
|
||||
def test_cli_prints_vendor_compatibility_audit(self) -> None:
|
||||
payload = engine_compatibility.cli_json(["--print-vendor-audit"])
|
||||
document = json.loads(payload)
|
||||
|
||||
self.assertIn("vendor_compatibility_audit", document)
|
||||
profiles = {entry["profile"] for entry in document["vendor_compatibility_audit"]["providers"]}
|
||||
self.assertIn("aws-s3tables", profiles)
|
||||
self.assertIn("minio-aistor", profiles)
|
||||
self.assertIn("cloudflare-r2-data-catalog", profiles)
|
||||
self.assertIn("oss-tables", profiles)
|
||||
|
||||
def test_cli_prints_vendor_spark_config(self) -> None:
|
||||
payload = engine_compatibility.cli_json(
|
||||
[
|
||||
@@ -192,6 +259,36 @@ class EngineCompatibilityTest(unittest.TestCase):
|
||||
self.assertNotIn("spark.sql.catalog.rustfs.s3.access-key-id", config)
|
||||
self.assertNotIn("spark.sql.catalog.rustfs.s3.secret-access-key", config)
|
||||
|
||||
def test_cli_prints_oss_tables_spark_config(self) -> None:
|
||||
payload = engine_compatibility.cli_json(
|
||||
[
|
||||
"--print-spark-config",
|
||||
"--profile",
|
||||
"oss-tables",
|
||||
"--endpoint",
|
||||
"https://cn-hangzhou.oss-tables.aliyuncs.com",
|
||||
"--region",
|
||||
"cn-hangzhou",
|
||||
"--account-id",
|
||||
"123456789012",
|
||||
"--table-bucket",
|
||||
"analytics",
|
||||
]
|
||||
)
|
||||
document = json.loads(payload)
|
||||
config = document["spark_config"]
|
||||
|
||||
self.assertEqual(config["spark.sql.catalog.rustfs.uri"], "https://cn-hangzhou.oss-tables.aliyuncs.com/iceberg")
|
||||
self.assertEqual(
|
||||
config["spark.sql.catalog.rustfs.warehouse"],
|
||||
"acs:osstables:cn-hangzhou:123456789012:bucket/analytics",
|
||||
)
|
||||
self.assertEqual(config["spark.sql.catalog.rustfs.rest.signing-name"], "osstables")
|
||||
self.assertEqual(config["spark.sql.catalog.rustfs.s3.endpoint"], "https://oss-cn-hangzhou.aliyuncs.com")
|
||||
self.assertEqual(config["spark.sql.catalog.rustfs.s3.access-key-id"], "rustfsadmin")
|
||||
self.assertEqual(config["spark.sql.catalog.rustfs.s3.secret-access-key"], "rustfsadmin")
|
||||
self.assertNotIn("spark.sql.catalog.rustfs.s3.path-style-access", config)
|
||||
|
||||
def test_cli_spark_config_keeps_explicit_rest_overrides(self) -> None:
|
||||
payload = engine_compatibility.cli_json(
|
||||
[
|
||||
|
||||
@@ -73,6 +73,7 @@ class PyIcebergSmokeConfigTest(unittest.TestCase):
|
||||
)
|
||||
self.assertEqual(profiles["minio-aistor"]["rest_signing_name"], "s3tables")
|
||||
self.assertEqual(profiles["cloudflare-r2-data-catalog"]["credential_mode"], "catalog-vended")
|
||||
self.assertEqual(profiles["oss-tables"]["rest_signing_name"], "osstables")
|
||||
|
||||
def test_vendor_profiles_publish_migration_boundaries(self) -> None:
|
||||
profiles = pyiceberg_smoke.vendor_profiles()
|
||||
@@ -89,6 +90,10 @@ class PyIcebergSmokeConfigTest(unittest.TestCase):
|
||||
self.assertEqual(cloudflare["credential_mode"], "catalog-vended")
|
||||
self.assertIn("live RustFS interoperability", cloudflare["not_claimed"])
|
||||
|
||||
oss = profiles["oss-tables"]
|
||||
self.assertEqual(oss["warehouse_shape"], "acs:osstables:{region}:{account_id}:bucket/{table_bucket}")
|
||||
self.assertIn("provider-error-code parity", oss["not_claimed"])
|
||||
|
||||
def test_aws_reference_profile_formats_s3tables_warehouse_arn(self) -> None:
|
||||
args = self.parse_with_args([
|
||||
"--profile",
|
||||
@@ -109,6 +114,26 @@ class PyIcebergSmokeConfigTest(unittest.TestCase):
|
||||
self.assertEqual(properties["warehouse"], "arn:aws:s3tables:us-east-1:123456789012:bucket/analytics")
|
||||
self.assertEqual(properties["rest.signing-name"], "s3tables")
|
||||
|
||||
def test_oss_tables_reference_profile_formats_warehouse_arn(self) -> None:
|
||||
args = self.parse_with_args([
|
||||
"--profile",
|
||||
"oss-tables",
|
||||
"--endpoint",
|
||||
"https://cn-hangzhou.oss-tables.aliyuncs.com",
|
||||
"--region",
|
||||
"cn-hangzhou",
|
||||
"--account-id",
|
||||
"123456789012",
|
||||
"--table-bucket",
|
||||
"analytics",
|
||||
])
|
||||
|
||||
properties = pyiceberg_smoke.catalog_properties(args)
|
||||
|
||||
self.assertEqual(properties["uri"], "https://cn-hangzhou.oss-tables.aliyuncs.com/iceberg")
|
||||
self.assertEqual(properties["warehouse"], "acs:osstables:cn-hangzhou:123456789012:bucket/analytics")
|
||||
self.assertEqual(properties["rest.signing-name"], "osstables")
|
||||
|
||||
def test_cloudflare_reference_profile_uses_catalog_uri_and_warehouse_name(self) -> None:
|
||||
args = self.parse_with_args([
|
||||
"--profile",
|
||||
|
||||
Reference in New Issue
Block a user