test(table-catalog): generate DuckDB REST attach SQL (#6749)

2026-08-28 16:07:05 +00:00 · 2026-08-27 22:25:33 +08:00
parent 2e6c820f53
commit e281ed2f6d
4 changed files with 246 additions and 10 deletions
@@ -43,7 +43,7 @@ catalog extension.
 | PyIceberg | Automated | Creates namespace and table, appends rows, reloads, scans, probes metadata-location, refs, views, maintenance, diagnostics, and optional catalog-vended table credentials with an exact-prefix data-plane scope check. |
 | Spark Iceberg REST catalog | Manual/live harness | RustFS can generate pinned Spark/Iceberg package inputs, REST catalog properties, SQL, run commands, expected `row_count=2`, and a CI opt-in gate for namespace creation, table creation, append, refresh, count, and cleanup. Live Spark execution and commit-conflict probing are still manual validation items unless explicitly enabled in the runner. |
 | Trino Iceberg REST catalog | Manual/live harness | RustFS can generate catalog properties and a read-only `SELECT COUNT(*)` command for a table created by PyIceberg or Spark. Write compatibility is not claimed. |
-| DuckDB Iceberg | Manual/live harness | RustFS can generate `httpfs` and `iceberg` SQL using an operator-supplied current metadata location. Write and commit compatibility are not claimed. |
+| DuckDB Iceberg | Manual/live harness | RustFS can generate the read-only `iceberg_scan` path using an operator-supplied current metadata location and a generic signed Iceberg REST Catalog profile for `/iceberg` or `/_iceberg`. The REST profile disables staged create, post-create metadata updates, multi-table commit, client-side file removal, and purge-on-drop. Write and commit compatibility remain not claimed until repeatable live evidence is automated. |
 | StarRocks Iceberg REST catalog | Documented, not automated | External catalog read-path reference only. Write compatibility is not claimed. |
 | Databend | Manual/live harness | RustFS can generate an S3 stage read probe for table data files. RustFS does not claim Databend Iceberg REST Catalog integration yet. |
 | Snowflake Open Catalog / Iceberg integrations | Generated harness | RustFS can generate an operator-adapted external volume/catalog SQL template. Live RustFS interoperability is not claimed. |
@@ -250,6 +250,7 @@ python3 scripts/table-catalog/pyiceberg_smoke.py --print-vendor-profiles
 python3 scripts/table-catalog/pyiceberg_smoke.py --print-production-readiness
 python3 scripts/table-catalog/engine_compatibility.py --print-vendor-audit
 python3 scripts/table-catalog/engine_compatibility.py --print-spark-config
+python3 scripts/table-catalog/engine_compatibility.py --print-duckdb-rest-sql
 python3 scripts/table-catalog/engine_compatibility.py \
  --profile aws-s3tables \
  --region us-east-1 \
@@ -241,6 +241,7 @@ python3 scripts/table-catalog/engine_compatibility.py \
  --table-bucket analytics \
  --print-spark-config
 python3 scripts/table-catalog/engine_compatibility.py --print-spark-sql --cleanup
+python3 scripts/table-catalog/engine_compatibility.py --print-duckdb-rest-sql
 python3 scripts/table-catalog/engine_compatibility.py --print-live-conformance --cleanup
 python3 scripts/table-catalog/engine_compatibility.py --print-operations-guide
 ```
@@ -310,7 +311,7 @@ The smoke test also probes catalog-backed advanced Iceberg surfaces:
 | PyIceberg | Automated smoke target | create namespace, create table, append, reload, scan, metadata-location, refs, views, maintenance, diagnostics, optional catalog-vended table credentials with exact-prefix data-plane scope probe |
 | Spark Iceberg REST catalog | Manual/live harness | pinned Spark and Iceberg package inputs, configuration, SQL, run command, expected row count, and cleanup can be generated for a running RustFS endpoint; CI execution is opt-in |
 | Trino Iceberg REST catalog | Manual/live read probe | generated catalog properties and a read-only SELECT probe for a table created by PyIceberg or Spark; no write compatibility claim yet |
-| DuckDB Iceberg | Manual/live read probe | generated httpfs/iceberg SQL using an operator-supplied current metadata location; read-path only |
+| DuckDB Iceberg | Manual/live harness | generated metadata-location read SQL plus generic Iceberg REST Catalog attach SQL for `/iceberg` with `s3` signing or `/_iceberg` with `s3tables` signing; write compatibility still requires repeatable live evidence |
 | StarRocks Iceberg REST catalog | Documented, not automated | external catalog read-path reference only |
 | Databend | Manual/live S3 stage probe | generated S3 stage read probe for table data files; Iceberg REST catalog integration is not claimed |
 | Snowflake/Open Catalog integrations | Manual reference probe | generated external volume/catalog SQL template; live RustFS interoperability is not claimed |
@@ -503,6 +504,43 @@ RUSTFS_TABLE_CATALOG_CREDENTIAL_TTL_SECONDS=900

 The TTL is clamped to the supported short-lived range by the server.

+## DuckDB REST Catalog Profile
+
+DuckDB can read an individual Iceberg table with `iceberg_scan` or attach RustFS
+as a generic Iceberg REST Catalog. The metadata-location path remains read-only.
+The attached catalog path is the prerequisite for DuckDB writes.
+
+Generate the canonical RustFS REST Catalog profile:
+
+```bash
+python3 scripts/table-catalog/engine_compatibility.py \
+  --endpoint http://127.0.0.1:9000 \
+  --warehouse rustfs-s3table-smoke \
+  --namespace smoke \
+  --table events \
+  --rest-path /iceberg \
+  --rest-signing-name s3 \
+  --print-duckdb-rest-sql
+```
+
+Generate the compatibility alias profile by changing the last three arguments:
+
+```bash
+python3 scripts/table-catalog/engine_compatibility.py \
+  --rest-path /_iceberg \
+  --rest-signing-name s3tables \
+  --print-duckdb-rest-sql
+```
+
+The generated `ATTACH` disables staged create, post-create metadata updates,
+multi-table commit, client-side file removal, and purge-on-drop. These options
+keep DuckDB within RustFS's claimed single-table REST surface. Do not replace
+the explicit endpoint with DuckDB `ENDPOINT_TYPE S3_TABLES`; that shortcut is
+for AWS S3 Tables endpoint and warehouse shapes.
+
+This profile is generated conformance input. It does not promote DuckDB write
+compatibility until the repeatable live smoke records passing evidence.
+
 ## Spark Manual/Live Harness

 Spark validation should use the same RustFS endpoint and warehouse bucket as the
@@ -17,7 +17,7 @@ DEFAULT_SPARK_VERSION = "3.5.4"
 DEFAULT_ICEBERG_VERSION = "1.7.1"
 DEFAULT_SCALA_VERSION = "2.12"
 DEFAULT_TRINO_VERSION = "477"
-DEFAULT_DUCKDB_VERSION = "1.3.2"
+DEFAULT_DUCKDB_VERSION = "1.5.5"
 DEFAULT_SNOWFLAKE_CLIENT_VERSION = "operator-recorded"
 DEFAULT_DATABEND_VERSION = "operator-recorded"
 DEFAULT_TRINO_SERVER = "http://127.0.0.1:8080"
@@ -155,12 +155,13 @@ def engine_compatibility_matrix() -> list[dict[str, Any]]:
        },
        {
            "client": "DuckDB Iceberg",
-            "status": "manual-live-read-probe",
+            "status": "manual-live-harness",
            "entrypoint": "scripts/table-catalog/engine_compatibility.py --print-live-conformance",
            "scenarios": [
                scenario("metadata-read", "manual-live-probe", "read a supplied Iceberg metadata location through DuckDB iceberg_scan"),
-                scenario("read-table", "manual-live-probe", "read-path verification only"),
-                scenario("write-table", "not-claimed", "DuckDB write/commit compatibility is not claimed"),
+                scenario("catalog-attach", "generated-harness", "attach RustFS as a generic signed Iceberg REST catalog"),
+                scenario("read-table", "manual-live-probe", "read an existing table through the attached catalog"),
+                scenario("write-table", "manual-validation-required", "generated SQL does not promote DuckDB write compatibility"),
            ],
        },
        {
@@ -489,8 +490,66 @@ def duckdb_sql_probe(
    return "\n".join(statements) + "\n"


-def duckdb_command() -> str:
-    return shell_join(["duckdb", "-c", ".read /tmp/rustfs-s3tables-duckdb-read.sql"])
+def duckdb_rest_catalog_sql(
+    *,
+    endpoint: str,
+    warehouse: str,
+    access_key: str,
+    secret_key: str,
+    region: str,
+    catalog_name: str,
+    namespace: str,
+    table: str,
+    rest_path: str,
+    rest_signing_name: str,
+) -> str:
+    parsed = re.match(r"^(https?)://(.+)$", normalized_endpoint(endpoint))
+    if not parsed:
+        raise ValueError("DuckDB REST catalog endpoint must include http:// or https://")
+    scheme, endpoint_without_scheme = parsed.groups()
+    rest_path = normalized_rest_path(rest_path)
+    catalog_identifier = quote_double_identifier(catalog_name)
+    table_identifier = ".".join(
+        [catalog_identifier, quote_double_identifier(namespace), quote_double_identifier(table)]
+    )
+    statements = [
+        "INSTALL httpfs;",
+        "LOAD httpfs;",
+        "INSTALL iceberg;",
+        "LOAD iceberg;",
+        "CREATE OR REPLACE SECRET rustfs_s3 (",
+        "  TYPE s3,",
+        "  PROVIDER config,",
+        f"  KEY_ID {sql_string(access_key)},",
+        f"  SECRET {sql_string(secret_key)},",
+        f"  REGION {sql_string(region)},",
+        f"  ENDPOINT {sql_string(endpoint_without_scheme)},",
+        "  URL_STYLE 'path',",
+        f"  USE_SSL {'true' if scheme == 'https' else 'false'},",
+        f"  SCOPE {sql_string(f's3://{warehouse}')}",
+        ");",
+        f"ATTACH {sql_string(warehouse)} AS {catalog_identifier} (",
+        "  TYPE iceberg,",
+        f"  ENDPOINT {sql_string(f'{normalized_endpoint(endpoint)}{rest_path}')},",
+        "  AUTHORIZATION_TYPE 'sigv4',",
+        "  SECRET 'rustfs_s3',",
+        f"  SIGV4_REGION {sql_string(region)},",
+        f"  SIGV4_SERVICE {sql_string(rest_signing_name)},",
+        "  ACCESS_DELEGATION_MODE 'none',",
+        "  STAGE_CREATE_TABLES false,",
+        "  SKIP_CREATE_TABLE_METADATA_UPDATES true,",
+        "  DISABLE_MULTI_TABLE_COMMIT true,",
+        "  REMOVE_FILES_ON_DELETE false,",
+        "  PURGE_REQUESTED false,",
+        "  SUPPORT_NESTED_NAMESPACES false",
+        ");",
+        f"SELECT COUNT(*) AS row_count FROM {table_identifier};",
+    ]
+    return "\n".join(statements) + "\n"
+
+
+def duckdb_command(*, sql_file: str = "/tmp/rustfs-s3tables-duckdb-read.sql") -> str:
+    return shell_join(["duckdb", "-c", f".read {sql_file}"])


 def snowflake_sql_template(*, endpoint: str, warehouse: str, rest_path: str, namespace: str, table: str) -> str:
@@ -1425,6 +1484,18 @@ def live_conformance_harness(
        region=region,
        metadata_location=metadata_location,
    )
+    duckdb_rest_sql = duckdb_rest_catalog_sql(
+        endpoint=endpoint,
+        warehouse=warehouse,
+        access_key=access_key,
+        secret_key=secret_key,
+        region=region,
+        catalog_name=catalog_name,
+        namespace=namespace,
+        table=table,
+        rest_path=rest_path,
+        rest_signing_name=rest_signing_name,
+    )
    snowflake_sql = snowflake_sql_template(
        endpoint=endpoint,
        warehouse=warehouse,
@@ -1553,13 +1624,24 @@ def live_conformance_harness(
                    OrderedDict(
                        [
                            ("name", "DuckDB Iceberg"),
-                            ("status", "manual-live-read-probe"),
+                            ("status", "manual-live-harness"),
                            ("version", duckdb_version),
                            ("metadata_location", metadata_location),
                            ("sql_file", "/tmp/rustfs-s3tables-duckdb-read.sql"),
                            ("sql", duckdb_sql),
                            ("command", duckdb_command()),
                            ("expected", "iceberg_scan returns row_count=2 when metadata_location points at the current Iceberg metadata JSON"),
+                            ("rest_catalog_sql_file", "/tmp/rustfs-s3tables-duckdb-rest.sql"),
+                            ("rest_catalog_sql", duckdb_rest_sql),
+                            (
+                                "rest_catalog_command",
+                                duckdb_command(sql_file="/tmp/rustfs-s3tables-duckdb-rest.sql"),
+                            ),
+                            (
+                                "rest_catalog_expected",
+                                "generic Iceberg REST ATTACH returns row_count=2 for an existing RustFS table",
+                            ),
+                            ("rest_catalog_write_compatibility", "manual-live-validation-required"),
                            ("write_compatibility", "not-claimed"),
                        ]
                    ),
@@ -1627,6 +1709,7 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
    parser.add_argument("--print-operations-guide", action="store_true")
    parser.add_argument("--print-spark-config", action="store_true")
    parser.add_argument("--print-spark-sql", action="store_true")
+    parser.add_argument("--print-duckdb-rest-sql", action="store_true")
    return parser.parse_args(argv)


@@ -1734,6 +1817,24 @@ def run(args: argparse.Namespace, output: StringIO | None = None) -> None:
        else:
            output.write(sql)
        printed = True
+    if args.print_duckdb_rest_sql:
+        sql = duckdb_rest_catalog_sql(
+            endpoint=args.endpoint,
+            warehouse=args.warehouse,
+            access_key=args.access_key,
+            secret_key=args.secret_key,
+            region=args.region,
+            catalog_name=args.catalog_name,
+            namespace=args.namespace,
+            table=args.table,
+            rest_path=args.rest_path or "/iceberg",
+            rest_signing_name=args.rest_signing_name or "s3",
+        )
+        if output is None:
+            print(sql, end="")
+        else:
+            output.write(sql)
+        printed = True
    if not printed:
        print_json({"engine_compatibility": engine_compatibility_matrix()}, output)

@@ -41,6 +41,12 @@ class EngineCompatibilityTest(unittest.TestCase):
        self.assertEqual(trino["status"], "manual-live-read-probe")
        self.assertContainsScenario(trino, "catalog-load", "manual-live-probe")

+        duckdb = by_client["DuckDB Iceberg"]
+        self.assertEqual(duckdb["status"], "manual-live-harness")
+        self.assertContainsScenario(duckdb, "metadata-read", "manual-live-probe")
+        self.assertContainsScenario(duckdb, "catalog-attach", "generated-harness")
+        self.assertContainsScenario(duckdb, "write-table", "manual-validation-required")
+
    def test_spark_config_uses_rustfs_rest_catalog_and_s3fileio(self) -> None:
        config = engine_compatibility.spark_catalog_config(
            endpoint="http://127.0.0.1:9000",
@@ -158,6 +164,70 @@ class EngineCompatibilityTest(unittest.TestCase):
                table="orders",
            )

+    def test_duckdb_rest_catalog_sql_uses_rustfs_compatibility_options(self) -> None:
+        sql = engine_compatibility.duckdb_rest_catalog_sql(
+            endpoint="http://127.0.0.1:9000",
+            warehouse="rustfs-s3table-smoke",
+            access_key="rustfsadmin",
+            secret_key="rustfsadmin",
+            region="us-east-1",
+            catalog_name="rustfs",
+            namespace="smoke",
+            table="events",
+            rest_path="/iceberg",
+            rest_signing_name="s3",
+        )
+
+        self.assertIn("CREATE OR REPLACE SECRET rustfs_s3", sql)
+        self.assertIn("ENDPOINT '127.0.0.1:9000'", sql)
+        self.assertIn("SCOPE 's3://rustfs-s3table-smoke'", sql)
+        self.assertIn("ATTACH 'rustfs-s3table-smoke' AS \"rustfs\"", sql)
+        self.assertIn("ENDPOINT 'http://127.0.0.1:9000/iceberg'", sql)
+        self.assertIn("SIGV4_SERVICE 's3'", sql)
+        self.assertIn("STAGE_CREATE_TABLES false", sql)
+        self.assertIn("SKIP_CREATE_TABLE_METADATA_UPDATES true", sql)
+        self.assertIn("DISABLE_MULTI_TABLE_COMMIT true", sql)
+        self.assertIn("REMOVE_FILES_ON_DELETE false", sql)
+        self.assertIn("PURGE_REQUESTED false", sql)
+        self.assertIn('FROM "rustfs"."smoke"."events"', sql)
+        self.assertNotIn("ENDPOINT_TYPE", sql)
+
+    def test_duckdb_rest_catalog_sql_supports_s3tables_alias(self) -> None:
+        sql = engine_compatibility.duckdb_rest_catalog_sql(
+            endpoint="https://rustfs.example",
+            warehouse="analytics",
+            access_key="access'key",
+            secret_key="secret'key",
+            region="us-east-1",
+            catalog_name="rustfs_compat",
+            namespace="smoke",
+            table="events",
+            rest_path="/_iceberg",
+            rest_signing_name="s3tables",
+        )
+
+        self.assertIn("USE_SSL true", sql)
+        self.assertIn("ENDPOINT 'rustfs.example'", sql)
+        self.assertIn("ENDPOINT 'https://rustfs.example/_iceberg'", sql)
+        self.assertIn("SIGV4_SERVICE 's3tables'", sql)
+        self.assertIn("KEY_ID 'access''key'", sql)
+        self.assertIn("SECRET 'secret''key'", sql)
+
+    def test_duckdb_rest_catalog_sql_rejects_endpoint_without_scheme(self) -> None:
+        with self.assertRaisesRegex(ValueError, "must include http:// or https://"):
+            engine_compatibility.duckdb_rest_catalog_sql(
+                endpoint="127.0.0.1:9000",
+                warehouse="analytics",
+                access_key="rustfsadmin",
+                secret_key="rustfsadmin",
+                region="us-east-1",
+                catalog_name="rustfs",
+                namespace="smoke",
+                table="events",
+                rest_path="/iceberg",
+                rest_signing_name="s3",
+            )
+
    def test_cli_prints_machine_readable_engine_matrix(self) -> None:
        payload = engine_compatibility.cli_json(["--print-engine-matrix"])
        document = json.loads(payload)
@@ -308,6 +378,27 @@ class EngineCompatibilityTest(unittest.TestCase):
        self.assertEqual(config["spark.sql.catalog.rustfs.uri"], "http://127.0.0.1:9000/_iceberg")
        self.assertEqual(config["spark.sql.catalog.rustfs.rest.signing-name"], "s3tables")

+    def test_cli_prints_duckdb_rest_catalog_sql(self) -> None:
+        sql = engine_compatibility.cli_json(
+            [
+                "--print-duckdb-rest-sql",
+                "--endpoint",
+                "http://127.0.0.1:9000",
+                "--warehouse",
+                "analytics",
+                "--catalog-name",
+                "rustfs_compat",
+                "--rest-path",
+                "/_iceberg",
+                "--rest-signing-name",
+                "s3tables",
+            ]
+        )
+
+        self.assertIn("ATTACH 'analytics' AS \"rustfs_compat\"", sql)
+        self.assertIn("ENDPOINT 'http://127.0.0.1:9000/_iceberg'", sql)
+        self.assertIn("SIGV4_SERVICE 's3tables'", sql)
+
    def test_live_conformance_harness_pins_clients_and_records_commands(self) -> None:
        harness = engine_compatibility.live_conformance_harness(
            endpoint="http://127.0.0.1:9000",
@@ -359,10 +450,15 @@ class EngineCompatibilityTest(unittest.TestCase):
        self.assertEqual(trino["write_compatibility"], "not-claimed")

        duckdb = by_client["DuckDB Iceberg"]
-        self.assertEqual(duckdb["status"], "manual-live-read-probe")
+        self.assertEqual(duckdb["status"], "manual-live-harness")
+        self.assertEqual(duckdb["version"], "1.5.5")
        self.assertIn("LOAD httpfs", duckdb["sql"])
        self.assertIn("LOAD iceberg", duckdb["sql"])
        self.assertIn("iceberg_scan", duckdb["sql"])
+        self.assertIn("ATTACH 'rustfs-s3table-smoke'", duckdb["rest_catalog_sql"])
+        self.assertIn("STAGE_CREATE_TABLES false", duckdb["rest_catalog_sql"])
+        self.assertIn("SKIP_CREATE_TABLE_METADATA_UPDATES true", duckdb["rest_catalog_sql"])
+        self.assertEqual(duckdb["rest_catalog_write_compatibility"], "manual-live-validation-required")
        self.assertEqual(duckdb["write_compatibility"], "not-claimed")

        snowflake = by_client["Snowflake Open Catalog / Iceberg integrations"]