From db8f55cb97e60a8e91e185e3ce13c5a911a99341 Mon Sep 17 00:00:00 2001 From: Henry Guo Date: Sun, 16 Aug 2026 03:05:09 +0800 Subject: [PATCH] feat(table-catalog): finalize Iceberg REST behavior (#6072) * feat(table-catalog): finalize Iceberg REST behavior * fix(table-catalog): address REST finalization regressions * test(table-catalog): expect REST commit conflicts * test(table-catalog): avoid serialized view test deadlocks * fix(table-catalog): adapt shared test backend * fix(table-catalog): enforce Iceberg metadata invariants * fix(table-catalog): preserve manifest length in test * test(table-catalog): use valid metadata fixtures * test(table-catalog): seed manifests before manifest lists * fix(table-catalog): restore validation gates --------- Co-authored-by: Henry Guo Co-authored-by: overtrue --- Cargo.lock | 4 + Cargo.toml | 2 +- docs/architecture/s3-tables-support-matrix.md | 12 +- rustfs/Cargo.toml | 3 +- .../handlers/table_catalog/credentials.rs | 2 +- .../src/admin/handlers/table_catalog/mod.rs | 1559 ++++++--- .../src/admin/handlers/table_catalog/table.rs | 14 +- .../src/admin/handlers/table_catalog/tests.rs | 2943 +++++++++++++++-- .../src/admin/handlers/table_catalog/view.rs | 14 +- rustfs/src/storage/access.rs | 2 +- rustfs/src/table_catalog/iceberg/manifest.rs | 255 +- .../src/table_catalog/iceberg/validation.rs | 2036 +++++++++++- rustfs/src/table_catalog/mod.rs | 4 + rustfs/src/table_catalog/store/migration.rs | 6 +- rustfs/src/table_catalog/store/mod.rs | 128 +- rustfs/src/table_catalog/store/object.rs | 194 +- rustfs/src/table_catalog/store/strong.rs | 148 +- rustfs/src/table_catalog/test_support.rs | 151 +- rustfs/src/table_catalog/tests.rs | 1839 +++++++++- scripts/table-catalog/failure_coverage.py | 16 +- .../table-catalog/test_failure_coverage.py | 4 + 21 files changed, 8165 insertions(+), 1171 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4c7c5de82..38637d173 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -278,6 +278,7 @@ checksum = "312c1ea69e5fe9966e0029fb95aca8790100b85aff4f0d3b00a9337c74069a9c" dependencies = [ "bigdecimal", "bon", + "crc32fast", "digest 0.11.3", "log", "miniz_oxide 0.9.1", @@ -289,9 +290,11 @@ dependencies = [ "serde", "serde_bytes", "serde_json", + "snap", "strum", "thiserror 2.0.20", "uuid", + "zstd", ] [[package]] @@ -9200,6 +9203,7 @@ dependencies = [ "serial_test", "sha2 0.11.0", "shadow-rs", + "snap", "socket2", "subtle", "sysinfo", diff --git a/Cargo.toml b/Cargo.toml index 9e0524dd7..6bbd3acd1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -171,7 +171,7 @@ tower = { version = "0.5.3" } tower-http = { version = "0.7.0" } # Serialization and Data Formats -apache-avro = "0.22.0" +apache-avro = { version = "0.22.0", features = ["snappy", "zstandard"] } bytes = { version = "1.12.1" } bytesize = "2.7.0" byteorder = "1.5.0" diff --git a/docs/architecture/s3-tables-support-matrix.md b/docs/architecture/s3-tables-support-matrix.md index cb356350c..65eeb4be7 100644 --- a/docs/architecture/s3-tables-support-matrix.md +++ b/docs/architecture/s3-tables-support-matrix.md @@ -61,17 +61,17 @@ catalog extension. | Area | Status | Covered behavior | |---|---|---| -| Catalog config | Supported | `GET /v1/config` advertises RustFS catalog defaults and route capabilities. | +| Catalog config | Supported | `GET /v1/config` advertises RustFS catalog defaults and only the supported OpenAPI REST paths in `endpoints`. RustFS administration, maintenance, migration, diagnostics, refs, and metadata-location extensions remain available but are not presented as standard Iceberg REST endpoints. | | Table bucket discovery | Supported | `PUT` and `GET /v1/buckets/{warehouse}` enable and inspect table bucket state. | | Namespaces | Supported | Create, list, load, existence check, and drop namespace routes are registered on both catalog prefixes. List responses support Iceberg REST `pageSize`/`pageToken` pagination with context-bound tokens and bounded catalog-store reads. Namespace identifiers are limited to 512 ASCII characters so persisted paths and stateless continuation tokens remain bounded. | -| Tables | Supported | Create, register, list, load, existence check, commit, metadata-location get/update, and drop table routes are registered on both catalog prefixes. Table and view listings support Iceberg REST `pageSize`/`pageToken` pagination with context-bound tokens and bounded catalog-store reads. | -| Commit CAS | Supported | Single-table commits validate base metadata, expected version token, referenced object existence, warehouse scope, and Iceberg commit requirements before advancing the current metadata pointer. Standard commits preserve the normal commit-token file name and use an immutable-table-scoped fallback when rename followed by source-name reuse would otherwise collide at the same generation and commit ID. | +| Tables | Supported | Create, register, list, load, existence check, commit, metadata-location get/update, and drop table routes are registered on both catalog prefixes. Table and view listings support Iceberg REST `pageSize`/`pageToken` pagination with context-bound tokens and bounded catalog-store reads. Commit identifiers must match the URL resource; unknown requirements, updates, and snapshot operations fail as bad requests; staged create, register overwrite, purge-on-drop, and v3-only encryption-key updates return an explicit unsupported-operation response. Standard statistics, partition statistics, and schema/spec cleanup updates are accepted. | +| Commit CAS | Supported | Single-table commits validate base metadata, expected version token, referenced object existence, warehouse scope, and Iceberg commit requirements before advancing the current metadata pointer. Externally supplied metadata transitions preserve monotonic column, partition, and sequence assignment watermarks and immutable definitions for retained schemas, partition specs, sort orders, and snapshots. Standard commits preserve the normal commit-token file name and use an immutable-table-scoped fallback when rename followed by source-name reuse would otherwise collide at the same generation and commit ID. The catalog does not advertise `idempotency-key-lifetime`; clients must treat standard mutation-wide `Idempotency-Key` semantics as unsupported. | | Commit recovery | Supported | Commit log, idempotency lookup, diagnostics, and recovery routes expose staged/finalization gaps and repair safe idempotency gaps without moving the table pointer. | | Snapshot refs | Supported | Refs can be listed, created or replaced, and deleted through catalog commits. `main` is protected and refs with explicit retention require forced delete. | -| Iceberg views | Supported | Basic create, list, load, replace, existence check, and drop routes persist view metadata with view-scoped authorization. | -| Table credentials endpoint | Supported | Returns an empty `storage-credentials` list by default. Returns table-scoped temporary credentials only when credential vending is enabled. | +| Iceberg views | Supported | Basic create, list, load, replace, existence check, and drop routes persist view metadata with view-scoped authorization. Replace identifiers must match the URL resource, `schema-id: -1` resolves to the last added schema, one commit timestamp is used consistently, and only Iceberg view format version 1 is accepted. | +| Table credentials endpoint | Supported | Returns an empty `storage-credentials` list by default. Returns table-scoped temporary credentials only when credential vending is enabled. Credential responses set `Cache-Control: no-store, private`, `Pragma: no-cache`, and `Expires: 0`. | | Catalog diagnostics and export | Supported | Exposes recovery state, consistency state, backing manifest, recoverable commit-log WAL state, strong backing migration target, single-active-writer policy, and scale validation matrix. | -| Catalog import and rollback | Supported | Import/register and rollback use catalog validation and commit paths rather than direct pointer mutation. | +| Catalog import and rollback | Supported | Import/register and online rollback use catalog validation and commit paths rather than direct pointer mutation. Online rollback accepts only a forward-safe metadata target that preserves assignment watermarks and retained definitions. Restoring an older target that lowers those watermarks is an offline disaster-recovery operation and requires every writer to be stopped. | | External catalog bridge | Supported operator path | Operator-supplied metadata pointer sync/import is supported for external catalog identity boundaries. Online vendor SDK polling and policy mirroring are not claimed. | | Multi-table transactions | Not claimed | RustFS currently claims single-table commit atomicity only. | diff --git a/rustfs/Cargo.toml b/rustfs/Cargo.toml index b88db7aee..69916207e 100644 --- a/rustfs/Cargo.toml +++ b/rustfs/Cargo.toml @@ -278,6 +278,8 @@ rustfs-signer.workspace = true serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true, features = ["raw_value"] } serde_urlencoded = { workspace = true } +snap.workspace = true +zstd.workspace = true # Cryptography and Security rustls = { workspace = true, default-features = false, features = ["aws-lc-rs", "logging", "tls12", "prefer-post-quantum", "std"] } @@ -355,7 +357,6 @@ rcgen = { workspace = true } rustfs-test-utils.workspace = true # diagnose_e2e fixtures (archives are generated in-test, never checked in) zip = { workspace = true } -zstd = { workspace = true } # Enables the shared MockWarmBackend / xl.meta assertion helpers exposed via # the ecstore `api::tier::test_util` facade module (rustfs/backlog#1148 ilm-6). rustfs-ecstore = { workspace = true, features = ["test-util"] } diff --git a/rustfs/src/admin/handlers/table_catalog/credentials.rs b/rustfs/src/admin/handlers/table_catalog/credentials.rs index e997e39d7..62efde2e5 100644 --- a/rustfs/src/admin/handlers/table_catalog/credentials.rs +++ b/rustfs/src/admin/handlers/table_catalog/credentials.rs @@ -29,6 +29,6 @@ impl Operation for RestLoadCredentialsHandler { let issuer = IamTableCredentialIssuer::from_request(&req)?; let response = load_credentials_response(&store, &warehouse, &namespace, &table, &issuer, Some(&principal.credentials)).await?; - build_json_response(StatusCode::OK, &response) + build_sensitive_json_response(StatusCode::OK, &response) } } diff --git a/rustfs/src/admin/handlers/table_catalog/mod.rs b/rustfs/src/admin/handlers/table_catalog/mod.rs index f083d252b..1dd03db42 100644 --- a/rustfs/src/admin/handlers/table_catalog/mod.rs +++ b/rustfs/src/admin/handlers/table_catalog/mod.rs @@ -25,6 +25,7 @@ use crate::auth::{check_key_valid_with_context, get_session_token}; use crate::error::ApiError; use crate::server::{RemoteAddr, TABLE_CATALOG_COMPAT_PREFIX, TABLE_CATALOG_PREFIX}; use crate::table_catalog::{DEFAULT_WAREHOUSE_ID, TableCatalogStore}; +use bytes::Bytes; use futures::{StreamExt, TryStreamExt, stream}; use http::{HeaderMap, HeaderValue, StatusCode}; use hyper::Method; @@ -74,6 +75,9 @@ const ENV_TABLE_CATALOG_CREDENTIAL_TTL_SECONDS: &str = "RUSTFS_TABLE_CATALOG_CRE const DEFAULT_TABLE_CATALOG_CREDENTIAL_TTL_SECONDS: i64 = 15 * 60; const MIN_TABLE_CATALOG_CREDENTIAL_TTL_SECONDS: i64 = 60; const MAX_TABLE_CATALOG_CREDENTIAL_TTL_SECONDS: i64 = 60 * 60; +const TABLE_CATALOG_REQUEST_BODY_TIMEOUT: StdDuration = StdDuration::from_secs(30); +const TABLE_CATALOG_COMMIT_REQUIREMENT_MAX_COUNT: usize = 1_024; +const TABLE_CATALOG_COMMIT_UPDATE_MAX_COUNT: usize = 1_024; const NAMESPACE_REQUEST_BODY_MAX_SIZE: usize = MAX_ADMIN_REQUEST_BODY_SIZE; const NAMESPACE_REQUEST_BODY_TIMEOUT: StdDuration = StdDuration::from_secs(10); const RENAME_TABLE_BODY_MAX_SIZE: usize = 16 * 1024; @@ -96,6 +100,7 @@ const ICEBERG_ERROR_NO_SUCH_VIEW: &str = "NoSuchViewException"; const ICEBERG_ERROR_REST: &str = "RESTException"; const ICEBERG_ERROR_UNPROCESSABLE_ENTITY: &str = "UnprocessableEntityException"; const ICEBERG_ERROR_UNSUPPORTED_OPERATION: &str = "UnsupportedOperationException"; +const ICEBERG_VIEW_FORMAT_VERSION: i64 = 1; const REST_PAGE_TOKEN_VERSION: u8 = 1; const REST_PAGE_TOKEN_MAX_LENGTH: usize = 16 * 1024; const REST_DEFAULT_PAGE_SIZE: usize = 1000; @@ -159,52 +164,12 @@ const TABLE_CATALOG_ENDPOINTS: &[&str] = &[ "GET /v1/{prefix}/namespaces/{namespace}/tables/{table}/credentials", "POST /v1/{prefix}/namespaces/{namespace}/tables/{table}", "DELETE /v1/{prefix}/namespaces/{namespace}/tables/{table}", - "PUT /buckets/{warehouse}", - "GET /buckets/{warehouse}", - "GET /{warehouse}/catalog/migration", - "POST /{warehouse}/catalog/migration", - "DELETE /{warehouse}/catalog/migration", - "GET /{warehouse}/namespaces", - "POST /{warehouse}/namespaces", - "GET /{warehouse}/namespaces/{namespace}", - "HEAD /{warehouse}/namespaces/{namespace}", - "DELETE /{warehouse}/namespaces/{namespace}", - "GET /{warehouse}/namespaces/{namespace}/tables", - "POST /{warehouse}/namespaces/{namespace}/tables", - "POST /{warehouse}/namespaces/{namespace}/register", - "GET /{warehouse}/namespaces/{namespace}/views", - "POST /{warehouse}/namespaces/{namespace}/views", - "GET /{warehouse}/namespaces/{namespace}/tables/{table}", - "HEAD /{warehouse}/namespaces/{namespace}/tables/{table}", - "GET /{warehouse}/namespaces/{namespace}/tables/{table}/credentials", - "POST /{warehouse}/namespaces/{namespace}/tables/{table}", - "DELETE /{warehouse}/namespaces/{namespace}/tables/{table}", - "GET /{warehouse}/namespaces/{namespace}/views/{view}", - "HEAD /{warehouse}/namespaces/{namespace}/views/{view}", - "POST /{warehouse}/namespaces/{namespace}/views/{view}", - "DELETE /{warehouse}/namespaces/{namespace}/views/{view}", - "GET /{warehouse}/namespaces/{namespace}/tables/{table}/refs", - "PUT /{warehouse}/namespaces/{namespace}/tables/{table}/refs/{ref}", - "DELETE /{warehouse}/namespaces/{namespace}/tables/{table}/refs/{ref}", - "POST /{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/metadata", - "GET /{warehouse}/namespaces/{namespace}/tables/{table}/metadata-location", - "PUT /{warehouse}/namespaces/{namespace}/tables/{table}/metadata-location", - "GET /{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/config", - "PUT /{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/config", - "GET /{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/jobs/{job}", - "GET /{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/scheduler", - "POST /{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/scheduler/run", - "POST /{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/worker/run", - "POST /{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/jobs/{job}/heartbeat", - "POST /{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/jobs/{job}/quarantine", - "GET /{warehouse}/namespaces/{namespace}/tables/{table}/catalog/export", - "POST /{warehouse}/namespaces/{namespace}/tables/{table}/catalog/import", - "GET /{warehouse}/namespaces/{namespace}/tables/{table}/catalog/external", - "PUT /{warehouse}/namespaces/{namespace}/tables/{table}/catalog/external", - "POST /{warehouse}/namespaces/{namespace}/tables/{table}/catalog/external/sync", - "GET /{warehouse}/namespaces/{namespace}/tables/{table}/catalog/diagnostics", - "POST /{warehouse}/namespaces/{namespace}/tables/{table}/catalog/recovery", - "POST /{warehouse}/namespaces/{namespace}/tables/{table}/catalog/rollback", + "GET /v1/{prefix}/namespaces/{namespace}/views", + "POST /v1/{prefix}/namespaces/{namespace}/views", + "GET /v1/{prefix}/namespaces/{namespace}/views/{view}", + "HEAD /v1/{prefix}/namespaces/{namespace}/views/{view}", + "POST /v1/{prefix}/namespaces/{namespace}/views/{view}", + "DELETE /v1/{prefix}/namespaces/{namespace}/views/{view}", ]; const TABLE_CATALOG_DURABLE_STRONG_ENDPOINTS: &[&str] = &[ "POST /v1/{prefix}/namespaces/{namespace}/properties", @@ -341,7 +306,7 @@ struct CreateViewRequest { #[serde(deny_unknown_fields)] struct RestCommitTableRequest { #[serde(default, rename = "identifier")] - _identifier: Option, + identifier: Option, #[serde(default, rename = "commit-id")] commit_id: Option, #[serde(default, rename = "idempotency-key")] @@ -365,8 +330,10 @@ struct RestCommitTableRequest { #[derive(Debug, Deserialize)] #[serde(deny_unknown_fields)] struct RestCommitViewRequest { + #[serde(default, rename = "identifier")] + identifier: Option, #[serde(default, rename = "commit-id")] - commit_id: Option, + _commit_id: Option, #[serde(default, rename = "expected-version-token")] expected_version_token: Option, #[serde(default, rename = "expected-metadata-location")] @@ -705,6 +672,12 @@ enum RestPagination { }, } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum RestTableSnapshotSelection { + All, + Refs, +} + impl RestPagination { fn page_request(&self) -> Option<(Option<&str>, NonZeroUsize)> { match self { @@ -984,6 +957,18 @@ fn build_json_response(status: StatusCode, body: &T) -> S3Result(status: StatusCode, body: &T) -> S3Result> { + let mut response = build_json_response(status, body)?; + response + .headers + .insert(http::header::CACHE_CONTROL, HeaderValue::from_static("no-store, private")); + response + .headers + .insert(http::header::PRAGMA, HeaderValue::from_static("no-cache")); + response.headers.insert(http::header::EXPIRES, HeaderValue::from_static("0")); + Ok(response) +} + fn empty_response(status: StatusCode) -> S3Response<(StatusCode, Body)> { S3Response::new((status, Body::default())) } @@ -1267,7 +1252,7 @@ struct TableCommitPublicationState { bucket_fence: Option, table_fence: Option<(String, String, String)>, observed_objects: BTreeMap<(String, String), TableCommitObservedObject>, - guards: Vec>, + guards: Vec, } #[derive(Clone)] @@ -1730,7 +1715,7 @@ where &self, bucket: &str, object: &str, - ) -> crate::table_catalog::TableCatalogStoreResult> { + ) -> crate::table_catalog::TableCatalogStoreResult { self.backend.acquire_read_lock(bucket, object).await } @@ -1738,7 +1723,7 @@ where &self, bucket: &str, object: &str, - ) -> crate::table_catalog::TableCatalogStoreResult> { + ) -> crate::table_catalog::TableCatalogStoreResult { self.backend.acquire_write_lock(bucket, object).await } @@ -1750,7 +1735,8 @@ where } fn table_bucket_commit_publication_is_held(&self, table_bucket: &str) -> bool { - self.publication.lock().bucket_fence.as_deref() == Some(table_bucket) + let publication = self.publication.lock(); + publication.bucket_fence.as_deref() == Some(table_bucket) && publication.guards.iter().all(|guard| !guard.is_lock_lost()) } async fn prepare_table_commit_publication( @@ -1763,11 +1749,12 @@ where } fn table_commit_publication_is_held(&self, table_bucket: &str, namespace: &str, table: &str) -> bool { - self.publication - .lock() + let publication = self.publication.lock(); + publication .table_fence .as_ref() .is_some_and(|held| held.0 == table_bucket && held.1 == namespace && held.2 == table) + && publication.guards.iter().all(|guard| !guard.is_lock_lost()) } fn complete_table_commit_publication(&self) { @@ -1775,20 +1762,72 @@ where } } -async fn read_json_body(mut input: Body) -> S3Result { - let body = input - .store_all_limited(MAX_ADMIN_REQUEST_BODY_SIZE) +async fn read_limited_body(mut input: Body, max_size: usize, timeout: StdDuration, operation: Option<&str>) -> S3Result { + tokio::time::timeout(timeout, input.store_all_limited(max_size)) .await - .map_err(|err| s3_error!(InvalidRequest, "failed to read request body: {}", err))?; + .map_err(|_| { + operation.map_or_else( + || S3Error::from(ApiError::invalid_request("timed out reading request body")), + |operation| S3Error::from(ApiError::invalid_request(format!("timed out reading {operation} request body"))), + ) + })? + .map_err(|err| S3Error::from(ApiError::invalid_request(format!("failed to read request body: {err}")))) +} + +async fn read_json_body(input: Body) -> S3Result { + let body = read_limited_body(input, MAX_ADMIN_REQUEST_BODY_SIZE, TABLE_CATALOG_REQUEST_BODY_TIMEOUT, None).await?; if body.is_empty() { - return Err(s3_error!(InvalidRequest, "request body is required")); + return Err(S3Error::from(ApiError::invalid_request("request body is required"))); } - serde_json::from_slice(&body).map_err(|err| s3_error!(InvalidRequest, "invalid JSON: {}", err)) + serde_json::from_slice(&body).map_err(|err| S3Error::from(ApiError::invalid_request(format!("invalid JSON: {err}")))) +} + +fn validate_rest_commit_request_shape( + value: &serde_json::Value, + require_requirements: bool, + require_updates: bool, +) -> S3Result<()> { + let object = value + .as_object() + .ok_or_else(|| S3Error::from(ApiError::invalid_request("commit request must be a JSON object")))?; + if object.get("new-metadata-location").is_some_and(serde_json::Value::is_string) { + for field in ["requirements", "updates"] { + if object + .get(field) + .and_then(serde_json::Value::as_array) + .is_some_and(|values| !values.is_empty()) + { + return Err(S3Error::from(ApiError::invalid_request(format!( + "legacy metadata pointer commit must not include standard {field}" + )))); + } + } + return Ok(()); + } + if require_requirements && !object.contains_key("requirements") { + return Err(S3Error::from(ApiError::invalid_request("commit request requires requirements"))); + } + if require_updates && !object.contains_key("updates") { + return Err(S3Error::from(ApiError::invalid_request("commit request requires updates"))); + } + Ok(()) +} + +async fn read_rest_commit_table_request(input: Body) -> S3Result { + let value = read_json_body::(input).await?; + validate_rest_commit_request_shape(&value, true, true)?; + serde_json::from_value(value).map_err(|err| S3Error::from(ApiError::invalid_request(format!("invalid JSON: {err}")))) +} + +async fn read_rest_commit_view_request(input: Body) -> S3Result { + let value = read_json_body::(input).await?; + validate_rest_commit_request_shape(&value, false, true)?; + serde_json::from_value(value).map_err(|err| S3Error::from(ApiError::invalid_request(format!("invalid JSON: {err}")))) } async fn read_bounded_json_body( headers: &HeaderMap, - mut input: Body, + input: Body, max_size: usize, timeout: StdDuration, operation: &str, @@ -1803,34 +1842,28 @@ async fn read_bounded_json_body( return Err(S3Error::from(ApiError::invalid_request(format!("{operation} request body is too large")))); } } - let body = tokio::time::timeout(timeout, input.store_all_limited(max_size)) - .await - .map_err(|_| S3Error::from(ApiError::invalid_request(format!("timed out reading {operation} request body"))))? - .map_err(|err| S3Error::from(ApiError::invalid_request(format!("failed to read request body: {err}"))))?; + let body = read_limited_body(input, max_size, timeout, Some(operation)).await?; if body.is_empty() { return Err(S3Error::from(ApiError::invalid_request("request body is required"))); } serde_json::from_slice(&body).map_err(|err| S3Error::from(ApiError::invalid_request(format!("invalid JSON: {err}")))) } -async fn read_json_body_or_default(mut input: Body) -> S3Result +async fn read_json_body_or_default(input: Body) -> S3Result where T: Default + DeserializeOwned, { - let body = input - .store_all_limited(MAX_ADMIN_REQUEST_BODY_SIZE) - .await - .map_err(|err| s3_error!(InvalidRequest, "failed to read request body: {}", err))?; + let body = read_limited_body(input, MAX_ADMIN_REQUEST_BODY_SIZE, TABLE_CATALOG_REQUEST_BODY_TIMEOUT, None).await?; if body.is_empty() { return Ok(T::default()); } - serde_json::from_slice(&body).map_err(|err| s3_error!(InvalidRequest, "invalid JSON: {}", err)) + serde_json::from_slice(&body).map_err(|err| S3Error::from(ApiError::invalid_request(format!("invalid JSON: {err}")))) } fn warehouse_from_params(params: &Params<'_, '_>) -> S3Result { let warehouse = params.get("warehouse").unwrap_or(""); if warehouse.is_empty() { - return Err(s3_error!(InvalidRequest, "warehouse is required")); + return Err(S3Error::from(ApiError::invalid_request("warehouse is required"))); } Ok(warehouse.to_string()) } @@ -1855,6 +1888,95 @@ fn warehouse_from_config_query(uri: &http::Uri) -> S3Result> { Ok(warehouse) } +fn rest_purge_requested_from_query(uri: &http::Uri) -> S3Result { + let mut purge_requested = None; + if let Some(query) = uri.query() { + for (key, value) in url::form_urlencoded::parse(query.as_bytes()) { + if key != "purgeRequested" { + continue; + } + if purge_requested.is_some() { + return Err(iceberg_rest_error( + ICEBERG_ERROR_BAD_REQUEST, + StatusCode::BAD_REQUEST, + "purgeRequested query parameter must not be repeated", + )); + } + let value = if value.eq_ignore_ascii_case("true") { + true + } else if value.eq_ignore_ascii_case("false") { + false + } else { + return Err(iceberg_rest_error( + ICEBERG_ERROR_BAD_REQUEST, + StatusCode::BAD_REQUEST, + "purgeRequested query parameter must be true or false", + )); + }; + purge_requested = Some(value); + } + } + Ok(purge_requested.unwrap_or(false)) +} + +fn rest_table_snapshot_selection_from_query(uri: &http::Uri) -> S3Result { + let mut selection = None; + if let Some(query) = uri.query() { + for (key, value) in url::form_urlencoded::parse(query.as_bytes()) { + if key != "snapshots" { + continue; + } + if selection.is_some() { + return Err(iceberg_rest_error( + ICEBERG_ERROR_BAD_REQUEST, + StatusCode::BAD_REQUEST, + "snapshots query parameter must not be repeated", + )); + } + selection = Some(match value.as_ref() { + "all" => RestTableSnapshotSelection::All, + "refs" => RestTableSnapshotSelection::Refs, + _ => { + return Err(iceberg_rest_error( + ICEBERG_ERROR_BAD_REQUEST, + StatusCode::BAD_REQUEST, + "snapshots query parameter must be all or refs", + )); + } + }); + } + } + Ok(selection.unwrap_or(RestTableSnapshotSelection::All)) +} + +fn apply_rest_table_snapshot_selection(metadata: &mut serde_json::Value, selection: RestTableSnapshotSelection) { + if selection == RestTableSnapshotSelection::All { + return; + } + let mut referenced_snapshot_ids = metadata + .get("refs") + .and_then(serde_json::Value::as_object) + .into_iter() + .flat_map(|refs| refs.values()) + .filter_map(|reference| reference.get("snapshot-id").and_then(serde_json::Value::as_i64)) + .collect::>(); + if let Some(current_snapshot_id) = metadata + .get("current-snapshot-id") + .and_then(serde_json::Value::as_i64) + .filter(|snapshot_id| *snapshot_id != -1) + { + referenced_snapshot_ids.insert(current_snapshot_id); + } + if let Some(snapshots) = metadata.get_mut("snapshots").and_then(serde_json::Value::as_array_mut) { + snapshots.retain(|snapshot| { + snapshot + .get("snapshot-id") + .and_then(serde_json::Value::as_i64) + .is_some_and(|snapshot_id| referenced_snapshot_ids.contains(&snapshot_id)) + }); + } +} + fn rest_pagination_from_query(uri: &http::Uri, context: RestPageContext<'_>) -> S3Result { let mut page_token = None; let mut page_token_seen = false; @@ -2242,6 +2364,23 @@ fn namespace_segments(namespace: &crate::table_catalog::Namespace) -> Vec, + namespace: &crate::table_catalog::Namespace, + name: &str, +) -> S3Result<()> { + if let Some(identifier) = identifier + && (identifier.namespace != namespace_segments(namespace) || identifier.name != name) + { + return Err(iceberg_rest_error( + ICEBERG_ERROR_BAD_REQUEST, + StatusCode::BAD_REQUEST, + "request identifier must match the resource URL", + )); + } + Ok(()) +} + fn namespace_from_segments(segments: &[String]) -> S3Result { crate::table_catalog::Namespace::from_segments(segments.to_vec()) .map_err(|err| s3_error!(InvalidRequest, "invalid namespace: {}", err)) @@ -2487,13 +2626,15 @@ fn load_table_response_from_entry(entry: crate::table_catalog::TableEntry, metad fn load_view_response_from_entry(entry: crate::table_catalog::ViewEntry, metadata: serde_json::Value) -> RestLoadViewResponse { let mut config = BTreeMap::new(); let warehouse_location = entry.warehouse_location.clone(); + let metadata_location = table_metadata_location_for_client(&entry.table_bucket, &entry.metadata_location); + let metadata = table_metadata_for_client(&entry.table_bucket, metadata); config.insert("warehouse-location".to_string(), warehouse_location.clone()); config.insert(CREDENTIAL_SCOPE_CONFIG_KEY.to_string(), CREDENTIAL_SCOPE_TABLE_PREFIX.to_string()); config.insert(CREDENTIAL_SCOPE_PREFIX_CONFIG_KEY.to_string(), warehouse_location); config.insert(CREDENTIAL_MODE_CONFIG_KEY.to_string(), CREDENTIAL_MODE_CLIENT_PROVIDED.to_string()); RestLoadViewResponse { - metadata_location: entry.metadata_location, + metadata_location, metadata, config, } @@ -2662,13 +2803,59 @@ fn validate_metadata_view_location_in_bucket(bucket: &str, metadata: &serde_json validate_view_location_in_bucket(bucket, location) } +fn validate_persisted_table_metadata( + entry: &crate::table_catalog::TableEntry, + metadata: &serde_json::Value, + require_current_warehouse: bool, +) -> S3Result<()> { + crate::table_catalog::validate_supported_table_metadata(metadata).map_err(|_| persisted_metadata_error("table"))?; + validate_metadata_table_location_in_bucket(&entry.table_bucket, metadata).map_err(|_| persisted_metadata_error("table"))?; + let metadata_uuid = metadata_table_uuid(metadata).map_err(|_| persisted_metadata_error("table"))?; + let metadata_location = metadata_table_location(metadata).map_err(|_| persisted_metadata_error("table"))?; + let format_version = metadata_format_version(metadata).map_err(|_| persisted_metadata_error("table"))?; + if metadata_uuid != entry.table_uuid + || (require_current_warehouse && format_version < entry.format_version) + || (require_current_warehouse && metadata_location != entry.warehouse_location) + { + return Err(persisted_metadata_error("table")); + } + Ok(()) +} + +fn validate_persisted_table_metadata_location(entry: &crate::table_catalog::TableEntry, metadata_location: &str) -> S3Result<()> { + if !crate::table_catalog::is_valid_table_metadata_location_for_entry(entry, metadata_location) { + return Err(persisted_metadata_error("table")); + } + Ok(()) +} + +fn validate_persisted_view_metadata(entry: &crate::table_catalog::ViewEntry, metadata: &serde_json::Value) -> S3Result<()> { + validate_persisted_view_metadata_identity(entry, metadata)?; + crate::table_catalog::validate_supported_view_metadata(metadata).map_err(|_| persisted_metadata_error("view")) +} + +fn validate_persisted_view_metadata_identity( + entry: &crate::table_catalog::ViewEntry, + metadata: &serde_json::Value, +) -> S3Result<()> { + validate_metadata_view_location_in_bucket(&entry.table_bucket, metadata).map_err(|_| persisted_metadata_error("view"))?; + let metadata_uuid = metadata_view_uuid(metadata).map_err(|_| persisted_metadata_error("view"))?; + let metadata_location = metadata_table_location(metadata).map_err(|_| persisted_metadata_error("view"))?; + let format_version = metadata_format_version(metadata).map_err(|_| persisted_metadata_error("view"))?; + if metadata_uuid != entry.view_uuid || metadata_location != entry.warehouse_location || format_version != entry.format_version + { + return Err(persisted_metadata_error("view")); + } + Ok(()) +} + fn validate_metadata_matches_current_metadata( current_metadata: &serde_json::Value, target_metadata: &serde_json::Value, ) -> S3Result<()> { - crate::table_catalog::validate_supported_table_metadata(current_metadata).map_err(catalog_store_error)?; crate::table_catalog::validate_supported_table_metadata(target_metadata).map_err(catalog_store_error)?; - validate_metadata_identity_matches_current_metadata(current_metadata, target_metadata) + validate_metadata_identity_matches_current_metadata(current_metadata, target_metadata)?; + crate::table_catalog::validate_table_metadata_transition(current_metadata, target_metadata).map_err(catalog_store_error) } fn validate_metadata_identity_matches_current_metadata( @@ -2676,15 +2863,20 @@ fn validate_metadata_identity_matches_current_metadata( target_metadata: &serde_json::Value, ) -> S3Result<()> { let expected_table_uuid = metadata_table_uuid(current_metadata)?; - metadata_format_version(current_metadata)?; + let expected_format_version = metadata_format_version(current_metadata)?; let target_table_uuid = metadata_table_uuid(target_metadata)?; - metadata_format_version(target_metadata)?; + let target_format_version = metadata_format_version(target_metadata)?; if target_table_uuid != expected_table_uuid { return Err(s3_error!( InvalidRequest, "table metadata table-uuid does not match current table metadata" )); } + if target_format_version < expected_format_version { + return Err(S3Error::from(ApiError::invalid_request( + "table metadata format-version cannot be downgraded", + ))); + } Ok(()) } @@ -2727,7 +2919,11 @@ fn table_entry_from_register_request( request: RegisterTableRequest, ) -> S3Result { if request.overwrite { - return Err(s3_error!(NotImplemented, "register table overwrite is not supported")); + return Err(iceberg_rest_error( + ICEBERG_ERROR_UNSUPPORTED_OPERATION, + StatusCode::NOT_ACCEPTABLE, + "register table overwrite is not supported", + )); } let table = crate::table_catalog::IdentifierSegment::parse(request.name) .map_err(|err| s3_error!(InvalidRequest, "invalid table name: {}", err))?; @@ -2806,7 +3002,11 @@ fn table_entry_from_create_table_request( mut properties, } = request; if stage_create { - return Err(s3_error!(NotImplemented, "stage-create is not supported")); + return Err(iceberg_rest_error( + ICEBERG_ERROR_UNSUPPORTED_OPERATION, + StatusCode::NOT_ACCEPTABLE, + "stage-create is not supported", + )); } let table = crate::table_catalog::IdentifierSegment::parse(name) @@ -2899,13 +3099,7 @@ fn initial_table_metadata_json( let schema_object = schema .as_object_mut() .ok_or_else(|| s3_error!(InvalidRequest, "schema must be a JSON object"))?; - schema_object - .entry("schema-id".to_string()) - .or_insert_with(|| serde_json::Value::from(0)); - let schema_id = schema_object - .get("schema-id") - .and_then(serde_json::Value::as_i64) - .ok_or_else(|| s3_error!(InvalidRequest, "schema-id must be an integer"))?; + schema_object.insert("schema-id".to_string(), serde_json::Value::from(0)); let last_column_id = max_field_id(&schema); let mut spec = partition_spec.unwrap_or_else(|| { @@ -2917,17 +3111,11 @@ fn initial_table_metadata_json( let spec_object = spec .as_object_mut() .ok_or_else(|| s3_error!(InvalidRequest, "partition-spec must be a JSON object"))?; - spec_object - .entry("spec-id".to_string()) - .or_insert_with(|| serde_json::Value::from(0)); + spec_object.insert("spec-id".to_string(), serde_json::Value::from(0)); spec_object .entry("fields".to_string()) .or_insert_with(|| serde_json::Value::Array(Vec::new())); - let spec_id = spec_object - .get("spec-id") - .and_then(serde_json::Value::as_i64) - .ok_or_else(|| s3_error!(InvalidRequest, "partition spec-id must be an integer"))?; - let last_partition_id = max_partition_field_id(&spec); + let last_partition_id = assign_partition_field_ids(&mut spec, 999, &BTreeMap::new())?; let mut sort_order = write_order.unwrap_or_else(|| { serde_json::json!({ @@ -2938,17 +3126,19 @@ fn initial_table_metadata_json( let sort_order_object = sort_order .as_object_mut() .ok_or_else(|| s3_error!(InvalidRequest, "write-order must be a JSON object"))?; - sort_order_object - .entry("order-id".to_string()) - .or_insert_with(|| serde_json::Value::from(0)); - sort_order_object + let sort_order_fields = sort_order_object .entry("fields".to_string()) .or_insert_with(|| serde_json::Value::Array(Vec::new())); - let sort_order_id = sort_order_object - .get("order-id") - .and_then(serde_json::Value::as_i64) - .ok_or_else(|| s3_error!(InvalidRequest, "sort order-id must be an integer"))?; - + let sort_order_id = if sort_order_fields + .as_array() + .ok_or_else(|| S3Error::from(ApiError::invalid_request("write-order fields must be an array")))? + .is_empty() + { + 0 + } else { + 1 + }; + sort_order_object.insert("order-id".to_string(), serde_json::Value::from(sort_order_id)); let mut metadata = serde_json::json!({ "format-version": entry.format_version, "table-uuid": entry.table_uuid, @@ -2956,9 +3146,9 @@ fn initial_table_metadata_json( "last-updated-ms": current_time_millis(), "last-column-id": last_column_id, "schemas": [schema], - "current-schema-id": schema_id, + "current-schema-id": 0, "partition-specs": [spec], - "default-spec-id": spec_id, + "default-spec-id": 0, "last-partition-id": last_partition_id, "sort-orders": [sort_order], "default-sort-order-id": sort_order_id, @@ -2985,26 +3175,12 @@ fn initial_view_metadata_json( let schema_object = schema .as_object_mut() .ok_or_else(|| s3_error!(InvalidRequest, "schema must be a JSON object"))?; - schema_object - .entry("schema-id".to_string()) - .or_insert_with(|| serde_json::Value::from(0)); - let schema_id = schema_object - .get("schema-id") - .and_then(serde_json::Value::as_i64) - .ok_or_else(|| s3_error!(InvalidRequest, "schema-id must be an integer"))?; + schema_object.insert("schema-id".to_string(), serde_json::Value::from(0)); let view_version_object = view_version .as_object_mut() .ok_or_else(|| s3_error!(InvalidRequest, "view-version must be a JSON object"))?; - view_version_object - .entry("version-id".to_string()) - .or_insert_with(|| serde_json::Value::from(1)); - view_version_object - .entry("schema-id".to_string()) - .or_insert_with(|| serde_json::Value::from(schema_id)); - view_version_object - .entry("timestamp-ms".to_string()) - .or_insert_with(|| serde_json::Value::from(current_time_millis())); + view_version_object.insert("schema-id".to_string(), serde_json::Value::from(0)); let version_id = view_version_object .get("version-id") .and_then(serde_json::Value::as_i64) @@ -3012,13 +3188,12 @@ fn initial_view_metadata_json( let timestamp_ms = view_version_object .get("timestamp-ms") .and_then(serde_json::Value::as_i64) - .unwrap_or_else(current_time_millis); + .ok_or_else(|| S3Error::from(ApiError::invalid_request("view-version timestamp-ms must be an integer")))?; - Ok(serde_json::json!({ + let metadata = serde_json::json!({ "format-version": entry.format_version, "view-uuid": entry.view_uuid, "location": entry.warehouse_location, - "last-updated-ms": current_time_millis(), "current-version-id": version_id, "schemas": [schema], "versions": [view_version], @@ -3026,9 +3201,10 @@ fn initial_view_metadata_json( "timestamp-ms": timestamp_ms, "version-id": version_id }], - "metadata-log": [], "properties": properties - })) + }); + validate_supported_view_metadata(&metadata)?; + Ok(metadata) } fn current_time_millis() -> i64 { @@ -3047,8 +3223,10 @@ fn max_field_id(value: &serde_json::Value) -> i64 { fn collect_max_field_id(value: &serde_json::Value, max_id: &mut i64) { match value { serde_json::Value::Object(object) => { - if let Some(id) = object.get("id").and_then(serde_json::Value::as_i64) { - *max_id = (*max_id).max(id); + for field in ["id", "element-id", "key-id", "value-id"] { + if let Some(id) = object.get(field).and_then(serde_json::Value::as_i64) { + *max_id = (*max_id).max(id); + } } for child in object.values() { collect_max_field_id(child, max_id); @@ -3063,19 +3241,6 @@ fn collect_max_field_id(value: &serde_json::Value, max_id: &mut i64) { } } -fn max_partition_field_id(value: &serde_json::Value) -> i64 { - let mut max_id = 999; - let Some(fields) = value.get("fields").and_then(serde_json::Value::as_array) else { - return max_id; - }; - for field in fields { - if let Some(field_id) = field.get("field-id").and_then(serde_json::Value::as_i64) { - max_id = max_id.max(field_id); - } - } - max_id -} - fn standard_commit_ids(commit_id: Option) -> (String, String) { match commit_id { Some(commit_id) => match Uuid::parse_str(&commit_id) { @@ -3163,7 +3328,7 @@ fn validate_table_commit_requirements(metadata: &serde_json::Value, requirements .ok_or_else(|| s3_error!(InvalidRequest, "commit requirement type is required"))?; match requirement_type { "assert-create" => { - return Err(s3_error!(PreconditionFailed, "commit requirement failed: table already exists")); + return Err(commit_requirement_failed("commit requirement failed: table already exists")); } "assert-table-uuid" => { let expected = requirement @@ -3175,7 +3340,7 @@ fn validate_table_commit_requirements(metadata: &serde_json::Value, requirements .and_then(serde_json::Value::as_str) .ok_or_else(|| s3_error!(InvalidRequest, "current table metadata is missing table-uuid"))?; if actual != expected { - return Err(s3_error!(PreconditionFailed, "commit requirement failed: table uuid changed")); + return Err(commit_requirement_failed("commit requirement failed: table uuid changed")); } } "assert-current-schema-id" => { @@ -3206,8 +3371,11 @@ fn validate_table_commit_requirements(metadata: &serde_json::Value, requirements )?; } "assert-ref-snapshot-id" => validate_ref_snapshot_requirement(metadata, requirement)?, - "assert-current-snapshot-id" => validate_current_snapshot_requirement(metadata, requirement)?, - _ => return Err(s3_error!(NotImplemented, "unsupported commit requirement: {requirement_type}")), + _ => { + return Err(S3Error::from(ApiError::invalid_request(format!( + "unsupported commit requirement: {requirement_type}" + )))); + } } } Ok(()) @@ -3238,7 +3406,7 @@ fn validate_i64_requirement_with_metadata_key( .and_then(serde_json::Value::as_i64) .ok_or_else(|| s3_error!(InvalidRequest, "current table metadata is missing {metadata_key}"))?; if actual != expected { - return Err(s3_error!(PreconditionFailed, "commit requirement failed: {label} changed")); + return Err(commit_requirement_failed(format!("commit requirement failed: {label} changed"))); } Ok(()) } @@ -3255,7 +3423,7 @@ fn validate_ref_snapshot_requirement(metadata: &serde_json::Value, requirement: .and_then(serde_json::Value::as_i64); if requirement.get("snapshot-id").is_some_and(serde_json::Value::is_null) { if actual.is_some() { - return Err(s3_error!(PreconditionFailed, "commit requirement failed: snapshot ref exists")); + return Err(commit_requirement_failed("commit requirement failed: snapshot ref exists")); } return Ok(()); } @@ -3264,25 +3432,7 @@ fn validate_ref_snapshot_requirement(metadata: &serde_json::Value, requirement: .and_then(serde_json::Value::as_i64) .ok_or_else(|| s3_error!(InvalidRequest, "assert-ref-snapshot-id requires snapshot-id"))?; if actual != Some(expected) { - return Err(s3_error!(PreconditionFailed, "commit requirement failed: snapshot ref changed")); - } - Ok(()) -} - -fn validate_current_snapshot_requirement(metadata: &serde_json::Value, requirement: &serde_json::Value) -> S3Result<()> { - let actual = metadata.get("current-snapshot-id").and_then(serde_json::Value::as_i64); - if requirement.get("snapshot-id").is_some_and(serde_json::Value::is_null) { - if actual.is_some() { - return Err(s3_error!(PreconditionFailed, "commit requirement failed: current snapshot exists")); - } - return Ok(()); - } - let expected = requirement - .get("snapshot-id") - .and_then(serde_json::Value::as_i64) - .ok_or_else(|| s3_error!(InvalidRequest, "assert-current-snapshot-id requires snapshot-id"))?; - if actual != Some(expected) { - return Err(s3_error!(PreconditionFailed, "commit requirement failed: current snapshot changed")); + return Err(commit_requirement_failed("commit requirement failed: snapshot ref changed")); } Ok(()) } @@ -3304,6 +3454,16 @@ fn apply_table_commit_updates_at( if !metadata.is_object() { return Err(s3_error!(InvalidRequest, "current table metadata must be a JSON object")); } + if metadata.get("format-version").is_some() { + crate::table_catalog::synchronize_table_metadata_version_fields(&mut metadata).map_err(catalog_store_error)?; + } + let mut next_schema_id = next_catalog_id_for_updates(&metadata, updates, "add-schema", "schemas", "schema-id")?; + let mut next_spec_id = next_catalog_id_for_updates(&metadata, updates, "add-spec", "partition-specs", "spec-id")?; + let mut next_sort_order_id = next_catalog_id_for_updates(&metadata, updates, "add-sort-order", "sort-orders", "order-id")?; + let mut last_added_schema_id = None; + let mut last_added_spec_id = None; + let mut last_added_sort_order_id = None; + let mut added_snapshot_ids = BTreeSet::new(); for update in updates { let action = update @@ -3311,25 +3471,81 @@ fn apply_table_commit_updates_at( .and_then(serde_json::Value::as_str) .ok_or_else(|| s3_error!(InvalidRequest, "table update action is required"))?; match action { - "assign-uuid" => apply_assign_uuid_update(&mut metadata, update)?, + "assign-uuid" => apply_assign_uuid_update(&mut metadata, update, "table-uuid", "table")?, "upgrade-format-version" => apply_upgrade_format_version_update(&mut metadata, update)?, - "add-schema" => apply_add_schema_update(&mut metadata, update)?, - "set-current-schema" => apply_set_current_schema_update(&mut metadata, update)?, - "add-spec" => apply_add_spec_update(&mut metadata, update)?, - "set-default-spec" => apply_set_default_spec_update(&mut metadata, update)?, - "add-sort-order" => apply_add_sort_order_update(&mut metadata, update)?, - "set-default-sort-order" => apply_set_default_sort_order_update(&mut metadata, update)?, - "add-snapshot" => apply_add_snapshot_update(&mut metadata, update)?, - "set-snapshot-ref" => apply_set_snapshot_ref_update(&mut metadata, update)?, + "add-schema" => { + let schema_id = take_catalog_assigned_id(&mut next_schema_id, "schema-id")?; + apply_add_table_schema_update(&mut metadata, update, schema_id)?; + last_added_schema_id = Some(schema_id); + } + "set-current-schema" => { + apply_set_current_schema_update(&mut metadata, update, last_added_schema_id)?; + } + "add-spec" => { + let spec_id = take_catalog_assigned_id(&mut next_spec_id, "spec-id")?; + apply_add_spec_update(&mut metadata, update, spec_id)?; + last_added_spec_id = Some(spec_id); + } + "set-default-spec" => { + apply_set_default_spec_update(&mut metadata, update, last_added_spec_id)?; + } + "add-sort-order" => { + let sort_order_id = take_catalog_assigned_id(&mut next_sort_order_id, "sort order-id")?; + last_added_sort_order_id = Some(apply_add_sort_order_update(&mut metadata, update, sort_order_id)?); + } + "set-default-sort-order" => { + apply_set_default_sort_order_update(&mut metadata, update, last_added_sort_order_id)?; + } + "add-snapshot" => { + added_snapshot_ids.insert(apply_add_snapshot_update(&mut metadata, update)?); + } + "set-snapshot-ref" => { + apply_set_snapshot_ref_update(&mut metadata, update, &added_snapshot_ids, commit_timestamp_ms)?; + } "remove-snapshots" => apply_remove_snapshots_update(&mut metadata, update)?, "remove-snapshot-ref" => apply_remove_snapshot_ref_update(&mut metadata, update)?, "set-location" => apply_set_location_update(&mut metadata, update)?, "set-properties" => apply_set_properties_update(&mut metadata, update)?, "remove-properties" => apply_remove_properties_update(&mut metadata, update)?, - _ => return Err(s3_error!(NotImplemented, "unsupported table update: {action}")), + "set-statistics" => apply_set_snapshot_file_update( + &mut metadata, + update, + "statistics", + "statistics", + crate::table_catalog::IcebergStatisticsFileKind::Table, + )?, + "remove-statistics" => apply_remove_snapshot_file_update(&mut metadata, update, "statistics")?, + "set-partition-statistics" => { + apply_set_snapshot_file_update( + &mut metadata, + update, + "partition-statistics", + "partition-statistics", + crate::table_catalog::IcebergStatisticsFileKind::Partition, + )?; + } + "remove-partition-statistics" => { + apply_remove_snapshot_file_update(&mut metadata, update, "partition-statistics")?; + } + "remove-partition-specs" => { + apply_remove_metadata_ids_update(&mut metadata, update, "partition-specs", "spec-id", "spec-ids")?; + } + "remove-schemas" => { + apply_remove_metadata_ids_update(&mut metadata, update, "schemas", "schema-id", "schema-ids")?; + } + "add-encryption-key" | "remove-encryption-key" => { + return Err(iceberg_rest_error( + ICEBERG_ERROR_UNSUPPORTED_OPERATION, + StatusCode::NOT_ACCEPTABLE, + "table encryption keys require Iceberg format-version 3", + )); + } + _ => return Err(S3Error::from(ApiError::invalid_request(format!("unsupported table update: {action}")))), } } + prune_intermediate_snapshot_log_entries(&mut metadata, &added_snapshot_ids)?; + if metadata.get("format-version").is_some() { crate::table_catalog::synchronize_table_metadata_version_fields(&mut metadata).map_err(catalog_store_error)?; } @@ -3338,6 +3554,27 @@ fn apply_table_commit_updates_at( Ok(metadata) } +fn prune_intermediate_snapshot_log_entries(metadata: &mut serde_json::Value, added_snapshot_ids: &BTreeSet) -> S3Result<()> { + if added_snapshot_ids.is_empty() { + return Ok(()); + } + let current_snapshot_id = metadata.get("current-snapshot-id").and_then(serde_json::Value::as_i64); + let snapshot_log = ensure_array_field(metadata, "snapshot-log")?; + for entry in snapshot_log.iter() { + entry + .get("snapshot-id") + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| S3Error::from(ApiError::invalid_request("snapshot-log snapshot-id must be an integer")))?; + } + snapshot_log.retain(|entry| { + entry + .get("snapshot-id") + .and_then(serde_json::Value::as_i64) + .is_none_or(|snapshot_id| !added_snapshot_ids.contains(&snapshot_id) || Some(snapshot_id) == current_snapshot_id) + }); + Ok(()) +} + fn validate_view_commit_requirements(metadata: &serde_json::Value, requirements: &[serde_json::Value]) -> S3Result<()> { for requirement in requirements { let requirement_type = requirement @@ -3355,32 +3592,35 @@ fn validate_view_commit_requirements(metadata: &serde_json::Value, requirements: .and_then(serde_json::Value::as_str) .ok_or_else(|| s3_error!(InvalidRequest, "current view metadata is missing view-uuid"))?; if actual != expected { - return Err(s3_error!(PreconditionFailed, "commit requirement failed: view uuid changed")); + return Err(commit_requirement_failed("commit requirement failed: view uuid changed")); } } - "assert-current-view-version-id" => { - validate_i64_requirement_with_metadata_key( - metadata, - requirement, - "current-view-version-id", - "current-version-id", - "current view version id", - )?; + _ => { + return Err(S3Error::from(ApiError::invalid_request(format!( + "unsupported view commit requirement: {requirement_type}" + )))); } - _ => return Err(s3_error!(NotImplemented, "unsupported view commit requirement: {requirement_type}")), } } Ok(()) } -fn apply_view_commit_updates( +fn validate_supported_view_metadata(metadata: &serde_json::Value) -> S3Result<()> { + crate::table_catalog::validate_supported_view_metadata(metadata).map_err(catalog_store_error) +} + +fn apply_view_commit_updates_at( mut metadata: serde_json::Value, updates: &[serde_json::Value], - previous_metadata_location: &str, + commit_timestamp_ms: i64, ) -> S3Result { if !metadata.is_object() { return Err(s3_error!(InvalidRequest, "current view metadata must be a JSON object")); } + let mut next_schema_id = next_catalog_id_for_updates(&metadata, updates, "add-schema", "schemas", "schema-id")?; + let mut last_added_schema_id = None; + let mut last_added_view_version_id = None; + let mut added_view_version_timestamps = BTreeMap::new(); for update in updates { let action = update @@ -3388,40 +3628,138 @@ fn apply_view_commit_updates( .and_then(serde_json::Value::as_str) .ok_or_else(|| s3_error!(InvalidRequest, "view update action is required"))?; match action { - "assign-uuid" => apply_assign_uuid_update(&mut metadata, update)?, - "add-schema" => apply_add_schema_update(&mut metadata, update)?, - "set-current-schema" => apply_set_current_schema_update(&mut metadata, update)?, - "add-view-version" => apply_add_view_version_update(&mut metadata, update)?, - "set-current-view-version" => apply_set_current_view_version_update(&mut metadata, update)?, + "assign-uuid" => apply_assign_uuid_update(&mut metadata, update, "view-uuid", "view")?, + "upgrade-format-version" => apply_upgrade_view_format_version_update(update)?, + "add-schema" => { + let schema_id = take_catalog_assigned_id(&mut next_schema_id, "schema-id")?; + apply_add_view_schema_update(&mut metadata, update, schema_id)?; + last_added_schema_id = Some(schema_id); + } + "add-view-version" => { + let (version_id, timestamp_ms) = apply_add_view_version_update(&mut metadata, update, last_added_schema_id)?; + last_added_view_version_id = Some(version_id); + added_view_version_timestamps.insert(version_id, timestamp_ms); + } + "set-current-view-version" => { + apply_set_current_view_version_update( + &mut metadata, + update, + last_added_view_version_id, + &added_view_version_timestamps, + commit_timestamp_ms, + )?; + } "set-location" => apply_set_location_update(&mut metadata, update)?, "set-properties" => apply_set_properties_update(&mut metadata, update)?, "remove-properties" => apply_remove_properties_update(&mut metadata, update)?, - _ => return Err(s3_error!(NotImplemented, "unsupported view update: {action}")), + _ => return Err(S3Error::from(ApiError::invalid_request(format!("unsupported view update: {action}")))), } } - crate::table_catalog::validate_view_metadata_references(&metadata).map_err(catalog_store_error)?; - append_previous_metadata_log(&mut metadata, previous_metadata_location)?; - metadata_object_mut(&mut metadata)?.insert("last-updated-ms".to_string(), serde_json::Value::from(current_time_millis())); + validate_supported_view_metadata(&metadata)?; Ok(metadata) } -fn apply_assign_uuid_update(metadata: &mut serde_json::Value, update: &serde_json::Value) -> S3Result<()> { +fn apply_set_snapshot_file_update( + metadata: &mut serde_json::Value, + update: &serde_json::Value, + metadata_field: &str, + update_field: &str, + kind: crate::table_catalog::IcebergStatisticsFileKind, +) -> S3Result<()> { + let value = update + .get(update_field) + .cloned() + .ok_or_else(|| S3Error::from(ApiError::invalid_request(format!("{update_field} is required"))))?; + let snapshot_id = + crate::table_catalog::validate_iceberg_statistics_file(&value, update_field, kind).map_err(catalog_store_error)?; + if let Some(deprecated_snapshot_id) = update.get("snapshot-id") { + let deprecated_snapshot_id = deprecated_snapshot_id.as_i64().ok_or_else(|| { + iceberg_rest_error(ICEBERG_ERROR_BAD_REQUEST, StatusCode::BAD_REQUEST, "snapshot-id must be an integer") + })?; + if deprecated_snapshot_id != snapshot_id { + return Err(iceberg_rest_error( + ICEBERG_ERROR_BAD_REQUEST, + StatusCode::BAD_REQUEST, + format!("{update_field}.snapshot-id does not match snapshot-id"), + )); + } + } + let values = ensure_array_field(metadata, metadata_field)?; + values.retain(|value| value.get("snapshot-id").and_then(serde_json::Value::as_i64) != Some(snapshot_id)); + values.push(value); + Ok(()) +} + +fn apply_remove_snapshot_file_update( + metadata: &mut serde_json::Value, + update: &serde_json::Value, + metadata_field: &str, +) -> S3Result<()> { + let snapshot_id = update + .get("snapshot-id") + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| S3Error::from(ApiError::invalid_request("remove update requires snapshot-id")))?; + if let Some(values) = metadata.get_mut(metadata_field).and_then(serde_json::Value::as_array_mut) { + values.retain(|value| value.get("snapshot-id").and_then(serde_json::Value::as_i64) != Some(snapshot_id)); + } + Ok(()) +} + +fn apply_remove_metadata_ids_update( + metadata: &mut serde_json::Value, + update: &serde_json::Value, + metadata_field: &str, + id_field: &str, + update_field: &str, +) -> S3Result<()> { + let ids = update + .get(update_field) + .and_then(serde_json::Value::as_array) + .ok_or_else(|| S3Error::from(ApiError::invalid_request(format!("{update_field} must be an array"))))? + .iter() + .map(|value| { + value + .as_i64() + .ok_or_else(|| S3Error::from(ApiError::invalid_request(format!("{update_field} must contain integers")))) + }) + .collect::>>()?; + if let Some(values) = metadata.get_mut(metadata_field).and_then(serde_json::Value::as_array_mut) { + values.retain(|value| { + value + .get(id_field) + .and_then(serde_json::Value::as_i64) + .is_none_or(|id| !ids.contains(&id)) + }); + } + Ok(()) +} + +fn apply_assign_uuid_update( + metadata: &mut serde_json::Value, + update: &serde_json::Value, + uuid_field: &str, + entity: &str, +) -> S3Result<()> { let uuid = update .get("uuid") .and_then(serde_json::Value::as_str) .ok_or_else(|| s3_error!(InvalidRequest, "assign-uuid requires uuid"))?; let object = metadata_object_mut(metadata)?; - if let Some(existing) = object.get("table-uuid").and_then(serde_json::Value::as_str) + if let Some(existing) = object.get(uuid_field).and_then(serde_json::Value::as_str) && existing != uuid { - return Err(s3_error!(PreconditionFailed, "cannot reassign table uuid")); + return Err(commit_requirement_failed(format!("cannot reassign {entity} uuid"))); } - object.insert("table-uuid".to_string(), serde_json::Value::String(uuid.to_string())); + object.insert(uuid_field.to_string(), serde_json::Value::String(uuid.to_string())); Ok(()) } -fn apply_add_view_version_update(metadata: &mut serde_json::Value, update: &serde_json::Value) -> S3Result<()> { +fn apply_add_view_version_update( + metadata: &mut serde_json::Value, + update: &serde_json::Value, + last_added_schema_id: Option, +) -> S3Result<(i64, i64)> { let mut view_version = update .get("view-version") .cloned() @@ -3429,35 +3767,48 @@ fn apply_add_view_version_update(metadata: &mut serde_json::Value, update: &serd if !view_version.is_object() { return Err(s3_error!(InvalidRequest, "view-version must be a JSON object")); } - if view_version.get("version-id").is_none() { - let next_id = next_array_object_i64(metadata, "versions", "version-id")?; + let version_id = view_version + .get("version-id") + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| s3_error!(InvalidRequest, "view-version version-id must be an integer"))?; + if view_version.get("schema-id").and_then(serde_json::Value::as_i64) == Some(-1) { + let schema_id = resolve_last_added_update_id(-1, last_added_schema_id, "add-view-version", "add-schema")?; view_version .as_object_mut() .ok_or_else(|| s3_error!(InvalidRequest, "view-version must be a JSON object"))? - .insert("version-id".to_string(), serde_json::Value::from(next_id)); + .insert("schema-id".to_string(), serde_json::Value::from(schema_id)); } - view_version - .as_object_mut() - .ok_or_else(|| s3_error!(InvalidRequest, "view-version must be a JSON object"))? - .entry("timestamp-ms".to_string()) - .or_insert_with(|| serde_json::Value::from(current_time_millis())); + let timestamp_ms = view_version + .get("timestamp-ms") + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| s3_error!(InvalidRequest, "view-version timestamp-ms must be an integer"))?; ensure_array_field(metadata, "versions")?.push(view_version); - Ok(()) + Ok((version_id, timestamp_ms)) } -fn apply_set_current_view_version_update(metadata: &mut serde_json::Value, update: &serde_json::Value) -> S3Result<()> { +fn apply_set_current_view_version_update( + metadata: &mut serde_json::Value, + update: &serde_json::Value, + last_added_view_version_id: Option, + added_view_version_timestamps: &BTreeMap, + commit_timestamp_ms: i64, +) -> S3Result<()> { let requested_id = update .get("view-version-id") .and_then(serde_json::Value::as_i64) .ok_or_else(|| s3_error!(InvalidRequest, "set-current-view-version requires view-version-id"))?; - let version_id = if requested_id == -1 { - last_array_object_i64(metadata, "versions", "version-id")? - } else { - requested_id - }; + let version_id = + resolve_last_added_update_id(requested_id, last_added_view_version_id, "set-current-view-version", "add-view-version")?; + if metadata.get("current-version-id").and_then(serde_json::Value::as_i64) == Some(version_id) { + return Ok(()); + } + let history_timestamp_ms = added_view_version_timestamps + .get(&version_id) + .copied() + .unwrap_or(commit_timestamp_ms); metadata_object_mut(metadata)?.insert("current-version-id".to_string(), serde_json::Value::from(version_id)); ensure_array_field(metadata, "version-log")?.push(serde_json::json!({ - "timestamp-ms": current_time_millis(), + "timestamp-ms": history_timestamp_ms, "version-id": version_id })); Ok(()) @@ -3498,148 +3849,300 @@ fn apply_upgrade_format_version_update(metadata: &mut serde_json::Value, update: Ok(()) } -fn apply_add_schema_update(metadata: &mut serde_json::Value, update: &serde_json::Value) -> S3Result<()> { +fn apply_upgrade_view_format_version_update(update: &serde_json::Value) -> S3Result<()> { + let version = update + .get("format-version") + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| s3_error!(InvalidRequest, "upgrade-format-version requires format-version"))?; + if version != ICEBERG_VIEW_FORMAT_VERSION { + return Err(iceberg_rest_error( + ICEBERG_ERROR_UNSUPPORTED_OPERATION, + StatusCode::NOT_ACCEPTABLE, + format!("unsupported Iceberg view format-version: {version}"), + )); + } + Ok(()) +} + +fn catalog_assigned_schema(update: &serde_json::Value, schema_id: i64) -> S3Result { let mut schema = update .get("schema") .cloned() .ok_or_else(|| s3_error!(InvalidRequest, "add-schema requires schema"))?; - if !schema.is_object() { - return Err(s3_error!(InvalidRequest, "add-schema schema must be a JSON object")); - } - if schema.get("schema-id").is_none() { - let next_id = next_array_object_i64(metadata, "schemas", "schema-id")?; - schema - .as_object_mut() - .ok_or_else(|| s3_error!(InvalidRequest, "add-schema schema must be a JSON object"))? - .insert("schema-id".to_string(), serde_json::Value::from(next_id)); - } + let schema_object = schema + .as_object_mut() + .ok_or_else(|| s3_error!(InvalidRequest, "add-schema schema must be a JSON object"))?; + schema_object.insert("schema-id".to_string(), serde_json::Value::from(schema_id)); + Ok(schema) +} + +fn apply_add_table_schema_update(metadata: &mut serde_json::Value, update: &serde_json::Value, schema_id: i64) -> S3Result<()> { + let schema = catalog_assigned_schema(update, schema_id)?; let last_column_id = max_field_id(&schema); ensure_array_field(metadata, "schemas")?.push(schema); let object = metadata_object_mut(metadata)?; let current_last = object .get("last-column-id") .and_then(serde_json::Value::as_i64) - .unwrap_or_default(); + .ok_or_else(|| s3_error!(InvalidRequest, "current table metadata is missing last-column-id"))?; object.insert("last-column-id".to_string(), serde_json::Value::from(current_last.max(last_column_id))); Ok(()) } -fn apply_set_current_schema_update(metadata: &mut serde_json::Value, update: &serde_json::Value) -> S3Result<()> { +fn apply_add_view_schema_update(metadata: &mut serde_json::Value, update: &serde_json::Value, schema_id: i64) -> S3Result<()> { + let schema = catalog_assigned_schema(update, schema_id)?; + ensure_array_field(metadata, "schemas")?.push(schema); + Ok(()) +} + +fn apply_set_current_schema_update( + metadata: &mut serde_json::Value, + update: &serde_json::Value, + last_added_schema_id: Option, +) -> S3Result<()> { let requested_id = update .get("schema-id") .and_then(serde_json::Value::as_i64) .ok_or_else(|| s3_error!(InvalidRequest, "set-current-schema requires schema-id"))?; - let schema_id = if requested_id == -1 { - last_array_object_i64(metadata, "schemas", "schema-id")? - } else { - requested_id - }; + let schema_id = resolve_last_added_update_id(requested_id, last_added_schema_id, "set-current-schema", "add-schema")?; metadata_object_mut(metadata)?.insert("current-schema-id".to_string(), serde_json::Value::from(schema_id)); Ok(()) } -fn apply_add_spec_update(metadata: &mut serde_json::Value, update: &serde_json::Value) -> S3Result<()> { +fn apply_add_spec_update(metadata: &mut serde_json::Value, update: &serde_json::Value, spec_id: i64) -> S3Result<()> { let mut spec = update .get("spec") .cloned() .ok_or_else(|| s3_error!(InvalidRequest, "add-spec requires spec"))?; - if !spec.is_object() { - return Err(s3_error!(InvalidRequest, "add-spec spec must be a JSON object")); - } - if spec.get("spec-id").is_none() { - let next_id = next_array_object_i64(metadata, "partition-specs", "spec-id")?; - spec.as_object_mut() - .ok_or_else(|| s3_error!(InvalidRequest, "add-spec spec must be a JSON object"))? - .insert("spec-id".to_string(), serde_json::Value::from(next_id)); - } - let last_partition_id = max_partition_field_id(&spec); - ensure_array_field(metadata, "partition-specs")?.push(spec); - let object = metadata_object_mut(metadata)?; - let current_last = object + let spec_object = spec + .as_object_mut() + .ok_or_else(|| s3_error!(InvalidRequest, "add-spec spec must be a JSON object"))?; + spec_object.insert("spec-id".to_string(), serde_json::Value::from(spec_id)); + let current_last = metadata .get("last-partition-id") .and_then(serde_json::Value::as_i64) .unwrap_or(999); - object.insert( - "last-partition-id".to_string(), - serde_json::Value::from(current_last.max(last_partition_id)), - ); + let existing_fields = existing_partition_field_ids(metadata)?; + let last_partition_id = assign_partition_field_ids(&mut spec, current_last, &existing_fields)?; + crate::table_catalog::validate_partition_spec_sources_against_current_schema(metadata, &spec).map_err(catalog_store_error)?; + ensure_array_field(metadata, "partition-specs")?.push(spec); + let object = metadata_object_mut(metadata)?; + object.insert("last-partition-id".to_string(), serde_json::Value::from(last_partition_id)); Ok(()) } -fn apply_set_default_spec_update(metadata: &mut serde_json::Value, update: &serde_json::Value) -> S3Result<()> { +fn existing_partition_field_ids(metadata: &serde_json::Value) -> S3Result> { + let mut existing = BTreeMap::new(); + for spec in metadata + .get("partition-specs") + .and_then(serde_json::Value::as_array) + .into_iter() + .flatten() + { + for field in spec.get("fields").and_then(serde_json::Value::as_array).into_iter().flatten() { + let source_id = field + .get("source-id") + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| s3_error!(InvalidRequest, "partition source-id must be an integer"))?; + let transform = field + .get("transform") + .and_then(serde_json::Value::as_str) + .ok_or_else(|| s3_error!(InvalidRequest, "partition transform must be a string"))?; + let field_id = field + .get("field-id") + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| s3_error!(InvalidRequest, "partition field-id must be an integer"))?; + match existing.insert((source_id, transform.to_string()), field_id) { + Some(previous) if previous != field_id => { + return Err(s3_error!(InvalidRequest, "equivalent partition fields must reuse the same field-id")); + } + _ => {} + } + } + } + Ok(existing) +} + +fn assign_partition_field_ids( + spec: &mut serde_json::Value, + current_last: i64, + existing_fields: &BTreeMap<(i64, String), i64>, +) -> S3Result { + let fields = spec + .get_mut("fields") + .and_then(serde_json::Value::as_array_mut) + .ok_or_else(|| s3_error!(InvalidRequest, "partition spec fields must be an array"))?; + let mut assigned_ids = BTreeSet::new(); + let mut last_partition_id = current_last; + for field in fields.iter() { + let field = field + .as_object() + .ok_or_else(|| s3_error!(InvalidRequest, "partition spec fields must be JSON objects"))?; + let Some(field_id) = field.get("field-id") else { + continue; + }; + let field_id = field_id + .as_i64() + .ok_or_else(|| s3_error!(InvalidRequest, "partition field-id must be an integer"))?; + if i32::try_from(field_id).is_err() || !assigned_ids.insert(field_id) { + return Err(s3_error!(InvalidRequest, "partition field-id must be a unique signed 32-bit integer")); + } + let source_id = field + .get("source-id") + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| s3_error!(InvalidRequest, "partition source-id must be an integer"))?; + let transform = field + .get("transform") + .and_then(serde_json::Value::as_str) + .ok_or_else(|| s3_error!(InvalidRequest, "partition transform must be a string"))?; + if existing_fields + .get(&(source_id, transform.to_string())) + .is_some_and(|existing_id| *existing_id != field_id) + { + return Err(s3_error!(InvalidRequest, "equivalent partition fields must reuse the same field-id")); + } + last_partition_id = last_partition_id.max(field_id); + } + for field in fields.iter_mut().filter(|field| field.get("field-id").is_none()) { + let source_id = field + .get("source-id") + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| s3_error!(InvalidRequest, "partition source-id must be an integer"))?; + let transform = field + .get("transform") + .and_then(serde_json::Value::as_str) + .ok_or_else(|| s3_error!(InvalidRequest, "partition transform must be a string"))?; + let field_id = match existing_fields.get(&(source_id, transform.to_string())) { + Some(field_id) => *field_id, + None => { + last_partition_id = last_partition_id + .checked_add(1) + .filter(|field_id| i32::try_from(*field_id).is_ok()) + .ok_or_else(|| s3_error!(InvalidRequest, "partition field-id exceeds the signed 32-bit range"))?; + last_partition_id + } + }; + if !assigned_ids.insert(field_id) { + return Err(s3_error!(InvalidRequest, "partition field-id must be unique within a partition spec")); + } + field + .as_object_mut() + .ok_or_else(|| s3_error!(InvalidRequest, "partition spec fields must be JSON objects"))? + .insert("field-id".to_string(), serde_json::Value::from(field_id)); + } + Ok(last_partition_id) +} + +fn apply_set_default_spec_update( + metadata: &mut serde_json::Value, + update: &serde_json::Value, + last_added_spec_id: Option, +) -> S3Result<()> { let requested_id = update .get("spec-id") .and_then(serde_json::Value::as_i64) .ok_or_else(|| s3_error!(InvalidRequest, "set-default-spec requires spec-id"))?; - let spec_id = if requested_id == -1 { - last_array_object_i64(metadata, "partition-specs", "spec-id")? - } else { - requested_id - }; + let spec_id = resolve_last_added_update_id(requested_id, last_added_spec_id, "set-default-spec", "add-spec")?; metadata_object_mut(metadata)?.insert("default-spec-id".to_string(), serde_json::Value::from(spec_id)); Ok(()) } -fn apply_add_sort_order_update(metadata: &mut serde_json::Value, update: &serde_json::Value) -> S3Result<()> { +fn apply_add_sort_order_update( + metadata: &mut serde_json::Value, + update: &serde_json::Value, + sort_order_id: i64, +) -> S3Result { let mut sort_order = update .get("sort-order") .cloned() .ok_or_else(|| s3_error!(InvalidRequest, "add-sort-order requires sort-order"))?; - if !sort_order.is_object() { - return Err(s3_error!(InvalidRequest, "add-sort-order sort-order must be a JSON object")); + let sort_order_object = sort_order + .as_object_mut() + .ok_or_else(|| s3_error!(InvalidRequest, "add-sort-order sort-order must be a JSON object"))?; + let fields_are_empty = sort_order_object + .get("fields") + .and_then(serde_json::Value::as_array) + .ok_or_else(|| s3_error!(InvalidRequest, "sort-order fields must be an array"))? + .is_empty(); + let assigned_id = if fields_are_empty { 0 } else { sort_order_id }; + sort_order_object.insert("order-id".to_string(), serde_json::Value::from(assigned_id)); + crate::table_catalog::validate_sort_order_sources_against_current_schema(metadata, &sort_order) + .map_err(catalog_store_error)?; + let sort_orders = ensure_array_field(metadata, "sort-orders")?; + if assigned_id == 0 { + sort_orders.retain(|order| order.get("order-id").and_then(serde_json::Value::as_i64) != Some(0)); } - if sort_order.get("order-id").is_none() { - let next_id = next_array_object_i64(metadata, "sort-orders", "order-id")?; - sort_order - .as_object_mut() - .ok_or_else(|| s3_error!(InvalidRequest, "add-sort-order sort-order must be a JSON object"))? - .insert("order-id".to_string(), serde_json::Value::from(next_id)); - } - ensure_array_field(metadata, "sort-orders")?.push(sort_order); - Ok(()) + sort_orders.push(sort_order); + Ok(assigned_id) } -fn apply_set_default_sort_order_update(metadata: &mut serde_json::Value, update: &serde_json::Value) -> S3Result<()> { +fn apply_set_default_sort_order_update( + metadata: &mut serde_json::Value, + update: &serde_json::Value, + last_added_sort_order_id: Option, +) -> S3Result<()> { let requested_id = update .get("sort-order-id") .and_then(serde_json::Value::as_i64) .ok_or_else(|| s3_error!(InvalidRequest, "set-default-sort-order requires sort-order-id"))?; - let sort_order_id = if requested_id == -1 { - last_array_object_i64(metadata, "sort-orders", "order-id")? - } else { - requested_id - }; + let sort_order_id = + resolve_last_added_update_id(requested_id, last_added_sort_order_id, "set-default-sort-order", "add-sort-order")?; metadata_object_mut(metadata)?.insert("default-sort-order-id".to_string(), serde_json::Value::from(sort_order_id)); Ok(()) } -fn apply_add_snapshot_update(metadata: &mut serde_json::Value, update: &serde_json::Value) -> S3Result<()> { +fn apply_add_snapshot_update(metadata: &mut serde_json::Value, update: &serde_json::Value) -> S3Result { let snapshot = update .get("snapshot") .cloned() .ok_or_else(|| s3_error!(InvalidRequest, "add-snapshot requires snapshot"))?; + let format_version = metadata + .get("format-version") + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| s3_error!(InvalidRequest, "current table metadata is missing format-version"))?; + if format_version == 2 && snapshot.get("manifests").is_some() { + return Err(s3_error!(InvalidRequest, "Iceberg v2 snapshots require manifest-list")); + } let snapshot_id = snapshot .get("snapshot-id") .and_then(serde_json::Value::as_i64) .ok_or_else(|| s3_error!(InvalidRequest, "snapshot-id must be an integer"))?; - let sequence_number = snapshot - .get("sequence-number") - .and_then(serde_json::Value::as_i64) - .ok_or_else(|| s3_error!(InvalidRequest, "snapshot sequence-number must be an integer"))?; - let timestamp_ms = snapshot + let sequence_number = snapshot_sequence_number(&snapshot, format_version)?; + snapshot .get("timestamp-ms") .and_then(serde_json::Value::as_i64) - .unwrap_or_else(current_time_millis); - validate_added_snapshot(metadata, &snapshot, snapshot_id, sequence_number)?; + .ok_or_else(|| s3_error!(InvalidRequest, "snapshot timestamp-ms must be an integer"))?; + validate_added_snapshot(metadata, &snapshot, snapshot_id, sequence_number, format_version)?; ensure_array_field(metadata, "snapshots")?.push(snapshot); - let object = metadata_object_mut(metadata)?; - object.insert("last-sequence-number".to_string(), serde_json::Value::from(sequence_number)); - object.insert("current-snapshot-id".to_string(), serde_json::Value::from(snapshot_id)); - ensure_array_field(metadata, "snapshot-log")?.push(serde_json::json!({ - "timestamp-ms": timestamp_ms, - "snapshot-id": snapshot_id - })); - Ok(()) + if format_version > 1 { + metadata_object_mut(metadata)?.insert("last-sequence-number".to_string(), serde_json::Value::from(sequence_number)); + } + Ok(snapshot_id) +} + +fn snapshot_sequence_number(snapshot: &serde_json::Value, format_version: i64) -> S3Result { + let sequence_number = match snapshot.get("sequence-number") { + Some(sequence_number) => sequence_number + .as_i64() + .ok_or_else(|| s3_error!(InvalidRequest, "snapshot sequence-number must be an integer")), + None if format_version == 1 => Ok(0), + None => Err(s3_error!(InvalidRequest, "Iceberg v2 snapshot sequence-number is required")), + }?; + if format_version == 1 && sequence_number != 0 { + return Err(s3_error!(InvalidRequest, "Iceberg v1 snapshot sequence-number must be zero")); + } + Ok(sequence_number) +} + +fn snapshot_parent_id(snapshot: &serde_json::Value) -> S3Result> { + snapshot + .get("parent-snapshot-id") + .map(|parent_snapshot_id| { + parent_snapshot_id + .as_i64() + .ok_or_else(|| s3_error!(InvalidRequest, "snapshot parent-snapshot-id must be an integer")) + }) + .transpose() } fn validate_added_snapshot( @@ -3647,6 +4150,7 @@ fn validate_added_snapshot( snapshot: &serde_json::Value, snapshot_id: i64, sequence_number: i64, + format_version: i64, ) -> S3Result<()> { if metadata .get("snapshots") @@ -3657,22 +4161,31 @@ fn validate_added_snapshot( .any(|snapshot| snapshot.get("snapshot-id").and_then(serde_json::Value::as_i64) == Some(snapshot_id)) }) { - return Err(s3_error!(PreconditionFailed, "snapshot id already exists")); + return Err(commit_requirement_failed("snapshot id already exists")); } - let current_snapshot_id = metadata.get("current-snapshot-id").and_then(serde_json::Value::as_i64); - if let Some(parent_snapshot_id) = snapshot.get("parent-snapshot-id").and_then(serde_json::Value::as_i64) - && Some(parent_snapshot_id) != current_snapshot_id + let parent_snapshot_id = snapshot_parent_id(snapshot)?; + if let Some(parent_snapshot_id) = parent_snapshot_id + && !metadata + .get("snapshots") + .and_then(serde_json::Value::as_array) + .is_some_and(|snapshots| { + snapshots + .iter() + .any(|snapshot| snapshot.get("snapshot-id").and_then(serde_json::Value::as_i64) == Some(parent_snapshot_id)) + }) { - return Err(s3_error!(PreconditionFailed, "snapshot parent no longer matches current snapshot")); + return Err(commit_requirement_failed("snapshot parent does not exist")); } - let current_sequence_number = metadata - .get("last-sequence-number") - .and_then(serde_json::Value::as_i64) - .unwrap_or_default(); - if sequence_number <= current_sequence_number { - return Err(s3_error!(PreconditionFailed, "snapshot sequence number must advance")); + if format_version > 1 { + let current_sequence_number = metadata + .get("last-sequence-number") + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| s3_error!(InvalidRequest, "current Iceberg v2 metadata is missing last-sequence-number"))?; + if sequence_number <= current_sequence_number { + return Err(commit_requirement_failed("snapshot sequence number must advance")); + } } if !snapshot_has_manifest_references(snapshot) { @@ -3685,7 +4198,7 @@ fn validate_added_snapshot( .and_then(serde_json::Value::as_str) .ok_or_else(|| s3_error!(InvalidRequest, "snapshot summary.operation is required"))?; if !matches!(operation, "append" | "overwrite" | "delete" | "replace") { - return Err(s3_error!(NotImplemented, "unsupported snapshot operation: {operation}")); + return Err(s3_error!(InvalidRequest, "unsupported snapshot operation: {operation}")); } Ok(()) @@ -3785,31 +4298,56 @@ async fn validate_table_snapshot_commit_conflicts( where B: crate::table_catalog::TableCatalogObjectBackend, { - let Some(snapshot) = added_snapshot_update(updates)? else { - return Ok(()); - }; + let mut snapshot_state = current_metadata.clone(); + for update in updates { + match update.get("action").and_then(serde_json::Value::as_str) { + Some("add-snapshot") => { + let snapshot = update + .get("snapshot") + .ok_or_else(|| s3_error!(InvalidRequest, "add-snapshot requires snapshot"))?; + validate_snapshot_file_conflicts(metadata_backend, bucket, entry, &snapshot_state, snapshot).await?; + apply_add_snapshot_update(&mut snapshot_state, update)?; + } + Some("remove-snapshots") => apply_remove_snapshots_update(&mut snapshot_state, update)?, + _ => {} + } + } + Ok(()) +} + +async fn validate_snapshot_file_conflicts( + metadata_backend: &B, + bucket: &str, + entry: &crate::table_catalog::TableEntry, + snapshot_state: &serde_json::Value, + snapshot: &serde_json::Value, +) -> S3Result<()> +where + B: crate::table_catalog::TableCatalogObjectBackend, +{ let snapshot_id = snapshot .get("snapshot-id") .and_then(serde_json::Value::as_i64) .ok_or_else(|| s3_error!(InvalidRequest, "snapshot-id must be an integer"))?; - let sequence_number = snapshot - .get("sequence-number") + let format_version = snapshot_state + .get("format-version") .and_then(serde_json::Value::as_i64) - .ok_or_else(|| s3_error!(InvalidRequest, "snapshot sequence-number must be an integer"))?; + .ok_or_else(|| s3_error!(InvalidRequest, "current table metadata is missing format-version"))?; + let sequence_number = snapshot_sequence_number(snapshot, format_version)?; let operation = snapshot .get("summary") .and_then(|summary| summary.get("operation")) .and_then(serde_json::Value::as_str) .ok_or_else(|| s3_error!(InvalidRequest, "snapshot summary.operation is required"))?; - - let current_live_files = load_current_snapshot_live_files(metadata_backend, bucket, entry, current_metadata).await?; + let parent_snapshot_id = snapshot_parent_id(snapshot)?; + let parent_live_files = load_snapshot_live_files(metadata_backend, bucket, entry, snapshot_state, parent_snapshot_id).await?; let changes = load_snapshot_file_changes( metadata_backend, bucket, entry, snapshot, SnapshotChangeContext { - current_live_files: ¤t_live_files, + current_live_files: &parent_live_files, snapshot_id, sequence_number, }, @@ -3817,10 +4355,9 @@ where .await?; for location in changes.added_data_files.iter().chain(changes.added_delete_files.iter()) { - if current_live_files.contains(location) { - return Err(s3_error!( - PreconditionFailed, - "commit requirement failed: added file already exists in current snapshot" + if parent_live_files.contains(location) { + return Err(commit_requirement_failed( + "commit requirement failed: added file already exists in parent snapshot", )); } } @@ -3832,12 +4369,8 @@ where } } "overwrite" | "delete" | "replace" => { - if current_metadata - .get("current-snapshot-id") - .and_then(serde_json::Value::as_i64) - .is_none() - { - return Err(s3_error!(InvalidRequest, "row-level snapshot operation requires a current snapshot")); + if parent_snapshot_id.is_none() { + return Err(s3_error!(InvalidRequest, "row-level snapshot operation requires a parent snapshot")); } if operation == "overwrite" { if !changes.has_any_change() { @@ -3850,48 +4383,30 @@ where )); } for location in changes.deleted_data_files.iter().chain(changes.deleted_delete_files.iter()) { - if !current_live_files.contains(location) { - return Err(s3_error!(PreconditionFailed, "commit requirement failed: deleted file is not current")); + if !parent_live_files.contains(location) { + return Err(commit_requirement_failed( + "commit requirement failed: deleted file is not in the parent snapshot", + )); } } } - _ => return Err(s3_error!(NotImplemented, "unsupported snapshot operation: {operation}")), + _ => return Err(s3_error!(InvalidRequest, "unsupported snapshot operation: {operation}")), } Ok(()) } -fn added_snapshot_update(updates: &[serde_json::Value]) -> S3Result> { - let mut snapshot = None; - for update in updates { - if update.get("action").and_then(serde_json::Value::as_str) != Some("add-snapshot") { - continue; - } - if snapshot.is_some() { - return Err(s3_error!(InvalidRequest, "standard commit supports one add-snapshot update")); - } - snapshot = Some( - update - .get("snapshot") - .ok_or_else(|| s3_error!(InvalidRequest, "add-snapshot requires snapshot"))?, - ); - } - Ok(snapshot) -} - -async fn load_current_snapshot_live_files( +async fn load_snapshot_live_files( metadata_backend: &B, bucket: &str, entry: &crate::table_catalog::TableEntry, current_metadata: &serde_json::Value, + snapshot_id: Option, ) -> S3Result where B: crate::table_catalog::TableCatalogObjectBackend, { - let Some(current_snapshot_id) = current_metadata - .get("current-snapshot-id") - .and_then(serde_json::Value::as_i64) - else { + let Some(snapshot_id) = snapshot_id else { return Ok(SnapshotLiveFiles::default()); }; let snapshot = current_metadata @@ -3900,9 +4415,9 @@ where .and_then(|snapshots| { snapshots .iter() - .find(|snapshot| snapshot.get("snapshot-id").and_then(serde_json::Value::as_i64) == Some(current_snapshot_id)) + .find(|snapshot| snapshot.get("snapshot-id").and_then(serde_json::Value::as_i64) == Some(snapshot_id)) }) - .ok_or_else(|| s3_error!(InvalidRequest, "current snapshot metadata is missing"))?; + .ok_or_else(|| commit_requirement_failed("commit requirement failed: parent snapshot no longer exists"))?; let mut live_files = SnapshotLiveFiles::default(); for manifest in read_snapshot_manifest_references(metadata_backend, bucket, entry, snapshot).await? { @@ -4204,7 +4719,12 @@ fn table_commit_object_key( Ok(object_key) } -fn apply_set_snapshot_ref_update(metadata: &mut serde_json::Value, update: &serde_json::Value) -> S3Result<()> { +fn apply_set_snapshot_ref_update( + metadata: &mut serde_json::Value, + update: &serde_json::Value, + added_snapshot_ids: &BTreeSet, + commit_timestamp_ms: i64, +) -> S3Result<()> { let ref_name = update .get("ref-name") .and_then(serde_json::Value::as_str) @@ -4220,9 +4740,47 @@ fn apply_set_snapshot_ref_update(metadata: &mut serde_json::Value, update: &serd .filter(|(key, _)| key.as_str() != "action" && key.as_str() != "ref-name") .map(|(key, value)| (key.clone(), value.clone())) .collect::>(); - ensure_object_field(metadata, "refs")?.insert(ref_name.to_string(), serde_json::Value::Object(reference)); + if !metadata + .get("snapshots") + .and_then(serde_json::Value::as_array) + .is_some_and(|snapshots| { + snapshots + .iter() + .any(|snapshot| snapshot.get("snapshot-id").and_then(serde_json::Value::as_i64) == Some(snapshot_id)) + }) + { + return Err(s3_error!(InvalidRequest, "set-snapshot-ref targets an unknown snapshot")); + } + let next_reference = serde_json::Value::Object(reference); + let unchanged = metadata + .get("refs") + .and_then(serde_json::Value::as_object) + .and_then(|refs| refs.get(ref_name)) + == Some(&next_reference); + ensure_object_field(metadata, "refs")?.insert(ref_name.to_string(), next_reference); if ref_name == "main" { metadata_object_mut(metadata)?.insert("current-snapshot-id".to_string(), serde_json::Value::from(snapshot_id)); + if !unchanged { + let timestamp_ms = if added_snapshot_ids.contains(&snapshot_id) { + metadata + .get("snapshots") + .and_then(serde_json::Value::as_array) + .and_then(|snapshots| { + snapshots + .iter() + .find(|snapshot| snapshot.get("snapshot-id").and_then(serde_json::Value::as_i64) == Some(snapshot_id)) + }) + .and_then(|snapshot| snapshot.get("timestamp-ms")) + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| s3_error!(InvalidRequest, "snapshot timestamp-ms must be an integer"))? + } else { + commit_timestamp_ms + }; + ensure_array_field(metadata, "snapshot-log")?.push(serde_json::json!({ + "timestamp-ms": timestamp_ms, + "snapshot-id": snapshot_id + })); + } } Ok(()) } @@ -4233,19 +4791,75 @@ fn apply_remove_snapshots_update(metadata: &mut serde_json::Value, update: &serd .and_then(serde_json::Value::as_array) .ok_or_else(|| s3_error!(InvalidRequest, "remove-snapshots requires snapshot-ids"))? .iter() - .filter_map(serde_json::Value::as_i64) - .collect::>(); - ensure_array_field(metadata, "snapshots")?.retain(|snapshot| { + .map(|snapshot_id| { + snapshot_id + .as_i64() + .ok_or_else(|| s3_error!(InvalidRequest, "snapshot-ids must contain integers")) + }) + .collect::>>()?; + let snapshots = ensure_array_field(metadata, "snapshots")?; + let snapshot_count = snapshots.len(); + snapshots.retain(|snapshot| { snapshot .get("snapshot-id") .and_then(serde_json::Value::as_i64) .is_none_or(|snapshot_id| !ids.contains(&snapshot_id)) }); - ensure_array_field(metadata, "snapshot-log")?.retain(|log| { - log.get("snapshot-id") + let removed_snapshot = snapshots.len() != snapshot_count; + if removed_snapshot { + let remaining_snapshot_ids = snapshots + .iter() + .filter_map(|snapshot| snapshot.get("snapshot-id").and_then(serde_json::Value::as_i64)) + .collect::>(); + let snapshot_log = ensure_array_field(metadata, "snapshot-log")?; + let previous_log = std::mem::take(snapshot_log); + for log in previous_log { + let snapshot_id = log + .get("snapshot-id") + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| s3_error!(InvalidRequest, "snapshot-log snapshot-id must be an integer"))?; + if remaining_snapshot_ids.contains(&snapshot_id) { + snapshot_log.push(log); + } else { + snapshot_log.clear(); + } + } + } + let dangling_refs = metadata + .get("refs") + .and_then(serde_json::Value::as_object) + .into_iter() + .flat_map(|refs| refs.iter()) + .filter_map(|(name, reference)| { + reference + .get("snapshot-id") + .and_then(serde_json::Value::as_i64) + .filter(|snapshot_id| ids.contains(snapshot_id)) + .map(|_| name.clone()) + }) + .collect::>(); + let removed_main = dangling_refs.iter().any(|name| name == "main") + || metadata + .get("current-snapshot-id") .and_then(serde_json::Value::as_i64) - .is_none_or(|snapshot_id| !ids.contains(&snapshot_id)) - }); + .is_some_and(|snapshot_id| ids.contains(&snapshot_id)); + let refs = ensure_object_field(metadata, "refs")?; + for name in dangling_refs { + refs.remove(&name); + } + if removed_main { + metadata_object_mut(metadata)?.insert("current-snapshot-id".to_string(), serde_json::Value::from(-1)); + } + for field in ["statistics", "partition-statistics"] { + if let Some(values) = metadata.get_mut(field).and_then(serde_json::Value::as_array_mut) { + values.retain(|value| { + value + .get("snapshot-id") + .and_then(serde_json::Value::as_i64) + .is_none_or(|snapshot_id| !ids.contains(&snapshot_id)) + }); + } + } Ok(()) } @@ -4254,7 +4868,10 @@ fn apply_remove_snapshot_ref_update(metadata: &mut serde_json::Value, update: &s .get("ref-name") .and_then(serde_json::Value::as_str) .ok_or_else(|| s3_error!(InvalidRequest, "remove-snapshot-ref requires ref-name"))?; - ensure_object_field(metadata, "refs")?.remove(ref_name); + let removed = ensure_object_field(metadata, "refs")?.remove(ref_name).is_some(); + if removed && ref_name == "main" { + metadata_object_mut(metadata)?.insert("current-snapshot-id".to_string(), serde_json::Value::from(-1)); + } Ok(()) } @@ -4340,8 +4957,47 @@ fn ensure_object_field<'a>( .ok_or_else(|| s3_error!(InvalidRequest, "metadata field {key} must be an object")) } +fn validate_commit_item_count(label: &str, count: usize, max_count: usize) -> S3Result<()> { + if count > max_count { + return Err(s3_error!(InvalidRequest, "{label} exceeds the maximum count of {max_count}")); + } + Ok(()) +} + +fn validate_rest_commit_item_counts(requirements: &[serde_json::Value], updates: &[serde_json::Value]) -> S3Result<()> { + validate_commit_item_count("commit requirements", requirements.len(), TABLE_CATALOG_COMMIT_REQUIREMENT_MAX_COUNT)?; + validate_commit_item_count("commit updates", updates.len(), TABLE_CATALOG_COMMIT_UPDATE_MAX_COUNT) +} + +fn next_catalog_id_for_updates( + metadata: &serde_json::Value, + updates: &[serde_json::Value], + action: &str, + array_key: &str, + id_key: &str, +) -> S3Result> { + updates + .iter() + .any(|update| update.get("action").and_then(serde_json::Value::as_str) == Some(action)) + .then(|| next_array_object_i64(metadata, array_key, id_key)) + .transpose() +} + +fn take_catalog_assigned_id(next_id: &mut Option, label: &str) -> S3Result { + let next_id = next_id + .as_mut() + .ok_or_else(|| s3_error!(InternalError, "catalog-assigned {label} state is missing"))?; + let assigned_id = *next_id; + *next_id = next_id + .checked_add(1) + .ok_or_else(|| s3_error!(InvalidRequest, "catalog-assigned {label} exceeds the signed 64-bit range"))?; + Ok(assigned_id) +} + fn next_array_object_i64(metadata: &serde_json::Value, array_key: &str, id_key: &str) -> S3Result { - Ok(last_array_object_i64(metadata, array_key, id_key)?.saturating_add(1)) + last_array_object_i64(metadata, array_key, id_key)? + .checked_add(1) + .ok_or_else(|| s3_error!(InvalidRequest, "metadata field {array_key} {id_key} exceeds the signed 64-bit range")) } fn last_array_object_i64(metadata: &serde_json::Value, array_key: &str, id_key: &str) -> S3Result { @@ -4356,6 +5012,18 @@ fn last_array_object_i64(metadata: &serde_json::Value, array_key: &str, id_key: .ok_or_else(|| s3_error!(InvalidRequest, "metadata field {array_key} has no {id_key}")) } +fn resolve_last_added_update_id( + requested_id: i64, + last_added_id: Option, + update_action: &str, + required_action: &str, +) -> S3Result { + if requested_id != -1 { + return Ok(requested_id); + } + last_added_id.ok_or_else(|| s3_error!(InvalidRequest, "{update_action} id -1 requires a preceding {required_action} update")) +} + fn table_commit_operation(metadata: &serde_json::Value) -> String { metadata .get("snapshots") @@ -4392,6 +5060,18 @@ fn iceberg_rest_error(error_type: &str, status: StatusCode, message: impl Into) -> S3Error { + iceberg_rest_error(ICEBERG_ERROR_COMMIT_FAILED, StatusCode::CONFLICT, message) +} + +fn persisted_metadata_error(entity: &str) -> S3Error { + iceberg_rest_error( + ICEBERG_ERROR_REST, + StatusCode::INTERNAL_SERVER_ERROR, + format!("persisted {entity} metadata is invalid"), + ) +} + fn catalog_store_error(err: crate::table_catalog::TableCatalogStoreError) -> S3Error { match err { crate::table_catalog::TableCatalogStoreError::NotFound(message) => { @@ -4687,6 +5367,15 @@ where { let (entry, metadata) = view_entry_from_create_view_request(bucket, namespace, request)?; ensure_table_bucket_entry(store, bucket, table_bucket_enabled).await?; + crate::table_catalog::TableCommitPublication::begin_table_bucket(metadata_backend, bucket) + .await + .map_err(catalog_store_error)?; + if !crate::table_catalog::TableCommitPublication::holds_table_bucket(metadata_backend, bucket) { + return Err(catalog_store_error(crate::table_catalog::TableCatalogStoreError::Internal( + "view creation requires a table-bucket publication fence".to_string(), + ))); + } + let _publication_completion = crate::table_catalog::TableCommitPublicationCompletion::new(metadata_backend); let metadata_data = serde_json::to_vec(&metadata) .map_err(|err| s3_error!(InternalError, "failed to serialize initial view metadata: {}", err))?; metadata_backend @@ -4699,7 +5388,7 @@ where .await .map_err(catalog_store_already_exists_error)?; store - .create_view(entry.clone()) + .create_view_with_publication(entry.clone(), metadata_backend) .await .map_err(catalog_store_already_exists_error)?; Ok(load_view_response_from_entry(entry, metadata)) @@ -4719,6 +5408,17 @@ async fn read_table_metadata_json( Ok(metadata) } +async fn read_persisted_metadata_json( + metadata_backend: &impl crate::table_catalog::TableCatalogObjectBackend, + bucket: &str, + metadata_location: &str, + entity: &str, +) -> S3Result { + read_table_metadata_json(metadata_backend, bucket, metadata_location) + .await + .map_err(|_| persisted_metadata_error(entity)) +} + async fn read_generated_table_metadata_json( metadata_backend: &impl crate::table_catalog::TableCatalogObjectBackend, bucket: &str, @@ -4815,7 +5515,7 @@ where else { return Err(iceberg_rest_error(ICEBERG_ERROR_NO_SUCH_TABLE, StatusCode::NOT_FOUND, "table not found")); }; - let metadata = read_table_metadata_json(metadata_backend, bucket, &entry.metadata_location).await?; + let metadata = read_persisted_table_metadata_for_entry(metadata_backend, &entry, &entry.metadata_location, true).await?; Ok(load_table_response_from_entry(entry, metadata)) } @@ -4866,7 +5566,13 @@ where else { return Err(iceberg_rest_error(ICEBERG_ERROR_NO_SUCH_VIEW, StatusCode::NOT_FOUND, "view not found")); }; - let metadata = read_table_metadata_json(metadata_backend, bucket, &entry.metadata_location).await?; + let view_name = + crate::table_catalog::IdentifierSegment::parse(view.to_string()).map_err(|_| persisted_metadata_error("view"))?; + if !crate::table_catalog::is_valid_view_metadata_location(namespace, &view_name, &entry.metadata_location) { + return Err(persisted_metadata_error("view")); + } + let metadata = read_persisted_metadata_json(metadata_backend, bucket, &entry.metadata_location, "view").await?; + validate_persisted_view_metadata(&entry, &metadata)?; Ok(load_view_response_from_entry(entry, metadata)) } @@ -4898,6 +5604,8 @@ async fn replace_view_response( where S: crate::table_catalog::TableCatalogStore + ?Sized, { + validate_rest_commit_item_counts(&request.requirements, &request.updates)?; + validate_rest_commit_identifier(request.identifier.as_ref(), namespace, view)?; let Some(current) = store .load_view(bucket, &namespace.public_name(), view) .await @@ -4905,23 +5613,33 @@ where else { return Err(iceberg_rest_error(ICEBERG_ERROR_NO_SUCH_VIEW, StatusCode::NOT_FOUND, "view not found")); }; - let current_metadata = read_table_metadata_json(metadata_backend, bucket, ¤t.metadata_location).await?; - validate_view_commit_requirements(¤t_metadata, &request.requirements)?; let view_name = crate::table_catalog::IdentifierSegment::parse(view.to_string()) .map_err(|err| s3_error!(InvalidRequest, "invalid view name: {}", err))?; + if !crate::table_catalog::is_valid_view_metadata_location(namespace, &view_name, ¤t.metadata_location) { + return Err(persisted_metadata_error("view")); + } + let current_metadata = read_persisted_metadata_json(metadata_backend, bucket, ¤t.metadata_location, "view").await?; + if request.new_metadata_location.is_some() { + validate_persisted_view_metadata_identity(¤t, ¤t_metadata)?; + } else { + validate_persisted_view_metadata(¤t, ¤t_metadata)?; + } + validate_view_commit_requirements(¤t_metadata, &request.requirements)?; let (next_metadata_location, next_metadata) = if let Some(new_metadata_location) = request.new_metadata_location { + let new_metadata_location = table_metadata_location_for_catalog(bucket, &new_metadata_location)?; if !crate::table_catalog::is_valid_view_metadata_location(namespace, &view_name, &new_metadata_location) { return Err(s3_error!(InvalidRequest, "metadata location must be inside the view metadata directory")); } let target_metadata = read_table_metadata_json(metadata_backend, bucket, &new_metadata_location).await?; + validate_supported_view_metadata(&target_metadata)?; validate_metadata_view_location_in_bucket(bucket, &target_metadata)?; validate_metadata_matches_current_view_metadata(¤t_metadata, &target_metadata)?; (new_metadata_location, target_metadata) } else { - let next_metadata = apply_view_commit_updates(current_metadata.clone(), &request.updates, ¤t.metadata_location)?; + let next_metadata = apply_view_commit_updates_at(current_metadata.clone(), &request.updates, current_time_millis())?; validate_metadata_view_location_in_bucket(bucket, &next_metadata)?; validate_metadata_matches_current_view_metadata(¤t_metadata, &next_metadata)?; - let (_, metadata_file_token) = standard_commit_ids(request.commit_id); + let (_, metadata_file_token) = standard_commit_ids(None); let next_generation = current.generation.saturating_add(1); let next_metadata_location = crate::table_catalog::default_view_metadata_file_path( namespace, @@ -4942,19 +5660,29 @@ where (next_metadata_location, next_metadata) }; + let expected_metadata_location = request + .expected_metadata_location + .as_deref() + .map(|location| table_metadata_location_for_catalog(bucket, location)) + .transpose()? + .unwrap_or_else(|| current.metadata_location.clone()); + let table_bucket_fence_required = metadata_table_location(&next_metadata)? != current.warehouse_location; + let result = store - .replace_view(crate::table_catalog::ViewCommitRequest { - table_bucket: bucket.to_string(), - namespace: namespace.public_name(), - view: view.to_string(), - expected_version_token: request - .expected_version_token - .unwrap_or_else(|| current.version_token.clone()), - expected_metadata_location: request - .expected_metadata_location - .unwrap_or_else(|| current.metadata_location.clone()), - new_metadata_location: next_metadata_location, - }) + .replace_view_with_publication( + crate::table_catalog::ViewCommitRequest { + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + view: view.to_string(), + expected_version_token: request + .expected_version_token + .unwrap_or_else(|| current.version_token.clone()), + expected_metadata_location, + new_metadata_location: next_metadata_location, + }, + table_bucket_fence_required, + metadata_backend, + ) .await .map_err(catalog_store_error)?; Ok(load_view_response_from_entry(result.view, next_metadata)) @@ -5074,8 +5802,14 @@ where let previous_metadata_location = existing_commit .as_ref() .map_or_else(|| current.metadata_location.clone(), |commit| commit.previous_metadata_location.clone()); - let previous_metadata = read_table_metadata_json(metadata_backend, bucket, &previous_metadata_location).await?; - validate_metadata_table_location_in_bucket(bucket, &previous_metadata)?; + let require_current_warehouse = existing_commit.is_none(); + let previous_metadata = read_persisted_table_metadata_for_entry( + metadata_backend, + ¤t, + &previous_metadata_location, + require_current_warehouse, + ) + .await?; let target_metadata = read_table_metadata_json(metadata_backend, bucket, &metadata_location).await?; validate_metadata_table_location_in_bucket(bucket, &target_metadata)?; let table_bucket_fence_required = table_warehouse_location_changes(¤t, &target_metadata)?; @@ -5129,6 +5863,8 @@ async fn commit_table_response( where S: crate::table_catalog::TableCatalogStore + ?Sized, { + validate_rest_commit_item_counts(&request.requirements, &request.updates)?; + validate_rest_commit_identifier(request.identifier.as_ref(), namespace, table)?; if request.new_metadata_location.is_none() { return standard_commit_table_response(store, metadata_backend, bucket, namespace, table, request).await; } @@ -5151,8 +5887,8 @@ where if !crate::table_catalog::is_valid_table_metadata_location_for_entry(¤t, &request.new_metadata_location) { return Err(s3_error!(InvalidRequest, "metadata location must be inside the table metadata directory")); } - let current_metadata = read_table_metadata_json(metadata_backend, bucket, ¤t.metadata_location).await?; - validate_metadata_table_location_in_bucket(bucket, ¤t_metadata)?; + let current_metadata = + read_persisted_table_metadata_for_entry(metadata_backend, ¤t, ¤t.metadata_location, true).await?; let target_metadata = read_table_metadata_json(metadata_backend, bucket, &request.new_metadata_location).await?; validate_metadata_table_location_in_bucket(bucket, &target_metadata)?; let table_bucket_fence_required = table_warehouse_location_changes(¤t, &target_metadata)?; @@ -5165,9 +5901,13 @@ where "commit retry does not match the original request", )); } - let previous_metadata = - read_table_metadata_json(metadata_backend, bucket, &existing_commit.previous_metadata_location).await?; - validate_metadata_table_location_in_bucket(bucket, &previous_metadata)?; + let previous_metadata = read_persisted_table_metadata_for_entry( + metadata_backend, + ¤t, + &existing_commit.previous_metadata_location, + false, + ) + .await?; validate_table_commit_requirements(&previous_metadata, &client_requirements)?; validate_metadata_matches_current_metadata(&previous_metadata, &target_metadata)?; validate_table_metadata_snapshot_graph(metadata_backend, bucket, ¤t, Some(&previous_metadata), &target_metadata) @@ -5207,7 +5947,8 @@ where { return Ok(response); } - let current_metadata = read_table_metadata_json(metadata_backend, bucket, ¤t.metadata_location).await?; + let current_metadata = + read_persisted_table_metadata_for_entry(metadata_backend, ¤t, ¤t.metadata_location, true).await?; validate_table_commit_requirements(¤t_metadata, &request.requirements)?; let expected_metadata = current_metadata.clone(); let previous_metadata_location = table_metadata_location_for_client(bucket, ¤t.metadata_location); @@ -5395,6 +6136,7 @@ async fn commit_table_replay_response( } read_table_metadata_json(metadata_backend, bucket, &result.table.metadata_location).await? }; + validate_persisted_table_metadata(&result.table, &metadata, true)?; Ok(commit_table_response_from_result(result, metadata)) } @@ -5427,8 +6169,8 @@ where )); } - let previous_metadata = read_table_metadata_json(metadata_backend, bucket, &commit.previous_metadata_location).await?; - validate_metadata_table_location_in_bucket(bucket, &previous_metadata)?; + let previous_metadata = + read_persisted_table_metadata_for_entry(metadata_backend, current, &commit.previous_metadata_location, false).await?; validate_table_commit_requirements(&previous_metadata, &request.requirements)?; let target_metadata = read_table_metadata_json(metadata_backend, bucket, &commit.new_metadata_location).await?; validate_metadata_table_location_in_bucket(bucket, &target_metadata)?; @@ -5692,7 +6434,8 @@ where return Ok(report); } - let current_metadata = read_table_metadata_json(metadata_backend, bucket, ¤t.metadata_location).await?; + let current_metadata = + read_persisted_table_metadata_for_entry(metadata_backend, ¤t, ¤t.metadata_location, true).await?; let updates = [serde_json::json!({ "action": "remove-snapshots", "snapshot-ids": expired_snapshot_ids.clone() @@ -5758,7 +6501,7 @@ where else { return Err(iceberg_rest_error(ICEBERG_ERROR_NO_SUCH_TABLE, StatusCode::NOT_FOUND, "table not found")); }; - let metadata = read_table_metadata_json(metadata_backend, bucket, &entry.metadata_location).await?; + let metadata = read_persisted_table_metadata_for_entry(metadata_backend, &entry, &entry.metadata_location, true).await?; let current_snapshot_id = metadata.get("current-snapshot-id").and_then(serde_json::Value::as_i64); let refs = metadata .get("refs") @@ -5835,7 +6578,7 @@ where namespace, table, RestCommitTableRequest { - _identifier: None, + identifier: None, commit_id: request.commit_id, idempotency_key: request.idempotency_key, operation: Some("set-snapshot-ref".to_string()), @@ -5872,7 +6615,7 @@ where else { return Err(iceberg_rest_error(ICEBERG_ERROR_NO_SUCH_TABLE, StatusCode::NOT_FOUND, "table not found")); }; - let metadata = read_table_metadata_json(metadata_backend, bucket, &entry.metadata_location).await?; + let metadata = read_persisted_table_metadata_for_entry(metadata_backend, &entry, &entry.metadata_location, true).await?; let reference = metadata .get("refs") .and_then(serde_json::Value::as_object) @@ -5895,7 +6638,7 @@ where namespace, table, RestCommitTableRequest { - _identifier: None, + identifier: None, commit_id: request.commit_id, idempotency_key: request.idempotency_key, operation: Some("remove-snapshot-ref".to_string()), @@ -5960,6 +6703,18 @@ fn external_catalog_bridge_response_from_entry( } } +async fn read_persisted_table_metadata_for_entry( + metadata_backend: &impl crate::table_catalog::TableCatalogObjectBackend, + entry: &crate::table_catalog::TableEntry, + metadata_location: &str, + require_current_warehouse: bool, +) -> S3Result { + validate_persisted_table_metadata_location(entry, metadata_location)?; + let metadata = read_persisted_metadata_json(metadata_backend, &entry.table_bucket, metadata_location, "table").await?; + validate_persisted_table_metadata(entry, &metadata, require_current_warehouse)?; + Ok(metadata) +} + fn external_catalog_bridge_capabilities() -> Vec { EXTERNAL_CATALOG_BRIDGE_CAPABILITIES .iter() @@ -6208,8 +6963,8 @@ where .expected_metadata_location .clone() .ok_or_else(|| s3_error!(InvalidRequest, "external catalog sync requires expected-metadata-location"))?; - let current_metadata = read_table_metadata_json(metadata_backend, bucket, ¤t.metadata_location).await?; - validate_metadata_table_location_in_bucket(bucket, ¤t_metadata)?; + let current_metadata = + read_persisted_table_metadata_for_entry(metadata_backend, ¤t, ¤t.metadata_location, true).await?; validate_metadata_matches_current_metadata(¤t_metadata, &target_metadata)?; validate_table_metadata_snapshot_graph(metadata_backend, bucket, ¤t, Some(¤t_metadata), &target_metadata) .await?; @@ -6349,8 +7104,8 @@ where if !crate::table_catalog::is_valid_table_metadata_location_for_entry(¤t, &metadata_location) { return Err(s3_error!(InvalidRequest, "metadata location must be inside the table metadata directory")); } - let current_metadata = read_table_metadata_json(metadata_backend, bucket, ¤t.metadata_location).await?; - validate_metadata_table_location_in_bucket(bucket, ¤t_metadata)?; + let current_metadata = + read_persisted_table_metadata_for_entry(metadata_backend, ¤t, ¤t.metadata_location, true).await?; let target_metadata = read_table_metadata_json(metadata_backend, bucket, &metadata_location).await?; validate_metadata_table_location_in_bucket(bucket, &target_metadata)?; validate_metadata_matches_current_metadata(¤t_metadata, &target_metadata)?; diff --git a/rustfs/src/admin/handlers/table_catalog/table.rs b/rustfs/src/admin/handlers/table_catalog/table.rs index 21376fd81..b98b2fbfd 100644 --- a/rustfs/src/admin/handlers/table_catalog/table.rs +++ b/rustfs/src/admin/handlers/table_catalog/table.rs @@ -125,7 +125,9 @@ impl Operation for RestLoadTableHandler { ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?; let metadata_backend = table_catalog_backend_from_extensions(&req.extensions)?; let store = table_catalog_store_from_backend(metadata_backend.clone())?; - let response = load_table_response(&store, &metadata_backend, &warehouse, &namespace, &table).await?; + let snapshot_selection = rest_table_snapshot_selection_from_query(&req.uri)?; + let mut response = load_table_response(&store, &metadata_backend, &warehouse, &namespace, &table).await?; + apply_rest_table_snapshot_selection(&mut response.metadata, snapshot_selection); build_json_response(StatusCode::OK, &response) } } @@ -158,7 +160,7 @@ impl Operation for RestCommitTableHandler { let principal = authorize_table_catalog_resource_request(&req, &resource, AdminAction::CommitTableAction).await?; install_table_catalog_s3_request_info(&mut req, &principal)?; ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?; - let request = read_json_body::(std::mem::take(&mut req.input)).await?; + let request = read_rest_commit_table_request(std::mem::take(&mut req.input)).await?; let metadata_backend = table_catalog_backend_from_extensions(&req.extensions)?; let store = table_catalog_store_from_backend(metadata_backend.clone())?; let commit_backend = TableCommitObjectBackend::for_request(metadata_backend, req); @@ -178,6 +180,14 @@ impl Operation for RestDropTableHandler { let table = table_name_from_params(¶ms)?; let resource = TableCatalogResource::table(&warehouse, &namespace, &table); authorize_table_catalog_resource_request(&req, &resource, AdminAction::DeleteTableAction).await?; + let purge_requested = rest_purge_requested_from_query(&req.uri)?; + if purge_requested { + return Err(iceberg_rest_error( + ICEBERG_ERROR_UNSUPPORTED_OPERATION, + StatusCode::NOT_ACCEPTABLE, + "purgeRequested=true is not supported", + )); + } ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?; let store = table_catalog_store_from_extensions(&req.extensions)?; drop_table_in_store(&store, &warehouse, &namespace, &table).await?; diff --git a/rustfs/src/admin/handlers/table_catalog/tests.rs b/rustfs/src/admin/handlers/table_catalog/tests.rs index 454d12bad..013ae9304 100644 --- a/rustfs/src/admin/handlers/table_catalog/tests.rs +++ b/rustfs/src/admin/handlers/table_catalog/tests.rs @@ -166,106 +166,8 @@ fn catalog_config_response_lists_standard_rest_endpoints() { assert_eq!(response.admin_discovery.runtime_capabilities, "/rustfs/admin/v4/runtime/capabilities"); assert_eq!(response.admin_discovery.cluster_snapshot, "/rustfs/admin/v4/cluster/snapshot"); assert_eq!(response.admin_discovery.extensions_catalog, "/rustfs/admin/v4/extensions/catalog"); - assert!(response.endpoints.contains(&"GET /v1/{prefix}/namespaces")); - assert!(response.endpoints.contains(&"GET /{warehouse}/catalog/migration")); - assert!(response.endpoints.contains(&"POST /{warehouse}/catalog/migration")); - assert!(response.endpoints.contains(&"DELETE /{warehouse}/catalog/migration")); - assert!(response.endpoints.contains(&"HEAD /v1/{prefix}/namespaces/{namespace}")); - assert!( - response - .endpoints - .contains(&"GET /v1/{prefix}/namespaces/{namespace}/tables/{table}") - ); - assert!( - response - .endpoints - .contains(&"HEAD /v1/{prefix}/namespaces/{namespace}/tables/{table}") - ); - assert!( - response - .endpoints - .contains(&"GET /v1/{prefix}/namespaces/{namespace}/tables/{table}/credentials") - ); - assert!(response.endpoints.contains(&"GET /{warehouse}/namespaces")); - assert!(response.endpoints.contains(&"POST /{warehouse}/namespaces")); - assert!(response.endpoints.contains(&"HEAD /{warehouse}/namespaces/{namespace}")); - assert!( - response - .endpoints - .contains(&"POST /{warehouse}/namespaces/{namespace}/register") - ); - assert!( - response - .endpoints - .contains(&"POST /{warehouse}/namespaces/{namespace}/tables") - ); - assert!(response.endpoints.contains(&"GET /{warehouse}/namespaces/{namespace}/views")); - assert!(response.endpoints.contains(&"POST /{warehouse}/namespaces/{namespace}/views")); - assert!( - response - .endpoints - .contains(&"HEAD /{warehouse}/namespaces/{namespace}/views/{view}") - ); - assert!( - response - .endpoints - .contains(&"GET /{warehouse}/namespaces/{namespace}/tables/{table}") - ); - assert!( - response - .endpoints - .contains(&"HEAD /{warehouse}/namespaces/{namespace}/tables/{table}") - ); - assert!( - response - .endpoints - .contains(&"POST /{warehouse}/namespaces/{namespace}/tables/{table}") - ); - assert!( - response - .endpoints - .contains(&"GET /{warehouse}/namespaces/{namespace}/tables/{table}/credentials") - ); - assert!( - response - .endpoints - .contains(&"GET /{warehouse}/namespaces/{namespace}/views/{view}") - ); - assert!( - response - .endpoints - .contains(&"GET /{warehouse}/namespaces/{namespace}/tables/{table}/refs") - ); - assert!( - response - .endpoints - .contains(&"PUT /{warehouse}/namespaces/{namespace}/tables/{table}/refs/{ref}") - ); - assert!( - response - .endpoints - .contains(&"DELETE /{warehouse}/namespaces/{namespace}/tables/{table}/refs/{ref}") - ); - assert!( - response - .endpoints - .contains(&"GET /{warehouse}/namespaces/{namespace}/tables/{table}/catalog/external") - ); - assert!( - response - .endpoints - .contains(&"PUT /{warehouse}/namespaces/{namespace}/tables/{table}/catalog/external") - ); - assert!( - response - .endpoints - .contains(&"POST /{warehouse}/namespaces/{namespace}/tables/{table}/catalog/external/sync") - ); - assert!( - response - .endpoints - .contains(&"POST /{warehouse}/namespaces/{namespace}/tables/{table}/catalog/recovery") - ); + assert_eq!(response.endpoints.as_slice(), TABLE_CATALOG_ENDPOINTS); + assert!(response.endpoints.iter().all(|endpoint| endpoint.contains("/v1/{prefix}/"))); } #[test] @@ -313,6 +215,31 @@ fn warehouse_config_query_rejects_empty_and_repeated_values() { assert!(warehouse_from_config_query(&uri).is_err()); } +#[test] +fn drop_table_purge_query_is_explicit_and_strict() { + for (uri, expected) in [ + ("/iceberg/v1/analytics/namespaces/sales/tables/orders", false), + ("/iceberg/v1/analytics/namespaces/sales/tables/orders?purgeRequested=false", false), + ("/iceberg/v1/analytics/namespaces/sales/tables/orders?purgeRequested=true", true), + ("/iceberg/v1/analytics/namespaces/sales/tables/orders?purgeRequested=False", false), + ("/iceberg/v1/analytics/namespaces/sales/tables/orders?purgeRequested=True", true), + ] { + assert_eq!( + rest_purge_requested_from_query(&uri.parse().expect("URI")).expect("purge query should parse"), + expected + ); + } + + for uri in [ + "/iceberg/v1/analytics/namespaces/sales/tables/orders?purgeRequested=1", + "/iceberg/v1/analytics/namespaces/sales/tables/orders?purgeRequested=true&purgeRequested=false", + ] { + let error = rest_purge_requested_from_query(&uri.parse().expect("URI")).expect_err("invalid purge query should fail"); + assert_eq!(error.code(), &S3ErrorCode::Custom(ICEBERG_ERROR_BAD_REQUEST.into())); + assert_eq!(error.status_code(), Some(StatusCode::BAD_REQUEST)); + } +} + #[test] fn catalog_conflicts_use_operation_specific_iceberg_errors() { let already_exists = catalog_store_already_exists_error(crate::table_catalog::TableCatalogStoreError::Conflict( @@ -521,6 +448,22 @@ fn table_catalog_handlers_require_table_admin_actions() { } } +#[test] +fn standard_rest_handlers_wire_strict_response_guards() { + let src = table_catalog_handler_source(); + let drop_table = operation_block(&src, "RestDropTableHandler"); + assert!(drop_table.contains("rest_purge_requested_from_query(&req.uri)?")); + assert!(drop_table.contains("if purge_requested")); + assert!(drop_table.contains("StatusCode::NOT_ACCEPTABLE")); + + let load_table = operation_block(&src, "RestLoadTableHandler"); + assert!(load_table.contains("rest_table_snapshot_selection_from_query(&req.uri)?")); + assert!(load_table.contains("apply_rest_table_snapshot_selection(&mut response.metadata, snapshot_selection);")); + + let credentials = operation_block(&src, "RestLoadCredentialsHandler"); + assert!(credentials.contains("build_sensitive_json_response(StatusCode::OK, &response)")); +} + #[test] fn table_bucket_handlers_resolve_state_from_the_request_context() { let src = table_catalog_handler_source(); @@ -580,6 +523,7 @@ fn table_pointer_write_handlers_install_commit_publication_guard() { "ImportTableCatalogHandler", "PutTableRefHandler", "DeleteTableRefHandler", + "RestReplaceViewHandler", "RollbackTableCatalogHandler", "SyncExternalCatalogBridgeHandler", ] { @@ -1345,6 +1289,32 @@ async fn namespace_property_update_body_is_bounded_and_required() { .expect("maximum valid namespace properties should remain within the domain limit"); } +#[tokio::test(start_paused = true)] +async fn generic_json_body_readers_time_out_stalled_streams() { + let required_stream = futures::stream::pending::, std::io::Error>>(); + let required = tokio::spawn(read_json_body::(Body::http_body(http_body_util::StreamBody::new( + required_stream, + )))); + let optional_stream = futures::stream::pending::, std::io::Error>>(); + let optional = tokio::spawn(read_json_body_or_default::(Body::http_body( + http_body_util::StreamBody::new(optional_stream), + ))); + tokio::task::yield_now().await; + tokio::time::advance(TABLE_CATALOG_REQUEST_BODY_TIMEOUT).await; + + let required_error = required + .await + .expect("required body task should complete") + .expect_err("stalled required body should time out"); + let optional_error = optional + .await + .expect("optional body task should complete") + .expect_err("stalled optional body should time out"); + + assert_eq!(required_error.code(), &S3ErrorCode::InvalidRequest); + assert_eq!(optional_error.code(), &S3ErrorCode::InvalidRequest); +} + #[test] fn list_tables_response_uses_rest_identifier_shape() { let namespace = crate::table_catalog::Namespace::parse("analytics.daily_events").expect("namespace should parse"); @@ -1930,9 +1900,356 @@ fn create_table_request_honors_supported_format_version_property() { assert_eq!(metadata["current-schema-id"], 0); assert!(metadata.get("partition-specs").is_some()); assert!(metadata.get("sort-orders").is_some()); + assert_eq!(metadata["sort-orders"][0]["order-id"], 0); + assert_eq!(metadata["default-sort-order-id"], 0); assert!(metadata.get("last-sequence-number").is_none()); } +#[test] +fn catalog_assigns_read_only_schema_spec_and_sort_order_ids() { + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let request: CreateTableRequest = serde_json::from_value(serde_json::json!({ + "name": "events", + "schema": { + "type": "struct", + "schema-id": 41, + "fields": [{"id": 1, "name": "id", "required": true, "type": "long"}] + }, + "partition-spec": { + "spec-id": 42, + "fields": [{"source-id": 1, "name": "id", "transform": "identity"}] + }, + "write-order": { + "order-id": 43, + "fields": [{ + "source-id": 1, + "transform": "identity", + "direction": "asc", + "null-order": "nulls-first" + }] + }, + "properties": {} + })) + .expect("create table request should parse"); + let (_, metadata) = + table_entry_from_create_table_request("warehouse", &namespace, request).expect("table metadata should be created"); + + assert_eq!(metadata["schemas"][0]["schema-id"], 0); + assert_eq!(metadata["current-schema-id"], 0); + assert_eq!(metadata["partition-specs"][0]["spec-id"], 0); + assert_eq!(metadata["partition-specs"][0]["fields"][0]["field-id"], 1000); + assert_eq!(metadata["default-spec-id"], 0); + assert_eq!(metadata["last-partition-id"], 1000); + assert_eq!(metadata["sort-orders"][0]["order-id"], 1); + assert_eq!(metadata["default-sort-order-id"], 1); + + let updated = apply_table_commit_updates_at( + metadata, + &[ + serde_json::json!({ + "action": "add-schema", + "schema": { + "type": "struct", + "schema-id": 41, + "fields": [{"id": 1, "name": "id", "required": true, "type": "long"}] + } + }), + serde_json::json!({"action": "set-current-schema", "schema-id": -1}), + serde_json::json!({ + "action": "add-spec", + "spec": { + "spec-id": 42, + "fields": [ + {"source-id": 1, "name": "id", "transform": "identity"}, + {"source-id": 1, "name": "id_bucket", "transform": "bucket[16]"} + ] + } + }), + serde_json::json!({"action": "set-default-spec", "spec-id": -1}), + serde_json::json!({ + "action": "add-sort-order", + "sort-order": {"order-id": 43, "fields": []} + }), + serde_json::json!({"action": "set-default-sort-order", "sort-order-id": -1}), + ], + "s3://warehouse/tables/table-id/metadata/v1.metadata.json", + 2, + ) + .expect("catalog-assigned metadata IDs should apply"); + + assert_eq!(updated["schemas"][1]["schema-id"], 1); + assert_eq!(updated["current-schema-id"], 1); + assert_eq!(updated["partition-specs"][1]["spec-id"], 1); + assert_eq!(updated["partition-specs"][1]["fields"][0]["field-id"], 1000); + assert_eq!(updated["partition-specs"][1]["fields"][1]["field-id"], 1001); + assert_eq!(updated["default-spec-id"], 1); + assert_eq!(updated["last-partition-id"], 1001); + assert_eq!(updated["sort-orders"].as_array().map(Vec::len), Some(2)); + assert_eq!(updated["sort-orders"][0]["order-id"], 1); + assert_eq!(updated["sort-orders"][1]["order-id"], 0); + assert_eq!(updated["default-sort-order-id"], 0); +} + +#[test] +fn standard_commit_binds_new_specs_and_sort_orders_to_current_schema() { + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let request: CreateTableRequest = serde_json::from_value(serde_json::json!({ + "name": "events", + "schema": { + "type": "struct", + "schema-id": 0, + "fields": [{"id": 1, "name": "id", "required": true, "type": "long"}] + }, + "properties": {} + })) + .expect("create table request should parse"); + let (_, metadata) = + table_entry_from_create_table_request("warehouse", &namespace, request).expect("table metadata should be created"); + let schema_updates = [ + serde_json::json!({ + "action": "add-schema", + "schema": {"type": "struct", "schema-id": 1, "fields": []} + }), + serde_json::json!({"action": "set-current-schema", "schema-id": -1}), + ]; + + let mut spec_updates = schema_updates.to_vec(); + spec_updates.push(serde_json::json!({ + "action": "add-spec", + "spec": { + "spec-id": 1, + "fields": [{"source-id": 1, "name": "id", "transform": "identity"}] + } + })); + apply_table_commit_updates_at( + metadata.clone(), + &spec_updates, + "s3://warehouse/tables/table-id/metadata/v1.metadata.json", + 2, + ) + .expect_err("a new partition spec must bind to the current schema"); + spec_updates[2]["spec"]["fields"][0]["transform"] = serde_json::Value::from("void"); + apply_table_commit_updates_at( + metadata.clone(), + &spec_updates, + "s3://warehouse/tables/table-id/metadata/v1.metadata.json", + 2, + ) + .expect("a void partition field may retain a source removed from the current schema"); + + let mut sort_updates = schema_updates.to_vec(); + sort_updates.push(serde_json::json!({ + "action": "add-sort-order", + "sort-order": { + "order-id": 1, + "fields": [{ + "source-id": 1, + "transform": "identity", + "direction": "asc", + "null-order": "nulls-first" + }] + } + })); + apply_table_commit_updates_at(metadata, &sort_updates, "s3://warehouse/tables/table-id/metadata/v1.metadata.json", 2) + .expect_err("a new sort order must bind to the current schema"); + + let request: CreateTableRequest = serde_json::from_value(serde_json::json!({ + "name": "events_v1", + "schema": { + "type": "struct", + "schema-id": 0, + "fields": [{"id": 1, "name": "id", "required": true, "type": "long"}] + }, + "properties": {"format-version": "1"} + })) + .expect("v1 create table request should parse"); + let (_, v1_metadata) = + table_entry_from_create_table_request("warehouse", &namespace, request).expect("v1 table metadata should be created"); + spec_updates[2]["spec"]["fields"][0]["transform"] = serde_json::Value::from("identity"); + apply_table_commit_updates_at( + v1_metadata.clone(), + &spec_updates, + "s3://warehouse/tables/table-id/metadata/v1.metadata.json", + 2, + ) + .expect_err("a new v1 partition spec must bind to the updated current schema"); + apply_table_commit_updates_at( + v1_metadata.clone(), + &sort_updates, + "s3://warehouse/tables/table-id/metadata/v1.metadata.json", + 2, + ) + .expect_err("a new v1 sort order must bind to the updated current schema"); + + let mut singular_v1_metadata = v1_metadata.clone(); + let singular_v1_object = singular_v1_metadata + .as_object_mut() + .expect("v1 table metadata should be an object"); + for field in [ + "schemas", + "current-schema-id", + "partition-specs", + "default-spec-id", + "last-partition-id", + "sort-orders", + "default-sort-order-id", + ] { + singular_v1_object.remove(field); + } + let singular_v1_updates = [ + serde_json::json!({ + "action": "add-schema", + "schema": { + "type": "struct", + "schema-id": 1, + "fields": [{"id": 2, "name": "category", "required": true, "type": "string"}] + } + }), + serde_json::json!({ + "action": "add-spec", + "spec": { + "spec-id": 1, + "fields": [{"source-id": 1, "name": "id", "transform": "identity"}] + } + }), + ]; + let singular_v1_updated = apply_table_commit_updates_at( + singular_v1_metadata, + &singular_v1_updates, + "s3://warehouse/tables/table-id/metadata/v1.metadata.json", + 2, + ) + .expect("a singular v1 table may add schema history without changing its current schema"); + assert_eq!(singular_v1_updated["current-schema-id"], 0); + assert_eq!(singular_v1_updated["schema"]["schema-id"], 0); + + let valid_v1_updates = [ + serde_json::json!({ + "action": "add-schema", + "schema": { + "type": "struct", + "schema-id": 1, + "fields": [{"id": 2, "name": "category", "required": true, "type": "string"}] + } + }), + serde_json::json!({"action": "set-current-schema", "schema-id": -1}), + serde_json::json!({ + "action": "add-spec", + "spec": { + "spec-id": 1, + "fields": [{"source-id": 2, "name": "category", "transform": "identity"}] + } + }), + ]; + apply_table_commit_updates_at( + v1_metadata, + &valid_v1_updates, + "s3://warehouse/tables/table-id/metadata/v1.metadata.json", + 2, + ) + .expect("a new v1 partition spec may bind to a field in the updated current schema"); +} + +#[test] +fn last_added_table_ids_require_a_preceding_add_update() { + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let request: CreateTableRequest = serde_json::from_value(serde_json::json!({ + "name": "events", + "schema": { + "type": "struct", + "schema-id": 0, + "fields": [{"id": 1, "name": "id", "required": true, "type": "long"}] + }, + "properties": {} + })) + .expect("create table request should parse"); + let (_, metadata) = + table_entry_from_create_table_request("warehouse", &namespace, request).expect("table metadata should be created"); + let invalid_update_sequences = [ + vec![ + serde_json::json!({"action": "set-current-schema", "schema-id": -1}), + serde_json::json!({ + "action": "add-schema", + "schema": {"type": "struct", "schema-id": 1, "fields": []} + }), + ], + vec![ + serde_json::json!({"action": "set-default-spec", "spec-id": -1}), + serde_json::json!({ + "action": "add-spec", + "spec": {"spec-id": 1, "fields": []} + }), + ], + vec![ + serde_json::json!({"action": "set-default-sort-order", "sort-order-id": -1}), + serde_json::json!({ + "action": "add-sort-order", + "sort-order": {"order-id": 1, "fields": []} + }), + ], + ]; + + for updates in invalid_update_sequences { + let error = apply_table_commit_updates_at( + metadata.clone(), + &updates, + "s3://warehouse/tables/table-id/metadata/v1.metadata.json", + 2, + ) + .expect_err("a later add update must not satisfy an earlier -1 reference"); + assert_eq!(error.code(), &S3ErrorCode::InvalidRequest); + assert!( + error + .message() + .is_some_and(|message| message.contains("requires a preceding")) + ); + } +} + +#[test] +fn create_table_counts_collection_ids_in_last_column_id() { + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let request: CreateTableRequest = serde_json::from_value(serde_json::json!({ + "name": "events", + "schema": { + "type": "struct", + "schema-id": 0, + "fields": [ + { + "id": 1, + "name": "items", + "required": true, + "type": { + "type": "list", + "element-id": 7, + "element-required": true, + "element": "long" + } + }, + { + "id": 2, + "name": "lookup", + "required": true, + "type": { + "type": "map", + "key-id": 8, + "key": "string", + "value-id": 9, + "value-required": false, + "value": "string" + } + } + ] + } + })) + .expect("create table request should parse"); + + let (_, metadata) = + table_entry_from_create_table_request("warehouse", &namespace, request).expect("table metadata should be created"); + + assert_eq!(metadata["last-column-id"], 9); +} + #[test] fn commit_table_request_accepts_standard_iceberg_rest_shape() { let request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ @@ -1956,6 +2273,86 @@ fn commit_table_request_accepts_standard_iceberg_rest_shape() { assert_eq!(request.requirements.len(), 1); } +#[tokio::test] +async fn commit_request_readers_require_standard_arrays_and_preserve_legacy_pointer_shapes() { + let table_error = read_rest_commit_table_request(Body::from("{}".to_string())) + .await + .expect_err("standard table commits must include requirements and updates"); + assert_eq!(table_error.code(), &S3ErrorCode::InvalidRequest); + + let table = read_rest_commit_table_request(Body::from(r#"{"requirements":[],"updates":[]}"#.to_string())) + .await + .expect("standard table commits may provide empty requirements and updates"); + assert!(table.requirements.is_empty()); + assert!(table.updates.is_empty()); + + let legacy_table = read_rest_commit_table_request(Body::from( + r#"{"expected-version-token":"token-v1","expected-metadata-location":"s3://warehouse/tables/table-id/metadata/v1.metadata.json","new-metadata-location":"s3://warehouse/tables/table-id/metadata/v2.metadata.json"}"# + .to_string(), + )) + .await + .expect("legacy pointer commits may omit standard arrays"); + assert_eq!( + legacy_table.new_metadata_location.as_deref(), + Some("s3://warehouse/tables/table-id/metadata/v2.metadata.json") + ); + + let mixed_table = read_rest_commit_table_request(Body::from( + r#"{"expected-version-token":"token-v1","new-metadata-location":"s3://warehouse/tables/table-id/metadata/v2.metadata.json","requirements":[],"updates":[{"action":"set-properties","updates":{"owner":"lakehouse"}}]}"# + .to_string(), + )) + .await + .expect_err("legacy pointer commits must not silently discard standard updates"); + assert_eq!(mixed_table.code(), &S3ErrorCode::InvalidRequest); + + let view_error = read_rest_commit_view_request(Body::from("{}".to_string())) + .await + .expect_err("standard view commits must include updates"); + assert_eq!(view_error.code(), &S3ErrorCode::InvalidRequest); + + let view = read_rest_commit_view_request(Body::from(r#"{"updates":[]}"#.to_string())) + .await + .expect("standard view commits may omit requirements"); + assert!(view.requirements.is_empty()); + assert!(view.updates.is_empty()); + + let legacy_view = read_rest_commit_view_request(Body::from( + r#"{"commit-id":"legacy-view-update","new-metadata-location":"s3://warehouse/views/view-id/metadata/v2.metadata.json"}"# + .to_string(), + )) + .await + .expect("legacy view pointer commits may omit standard updates"); + assert_eq!(legacy_view._commit_id.as_deref(), Some("legacy-view-update")); +} + +#[test] +fn unsupported_create_and_register_modes_return_iceberg_errors() { + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let register_error = table_entry_from_register_request( + "warehouse", + &namespace, + RegisterTableRequest { + name: "events".to_string(), + metadata_location: "s3://warehouse/metadata/00001.metadata.json".to_string(), + overwrite: true, + }, + ) + .expect_err("register overwrite should remain unsupported"); + assert_eq!(register_error.code(), &S3ErrorCode::Custom(ICEBERG_ERROR_UNSUPPORTED_OPERATION.into())); + assert_eq!(register_error.status_code(), Some(StatusCode::NOT_ACCEPTABLE)); + + let create_request: CreateTableRequest = serde_json::from_value(serde_json::json!({ + "name": "events", + "schema": {"type": "struct", "schema-id": 0, "fields": []}, + "stage-create": true + })) + .expect("stage-create request should parse"); + let create_error = table_entry_from_create_table_request("warehouse", &namespace, create_request) + .expect_err("staged create should remain unsupported"); + assert_eq!(create_error.code(), &S3ErrorCode::Custom(ICEBERG_ERROR_UNSUPPORTED_OPERATION.into())); + assert_eq!(create_error.status_code(), Some(StatusCode::NOT_ACCEPTABLE)); +} + #[test] fn standard_commit_ids_use_uuid_for_metadata_file_when_provided() { let commit_id = "11111111-1111-4111-8111-111111111111"; @@ -1987,7 +2384,7 @@ fn format_upgrade_assigns_v1_snapshot_sequences_and_rejects_v3() { "schema-id": 0, "fields": [{"id": 1, "name": "id", "required": true, "type": "long"}] }, - "partition-spec": [], + "partition-spec": [{"source-id": 1, "name": "id", "transform": "identity"}], "snapshots": [{ "snapshot-id": 10, "timestamp-ms": 1, @@ -2005,6 +2402,8 @@ fn format_upgrade_assigns_v1_snapshot_sequences_and_rejects_v3() { .expect("upgraded metadata fields should synchronize"); assert_eq!(metadata["snapshots"][0]["sequence-number"], 0); + assert_eq!(metadata["partition-specs"][0]["fields"][0]["field-id"], 1000); + assert_eq!(metadata["last-partition-id"], 1000); crate::table_catalog::validate_supported_table_metadata(&metadata).expect("upgraded metadata should satisfy the v2 contract"); let error = apply_upgrade_format_version_update( @@ -2171,6 +2570,120 @@ async fn create_table_holds_bucket_fence_from_metadata_write_through_registratio ); } +#[tokio::test] +async fn create_view_holds_publication_fences_from_metadata_write_through_registration() { + let pause = TestCatalogPublishPause::default(); + let store = Arc::new(TestTableCatalogStore { + create_view_pause: Some(pause.clone()), + ..Default::default() + }); + let barrier = Arc::new(tokio::sync::Barrier::new(2)); + let metadata_backend = TestTableCatalogObjectBackend { + put_object_barrier: Some(Arc::clone(&barrier)), + ..Default::default() + }; + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + ensure_table_bucket_entry(store.as_ref(), "warehouse", true) + .await + .expect("table bucket entry should be seeded"); + create_namespace_response( + store.as_ref(), + "warehouse", + CreateNamespaceRequest { + namespace: vec!["analytics".to_string()], + properties: BTreeMap::new(), + }, + true, + ) + .await + .expect("namespace should be created"); + let request = serde_json::from_value::(serde_json::json!({ + "name": "recent_events", + "schema": {"type": "struct", "fields": []}, + "view-version": { + "version-id": 1, + "timestamp-ms": 1, + "schema-id": 0, + "summary": {}, + "default-namespace": ["analytics"], + "representations": [{"type": "sql", "sql": "SELECT 1", "dialect": "spark"}] + } + })) + .expect("create view request should parse"); + + let create_store = Arc::clone(&store); + let create_backend = TableCommitObjectBackend::trusted(metadata_backend.clone()); + let create_namespace = namespace.clone(); + let create = tokio::spawn(async move { + create_view_response(create_store.as_ref(), &create_backend, "warehouse", &create_namespace, request, true).await + }); + tokio::time::timeout(StdDuration::from_secs(2), async { + while metadata_backend.state.lock().await.objects.is_empty() { + tokio::task::yield_now().await; + } + }) + .await + .expect("initial view metadata write should reach its publication pause"); + + let bucket_lock = crate::table_catalog::default_table_bucket_publication_lock_path(); + assert!( + metadata_backend.write_lock_is_held("warehouse", &bucket_lock).await, + "view publication must fence data-plane writers before initial metadata is visible" + ); + barrier.wait().await; + tokio::time::timeout(StdDuration::from_secs(2), pause.wait_started()) + .await + .expect("view creation should reach catalog publication"); + + let view_name = crate::table_catalog::IdentifierSegment::parse("recent_events").expect("view should parse"); + let view_lock = crate::table_catalog::default_table_publication_lock_path(&namespace, &view_name); + assert!( + metadata_backend.write_lock_is_held("warehouse", &bucket_lock).await, + "view creation must retain the bucket fence until catalog publication" + ); + assert!( + metadata_backend.write_lock_is_held("warehouse", &view_lock).await, + "view creation must hold the view publication fence before registration" + ); + assert!( + store + .load_view("warehouse", "analytics", "recent_events") + .await + .expect("view lookup should succeed") + .is_none(), + "the view must remain invisible before catalog publication" + ); + + metadata_backend.lock_attempts.lock().await.clear(); + let writer_backend = metadata_backend.clone(); + let writer_lock = bucket_lock.clone(); + let writer = tokio::spawn(async move { + crate::table_catalog::TableCatalogObjectBackend::acquire_read_lock(&writer_backend, "warehouse", &writer_lock).await + }); + metadata_backend.wait_for_lock_attempts(1).await; + assert!(!writer.is_finished(), "a data-plane writer must wait for view registration"); + + pause.release(); + tokio::time::timeout(StdDuration::from_secs(2), create) + .await + .expect("view creation should complete") + .expect("view creation task should join") + .expect("view creation should succeed"); + tokio::time::timeout(StdDuration::from_secs(2), writer) + .await + .expect("writer should continue after view registration") + .expect("writer task should join") + .expect("writer lock acquisition should succeed"); + assert!( + store + .load_view("warehouse", "analytics", "recent_events") + .await + .expect("view lookup should succeed") + .is_some(), + "the view must become visible after catalog publication" + ); +} + #[tokio::test] async fn create_table_response_recreates_dropped_identifier_without_overwriting_retained_metadata() { let metadata_backend = TestTableCatalogObjectBackend::content_addressed(); @@ -2189,6 +2702,7 @@ async fn create_table_response_recreates_dropped_identifier_without_overwriting_ .expect("first metadata should exist"); let commit_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "requirements": [], "commit-id": "11111111-1111-4111-8111-111111111111", "updates": [ { @@ -3199,6 +3713,7 @@ async fn standard_commit_uses_client_uuid_commit_id_in_metadata_file_name() { let commit_id = "11111111-1111-4111-8111-111111111111"; let commit_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "requirements": [], "commit-id": commit_id, "updates": [ { @@ -3247,6 +3762,7 @@ async fn standard_commit_accepts_non_uuid_client_commit_id_without_using_it_in_m create_standard_events_table(&store, &metadata_backend, &namespace).await; let commit_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "requirements": [], "commit-id": "commit-1", "updates": [ { @@ -3620,7 +4136,25 @@ async fn commit_publication_authorizes_referenced_objects() { let manifest_list = format!("{table_location}/metadata/snap-10.avro"); let manifest = format!("{table_location}/metadata/manifest-snap-10.avro"); let data_file = format!("{table_location}/data/part-10.parquet"); + let statistics_file = format!("{table_location}/metadata/stats-10.puffin"); + let partition_statistics_file = format!("{table_location}/metadata/partition-stats-10.parquet"); + let statistics_bytes = b"PFA1PFA1".to_vec(); + let partition_statistics_bytes = test_parquet_i32_bytes(&[1]); seed_test_snapshot_manifest(&metadata_backend, "warehouse", &manifest_list, 10, 1, &[(&data_file, 0, 1, 10, 1)]).await; + metadata_backend + .put_bytes( + "warehouse", + &test_snapshot_object_key("warehouse", &statistics_file), + statistics_bytes.clone(), + ) + .await; + metadata_backend + .put_bytes( + "warehouse", + &test_snapshot_object_key("warehouse", &partition_statistics_file), + partition_statistics_bytes.clone(), + ) + .await; let request = serde_json::from_value(serde_json::json!({ "commit-id": "22222222-2222-4222-8222-222222222222", "requirements": [], @@ -3640,6 +4174,24 @@ async fn commit_publication_authorizes_referenced_objects() { "ref-name": "main", "snapshot-id": 10, "type": "branch" + }, + { + "action": "set-statistics", + "statistics": { + "snapshot-id": 10, + "statistics-path": statistics_file, + "file-size-in-bytes": statistics_bytes.len(), + "file-footer-size-in-bytes": 0, + "blob-metadata": [] + } + }, + { + "action": "set-partition-statistics", + "partition-statistics": { + "snapshot-id": 10, + "statistics-path": partition_statistics_file, + "file-size-in-bytes": partition_statistics_bytes.len() + } } ] })) @@ -3659,6 +4211,8 @@ async fn commit_publication_authorizes_referenced_objects() { test_snapshot_object_key("warehouse", &manifest_list), test_snapshot_object_key("warehouse", &manifest), test_snapshot_object_key("warehouse", &data_file), + test_snapshot_object_key("warehouse", &statistics_file), + test_snapshot_object_key("warehouse", &partition_statistics_file), ]; for object in expected_reads { assert!( @@ -4316,19 +4870,16 @@ async fn standard_commit_publishes_more_than_ten_thousand_live_files() { .map(|index| format!("{table_location}/data/part-{index:05}.parquet")) .collect::>(); let manifest_files = data_files.iter().map(|file| (file.as_str(), 0, 1, 20, 1)).collect::>(); + let manifest_bytes = test_manifest_avro_bytes(&manifest_files); metadata_backend .put_bytes( "warehouse", &test_snapshot_object_key("warehouse", &manifest_list), - test_manifest_list_avro_bytes(&[&manifest], 1, 20), + test_manifest_list_avro_bytes(&[(&manifest, manifest_bytes.len())], 1, 20), ) .await; metadata_backend - .put_bytes( - "warehouse", - &test_snapshot_object_key("warehouse", &manifest), - test_manifest_avro_bytes(&manifest_files), - ) + .put_bytes("warehouse", &test_snapshot_object_key("warehouse", &manifest), manifest_bytes) .await; { let mut state = metadata_backend.state.lock().await; @@ -4437,6 +4988,7 @@ async fn standard_commit_ignores_generation_only_orphan_metadata_file() { let commit_id = "22222222-2222-4222-8222-222222222222"; let commit_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "requirements": [], "commit-id": commit_id, "updates": [ { @@ -4482,6 +5034,7 @@ async fn concurrent_standard_commits_write_distinct_metadata_files_before_pointe let first_commit_id = "33333333-3333-4333-8333-333333333333"; let second_commit_id = "44444444-4444-4444-8444-444444444444"; let first_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "requirements": [], "commit-id": first_commit_id, "updates": [ { @@ -4494,6 +5047,7 @@ async fn concurrent_standard_commits_write_distinct_metadata_files_before_pointe })) .expect("first standard commit table request should parse"); let second_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "requirements": [], "commit-id": second_commit_id, "updates": [ { @@ -4536,7 +5090,7 @@ async fn concurrent_standard_commits_write_distinct_metadata_files_before_pointe } #[tokio::test] -async fn standard_commit_accepts_legacy_catalog_uuid_when_current_metadata_matches() { +async fn standard_commit_rejects_unbound_legacy_catalog_identity() { let store = TestTableCatalogStore::default(); let metadata_backend = TestTableCatalogObjectBackend::content_addressed(); let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); @@ -4579,6 +5133,7 @@ async fn standard_commit_accepts_legacy_catalog_uuid_when_current_metadata_match .await; let commit_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "requirements": [], "updates": [ { "action": "set-properties", @@ -4589,7 +5144,7 @@ async fn standard_commit_accepts_legacy_catalog_uuid_when_current_metadata_match ] })) .expect("standard commit table request should parse"); - let committed = commit_table_response( + let error = commit_table_response( &store, &trusted_table_commit_backend(&metadata_backend), "warehouse", @@ -4598,15 +5153,15 @@ async fn standard_commit_accepts_legacy_catalog_uuid_when_current_metadata_match commit_request, ) .await - .expect("legacy catalog uuid should not block standard commit"); + .expect_err("a legacy catalog identity that does not match persisted metadata must fail closed"); - assert_eq!(committed.metadata["table-uuid"], "metadata-table-uuid"); - assert_eq!(committed.metadata["properties"]["owner"], "lakehouse"); - assert_eq!(committed.generation, legacy_entry.generation + 1); + assert_eq!(error.code(), &S3ErrorCode::Custom(ICEBERG_ERROR_REST.into())); + assert_eq!(error.status_code(), Some(StatusCode::INTERNAL_SERVER_ERROR)); + assert_events_table_entry_unchanged(&store, &legacy_entry).await; } #[tokio::test] -async fn metadata_location_api_accepts_legacy_catalog_uuid_when_target_matches_current_metadata() { +async fn metadata_location_api_rejects_unbound_legacy_catalog_identity() { let store = TestTableCatalogStore::default(); let metadata_backend = TestTableCatalogObjectBackend::content_addressed(); let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); @@ -4652,7 +5207,7 @@ async fn metadata_location_api_accepts_legacy_catalog_uuid_when_target_matches_c next_metadata["last-sequence-number"] = serde_json::Value::from(2); metadata_backend.put_json("warehouse", next_location, next_metadata).await; - let updated = update_table_metadata_location_response( + let error = update_table_metadata_location_response( &store, &trusted_table_commit_backend(&metadata_backend), "warehouse", @@ -4660,16 +5215,17 @@ async fn metadata_location_api_accepts_legacy_catalog_uuid_when_target_matches_c "events", UpdateTableMetadataLocationRequest { metadata_location: next_location.to_string(), - version_token: legacy_entry.version_token, + version_token: legacy_entry.version_token.clone(), commit_id: Some("commit-1".to_string()), idempotency_key: None, }, ) .await - .expect("legacy catalog uuid should not block metadata-location update"); + .expect_err("metadata-location updates must reject unbound legacy catalog identity"); - assert_eq!(updated.metadata_location, table_metadata_location_for_client("warehouse", next_location)); - assert_eq!(updated.generation, legacy_entry.generation + 1); + assert_eq!(error.code(), &S3ErrorCode::Custom(ICEBERG_ERROR_REST.into())); + assert_eq!(error.status_code(), Some(StatusCode::INTERNAL_SERVER_ERROR)); + assert_events_table_entry_unchanged(&store, &legacy_entry).await; } #[tokio::test] @@ -5016,18 +5572,17 @@ async fn table_metadata_maintenance_helper_commits_compaction_through_publicatio let manifest = format!("{metadata_dir}/manifest-20.avro"); let left_data = format!("{data_dir}/part-left.parquet"); let right_data = format!("{data_dir}/part-right.parquet"); + let manifest_bytes = test_manifest_avro_bytes(&[(&left_data, 0, 0, 20, 7), (&right_data, 0, 0, 20, 7)]); seed_object_table_for_metadata_maintenance(&store, &backend, bucket, &namespace, &table, current.clone()).await; - backend - .put_bytes(bucket, &manifest_list, test_manifest_list_avro_bytes(&[&manifest], 7, 20)) - .await; backend .put_bytes( bucket, - &manifest, - test_manifest_avro_bytes(&[(&left_data, 0, 0, 20, 7), (&right_data, 0, 0, 20, 7)]), + &manifest_list, + test_manifest_list_avro_bytes(&[(&manifest, manifest_bytes.len())], 7, 20), ) .await; + backend.put_bytes(bucket, &manifest, manifest_bytes).await; backend.put_bytes(bucket, &left_data, test_parquet_i32_bytes(&[1, 2])).await; backend.put_bytes(bucket, &right_data, test_parquet_i32_bytes(&[3, 4])).await; backend @@ -5294,25 +5849,36 @@ async fn table_refs_response_reports_current_and_user_defined_refs() { let table = crate::table_catalog::IdentifierSegment::parse("events").expect("table should parse"); let current = crate::table_catalog::default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); seed_object_table_for_metadata_maintenance(&store, &backend, bucket, &namespace, &table, current.clone()).await; + let mut metadata = test_table_metadata_json("table-uuid", "s3://warehouse/tables/table-id"); + metadata["last-sequence-number"] = serde_json::Value::from(2); + metadata["snapshots"] = serde_json::json!([ + { + "snapshot-id": 9, + "sequence-number": 1, + "timestamp-ms": 1, + "manifest-list": "s3://warehouse/tables/table-id/metadata/snap-9.avro", + "summary": {"operation": "append"} + }, + { + "snapshot-id": 10, + "parent-snapshot-id": 9, + "sequence-number": 2, + "timestamp-ms": 2, + "manifest-list": "s3://warehouse/tables/table-id/metadata/snap-10.avro", + "summary": {"operation": "append"} + } + ]); + metadata["current-snapshot-id"] = serde_json::Value::from(10); + metadata["snapshot-log"] = serde_json::json!([ + {"timestamp-ms": 1, "snapshot-id": 9}, + {"timestamp-ms": 2, "snapshot-id": 10} + ]); + metadata["refs"] = serde_json::json!({ + "main": {"snapshot-id": 10, "type": "branch"}, + "audit": {"snapshot-id": 9, "type": "tag"} + }); backend - .put_json_with_mod_time( - bucket, - ¤t, - serde_json::json!({ - "current-snapshot-id": 10, - "refs": { - "main": { - "snapshot-id": 10, - "type": "branch" - }, - "audit": { - "snapshot-id": 9, - "type": "tag" - } - } - }), - Some(OffsetDateTime::UNIX_EPOCH), - ) + .put_json_with_mod_time(bucket, ¤t, metadata, Some(OffsetDateTime::UNIX_EPOCH)) .await; let response = table_refs_response(&store, &backend, bucket, &namespace, "events") @@ -5695,39 +6261,30 @@ async fn external_catalog_bridge_sync_conflicts_leave_pointer_unchanged() { } #[test] -fn commit_requirements_reject_mismatched_table_uuid() { +fn snapshot_conflict_requirements_validate_snapshot_ref_id() { let metadata = serde_json::json!({ - "table-uuid": "actual-table-uuid" - }); - let requirements = vec![serde_json::json!({ - "type": "assert-table-uuid", - "uuid": "stale-table-uuid" - })]; - - assert!(validate_table_commit_requirements(&metadata, &requirements).is_err()); -} - -#[test] -fn snapshot_conflict_requirements_validate_current_snapshot_id() { - let metadata = serde_json::json!({ - "current-snapshot-id": 10 + "current-snapshot-id": 10, + "refs": {"main": {"type": "branch", "snapshot-id": 10}} }); let matching = vec![serde_json::json!({ - "type": "assert-current-snapshot-id", + "type": "assert-ref-snapshot-id", + "ref": "main", "snapshot-id": 10 })]; validate_table_commit_requirements(&metadata, &matching).expect("matching current snapshot should pass"); let stale = vec![serde_json::json!({ - "type": "assert-current-snapshot-id", + "type": "assert-ref-snapshot-id", + "ref": "main", "snapshot-id": 9 })]; assert!(validate_table_commit_requirements(&metadata, &stale).is_err()); let no_snapshot_metadata = serde_json::json!({}); let create_like = vec![serde_json::json!({ - "type": "assert-current-snapshot-id", + "type": "assert-ref-snapshot-id", + "ref": "main", "snapshot-id": null })]; validate_table_commit_requirements(&no_snapshot_metadata, &create_like) @@ -5735,26 +6292,20 @@ fn snapshot_conflict_requirements_validate_current_snapshot_id() { } #[test] -fn snapshot_conflict_rejects_stale_parent_or_sequence_number() { - let metadata = serde_json::json!({ - "current-snapshot-id": 10, - "last-sequence-number": 4, - "snapshots": [ - { - "snapshot-id": 10, - "sequence-number": 4, - "timestamp-ms": 1234, - "manifest-list": "s3://warehouse/tables/table-id/metadata/snap-10.avro", - "summary": { - "operation": "append" - } - } - ], - "snapshot-log": [], - "metadata-log": [] - }); +fn snapshot_conflict_rejects_unknown_parent_or_stale_sequence_number() { + let mut metadata = test_table_metadata_json("table-uuid", "s3://warehouse/tables/table-id"); + metadata["current-snapshot-id"] = serde_json::Value::from(10); + metadata["last-sequence-number"] = serde_json::Value::from(4); + metadata["snapshots"] = serde_json::json!([{ + "snapshot-id": 10, + "sequence-number": 4, + "timestamp-ms": 1234, + "manifest-list": "s3://warehouse/tables/table-id/metadata/snap-10.avro", + "summary": {"operation": "append"} + }]); + metadata["refs"] = serde_json::json!({"main": {"snapshot-id": 10, "type": "branch"}}); - let stale_parent = vec![serde_json::json!({ + let unknown_parent = vec![serde_json::json!({ "action": "add-snapshot", "snapshot": { "snapshot-id": 11, @@ -5767,7 +6318,11 @@ fn snapshot_conflict_rejects_stale_parent_or_sequence_number() { } } })]; - assert!(apply_table_commit_updates(metadata.clone(), &stale_parent, "metadata/00001.metadata.json").is_err()); + let error = apply_table_commit_updates(metadata.clone(), &unknown_parent, "metadata/00001.metadata.json") + .expect_err("unknown snapshot parents must fail"); + assert_eq!(error.code(), &S3ErrorCode::Custom(ICEBERG_ERROR_COMMIT_FAILED.into())); + assert_eq!(error.status_code(), Some(StatusCode::CONFLICT)); + assert_eq!(error.message(), Some("snapshot parent does not exist")); let stale_sequence = vec![serde_json::json!({ "action": "add-snapshot", @@ -5782,28 +6337,341 @@ fn snapshot_conflict_rejects_stale_parent_or_sequence_number() { } } })]; - assert!(apply_table_commit_updates(metadata, &stale_sequence, "metadata/00001.metadata.json").is_err()); + let error = apply_table_commit_updates(metadata.clone(), &stale_sequence, "metadata/00001.metadata.json") + .expect_err("snapshot sequence numbers must advance"); + assert_eq!(error.code(), &S3ErrorCode::Custom(ICEBERG_ERROR_COMMIT_FAILED.into())); + assert_eq!(error.status_code(), Some(StatusCode::CONFLICT)); + assert_eq!(error.message(), Some("snapshot sequence number must advance")); + + let stale_root_sequence = vec![serde_json::json!({ + "action": "add-snapshot", + "snapshot": { + "snapshot-id": 11, + "sequence-number": 4, + "timestamp-ms": 2234, + "manifest-list": "s3://warehouse/tables/table-id/metadata/snap-11.avro", + "summary": { + "operation": "append" + } + } + })]; + let error = apply_table_commit_updates(metadata, &stale_root_sequence, "metadata/00001.metadata.json") + .expect_err("root snapshot sequence numbers must advance"); + assert_eq!(error.code(), &S3ErrorCode::Custom(ICEBERG_ERROR_COMMIT_FAILED.into())); + assert_eq!(error.status_code(), Some(StatusCode::CONFLICT)); + assert_eq!(error.message(), Some("snapshot sequence number must advance")); +} + +#[test] +fn snapshot_updates_move_only_the_declared_reference() { + let mut metadata = test_table_metadata_json("table-uuid", "s3://warehouse/tables/table-id"); + metadata["current-snapshot-id"] = serde_json::Value::from(10); + metadata["last-sequence-number"] = serde_json::Value::from(4); + metadata["snapshots"] = serde_json::json!([ + { + "snapshot-id": 9, + "sequence-number": 3, + "timestamp-ms": 1000, + "manifest-list": "s3://warehouse/tables/table-id/metadata/snap-9.avro", + "summary": {"operation": "append"} + }, + { + "snapshot-id": 10, + "parent-snapshot-id": 9, + "sequence-number": 4, + "timestamp-ms": 1234, + "manifest-list": "s3://warehouse/tables/table-id/metadata/snap-10.avro", + "summary": {"operation": "append"} + } + ]); + metadata["refs"] = serde_json::json!({"main": {"snapshot-id": 10, "type": "branch"}}); + metadata["snapshot-log"] = serde_json::json!([{"timestamp-ms": 1234, "snapshot-id": 10}]); + let add_snapshot = serde_json::json!({ + "action": "add-snapshot", + "snapshot": { + "snapshot-id": 11, + "parent-snapshot-id": 9, + "sequence-number": 5, + "timestamp-ms": 2234, + "manifest-list": "s3://warehouse/tables/table-id/metadata/snap-11.avro", + "summary": {"operation": "append"} + } + }); + + let added = + apply_table_commit_updates_at(metadata, std::slice::from_ref(&add_snapshot), "metadata/00001.metadata.json", 3000) + .expect("a snapshot may branch from any retained parent"); + assert_eq!(added["current-snapshot-id"], 10); + assert_eq!(added["refs"]["main"]["snapshot-id"], 10); + assert_eq!(added["snapshot-log"].as_array().map(Vec::len), Some(1)); + + let branch = apply_table_commit_updates_at( + added, + &[serde_json::json!({ + "action": "set-snapshot-ref", + "ref-name": "audit", + "snapshot-id": 11, + "type": "branch" + })], + "metadata/00002.metadata.json", + 3001, + ) + .expect("a non-main branch should be updated"); + assert_eq!(branch["current-snapshot-id"], 10); + assert_eq!(branch["refs"]["main"]["snapshot-id"], 10); + assert_eq!(branch["refs"]["audit"]["snapshot-id"], 11); + assert_eq!(branch["snapshot-log"].as_array().map(Vec::len), Some(1)); + + let main = apply_table_commit_updates_at( + branch, + &[serde_json::json!({ + "action": "set-snapshot-ref", + "ref-name": "main", + "snapshot-id": 11, + "type": "branch" + })], + "metadata/00003.metadata.json", + 3002, + ) + .expect("main should move to a retained snapshot"); + assert_eq!(main["current-snapshot-id"], 11); + assert_eq!(main["snapshot-log"].as_array().map(Vec::len), Some(2)); + assert_eq!(main["snapshot-log"][1], serde_json::json!({"timestamp-ms": 3002, "snapshot-id": 11})); + + let unchanged = apply_table_commit_updates_at( + main, + &[serde_json::json!({ + "action": "set-snapshot-ref", + "ref-name": "main", + "snapshot-id": 11, + "type": "branch" + })], + "metadata/00004.metadata.json", + 3003, + ) + .expect("replaying an unchanged main reference should be a no-op for snapshot history"); + assert_eq!(unchanged["snapshot-log"].as_array().map(Vec::len), Some(2)); +} + +#[test] +fn newly_added_main_snapshot_uses_its_snapshot_timestamp_in_history() { + let mut metadata = test_table_metadata_json("table-uuid", "s3://warehouse/tables/table-id"); + metadata["current-snapshot-id"] = serde_json::Value::from(-1); + let updated = apply_table_commit_updates_at( + metadata, + &[ + serde_json::json!({ + "action": "add-snapshot", + "snapshot": { + "snapshot-id": 11, + "sequence-number": 1, + "timestamp-ms": 2234, + "manifest-list": "s3://warehouse/tables/table-id/metadata/snap-11.avro", + "summary": {"operation": "append"} + } + }), + serde_json::json!({ + "action": "set-snapshot-ref", + "ref-name": "main", + "snapshot-id": 11, + "type": "branch" + }), + ], + "metadata/00001.metadata.json", + 3000, + ) + .expect("new snapshot and main reference should apply"); + + assert_eq!(updated["current-snapshot-id"], 11); + assert_eq!(updated["snapshot-log"], serde_json::json!([{"timestamp-ms": 2234, "snapshot-id": 11}])); +} + +#[test] +fn only_v1_snapshots_may_omit_sequence_number() { + let v1 = serde_json::json!({ + "format-version": 1, + "table-uuid": "table-uuid", + "location": "s3://warehouse/tables/table-id", + "last-updated-ms": 1, + "last-column-id": 1, + "schema": { + "type": "struct", + "schema-id": 0, + "fields": [{"id": 1, "name": "id", "required": true, "type": "long"}] + }, + "partition-spec": [], + "properties": {}, + "snapshots": [], + "snapshot-log": [], + "metadata-log": [] + }); + let v1_updated = apply_table_commit_updates_at( + v1, + &[serde_json::json!({ + "action": "add-snapshot", + "snapshot": { + "snapshot-id": 10, + "timestamp-ms": 2234, + "manifests": ["s3://warehouse/tables/table-id/metadata/manifest-10.avro"], + "summary": {"operation": "append"} + } + })], + "metadata/00001.metadata.json", + 3000, + ) + .expect("an Iceberg v1 zero sequence snapshot may omit sequence-number"); + assert!(v1_updated["snapshots"][0].get("sequence-number").is_none()); + assert!(v1_updated.get("last-sequence-number").is_none()); + + let v2 = test_table_metadata_json("table-uuid", "s3://warehouse/tables/table-id"); + let v2_error = apply_table_commit_updates_at( + v2, + &[serde_json::json!({ + "action": "add-snapshot", + "snapshot": { + "snapshot-id": 10, + "timestamp-ms": 2234, + "manifest-list": "s3://warehouse/tables/table-id/metadata/snap-10.avro", + "summary": {"operation": "append"} + } + })], + "metadata/00001.metadata.json", + 3000, + ) + .expect_err("new Iceberg v2 snapshots must include sequence-number"); + assert_eq!(v2_error.code(), &S3ErrorCode::InvalidRequest); + assert_eq!(v2_error.message(), Some("Iceberg v2 snapshot sequence-number is required")); +} + +#[test] +fn snapshot_updates_reject_non_integer_parent_ids() { + let metadata = test_table_metadata_json("table-uuid", "s3://warehouse/tables/table-id"); + let error = apply_table_commit_updates_at( + metadata, + &[serde_json::json!({ + "action": "add-snapshot", + "snapshot": { + "snapshot-id": 10, + "parent-snapshot-id": "invalid", + "sequence-number": 1, + "timestamp-ms": 2234, + "manifest-list": "s3://warehouse/tables/table-id/metadata/snap-10.avro", + "summary": {"operation": "append"} + } + })], + "metadata/00001.metadata.json", + 3000, + ) + .expect_err("snapshot parent IDs must be integers"); + + assert_eq!(error.code(), &S3ErrorCode::InvalidRequest); +} + +#[tokio::test] +async fn standard_commit_accepts_multiple_ordered_snapshots() { + let store = TestTableCatalogStore::default(); + let metadata_backend = TestTableCatalogObjectBackend::default(); + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let created = create_standard_events_table(&store, &metadata_backend, &namespace).await; + let table_location = created.metadata["location"] + .as_str() + .expect("created metadata should have table location"); + let first_manifest_list = format!("{table_location}/metadata/snap-10.avro"); + let first_manifest = format!("{table_location}/metadata/manifest-snap-10.avro"); + let first_data_file = format!("{table_location}/data/part-10.parquet"); + seed_test_snapshot_manifest( + &metadata_backend, + "warehouse", + &first_manifest_list, + 10, + 1, + &[(&first_data_file, 0, 1, 10, 1)], + ) + .await; + let second_manifest_list = format!("{table_location}/metadata/snap-11.avro"); + let second_manifest = format!("{table_location}/metadata/manifest-11.avro"); + let second_data_file = format!("{table_location}/data/part-11.parquet"); + seed_test_manifest(&metadata_backend, "warehouse", &second_manifest, &[(&second_data_file, 0, 1, 11, 2)]).await; + seed_test_manifest_list_entries( + &metadata_backend, + "warehouse", + &second_manifest_list, + &[(&first_manifest, 1, 10), (&second_manifest, 2, 11)], + ) + .await; + let request = serde_json::from_value(serde_json::json!({ + "requirements": [], + "updates": [ + { + "action": "add-snapshot", + "snapshot": { + "snapshot-id": 10, + "sequence-number": 1, + "timestamp-ms": 1234, + "manifest-list": first_manifest_list, + "summary": {"operation": "append"} + } + }, + { + "action": "set-snapshot-ref", + "ref-name": "main", + "snapshot-id": 10, + "type": "branch" + }, + { + "action": "add-snapshot", + "snapshot": { + "snapshot-id": 11, + "parent-snapshot-id": 10, + "sequence-number": 2, + "timestamp-ms": 2234, + "manifest-list": second_manifest_list, + "summary": {"operation": "append"} + } + }, + { + "action": "set-snapshot-ref", + "ref-name": "main", + "snapshot-id": 11, + "type": "branch" + } + ] + })) + .expect("multi-snapshot commit request should parse"); + + let committed = standard_commit_table_response( + &store, + &trusted_table_commit_backend(&metadata_backend), + "warehouse", + &namespace, + "events", + request, + ) + .await + .expect("ordered intermediate snapshots should commit"); + + assert_eq!(committed.metadata["snapshots"].as_array().map(Vec::len), Some(2)); + assert_eq!(committed.metadata["current-snapshot-id"], 11); + assert_eq!(committed.metadata["last-sequence-number"], 2); + assert_eq!( + committed.metadata["snapshot-log"], + serde_json::json!([{"timestamp-ms": 2234, "snapshot-id": 11}]) + ); } #[test] fn snapshot_conflict_rejects_unknown_snapshot_operations() { - let metadata = serde_json::json!({ - "current-snapshot-id": 10, - "last-sequence-number": 4, - "snapshots": [ - { - "snapshot-id": 10, - "sequence-number": 4, - "timestamp-ms": 1234, - "manifest-list": "s3://warehouse/tables/table-id/metadata/snap-10.avro", - "summary": { - "operation": "append" - } - } - ], - "snapshot-log": [], - "metadata-log": [] - }); + let mut metadata = test_table_metadata_json("table-uuid", "s3://warehouse/tables/table-id"); + metadata["current-snapshot-id"] = serde_json::Value::from(10); + metadata["last-sequence-number"] = serde_json::Value::from(4); + metadata["snapshots"] = serde_json::json!([{ + "snapshot-id": 10, + "sequence-number": 4, + "timestamp-ms": 1234, + "manifest-list": "s3://warehouse/tables/table-id/metadata/snap-10.avro", + "summary": {"operation": "append"} + }]); + metadata["refs"] = serde_json::json!({"main": {"snapshot-id": 10, "type": "branch"}}); let updates = vec![serde_json::json!({ "action": "add-snapshot", @@ -5818,7 +6686,10 @@ fn snapshot_conflict_rejects_unknown_snapshot_operations() { } } })]; - assert!(apply_table_commit_updates(metadata, &updates, "metadata/00001.metadata.json").is_err()); + let error = apply_table_commit_updates(metadata, &updates, "metadata/00001.metadata.json") + .expect_err("unknown snapshot operations must fail"); + assert_eq!(error.code(), &S3ErrorCode::InvalidRequest); + assert_eq!(error.message(), Some("unsupported snapshot operation: unknown")); } #[tokio::test] @@ -5842,6 +6713,7 @@ async fn row_level_conflict_allows_overwrite_when_deleted_file_is_current() { ) .await; let append_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "requirements": [], "updates": [ { "action": "add-snapshot", @@ -5889,7 +6761,8 @@ async fn row_level_conflict_allows_overwrite_when_deleted_file_is_current() { let overwrite_request_json = serde_json::json!({ "requirements": [ { - "type": "assert-current-snapshot-id", + "type": "assert-ref-snapshot-id", + "ref": "main", "snapshot-id": 10 } ], @@ -5958,7 +6831,7 @@ async fn row_level_conflict_allows_overwrite_when_deleted_file_is_current() { } #[tokio::test] -async fn row_level_conflict_allows_v1_manifest_snapshot() { +async fn row_level_conflict_rejects_embedded_manifests_for_v2_snapshot() { let store = TestTableCatalogStore::default(); let metadata_backend = TestTableCatalogObjectBackend::content_addressed(); let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); @@ -5967,9 +6840,13 @@ async fn row_level_conflict_allows_v1_manifest_snapshot() { .as_str() .expect("created metadata should have table location"); let manifest = format!("{table_location}/metadata/manifest-10.avro"); - let data_file = format!("{table_location}/data/part-10.parquet"); - seed_test_manifest(&metadata_backend, "warehouse", &manifest, &[(&data_file, 0, 1, 10, 1)]).await; + let current = store + .load_table("warehouse", "analytics", "events") + .await + .expect("table lookup should succeed") + .expect("table should exist"); let append_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "requirements": [], "updates": [ { "action": "add-snapshot", @@ -5995,7 +6872,7 @@ async fn row_level_conflict_allows_v1_manifest_snapshot() { })) .expect("append request should parse"); - let commit = commit_table_response( + let error = commit_table_response( &store, &trusted_table_commit_backend(&metadata_backend), "warehouse", @@ -6004,67 +6881,17 @@ async fn row_level_conflict_allows_v1_manifest_snapshot() { append_request, ) .await - .expect("v1 manifests snapshot should commit"); + .expect_err("new v2 snapshots must use a manifest list"); - assert_eq!(commit.metadata["current-snapshot-id"], 10); - assert_eq!(commit.metadata["last-sequence-number"], 1); - - let second_manifest_list = format!("{table_location}/metadata/snap-11.avro"); - let second_manifest = format!("{table_location}/metadata/manifest-11.avro"); - let second_data_file = format!("{table_location}/data/part-11.parquet"); - let second_manifest_list_key = test_snapshot_object_key("warehouse", &second_manifest_list); - metadata_backend - .put_bytes( - "warehouse", - &second_manifest_list_key, - test_manifest_list_avro_entries(&[(&manifest, 1, 10), (&second_manifest, 2, 11)]), - ) - .await; - seed_test_manifest(&metadata_backend, "warehouse", &second_manifest, &[(&second_data_file, 0, 1, 11, 2)]).await; - let second_append: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ - "requirements": [ - { - "type": "assert-current-snapshot-id", - "snapshot-id": 10 - } - ], - "updates": [ - { - "action": "add-snapshot", - "snapshot": { - "snapshot-id": 11, - "parent-snapshot-id": 10, - "sequence-number": 2, - "timestamp-ms": 2234, - "manifest-list": second_manifest_list, - "summary": { - "operation": "append" - } - } - }, - { - "action": "set-snapshot-ref", - "ref-name": "main", - "snapshot-id": 11, - "type": "branch" - } - ] - })) - .expect("second append request should parse"); - - let upgraded = commit_table_response( - &store, - &trusted_table_commit_backend(&metadata_backend), - "warehouse", - &namespace, - "events", - second_append, - ) - .await - .expect("manifest-list snapshot should inherit a legacy manifest with unknown provenance"); - - assert_eq!(upgraded.metadata["current-snapshot-id"], 11); - assert_eq!(upgraded.metadata["last-sequence-number"], 2); + assert_eq!(error.code(), &s3s::S3ErrorCode::InvalidRequest); + let unchanged = store + .load_table("warehouse", "analytics", "events") + .await + .expect("table lookup should succeed") + .expect("table should still exist"); + assert_eq!(unchanged.metadata_location, current.metadata_location); + assert_eq!(unchanged.version_token, current.version_token); + assert_eq!(unchanged.generation, current.generation); } #[tokio::test] @@ -6079,9 +6906,10 @@ async fn row_level_conflict_inherits_manifest_list_sequence_numbers() { let manifest_list = format!("{table_location}/metadata/snap-10.avro"); let manifest = format!("{table_location}/metadata/manifest-10.avro"); let data_file = format!("{table_location}/data/part-10.parquet"); - seed_test_manifest_list(&metadata_backend, "warehouse", &manifest_list, &[&manifest], 1, 10).await; seed_test_manifest_with_nullable_sequences(&metadata_backend, "warehouse", &manifest, &[(&data_file, 0, 1, 10, None)]).await; + seed_test_manifest_list(&metadata_backend, "warehouse", &manifest_list, &[&manifest], 1, 10).await; let append_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "requirements": [], "updates": [ { "action": "add-snapshot", @@ -6132,9 +6960,10 @@ async fn row_level_conflict_allows_inherited_manifests_on_append() { let first_manifest_list = format!("{table_location}/metadata/snap-10.avro"); let first_manifest = format!("{table_location}/metadata/manifest-10.avro"); let first_data_file = format!("{table_location}/data/part-10.parquet"); - seed_test_manifest_list(&metadata_backend, "warehouse", &first_manifest_list, &[&first_manifest], 1, 10).await; seed_test_manifest(&metadata_backend, "warehouse", &first_manifest, &[(&first_data_file, 0, 1, 10, 1)]).await; + seed_test_manifest_list(&metadata_backend, "warehouse", &first_manifest_list, &[&first_manifest], 1, 10).await; let first_append: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "requirements": [], "updates": [ { "action": "add-snapshot", @@ -6171,19 +7000,19 @@ async fn row_level_conflict_allows_inherited_manifests_on_append() { let second_manifest_list = format!("{table_location}/metadata/snap-11.avro"); let second_manifest = format!("{table_location}/metadata/manifest-11.avro"); let second_data_file = format!("{table_location}/data/part-11.parquet"); - let second_manifest_list_key = test_snapshot_object_key("warehouse", &second_manifest_list); - metadata_backend - .put_bytes( - "warehouse", - &second_manifest_list_key, - test_manifest_list_avro_entries(&[(&first_manifest, 1, 10), (&second_manifest, 2, 11)]), - ) - .await; seed_test_manifest(&metadata_backend, "warehouse", &second_manifest, &[(&second_data_file, 0, 1, 11, 2)]).await; + seed_test_manifest_list_entries( + &metadata_backend, + "warehouse", + &second_manifest_list, + &[(&first_manifest, 1, 10), (&second_manifest, 2, 11)], + ) + .await; let second_append: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ "requirements": [ { - "type": "assert-current-snapshot-id", + "type": "assert-ref-snapshot-id", + "ref": "main", "snapshot-id": 10 } ], @@ -6238,9 +7067,10 @@ async fn row_level_conflict_rejects_changed_inherited_manifest_identity() { let first_manifest_list = format!("{table_location}/metadata/snap-10.avro"); let first_manifest = format!("{table_location}/metadata/manifest-10.avro"); let first_data_file = format!("{table_location}/data/part-10.parquet"); - seed_test_manifest_list(&metadata_backend, "warehouse", &first_manifest_list, &[&first_manifest], 1, 10).await; seed_test_manifest(&metadata_backend, "warehouse", &first_manifest, &[(&first_data_file, 0, 1, 10, 1)]).await; + seed_test_manifest_list(&metadata_backend, "warehouse", &first_manifest_list, &[&first_manifest], 1, 10).await; let first_append: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "requirements": [], "updates": [ { "action": "add-snapshot", @@ -6284,7 +7114,8 @@ async fn row_level_conflict_rejects_changed_inherited_manifest_identity() { let second_append: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ "requirements": [ { - "type": "assert-current-snapshot-id", + "type": "assert-ref-snapshot-id", + "ref": "main", "snapshot-id": 10 } ], @@ -6328,10 +7159,7 @@ async fn row_level_conflict_rejects_changed_inherited_manifest_identity() { assert_eq!(unchanged.generation, current.generation); } -/// Table-driven fold of the three commit-rejection cases whose bodies were -/// identical apart from four literals (backlog#1837 PR3). Each row keeps its -/// original manifest-list sequence, data-file name, manifest-entry snapshot -/// id, and failure message, so no poison combination is lost. +/// Table-driven coverage for stale or historical manifest sequence failures. #[tokio::test] async fn row_level_conflict_rejects_stale_or_historical_manifest_sequences() { // (case, manifest-list sequence, data-file suffix, manifest-entry snapshot id, expected failure) @@ -6375,8 +7203,8 @@ async fn row_level_conflict_rejects_stale_or_historical_manifest_sequences() { let manifest_list = format!("{table_location}/metadata/snap-11.avro"); let manifest = format!("{table_location}/metadata/manifest-11.avro"); let data_file = format!("{table_location}/data/part-{data_file_suffix}.parquet"); - seed_test_manifest_list(&metadata_backend, "warehouse", &manifest_list, &[&manifest], *manifest_list_sequence, 11).await; seed_test_manifest(&metadata_backend, "warehouse", &manifest, &[(&data_file, 0, 1, *entry_snapshot_id, 1)]).await; + seed_test_manifest_list(&metadata_backend, "warehouse", &manifest_list, &[&manifest], *manifest_list_sequence, 11).await; let append_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ "updates": [ { @@ -6441,6 +7269,7 @@ async fn row_level_conflict_allows_add_only_overwrite_snapshot() { ) .await; let append_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "requirements": [], "updates": [ { "action": "add-snapshot", @@ -6488,7 +7317,8 @@ async fn row_level_conflict_allows_add_only_overwrite_snapshot() { let overwrite_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ "requirements": [ { - "type": "assert-current-snapshot-id", + "type": "assert-ref-snapshot-id", + "ref": "main", "snapshot-id": 10 } ], @@ -6552,6 +7382,7 @@ async fn row_level_conflict_rejects_delete_of_non_current_file() { ) .await; let append_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "requirements": [], "updates": [ { "action": "add-snapshot", @@ -6606,7 +7437,8 @@ async fn row_level_conflict_rejects_delete_of_non_current_file() { let overwrite_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ "requirements": [ { - "type": "assert-current-snapshot-id", + "type": "assert-ref-snapshot-id", + "ref": "main", "snapshot-id": 10 } ], @@ -6639,7 +7471,8 @@ async fn row_level_conflict_rejects_delete_of_non_current_file() { .await .expect_err("stale row-level delete should conflict"); - assert_eq!(error.code(), &s3s::S3ErrorCode::PreconditionFailed); + assert_eq!(error.code(), &s3s::S3ErrorCode::Custom(ICEBERG_ERROR_COMMIT_FAILED.into())); + assert_eq!(error.status_code(), Some(StatusCode::CONFLICT)); let unchanged = store .load_table("warehouse", "analytics", "events") .await @@ -6668,6 +7501,7 @@ async fn row_level_conflict_rejects_append_with_delete_files() { let delete_file = format!("{table_location}/delete/delete-10.parquet"); seed_test_snapshot_manifest(&metadata_backend, "warehouse", &manifest_list, 10, 1, &[(&delete_file, 1, 1, 10, 1)]).await; let append_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "requirements": [], "updates": [ { "action": "add-snapshot", @@ -6728,6 +7562,7 @@ async fn row_level_conflict_rejects_missing_manifest_before_pointer_update() { ) .await; let append_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "requirements": [], "updates": [ { "action": "add-snapshot", @@ -6769,7 +7604,8 @@ async fn row_level_conflict_rejects_missing_manifest_before_pointer_update() { let overwrite_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ "requirements": [ { - "type": "assert-current-snapshot-id", + "type": "assert-ref-snapshot-id", + "ref": "main", "snapshot-id": 10 } ], @@ -6834,6 +7670,7 @@ async fn row_level_conflict_rejects_manifest_outside_table_warehouse() { ) .await; let append_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "requirements": [], "updates": [ { "action": "add-snapshot", @@ -6875,7 +7712,8 @@ async fn row_level_conflict_rejects_manifest_outside_table_warehouse() { let overwrite_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ "requirements": [ { - "type": "assert-current-snapshot-id", + "type": "assert-ref-snapshot-id", + "ref": "main", "snapshot-id": 10 } ], @@ -6919,6 +7757,110 @@ async fn row_level_conflict_rejects_manifest_outside_table_warehouse() { assert_eq!(unchanged.generation, committed.generation); } +#[tokio::test] +async fn statistics_updates_reject_unpublished_objects_before_pointer_update() { + let store = TestTableCatalogStore::default(); + let metadata_backend = TestTableCatalogObjectBackend::default(); + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let created = create_standard_events_table(&store, &metadata_backend, &namespace).await; + let table_location = created.metadata["location"] + .as_str() + .expect("created metadata should have table location"); + let manifest_list = format!("{table_location}/metadata/snap-10.avro"); + let data_file = format!("{table_location}/data/part-10.parquet"); + seed_test_snapshot_manifest(&metadata_backend, "warehouse", &manifest_list, 10, 1, &[(&data_file, 0, 1, 10, 1)]).await; + let append_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "requirements": [], + "updates": [ + { + "action": "add-snapshot", + "snapshot": { + "snapshot-id": 10, + "sequence-number": 1, + "timestamp-ms": 1234, + "manifest-list": manifest_list, + "summary": {"operation": "append"} + } + }, + { + "action": "set-snapshot-ref", + "ref-name": "main", + "snapshot-id": 10, + "type": "branch" + } + ] + })) + .expect("append request should parse"); + commit_table_response( + &store, + &trusted_table_commit_backend(&metadata_backend), + "warehouse", + &namespace, + "events", + append_request, + ) + .await + .expect("append commit should succeed"); + let committed = store + .load_table("warehouse", "analytics", "events") + .await + .expect("table lookup should succeed") + .expect("table should exist"); + let outside_statistics_file = "s3://warehouse/tables/other-table/metadata/stats-10.puffin"; + metadata_backend + .put_bytes( + "warehouse", + &test_snapshot_object_key("warehouse", outside_statistics_file), + b"outside-stats".to_vec(), + ) + .await; + + for (commit_id, statistics_file) in [ + ( + "55555555-5555-4555-8555-555555555551", + format!("{table_location}/metadata/missing-stats-10.puffin"), + ), + ("55555555-5555-4555-8555-555555555552", outside_statistics_file.to_string()), + ] { + let request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "commit-id": commit_id, + "requirements": [{"type": "assert-ref-snapshot-id", "ref": "main", "snapshot-id": 10}], + "updates": [{ + "action": "set-statistics", + "statistics": { + "snapshot-id": 10, + "statistics-path": statistics_file, + "file-size-in-bytes": 5, + "file-footer-size-in-bytes": 0, + "blob-metadata": [] + } + }] + })) + .expect("statistics request should parse"); + + let error = commit_table_response( + &store, + &trusted_table_commit_backend(&metadata_backend), + "warehouse", + &namespace, + "events", + request, + ) + .await + .expect_err("unpublished statistics object should fail before pointer update"); + + assert_eq!(error.code(), &S3ErrorCode::InvalidRequest); + let unchanged = store + .load_table("warehouse", "analytics", "events") + .await + .expect("table lookup should succeed") + .expect("table should still exist"); + assert_eq!(unchanged.metadata_location, committed.metadata_location); + assert_eq!(unchanged.version_token, committed.version_token); + assert_eq!(unchanged.generation, committed.generation); + } +} + #[tokio::test] async fn bodyless_ref_delete_uses_default_request_options() { let request: DeleteTableRefRequest = read_json_body_or_default(Body::empty()) @@ -6933,15 +7875,443 @@ async fn bodyless_ref_delete_uses_default_request_options() { } #[test] -fn table_updates_reject_unknown_actions() { +fn unknown_commit_requirements_and_updates_are_bad_requests() { + let unknown_requirement = vec![serde_json::json!({"type": "unknown-requirement"})]; + let nonstandard_table_requirement = vec![serde_json::json!({"type": "assert-current-snapshot-id", "snapshot-id": 10})]; + let unknown_update = vec![serde_json::json!({"action": "unknown-update"})]; + let table_requirement_error = validate_table_commit_requirements(&serde_json::json!({}), &unknown_requirement) + .expect_err("unknown table requirement should fail"); + let nonstandard_table_requirement_error = + validate_table_commit_requirements(&serde_json::json!({"current-snapshot-id": 10}), &nonstandard_table_requirement) + .expect_err("nonstandard table requirement should fail"); + let table_update_error = apply_table_commit_updates(serde_json::json!({}), &unknown_update, "metadata/00001.metadata.json") + .expect_err("unknown table update should fail"); + let view_requirement_error = validate_view_commit_requirements(&serde_json::json!({}), &unknown_requirement) + .expect_err("unknown view requirement should fail"); + let nonstandard_view_requirement_error = validate_view_commit_requirements( + &serde_json::json!({"current-version-id": 1}), + &[serde_json::json!({"type": "assert-current-view-version-id", "current-view-version-id": 1})], + ) + .expect_err("nonstandard view requirement should fail"); + let nonstandard_view_update_error = apply_view_commit_updates_at( + serde_json::json!({}), + &[serde_json::json!({"action": "set-current-schema", "schema-id": 1})], + 0, + ) + .expect_err("nonstandard view update should fail"); + assert_eq!( + nonstandard_view_update_error.message(), + Some("unsupported view update: set-current-schema") + ); + let view_update_error = + apply_view_commit_updates_at(serde_json::json!({}), &unknown_update, 0).expect_err("unknown view update should fail"); + + for error in [ + table_requirement_error, + nonstandard_table_requirement_error, + table_update_error, + view_requirement_error, + nonstandard_view_requirement_error, + nonstandard_view_update_error, + view_update_error, + ] { + assert_eq!(error.code(), &S3ErrorCode::InvalidRequest); + } +} + +#[test] +fn failed_commit_requirements_use_iceberg_conflict_errors() { + let table_error = validate_table_commit_requirements( + &serde_json::json!({"table-uuid": "current"}), + &[serde_json::json!({"type": "assert-table-uuid", "uuid": "stale"})], + ) + .expect_err("stale table requirement should fail"); + let view_error = validate_view_commit_requirements( + &serde_json::json!({"view-uuid": "current"}), + &[serde_json::json!({"type": "assert-view-uuid", "uuid": "stale"})], + ) + .expect_err("stale view requirement should fail"); + + for error in [table_error, view_error] { + assert_eq!(error.code(), &S3ErrorCode::Custom(ICEBERG_ERROR_COMMIT_FAILED.into())); + assert_eq!(error.status_code(), Some(StatusCode::CONFLICT)); + } +} + +#[test] +fn commit_identifier_must_match_the_resource_url() { + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let matching = RestTableIdentifier { + namespace: vec!["analytics".to_string()], + name: "events".to_string(), + }; + validate_rest_commit_identifier(Some(&matching), &namespace, "events").expect("matching identifier should be accepted"); + + for identifier in [ + RestTableIdentifier { + namespace: vec!["staging".to_string()], + name: "events".to_string(), + }, + RestTableIdentifier { + namespace: vec!["analytics".to_string()], + name: "other".to_string(), + }, + ] { + assert_eq!( + validate_rest_commit_identifier(Some(&identifier), &namespace, "events") + .expect_err("identifier mismatch should fail") + .code(), + &S3ErrorCode::Custom(ICEBERG_ERROR_BAD_REQUEST.into()) + ); + } +} + +#[tokio::test] +async fn mismatched_commit_identifiers_leave_catalog_pointers_unchanged() { + let store = TestTableCatalogStore::default(); + let metadata_backend = TestTableCatalogObjectBackend::default(); + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + create_standard_events_table(&store, &metadata_backend, &namespace).await; + let table_before = store + .load_table("warehouse", "analytics", "events") + .await + .expect("table lookup should succeed") + .expect("table should exist"); + let table_error = commit_table_response( + &store, + &trusted_table_commit_backend(&metadata_backend), + "warehouse", + &namespace, + "events", + RestCommitTableRequest { + identifier: Some(RestTableIdentifier { + namespace: vec!["analytics".to_string()], + name: "other".to_string(), + }), + commit_id: None, + idempotency_key: None, + operation: None, + expected_version_token: None, + expected_metadata_location: None, + new_metadata_location: None, + requirements: Vec::new(), + updates: vec![serde_json::json!({"action": "set-properties", "updates": {"owner": "bad"}})], + writer: None, + }, + ) + .await + .expect_err("mismatched table identifier should fail"); + assert_eq!(table_error.status_code(), Some(StatusCode::BAD_REQUEST)); + assert_eq!( + store + .load_table("warehouse", "analytics", "events") + .await + .expect("table lookup should succeed") + .expect("table should exist"), + table_before + ); + + create_standard_recent_events_view(&store, &metadata_backend, &namespace).await; + let view_before = store + .load_view("warehouse", "analytics", "recent_events") + .await + .expect("view lookup should succeed") + .expect("view should exist"); + let view_error = replace_view_response( + &store, + &metadata_backend, + "warehouse", + &namespace, + "recent_events", + RestCommitViewRequest { + identifier: Some(RestTableIdentifier { + namespace: vec!["analytics".to_string()], + name: "other".to_string(), + }), + _commit_id: None, + expected_version_token: None, + expected_metadata_location: None, + new_metadata_location: None, + requirements: Vec::new(), + updates: vec![serde_json::json!({"action": "set-properties", "updates": {"owner": "bad"}})], + }, + ) + .await + .expect_err("mismatched view identifier should fail"); + assert_eq!(view_error.status_code(), Some(StatusCode::BAD_REQUEST)); + assert_eq!( + store + .load_view("warehouse", "analytics", "recent_events") + .await + .expect("view lookup should succeed") + .expect("view should exist"), + view_before + ); +} + +#[test] +fn table_updates_apply_standard_statistics_and_metadata_cleanup_actions() { let metadata = serde_json::json!({ + "last-updated-ms": 1, + "schemas": [ + {"type": "struct", "schema-id": 0, "fields": []}, + {"type": "struct", "schema-id": 1, "fields": []} + ], + "current-schema-id": 0, + "partition-specs": [ + {"spec-id": 0, "fields": []}, + {"spec-id": 1, "fields": []} + ], + "default-spec-id": 0, + "sort-orders": [{"order-id": 0, "fields": []}], + "default-sort-order-id": 0, + "snapshots": [{"snapshot-id": 10, "schema-id": 0}], + "current-snapshot-id": 10, + "refs": {"main": {"type": "branch", "snapshot-id": 10}}, "metadata-log": [] }); - let updates = vec![serde_json::json!({ - "action": "rewrite-everything" - })]; + let updated = apply_table_commit_updates_at( + metadata, + &[ + serde_json::json!({ + "action": "set-statistics", + "statistics": { + "snapshot-id": 10, + "statistics-path": "s3://warehouse/tables/table-id/metadata/stats.puffin", + "file-size-in-bytes": 128, + "file-footer-size-in-bytes": 16, + "blob-metadata": [{ + "type": "apache-datasketches-theta-v1", + "snapshot-id": 10, + "sequence-number": 1, + "fields": [1], + "properties": {"compression-codec": "zstd"} + }] + } + }), + serde_json::json!({ + "action": "set-partition-statistics", + "partition-statistics": { + "snapshot-id": 10, + "statistics-path": "s3://warehouse/tables/table-id/metadata/partition-stats.parquet", + "file-size-in-bytes": 64 + } + }), + serde_json::json!({"action": "remove-partition-specs", "spec-ids": [1]}), + serde_json::json!({"action": "remove-schemas", "schema-ids": [1]}), + ], + "metadata/00001.metadata.json", + 100, + ) + .expect("standard table updates should apply"); + assert_eq!(updated["statistics"][0]["snapshot-id"], 10); + assert_eq!(updated["partition-statistics"][0]["snapshot-id"], 10); + assert_eq!(updated["partition-specs"].as_array().map(Vec::len), Some(1)); + assert_eq!(updated["schemas"].as_array().map(Vec::len), Some(1)); + let removed = apply_table_commit_updates_at( + updated, + &[ + serde_json::json!({"action": "remove-statistics", "snapshot-id": 10}), + serde_json::json!({"action": "remove-partition-statistics", "snapshot-id": 10}), + ], + "metadata/00002.metadata.json", + 101, + ) + .expect("standard table removals should apply"); + assert!(removed["statistics"].as_array().is_some_and(Vec::is_empty)); + assert!(removed["partition-statistics"].as_array().is_some_and(Vec::is_empty)); +} - assert!(apply_table_commit_updates(metadata, &updates, "metadata/00001.metadata.json").is_err()); +#[test] +fn remove_snapshots_rejects_mixed_snapshot_id_types() { + let metadata = serde_json::json!({ + "snapshots": [{"snapshot-id": 10}, {"snapshot-id": 11}], + "snapshot-log": [{"snapshot-id": 10}, {"snapshot-id": 11}], + "metadata-log": [] + }); + let error = apply_table_commit_updates_at( + metadata, + &[serde_json::json!({"action": "remove-snapshots", "snapshot-ids": [10, "bad"]})], + "metadata/00001.metadata.json", + 100, + ) + .expect_err("mixed snapshot id types must fail before removing snapshots"); + + assert_eq!(error.code(), &S3ErrorCode::InvalidRequest); +} + +#[test] +fn remove_snapshots_removes_associated_statistics_entries() { + let metadata = serde_json::json!({ + "last-updated-ms": 1, + "snapshots": [{"snapshot-id": 10}, {"snapshot-id": 11}], + "snapshot-log": [{"snapshot-id": 10}, {"snapshot-id": 11}], + "statistics": [ + {"snapshot-id": 10, "statistics-path": "s3://warehouse/stats-10.puffin"}, + {"snapshot-id": 11, "statistics-path": "s3://warehouse/stats-11.puffin"} + ], + "partition-statistics": [ + {"snapshot-id": 10, "statistics-path": "s3://warehouse/partition-stats-10.parquet"}, + {"snapshot-id": 11, "statistics-path": "s3://warehouse/partition-stats-11.parquet"} + ], + "metadata-log": [] + }); + let updated = apply_table_commit_updates_at( + metadata, + &[serde_json::json!({"action": "remove-snapshots", "snapshot-ids": [10]})], + "metadata/00001.metadata.json", + 100, + ) + .expect("snapshot expiration should remove associated statistics entries"); + + for field in ["snapshots", "snapshot-log", "statistics", "partition-statistics"] { + assert_eq!(updated[field].as_array().map(Vec::len), Some(1)); + assert_eq!(updated[field][0]["snapshot-id"], 11); + } +} + +#[test] +fn removing_snapshots_clears_dangling_references_and_current_snapshot() { + let metadata = serde_json::json!({ + "last-updated-ms": 1, + "current-snapshot-id": 10, + "snapshots": [{"snapshot-id": 10}, {"snapshot-id": 11}], + "refs": { + "main": {"snapshot-id": 10, "type": "branch"}, + "release": {"snapshot-id": 10, "type": "tag"}, + "audit": {"snapshot-id": 11, "type": "branch"} + }, + "snapshot-log": [{"snapshot-id": 10}, {"snapshot-id": 11}], + "metadata-log": [] + }); + let updated = apply_table_commit_updates_at( + metadata, + &[serde_json::json!({"action": "remove-snapshots", "snapshot-ids": [10]})], + "metadata/00001.metadata.json", + 100, + ) + .expect("snapshot removal should clean references"); + + assert_eq!(updated["current-snapshot-id"], -1); + assert!(updated["refs"].get("main").is_none()); + assert!(updated["refs"].get("release").is_none()); + assert_eq!(updated["refs"]["audit"]["snapshot-id"], 11); +} + +#[test] +fn removing_an_intermediate_snapshot_truncates_earlier_history() { + let metadata = serde_json::json!({ + "last-updated-ms": 1, + "current-snapshot-id": 12, + "snapshots": [{"snapshot-id": 10}, {"snapshot-id": 11}, {"snapshot-id": 12}], + "refs": {"main": {"snapshot-id": 12, "type": "branch"}}, + "snapshot-log": [ + {"timestamp-ms": 10, "snapshot-id": 10}, + {"timestamp-ms": 11, "snapshot-id": 11}, + {"timestamp-ms": 12, "snapshot-id": 12} + ], + "metadata-log": [] + }); + let updated = apply_table_commit_updates_at( + metadata, + &[serde_json::json!({"action": "remove-snapshots", "snapshot-ids": [11]})], + "metadata/00001.metadata.json", + 100, + ) + .expect("snapshot removal should preserve valid time-travel history only"); + + assert_eq!(updated["snapshot-log"], serde_json::json!([{"timestamp-ms": 12, "snapshot-id": 12}])); +} + +#[test] +fn removing_main_snapshot_reference_clears_current_snapshot() { + let metadata = serde_json::json!({ + "last-updated-ms": 1, + "current-snapshot-id": 10, + "snapshots": [{"snapshot-id": 10}], + "refs": {"main": {"snapshot-id": 10, "type": "branch"}}, + "snapshot-log": [{"snapshot-id": 10}], + "metadata-log": [] + }); + let updated = apply_table_commit_updates_at( + metadata, + &[serde_json::json!({"action": "remove-snapshot-ref", "ref-name": "main"})], + "metadata/00001.metadata.json", + 100, + ) + .expect("main reference removal should succeed"); + + assert_eq!(updated["current-snapshot-id"], -1); + assert!(updated["refs"].get("main").is_none()); +} + +#[test] +fn table_statistics_updates_reject_malformed_standard_files() { + let metadata = serde_json::json!({"metadata-log": []}); + for update in [ + serde_json::json!({ + "action": "set-statistics", + "statistics": { + "snapshot-id": 10, + "statistics-path": "s3://warehouse/stats.puffin", + "file-footer-size-in-bytes": 1, + "blob-metadata": [] + } + }), + serde_json::json!({ + "action": "set-statistics", + "statistics": { + "snapshot-id": 10, + "statistics-path": "s3://warehouse/stats.puffin", + "file-size-in-bytes": 1, + "file-footer-size-in-bytes": 2, + "blob-metadata": [] + } + }), + serde_json::json!({ + "action": "set-partition-statistics", + "partition-statistics": { + "snapshot-id": 10, + "statistics-path": "s3://warehouse/partition-stats.parquet" + } + }), + serde_json::json!({ + "action": "set-statistics", + "snapshot-id": 11, + "statistics": { + "snapshot-id": 10, + "statistics-path": "s3://warehouse/stats.puffin", + "file-size-in-bytes": 1, + "file-footer-size-in-bytes": 0, + "blob-metadata": [] + } + }), + ] { + let error = apply_table_commit_updates_at(metadata.clone(), &[update], "metadata/00001.metadata.json", 100) + .expect_err("malformed statistics updates must fail"); + assert_eq!(error.code(), &S3ErrorCode::Custom(ICEBERG_ERROR_BAD_REQUEST.into())); + assert_eq!(error.status_code(), Some(StatusCode::BAD_REQUEST)); + } +} + +#[test] +fn table_encryption_key_updates_require_format_version_three() { + for update in [ + serde_json::json!({ + "action": "add-encryption-key", + "encryption-key": {"key-id": "key-1", "encrypted-key-metadata": "AQID"} + }), + serde_json::json!({"action": "remove-encryption-key", "key-id": "key-1"}), + ] { + let error = apply_table_commit_updates_at( + test_table_metadata_json("table-uuid", "s3://warehouse/tables/table-id"), + &[update], + "metadata/00001.metadata.json", + 100, + ) + .expect_err("Iceberg v2 tables must reject v3 encryption-key updates"); + assert_eq!(error.status_code(), Some(StatusCode::NOT_ACCEPTABLE)); + } } #[test] @@ -6980,6 +8350,7 @@ fn create_view_request_accepts_standard_iceberg_rest_shape() { }, "view-version": { "version-id": 1, + "timestamp-ms": 1, "schema-id": 0, "summary": { "engine-name": "spark", @@ -7005,6 +8376,395 @@ fn create_view_request_accepts_standard_iceberg_rest_shape() { assert_eq!(request.properties.get("comment").map(String::as_str), Some("recent event ids")); } +#[test] +fn view_versions_use_the_created_schema_and_resolve_minus_one() { + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let request: CreateViewRequest = serde_json::from_value(serde_json::json!({ + "name": "recent_events", + "schema": {"type": "struct", "schema-id": 3, "fields": []}, + "view-version": { + "version-id": 1, + "timestamp-ms": 1, + "schema-id": 99, + "summary": {"engine-name": "spark"}, + "default-catalog": "warehouse", + "default-namespace": ["analytics"], + "representations": [{"type": "sql", "sql": "SELECT 1", "dialect": "spark"}] + }, + "properties": {} + })) + .expect("view request should parse"); + let (_, metadata) = view_entry_from_create_view_request("warehouse", &namespace, request) + .expect("create should resolve the current schema placeholder"); + assert_eq!(metadata["schemas"][0]["schema-id"], 0); + assert_eq!(metadata["versions"][0]["schema-id"], 0); + + let updated = apply_view_commit_updates_at( + metadata, + &[ + serde_json::json!({ + "action": "add-schema", + "schema": {"type": "struct", "schema-id": 99, "fields": []} + }), + serde_json::json!({ + "action": "add-view-version", + "view-version": { + "version-id": 2, + "timestamp-ms": 2, + "schema-id": -1, + "summary": {"engine-name": "spark"}, + "default-catalog": "warehouse", + "default-namespace": ["analytics"], + "representations": [{"type": "sql", "sql": "SELECT 2", "dialect": "spark"}] + } + }), + ], + 2, + ) + .expect("view commit should resolve the current schema placeholder"); + assert_eq!(updated["schemas"][1]["schema-id"], 1); + assert_eq!(updated["versions"][1]["schema-id"], 1); + assert_eq!(updated["versions"][1]["timestamp-ms"], 2); + assert!(updated.get("last-updated-ms").is_none()); + assert!(updated.get("metadata-log").is_none()); + assert!(updated.get("last-column-id").is_none()); +} + +#[test] +fn current_view_version_minus_one_selects_the_last_added_version() { + let metadata = serde_json::json!({ + "format-version": 1, + "view-uuid": "view-uuid", + "location": "s3://warehouse/views/view-id", + "current-version-id": 5, + "schemas": [{"type": "struct", "schema-id": 0, "fields": []}], + "versions": [{ + "version-id": 5, + "timestamp-ms": 1, + "schema-id": 0, + "summary": {}, + "default-namespace": ["analytics"], + "representations": [{"type": "sql", "sql": "SELECT 5", "dialect": "spark"}] + }], + "version-log": [{"version-id": 5, "timestamp-ms": 1}], + "properties": {} + }); + let updated = apply_view_commit_updates_at( + metadata, + &[ + serde_json::json!({ + "action": "add-view-version", + "view-version": { + "version-id": 3, + "timestamp-ms": 2, + "schema-id": 0, + "summary": {}, + "default-namespace": ["analytics"], + "representations": [{"type": "sql", "sql": "SELECT 3", "dialect": "spark"}] + } + }), + serde_json::json!({"action": "set-current-view-version", "view-version-id": -1}), + ], + 2, + ) + .expect("minus one should resolve to the last added view version"); + + assert_eq!(updated["current-version-id"], 3); + assert_eq!( + updated["version-log"] + .as_array() + .and_then(|log| log.last()) + .map(|entry| &entry["version-id"]), + Some(&serde_json::Value::from(3)) + ); + assert_eq!( + updated["version-log"] + .as_array() + .and_then(|log| log.last()) + .map(|entry| &entry["timestamp-ms"]), + Some(&serde_json::Value::from(2)) + ); +} + +#[test] +fn last_added_view_ids_require_a_preceding_add_update() { + let metadata = serde_json::json!({ + "format-version": 1, + "view-uuid": "view-uuid", + "location": "s3://warehouse/views/view-id", + "current-version-id": 1, + "schemas": [{"type": "struct", "schema-id": 0, "fields": []}], + "versions": [{ + "version-id": 1, + "timestamp-ms": 1, + "schema-id": 0, + "summary": {}, + "default-namespace": ["analytics"], + "representations": [{"type": "sql", "sql": "SELECT 1", "dialect": "spark"}] + }], + "version-log": [{"version-id": 1, "timestamp-ms": 1}], + "properties": {} + }); + let version = serde_json::json!({ + "action": "add-view-version", + "view-version": { + "version-id": 2, + "timestamp-ms": 2, + "schema-id": 0, + "summary": {}, + "default-namespace": ["analytics"], + "representations": [{"type": "sql", "sql": "SELECT 2", "dialect": "spark"}] + } + }); + let invalid_update_sequences = [ + vec![serde_json::json!({ + "action": "add-view-version", + "view-version": { + "version-id": 2, + "timestamp-ms": 2, + "schema-id": -1, + "summary": {}, + "default-namespace": ["analytics"], + "representations": [{"type": "sql", "sql": "SELECT 2", "dialect": "spark"}] + } + })], + vec![serde_json::json!({"action": "set-current-view-version", "view-version-id": -1})], + vec![ + serde_json::json!({"action": "set-current-view-version", "view-version-id": -1}), + version, + ], + ]; + + for updates in invalid_update_sequences { + let error = apply_view_commit_updates_at(metadata.clone(), &updates, 3) + .expect_err("historical or later view objects must not satisfy a -1 reference"); + assert_eq!(error.code(), &S3ErrorCode::InvalidRequest); + assert!( + error + .message() + .is_some_and(|message| message.contains("requires a preceding")) + ); + } +} + +#[test] +fn view_history_uses_added_version_time_and_skips_current_version_noops() { + let metadata = serde_json::json!({ + "format-version": 1, + "view-uuid": "view-uuid", + "location": "s3://warehouse/views/view-id", + "current-version-id": 1, + "schemas": [{"type": "struct", "schema-id": 0, "fields": []}], + "versions": [{ + "version-id": 1, + "timestamp-ms": 1, + "schema-id": 0, + "summary": {}, + "default-namespace": ["analytics"], + "representations": [{"type": "sql", "sql": "SELECT 1", "dialect": "spark"}] + }], + "version-log": [{"version-id": 1, "timestamp-ms": 1}], + "properties": {} + }); + let unchanged = apply_view_commit_updates_at( + metadata.clone(), + &[serde_json::json!({"action": "set-current-view-version", "view-version-id": 1})], + 100, + ) + .expect("setting the current view version again should be a no-op"); + assert_eq!(unchanged["version-log"], metadata["version-log"]); + + let updated = apply_view_commit_updates_at( + metadata, + &[ + serde_json::json!({ + "action": "add-view-version", + "view-version": { + "version-id": 2, + "timestamp-ms": 20, + "schema-id": 0, + "summary": {}, + "default-namespace": ["analytics"], + "representations": [{"type": "sql", "sql": "SELECT 2", "dialect": "spark"}] + } + }), + serde_json::json!({ + "action": "add-view-version", + "view-version": { + "version-id": 3, + "timestamp-ms": 30, + "schema-id": 0, + "summary": {}, + "default-namespace": ["analytics"], + "representations": [{"type": "sql", "sql": "SELECT 3", "dialect": "spark"}] + } + }), + serde_json::json!({"action": "set-current-view-version", "view-version-id": 2}), + ], + 100, + ) + .expect("an explicitly selected version added in this commit should use its own timestamp"); + assert_eq!(updated["current-version-id"], 2); + assert_eq!( + updated["version-log"].as_array().and_then(|log| log.last()), + Some(&serde_json::json!({ + "timestamp-ms": 20, + "version-id": 2 + })) + ); +} + +#[test] +fn view_requests_and_metadata_require_standard_fields() { + let request_without_properties = serde_json::from_value::(serde_json::json!({ + "name": "recent_events", + "schema": {"type": "struct", "schema-id": 0, "fields": []}, + "view-version": { + "version-id": 1, + "timestamp-ms": 1, + "schema-id": 0, + "summary": {}, + "default-namespace": ["analytics"], + "representations": [{"type": "sql", "sql": "SELECT 1", "dialect": "spark"}] + } + })) + .expect("the Java REST serializer omits empty view properties"); + assert!(request_without_properties.properties.is_empty()); + + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let request = serde_json::from_value::(serde_json::json!({ + "name": "recent_events", + "schema": {"type": "struct", "schema-id": 0, "fields": []}, + "view-version": { + "version-id": 1, + "schema-id": 0, + "summary": {}, + "default-namespace": ["analytics"], + "representations": [{"type": "sql", "sql": "SELECT 1", "dialect": "spark"}] + }, + "properties": {} + })) + .expect("request shape should parse before metadata validation"); + let error = view_entry_from_create_view_request("warehouse", &namespace, request) + .expect_err("view-version timestamp-ms must be required"); + assert_eq!(error.code(), &S3ErrorCode::InvalidRequest); + + let malformed_schema_request = serde_json::from_value::(serde_json::json!({ + "name": "recent_events", + "schema": {}, + "view-version": { + "version-id": 1, + "timestamp-ms": 1, + "schema-id": 0, + "summary": {}, + "default-namespace": ["analytics"], + "representations": [{"type": "sql", "sql": "SELECT 1", "dialect": "spark"}] + }, + "properties": {} + })) + .expect("request shape should parse before schema validation"); + view_entry_from_create_view_request("warehouse", &namespace, malformed_schema_request) + .expect_err("create view must reject schemas missing type and fields"); + + let metadata = serde_json::json!({ + "format-version": 1, + "view-uuid": "view-uuid", + "location": "s3://warehouse/views/view-id", + "current-version-id": 1, + "schemas": [{"type": "struct", "schema-id": 0, "fields": []}], + "versions": [{ + "version-id": 1, + "timestamp-ms": 1, + "schema-id": 0, + "summary": {}, + "default-namespace": ["analytics"], + "representations": [{"type": "sql", "sql": "SELECT 1", "dialect": "spark"}] + }], + "version-log": [{"version-id": 1, "timestamp-ms": 1}], + "properties": {} + }); + let mut missing_current = metadata.clone(); + missing_current + .as_object_mut() + .expect("metadata should be an object") + .remove("current-version-id"); + validate_supported_view_metadata(&missing_current).expect_err("current-version-id must be required"); + + let mut unsupported_representation = metadata.clone(); + unsupported_representation["versions"][0]["representations"][0]["type"] = serde_json::Value::from("python"); + validate_supported_view_metadata(&unsupported_representation).expect_err("non-SQL view representations must be rejected"); + + let mut duplicate_dialect = metadata.clone(); + duplicate_dialect["versions"][0]["representations"] = serde_json::json!([ + {"type": "sql", "sql": "SELECT 1", "dialect": "spark"}, + {"type": "sql", "sql": "SELECT 2", "dialect": "SPARK"} + ]); + validate_supported_view_metadata(&duplicate_dialect).expect_err("a view version must not contain duplicate SQL dialects"); + + let mut malformed_schema = metadata.clone(); + malformed_schema["schemas"] = serde_json::json!([{"schema-id": 0}]); + validate_supported_view_metadata(&malformed_schema).expect_err("view schemas must include type and fields"); + + let updated = apply_view_commit_updates_at( + metadata.clone(), + &[serde_json::json!({"action": "set-properties", "updates": {"owner": "analytics"}})], + 2, + ) + .expect("standard view metadata without table-only timestamps must remain mutable"); + assert_eq!(updated["properties"]["owner"], "analytics"); + assert!(updated.get("last-updated-ms").is_none()); + assert!(updated.get("metadata-log").is_none()); + + let missing_version_id = apply_view_commit_updates_at( + metadata, + &[serde_json::json!({ + "action": "add-view-version", + "view-version": { + "timestamp-ms": 2, + "schema-id": 0, + "summary": {}, + "default-namespace": ["analytics"], + "representations": [{"type": "sql", "sql": "SELECT 2", "dialect": "spark"}] + } + })], + 2, + ) + .expect_err("add-view-version must not synthesize version-id"); + assert_eq!(missing_version_id.code(), &S3ErrorCode::InvalidRequest); +} + +#[test] +fn view_commit_rejects_unsupported_format_version_upgrade() { + let metadata = serde_json::json!({ + "format-version": 1, + "view-uuid": "view-uuid", + "location": "s3://warehouse/views/view-id", + "schemas": [{"schema-id": 0, "type": "struct", "fields": []}], + "versions": [{ + "version-id": 1, + "timestamp-ms": 1, + "schema-id": 0, + "summary": {"engine-name": "spark"}, + "default-namespace": ["analytics"], + "representations": [{"type": "sql", "sql": "SELECT 1", "dialect": "spark"}] + }], + "current-version-id": 1, + "version-log": [], + "metadata-log": [], + "properties": {} + }); + + let error = apply_view_commit_updates_at( + metadata, + &[serde_json::json!({"action": "upgrade-format-version", "format-version": 2})], + 0, + ) + .expect_err("Iceberg view format-version 2 is unsupported"); + + assert_eq!(error.code(), &S3ErrorCode::Custom(ICEBERG_ERROR_UNSUPPORTED_OPERATION.into())); + assert_eq!(error.status_code(), Some(StatusCode::NOT_ACCEPTABLE)); +} + #[test] fn create_view_request_accepts_deep_warehouse_location() { let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); @@ -7021,6 +8781,7 @@ fn create_view_request_accepts_deep_warehouse_location() { }, "view-version": { "version-id": 1, + "timestamp-ms": 1, "schema-id": 0, "summary": { "engine-name": "spark" @@ -7034,7 +8795,8 @@ fn create_view_request_accepts_deep_warehouse_location() { "dialect": "spark" } ] - } + }, + "properties": {} })) .expect("deep create view request should parse"); @@ -7082,6 +8844,7 @@ async fn view_catalog_responses_persist_replace_and_drop_view_metadata() { }, "view-version": { "version-id": 1, + "timestamp-ms": 1, "schema-id": 0, "summary": { "engine-name": "spark" @@ -7095,19 +8858,23 @@ async fn view_catalog_responses_persist_replace_and_drop_view_metadata() { "dialect": "spark" } ] - } + }, + "properties": {} })) .expect("standard create view request should parse"); - let created = create_view_response(&store, &metadata_backend, "warehouse", &namespace, create_request, true) + let create_backend = TableCommitObjectBackend::trusted(metadata_backend.clone()); + let created = create_view_response(&store, &create_backend, "warehouse", &namespace, create_request, true) .await .expect("view should be created"); assert_eq!(created.metadata["format-version"], 1); assert_eq!(created.metadata["current-version-id"], 1); assert_eq!(created.metadata["versions"][0]["representations"][0]["dialect"], "spark"); + let created_metadata_key = table_metadata_location_for_catalog("warehouse", &created.metadata_location) + .expect("created metadata location should normalize for the object backend"); assert!( metadata_backend - .object_exists("warehouse", &created.metadata_location) + .object_exists("warehouse", &created_metadata_key) .await .expect("view metadata object lookup should succeed") ); @@ -7123,12 +8890,51 @@ async fn view_catalog_responses_persist_replace_and_drop_view_metadata() { .await .expect("view should load"); assert_eq!(loaded.metadata_location, created.metadata_location); + + let metadata_directory = created_metadata_key + .rsplit_once('/') + .map(|(directory, _)| directory) + .expect("created metadata key should have a directory"); + let invalid_target_key = format!("{metadata_directory}/invalid.metadata.json"); + let mut invalid_target = created.metadata.clone(); + invalid_target["format-version"] = serde_json::Value::from(2); + metadata_backend + .put_json("warehouse", &invalid_target_key, invalid_target) + .await; + let invalid_replace = serde_json::from_value::(serde_json::json!({ + "new-metadata-location": table_metadata_location_for_client("warehouse", &invalid_target_key), + "updates": [] + })) + .expect("external replace request should parse"); + let invalid_replace_backend = TableCommitObjectBackend::trusted(metadata_backend.clone()); + let error = replace_view_response( + &store, + &invalid_replace_backend, + "warehouse", + &namespace, + "recent_events", + invalid_replace, + ) + .await + .expect_err("unsupported external view metadata must fail before pointer publication"); + assert_eq!(error.status_code(), Some(StatusCode::NOT_ACCEPTABLE)); + assert_eq!( + store + .load_view("warehouse", "analytics", "recent_events") + .await + .expect("view lookup should succeed") + .expect("view should remain registered") + .metadata_location, + created_metadata_key + ); + let replace_request: RestCommitViewRequest = serde_json::from_value(serde_json::json!({ "updates": [ { "action": "add-view-version", "view-version": { "version-id": 2, + "timestamp-ms": 2, "schema-id": 0, "summary": { "engine-name": "spark" @@ -7151,7 +8957,8 @@ async fn view_catalog_responses_persist_replace_and_drop_view_metadata() { ] })) .expect("replace view request should parse"); - let replaced = replace_view_response(&store, &metadata_backend, "warehouse", &namespace, "recent_events", replace_request) + let replace_backend = TableCommitObjectBackend::trusted(metadata_backend.clone()); + let replaced = replace_view_response(&store, &replace_backend, "warehouse", &namespace, "recent_events", replace_request) .await .expect("view should replace"); assert_ne!(replaced.metadata_location, created.metadata_location); @@ -7188,6 +8995,7 @@ async fn view_catalog_responses_persist_replace_and_drop_view_metadata() { }, "view-version": { "version-id": 1, + "timestamp-ms": 1, "schema-id": 0, "summary": { "engine-name": "spark" @@ -7201,15 +9009,217 @@ async fn view_catalog_responses_persist_replace_and_drop_view_metadata() { "dialect": "spark" } ] - } + }, + "properties": {} })) .expect("standard recreate view request should parse"); - let recreated = create_view_response(&store, &metadata_backend, "warehouse", &namespace, recreate_request, true) + let recreate_backend = TableCommitObjectBackend::trusted(metadata_backend.clone()); + let recreated = create_view_response(&store, &recreate_backend, "warehouse", &namespace, recreate_request, true) .await .expect("dropped view name should be reusable"); assert_ne!(recreated.metadata_location, created.metadata_location); } +#[tokio::test] +async fn replace_view_holds_target_metadata_and_view_fences_until_pointer_publish() { + let pause = TestCatalogPublishPause::default(); + let store = Arc::new(TestTableCatalogStore { + replace_view_pause: Some(pause.clone()), + ..Default::default() + }); + let metadata_backend = TestTableCatalogObjectBackend::default(); + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let created = create_standard_recent_events_view(store.as_ref(), &metadata_backend, &namespace).await; + let current_metadata_key = table_metadata_location_for_catalog("warehouse", &created.metadata_location) + .expect("created metadata location should normalize"); + let metadata_directory = current_metadata_key + .rsplit_once('/') + .map(|(directory, _)| directory) + .expect("created metadata key should have a directory"); + let target_metadata_key = format!("{metadata_directory}/external.metadata.json"); + metadata_backend + .put_json("warehouse", &target_metadata_key, created.metadata.clone()) + .await; + let request = serde_json::from_value::(serde_json::json!({ + "new-metadata-location": table_metadata_location_for_client("warehouse", &target_metadata_key), + "requirements": [], + "updates": [] + })) + .expect("replace view request should parse"); + + let replace_store = Arc::clone(&store); + let replace_backend = TableCommitObjectBackend::trusted(metadata_backend.clone()); + let replace_namespace = namespace.clone(); + let replace = tokio::spawn(async move { + let result = replace_view_response( + replace_store.as_ref(), + &replace_backend, + "warehouse", + &replace_namespace, + "recent_events", + request, + ) + .await; + replace_backend.finish(result).await + }); + tokio::time::timeout(StdDuration::from_secs(2), pause.wait_started()) + .await + .expect("view replacement should reach catalog publication"); + + let view_name = crate::table_catalog::IdentifierSegment::parse("recent_events").expect("view should parse"); + let view_lock = crate::table_catalog::default_table_publication_lock_path(&namespace, &view_name); + let bucket_lock = crate::table_catalog::default_table_bucket_publication_lock_path(); + assert!( + !metadata_backend.write_lock_is_held("warehouse", &bucket_lock).await, + "view replacement without warehouse relocation must not serialize the table bucket" + ); + assert!( + metadata_backend.write_lock_is_held("warehouse", &view_lock).await, + "view replacement must retain its publication fence until pointer publication" + ); + assert!( + metadata_backend.write_lock_is_held("warehouse", &target_metadata_key).await, + "target view metadata must remain stable until pointer publication" + ); + + metadata_backend.lock_attempts.lock().await.clear(); + let writer_backend = metadata_backend.clone(); + let writer_target = target_metadata_key.clone(); + let writer = tokio::spawn(async move { + crate::table_catalog::TableCatalogObjectBackend::acquire_write_lock(&writer_backend, "warehouse", &writer_target).await + }); + metadata_backend.wait_for_lock_attempts(1).await; + assert!(!writer.is_finished(), "a target metadata writer must wait for pointer publication"); + + pause.release(); + let replaced = tokio::time::timeout(StdDuration::from_secs(2), replace) + .await + .expect("view replacement should complete") + .expect("view replacement task should join") + .expect("view replacement should succeed"); + tokio::time::timeout(StdDuration::from_secs(2), writer) + .await + .expect("target metadata writer should continue after publication") + .expect("target metadata writer task should join") + .expect("target metadata writer lock acquisition should succeed"); + assert_eq!( + table_metadata_location_for_catalog("warehouse", &replaced.metadata_location) + .expect("replaced metadata location should normalize"), + target_metadata_key + ); +} + +#[tokio::test] +async fn replace_view_holds_table_bucket_fence_for_warehouse_relocation() { + let pause = TestCatalogPublishPause::default(); + let store = Arc::new(TestTableCatalogStore { + replace_view_pause: Some(pause.clone()), + ..Default::default() + }); + let metadata_backend = TestTableCatalogObjectBackend::default(); + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let created = create_standard_recent_events_view(store.as_ref(), &metadata_backend, &namespace).await; + let current_metadata_key = table_metadata_location_for_catalog("warehouse", &created.metadata_location) + .expect("created metadata location should normalize"); + let metadata_directory = current_metadata_key + .rsplit_once('/') + .map(|(directory, _)| directory) + .expect("created metadata key should have a directory"); + let target_metadata_key = format!("{metadata_directory}/relocated.metadata.json"); + let mut target_metadata = created.metadata; + target_metadata["location"] = serde_json::Value::String("s3://warehouse/views/relocated".to_string()); + metadata_backend + .put_json("warehouse", &target_metadata_key, target_metadata) + .await; + let request = serde_json::from_value::(serde_json::json!({ + "new-metadata-location": table_metadata_location_for_client("warehouse", &target_metadata_key), + "requirements": [], + "updates": [] + })) + .expect("replace view request should parse"); + + let replace_store = Arc::clone(&store); + let replace_backend = TableCommitObjectBackend::trusted(metadata_backend.clone()); + let replace_namespace = namespace.clone(); + let replace = tokio::spawn(async move { + let result = replace_view_response( + replace_store.as_ref(), + &replace_backend, + "warehouse", + &replace_namespace, + "recent_events", + request, + ) + .await; + replace_backend.finish(result).await + }); + tokio::time::timeout(StdDuration::from_secs(2), pause.wait_started()) + .await + .expect("view replacement should reach catalog publication"); + + let bucket_lock = crate::table_catalog::default_table_bucket_publication_lock_path(); + assert!( + metadata_backend.write_lock_is_held("warehouse", &bucket_lock).await, + "view warehouse relocation must retain the table-bucket publication fence" + ); + + pause.release(); + tokio::time::timeout(StdDuration::from_secs(2), replace) + .await + .expect("view replacement should complete") + .expect("view replacement task should join") + .expect("view replacement should succeed"); +} + +#[tokio::test] +async fn external_view_metadata_replacement_repairs_legacy_incomplete_metadata() { + let store = TestTableCatalogStore::default(); + let metadata_backend = TestTableCatalogObjectBackend::default(); + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let created = create_standard_recent_events_view(&store, &metadata_backend, &namespace).await; + let current = store + .load_view("warehouse", "analytics", "recent_events") + .await + .expect("view lookup should succeed") + .expect("view should exist"); + let mut incomplete = created.metadata.clone(); + incomplete + .as_object_mut() + .expect("view metadata should be an object") + .remove("versions"); + metadata_backend + .put_json("warehouse", ¤t.metadata_location, incomplete) + .await; + load_view_response(&store, &metadata_backend, "warehouse", &namespace, "recent_events") + .await + .expect_err("legacy incomplete view metadata must not be served"); + + let metadata_directory = current + .metadata_location + .rsplit_once('/') + .map(|(directory, _)| directory) + .expect("current metadata location should have a directory"); + let target = format!("{metadata_directory}/repaired.metadata.json"); + metadata_backend + .put_json("warehouse", &target, created.metadata.clone()) + .await; + let request = serde_json::from_value::(serde_json::json!({ + "new-metadata-location": table_metadata_location_for_client("warehouse", &target), + "requirements": [{"type": "assert-view-uuid", "uuid": current.view_uuid}], + "updates": [] + })) + .expect("view repair request should parse"); + let publication_backend = TableCommitObjectBackend::trusted(metadata_backend.clone()); + let repaired = replace_view_response(&store, &publication_backend, "warehouse", &namespace, "recent_events", request) + .await + .expect("a valid external metadata target should repair legacy incomplete metadata"); + + assert_eq!(repaired.metadata, created.metadata); + load_view_response(&store, &metadata_backend, "warehouse", &namespace, "recent_events") + .await + .expect("the repaired view should load"); +} + #[tokio::test] async fn table_ref_write_responses_use_commit_guard_and_protect_deletes() { let store = TestTableCatalogStore::default(); @@ -7224,6 +9234,7 @@ async fn table_ref_write_responses_use_commit_guard_and_protect_deletes() { seed_test_snapshot_manifest(&metadata_backend, "warehouse", &manifest_list, 10, 1, &[(&data_file, 0, 1, 10, 1)]).await; let append_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "requirements": [], "updates": [ { "action": "add-snapshot", @@ -7436,6 +9447,75 @@ fn load_table_response_preserves_format_v4_relative_metadata_log() { ); } +#[test] +fn load_table_snapshot_selection_validates_and_filters_refs() { + assert_eq!( + rest_table_snapshot_selection_from_query(&"/".parse().expect("URI should parse")) + .expect("omitted snapshots selection should parse"), + RestTableSnapshotSelection::All + ); + assert_eq!( + rest_table_snapshot_selection_from_query(&"/?snapshots=all".parse().expect("URI should parse")) + .expect("all snapshots selection should parse"), + RestTableSnapshotSelection::All + ); + assert_eq!( + rest_table_snapshot_selection_from_query(&"/?snapshots=refs".parse().expect("URI should parse")) + .expect("referenced snapshots selection should parse"), + RestTableSnapshotSelection::Refs + ); + for uri in ["/?snapshots=unknown", "/?snapshots=all&snapshots=refs"] { + let error = rest_table_snapshot_selection_from_query(&uri.parse().expect("URI should parse")) + .expect_err("invalid snapshots selections must fail"); + assert_eq!(error.code(), &S3ErrorCode::Custom(ICEBERG_ERROR_BAD_REQUEST.into())); + } + + let metadata = serde_json::json!({ + "snapshots": [ + {"snapshot-id": 1}, + {"snapshot-id": 2}, + {"snapshot-id": 3} + ], + "refs": { + "audit": {"type": "tag", "snapshot-id": 1}, + "main": {"type": "branch", "snapshot-id": 3} + } + }); + let mut all = metadata.clone(); + apply_rest_table_snapshot_selection(&mut all, RestTableSnapshotSelection::All); + assert_eq!(all["snapshots"].as_array().map(Vec::len), Some(3)); + + let mut referenced = metadata; + apply_rest_table_snapshot_selection(&mut referenced, RestTableSnapshotSelection::Refs); + assert_eq!( + referenced["snapshots"] + .as_array() + .expect("snapshots should remain an array") + .iter() + .filter_map(|snapshot| snapshot["snapshot-id"].as_i64()) + .collect::>(), + vec![1, 3] + ); + + let mut implicit_main = serde_json::json!({ + "current-snapshot-id": 2, + "snapshots": [ + {"snapshot-id": 1}, + {"snapshot-id": 2} + ] + }); + apply_rest_table_snapshot_selection(&mut implicit_main, RestTableSnapshotSelection::Refs); + assert_eq!( + implicit_main["snapshots"] + .as_array() + .expect("snapshots should remain an array") + .iter() + .filter_map(|snapshot| snapshot["snapshot-id"].as_i64()) + .collect::>(), + vec![2] + ); +} + #[test] fn table_metadata_location_for_catalog_accepts_only_the_table_bucket() { let object_key = ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00001.metadata.json"; @@ -7640,6 +9720,19 @@ async fn credential_response_serializes_sensitive_config_only_inside_storage_cre ); } +#[test] +fn credential_http_response_disables_caching() { + let response = build_sensitive_json_response(StatusCode::OK, &serde_json::json!({"storage-credentials": []})) + .expect("sensitive response should build"); + + assert_eq!( + response.headers.get(http::header::CACHE_CONTROL), + Some(&HeaderValue::from_static("no-store, private")) + ); + assert_eq!(response.headers.get(http::header::PRAGMA), Some(&HeaderValue::from_static("no-cache"))); + assert_eq!(response.headers.get(http::header::EXPIRES), Some(&HeaderValue::from_static("0"))); +} + #[test] fn table_credentials_do_not_snapshot_parent_groups() { let principal = rustfs_credentials::Credentials { @@ -7817,10 +9910,12 @@ fn commit_table_request_uses_rest_commit_fields() { "new-metadata-location": ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00002.metadata.json", "requirements": [ { - "type": "assert-current-snapshot-id", + "type": "assert-ref-snapshot-id", + "ref": "main", "snapshot-id": 10 } ], + "updates": [], "writer": "pyiceberg" })) .expect("commit request should parse"); @@ -7837,6 +9932,26 @@ fn commit_table_request_uses_rest_commit_fields() { assert_eq!(request.writer.as_deref(), Some("pyiceberg")); } +#[test] +fn rest_commit_item_counts_are_bounded_before_processing() { + let allowed = vec![serde_json::Value::Null; TABLE_CATALOG_COMMIT_REQUIREMENT_MAX_COUNT]; + validate_rest_commit_item_counts(&allowed, &[]).expect("the documented requirement limit should be accepted"); + let too_many_requirements = vec![serde_json::Value::Null; TABLE_CATALOG_COMMIT_REQUIREMENT_MAX_COUNT + 1]; + assert_eq!( + validate_rest_commit_item_counts(&too_many_requirements, &[]) + .expect_err("excess requirements must be rejected") + .code(), + &S3ErrorCode::InvalidRequest + ); + let too_many_updates = vec![serde_json::Value::Null; TABLE_CATALOG_COMMIT_UPDATE_MAX_COUNT + 1]; + assert_eq!( + validate_rest_commit_item_counts(&[], &too_many_updates) + .expect_err("excess updates must be rejected") + .code(), + &S3ErrorCode::InvalidRequest + ); +} + fn trusted_table_commit_backend( backend: &TestTableCatalogObjectBackend, ) -> TableCommitObjectBackend { @@ -7870,15 +9985,53 @@ async fn seed_test_manifest_list( snapshot_id: i64, ) { let manifest_list_key = test_snapshot_object_key(bucket, manifest_list_location); + let mut manifests = Vec::with_capacity(manifest_locations.len()); + for manifest_location in manifest_locations { + let manifest_key = test_snapshot_object_key(bucket, manifest_location); + let manifest_length = backend + .state + .lock() + .await + .objects + .get(&(bucket.to_string(), manifest_key)) + .map(|object| object.data.len()) + .expect("test manifest must be seeded before its manifest list"); + manifests.push((*manifest_location, manifest_length)); + } backend .put_bytes( bucket, &manifest_list_key, - test_manifest_list_avro_bytes(manifest_locations, sequence_number, snapshot_id), + test_manifest_list_avro_bytes(&manifests, sequence_number, snapshot_id), ) .await; } +async fn seed_test_manifest_list_entries( + backend: &TestTableCatalogObjectBackend, + bucket: &str, + manifest_list_location: &str, + manifest_entries: &[(&str, i64, i64)], +) { + let manifest_list_key = test_snapshot_object_key(bucket, manifest_list_location); + let mut manifests = Vec::with_capacity(manifest_entries.len()); + for (manifest_location, sequence_number, snapshot_id) in manifest_entries { + let manifest_key = test_snapshot_object_key(bucket, manifest_location); + let manifest_length = backend + .state + .lock() + .await + .objects + .get(&(bucket.to_string(), manifest_key)) + .map(|object| object.data.len()) + .expect("test manifest must be seeded before its manifest list"); + manifests.push((*manifest_location, manifest_length, *sequence_number, *snapshot_id)); + } + backend + .put_bytes(bucket, &manifest_list_key, test_manifest_list_avro_entries(&manifests)) + .await; +} + async fn seed_test_snapshot_manifest( backend: &TestTableCatalogObjectBackend, bucket: &str, @@ -7893,16 +10046,15 @@ async fn seed_test_snapshot_manifest( .expect("manifest list location should include a file name"); let manifest_key = test_snapshot_object_key(bucket, &manifest_location); let manifest_list_key = test_snapshot_object_key(bucket, manifest_list_location); + let manifest_bytes = test_manifest_avro_bytes(files); backend .put_bytes( bucket, &manifest_list_key, - test_manifest_list_avro_bytes(&[&manifest_location], sequence_number, snapshot_id), + test_manifest_list_avro_bytes(&[(&manifest_location, manifest_bytes.len())], sequence_number, snapshot_id), ) .await; - backend - .put_bytes(bucket, &manifest_key, test_manifest_avro_bytes(files)) - .await; + backend.put_bytes(bucket, &manifest_key, manifest_bytes).await; seed_test_manifest_data_files(backend, bucket, files).await; } @@ -7995,6 +10147,55 @@ where .expect("table should be created") } +async fn create_standard_recent_events_view( + store: &S, + metadata_backend: &TestTableCatalogObjectBackend, + namespace: &crate::table_catalog::Namespace, +) -> RestLoadViewResponse +where + S: crate::table_catalog::TableCatalogStore + ?Sized, +{ + ensure_table_bucket_entry(store, "warehouse", true) + .await + .expect("table bucket entry should be seeded"); + if store + .get_namespace("warehouse", &namespace.public_name()) + .await + .expect("namespace lookup should succeed") + .is_none() + { + create_namespace_response( + store, + "warehouse", + CreateNamespaceRequest { + namespace: namespace.public_name().split('.').map(str::to_string).collect(), + properties: BTreeMap::new(), + }, + true, + ) + .await + .expect("namespace should be created"); + } + let request: CreateViewRequest = serde_json::from_value(serde_json::json!({ + "name": "recent_events", + "schema": {"type": "struct", "fields": []}, + "view-version": { + "version-id": 1, + "timestamp-ms": 1, + "schema-id": 0, + "summary": {}, + "default-namespace": ["analytics"], + "representations": [{"type": "sql", "sql": "SELECT 1", "dialect": "spark"}] + }, + "properties": {} + })) + .expect("standard create view request should parse"); + let publication_backend = TableCommitObjectBackend::trusted(metadata_backend.clone()); + create_view_response(store, &publication_backend, "warehouse", namespace, request, true) + .await + .expect("view should be created") +} + fn standard_property_commit_request(commit_id: &str, table_uuid: &str, owner: &str) -> RestCommitTableRequest { serde_json::from_value(serde_json::json!({ "commit-id": commit_id, @@ -8403,7 +10604,7 @@ async fn table_helpers_call_catalog_store() { new_metadata_location: Some(table_metadata_location_for_client("warehouse", next_metadata_location)), requirements: client_requirements.clone(), updates: Vec::new(), - _identifier: None, + identifier: None, writer: Some("pyiceberg".to_string()), }, ) @@ -8445,6 +10646,121 @@ async fn table_helpers_call_catalog_store() { ); } +#[tokio::test] +async fn load_table_rejects_invalid_persisted_metadata() { + let store = TestTableCatalogStore::default(); + let metadata_backend = TestTableCatalogObjectBackend::default(); + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let created = create_standard_events_table(&store, &metadata_backend, &namespace).await; + let metadata_key = table_metadata_location_for_catalog("warehouse", &created.metadata_location) + .expect("created metadata location should normalize"); + let mut invalid_metadata = created.metadata; + invalid_metadata["location"] = serde_json::Value::from("s3://other-warehouse/tables/table-id"); + metadata_backend.put_json("warehouse", &metadata_key, invalid_metadata).await; + + let error = load_table_response(&store, &metadata_backend, "warehouse", &namespace, "events") + .await + .expect_err("load table must reject persisted metadata outside the table bucket"); + assert_eq!(error.code(), &S3ErrorCode::Custom(ICEBERG_ERROR_REST.into())); + assert_eq!(error.status_code(), Some(StatusCode::INTERNAL_SERVER_ERROR)); +} + +#[test] +fn table_format_upgrade_accepts_historical_v1_metadata_and_rejects_downgrade() { + let entry = crate::table_catalog::TableEntry { + version: crate::table_catalog::TABLE_CATALOG_ENTRY_VERSION, + table_bucket: "warehouse".to_string(), + namespace: "analytics".to_string(), + table: "events".to_string(), + table_id: "table-id".to_string(), + table_uuid: "table-uuid".to_string(), + format: "ICEBERG".to_string(), + format_version: 2, + warehouse_location: "s3://warehouse/tables/table-id".to_string(), + metadata_location: "tables/table-id/metadata/00002.metadata.json".to_string(), + version_token: "token-v2".to_string(), + generation: 2, + state: crate::table_catalog::TableCatalogEntryState::Active, + properties: BTreeMap::new(), + created_at: None, + updated_at: None, + }; + let historical_v1 = serde_json::json!({ + "format-version": 1, + "table-uuid": "table-uuid", + "location": "s3://warehouse/tables/table-id", + "last-updated-ms": 1, + "last-column-id": 1, + "schema": { + "type": "struct", + "schema-id": 0, + "fields": [{"id": 1, "name": "id", "required": true, "type": "long"}] + }, + "partition-spec": [], + "properties": {}, + "snapshots": [], + "snapshot-log": [], + "metadata-log": [] + }); + let current_v2 = test_table_metadata_json("table-uuid", "s3://warehouse/tables/table-id"); + + validate_persisted_table_metadata(&entry, &historical_v1, false) + .expect("historical v1 metadata should remain readable after a v2 upgrade"); + validate_persisted_table_metadata(&entry, &historical_v1, true) + .expect_err("the current pointer must match the catalog format version"); + let mut legacy_entry = entry; + legacy_entry.format_version = 1; + validate_persisted_table_metadata(&legacy_entry, ¤t_v2, true) + .expect("a current v2 metadata file committed before format persistence must remain readable"); + validate_persisted_table_metadata(&legacy_entry, ¤t_v2, false) + .expect("post-upgrade v2 metadata must remain readable as a historical commit base"); + validate_metadata_identity_matches_current_metadata(&historical_v1, ¤t_v2).expect("a v1 table may upgrade to v2"); + validate_metadata_identity_matches_current_metadata(¤t_v2, &historical_v1) + .expect_err("a v2 table must not downgrade to v1"); +} + +#[tokio::test] +async fn load_responses_reject_persisted_metadata_for_another_catalog_identity() { + let store = TestTableCatalogStore::default(); + let metadata_backend = TestTableCatalogObjectBackend::default(); + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let table = create_standard_events_table(&store, &metadata_backend, &namespace).await; + let table_entry = store + .load_table("warehouse", "analytics", "events") + .await + .expect("table lookup should succeed") + .expect("table should exist"); + let mut foreign_table_metadata = table.metadata; + foreign_table_metadata["table-uuid"] = serde_json::Value::from("foreign-table-uuid"); + metadata_backend + .put_json("warehouse", &table_entry.metadata_location, foreign_table_metadata) + .await; + + let table_error = load_table_response(&store, &metadata_backend, "warehouse", &namespace, "events") + .await + .expect_err("load table must bind persisted metadata to the catalog identity"); + assert_eq!(table_error.code(), &S3ErrorCode::Custom(ICEBERG_ERROR_REST.into())); + assert_eq!(table_error.status_code(), Some(StatusCode::INTERNAL_SERVER_ERROR)); + + let view = create_standard_recent_events_view(&store, &metadata_backend, &namespace).await; + let view_entry = store + .load_view("warehouse", "analytics", "recent_events") + .await + .expect("view lookup should succeed") + .expect("view should exist"); + let mut foreign_view_metadata = view.metadata; + foreign_view_metadata["view-uuid"] = serde_json::Value::from("foreign-view-uuid"); + metadata_backend + .put_json("warehouse", &view_entry.metadata_location, foreign_view_metadata) + .await; + + let view_error = load_view_response(&store, &metadata_backend, "warehouse", &namespace, "recent_events") + .await + .expect_err("load view must bind persisted metadata to the catalog identity"); + assert_eq!(view_error.code(), &S3ErrorCode::Custom(ICEBERG_ERROR_REST.into())); + assert_eq!(view_error.status_code(), Some(StatusCode::INTERNAL_SERVER_ERROR)); +} + #[tokio::test] async fn register_table_response_adopts_metadata_table_uuid() { let store = TestTableCatalogStore::default(); @@ -8805,13 +11121,10 @@ async fn metadata_location_api_loads_and_updates_current_pointer() { ) .expect("table entry should build"); let table_uuid = entry.table_uuid.clone(); + let warehouse_location = entry.warehouse_location.clone(); store.register_table(entry).await.expect("table should register"); metadata_backend - .put_json( - "warehouse", - current_location, - test_table_metadata_json(&table_uuid, "s3://warehouse/tables/table-id"), - ) + .put_json("warehouse", current_location, test_table_metadata_json(&table_uuid, &warehouse_location)) .await; let current = get_table_metadata_location_response(&store, "warehouse", &namespace, "events") .await @@ -8822,11 +11135,7 @@ async fn metadata_location_api_loads_and_updates_current_pointer() { ); let next_location = ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00002.metadata.json"; metadata_backend - .put_json( - "warehouse", - next_location, - test_table_metadata_json(&table_uuid, "s3://warehouse/tables/table-id"), - ) + .put_json("warehouse", next_location, test_table_metadata_json(&table_uuid, &warehouse_location)) .await; let updated = update_table_metadata_location_response( @@ -9573,7 +11882,7 @@ async fn legacy_commit_rejects_mismatched_table_uuid_before_commit() { new_metadata_location: Some(mismatched_location.to_string()), requirements: Vec::new(), updates: Vec::new(), - _identifier: None, + identifier: None, writer: Some("pyiceberg".to_string()), }, ) diff --git a/rustfs/src/admin/handlers/table_catalog/view.rs b/rustfs/src/admin/handlers/table_catalog/view.rs index 2b4ca5d31..b18b7e5ce 100644 --- a/rustfs/src/admin/handlers/table_catalog/view.rs +++ b/rustfs/src/admin/handlers/table_catalog/view.rs @@ -43,8 +43,9 @@ impl Operation for RestCreateViewHandler { let metadata_backend = table_catalog_backend_from_extensions(&req.extensions)?; let store = table_catalog_store_from_backend(metadata_backend.clone())?; let table_bucket_enabled = table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?; + let publication_backend = TableCommitObjectBackend::preauthorized(metadata_backend); let response = - create_view_response(&store, &metadata_backend, &warehouse, &namespace, request, table_bucket_enabled).await?; + create_view_response(&store, &publication_backend, &warehouse, &namespace, request, table_bucket_enabled).await?; build_json_response(StatusCode::OK, &response) } } @@ -87,17 +88,20 @@ pub struct RestReplaceViewHandler {} #[async_trait::async_trait] impl Operation for RestReplaceViewHandler { - async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + async fn call(&self, mut req: S3Request, params: Params<'_, '_>) -> S3Result> { let warehouse = warehouse_from_params(¶ms)?; let namespace = namespace_from_params(¶ms)?; let view = view_name_from_params(¶ms)?; let resource = TableCatalogResource::view(&warehouse, &namespace, &view); - authorize_table_catalog_resource_request(&req, &resource, AdminAction::CommitTableAction).await?; + let principal = authorize_table_catalog_resource_request(&req, &resource, AdminAction::CommitTableAction).await?; + install_table_catalog_s3_request_info(&mut req, &principal)?; ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?; - let request = read_json_body::(req.input).await?; + let request = read_rest_commit_view_request(std::mem::take(&mut req.input)).await?; let metadata_backend = table_catalog_backend_from_extensions(&req.extensions)?; let store = table_catalog_store_from_backend(metadata_backend.clone())?; - let response = replace_view_response(&store, &metadata_backend, &warehouse, &namespace, &view, request).await?; + let commit_backend = TableCommitObjectBackend::for_request(metadata_backend, req); + let result = replace_view_response(&store, &commit_backend, &warehouse, &namespace, &view, request).await; + let response = commit_backend.finish(result).await?; build_json_response(StatusCode::OK, &response) } } diff --git a/rustfs/src/storage/access.rs b/rustfs/src/storage/access.rs index 806035b70..2d62f16c9 100644 --- a/rustfs/src/storage/access.rs +++ b/rustfs/src/storage/access.rs @@ -1478,7 +1478,7 @@ async fn retain_table_data_plane_publication_guard( .map_err(|err| s3_error!(InternalError, "failed to acquire table publication guard: {}", err))?; let mut state = retained.state.lock(); state.keys.insert(key); - state.guards.push(guard); + state.guards.push(Box::new(guard)); drop(state); req.extensions.insert(retained); Ok(()) diff --git a/rustfs/src/table_catalog/iceberg/manifest.rs b/rustfs/src/table_catalog/iceberg/manifest.rs index cdc0a2635..82960d255 100644 --- a/rustfs/src/table_catalog/iceberg/manifest.rs +++ b/rustfs/src/table_catalog/iceberg/manifest.rs @@ -16,6 +16,8 @@ use std::io::Read; use super::super::*; +const AVRO_ZSTANDARD_MAX_WINDOW_LOG: u32 = 27; + #[derive(Debug, Clone, PartialEq)] pub(crate) struct ManifestDataFileReference { pub location: String, @@ -66,6 +68,7 @@ pub(crate) struct DecodedManifestList { pub(crate) struct DecodedManifest { pub references: Vec, pub decoded_size: usize, + pub partition_spec_id: Option, } pub(crate) fn manifest_paths_from_manifest_list_avro(data: &[u8]) -> TableCatalogStoreResult> { @@ -92,6 +95,25 @@ pub(crate) fn decode_manifest_list_avro(data: &[u8]) -> TableCatalogStoreResult< .map_err(|err| TableCatalogStoreError::Invalid(format!("failed to read manifest list Avro: {err}")))?; let format_version = avro_record_format_version(reader.writer_schema(), &["sequence_number", "min_sequence_number"], "manifest list")?; + if format_version == 2 { + let apache_avro::Schema::Record(record) = reader.writer_schema() else { + return Err(TableCatalogStoreError::Invalid("manifest list Avro schema must be a record".to_string())); + }; + for field in [ + "added_files_count", + "existing_files_count", + "deleted_files_count", + "added_rows_count", + "existing_rows_count", + "deleted_rows_count", + ] { + if !record.lookup.contains_key(field) { + return Err(TableCatalogStoreError::Invalid(format!( + "Iceberg v2 manifest list Avro schema is missing {field}" + ))); + } + } + } let mut manifest_paths = Vec::new(); for value in reader { if manifest_paths.len() >= TABLE_MANIFEST_AVRO_MAX_RECORDS { @@ -115,24 +137,12 @@ pub(crate) fn decode_manifest_list_avro(data: &[u8]) -> TableCatalogStoreResult< sequence_number: avro_record_field(&value, "sequence_number").and_then(avro_i64_value), min_sequence_number: avro_record_field(&value, "min_sequence_number").and_then(avro_i64_value), added_snapshot_id: avro_record_field(&value, "added_snapshot_id").and_then(avro_i64_value), - added_files_count: avro_record_field(&value, "added_files_count") - .and_then(avro_i32_value) - .and_then(|value| u64::try_from(value).ok()), - existing_files_count: avro_record_field(&value, "existing_files_count") - .and_then(avro_i32_value) - .and_then(|value| u64::try_from(value).ok()), - deleted_files_count: avro_record_field(&value, "deleted_files_count") - .and_then(avro_i32_value) - .and_then(|value| u64::try_from(value).ok()), - added_rows_count: avro_record_field(&value, "added_rows_count") - .and_then(avro_i64_value) - .and_then(|value| u64::try_from(value).ok()), - existing_rows_count: avro_record_field(&value, "existing_rows_count") - .and_then(avro_i64_value) - .and_then(|value| u64::try_from(value).ok()), - deleted_rows_count: avro_record_field(&value, "deleted_rows_count") - .and_then(avro_i64_value) - .and_then(|value| u64::try_from(value).ok()), + added_files_count: avro_nullable_non_negative_i32(&value, "added_files_count", "manifest list")?, + existing_files_count: avro_nullable_non_negative_i32(&value, "existing_files_count", "manifest list")?, + deleted_files_count: avro_nullable_non_negative_i32(&value, "deleted_files_count", "manifest list")?, + added_rows_count: avro_nullable_non_negative_i64(&value, "added_rows_count", "manifest list")?, + existing_rows_count: avro_nullable_non_negative_i64(&value, "existing_rows_count", "manifest list")?, + deleted_rows_count: avro_nullable_non_negative_i64(&value, "deleted_rows_count", "manifest list")?, }); } Ok(DecodedManifestList { @@ -171,6 +181,17 @@ pub(crate) fn decode_manifest_avro(data: &[u8]) -> TableCatalogStoreResult().ok()) + .filter(|value| *value >= 0) + .ok_or_else(|| TableCatalogStoreError::Invalid("manifest partition-spec-id metadata is invalid".to_string())) + }) + .transpose()?; let mut files = Vec::new(); for value in reader { if files.len() >= TABLE_MANIFEST_AVRO_MAX_RECORDS { @@ -202,6 +223,9 @@ pub(crate) fn decode_manifest_avro(data: &[u8]) -> TableCatalogStoreResult TableCatalogStoreResult) -> TableCatalogSto enum AvroContainerCodec { Null, Deflate, + Snappy, + Zstandard, } fn validate_avro_container(data: &[u8]) -> TableCatalogStoreResult { @@ -300,6 +325,8 @@ fn validate_avro_container(data: &[u8]) -> TableCatalogStoreResult { let codec = match codec.unwrap_or(b"null") { b"null" => AvroContainerCodec::Null, b"deflate" => AvroContainerCodec::Deflate, + b"snappy" => AvroContainerCodec::Snappy, + b"zstandard" => AvroContainerCodec::Zstandard, codec => { return Err(TableCatalogStoreError::Unsupported(format!( "Avro codec {} is not supported for table commit validation", @@ -369,6 +396,34 @@ fn avro_block_decoded_size(codec: AvroContainerCodec, block: &[u8], remaining_si usize::try_from(decoded_size) .map_err(|_| TableCatalogStoreError::Invalid("Avro decoded data size is invalid".to_string())) } + AvroContainerCodec::Snappy => { + let data_end = block + .len() + .checked_sub(4) + .ok_or_else(|| TableCatalogStoreError::Invalid("Avro snappy block is missing its checksum".to_string()))?; + let decoded_size = snap::raw::decompress_len(&block[..data_end]) + .map_err(|err| TableCatalogStoreError::Invalid(format!("failed to inspect Avro snappy block: {err}")))?; + if decoded_size > remaining_size { + return Err(TableCatalogStoreError::Invalid("Avro decoded data exceeds the commit limit".to_string())); + } + Ok(decoded_size) + } + AvroContainerCodec::Zstandard => { + let limit = remaining_size + .checked_add(1) + .ok_or_else(|| TableCatalogStoreError::Invalid("Avro decoded data size limit overflowed".to_string()))?; + let limit = u64::try_from(limit) + .map_err(|_| TableCatalogStoreError::Invalid("Avro decoded data size limit is invalid".to_string()))?; + let mut decoder = zstd::stream::read::Decoder::new(block) + .map_err(|err| TableCatalogStoreError::Invalid(format!("failed to decompress Avro zstandard block: {err}")))?; + decoder + .window_log_max(AVRO_ZSTANDARD_MAX_WINDOW_LOG) + .map_err(|err| TableCatalogStoreError::Invalid(format!("failed to bound Avro zstandard window: {err}")))?; + let decoded_size = std::io::copy(&mut decoder.take(limit), &mut std::io::sink()) + .map_err(|err| TableCatalogStoreError::Invalid(format!("failed to decompress Avro zstandard block: {err}")))?; + usize::try_from(decoded_size) + .map_err(|_| TableCatalogStoreError::Invalid("Avro decoded data size is invalid".to_string())) + } } } @@ -445,6 +500,40 @@ fn avro_record_value_fields(value: &apache_avro::types::Value) -> Option TableCatalogStoreResult> { + let Some(value) = avro_record_field(value, field) else { + return Ok(None); + }; + match avro_non_union_value(value) { + apache_avro::types::Value::Null => Ok(None), + apache_avro::types::Value::Int(value) => u64::try_from(*value) + .map(Some) + .map_err(|_| TableCatalogStoreError::Invalid(format!("{label} field {field} must be a non-negative int"))), + _ => Err(TableCatalogStoreError::Invalid(format!("{label} field {field} must be a nullable int"))), + } +} + +fn avro_nullable_non_negative_i64( + value: &apache_avro::types::Value, + field: &str, + label: &str, +) -> TableCatalogStoreResult> { + let Some(value) = avro_record_field(value, field) else { + return Ok(None); + }; + match avro_non_union_value(value) { + apache_avro::types::Value::Null => Ok(None), + apache_avro::types::Value::Long(value) => u64::try_from(*value) + .map(Some) + .map_err(|_| TableCatalogStoreError::Invalid(format!("{label} field {field} must be a non-negative long"))), + _ => Err(TableCatalogStoreError::Invalid(format!("{label} field {field} must be a nullable long"))), + } +} + pub(crate) fn avro_non_union_value(value: &apache_avro::types::Value) -> &apache_avro::types::Value { match value { apache_avro::types::Value::Union(_, inner) => avro_non_union_value(inner), @@ -472,3 +561,127 @@ fn avro_i64_value(value: &apache_avro::types::Value) -> Option { _ => None, } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rejects_v2_manifest_lists_without_required_count_fields() { + let schema = apache_avro::Schema::parse_str( + r#"{ + "type": "record", + "name": "manifest_file", + "fields": [ + {"name": "manifest_path", "type": "string"}, + {"name": "manifest_length", "type": "long"}, + {"name": "partition_spec_id", "type": "int"}, + {"name": "content", "type": "int"}, + {"name": "sequence_number", "type": "long"}, + {"name": "min_sequence_number", "type": "long"}, + {"name": "added_snapshot_id", "type": "long"} + ] + }"#, + ) + .expect("incomplete manifest-list schema should parse"); + let data = apache_avro::Writer::new(&schema, Vec::new()) + .expect("manifest-list writer should initialize") + .into_inner() + .expect("manifest-list bytes should flush"); + + let error = match decode_manifest_list_avro(&data) { + Ok(_) => panic!("v2 count fields must be declared in the writer schema"), + Err(error) => error, + }; + assert_eq!( + error, + TableCatalogStoreError::Invalid("Iceberg v2 manifest list Avro schema is missing added_files_count".to_string()) + ); + } + + #[test] + fn rejects_negative_nullable_manifest_list_counts() { + let value = apache_avro::types::Value::Record(vec![ + ("added_files_count".to_string(), apache_avro::types::Value::Int(-1)), + ("added_rows_count".to_string(), apache_avro::types::Value::Long(-1)), + ]); + + assert_eq!( + avro_nullable_non_negative_i32(&value, "added_files_count", "manifest list") + .expect_err("negative file counts must be rejected"), + TableCatalogStoreError::Invalid("manifest list field added_files_count must be a non-negative int".to_string()) + ); + assert_eq!( + avro_nullable_non_negative_i64(&value, "added_rows_count", "manifest list") + .expect_err("negative row counts must be rejected"), + TableCatalogStoreError::Invalid("manifest list field added_rows_count must be a non-negative long".to_string()) + ); + } + + #[test] + fn rejects_manifest_partition_with_non_record_schema() { + let schema = apache_avro::Schema::parse_str( + r#"{ + "type": "record", + "name": "manifest_entry", + "fields": [ + {"name": "status", "type": "int"}, + {"name": "snapshot_id", "type": "long"}, + { + "name": "data_file", + "type": { + "type": "record", + "name": "data_file", + "fields": [ + {"name": "file_path", "type": "string"}, + {"name": "record_count", "type": "long"}, + {"name": "file_size_in_bytes", "type": "long"}, + {"name": "partition", "type": "string"} + ] + } + } + ] + }"#, + ) + .expect("manifest schema should parse"); + let mut writer = apache_avro::Writer::new(&schema, Vec::new()).expect("manifest writer should initialize"); + writer + .append_value(apache_avro::types::Value::Record(vec![ + ("status".to_string(), apache_avro::types::Value::Int(1)), + ("snapshot_id".to_string(), apache_avro::types::Value::Long(1)), + ( + "data_file".to_string(), + apache_avro::types::Value::Record(vec![ + ( + "file_path".to_string(), + apache_avro::types::Value::String("s3://warehouse/tables/table-id/data/file.parquet".to_string()), + ), + ("record_count".to_string(), apache_avro::types::Value::Long(1)), + ("file_size_in_bytes".to_string(), apache_avro::types::Value::Long(1)), + ("partition".to_string(), apache_avro::types::Value::String("not-a-record".to_string())), + ]), + ), + ])) + .expect("manifest record should append"); + let data = writer.into_inner().expect("manifest bytes should flush"); + + let error = match decode_manifest_avro(&data) { + Ok(_) => panic!("manifest partitions must preserve their record shape"), + Err(error) => error, + }; + assert_eq!( + error, + TableCatalogStoreError::Invalid("manifest data file partition must be a record".to_string()) + ); + } + + #[test] + fn rejects_oversized_zstandard_windows() { + // Non-single-segment frame with a 2^28-byte window and one empty final block. + let compressed = [0x28, 0xb5, 0x2f, 0xfd, 0x00, 0x90, 0x01, 0x00, 0x00]; + + let error = avro_block_decoded_size(AvroContainerCodec::Zstandard, &compressed, TABLE_MANIFEST_AVRO_MAX_DECODED_SIZE) + .expect_err("zstandard windows larger than the manifest decode budget must be rejected"); + assert!(matches!(error, TableCatalogStoreError::Invalid(_))); + } +} diff --git a/rustfs/src/table_catalog/iceberg/validation.rs b/rustfs/src/table_catalog/iceberg/validation.rs index 0b96f4078..c10c30237 100644 --- a/rustfs/src/table_catalog/iceberg/validation.rs +++ b/rustfs/src/table_catalog/iceberg/validation.rs @@ -18,6 +18,8 @@ use futures::{StreamExt, TryStreamExt, stream}; use super::super::*; +const ICEBERG_MAX_USER_FIELD_ID: i32 = i32::MAX - 200; + fn normalize_warehouse_object_prefix(object_prefix: &str, max_prefix_depth: Option) -> TableCatalogStoreResult { let object_prefix = object_prefix.strip_suffix('/').unwrap_or(object_prefix); if object_prefix.is_empty() { @@ -109,18 +111,33 @@ pub(crate) fn table_object_s3_location(table_bucket: &str, object_key: &str) -> format!("s3://{table_bucket}/{object_key}") } -fn metadata_warehouse_location( +pub(crate) struct TableMetadataCommitState { + pub(crate) warehouse_location: Option, + pub(crate) format_version: Option, +} + +pub(crate) fn table_metadata_commit_state( table_bucket: &str, metadata_location: &str, metadata_object: &TableCatalogObject, - validate_location: fn(&str, &str) -> TableCatalogStoreResult<()>, -) -> TableCatalogStoreResult> { +) -> TableCatalogStoreResult { let metadata = decode_table_metadata_json(metadata_location, &metadata_object.data)?; - let Some(location) = metadata.get("location").and_then(serde_json::Value::as_str) else { - return Ok(None); - }; - validate_location(table_bucket, location)?; - Ok(Some(location.to_string())) + let warehouse_location = metadata + .get("location") + .and_then(serde_json::Value::as_str) + .map(|location| { + validate_table_warehouse_location(table_bucket, location)?; + Ok(location.to_string()) + }) + .transpose()?; + let format_version = metadata + .get("format-version") + .map(|_| table_metadata_format_version(&metadata)) + .transpose()?; + Ok(TableMetadataCommitState { + warehouse_location, + format_version, + }) } pub(crate) fn decode_table_metadata_json(metadata_location: &str, data: &[u8]) -> TableCatalogStoreResult { @@ -167,14 +184,6 @@ fn table_metadata_location_is_gzip(metadata_location: &str) -> bool { metadata_location.ends_with(".gz.metadata.json") || metadata_location.ends_with(".metadata.json.gz") } -pub(crate) fn table_metadata_warehouse_location( - table_bucket: &str, - metadata_location: &str, - metadata_object: &TableCatalogObject, -) -> TableCatalogStoreResult> { - metadata_warehouse_location(table_bucket, metadata_location, metadata_object, validate_table_warehouse_location) -} - pub(crate) fn canonical_json_sha256(metadata: &serde_json::Value) -> TableCatalogStoreResult { let canonical = serde_json::to_vec(metadata) .map_err(|err| TableCatalogStoreError::Internal(format!("failed to encode metadata digest input: {err}")))?; @@ -220,7 +229,12 @@ pub(crate) fn view_metadata_warehouse_location( metadata_location: &str, metadata_object: &TableCatalogObject, ) -> TableCatalogStoreResult> { - metadata_warehouse_location(table_bucket, metadata_location, metadata_object, validate_view_warehouse_location) + let metadata = decode_table_metadata_json(metadata_location, &metadata_object.data)?; + let Some(location) = metadata.get("location").and_then(serde_json::Value::as_str) else { + return Ok(None); + }; + validate_view_warehouse_location(table_bucket, location)?; + Ok(Some(location.to_string())) } pub(crate) fn warehouse_index_candidate_prefixes(object: &str) -> Vec<&str> { @@ -334,9 +348,99 @@ pub(crate) fn table_metadata_format_version(metadata: &serde_json::Value) -> Tab Ok(version) } +fn normalize_v1_table_metadata_update_fields(metadata: &mut serde_json::Value) -> TableCatalogStoreResult<()> { + if metadata.get("schemas").is_none() { + let mut schema = metadata + .get("schema") + .cloned() + .ok_or_else(|| TableCatalogStoreError::Invalid("Iceberg v1 table metadata is missing schema".to_string()))?; + schema + .as_object_mut() + .ok_or_else(|| TableCatalogStoreError::Invalid("Iceberg v1 schema must be an object".to_string()))? + .entry("schema-id".to_string()) + .or_insert_with(|| serde_json::Value::from(0)); + let schema_id = schema + .get("schema-id") + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| TableCatalogStoreError::Invalid("Iceberg v1 schema-id must be an integer".to_string()))?; + let object = metadata_object_mut(metadata)?; + object.insert("schemas".to_string(), serde_json::json!([schema])); + object.insert("current-schema-id".to_string(), serde_json::Value::from(schema_id)); + } else if metadata.get("current-schema-id").is_none() { + let schema_id = metadata + .get("schema") + .and_then(|schema| schema.get("schema-id")) + .and_then(serde_json::Value::as_i64) + .unwrap_or(0); + if !require_metadata_array(metadata, "schemas")? + .iter() + .any(|schema| schema.get("schema-id").and_then(serde_json::Value::as_i64) == Some(schema_id)) + { + return Err(TableCatalogStoreError::Invalid("Iceberg v1 current schema does not exist".to_string())); + } + metadata_object_mut(metadata)?.insert("current-schema-id".to_string(), serde_json::Value::from(schema_id)); + } + + if metadata.get("partition-specs").is_none() { + let mut fields = metadata + .get("partition-spec") + .cloned() + .ok_or_else(|| TableCatalogStoreError::Invalid("Iceberg v1 table metadata is missing partition-spec".to_string()))?; + let fields = fields + .as_array_mut() + .ok_or_else(|| TableCatalogStoreError::Invalid("Iceberg v1 partition-spec must be an array".to_string()))?; + for (index, field) in fields.iter_mut().enumerate() { + let field_id = i32::try_from(index) + .ok() + .and_then(|index| 1000_i32.checked_add(index)) + .ok_or_else(|| TableCatalogStoreError::Invalid("Iceberg v1 partition spec has too many fields".to_string()))?; + field + .as_object_mut() + .ok_or_else(|| TableCatalogStoreError::Invalid("Iceberg v1 partition spec fields must be objects".to_string()))? + .entry("field-id".to_string()) + .or_insert_with(|| serde_json::Value::from(field_id)); + } + let object = metadata_object_mut(metadata)?; + object.insert("partition-specs".to_string(), serde_json::json!([{"spec-id": 0, "fields": fields}])); + object.insert("default-spec-id".to_string(), serde_json::Value::from(0)); + } else if metadata.get("default-spec-id").is_none() { + let default_spec_id = require_metadata_array(metadata, "partition-specs")? + .last() + .and_then(|spec| spec.get("spec-id")) + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| TableCatalogStoreError::Invalid("Iceberg v1 default partition spec does not exist".to_string()))?; + metadata_object_mut(metadata)?.insert("default-spec-id".to_string(), serde_json::Value::from(default_spec_id)); + } + + if metadata.get("sort-orders").is_none() { + let object = metadata_object_mut(metadata)?; + object.insert("sort-orders".to_string(), serde_json::json!([{"order-id": 0, "fields": []}])); + object.insert("default-sort-order-id".to_string(), serde_json::Value::from(0)); + } else if metadata.get("default-sort-order-id").is_none() { + let default_sort_order_id = require_metadata_array(metadata, "sort-orders")? + .last() + .and_then(|order| order.get("order-id")) + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| TableCatalogStoreError::Invalid("Iceberg v1 default sort order does not exist".to_string()))?; + metadata_object_mut(metadata)? + .insert("default-sort-order-id".to_string(), serde_json::Value::from(default_sort_order_id)); + } + + if metadata.get("last-partition-id").is_none() { + let last_partition_id = require_metadata_array(metadata, "partition-specs")? + .iter() + .map(max_partition_field_id) + .max() + .unwrap_or(999); + metadata_object_mut(metadata)?.insert("last-partition-id".to_string(), serde_json::Value::from(last_partition_id)); + } + Ok(()) +} + pub(crate) fn synchronize_table_metadata_version_fields(metadata: &mut serde_json::Value) -> TableCatalogStoreResult<()> { match table_metadata_format_version(metadata)? { 1 => { + normalize_v1_table_metadata_update_fields(metadata)?; if let Some(schemas) = metadata.get("schemas").and_then(serde_json::Value::as_array) && !schemas.is_empty() { @@ -404,9 +508,27 @@ pub(crate) fn synchronize_table_metadata_version_fields(metadata: &mut serde_jso metadata_object_mut(metadata)?.insert("current-schema-id".to_string(), serde_json::Value::from(schema_id)); } if metadata.get("partition-specs").is_none() { - let fields = metadata.get("partition-spec").cloned().ok_or_else(|| { + let mut fields = metadata.get("partition-spec").cloned().ok_or_else(|| { TableCatalogStoreError::Invalid("Iceberg v1 table metadata is missing partition-spec".to_string()) })?; + let fields = fields + .as_array_mut() + .ok_or_else(|| TableCatalogStoreError::Invalid("Iceberg v1 partition-spec must be an array".to_string()))?; + for (index, field) in fields.iter_mut().enumerate() { + let field_id = i32::try_from(index) + .ok() + .and_then(|index| 1000_i32.checked_add(index)) + .ok_or_else(|| { + TableCatalogStoreError::Invalid("Iceberg v1 partition spec has too many fields".to_string()) + })?; + field + .as_object_mut() + .ok_or_else(|| { + TableCatalogStoreError::Invalid("Iceberg v1 partition spec fields must be objects".to_string()) + })? + .entry("field-id".to_string()) + .or_insert_with(|| serde_json::Value::from(field_id)); + } metadata_object_mut(metadata)? .insert("partition-specs".to_string(), serde_json::json!([{"spec-id": 0, "fields": fields}])); metadata_object_mut(metadata)?.insert("default-spec-id".to_string(), serde_json::Value::from(0)); @@ -452,11 +574,456 @@ pub(crate) fn synchronize_table_metadata_version_fields(metadata: &mut serde_jso Ok(()) } +fn validate_table_history_logs(metadata: &serde_json::Value) -> TableCatalogStoreResult<()> { + for field in ["snapshot-log", "metadata-log"] { + let Some(entries) = metadata.get(field) else { + continue; + }; + let entries = entries + .as_array() + .ok_or_else(|| TableCatalogStoreError::Invalid(format!("{field} must be an array")))?; + for entry in entries { + let entry = entry + .as_object() + .ok_or_else(|| TableCatalogStoreError::Invalid(format!("{field} entries must be JSON objects")))?; + if entry.get("timestamp-ms").and_then(serde_json::Value::as_i64).is_none() { + return Err(TableCatalogStoreError::Invalid(format!("{field} entries require integer timestamp-ms"))); + } + match field { + "snapshot-log" if entry.get("snapshot-id").and_then(serde_json::Value::as_i64).is_none() => { + return Err(TableCatalogStoreError::Invalid( + "snapshot-log entries require integer snapshot-id".to_string(), + )); + } + "metadata-log" + if !entry + .get("metadata-file") + .and_then(serde_json::Value::as_str) + .is_some_and(|location| !location.is_empty()) => + { + return Err(TableCatalogStoreError::Invalid( + "metadata-log entries require non-empty metadata-file".to_string(), + )); + } + _ => {} + } + } + } + Ok(()) +} + pub(crate) fn validate_supported_table_metadata(metadata: &serde_json::Value) -> TableCatalogStoreResult<()> { validate_supported_table_metadata_fields(metadata)?; + validate_table_history_logs(metadata)?; validate_table_metadata_references(metadata) } +pub(crate) fn validate_table_metadata_transition( + current_metadata: &serde_json::Value, + target_metadata: &serde_json::Value, +) -> TableCatalogStoreResult<()> { + let current_last_column_id = require_metadata_i32(current_metadata, "last-column-id")?; + let target_last_column_id = require_metadata_i32(target_metadata, "last-column-id")?; + if target_last_column_id < current_last_column_id { + return Err(TableCatalogStoreError::Invalid( + "last-column-id must not decrease across table metadata commits".to_string(), + )); + } + + let current_format_version = table_metadata_format_version(current_metadata)?; + let target_format_version = table_metadata_format_version(target_metadata)?; + let current_last_partition_id = table_metadata_last_partition_id(current_metadata, current_format_version)?; + let target_last_partition_id = table_metadata_last_partition_id(target_metadata, target_format_version)?; + if target_last_partition_id < current_last_partition_id { + return Err(TableCatalogStoreError::Invalid( + "last-partition-id must not decrease across table metadata commits".to_string(), + )); + } + let current_last_sequence_number = table_metadata_last_sequence_number(current_metadata, current_format_version)?; + let target_last_sequence_number = table_metadata_last_sequence_number(target_metadata, target_format_version)?; + if target_last_sequence_number < current_last_sequence_number { + return Err(TableCatalogStoreError::Invalid( + "last-sequence-number must not decrease across table metadata commits".to_string(), + )); + } + + validate_existing_partition_specs_unchanged(current_metadata, target_metadata)?; + validate_existing_metadata_entries_unchanged( + &normalized_sort_order_definitions(current_metadata, current_format_version)?, + &normalized_sort_order_definitions(target_metadata, target_format_version)?, + "sort order", + )?; + validate_existing_snapshots_unchanged(current_metadata, target_metadata, current_format_version, target_format_version)?; + + let current_schemas = table_metadata_schemas_by_id(current_metadata, current_format_version)?; + let target_schemas = table_metadata_schemas_by_id(target_metadata, target_format_version)?; + for (schema_id, current_schema) in ¤t_schemas { + if let Some(target_schema) = target_schemas.get(schema_id) + && normalized_schema_definition(current_schema, *schema_id)? + != normalized_schema_definition(target_schema, *schema_id)? + { + return Err(TableCatalogStoreError::Invalid(format!( + "existing schema {schema_id} must not be modified" + ))); + } + } + + let current_schema_id = table_metadata_current_schema_id(current_metadata, current_format_version)?; + let current_schema = current_schemas + .get(¤t_schema_id) + .ok_or_else(|| TableCatalogStoreError::Invalid("current table metadata schema does not exist".to_string()))?; + let current_fields = validate_iceberg_schema_fields(current_schema, "current schema")?; + for (target_schema_id, target_schema) in &target_schemas { + if current_schemas.contains_key(target_schema_id) { + continue; + } + let target_fields = validate_iceberg_schema_fields(target_schema, "target schema")?; + for (field_id, target_field) in &target_fields.descriptors { + match current_fields.descriptors.get(field_id) { + Some(current_field) => validate_schema_field_evolution(*field_id, current_field, target_field)?, + None if *field_id <= current_last_column_id => { + return Err(TableCatalogStoreError::Invalid(format!( + "schema field {field_id} cannot reuse a previously assigned field id" + ))); + } + None => {} + } + } + } + Ok(()) +} + +fn table_metadata_last_sequence_number(metadata: &serde_json::Value, format_version: u16) -> TableCatalogStoreResult { + if format_version == 1 { + return Ok(0); + } + require_metadata_i64(metadata, "last-sequence-number") +} + +fn normalized_sort_order_definitions( + metadata: &serde_json::Value, + format_version: u16, +) -> TableCatalogStoreResult> { + let Some(sort_orders) = metadata.get("sort-orders") else { + return if format_version == 1 { + Ok(BTreeMap::from([(0, serde_json::json!({"order-id": 0, "fields": []}))])) + } else { + Err(TableCatalogStoreError::Invalid("sort-orders must be an array".to_string())) + }; + }; + metadata_entries_by_id(sort_orders, "order-id", "sort order") +} + +fn validate_existing_snapshots_unchanged( + current_metadata: &serde_json::Value, + target_metadata: &serde_json::Value, + current_format_version: u16, + target_format_version: u16, +) -> TableCatalogStoreResult<()> { + let current = current_metadata + .get("snapshots") + .map(|snapshots| metadata_entries_by_id(snapshots, "snapshot-id", "snapshot")) + .transpose()? + .unwrap_or_default(); + let target = target_metadata + .get("snapshots") + .map(|snapshots| metadata_entries_by_id(snapshots, "snapshot-id", "snapshot")) + .transpose()? + .unwrap_or_default(); + for (snapshot_id, current_snapshot) in current { + let Some(target_snapshot) = target.get(&snapshot_id) else { + continue; + }; + let mut current_snapshot = current_snapshot; + let mut target_snapshot = target_snapshot.clone(); + if current_format_version == 1 && target_format_version == 2 { + for snapshot in [&mut current_snapshot, &mut target_snapshot] { + if snapshot.get("sequence-number").and_then(serde_json::Value::as_i64) == Some(0) + && let Some(object) = snapshot.as_object_mut() + { + object.remove("sequence-number"); + } + } + } + if current_snapshot != target_snapshot { + return Err(TableCatalogStoreError::Invalid(format!( + "existing snapshot {snapshot_id} must not be modified" + ))); + } + } + Ok(()) +} + +fn validate_existing_metadata_entries_unchanged( + current: &BTreeMap, + target: &BTreeMap, + label: &str, +) -> TableCatalogStoreResult<()> { + for (id, current_value) in current { + if let Some(target_value) = target.get(id) + && target_value != current_value + { + return Err(TableCatalogStoreError::Invalid(format!("existing {label} {id} must not be modified"))); + } + } + Ok(()) +} + +fn validate_existing_partition_specs_unchanged( + current_metadata: &serde_json::Value, + target_metadata: &serde_json::Value, +) -> TableCatalogStoreResult<()> { + let current = normalized_partition_spec_definitions(current_metadata)?; + let target = normalized_partition_spec_definitions(target_metadata)?; + for (spec_id, current_fields) in current { + if let Some(target_fields) = target.get(&spec_id) + && target_fields != ¤t_fields + { + return Err(TableCatalogStoreError::Invalid(format!( + "existing partition spec {spec_id} must not be modified" + ))); + } + } + Ok(()) +} + +fn metadata_entries_by_id( + value: &serde_json::Value, + id_field: &str, + label: &str, +) -> TableCatalogStoreResult> { + if value.is_null() { + return Ok(BTreeMap::new()); + } + let values = value + .as_array() + .ok_or_else(|| TableCatalogStoreError::Invalid(format!("{label}s must be an array")))?; + let mut entries = BTreeMap::new(); + for value in values { + let id = value + .get(id_field) + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| TableCatalogStoreError::Invalid(format!("{label} is missing {id_field}")))?; + if entries.insert(id, value.clone()).is_some() { + return Err(TableCatalogStoreError::Invalid(format!("duplicate {label} id {id}"))); + } + } + Ok(entries) +} + +fn normalized_schema_definition(schema: &serde_json::Value, schema_id: i64) -> TableCatalogStoreResult { + let mut schema = schema.clone(); + schema + .as_object_mut() + .ok_or_else(|| TableCatalogStoreError::Invalid("schema must be a JSON object".to_string()))? + .entry("schema-id".to_string()) + .or_insert_with(|| serde_json::Value::from(schema_id)); + Ok(schema) +} + +fn table_metadata_last_partition_id(metadata: &serde_json::Value, format_version: u16) -> TableCatalogStoreResult { + if format_version != 1 { + return require_metadata_i32(metadata, "last-partition-id"); + } + let field_count = require_metadata_array(metadata, "partition-spec")?.len(); + i32::try_from(field_count) + .ok() + .and_then(|field_count| 999_i32.checked_add(field_count)) + .ok_or_else(|| TableCatalogStoreError::Invalid("Iceberg v1 partition spec has too many fields".to_string())) +} + +fn table_metadata_current_schema_id(metadata: &serde_json::Value, format_version: u16) -> TableCatalogStoreResult { + if format_version == 1 { + return Ok(metadata + .get("schema") + .and_then(|schema| schema.get("schema-id")) + .and_then(serde_json::Value::as_i64) + .unwrap_or(0)); + } + metadata + .get("current-schema-id") + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| TableCatalogStoreError::Invalid("current-schema-id must be an integer".to_string())) +} + +fn table_metadata_schemas_by_id( + metadata: &serde_json::Value, + format_version: u16, +) -> TableCatalogStoreResult> { + let mut schemas = BTreeMap::new(); + if let Some(values) = metadata.get("schemas") { + for schema in values + .as_array() + .ok_or_else(|| TableCatalogStoreError::Invalid("schemas must be an array".to_string()))? + { + let schema_id = schema + .get("schema-id") + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| TableCatalogStoreError::Invalid("schema-id must be an integer".to_string()))?; + if schemas.insert(schema_id, schema).is_some() { + return Err(TableCatalogStoreError::Invalid(format!("duplicate schema id {schema_id}"))); + } + } + } + if format_version == 1 { + let schema = metadata + .get("schema") + .ok_or_else(|| TableCatalogStoreError::Invalid("Iceberg v1 table metadata is missing schema".to_string()))?; + let schema_id = schema.get("schema-id").and_then(serde_json::Value::as_i64).unwrap_or(0); + if let Some(known_schema) = schemas.get(&schema_id) { + let mut normalized_schema = schema.clone(); + normalized_schema + .as_object_mut() + .ok_or_else(|| TableCatalogStoreError::Invalid("Iceberg v1 schema must be an object".to_string()))? + .entry("schema-id".to_string()) + .or_insert_with(|| serde_json::Value::from(schema_id)); + if *known_schema != &normalized_schema { + return Err(TableCatalogStoreError::Invalid(format!( + "Iceberg v1 current schema {schema_id} does not match schemas" + ))); + } + } else { + schemas.insert(schema_id, schema); + } + } + Ok(schemas) +} + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum IcebergStatisticsFileKind { + Table, + Partition, +} + +pub(crate) fn validate_iceberg_statistics_file( + value: &serde_json::Value, + label: &str, + kind: IcebergStatisticsFileKind, +) -> TableCatalogStoreResult { + let object = value + .as_object() + .ok_or_else(|| TableCatalogStoreError::Invalid(format!("{label} must be a JSON object")))?; + let snapshot_id = object + .get("snapshot-id") + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| TableCatalogStoreError::Invalid(format!("{label}.snapshot-id must be an integer")))?; + if !object + .get("statistics-path") + .and_then(serde_json::Value::as_str) + .is_some_and(|path| !path.is_empty()) + { + return Err(TableCatalogStoreError::Invalid(format!( + "{label}.statistics-path must be a non-empty string" + ))); + } + let file_size = statistics_non_negative_i64(object, "file-size-in-bytes", label)?; + if matches!(kind, IcebergStatisticsFileKind::Table) { + let footer_size = statistics_non_negative_i64(object, "file-footer-size-in-bytes", label)?; + if footer_size > file_size { + return Err(TableCatalogStoreError::Invalid(format!( + "{label}.file-footer-size-in-bytes must not exceed file-size-in-bytes" + ))); + } + let blobs = object + .get("blob-metadata") + .and_then(serde_json::Value::as_array) + .ok_or_else(|| TableCatalogStoreError::Invalid(format!("{label}.blob-metadata must be an array")))?; + for blob in blobs { + validate_statistics_blob_metadata(blob, label)?; + } + } + Ok(snapshot_id) +} + +fn validate_statistics_blob_metadata(value: &serde_json::Value, label: &str) -> TableCatalogStoreResult<()> { + let object = value + .as_object() + .ok_or_else(|| TableCatalogStoreError::Invalid(format!("{label}.blob-metadata entries must be JSON objects")))?; + if !object + .get("type") + .and_then(serde_json::Value::as_str) + .is_some_and(|value| !value.is_empty()) + { + return Err(TableCatalogStoreError::Invalid(format!( + "{label}.blob-metadata type must be a non-empty string" + ))); + } + for field in ["snapshot-id", "sequence-number"] { + if object.get(field).and_then(serde_json::Value::as_i64).is_none() { + return Err(TableCatalogStoreError::Invalid(format!( + "{label}.blob-metadata {field} must be an integer" + ))); + } + } + let fields = object + .get("fields") + .and_then(serde_json::Value::as_array) + .ok_or_else(|| TableCatalogStoreError::Invalid(format!("{label}.blob-metadata fields must be an array")))?; + if fields.iter().any(|field| field.as_i64().is_none()) { + return Err(TableCatalogStoreError::Invalid(format!( + "{label}.blob-metadata fields must contain integers" + ))); + } + if let Some(properties) = object.get("properties") { + let properties = properties + .as_object() + .ok_or_else(|| TableCatalogStoreError::Invalid(format!("{label}.blob-metadata properties must be a JSON object")))?; + if properties.values().any(|value| !value.is_string()) { + return Err(TableCatalogStoreError::Invalid(format!( + "{label}.blob-metadata property values must be strings" + ))); + } + } + Ok(()) +} + +fn statistics_non_negative_i64( + object: &serde_json::Map, + field: &str, + label: &str, +) -> TableCatalogStoreResult { + let value = object + .get(field) + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| TableCatalogStoreError::Invalid(format!("{label}.{field} must be an integer")))?; + if value < 0 { + return Err(TableCatalogStoreError::Invalid(format!("{label}.{field} must not be negative"))); + } + Ok(value) +} + +fn validate_table_statistics_references( + metadata: &serde_json::Value, + snapshot_ids: &BTreeSet, +) -> TableCatalogStoreResult<()> { + for (field, kind) in [ + ("statistics", IcebergStatisticsFileKind::Table), + ("partition-statistics", IcebergStatisticsFileKind::Partition), + ] { + let Some(values) = metadata.get(field) else { + continue; + }; + let values = values + .as_array() + .ok_or_else(|| TableCatalogStoreError::Invalid(format!("table metadata field {field} must be an array")))?; + let mut snapshot_ids_with_statistics = BTreeSet::new(); + for value in values { + let snapshot_id = validate_iceberg_statistics_file(value, field, kind)?; + if !snapshot_ids.contains(&snapshot_id) { + return Err(TableCatalogStoreError::Invalid(format!( + "{field} references missing snapshot {snapshot_id}" + ))); + } + if !snapshot_ids_with_statistics.insert(snapshot_id) { + return Err(TableCatalogStoreError::Invalid(format!( + "{field} contains duplicate entries for snapshot {snapshot_id}" + ))); + } + } + } + Ok(()) +} + fn validate_supported_table_metadata_fields(metadata: &serde_json::Value) -> TableCatalogStoreResult<()> { table_metadata_uuid(metadata)?; table_metadata_location(metadata)?; @@ -508,12 +1075,16 @@ fn validate_supported_table_metadata_fields(metadata: &serde_json::Value) -> Tab if let Some(snapshots) = metadata.get("snapshots").and_then(serde_json::Value::as_array) { for snapshot in snapshots { validate_table_snapshot_fields(snapshot, 2)?; - let sequence_number = snapshot - .get("sequence-number") - .and_then(serde_json::Value::as_i64) - .ok_or_else(|| { + let sequence_number = match snapshot.get("sequence-number") { + Some(sequence_number) => sequence_number.as_i64().ok_or_else(|| { TableCatalogStoreError::Invalid("Iceberg v2 snapshot sequence-number must be an integer".to_string()) - })?; + })?, + None => { + return Err(TableCatalogStoreError::Invalid( + "Iceberg v2 snapshot sequence-number is required".to_string(), + )); + } + }; if sequence_number < 0 || sequence_number > last_sequence_number { return Err(TableCatalogStoreError::Invalid( "Iceberg v2 snapshot sequence-number must be between zero and last-sequence-number".to_string(), @@ -533,6 +1104,19 @@ fn validate_supported_table_metadata_fields(metadata: &serde_json::Value) -> Tab pub(crate) fn validate_table_metadata_references(metadata: &serde_json::Value) -> TableCatalogStoreResult<()> { let format_version = table_metadata_format_version(metadata)?; + let schema_fields = validate_table_schemas(metadata, format_version)?; + let current_schema_fields = current_table_schema_fields(metadata, format_version)?; + let last_column_id = require_metadata_i32(metadata, "last-column-id")?; + if last_column_id < 0 + || schema_fields + .field_ids + .last() + .is_some_and(|field_id| *field_id > last_column_id) + { + return Err(TableCatalogStoreError::Invalid( + "last-column-id must be non-negative and cover every assigned schema field id".to_string(), + )); + } let mut schema_ids = metadata_array_i32_ids(metadata, "schemas", "schema-id", "schema")?; if format_version == 1 && schema_ids.is_empty() { let schema = metadata @@ -545,6 +1129,9 @@ pub(crate) fn validate_table_metadata_references(metadata: &serde_json::Value) - .ok_or_else(|| TableCatalogStoreError::Invalid("Iceberg v1 schema-id must be an integer".to_string()))?, None => 0, }; + if schema_id < 0 { + return Err(TableCatalogStoreError::Invalid(format!("schema id {schema_id} must not be negative"))); + } if i32::try_from(schema_id).is_err() { return Err(TableCatalogStoreError::Invalid(format!( "schema id {schema_id} exceeds the signed 32-bit range" @@ -553,11 +1140,14 @@ pub(crate) fn validate_table_metadata_references(metadata: &serde_json::Value) - schema_ids.insert(schema_id); } validate_metadata_id_reference(metadata, "current-schema-id", &schema_ids, "schema")?; + validate_partition_specs(metadata, format_version, &schema_fields, ¤t_schema_fields)?; let spec_ids = metadata_array_i32_ids(metadata, "partition-specs", "spec-id", "partition spec")?; validate_metadata_id_reference(metadata, "default-spec-id", &spec_ids, "partition spec")?; + validate_sort_orders(metadata, &schema_fields, ¤t_schema_fields)?; let sort_order_ids = metadata_array_i32_ids(metadata, "sort-orders", "order-id", "sort order")?; validate_metadata_id_reference(metadata, "default-sort-order-id", &sort_order_ids, "sort order")?; let snapshot_ids = metadata_array_ids(metadata, "snapshots", "snapshot-id", "snapshot")?; + validate_table_statistics_references(metadata, &snapshot_ids)?; let current_snapshot_id = match metadata.get("current-snapshot-id").filter(|value| !value.is_null()) { Some(current_snapshot_id) => { @@ -623,6 +1213,714 @@ pub(crate) fn validate_table_metadata_references(metadata: &serde_json::Value) - Ok(()) } +fn validate_table_schemas(metadata: &serde_json::Value, format_version: u16) -> TableCatalogStoreResult { + let mut schemas = Vec::new(); + if format_version == 1 { + let schema = metadata + .get("schema") + .ok_or_else(|| TableCatalogStoreError::Invalid("Iceberg v1 table metadata is missing schema".to_string()))?; + let schema_id = schema.get("schema-id").and_then(serde_json::Value::as_i64).unwrap_or(0); + schemas.push((schema_id, schema)); + } + if let Some(metadata_schemas) = metadata.get("schemas") { + let metadata_schemas = metadata_schemas + .as_array() + .ok_or_else(|| TableCatalogStoreError::Invalid("schemas must be an array".to_string()))?; + for schema in metadata_schemas { + let schema_id = schema + .get("schema-id") + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| TableCatalogStoreError::Invalid("schema-id must be an integer".to_string()))?; + schemas.push((schema_id, schema)); + } + } + schemas.sort_by_key(|(schema_id, _)| *schema_id); + + let mut historical = BTreeMap::new(); + let mut active_fields = BTreeSet::new(); + let mut retired_fields = BTreeSet::new(); + let mut all_fields = IcebergSchemaFields::default(); + for (_, schema) in schemas { + let schema_fields = validate_iceberg_schema_fields(schema, "schema")?; + if let Some(field_id) = schema_fields + .field_ids + .iter() + .find(|field_id| retired_fields.contains(*field_id)) + { + return Err(TableCatalogStoreError::Invalid(format!( + "schema field {field_id} cannot be reused after removal" + ))); + } + for (field_id, descriptor) in &schema_fields.descriptors { + if let Some(previous) = historical.get(field_id) { + validate_schema_field_evolution(*field_id, previous, descriptor)?; + } + historical.insert(*field_id, descriptor.clone()); + } + retired_fields.extend(active_fields.difference(&schema_fields.field_ids).copied()); + active_fields = schema_fields.field_ids.clone(); + all_fields.field_ids.extend(schema_fields.field_ids); + all_fields.identifier_eligible.extend(schema_fields.identifier_eligible); + all_fields.descriptors.extend(schema_fields.descriptors); + } + Ok(all_fields) +} + +fn current_table_schema_fields( + metadata: &serde_json::Value, + format_version: u16, +) -> TableCatalogStoreResult { + if metadata.get("schemas").is_some() { + let current_schema_id = require_metadata_i32(metadata, "current-schema-id")?; + let schema = require_metadata_array(metadata, "schemas")? + .iter() + .find(|schema| schema.get("schema-id").and_then(serde_json::Value::as_i64) == Some(i64::from(current_schema_id))) + .ok_or_else(|| { + TableCatalogStoreError::Invalid(format!( + "current-schema-id targets schema {current_schema_id}, which does not exist" + )) + })?; + return validate_iceberg_schema_fields(schema, "current schema"); + } + if format_version == 1 { + let schema = metadata + .get("schema") + .ok_or_else(|| TableCatalogStoreError::Invalid("Iceberg v1 table metadata is missing schema".to_string()))?; + return validate_iceberg_schema_fields(schema, "schema"); + } + Err(TableCatalogStoreError::Invalid("schemas must be an array".to_string())) +} + +pub(crate) fn validate_partition_spec_sources_against_current_schema( + metadata: &serde_json::Value, + spec: &serde_json::Value, +) -> TableCatalogStoreResult<()> { + let current_schema_fields = current_table_schema_fields(metadata, table_metadata_format_version(metadata)?)?; + let fields = spec + .get("fields") + .and_then(serde_json::Value::as_array) + .ok_or_else(|| TableCatalogStoreError::Invalid("partition spec fields must be an array".to_string()))?; + for field in fields { + let field = field + .as_object() + .ok_or_else(|| TableCatalogStoreError::Invalid("partition spec fields must be JSON objects".to_string()))?; + let source_id = required_positive_i32_value(field, "source-id", "partition source-id")?; + let transform = field + .get("transform") + .and_then(serde_json::Value::as_str) + .filter(|value| !value.is_empty()) + .ok_or_else(|| TableCatalogStoreError::Invalid("partition field transform must be a non-empty string".to_string()))?; + if transform == "void" { + continue; + } + let source_type = current_schema_fields.descriptors.get(&source_id).ok_or_else(|| { + TableCatalogStoreError::Invalid(format!("partition source-id {source_id} does not reference the current schema")) + })?; + if source_type.inside_collection { + return Err(TableCatalogStoreError::Invalid(format!( + "partition source-id {source_id} must not be nested in a list or map" + ))); + } + validate_transform_for_source(transform, source_type, "partition field")?; + } + Ok(()) +} + +pub(crate) fn validate_sort_order_sources_against_current_schema( + metadata: &serde_json::Value, + sort_order: &serde_json::Value, +) -> TableCatalogStoreResult<()> { + let current_schema_fields = current_table_schema_fields(metadata, table_metadata_format_version(metadata)?)?; + let fields = sort_order + .get("fields") + .and_then(serde_json::Value::as_array) + .ok_or_else(|| TableCatalogStoreError::Invalid("sort order fields must be an array".to_string()))?; + for field in fields { + let field = field + .as_object() + .ok_or_else(|| TableCatalogStoreError::Invalid("sort order fields must be JSON objects".to_string()))?; + let source_id = required_positive_i32_value(field, "source-id", "sort field source-id")?; + let source_type = current_schema_fields.descriptors.get(&source_id).ok_or_else(|| { + TableCatalogStoreError::Invalid(format!("sort field source-id {source_id} does not reference the current schema")) + })?; + let transform = field + .get("transform") + .and_then(serde_json::Value::as_str) + .filter(|value| !value.is_empty()) + .ok_or_else(|| TableCatalogStoreError::Invalid("sort field transform must be a non-empty string".to_string()))?; + validate_transform_for_source(transform, source_type, "sort field")?; + } + Ok(()) +} + +fn validate_iceberg_schema(schema: &serde_json::Value, label: &str) -> TableCatalogStoreResult> { + Ok(validate_iceberg_schema_fields(schema, label)?.field_ids) +} + +fn validate_iceberg_schema_fields(schema: &serde_json::Value, label: &str) -> TableCatalogStoreResult { + let schema = schema + .as_object() + .ok_or_else(|| TableCatalogStoreError::Invalid(format!("{label} must be a JSON object")))?; + if schema.get("type").and_then(serde_json::Value::as_str) != Some("struct") { + return Err(TableCatalogStoreError::Invalid(format!("{label} type must be struct"))); + } + let fields = schema + .get("fields") + .and_then(serde_json::Value::as_array) + .ok_or_else(|| TableCatalogStoreError::Invalid(format!("{label} fields must be an array")))?; + let mut schema_fields = IcebergSchemaFields::default(); + validate_struct_fields(fields, label, true, false, &mut schema_fields)?; + if let Some(identifier_field_ids) = schema.get("identifier-field-ids") { + let identifier_field_ids = identifier_field_ids + .as_array() + .ok_or_else(|| TableCatalogStoreError::Invalid(format!("{label} identifier-field-ids must be an array")))?; + let mut seen = BTreeSet::new(); + for field_id in identifier_field_ids { + let field_id = required_positive_i32(field_id, &format!("{label} identifier field id"))?; + if !seen.insert(field_id) || schema_fields.identifier_eligible.get(&field_id) != Some(&true) { + return Err(TableCatalogStoreError::Invalid(format!( + "{label} identifier field id {field_id} must uniquely reference a required non-floating primitive outside lists, maps, and optional structs" + ))); + } + } + } + Ok(schema_fields) +} + +#[derive(Default)] +struct IcebergSchemaFields { + field_ids: BTreeSet, + identifier_eligible: BTreeMap, + descriptors: BTreeMap, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +struct IcebergSchemaFieldDescriptor { + field_type: IcebergFieldType, + required: bool, + inside_collection: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +enum IcebergFieldType { + Primitive(String), + Struct, + List, + Map, +} + +fn iceberg_field_type(value: &serde_json::Value, label: &str) -> TableCatalogStoreResult { + if let Some(primitive) = value.as_str() { + validate_iceberg_primitive_type(primitive, label)?; + return Ok(IcebergFieldType::Primitive(primitive.to_string())); + } + match value.get("type").and_then(serde_json::Value::as_str) { + Some("struct") => Ok(IcebergFieldType::Struct), + Some("list") => Ok(IcebergFieldType::List), + Some("map") => Ok(IcebergFieldType::Map), + _ => Err(TableCatalogStoreError::Invalid(format!("{label} contains an unsupported field type"))), + } +} + +fn insert_schema_field( + schema_fields: &mut IcebergSchemaFields, + field_id: i32, + field_type: &serde_json::Value, + required: bool, + inside_collection: bool, + label: &str, +) -> TableCatalogStoreResult<()> { + if !schema_fields.field_ids.insert(field_id) { + return Err(TableCatalogStoreError::Invalid(format!("duplicate {label} field id {field_id}"))); + } + schema_fields.descriptors.insert( + field_id, + IcebergSchemaFieldDescriptor { + field_type: iceberg_field_type(field_type, label)?, + required, + inside_collection, + }, + ); + Ok(()) +} + +fn validate_schema_field_evolution( + field_id: i32, + previous: &IcebergSchemaFieldDescriptor, + next: &IcebergSchemaFieldDescriptor, +) -> TableCatalogStoreResult<()> { + if !previous.required && next.required { + return Err(TableCatalogStoreError::Invalid(format!( + "schema field {field_id} cannot evolve from optional to required" + ))); + } + if previous.inside_collection != next.inside_collection { + return Err(TableCatalogStoreError::Invalid(format!( + "schema field {field_id} cannot move into or out of a list or map" + ))); + } + if previous.field_type == next.field_type { + return Ok(()); + } + let compatible = match (&previous.field_type, &next.field_type) { + (IcebergFieldType::Primitive(previous), IcebergFieldType::Primitive(next)) => { + primitive_type_promotion_is_valid(previous, next) + } + _ => false, + }; + if !compatible { + return Err(TableCatalogStoreError::Invalid(format!( + "schema field {field_id} has an incompatible type evolution" + ))); + } + Ok(()) +} + +fn primitive_type_promotion_is_valid(previous: &str, next: &str) -> bool { + if matches!((previous, next), ("int", "long") | ("float", "double")) { + return true; + } + let decimal = |value: &str| { + value + .strip_prefix("decimal(") + .and_then(|value| value.strip_suffix(')')) + .and_then(|parameters| parameters.split_once(',')) + .and_then(|(precision, scale)| Some((precision.trim().parse::().ok()?, scale.trim().parse::().ok()?))) + }; + matches!((decimal(previous), decimal(next)), (Some((previous_precision, previous_scale)), Some((next_precision, next_scale))) if previous_scale == next_scale && next_precision >= previous_precision) +} + +fn validate_struct_fields( + fields: &[serde_json::Value], + label: &str, + required_ancestors: bool, + inside_collection: bool, + schema_fields: &mut IcebergSchemaFields, +) -> TableCatalogStoreResult<()> { + for field in fields { + let field = field + .as_object() + .ok_or_else(|| TableCatalogStoreError::Invalid(format!("{label} fields must be JSON objects")))?; + let field_id = required_schema_field_id_value(field, "id", &format!("{label} field id"))?; + if !field + .get("name") + .and_then(serde_json::Value::as_str) + .is_some_and(|name| !name.is_empty()) + { + return Err(TableCatalogStoreError::Invalid(format!("{label} field name must be a non-empty string"))); + } + let required = field + .get("required") + .and_then(serde_json::Value::as_bool) + .ok_or_else(|| TableCatalogStoreError::Invalid(format!("{label} field required must be a boolean")))?; + let field_type = field + .get("type") + .ok_or_else(|| TableCatalogStoreError::Invalid(format!("{label} field type is required")))?; + insert_schema_field(schema_fields, field_id, field_type, required, inside_collection, label)?; + let identifier_eligible = required_ancestors + && required + && !inside_collection + && field_type + .as_str() + .is_some_and(|primitive| !matches!(primitive, "float" | "double")); + schema_fields.identifier_eligible.insert(field_id, identifier_eligible); + validate_iceberg_type(field_type, label, required_ancestors && required, inside_collection, schema_fields)?; + } + Ok(()) +} + +fn validate_iceberg_type( + field_type: &serde_json::Value, + label: &str, + required_ancestors: bool, + inside_collection: bool, + schema_fields: &mut IcebergSchemaFields, +) -> TableCatalogStoreResult<()> { + if let Some(primitive) = field_type.as_str() { + return validate_iceberg_primitive_type(primitive, label); + } + let field_type = field_type + .as_object() + .ok_or_else(|| TableCatalogStoreError::Invalid(format!("{label} field type must be a string or JSON object")))?; + match field_type.get("type").and_then(serde_json::Value::as_str) { + Some("struct") => { + let fields = field_type + .get("fields") + .and_then(serde_json::Value::as_array) + .ok_or_else(|| TableCatalogStoreError::Invalid(format!("{label} struct fields must be an array")))?; + validate_struct_fields(fields, label, required_ancestors, inside_collection, schema_fields) + } + Some("list") => { + let element_id = required_schema_field_id_value(field_type, "element-id", &format!("{label} list element-id"))?; + let element_required = field_type + .get("element-required") + .and_then(serde_json::Value::as_bool) + .ok_or_else(|| TableCatalogStoreError::Invalid(format!("{label} list element-required must be a boolean")))?; + let element = field_type + .get("element") + .ok_or_else(|| TableCatalogStoreError::Invalid(format!("{label} list element is required")))?; + insert_schema_field(schema_fields, element_id, element, element_required, true, label)?; + validate_iceberg_type(element, label, false, true, schema_fields) + } + Some("map") => { + let value_required = field_type + .get("value-required") + .and_then(serde_json::Value::as_bool) + .ok_or_else(|| TableCatalogStoreError::Invalid(format!("{label} map value-required must be a boolean")))?; + for (id_field, value_field, required) in [("key-id", "key", true), ("value-id", "value", value_required)] { + let field_id = required_schema_field_id_value(field_type, id_field, &format!("{label} map {id_field}"))?; + let value = field_type + .get(value_field) + .ok_or_else(|| TableCatalogStoreError::Invalid(format!("{label} map {value_field} is required")))?; + insert_schema_field(schema_fields, field_id, value, required, true, label)?; + validate_iceberg_type(value, label, false, true, schema_fields)?; + } + Ok(()) + } + _ => Err(TableCatalogStoreError::Invalid(format!("{label} contains an unsupported field type"))), + } +} + +fn validate_iceberg_primitive_type(primitive: &str, label: &str) -> TableCatalogStoreResult<()> { + if matches!( + primitive, + "boolean" + | "int" + | "long" + | "float" + | "double" + | "date" + | "time" + | "timestamp" + | "timestamptz" + | "string" + | "uuid" + | "binary" + ) { + return Ok(()); + } + if let Some(length) = primitive.strip_prefix("fixed[").and_then(|value| value.strip_suffix(']')) + && length.trim().parse::().is_ok_and(|length| length > 0) + { + return Ok(()); + } + if let Some(parameters) = primitive.strip_prefix("decimal(").and_then(|value| value.strip_suffix(')')) + && let Some((precision, scale)) = parameters.split_once(',') + && !scale.contains(',') + && let (Ok(precision), Ok(scale)) = (precision.trim().parse::(), scale.trim().parse::()) + && (1..=38).contains(&precision) + && scale <= precision + { + return Ok(()); + } + Err(TableCatalogStoreError::Invalid(format!( + "{label} contains unsupported primitive type {primitive}" + ))) +} + +fn required_i32_value( + object: &serde_json::Map, + field: &str, + label: &str, +) -> TableCatalogStoreResult { + let value = object + .get(field) + .ok_or_else(|| TableCatalogStoreError::Invalid(format!("{label} is required")))?; + required_i32(value, label) +} + +fn required_positive_i32_value( + object: &serde_json::Map, + field: &str, + label: &str, +) -> TableCatalogStoreResult { + let value = object + .get(field) + .ok_or_else(|| TableCatalogStoreError::Invalid(format!("{label} is required")))?; + required_positive_i32(value, label) +} + +fn required_schema_field_id_value( + object: &serde_json::Map, + field: &str, + label: &str, +) -> TableCatalogStoreResult { + let field_id = required_positive_i32_value(object, field, label)?; + if field_id > ICEBERG_MAX_USER_FIELD_ID { + return Err(TableCatalogStoreError::Invalid(format!( + "{label} must not use the reserved Iceberg field ID range" + ))); + } + Ok(field_id) +} + +fn required_i32(value: &serde_json::Value, label: &str) -> TableCatalogStoreResult { + let value = value + .as_i64() + .ok_or_else(|| TableCatalogStoreError::Invalid(format!("{label} must be an integer")))?; + i32::try_from(value).map_err(|_| TableCatalogStoreError::Invalid(format!("{label} exceeds the signed 32-bit range"))) +} + +fn required_positive_i32(value: &serde_json::Value, label: &str) -> TableCatalogStoreResult { + let value = required_i32(value, label)?; + if value <= 0 { + return Err(TableCatalogStoreError::Invalid(format!("{label} must be positive"))); + } + Ok(value) +} + +fn validate_partition_specs( + metadata: &serde_json::Value, + format_version: u16, + schema_fields: &IcebergSchemaFields, + current_schema_fields: &IcebergSchemaFields, +) -> TableCatalogStoreResult<()> { + let spec_fields = if format_version == 1 { + vec![(0, require_metadata_array(metadata, "partition-spec")?)] + } else { + metadata + .get("partition-specs") + .and_then(serde_json::Value::as_array) + .ok_or_else(|| TableCatalogStoreError::Invalid("partition-specs must be an array".to_string()))? + .iter() + .map(|spec| { + let spec_id = required_i32_value( + spec.as_object() + .ok_or_else(|| TableCatalogStoreError::Invalid("partition specs must be JSON objects".to_string()))?, + "spec-id", + "partition spec-id", + )?; + let fields = spec + .get("fields") + .and_then(serde_json::Value::as_array) + .ok_or_else(|| TableCatalogStoreError::Invalid("partition spec fields must be an array".to_string()))?; + Ok((spec_id, fields)) + }) + .collect::>>()? + }; + let default_spec_id = if format_version == 1 { + Some(0) + } else { + Some(require_metadata_i32(metadata, "default-spec-id")?) + }; + let last_partition_id = (format_version != 1) + .then(|| require_metadata_i32(metadata, "last-partition-id")) + .transpose()?; + if last_partition_id.is_some_and(|last_partition_id| last_partition_id < 0) { + return Err(TableCatalogStoreError::Invalid("last-partition-id must not be negative".to_string())); + } + let mut assigned_fields = BTreeMap::new(); + for (spec_id, fields) in spec_fields { + let mut field_ids = BTreeSet::new(); + for (field_index, field) in fields.iter().enumerate() { + let field = field + .as_object() + .ok_or_else(|| TableCatalogStoreError::Invalid("partition spec fields must be JSON objects".to_string()))?; + let source_id = required_positive_i32_value(field, "source-id", "partition source-id")?; + let transform = field + .get("transform") + .and_then(serde_json::Value::as_str) + .filter(|value| !value.is_empty()) + .ok_or_else(|| { + TableCatalogStoreError::Invalid("partition field transform must be a non-empty string".to_string()) + })?; + let source_fields = if default_spec_id == Some(spec_id) { + current_schema_fields + } else { + schema_fields + }; + let source_type = if transform == "void" { + None + } else { + let source_type = source_fields.descriptors.get(&source_id).ok_or_else(|| { + TableCatalogStoreError::Invalid(format!("partition source-id {source_id} does not reference a schema field")) + })?; + if source_type.inside_collection { + return Err(TableCatalogStoreError::Invalid(format!( + "partition source-id {source_id} must not be nested in a list or map" + ))); + } + Some(source_type) + }; + let field_id = if format_version == 1 { + let expected_field_id = i32::try_from(field_index) + .ok() + .and_then(|field_index| 1000_i32.checked_add(field_index)) + .ok_or_else(|| { + TableCatalogStoreError::Invalid("Iceberg v1 partition spec has too many fields".to_string()) + })?; + match field.get("field-id") { + Some(field_id) => { + let field_id = required_positive_i32(field_id, "partition field-id")?; + if field_id != expected_field_id { + return Err(TableCatalogStoreError::Invalid( + "Iceberg v1 partition field-id must be sequential from 1000".to_string(), + )); + } + field_id + } + None => expected_field_id, + } + } else { + required_positive_i32_value(field, "field-id", "partition field-id")? + }; + if last_partition_id.is_some_and(|last_partition_id| field_id > last_partition_id) || !field_ids.insert(field_id) { + return Err(TableCatalogStoreError::Invalid(format!( + "partition field-id {field_id} must be unique and not exceed last-partition-id" + ))); + } + if !field + .get("name") + .and_then(serde_json::Value::as_str) + .is_some_and(|value| !value.is_empty()) + { + return Err(TableCatalogStoreError::Invalid( + "partition field name must be a non-empty string".to_string(), + )); + } + if let Some(source_type) = source_type { + validate_transform_for_source(transform, source_type, "partition field")?; + } + let identity = (source_id, transform); + if format_version != 1 + && let Some(previous) = assigned_fields.insert(field_id, identity) + && previous != identity + { + return Err(TableCatalogStoreError::Invalid(format!( + "partition field-id {field_id} is assigned to multiple partition fields" + ))); + } + } + } + Ok(()) +} + +fn validate_sort_orders( + metadata: &serde_json::Value, + schema_fields: &IcebergSchemaFields, + current_schema_fields: &IcebergSchemaFields, +) -> TableCatalogStoreResult<()> { + let Some(sort_orders) = metadata.get("sort-orders") else { + return Ok(()); + }; + let sort_orders = sort_orders + .as_array() + .ok_or_else(|| TableCatalogStoreError::Invalid("sort-orders must be an array".to_string()))?; + let default_sort_order_id = metadata.get("default-sort-order-id").and_then(serde_json::Value::as_i64); + for sort_order in sort_orders { + let sort_order = sort_order + .as_object() + .ok_or_else(|| TableCatalogStoreError::Invalid("sort orders must be JSON objects".to_string()))?; + let order_id = required_i32_value(sort_order, "order-id", "sort order-id")?; + if order_id < 0 { + return Err(TableCatalogStoreError::Invalid("sort order-id must not be negative".to_string())); + } + let fields = sort_order + .get("fields") + .and_then(serde_json::Value::as_array) + .ok_or_else(|| TableCatalogStoreError::Invalid("sort order fields must be an array".to_string()))?; + if order_id == 0 && !fields.is_empty() { + return Err(TableCatalogStoreError::Invalid( + "sort order 0 is reserved for the unsorted order".to_string(), + )); + } + if order_id > 0 && fields.is_empty() { + return Err(TableCatalogStoreError::Invalid( + "empty sort orders must use the reserved unsorted order-id 0".to_string(), + )); + } + for field in fields { + let field = field + .as_object() + .ok_or_else(|| TableCatalogStoreError::Invalid("sort order fields must be JSON objects".to_string()))?; + let source_id = required_positive_i32_value(field, "source-id", "sort field source-id")?; + let source_fields = if default_sort_order_id == Some(i64::from(order_id)) { + current_schema_fields + } else { + schema_fields + }; + let source_type = source_fields.descriptors.get(&source_id).ok_or_else(|| { + TableCatalogStoreError::Invalid(format!("sort field source-id {source_id} does not reference a schema field")) + })?; + let transform = field + .get("transform") + .and_then(serde_json::Value::as_str) + .filter(|transform| !transform.is_empty()) + .ok_or_else(|| TableCatalogStoreError::Invalid("sort field transform must be a non-empty string".to_string()))?; + validate_transform_for_source(transform, source_type, "sort field")?; + if !field + .get("direction") + .and_then(serde_json::Value::as_str) + .is_some_and(|direction| matches!(direction, "asc" | "desc")) + { + return Err(TableCatalogStoreError::Invalid("sort field direction must be asc or desc".to_string())); + } + if !field + .get("null-order") + .and_then(serde_json::Value::as_str) + .is_some_and(|null_order| matches!(null_order, "nulls-first" | "nulls-last")) + { + return Err(TableCatalogStoreError::Invalid( + "sort field null-order must be nulls-first or nulls-last".to_string(), + )); + } + } + } + Ok(()) +} + +fn validate_transform_for_source( + transform: &str, + source: &IcebergSchemaFieldDescriptor, + label: &str, +) -> TableCatalogStoreResult<()> { + if transform == "void" { + return Ok(()); + } + let IcebergFieldType::Primitive(source_type) = &source.field_type else { + return Err(TableCatalogStoreError::Invalid(format!( + "{label} transform {transform} requires a primitive source type" + ))); + }; + let valid = match transform { + "identity" => true, + "year" | "month" | "day" => matches!(source_type.as_str(), "date" | "timestamp" | "timestamptz"), + "hour" => matches!(source_type.as_str(), "timestamp" | "timestamptz"), + _ => match transform_parameter(transform, "bucket") { + Some(width) => { + width > 0 + && (matches!( + source_type.as_str(), + "int" | "long" | "date" | "time" | "timestamp" | "timestamptz" | "string" | "uuid" | "binary" + ) || source_type.starts_with("decimal(") + || source_type.starts_with("fixed[")) + } + None => match transform_parameter(transform, "truncate") { + Some(width) => { + width > 0 + && (matches!(source_type.as_str(), "int" | "long" | "string" | "binary") + || source_type.starts_with("decimal(")) + } + None => false, + }, + }, + }; + if !valid { + return Err(TableCatalogStoreError::Invalid(format!( + "{label} transform {transform} is invalid for source type {source_type}" + ))); + } + Ok(()) +} + +fn transform_parameter(transform: &str, name: &str) -> Option { + transform + .strip_prefix(name) + .and_then(|value| value.strip_prefix('[')) + .and_then(|value| value.strip_suffix(']')) + .and_then(|value| value.parse::().ok()) +} + fn validate_table_snapshot_fields(snapshot: &serde_json::Value, format_version: u16) -> TableCatalogStoreResult<()> { let snapshot = snapshot .as_object() @@ -633,6 +1931,14 @@ fn validate_table_snapshot_fields(snapshot: &serde_json::Value, format_version: if snapshot.get("timestamp-ms").and_then(serde_json::Value::as_i64).is_none() { return Err(TableCatalogStoreError::Invalid("snapshot timestamp-ms must be an integer".to_string())); } + if snapshot + .get("parent-snapshot-id") + .is_some_and(|parent_snapshot_id| parent_snapshot_id.as_i64().is_none()) + { + return Err(TableCatalogStoreError::Invalid( + "snapshot parent-snapshot-id must be an integer".to_string(), + )); + } let manifest_list = snapshot .get("manifest-list") .and_then(serde_json::Value::as_str) @@ -641,6 +1947,10 @@ fn validate_table_snapshot_fields(snapshot: &serde_json::Value, format_version: if manifests.is_some_and(|manifests| !manifests.is_array()) { return Err(TableCatalogStoreError::Invalid("snapshot manifests must be an array".to_string())); } + let summary = snapshot.get("summary"); + if let Some(summary) = summary { + validate_string_map(summary, "snapshot summary")?; + } match format_version { 1 => { @@ -666,8 +1976,7 @@ fn validate_table_snapshot_fields(snapshot: &serde_json::Value, format_version: "Iceberg v2 snapshot requires manifest-list or v1-compatible manifests".to_string(), )); } - let summary = snapshot - .get("summary") + let summary = summary .and_then(serde_json::Value::as_object) .ok_or_else(|| TableCatalogStoreError::Invalid("Iceberg v2 snapshot requires summary".to_string()))?; if !summary @@ -731,24 +2040,162 @@ pub(crate) fn table_metadata_partition_spec_ids(metadata: &serde_json::Value) -> Err(TableCatalogStoreError::Invalid("table metadata has no partition specs".to_string())) } -pub(crate) fn validate_view_metadata_references(metadata: &serde_json::Value) -> TableCatalogStoreResult<()> { +pub(crate) fn validate_supported_view_metadata(metadata: &serde_json::Value) -> TableCatalogStoreResult<()> { + let object = metadata + .as_object() + .ok_or_else(|| TableCatalogStoreError::Invalid("view metadata must be a JSON object".to_string()))?; + let format_version = object + .get("format-version") + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| TableCatalogStoreError::Invalid("view metadata is missing integer field format-version".to_string()))?; + if format_version != 1 { + return Err(TableCatalogStoreError::Unsupported(format!( + "Iceberg view format-version {format_version}" + ))); + } + for field in ["view-uuid", "location"] { + if !object + .get(field) + .and_then(serde_json::Value::as_str) + .is_some_and(|value| !value.is_empty()) + { + return Err(TableCatalogStoreError::Invalid(format!( + "view metadata is missing non-empty string field {field}" + ))); + } + } + let schemas = view_metadata_array(metadata, "schemas")?; + if schemas.is_empty() { + return Err(TableCatalogStoreError::Invalid("view metadata schemas must not be empty".to_string())); + } + for schema in schemas { + validate_iceberg_schema(schema, "view schema")?; + } let schema_ids = metadata_array_i32_ids(metadata, "schemas", "schema-id", "schema")?; validate_metadata_id_reference(metadata, "current-schema-id", &schema_ids, "schema")?; + let versions = view_metadata_array(metadata, "versions")?; + if versions.is_empty() { + return Err(TableCatalogStoreError::Invalid("view metadata versions must not be empty".to_string())); + } + if object.get("current-version-id").and_then(serde_json::Value::as_i64).is_none() { + return Err(TableCatalogStoreError::Invalid( + "view metadata is missing integer field current-version-id".to_string(), + )); + } let version_ids = metadata_array_i32_ids(metadata, "versions", "version-id", "view version")?; validate_metadata_id_reference(metadata, "current-version-id", &version_ids, "view version")?; - if let Some(versions) = metadata.get("versions").and_then(serde_json::Value::as_array) { - for version in versions { - let schema_id = version - .get("schema-id") - .and_then(serde_json::Value::as_i64) - .ok_or_else(|| TableCatalogStoreError::Invalid("view version is missing schema-id".to_string()))?; - if !schema_ids.contains(&schema_id) { - return Err(TableCatalogStoreError::Invalid(format!( - "view version schema-id targets schema {schema_id}, which does not exist" - ))); - } + for version in versions { + validate_view_version(version)?; + let schema_id = version + .get("schema-id") + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| TableCatalogStoreError::Invalid("view version is missing schema-id".to_string()))?; + if !schema_ids.contains(&schema_id) { + return Err(TableCatalogStoreError::Invalid(format!( + "view version schema-id targets schema {schema_id}, which does not exist" + ))); } } + let version_log = view_metadata_array(metadata, "version-log")?; + for entry in version_log { + let version_id = entry + .get("version-id") + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| TableCatalogStoreError::Invalid("view version-log entry is missing version-id".to_string()))?; + if entry.get("timestamp-ms").and_then(serde_json::Value::as_i64).is_none() { + return Err(TableCatalogStoreError::Invalid( + "view version-log entries require integer version-id and timestamp-ms".to_string(), + )); + } + if !version_ids.contains(&version_id) { + return Err(TableCatalogStoreError::Invalid(format!( + "view version-log targets view version {version_id}, which does not exist" + ))); + } + } + if let Some(properties) = object.get("properties") { + validate_string_map(properties, "view metadata properties")?; + } + Ok(()) +} + +fn view_metadata_array<'a>(metadata: &'a serde_json::Value, field: &str) -> TableCatalogStoreResult<&'a Vec> { + metadata + .get(field) + .and_then(serde_json::Value::as_array) + .ok_or_else(|| TableCatalogStoreError::Invalid(format!("view metadata is missing array field {field}"))) +} + +fn validate_view_version(version: &serde_json::Value) -> TableCatalogStoreResult<()> { + let version = version + .as_object() + .ok_or_else(|| TableCatalogStoreError::Invalid("view version must be a JSON object".to_string()))?; + for field in ["version-id", "timestamp-ms", "schema-id"] { + if version.get(field).and_then(serde_json::Value::as_i64).is_none() { + return Err(TableCatalogStoreError::Invalid(format!("view version is missing integer field {field}"))); + } + } + let summary = version + .get("summary") + .ok_or_else(|| TableCatalogStoreError::Invalid("view version is missing summary".to_string()))?; + validate_string_map(summary, "view version summary")?; + let default_namespace = version + .get("default-namespace") + .and_then(serde_json::Value::as_array) + .ok_or_else(|| TableCatalogStoreError::Invalid("view version is missing default-namespace".to_string()))?; + if default_namespace.iter().any(|segment| !segment.is_string()) { + return Err(TableCatalogStoreError::Invalid( + "view version default-namespace must contain strings".to_string(), + )); + } + if version.get("default-catalog").is_some_and(|value| !value.is_string()) { + return Err(TableCatalogStoreError::Invalid( + "view version default-catalog must be a string".to_string(), + )); + } + let representations = version + .get("representations") + .and_then(serde_json::Value::as_array) + .ok_or_else(|| TableCatalogStoreError::Invalid("view version is missing representations".to_string()))?; + let mut dialects = BTreeSet::new(); + for representation in representations { + let representation = representation + .as_object() + .ok_or_else(|| TableCatalogStoreError::Invalid("view representation must be a JSON object".to_string()))?; + let dialect = representation + .get("dialect") + .and_then(serde_json::Value::as_str) + .filter(|dialect| !dialect.is_empty()); + if representation.get("type").and_then(serde_json::Value::as_str) != Some("sql") + || representation.get("sql").and_then(serde_json::Value::as_str).is_none() + || dialect.is_none() + { + return Err(TableCatalogStoreError::Invalid( + "view representation requires type sql, sql, and dialect strings".to_string(), + )); + } + if !dialects.insert( + dialect + .ok_or_else(|| { + TableCatalogStoreError::Invalid("view representation dialect must be a non-empty string".to_string()) + })? + .to_lowercase(), + ) { + return Err(TableCatalogStoreError::Invalid( + "view version contains duplicate SQL dialect representations".to_string(), + )); + } + } + Ok(()) +} + +fn validate_string_map(value: &serde_json::Value, label: &str) -> TableCatalogStoreResult<()> { + let values = value + .as_object() + .ok_or_else(|| TableCatalogStoreError::Invalid(format!("{label} must be a JSON object")))?; + if values.values().any(|value| !value.is_string()) { + return Err(TableCatalogStoreError::Invalid(format!("{label} values must be strings"))); + } Ok(()) } @@ -792,6 +2239,9 @@ fn metadata_array_i32_ids( label: &str, ) -> TableCatalogStoreResult> { let ids = metadata_array_ids(metadata, array_field, id_field, label)?; + if let Some(id) = ids.iter().find(|id| **id < 0) { + return Err(TableCatalogStoreError::Invalid(format!("{label} id {id} must not be negative"))); + } if let Some(id) = ids.iter().find(|id| i32::try_from(**id).is_err()) { return Err(TableCatalogStoreError::Invalid(format!( "{label} id {id} exceeds the signed 32-bit range" @@ -875,14 +2325,22 @@ impl<'a, B> TableSnapshotGraphValidationContext<'a, B> { #[derive(Default)] struct SnapshotGraphReadBudget { manifest_count: usize, + manifest_traversal_count: usize, avro_bytes: usize, decoded_avro_bytes: usize, file_reference_count: usize, - manifest_lists: BTreeMap>, - manifests: BTreeMap>, + manifest_lists: BTreeMap>>, + manifests: BTreeMap, validated_live_objects: BTreeSet, } +#[derive(Clone)] +struct CachedSnapshotGraphManifest { + object_size: usize, + partition_spec_id: Option, + references: Arc>, +} + impl SnapshotGraphReadBudget { fn charge_manifests(&mut self, count: usize) -> TableCatalogStoreResult<()> { self.manifest_count = self @@ -897,6 +2355,18 @@ impl SnapshotGraphReadBudget { Ok(()) } + fn charge_manifest_traversals(&mut self, count: usize) -> TableCatalogStoreResult<()> { + self.manifest_traversal_count = self.manifest_traversal_count.checked_add(count).ok_or_else(|| { + TableCatalogStoreError::Invalid("snapshot manifest traversal count exceeds the commit limit".to_string()) + })?; + if self.manifest_traversal_count > TABLE_COMMIT_MAX_MANIFEST_TRAVERSALS { + return Err(TableCatalogStoreError::Invalid( + "snapshot manifest traversal count exceeds the commit limit".to_string(), + )); + } + Ok(()) + } + fn charge_avro_bytes(&mut self, count: usize) -> TableCatalogStoreResult<()> { self.avro_bytes = self .avro_bytes @@ -962,14 +2432,25 @@ where B: TableCatalogObjectBackend, { validate_supported_table_metadata_fields(metadata)?; + let snapshot_ids = metadata_array_ids(metadata, "snapshots", "snapshot-id", "snapshot")?; + validate_table_statistics_references(metadata, &snapshot_ids)?; + let mut budget = SnapshotGraphReadBudget::default(); + validate_table_statistics_objects(context, metadata).await?; let format_version = table_metadata_format_version(metadata)?; let snapshots = snapshots_requiring_graph_validation(current_metadata, metadata)?; - let mut budget = SnapshotGraphReadBudget::default(); for snapshot in snapshots { snapshot .get("snapshot-id") .and_then(serde_json::Value::as_i64) .ok_or_else(|| TableCatalogStoreError::Invalid("snapshot-id must be an integer".to_string()))?; + if format_version == 2 + && snapshot.get("manifests").is_some() + && !snapshot_is_retained_v1_history(current_metadata, snapshot) + { + return Err(TableCatalogStoreError::Invalid( + "new Iceberg v2 snapshots require manifest-list".to_string(), + )); + } let snapshot_sequence_number = snapshot .get("sequence-number") .and_then(serde_json::Value::as_i64) @@ -984,9 +2465,9 @@ where ) .await?; let mut seen_files = BTreeSet::new(); - for references in manifests { - for reference in references { - if !seen_files.insert(reference.location) { + for references in &manifests { + for reference in references.iter() { + if !seen_files.insert(reference.location.as_str()) { return Err(TableCatalogStoreError::Invalid( "snapshot contains a duplicate file reference".to_string(), )); @@ -997,6 +2478,163 @@ where Ok(()) } +fn snapshot_is_retained_v1_history(current_metadata: Option<&serde_json::Value>, target_snapshot: &serde_json::Value) -> bool { + let Some(current_metadata) = current_metadata else { + return false; + }; + if table_metadata_format_version(current_metadata).ok() != Some(1) { + return false; + } + let Some(snapshot_id) = target_snapshot.get("snapshot-id").and_then(serde_json::Value::as_i64) else { + return false; + }; + let Some(current_snapshot) = current_metadata + .get("snapshots") + .and_then(serde_json::Value::as_array) + .and_then(|snapshots| { + snapshots + .iter() + .find(|snapshot| snapshot.get("snapshot-id").and_then(serde_json::Value::as_i64) == Some(snapshot_id)) + }) + else { + return false; + }; + let mut current_snapshot = current_snapshot.clone(); + let mut target_snapshot = target_snapshot.clone(); + for snapshot in [&mut current_snapshot, &mut target_snapshot] { + if snapshot.get("sequence-number").and_then(serde_json::Value::as_i64) == Some(0) + && let Some(object) = snapshot.as_object_mut() + { + object.remove("sequence-number"); + } + } + current_snapshot == target_snapshot +} + +async fn validate_table_statistics_objects( + context: &TableSnapshotGraphValidationContext<'_, B>, + metadata: &serde_json::Value, +) -> TableCatalogStoreResult<()> +where + B: TableCatalogObjectBackend, +{ + if context.entry.table_bucket != context.table_bucket { + return Err(TableCatalogStoreError::Invalid( + "statistics object is outside the table bucket".to_string(), + )); + } + let warehouse_object_prefix = table_warehouse_object_prefix(context.entry)?; + let mut objects = BTreeMap::new(); + let mut total_size = 0usize; + for (field, kind) in [ + ("statistics", IcebergStatisticsFileKind::Table), + ("partition-statistics", IcebergStatisticsFileKind::Partition), + ] { + let Some(values) = metadata.get(field).and_then(serde_json::Value::as_array) else { + continue; + }; + for value in values { + let location = value + .get("statistics-path") + .and_then(serde_json::Value::as_str) + .ok_or_else(|| TableCatalogStoreError::Invalid(format!("{field}.statistics-path must be a string")))?; + let object_key = table_catalog_object_key_from_location(context.table_bucket, location) + .ok_or_else(|| TableCatalogStoreError::Invalid(format!("{field} object location is invalid")))?; + if !object_key.starts_with(&warehouse_object_prefix) { + return Err(TableCatalogStoreError::Invalid(format!("{field} object is outside the table warehouse"))); + } + let file_size = value + .get("file-size-in-bytes") + .and_then(serde_json::Value::as_u64) + .and_then(|size| usize::try_from(size).ok()) + .filter(|size| *size <= TABLE_STATISTICS_FILE_MAX_SIZE) + .ok_or_else(|| { + TableCatalogStoreError::Invalid(format!("{field} file-size-in-bytes exceeds the validation limit")) + })?; + if let Some(previous) = objects.get(&object_key) { + if *previous != (file_size, kind) { + return Err(TableCatalogStoreError::Invalid( + "statistics object is declared with inconsistent metadata".to_string(), + )); + } + continue; + } + total_size = total_size + .checked_add(file_size) + .filter(|size| *size <= TABLE_COMMIT_MAX_STATISTICS_BYTES) + .ok_or_else(|| { + TableCatalogStoreError::Invalid("statistics bytes exceed the commit validation limit".to_string()) + })?; + objects.insert(object_key, (file_size, kind)); + } + } + + if objects.len() > TABLE_COMMIT_MAX_STATISTICS_OBJECTS { + return Err(TableCatalogStoreError::Invalid( + "statistics object count exceeds the commit limit".to_string(), + )); + } + let backend = context.backend.clone(); + let bucket = context.table_bucket.to_string(); + stream::iter(objects) + .map(move |(object_key, (expected_size, kind))| { + let backend = backend.clone(); + let bucket = bucket.clone(); + async move { + let object = backend + .read_object_limited(&bucket, &object_key, expected_size) + .await? + .ok_or_else(|| TableCatalogStoreError::Invalid("statistics object is missing".to_string()))?; + if object.data.len() != expected_size { + return Err(TableCatalogStoreError::Invalid( + "statistics file-size-in-bytes does not match the object".to_string(), + )); + } + let valid_magic = match kind { + IcebergStatisticsFileKind::Table => object.data.starts_with(b"PFA1") && object.data.ends_with(b"PFA1"), + IcebergStatisticsFileKind::Partition => object.data.starts_with(b"PAR1") && object.data.ends_with(b"PAR1"), + }; + if !valid_magic { + return Err(TableCatalogStoreError::Invalid(match kind { + IcebergStatisticsFileKind::Table => "table statistics object is not a Puffin file".to_string(), + IcebergStatisticsFileKind::Partition => "partition statistics object is not a Parquet file".to_string(), + })); + } + Ok(()) + } + }) + .buffered(TABLE_COMMIT_OBJECT_VALIDATION_CONCURRENCY) + .try_for_each(|()| async { Ok(()) }) + .await +} + +async fn validate_object_keys_exist( + backend: &B, + bucket: &str, + object_keys: impl IntoIterator, + missing_message: &'static str, +) -> TableCatalogStoreResult<()> +where + B: TableCatalogObjectBackend, +{ + let backend = backend.clone(); + let bucket = bucket.to_string(); + stream::iter(object_keys) + .map(move |object_key| { + let backend = backend.clone(); + let bucket = bucket.clone(); + async move { + if !backend.object_exists(&bucket, &object_key).await? { + return Err(TableCatalogStoreError::Invalid(missing_message.to_string())); + } + Ok(()) + } + }) + .buffered(TABLE_COMMIT_OBJECT_VALIDATION_CONCURRENCY) + .try_for_each(|()| async { Ok(()) }) + .await +} + fn snapshots_requiring_graph_validation<'a>( current_metadata: Option<&serde_json::Value>, metadata: &'a serde_json::Value, @@ -1009,6 +2647,9 @@ fn snapshots_requiring_graph_validation<'a>( .ok_or_else(|| TableCatalogStoreError::Invalid("snapshots must be an array".to_string()))?; if let Some(current_metadata) = current_metadata { + if !partition_specs_preserve_existing_definitions(current_metadata, metadata)? { + return Ok(snapshots.iter().collect()); + } let current_snapshots = current_metadata .get("snapshots") .and_then(serde_json::Value::as_array) @@ -1030,29 +2671,81 @@ fn snapshots_requiring_graph_validation<'a>( .collect()); } - let mut active_snapshot_ids = BTreeSet::new(); - if let Some(snapshot_id) = metadata - .get("current-snapshot-id") - .and_then(serde_json::Value::as_i64) - .filter(|snapshot_id| *snapshot_id != -1) - { - active_snapshot_ids.insert(snapshot_id); - } - if let Some(refs) = metadata.get("refs").and_then(serde_json::Value::as_object) { - active_snapshot_ids.extend( - refs.values() - .filter_map(|reference| reference.get("snapshot-id").and_then(serde_json::Value::as_i64)), - ); - } - Ok(snapshots + Ok(snapshots.iter().collect()) +} + +fn partition_specs_preserve_existing_definitions( + current_metadata: &serde_json::Value, + metadata: &serde_json::Value, +) -> TableCatalogStoreResult { + let current_specs = normalized_partition_spec_definitions(current_metadata)?; + let target_specs = normalized_partition_spec_definitions(metadata)?; + Ok(current_specs .iter() - .filter(|snapshot| { - snapshot - .get("snapshot-id") - .and_then(serde_json::Value::as_i64) - .is_some_and(|snapshot_id| active_snapshot_ids.contains(&snapshot_id)) + .all(|(spec_id, fields)| target_specs.get(spec_id) == Some(fields))) +} + +type NormalizedPartitionFieldDefinition = (i32, i32, String, String); +type NormalizedPartitionSpecDefinitions = BTreeMap>; + +fn normalized_partition_spec_definitions( + metadata: &serde_json::Value, +) -> TableCatalogStoreResult { + let format_version = table_metadata_format_version(metadata)?; + let specs = if format_version == 1 { + vec![(0, require_metadata_array(metadata, "partition-spec")?)] + } else { + require_metadata_array(metadata, "partition-specs")? + .iter() + .map(|spec| { + let spec = spec + .as_object() + .ok_or_else(|| TableCatalogStoreError::Invalid("partition specs must be JSON objects".to_string()))?; + let spec_id = required_i32_value(spec, "spec-id", "partition spec-id")?; + let fields = spec + .get("fields") + .and_then(serde_json::Value::as_array) + .ok_or_else(|| TableCatalogStoreError::Invalid("partition spec fields must be an array".to_string()))?; + Ok((spec_id, fields)) + }) + .collect::>>()? + }; + specs + .into_iter() + .map(|(spec_id, fields)| { + let fields = fields + .iter() + .enumerate() + .map(|(index, field)| { + let field = field.as_object().ok_or_else(|| { + TableCatalogStoreError::Invalid("partition spec fields must be JSON objects".to_string()) + })?; + let source_id = required_positive_i32_value(field, "source-id", "partition source-id")?; + let field_id = match field.get("field-id") { + Some(field_id) => required_positive_i32(field_id, "partition field-id")?, + None if format_version == 1 => i32::try_from(index) + .ok() + .and_then(|index| 1000_i32.checked_add(index)) + .ok_or_else(|| { + TableCatalogStoreError::Invalid("Iceberg v1 partition spec has too many fields".to_string()) + })?, + None => { + return Err(TableCatalogStoreError::Invalid("partition field-id is required".to_string())); + } + }; + let transform = field.get("transform").and_then(serde_json::Value::as_str).ok_or_else(|| { + TableCatalogStoreError::Invalid("partition field transform must be a string".to_string()) + })?; + let name = field + .get("name") + .and_then(serde_json::Value::as_str) + .ok_or_else(|| TableCatalogStoreError::Invalid("partition field name must be a string".to_string()))?; + Ok((source_id, field_id, name.to_string(), transform.to_string())) + }) + .collect::>>()?; + Ok((spec_id, fields)) }) - .collect()) + .collect() } async fn snapshot_graph_manifest_references( @@ -1062,7 +2755,7 @@ async fn snapshot_graph_manifest_references( format_version: u16, snapshot_sequence_number: i64, budget: &mut SnapshotGraphReadBudget, -) -> TableCatalogStoreResult>> +) -> TableCatalogStoreResult>>> where B: TableCatalogObjectBackend, { @@ -1071,8 +2764,9 @@ where let partition_spec_ids = table_metadata_partition_spec_ids(metadata)?; let mut manifests = Vec::with_capacity(manifest_locations.len()); let mut seen_manifest_paths = BTreeSet::new(); - for manifest_location in manifest_locations { - validate_snapshot_graph_manifest_location(&manifest_location, format_version, snapshot_sequence_number)?; + for manifest_location in manifest_locations.iter() { + budget.charge_manifest_traversals(1)?; + validate_snapshot_graph_manifest_location(manifest_location, format_version, snapshot_sequence_number)?; match manifest_location.partition_spec_id { Some(partition_spec_id) if !partition_spec_ids.contains(&partition_spec_id) => { return Err(TableCatalogStoreError::Invalid(format!( @@ -1086,7 +2780,7 @@ where } _ => {} } - if !seen_manifest_paths.insert(manifest_location.manifest_path.clone()) { + if !seen_manifest_paths.insert(manifest_location.manifest_path.as_str()) { return Err(TableCatalogStoreError::Invalid( "snapshot contains a duplicate manifest reference".to_string(), )); @@ -1096,8 +2790,8 @@ where &manifest_location.manifest_path, TableMetadataMaintenanceObjectKind::ManifestFile, )?; - let references = if let Some(references) = budget.manifests.get(&manifest_key).cloned() { - references + let cached_manifest = if let Some(manifest) = budget.manifests.get(&manifest_key) { + manifest.clone() } else { budget.charge_manifests(1)?; let manifest_object = context @@ -1109,13 +2803,37 @@ where budget.charge_avro_bytes(manifest_size)?; let decoded_manifest = decode_manifest_avro_async(manifest_object.data).await?; budget.charge_decoded_avro_bytes(decoded_manifest.decoded_size)?; - budget.charge_file_references(decoded_manifest.references.len())?; - budget.manifests.insert(manifest_key, decoded_manifest.references.clone()); - decoded_manifest.references + let manifest = CachedSnapshotGraphManifest { + object_size: manifest_size, + partition_spec_id: decoded_manifest.partition_spec_id, + references: Arc::new(decoded_manifest.references), + }; + budget.manifests.insert(manifest_key, manifest.clone()); + manifest }; + if manifest_location + .manifest_length + .is_some_and(|declared| u64::try_from(cached_manifest.object_size).ok() != Some(declared)) + { + return Err(TableCatalogStoreError::Invalid( + "manifest-list manifest_length does not match the manifest object".to_string(), + )); + } + if manifest_location.from_manifest_list + && cached_manifest + .partition_spec_id + .is_some_and(|manifest_spec_id| Some(manifest_spec_id) != manifest_location.partition_spec_id) + { + return Err(TableCatalogStoreError::Invalid( + "manifest partition-spec-id does not match its manifest-list entry".to_string(), + )); + } + let references = cached_manifest.references; + validate_snapshot_graph_manifest_content(manifest_location, references.as_ref())?; + budget.charge_file_references(references.len())?; validate_snapshot_graph_data_files( context, - &references, + references.as_ref(), budget, format_version, if manifest_location.from_manifest_list && manifest_location.format_version == 1 { @@ -1130,21 +2848,43 @@ where Ok(manifests) } +fn validate_snapshot_graph_manifest_content( + manifest: &SnapshotGraphManifestLocation, + references: &[ManifestDataFileReference], +) -> TableCatalogStoreResult<()> { + let content_matches = match manifest.content { + Some(0) => references + .iter() + .all(|reference| reference.content == ManifestDataFileContent::Data), + Some(1) => references + .iter() + .all(|reference| reference.content != ManifestDataFileContent::Data), + None => true, + Some(_) => false, + }; + if !content_matches { + return Err(TableCatalogStoreError::Invalid( + "manifest-list content does not match manifest file content".to_string(), + )); + } + Ok(()) +} + async fn snapshot_graph_manifest_locations( context: &TableSnapshotGraphValidationContext<'_, B>, snapshot: &serde_json::Value, format_version: u16, snapshot_sequence_number: i64, budget: &mut SnapshotGraphReadBudget, -) -> TableCatalogStoreResult> +) -> TableCatalogStoreResult>> where B: TableCatalogObjectBackend, { if let Some(manifest_list_location) = snapshot.get("manifest-list").and_then(serde_json::Value::as_str) { let manifest_list_key = snapshot_graph_object_key(context, manifest_list_location, TableMetadataMaintenanceObjectKind::ManifestList)?; - let references = if let Some(references) = budget.manifest_lists.get(&manifest_list_key).cloned() { - references + let references = if let Some(references) = budget.manifest_lists.get(&manifest_list_key) { + return Ok(Arc::clone(references)); } else { let manifest_list_object = context .backend @@ -1154,62 +2894,65 @@ where budget.charge_avro_bytes(manifest_list_object.data.len())?; let decoded_manifest_list = decode_manifest_list_avro_async(manifest_list_object.data).await?; budget.charge_decoded_avro_bytes(decoded_manifest_list.decoded_size)?; - budget - .manifest_lists - .insert(manifest_list_key, decoded_manifest_list.references.clone()); decoded_manifest_list.references }; - return Ok(references - .into_iter() - .map(|reference| SnapshotGraphManifestLocation { - manifest_path: reference.manifest_path, - format_version: reference.format_version, - manifest_length: reference.manifest_length, - partition_spec_id: reference.partition_spec_id, - content: reference.content, - sequence_number: reference.sequence_number, - min_sequence_number: reference.min_sequence_number, - added_snapshot_id: reference.added_snapshot_id, - added_files_count: reference.added_files_count, - existing_files_count: reference.existing_files_count, - deleted_files_count: reference.deleted_files_count, - added_rows_count: reference.added_rows_count, - existing_rows_count: reference.existing_rows_count, - deleted_rows_count: reference.deleted_rows_count, - from_manifest_list: true, - }) - .collect()); + let references = Arc::new( + references + .into_iter() + .map(|reference| SnapshotGraphManifestLocation { + manifest_path: reference.manifest_path, + format_version: reference.format_version, + manifest_length: reference.manifest_length, + partition_spec_id: reference.partition_spec_id, + content: reference.content, + sequence_number: reference.sequence_number, + min_sequence_number: reference.min_sequence_number, + added_snapshot_id: reference.added_snapshot_id, + added_files_count: reference.added_files_count, + existing_files_count: reference.existing_files_count, + deleted_files_count: reference.deleted_files_count, + added_rows_count: reference.added_rows_count, + existing_rows_count: reference.existing_rows_count, + deleted_rows_count: reference.deleted_rows_count, + from_manifest_list: true, + }) + .collect(), + ); + budget.manifest_lists.insert(manifest_list_key, Arc::clone(&references)); + return Ok(references); } let Some(manifests) = snapshot.get("manifests").and_then(serde_json::Value::as_array) else { return Err(TableCatalogStoreError::Invalid("snapshot manifest-list is required".to_string())); }; - manifests - .iter() - .map(|manifest| { - manifest - .as_str() - .filter(|manifest| !manifest.is_empty()) - .map(|manifest| SnapshotGraphManifestLocation { - manifest_path: manifest.to_string(), - format_version, - manifest_length: None, - partition_spec_id: None, - content: None, - sequence_number: Some(snapshot_sequence_number), - min_sequence_number: None, - added_snapshot_id: None, - added_files_count: None, - existing_files_count: None, - deleted_files_count: None, - added_rows_count: None, - existing_rows_count: None, - deleted_rows_count: None, - from_manifest_list: false, - }) - .ok_or_else(|| TableCatalogStoreError::Invalid("snapshot manifest location must be a string".to_string())) - }) - .collect() + Ok(Arc::new( + manifests + .iter() + .map(|manifest| { + manifest + .as_str() + .filter(|manifest| !manifest.is_empty()) + .map(|manifest| SnapshotGraphManifestLocation { + manifest_path: manifest.to_string(), + format_version, + manifest_length: None, + partition_spec_id: None, + content: None, + sequence_number: Some(snapshot_sequence_number), + min_sequence_number: None, + added_snapshot_id: None, + added_files_count: None, + existing_files_count: None, + deleted_files_count: None, + added_rows_count: None, + existing_rows_count: None, + deleted_rows_count: None, + from_manifest_list: false, + }) + .ok_or_else(|| TableCatalogStoreError::Invalid("snapshot manifest location must be a string".to_string())) + }) + .collect::>>()?, + )) } fn validate_snapshot_graph_manifest_location( @@ -1265,21 +3008,6 @@ fn validate_snapshot_graph_manifest_location( "Iceberg v2 manifest-list sequence numbers are inconsistent with the snapshot".to_string(), )); } - if [ - manifest.added_files_count, - manifest.existing_files_count, - manifest.deleted_files_count, - manifest.added_rows_count, - manifest.existing_rows_count, - manifest.deleted_rows_count, - ] - .into_iter() - .any(|count| count.is_none()) - { - return Err(TableCatalogStoreError::Invalid( - "Iceberg v2 manifest-list entry is missing required file or row counts".to_string(), - )); - } } _ => { return Err(TableCatalogStoreError::Internal(format!( @@ -1398,25 +3126,13 @@ where } } - for object_keys in live_object_keys.chunks(TABLE_COMMIT_OBJECT_VALIDATION_CONCURRENCY) { - let backend = context.backend.clone(); - let bucket = context.table_bucket.to_string(); - stream::iter(object_keys.iter().cloned()) - .map(move |object_key| { - let backend = backend.clone(); - let bucket = bucket.clone(); - async move { - if !backend.object_exists(&bucket, &object_key).await? { - return Err(TableCatalogStoreError::Invalid("manifest referenced data file is missing".to_string())); - } - Ok(()) - } - }) - .buffered(TABLE_COMMIT_OBJECT_VALIDATION_CONCURRENCY) - .try_for_each(|()| async { Ok(()) }) - .await?; - } - Ok(()) + validate_object_keys_exist( + context.backend, + context.table_bucket, + live_object_keys, + "manifest referenced data file is missing", + ) + .await } fn snapshot_graph_object_key( diff --git a/rustfs/src/table_catalog/mod.rs b/rustfs/src/table_catalog/mod.rs index af560459d..4011639ce 100644 --- a/rustfs/src/table_catalog/mod.rs +++ b/rustfs/src/table_catalog/mod.rs @@ -111,8 +111,12 @@ const TABLE_MANIFEST_AVRO_MAX_DECODED_SIZE: usize = 128 * 1024 * 1024; const TABLE_MANIFEST_AVRO_MAX_RECORDS: usize = 1_000_000; const TABLE_MANIFEST_AVRO_MAX_HEADER_ENTRIES: usize = 1_024; const TABLE_COMMIT_MAX_MANIFESTS: usize = 10_000; +const TABLE_COMMIT_MAX_MANIFEST_TRAVERSALS: usize = 20_000; const TABLE_COMMIT_MAX_AVRO_BYTES: usize = 512 * 1024 * 1024; const TABLE_COMMIT_MAX_FILE_REFERENCES: usize = 1_000_000; +const TABLE_COMMIT_MAX_STATISTICS_OBJECTS: usize = 1_024; +const TABLE_COMMIT_MAX_STATISTICS_BYTES: usize = 512 * 1024 * 1024; +const TABLE_STATISTICS_FILE_MAX_SIZE: usize = 128 * 1024 * 1024; pub(crate) const TABLE_COMMIT_OBJECT_VALIDATION_CONCURRENCY: usize = 16; pub const TABLE_RESERVED_PREFIX: &str = BUCKET_TABLE_RESERVED_PREFIX; const WAREHOUSE_ROOT: &str = "warehouses"; diff --git a/rustfs/src/table_catalog/store/migration.rs b/rustfs/src/table_catalog/store/migration.rs index d8224e471..398285f88 100644 --- a/rustfs/src/table_catalog/store/migration.rs +++ b/rustfs/src/table_catalog/store/migration.rs @@ -216,7 +216,7 @@ where Ok(fence) } - pub(super) async fn acquire_table_bucket_registry_write_permit(&self) -> TableCatalogStoreResult> { + pub(super) async fn acquire_table_bucket_registry_write_permit(&self) -> TableCatalogStoreResult { let fence_path = self.paths.backing_migration_global_fence_path(); let lock_path = self.paths.backing_migration_global_fence_lock_path(); let guard = self.backend.acquire_read_lock(self.catalog_bucket(), &lock_path).await?; @@ -235,7 +235,7 @@ where pub(super) async fn acquire_object_backed_catalog_write_permit( &self, table_bucket: &str, - ) -> TableCatalogStoreResult> { + ) -> TableCatalogStoreResult { let lock_path = self.paths.backing_migration_fence_lock_path(table_bucket); let guard = self.backend.acquire_read_lock(self.catalog_bucket(), &lock_path).await?; if self.read_backing_migration_fence(table_bucket).await?.is_some() { @@ -290,7 +290,7 @@ where async fn collect_bucket_snapshot_with_locks( &self, table_bucket: &str, - guards: &mut Vec>, + guards: &mut Vec, ) -> TableCatalogStoreResult { let bucket_path = self.paths.table_bucket_entry_path(table_bucket); guards.push(self.backend.acquire_write_lock(self.catalog_bucket(), &bucket_path).await?); diff --git a/rustfs/src/table_catalog/store/mod.rs b/rustfs/src/table_catalog/store/mod.rs index 4e358651b..9e8d57129 100644 --- a/rustfs/src/table_catalog/store/mod.rs +++ b/rustfs/src/table_catalog/store/mod.rs @@ -262,6 +262,29 @@ pub(crate) trait TableCatalogStore: Send + Sync { async fn create_view(&self, entry: ViewEntry) -> TableCatalogStoreResult<()>; + async fn create_view_with_publication( + &self, + entry: ViewEntry, + publication: &(dyn TableCommitPublication + Sync), + ) -> TableCatalogStoreResult<()> { + publication.begin_table_bucket(&entry.table_bucket).await?; + if !publication.holds_table_bucket(&entry.table_bucket) { + return Err(TableCatalogStoreError::Internal( + "view creation requires a table-bucket publication fence".to_string(), + )); + } + publication + .prepare(&entry.table_bucket, &entry.namespace, &entry.view) + .await?; + if !publication.holds_table(&entry.table_bucket, &entry.namespace, &entry.view) { + return Err(TableCatalogStoreError::Internal( + "view creation requires a view publication fence".to_string(), + )); + } + let _publication_completion = TableCommitPublicationCompletion::new(publication); + self.create_view(entry).await + } + async fn list_views(&self, table_bucket: &str, namespace: &str) -> TableCatalogStoreResult>; async fn list_views_page( @@ -283,6 +306,32 @@ pub(crate) trait TableCatalogStore: Send + Sync { async fn replace_view(&self, request: ViewCommitRequest) -> TableCatalogStoreResult; + async fn replace_view_with_publication( + &self, + request: ViewCommitRequest, + table_bucket_fence_required: bool, + publication: &(dyn TableCommitPublication + Sync), + ) -> TableCatalogStoreResult { + if table_bucket_fence_required { + publication.begin_table_bucket(&request.table_bucket).await?; + if !publication.holds_table_bucket(&request.table_bucket) { + return Err(TableCatalogStoreError::Internal( + "view replacement requires a table-bucket publication fence".to_string(), + )); + } + } + publication + .prepare(&request.table_bucket, &request.namespace, &request.view) + .await?; + if !publication.holds_table(&request.table_bucket, &request.namespace, &request.view) { + return Err(TableCatalogStoreError::Internal( + "view replacement requires a view publication fence".to_string(), + )); + } + let _publication_completion = TableCommitPublicationCompletion::new(publication); + self.replace_view(request).await + } + async fn drop_view(&self, table_bucket: &str, namespace: &str, view: &str) -> TableCatalogStoreResult<()>; async fn get_commit_by_id( @@ -338,7 +387,7 @@ struct TableCommitLockPublication<'a, B> { struct TableCommitLockPublicationState { table_bucket: Option, table: Option<(String, String, String)>, - guards: Vec>, + guards: Vec, } impl<'a, B> TableCommitLockPublication<'a, B> { @@ -409,15 +458,17 @@ where } fn holds_table_bucket(&self, table_bucket: &str) -> bool { - self.state.lock().table_bucket.as_deref() == Some(table_bucket) + let state = self.state.lock(); + state.table_bucket.as_deref() == Some(table_bucket) && state.guards.iter().all(|guard| !guard.is_lock_lost()) } fn holds_table(&self, table_bucket: &str, namespace: &str, table: &str) -> bool { - self.state - .lock() + let state = self.state.lock(); + state .table .as_ref() .is_some_and(|held| held.0 == table_bucket && held.1 == namespace && held.2 == table) + && state.guards.iter().all(|guard| !guard.is_lock_lost()) } fn complete(&self) { @@ -438,6 +489,32 @@ pub(crate) struct TableCatalogObjectMetadata { pub mod_time: Option, } +pub(crate) struct TableCatalogLockGuard { + _guard: Box, + lock_lost: Option>, +} + +impl TableCatalogLockGuard { + pub(crate) fn stable(guard: impl Send + 'static) -> Self { + Self { + _guard: Box::new(guard), + lock_lost: None, + } + } + + fn namespace(guard: rustfs_lock::NamespaceLockGuard) -> Self { + let lock_lost = guard.lock_lost_signal(); + Self { + _guard: Box::new(guard), + lock_lost, + } + } + + pub(crate) fn is_lock_lost(&self) -> bool { + self.lock_lost.as_ref().is_some_and(|signal| signal.is_lost()) + } +} + #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) struct TableCatalogObjectListPage { pub objects: Vec, @@ -588,11 +665,11 @@ pub(crate) trait TableCatalogObjectBackend: Clone + Send + Sync + 'static { Ok(TableCatalogObjectListPage { objects, is_truncated }) } - async fn acquire_read_lock(&self, bucket: &str, object: &str) -> TableCatalogStoreResult> { + async fn acquire_read_lock(&self, bucket: &str, object: &str) -> TableCatalogStoreResult { self.acquire_write_lock(bucket, object).await } - async fn acquire_write_lock(&self, bucket: &str, object: &str) -> TableCatalogStoreResult>; + async fn acquire_write_lock(&self, bucket: &str, object: &str) -> TableCatalogStoreResult; async fn begin_table_bucket_commit_publication(&self, _table_bucket: &str) -> TableCatalogStoreResult<()> { Ok(()) @@ -1169,6 +1246,17 @@ where } } + async fn create_view_with_publication( + &self, + entry: ViewEntry, + publication: &(dyn TableCommitPublication + Sync), + ) -> TableCatalogStoreResult<()> { + match self { + Self::ObjectBacked(store) => store.create_view_with_publication(entry, publication).await, + Self::DurableStrong(store) => store.create_view_with_publication(entry, publication).await, + } + } + async fn list_views(&self, table_bucket: &str, namespace: &str) -> TableCatalogStoreResult> { match self { Self::ObjectBacked(store) => store.list_views(table_bucket, namespace).await, @@ -1203,6 +1291,26 @@ where } } + async fn replace_view_with_publication( + &self, + request: ViewCommitRequest, + table_bucket_fence_required: bool, + publication: &(dyn TableCommitPublication + Sync), + ) -> TableCatalogStoreResult { + match self { + Self::ObjectBacked(store) => { + store + .replace_view_with_publication(request, table_bucket_fence_required, publication) + .await + } + Self::DurableStrong(store) => { + store + .replace_view_with_publication(request, table_bucket_fence_required, publication) + .await + } + } + } + async fn drop_view(&self, table_bucket: &str, namespace: &str, view: &str) -> TableCatalogStoreResult<()> { match self { Self::ObjectBacked(store) => store.drop_view(table_bucket, namespace, view).await, @@ -1686,7 +1794,7 @@ where }) } - async fn acquire_write_lock(&self, bucket: &str, object: &str) -> TableCatalogStoreResult> { + async fn acquire_write_lock(&self, bucket: &str, object: &str) -> TableCatalogStoreResult { let lock = self .store .new_ns_lock(bucket, object) @@ -1696,10 +1804,10 @@ where .get_write_lock(get_lock_acquire_timeout()) .await .map_err(|err| TableCatalogStoreError::Internal(format!("failed to acquire catalog table lock: {err}")))?; - Ok(Box::new(guard)) + Ok(TableCatalogLockGuard::namespace(guard)) } - async fn acquire_read_lock(&self, bucket: &str, object: &str) -> TableCatalogStoreResult> { + async fn acquire_read_lock(&self, bucket: &str, object: &str) -> TableCatalogStoreResult { let lock = self .store .new_ns_lock(bucket, object) @@ -1709,7 +1817,7 @@ where .get_read_lock(get_lock_acquire_timeout()) .await .map_err(|err| TableCatalogStoreError::Internal(format!("failed to acquire catalog migration lock: {err}")))?; - Ok(Box::new(guard)) + Ok(TableCatalogLockGuard::namespace(guard)) } } diff --git a/rustfs/src/table_catalog/store/object.rs b/rustfs/src/table_catalog/store/object.rs index b538ae85f..3cd08f321 100644 --- a/rustfs/src/table_catalog/store/object.rs +++ b/rustfs/src/table_catalog/store/object.rs @@ -1036,6 +1036,20 @@ where .await } + async fn restore_table_warehouse_index_after_failed_drop(&self, entry: &TableEntry, reason: &'static str) { + if let Err(err) = self.reserve_table_warehouse_index(entry).await { + tracing::warn!( + table_bucket = %entry.table_bucket, + namespace = %entry.namespace, + table = %entry.table, + table_id = %entry.table_id, + reason, + error = %err, + "failed to restore table warehouse index after table drop stopped" + ); + } + } + async fn delete_table_warehouse_index_if_changed(&self, current: &TableEntry, next: &TableEntry) { let Ok(current_index) = table_warehouse_index_entry(current) else { return; @@ -1316,6 +1330,15 @@ where } self.ensure_table_warehouse_prefix_available(&entry).await?; let reservation = self.reserve_table_warehouse_index(&entry).await?; + if !publication.holds_table_bucket(&entry.table_bucket) + || !publication.holds_table(&entry.table_bucket, &entry.namespace, &entry.table) + { + self.delete_created_table_warehouse_index(&entry, reservation, "table publication fence lost") + .await; + return Err(TableCatalogStoreError::Internal( + "table registration publication fence was lost before catalog update".to_string(), + )); + } let result = self .write_entry_unlocked(self.catalog_bucket(), &table_path, &entry, precondition) .await; @@ -1327,7 +1350,25 @@ where } async fn write_view_entry(&self, entry: ViewEntry, precondition: TableCatalogPutPrecondition) -> TableCatalogStoreResult<()> { + let publication = TableCommitLockPublication::new(&self.backend); + self.write_view_entry_with_publication(entry, precondition, &publication) + .await + } + + async fn write_view_entry_with_publication( + &self, + entry: ViewEntry, + precondition: TableCatalogPutPrecondition, + publication: &(dyn TableCommitPublication + Sync), + ) -> TableCatalogStoreResult<()> { validate_view_entry_version_and_id(&entry)?; + publication.begin_table_bucket(&entry.table_bucket).await?; + if !publication.holds_table_bucket(&entry.table_bucket) { + return Err(TableCatalogStoreError::Internal( + "view creation requires a table-bucket publication fence".to_string(), + )); + } + let _publication_completion = TableCommitPublicationCompletion::new(publication); self.require_table_bucket(&entry.table_bucket).await?; let namespace = parse_namespace_for_store(&entry.namespace)?; let view = parse_table_for_store(&entry.view)?; @@ -1353,6 +1394,17 @@ where entry.table_bucket, entry.namespace, entry.view ))); } + // Preserve catalog -> publication -> object lock order across rolling upgrades. + publication + .prepare(&entry.table_bucket, &entry.namespace, &entry.view) + .await?; + if !publication.holds_table_bucket(&entry.table_bucket) + || !publication.holds_table(&entry.table_bucket, &entry.namespace, &entry.view) + { + return Err(TableCatalogStoreError::Internal( + "view creation publication fence was lost before catalog update".to_string(), + )); + } self.write_entry_unlocked(self.catalog_bucket(), &view_path, &entry, precondition) .await } @@ -4493,16 +4545,16 @@ where validate_commit_metadata_digest(&request, &new_metadata_object)?; let table_bucket = request.table_bucket.clone(); let metadata_location = request.new_metadata_location.clone(); - let next_warehouse_location = tokio::task::spawn_blocking(move || { - table_metadata_warehouse_location(&table_bucket, &metadata_location, &new_metadata_object) + let next_metadata_state = tokio::task::spawn_blocking(move || { + table_metadata_commit_state(&table_bucket, &metadata_location, &new_metadata_object) }) .await .map_err(|err| TableCatalogStoreError::Internal(format!("table metadata parser task failed: {err}")))??; - if next_warehouse_location + let warehouse_relocation = next_metadata_state + .warehouse_location .as_ref() - .is_some_and(|warehouse_location| warehouse_location != ¤t.warehouse_location) - && !publication.holds_table_bucket(&request.table_bucket) - { + .is_some_and(|warehouse_location| warehouse_location != ¤t.warehouse_location); + if warehouse_relocation && !publication.holds_table_bucket(&request.table_bucket) { return table_commit_result( &request.table_bucket, &request.namespace, @@ -4537,9 +4589,12 @@ where let mut next = current.clone(); next.metadata_location = staged_commit_log.new_metadata_location.clone(); - if let Some(warehouse_location) = next_warehouse_location { + if let Some(warehouse_location) = next_metadata_state.warehouse_location { next.warehouse_location = warehouse_location; } + if let Some(format_version) = next_metadata_state.format_version { + next.format_version = format_version; + } next.version_token = staged_commit_log.new_version_token.clone(); next.generation = current.generation.saturating_add(1); if next.warehouse_location != current.warehouse_location { @@ -4585,6 +4640,24 @@ where ); } + if !publication.holds_table(&request.table_bucket, &request.namespace, &request.table) + || (warehouse_relocation && !publication.holds_table_bucket(&request.table_bucket)) + { + self.delete_created_table_warehouse_index(&next, reservation, "table publication fence lost") + .await; + return table_commit_result( + &request.table_bucket, + &request.namespace, + &request.table, + &request.commit_id, + &request.operation, + commit_started, + Err(TableCatalogStoreError::Internal( + "table commit publication fence was lost before pointer update".to_string(), + )), + ); + } + let cas_started = Instant::now(); let cas_result = self .write_entry_unlocked( @@ -4662,20 +4735,21 @@ where ))); }; self.delete_owned_table_warehouse_index_for_drop(&entry).await?; + if !publication.holds_table_bucket(table_bucket) + || !publication.holds_table(table_bucket, &namespace.public_name(), table.as_str()) + { + self.restore_table_warehouse_index_after_failed_drop(&entry, "table publication fence lost") + .await; + return Err(TableCatalogStoreError::Internal( + "table drop publication fence was lost before catalog update".to_string(), + )); + } if let Err(err) = self.backend.delete_object_unlocked(self.catalog_bucket(), &object).await { match self.read_table_with_etag_unlocked(table_bucket, &namespace, &table).await { Ok(None) => return Ok(()), Ok(Some((current, _))) if current == entry => { - if let Err(restore_err) = self.reserve_table_warehouse_index(&entry).await { - tracing::warn!( - table_bucket = %entry.table_bucket, - namespace = %entry.namespace, - table = %entry.table, - table_id = %entry.table_id, - error = %restore_err, - "failed to restore table warehouse index after table entry delete failure" - ); - } + self.restore_table_warehouse_index_after_failed_drop(&entry, "table entry delete failed") + .await; } Ok(Some(_)) => { return Err(TableCatalogStoreError::Internal(format!( @@ -4703,6 +4777,15 @@ where self.write_view_entry(entry, TableCatalogPutPrecondition::IfAbsent).await } + async fn create_view_with_publication( + &self, + entry: ViewEntry, + publication: &(dyn TableCommitPublication + Sync), + ) -> TableCatalogStoreResult<()> { + self.write_view_entry_with_publication(entry, TableCatalogPutPrecondition::IfAbsent, publication) + .await + } + async fn list_views(&self, table_bucket: &str, namespace: &str) -> TableCatalogStoreResult> { let namespace = parse_namespace_for_store(namespace)?; let mut entries = Vec::new(); @@ -4757,8 +4840,26 @@ where } async fn replace_view(&self, request: ViewCommitRequest) -> TableCatalogStoreResult { + let publication = TableCommitLockPublication::new(&self.backend); + self.replace_view_with_publication(request, true, &publication).await + } + + async fn replace_view_with_publication( + &self, + request: ViewCommitRequest, + table_bucket_fence_required: bool, + publication: &(dyn TableCommitPublication + Sync), + ) -> TableCatalogStoreResult { let namespace = parse_namespace_for_store(&request.namespace)?; let view = parse_table_for_store(&request.view)?; + if table_bucket_fence_required { + publication.begin_table_bucket(&request.table_bucket).await?; + if !publication.holds_table_bucket(&request.table_bucket) { + return Err(TableCatalogStoreError::Internal( + "view replacement requires a table-bucket publication fence".to_string(), + )); + } + } let _migration_guard = self.acquire_object_backed_catalog_write_permit(&request.table_bucket).await?; let namespace_path = self.paths.namespace_entry_path(&request.table_bucket, &namespace); let _namespace_guard = self @@ -4767,6 +4868,16 @@ where .await?; let view_path = self.paths.view_entry_path(&request.table_bucket, &namespace, &view); let _guard = self.backend.acquire_write_lock(self.catalog_bucket(), &view_path).await?; + // Preserve catalog -> publication -> object lock order across rolling upgrades. + publication + .prepare(&request.table_bucket, &request.namespace, &request.view) + .await?; + if !publication.holds_table(&request.table_bucket, &request.namespace, &request.view) { + return Err(TableCatalogStoreError::Internal( + "view replacement requires a table publication fence".to_string(), + )); + } + let _publication_completion = TableCommitPublicationCompletion::new(publication); let Some((current, current_etag)) = self .read_view_with_etag_unlocked(&request.table_bucket, &namespace, &view) .await? @@ -4814,6 +4925,14 @@ where }) .await .map_err(|err| TableCatalogStoreError::Internal(format!("view metadata parser task failed: {err}")))??; + let warehouse_relocation = next_warehouse_location + .as_deref() + .is_some_and(|location| location != current.warehouse_location); + if warehouse_relocation && !publication.holds_table_bucket(&request.table_bucket) { + return Err(TableCatalogStoreError::Internal( + "view warehouse relocation requires a table-bucket publication fence".to_string(), + )); + } let mut next = current; next.metadata_location = request.new_metadata_location; @@ -4822,13 +4941,40 @@ where } next.version_token = format!("token-{}", Uuid::new_v4()); next.generation = next.generation.saturating_add(1); - self.write_entry_unlocked( - self.catalog_bucket(), - &view_path, - &next, - TableCatalogPutPrecondition::IfMatch(current_etag), - ) - .await?; + if !publication.holds_table(&request.table_bucket, &request.namespace, &request.view) + || ((table_bucket_fence_required || warehouse_relocation) && !publication.holds_table_bucket(&request.table_bucket)) + { + return Err(TableCatalogStoreError::Internal( + "view replacement publication fence was lost before catalog update".to_string(), + )); + } + let write_result = self + .write_entry_unlocked( + self.catalog_bucket(), + &view_path, + &next, + TableCatalogPutPrecondition::IfMatch(current_etag), + ) + .await; + if let Err(err) = write_result { + match self + .read_view_with_etag_unlocked(&request.table_bucket, &namespace, &view) + .await + { + Ok(Some((persisted, _))) if persisted == next => {} + Ok(_) => return Err(err), + Err(read_err) => { + tracing::warn!( + table_bucket = %request.table_bucket, + namespace = %request.namespace, + view = %request.view, + error = %read_err, + "failed to verify view state after an ambiguous catalog update" + ); + return Err(err); + } + } + } Ok(ViewCommitResult { view: next }) } diff --git a/rustfs/src/table_catalog/store/strong.rs b/rustfs/src/table_catalog/store/strong.rs index e1d313129..2ac5e94d6 100644 --- a/rustfs/src/table_catalog/store/strong.rs +++ b/rustfs/src/table_catalog/store/strong.rs @@ -554,7 +554,7 @@ where // Ordinary mutations hold the global migration read lock before the local write lock; migration takes the // write side before invoking its dedicated snapshot mutation methods. - async fn acquire_snapshot_write_permit(&self) -> TableCatalogStoreResult> { + async fn acquire_snapshot_write_permit(&self) -> TableCatalogStoreResult { let lock_path = TableCatalogObjectPaths::default().backing_migration_global_fence_lock_path(); self.object_backend.acquire_read_lock(RUSTFS_META_BUCKET, &lock_path).await } @@ -1775,7 +1775,7 @@ where request: &TableCommitRequest, namespace: &Namespace, table: &IdentifierSegment, - next_warehouse_location: Option, + next_metadata_state: TableMetadataCommitState, ) -> TableCatalogStoreResult { let key = Self::table_key(&request.table_bucket, namespace, table); let current = Self::validate_new_table_commit_locked(state, &key, request)?; @@ -1802,9 +1802,12 @@ where let mut next = current; next.metadata_location = commit_log.new_metadata_location.clone(); - if let Some(warehouse_location) = next_warehouse_location { + if let Some(warehouse_location) = next_metadata_state.warehouse_location { next.warehouse_location = warehouse_location; } + if let Some(format_version) = next_metadata_state.format_version { + next.format_version = format_version; + } Self::ensure_table_warehouse_prefix_available_locked(state, &next, &key)?; next.version_token = commit_log.new_version_token.clone(); next.generation = next.generation.saturating_add(1); @@ -2170,6 +2173,7 @@ where let _write_guard = self.write_lock.lock().await; self.hydrate_state().await?; let key = Self::table_key(&entry.table_bucket, &namespace, &table); + let publication_identity = (entry.table_bucket.clone(), entry.namespace.clone(), entry.table.clone()); let (snapshot, precondition, postcondition) = { let state = self.state.lock().await; Self::require_table_bucket_in_state(&state, &entry.table_bucket)?; @@ -2198,6 +2202,13 @@ where StrongSnapshotWritePostcondition::TablePresent(entry), ) }; + if !publication.holds_table_bucket(&publication_identity.0) + || !publication.holds_table(&publication_identity.0, &publication_identity.1, &publication_identity.2) + { + return Err(TableCatalogStoreError::Internal( + "table registration publication fence was lost before snapshot update".to_string(), + )); + } self.finalize_snapshot_write(snapshot, precondition, postcondition).await } @@ -2479,9 +2490,15 @@ where let result = match prepared_result { Ok((result, Some((snapshot, precondition)))) => { let postcondition = Self::commit_write_postcondition(&request.table_bucket, &result.commit_log); - self.finalize_snapshot_write(snapshot, precondition, postcondition) - .await - .map(|_| result) + if !publication.holds_table(&request.table_bucket, &request.namespace, &request.table) { + Err(TableCatalogStoreError::Internal( + "table commit publication fence was lost before snapshot update".to_string(), + )) + } else { + self.finalize_snapshot_write(snapshot, precondition, postcondition) + .await + .map(|_| result) + } } Ok((result, None)) => Ok(result), Err(err) => Err(err), @@ -2519,8 +2536,8 @@ where validate_commit_metadata_digest(&request, &new_metadata_object)?; let table_bucket = request.table_bucket.clone(); let metadata_location = request.new_metadata_location.clone(); - let next_warehouse_location = tokio::task::spawn_blocking(move || { - table_metadata_warehouse_location(&table_bucket, &metadata_location, &new_metadata_object) + let next_metadata_state = tokio::task::spawn_blocking(move || { + table_metadata_commit_state(&table_bucket, &metadata_location, &new_metadata_object) }) .await .map_err(|err| TableCatalogStoreError::Internal(format!("table metadata parser task failed: {err}")))??; @@ -2539,11 +2556,11 @@ where )) })? }; - if next_warehouse_location + let warehouse_relocation = next_metadata_state + .warehouse_location .as_ref() - .is_some_and(|warehouse_location| warehouse_location != ¤t_warehouse_location) - && !publication.holds_table_bucket(&request.table_bucket) - { + .is_some_and(|warehouse_location| warehouse_location != ¤t_warehouse_location); + if warehouse_relocation && !publication.holds_table_bucket(&request.table_bucket) { return table_commit_result( &request.table_bucket, &request.namespace, @@ -2561,7 +2578,7 @@ where let prepared_result = { let state = self.state.lock().await; let (precondition, mut draft_state) = Self::snapshot_draft_context_locked(&state); - match Self::apply_commit_locked(&mut draft_state, &request, &namespace, &table, next_warehouse_location) { + match Self::apply_commit_locked(&mut draft_state, &request, &namespace, &table, next_metadata_state) { Ok(result) => Self::snapshot_from_mutated_state_locked(&mut draft_state, self.snapshot_write_version) .map(|snapshot| (result, snapshot, precondition)), Err(err) => Err(err), @@ -2570,7 +2587,16 @@ where let result = match prepared_result { Ok((result, snapshot, precondition)) => { let postcondition = Self::commit_write_postcondition(&request.table_bucket, &result.commit_log); - match self.finalize_snapshot_write(snapshot, precondition, postcondition).await { + let snapshot_result = if publication.holds_table(&request.table_bucket, &request.namespace, &request.table) + && (!warehouse_relocation || publication.holds_table_bucket(&request.table_bucket)) + { + self.finalize_snapshot_write(snapshot, precondition, postcondition).await + } else { + Err(TableCatalogStoreError::Internal( + "table commit publication fence was lost before snapshot update".to_string(), + )) + }; + match snapshot_result { Ok(()) => Ok(result), Err(err) => { let replay = { @@ -2647,13 +2673,26 @@ where }, ) }; + if !publication.holds_table_bucket(table_bucket) + || !publication.holds_table(table_bucket, &namespace.public_name(), table.as_str()) + { + return Err(TableCatalogStoreError::Internal( + "table drop publication fence was lost before snapshot update".to_string(), + )); + } self.finalize_snapshot_write(snapshot, precondition, postcondition).await } async fn create_view(&self, entry: ViewEntry) -> TableCatalogStoreResult<()> { - let _migration_guard = self.acquire_snapshot_write_permit().await?; - let _write_guard = self.write_lock.lock().await; - self.hydrate_state().await?; + let publication = TableCommitLockPublication::new(&self.object_backend); + self.create_view_with_publication(entry, &publication).await + } + + async fn create_view_with_publication( + &self, + entry: ViewEntry, + publication: &(dyn TableCommitPublication + Sync), + ) -> TableCatalogStoreResult<()> { validate_view_entry_version_and_id(&entry)?; validate_view_warehouse_location(&entry.table_bucket, &entry.warehouse_location)?; let namespace = parse_namespace_for_store(&entry.namespace)?; @@ -2663,7 +2702,26 @@ where "view metadata location must be inside the view metadata directory".to_string(), )); } + publication.begin_table_bucket(&entry.table_bucket).await?; + if !publication.holds_table_bucket(&entry.table_bucket) { + return Err(TableCatalogStoreError::Internal( + "view creation requires a table-bucket publication fence".to_string(), + )); + } + let _publication_completion = TableCommitPublicationCompletion::new(publication); + let _migration_guard = self.acquire_snapshot_write_permit().await?; + publication + .prepare(&entry.table_bucket, &entry.namespace, &entry.view) + .await?; + if !publication.holds_table(&entry.table_bucket, &entry.namespace, &entry.view) { + return Err(TableCatalogStoreError::Internal( + "view creation requires a table publication fence".to_string(), + )); + } + let _write_guard = self.write_lock.lock().await; + self.hydrate_state().await?; let key = Self::table_key(&entry.table_bucket, &namespace, &view); + let publication_identity = (entry.table_bucket.clone(), entry.namespace.clone(), entry.view.clone()); let (snapshot, precondition, postcondition) = { let state = self.state.lock().await; Self::require_table_bucket_in_state(&state, &entry.table_bucket)?; @@ -2682,6 +2740,13 @@ where StrongSnapshotWritePostcondition::ViewPresent(entry), ) }; + if !publication.holds_table_bucket(&publication_identity.0) + || !publication.holds_table(&publication_identity.0, &publication_identity.1, &publication_identity.2) + { + return Err(TableCatalogStoreError::Internal( + "view creation publication fence was lost before snapshot update".to_string(), + )); + } self.finalize_snapshot_write(snapshot, precondition, postcondition).await } @@ -2751,11 +2816,38 @@ where } async fn replace_view(&self, request: ViewCommitRequest) -> TableCatalogStoreResult { + let publication = TableCommitLockPublication::new(&self.object_backend); + self.replace_view_with_publication(request, true, &publication).await + } + + async fn replace_view_with_publication( + &self, + request: ViewCommitRequest, + table_bucket_fence_required: bool, + publication: &(dyn TableCommitPublication + Sync), + ) -> TableCatalogStoreResult { + if table_bucket_fence_required { + publication.begin_table_bucket(&request.table_bucket).await?; + if !publication.holds_table_bucket(&request.table_bucket) { + return Err(TableCatalogStoreError::Internal( + "view replacement requires a table-bucket publication fence".to_string(), + )); + } + } + let _publication_completion = TableCommitPublicationCompletion::new(publication); let _migration_guard = self.acquire_snapshot_write_permit().await?; - let write_guard = self.write_lock.lock().await; - self.hydrate_state().await?; let namespace = parse_namespace_for_store(&request.namespace)?; let view = parse_table_for_store(&request.view)?; + publication + .prepare(&request.table_bucket, &request.namespace, &request.view) + .await?; + if !publication.holds_table(&request.table_bucket, &request.namespace, &request.view) { + return Err(TableCatalogStoreError::Internal( + "view replacement requires a table publication fence".to_string(), + )); + } + let write_guard = self.write_lock.lock().await; + self.hydrate_state().await?; let key = Self::table_key(&request.table_bucket, &namespace, &view); let expected_view_id = { let state = self.state.lock().await; @@ -2798,7 +2890,7 @@ where let _write_guard = self.write_lock.lock().await; self.hydrate_state().await?; - let (snapshot, precondition, next, postcondition) = { + let (snapshot, precondition, next, postcondition, warehouse_relocation) = { let state = self.state.lock().await; Self::ensure_identifier_is_unambiguous_locked(&state, &key)?; let Some(current) = state.views.get(&key).cloned() else { @@ -2828,6 +2920,14 @@ where "current view metadata location does not match expected location".to_string(), )); } + let warehouse_relocation = next_warehouse_location + .as_deref() + .is_some_and(|location| location != current.warehouse_location); + if warehouse_relocation && !publication.holds_table_bucket(&request.table_bucket) { + return Err(TableCatalogStoreError::Internal( + "view warehouse relocation requires a table-bucket publication fence".to_string(), + )); + } let mut next = current; next.metadata_location = request.new_metadata_location; @@ -2843,8 +2943,16 @@ where precondition, next.clone(), StrongSnapshotWritePostcondition::ViewPresent(next), + warehouse_relocation, ) }; + if !publication.holds_table(&request.table_bucket, &request.namespace, &request.view) + || ((table_bucket_fence_required || warehouse_relocation) && !publication.holds_table_bucket(&request.table_bucket)) + { + return Err(TableCatalogStoreError::Internal( + "view replacement publication fence was lost before snapshot update".to_string(), + )); + } self.finalize_snapshot_write(snapshot, precondition, postcondition).await?; Ok(ViewCommitResult { view: next }) } diff --git a/rustfs/src/table_catalog/test_support.rs b/rustfs/src/table_catalog/test_support.rs index 3dac79309..9e49a39a2 100644 --- a/rustfs/src/table_catalog/test_support.rs +++ b/rustfs/src/table_catalog/test_support.rs @@ -55,23 +55,35 @@ pub(crate) fn table_metadata_json(table_uuid: &str, location: &str) -> serde_jso }) } -pub(crate) fn manifest_list_avro_bytes(manifest_paths: &[&str], sequence_number: i64, snapshot_id: i64) -> Vec { - let manifests = manifest_paths - .iter() - .map(|manifest_path| (*manifest_path, 0, sequence_number, snapshot_id)) - .collect::>(); - manifest_list_avro_entries_with_partition_specs(&manifests) -} - -pub(crate) fn manifest_list_avro_entries(manifests: &[(&str, i64, i64)]) -> Vec { +pub(crate) fn manifest_list_avro_bytes(manifests: &[(&str, usize)], sequence_number: i64, snapshot_id: i64) -> Vec { let manifests = manifests .iter() - .map(|(manifest_path, sequence_number, snapshot_id)| (*manifest_path, 0, *sequence_number, *snapshot_id)) + .map(|(manifest_path, manifest_length)| (*manifest_path, *manifest_length, 0, sequence_number, snapshot_id)) .collect::>(); manifest_list_avro_entries_with_partition_specs(&manifests) } -pub(crate) fn manifest_list_avro_entries_with_partition_specs(manifests: &[(&str, i32, i64, i64)]) -> Vec { +pub(crate) fn manifest_list_avro_entries(manifests: &[(&str, usize, i64, i64)]) -> Vec { + let manifests = manifests + .iter() + .map(|(manifest_path, manifest_length, sequence_number, snapshot_id)| { + (*manifest_path, *manifest_length, 0, *sequence_number, *snapshot_id) + }) + .collect::>(); + manifest_list_avro_entries_with_partition_specs(&manifests) +} + +pub(crate) fn manifest_list_avro_entries_with_partition_specs(manifests: &[(&str, usize, i32, i64, i64)]) -> Vec { + let manifests = manifests + .iter() + .map(|(path, length, spec_id, sequence_number, snapshot_id)| { + (*path, *length, *spec_id, 0, *sequence_number, *snapshot_id) + }) + .collect::>(); + manifest_list_avro_entries_with_content(&manifests) +} + +pub(crate) fn manifest_list_avro_entries_with_content(manifests: &[(&str, usize, i32, i32, i64, i64)]) -> Vec { let schema = apache_avro::Schema::parse_str( r#" { @@ -97,16 +109,19 @@ pub(crate) fn manifest_list_avro_entries_with_partition_specs(manifests: &[(&str ) .expect("manifest list avro schema should parse"); let mut writer = apache_avro::Writer::new(&schema, Vec::new()).expect("manifest list writer should initialize"); - for (manifest_path, partition_spec_id, sequence_number, snapshot_id) in manifests { + for (manifest_path, manifest_length, partition_spec_id, content, sequence_number, snapshot_id) in manifests { writer .append_value(apache_avro::types::Value::Record(vec![ ( "manifest_path".to_string(), apache_avro::types::Value::String((*manifest_path).to_string()), ), - ("manifest_length".to_string(), apache_avro::types::Value::Long(1)), + ( + "manifest_length".to_string(), + apache_avro::types::Value::Long(i64::try_from(*manifest_length).expect("test manifest length should fit")), + ), ("partition_spec_id".to_string(), apache_avro::types::Value::Int(*partition_spec_id)), - ("content".to_string(), apache_avro::types::Value::Int(0)), + ("content".to_string(), apache_avro::types::Value::Int(*content)), ("sequence_number".to_string(), apache_avro::types::Value::Long(*sequence_number)), ("min_sequence_number".to_string(), apache_avro::types::Value::Long(*sequence_number)), ("added_snapshot_id".to_string(), apache_avro::types::Value::Long(*snapshot_id)), @@ -122,7 +137,86 @@ pub(crate) fn manifest_list_avro_entries_with_partition_specs(manifests: &[(&str writer.into_inner().expect("manifest list avro bytes should flush") } +pub(crate) fn manifest_list_avro_entries_with_nullable_counts(manifests: &[(&str, usize, i32, i64, i64)]) -> Vec { + let schema = apache_avro::Schema::parse_str( + r#" + { + "type": "record", + "name": "manifest_file", + "fields": [ + {"name": "manifest_path", "type": "string"}, + {"name": "manifest_length", "type": "long"}, + {"name": "partition_spec_id", "type": "int"}, + {"name": "content", "type": "int"}, + {"name": "sequence_number", "type": "long"}, + {"name": "min_sequence_number", "type": "long"}, + {"name": "added_snapshot_id", "type": "long"}, + {"name": "added_files_count", "type": ["null", "int"], "default": null}, + {"name": "existing_files_count", "type": ["null", "int"], "default": null}, + {"name": "deleted_files_count", "type": ["null", "int"], "default": null}, + {"name": "added_rows_count", "type": ["null", "long"], "default": null}, + {"name": "existing_rows_count", "type": ["null", "long"], "default": null}, + {"name": "deleted_rows_count", "type": ["null", "long"], "default": null} + ] + } + "#, + ) + .expect("manifest list avro schema should parse"); + let mut writer = apache_avro::Writer::new(&schema, Vec::new()).expect("manifest list writer should initialize"); + for (manifest_path, manifest_length, partition_spec_id, sequence_number, snapshot_id) in manifests { + writer + .append_value(apache_avro::types::Value::Record(vec![ + ( + "manifest_path".to_string(), + apache_avro::types::Value::String((*manifest_path).to_string()), + ), + ( + "manifest_length".to_string(), + apache_avro::types::Value::Long(i64::try_from(*manifest_length).expect("test manifest length should fit")), + ), + ("partition_spec_id".to_string(), apache_avro::types::Value::Int(*partition_spec_id)), + ("content".to_string(), apache_avro::types::Value::Int(0)), + ("sequence_number".to_string(), apache_avro::types::Value::Long(*sequence_number)), + ("min_sequence_number".to_string(), apache_avro::types::Value::Long(*sequence_number)), + ("added_snapshot_id".to_string(), apache_avro::types::Value::Long(*snapshot_id)), + ( + "added_files_count".to_string(), + apache_avro::types::Value::Union(0, Box::new(apache_avro::types::Value::Null)), + ), + ( + "existing_files_count".to_string(), + apache_avro::types::Value::Union(0, Box::new(apache_avro::types::Value::Null)), + ), + ( + "deleted_files_count".to_string(), + apache_avro::types::Value::Union(0, Box::new(apache_avro::types::Value::Null)), + ), + ( + "added_rows_count".to_string(), + apache_avro::types::Value::Union(0, Box::new(apache_avro::types::Value::Null)), + ), + ( + "existing_rows_count".to_string(), + apache_avro::types::Value::Union(0, Box::new(apache_avro::types::Value::Null)), + ), + ( + "deleted_rows_count".to_string(), + apache_avro::types::Value::Union(0, Box::new(apache_avro::types::Value::Null)), + ), + ])) + .expect("manifest list record should append"); + } + writer.into_inner().expect("manifest list avro bytes should flush") +} + pub(crate) fn manifest_avro_bytes(files: &[(&str, i32, i32, i64, i64)]) -> Vec { + manifest_avro_bytes_with_partition_spec(files, None) +} + +pub(crate) fn manifest_avro_bytes_with_partition_spec( + files: &[(&str, i32, i32, i64, i64)], + partition_spec_id: Option, +) -> Vec { let schema = apache_avro::Schema::parse_str( r#" { @@ -141,6 +235,7 @@ pub(crate) fn manifest_avro_bytes(files: &[(&str, i32, i32, i64, i64)]) -> Vec Vec Vec, - guard: Arc>>>, + guard: Arc>>, } impl BlockingObjectPublication { @@ -812,7 +915,7 @@ impl TableCatalogObjectBackend for TestCatalogObjectBackend { .collect()) } - async fn acquire_write_lock(&self, bucket: &str, object: &str) -> TableCatalogStoreResult> { + async fn acquire_write_lock(&self, bucket: &str, object: &str) -> TableCatalogStoreResult { self.lock_attempts.lock().await.push((bucket.to_string(), object.to_string())); { let mut state = self.state.lock().await; @@ -828,10 +931,10 @@ impl TableCatalogObjectBackend for TestCatalogObjectBackend { .or_insert_with(|| std::sync::Arc::new(tokio::sync::RwLock::new(()))) .clone() }; - Ok(Box::new(lock.write_owned().await)) + Ok(TableCatalogLockGuard::stable(lock.write_owned().await)) } - async fn acquire_read_lock(&self, bucket: &str, object: &str) -> TableCatalogStoreResult> { + async fn acquire_read_lock(&self, bucket: &str, object: &str) -> TableCatalogStoreResult { // The admin fake implemented only acquire_write_lock, so the trait's // default read->write delegation made read acquisitions observable in // lock_attempts as well; keep that (backlog#1837 PR2). @@ -850,7 +953,7 @@ impl TableCatalogObjectBackend for TestCatalogObjectBackend { .or_insert_with(|| std::sync::Arc::new(tokio::sync::RwLock::new(()))) .clone() }; - Ok(Box::new(lock.read_owned().await)) + Ok(TableCatalogLockGuard::stable(lock.read_owned().await)) } } @@ -1165,6 +1268,8 @@ pub(crate) struct TestTableCatalogStore { pub(crate) fail_put_table_bucket: tokio::sync::Mutex, pub(crate) register_table_pause: Option, pub(crate) commit_table_pause: Option, + pub(crate) create_view_pause: Option, + pub(crate) replace_view_pause: Option, } #[async_trait::async_trait] @@ -1473,6 +1578,10 @@ impl crate::table_catalog::TableCatalogStore for TestTableCatalogStore { entry.table_bucket, entry.namespace ))); } + if let Some(pause) = &self.create_view_pause { + pause.started.notify_one(); + pause.release.notified().await; + } self.views.lock().await.push(entry); Ok(()) } @@ -1531,6 +1640,10 @@ impl crate::table_catalog::TableCatalogStore for TestTableCatalogStore { "current view metadata location does not match expected location".to_string(), )); } + if let Some(pause) = &self.replace_view_pause { + pause.started.notify_one(); + pause.release.notified().await; + } let mut next = current; next.metadata_location = request.new_metadata_location; next.version_token = "token-view-committed".to_string(); diff --git a/rustfs/src/table_catalog/tests.rs b/rustfs/src/table_catalog/tests.rs index 1572c3b16..cfc050343 100644 --- a/rustfs/src/table_catalog/tests.rs +++ b/rustfs/src/table_catalog/tests.rs @@ -338,6 +338,107 @@ async fn catalog_backings_fence_direct_commits_with_publication_lock() { assert_direct_commit_uses_publication_lock(&strong_store, &strong_backend).await; } +#[derive(Default)] +struct LosingTestPublication { + table_checks: std::sync::atomic::AtomicUsize, +} + +#[async_trait::async_trait] +impl TableCommitPublication for LosingTestPublication { + async fn begin_table_bucket(&self, _table_bucket: &str) -> TableCatalogStoreResult<()> { + Ok(()) + } + + async fn prepare(&self, _table_bucket: &str, _namespace: &str, _table: &str) -> TableCatalogStoreResult<()> { + Ok(()) + } + + fn holds_table_bucket(&self, _table_bucket: &str) -> bool { + true + } + + fn holds_table(&self, _table_bucket: &str, _namespace: &str, _table: &str) -> bool { + self.table_checks.fetch_add(1, std::sync::atomic::Ordering::SeqCst) == 0 + } + + fn complete(&self) {} +} + +async fn assert_view_replacement_rechecks_publication_fence(store: &S, backend: &TestCatalogObjectBackend) +where + S: TableCatalogStore, +{ + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let view = IdentifierSegment::parse("recent_orders").expect("view should parse"); + let current_metadata = default_view_metadata_file_path(&namespace, &view, "00001.metadata.json"); + let next_metadata = default_view_metadata_file_path(&namespace, &view, "00002.metadata.json"); + store + .put_table_bucket(test_bucket_entry(bucket)) + .await + .expect("table bucket should be created"); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .expect("namespace should be created"); + store + .create_view(test_view_entry(bucket, &namespace, &view, current_metadata.clone())) + .await + .expect("view should be created"); + backend + .seed_object( + bucket, + &next_metadata, + serde_json::to_vec(&serde_json::json!({ + "format-version": 1, + "view-uuid": "view-uuid", + "location": format!("s3://{bucket}/views/view-id") + })) + .expect("view metadata should encode"), + ) + .await; + + let error = store + .replace_view_with_publication( + ViewCommitRequest { + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + view: view.as_str().to_string(), + expected_version_token: "token-v1".to_string(), + expected_metadata_location: current_metadata.clone(), + new_metadata_location: next_metadata, + }, + true, + &LosingTestPublication::default(), + ) + .await + .expect_err("a lost publication fence must stop the view replacement"); + assert_matches!( + error, + TableCatalogStoreError::Internal(message) if message.contains("publication fence was lost") + ); + + let loaded = store + .load_view(bucket, &namespace.public_name(), view.as_str()) + .await + .expect("view lookup should succeed") + .expect("view should remain present"); + assert_eq!(loaded.metadata_location, current_metadata); + assert_eq!(loaded.version_token, "token-v1"); + assert_eq!(loaded.generation, 1); +} + +#[tokio::test] +async fn catalog_backings_stop_view_replacement_after_publication_fence_loss() { + let object_backend = TestCatalogObjectBackend::default(); + let object_store = ObjectTableCatalogStore::new(object_backend.clone()); + assert_view_replacement_rechecks_publication_fence(&object_store, &object_backend).await; + + let strong_backend = TestCatalogObjectBackend::default(); + let strong_store = StrongTableCatalogStore::new(strong_backend.clone()); + assert_view_replacement_rechecks_publication_fence(&strong_store, &strong_backend).await; +} + #[tokio::test] async fn strong_table_registration_and_drop_acquire_publication_before_migration_read_lock() { let backend = TestCatalogObjectBackend::default(); @@ -795,17 +896,23 @@ async fn strong_catalog_view_replace_rejects_identity_recreation_during_metadata let replace_current_metadata = current_metadata.clone(); let replace = tokio::spawn(async move { replace_store - .replace_view(ViewCommitRequest { - table_bucket: bucket.to_string(), - namespace: replace_namespace.public_name(), - view: replace_view.as_str().to_string(), - expected_version_token: "token-v1".to_string(), - expected_metadata_location: replace_current_metadata, - new_metadata_location: new_metadata, - }) + .replace_view_with_publication( + ViewCommitRequest { + table_bucket: bucket.to_string(), + namespace: replace_namespace.public_name(), + view: replace_view.as_str().to_string(), + expected_version_token: "token-v1".to_string(), + expected_metadata_location: replace_current_metadata, + new_metadata_location: new_metadata, + }, + false, + &UnserializedTestPublication, + ) .await }); - metadata_read.wait_started().await; + tokio::time::timeout(TABLE_CATALOG_TEST_TIMEOUT, metadata_read.wait_started()) + .await + .expect("the replacement should reach the paused metadata read"); store .drop_view(bucket, &namespace.public_name(), view.as_str()) @@ -927,20 +1034,38 @@ fn object_cleanup_report<'a>( .expect("metadata maintenance object cleanup report should exist") } -fn manifest_list_avro_bytes(manifest_paths: &[&str]) -> Vec { - manifest_list_avro_bytes_with_spec(manifest_paths, 0) +fn manifest_list_avro_bytes(manifests: &[(&str, usize)]) -> Vec { + manifest_list_avro_bytes_with_spec(manifests, 0) } -fn manifest_list_avro_bytes_with_spec(manifest_paths: &[&str], partition_spec_id: i32) -> Vec { - // Historical fixed values of this file's fixtures: sequence 7, snapshot 20. - let manifests = manifest_paths +fn manifest_list_avro_bytes_with_spec(manifests: &[(&str, usize)], partition_spec_id: i32) -> Vec { + manifest_list_avro_bytes_with_spec_and_content(manifests, partition_spec_id, 0) +} + +fn manifest_list_avro_bytes_with_spec_and_content(manifests: &[(&str, usize)], partition_spec_id: i32, content: i32) -> Vec { + let manifests = manifests .iter() - .map(|path| (*path, partition_spec_id, 7_i64, 20_i64)) + .map(|(path, length)| (*path, *length, partition_spec_id, content, 7_i64, 20_i64)) .collect::>(); - crate::table_catalog::test_support::manifest_list_avro_entries_with_partition_specs(&manifests) + crate::table_catalog::test_support::manifest_list_avro_entries_with_content(&manifests) } -fn v1_manifest_list_avro_bytes(manifest_path: &str) -> Vec { +fn manifest_list_avro_bytes_with_spec_and_null_counts( + manifests: &[(&str, usize)], + partition_spec_id: i32, + null_counts: bool, +) -> Vec { + if !null_counts { + return manifest_list_avro_bytes_with_spec(manifests, partition_spec_id); + } + let manifests = manifests + .iter() + .map(|(path, length)| (*path, *length, partition_spec_id, 7_i64, 20_i64)) + .collect::>(); + crate::table_catalog::test_support::manifest_list_avro_entries_with_nullable_counts(&manifests) +} + +fn v1_manifest_list_avro_bytes(manifest_path: &str, manifest_length: usize) -> Vec { let schema = apache_avro::Schema::parse_str( r#" { @@ -960,7 +1085,10 @@ fn v1_manifest_list_avro_bytes(manifest_path: &str) -> Vec { writer .append_value(apache_avro::types::Value::Record(vec![ ("manifest_path".to_string(), apache_avro::types::Value::String(manifest_path.to_string())), - ("manifest_length".to_string(), apache_avro::types::Value::Long(1)), + ( + "manifest_length".to_string(), + apache_avro::types::Value::Long(i64::try_from(manifest_length).expect("test manifest length should fit")), + ), ("partition_spec_id".to_string(), apache_avro::types::Value::Int(0)), ("added_snapshot_id".to_string(), apache_avro::types::Value::Long(10)), ])) @@ -997,6 +1125,9 @@ fn v1_manifest_avro_bytes(data_file_path: &str) -> Vec { ) .expect("v1 manifest schema should parse"); let mut writer = apache_avro::Writer::new(&schema, Vec::new()).expect("v1 manifest writer should initialize"); + writer + .add_user_metadata("partition-spec-id".to_string(), "0") + .expect("v1 manifest partition spec metadata should write"); writer .append_value(apache_avro::types::Value::Record(vec![ ("status".to_string(), apache_avro::types::Value::Int(1)), @@ -1059,13 +1190,31 @@ fn iceberg_metadata_validation_accepts_complete_v1_and_v2_shapes() { "schema-id": 0, "fields": [{"id": 1, "name": "id", "required": true, "type": "long"}] }, - "partition-spec": [], + "partition-spec": [{"source-id": 1, "name": "id", "transform": "identity"}], "properties": {}, "snapshots": [], "snapshot-log": [], "metadata-log": [] }); validate_supported_table_metadata(&v1).expect("complete Iceberg v1 metadata should validate"); + + let mut v1_retired_partition_source = v1.clone(); + v1_retired_partition_source["schemas"] = serde_json::json!([ + { + "type": "struct", + "schema-id": 0, + "fields": [{"id": 1, "name": "id", "required": true, "type": "long"}] + }, + {"type": "struct", "schema-id": 1, "fields": []} + ]); + v1_retired_partition_source["current-schema-id"] = serde_json::Value::from(1); + v1_retired_partition_source["schema"] = serde_json::json!({"type": "struct", "schema-id": 1, "fields": []}); + validate_supported_table_metadata(&v1_retired_partition_source) + .expect_err("the v1 partition spec must bind to the current schema rather than a historical schema"); + + let mut negative_v1_schema_id = v1; + negative_v1_schema_id["schema"]["schema-id"] = serde_json::Value::from(-1); + validate_supported_table_metadata(&negative_v1_schema_id).expect_err("Iceberg v1 schema IDs must not be negative"); } #[test] @@ -1099,6 +1248,529 @@ fn iceberg_metadata_validation_rejects_incomplete_v2_and_dangling_references() { assert!(matches!(error, TableCatalogStoreError::Invalid(_))); } +#[test] +fn iceberg_metadata_validation_rejects_invalid_primitive_types_and_field_ids() { + let metadata = table_metadata_json_for_validation(); + for invalid_type in [ + "banana", + "decimal(0,0)", + "decimal(10,11)", + "fixed[0]", + "fixed[2147483648]", + "decimal(10)", + ] { + let mut invalid = metadata.clone(); + invalid["schemas"][0]["fields"][0]["type"] = serde_json::Value::from(invalid_type); + validate_supported_table_metadata(&invalid).expect_err("invalid Iceberg primitive types must be rejected"); + } + for valid_type in [ + "decimal(38,38)", + "decimal(9, 2)", + "decimal( 9 , 2 )", + "fixed[16]", + "fixed[ 16 ]", + "timestamptz", + "uuid", + ] { + let mut valid = metadata.clone(); + valid["schemas"][0]["fields"][0]["type"] = serde_json::Value::from(valid_type); + validate_supported_table_metadata(&valid).expect("standard Iceberg primitive types should validate"); + } + for invalid_id in [0, -1] { + let mut invalid = metadata.clone(); + invalid["schemas"][0]["fields"][0]["id"] = serde_json::Value::from(invalid_id); + validate_supported_table_metadata(&invalid).expect_err("Iceberg field IDs must be positive"); + } + let mut reserved_id = metadata; + reserved_id["schemas"][0]["fields"][0]["id"] = serde_json::Value::from(2_147_483_448_i64); + validate_supported_table_metadata(&reserved_id).expect_err("reserved Iceberg metadata field IDs must be rejected"); + + let mut negative_schema_id = table_metadata_json_for_validation(); + negative_schema_id["schemas"][0]["schema-id"] = serde_json::Value::from(-1); + negative_schema_id["current-schema-id"] = serde_json::Value::from(-1); + validate_supported_table_metadata(&negative_schema_id).expect_err("Iceberg schema IDs must not be negative"); + + let mut negative_last_partition_id = table_metadata_json_for_validation(); + negative_last_partition_id["last-partition-id"] = serde_json::Value::from(-1); + validate_supported_table_metadata(&negative_last_partition_id).expect_err("Iceberg last-partition-id must not be negative"); +} + +#[test] +fn iceberg_metadata_validation_enforces_schema_evolution() { + let mut promoted = table_metadata_json_for_validation(); + promoted["schemas"][0]["fields"][0]["type"] = serde_json::Value::from("int"); + promoted["schemas"] + .as_array_mut() + .expect("schemas should be an array") + .push(serde_json::json!({ + "type": "struct", + "schema-id": 1, + "fields": [{"id": 1, "name": "id", "required": true, "type": "long"}] + })); + promoted["current-schema-id"] = serde_json::Value::from(1); + validate_supported_table_metadata(&promoted).expect("int fields may promote to long"); + + let mut decimal_promotion = table_metadata_json_for_validation(); + decimal_promotion["schemas"][0]["fields"][0]["type"] = serde_json::Value::from("decimal(9, 2)"); + decimal_promotion["schemas"] + .as_array_mut() + .expect("schemas should be an array") + .push(serde_json::json!({ + "type": "struct", + "schema-id": 1, + "fields": [{"id": 1, "name": "id", "required": true, "type": "decimal( 10 , 2 )"}] + })); + decimal_promotion["current-schema-id"] = serde_json::Value::from(1); + validate_supported_table_metadata(&decimal_promotion) + .expect("decimal precision promotion must accept optional parameter whitespace"); + + let mut incompatible = promoted; + incompatible["schemas"][1]["fields"][0]["type"] = serde_json::Value::from("string"); + let error = validate_supported_table_metadata(&incompatible).expect_err("field IDs must retain compatible types"); + assert_eq!( + error, + TableCatalogStoreError::Invalid("schema field 1 has an incompatible type evolution".to_string()) + ); + + let mut moved_into_collection = table_metadata_json_for_validation(); + moved_into_collection["last-column-id"] = serde_json::Value::from(2); + moved_into_collection["schemas"] + .as_array_mut() + .expect("schemas should be an array") + .push(serde_json::json!({ + "type": "struct", + "schema-id": 1, + "fields": [{ + "id": 2, + "name": "items", + "required": true, + "type": { + "type": "list", + "element-id": 1, + "element-required": true, + "element": "long" + } + }] + })); + moved_into_collection["current-schema-id"] = serde_json::Value::from(1); + validate_supported_table_metadata(&moved_into_collection).expect_err("a schema field ID must not move into a list or map"); + + let mut reused = table_metadata_json_for_validation(); + reused["schemas"] = serde_json::json!([ + { + "type": "struct", + "schema-id": 0, + "fields": [{"id": 1, "name": "id", "required": true, "type": "long"}] + }, + {"type": "struct", "schema-id": 1, "fields": []}, + { + "type": "struct", + "schema-id": 2, + "fields": [{"id": 1, "name": "replacement", "required": true, "type": "long"}] + } + ]); + reused["current-schema-id"] = serde_json::Value::from(2); + let error = validate_supported_table_metadata(&reused).expect_err("removed Iceberg field IDs must never be reused"); + assert_eq!( + error, + TableCatalogStoreError::Invalid("schema field 1 cannot be reused after removal".to_string()) + ); + + let mut stale_last_column_id = table_metadata_json_for_validation(); + stale_last_column_id["last-column-id"] = serde_json::Value::from(0); + let error = validate_supported_table_metadata(&stale_last_column_id) + .expect_err("last-column-id must cover nested and top-level assigned field IDs"); + assert_eq!( + error, + TableCatalogStoreError::Invalid( + "last-column-id must be non-negative and cover every assigned schema field id".to_string() + ) + ); +} + +#[test] +fn iceberg_metadata_transition_preserves_assignment_watermarks_and_schema_history() { + let current = table_metadata_json_for_validation(); + + let mut lower_column_watermark = current.clone(); + lower_column_watermark["last-column-id"] = serde_json::Value::from(0); + let error = validate_table_metadata_transition(¤t, &lower_column_watermark) + .expect_err("last-column-id must not decrease across commits"); + assert_eq!( + error, + TableCatalogStoreError::Invalid("last-column-id must not decrease across table metadata commits".to_string()) + ); + + let mut lower_partition_watermark = current.clone(); + lower_partition_watermark["last-partition-id"] = serde_json::Value::from(998); + let error = validate_table_metadata_transition(¤t, &lower_partition_watermark) + .expect_err("last-partition-id must not decrease across commits"); + assert_eq!( + error, + TableCatalogStoreError::Invalid("last-partition-id must not decrease across table metadata commits".to_string()) + ); + + let mut current_sequence = current.clone(); + current_sequence["last-sequence-number"] = serde_json::Value::from(2); + let mut lower_sequence_watermark = current_sequence.clone(); + lower_sequence_watermark["last-sequence-number"] = serde_json::Value::from(1); + let error = validate_table_metadata_transition(¤t_sequence, &lower_sequence_watermark) + .expect_err("last-sequence-number must not decrease across commits"); + assert_eq!( + error, + TableCatalogStoreError::Invalid("last-sequence-number must not decrease across table metadata commits".to_string()) + ); + + let mut current_partitioned = current.clone(); + current_partitioned["partition-specs"] = serde_json::json!([{ + "spec-id": 0, + "fields": [{ + "source-id": 1, + "field-id": 1000, + "name": "id", + "transform": "identity" + }] + }]); + current_partitioned["last-partition-id"] = serde_json::Value::from(1000); + let mut modified_partition = current_partitioned.clone(); + modified_partition["partition-specs"][0]["fields"][0]["name"] = serde_json::Value::from("renamed_id"); + let error = validate_table_metadata_transition(¤t_partitioned, &modified_partition) + .expect_err("published partition specs must be immutable"); + assert_eq!( + error, + TableCatalogStoreError::Invalid("existing partition spec 0 must not be modified".to_string()) + ); + + let mut current_sorted = current.clone(); + current_sorted["sort-orders"] = serde_json::json!([ + {"order-id": 0, "fields": []}, + { + "order-id": 1, + "fields": [{ + "source-id": 1, + "transform": "identity", + "direction": "asc", + "null-order": "nulls-first" + }] + } + ]); + current_sorted["default-sort-order-id"] = serde_json::Value::from(1); + let mut modified_sort = current_sorted.clone(); + modified_sort["sort-orders"][1]["fields"][0]["direction"] = serde_json::Value::from("desc"); + let error = + validate_table_metadata_transition(¤t_sorted, &modified_sort).expect_err("published sort orders must be immutable"); + assert_eq!( + error, + TableCatalogStoreError::Invalid("existing sort order 1 must not be modified".to_string()) + ); + + let mut current_with_snapshot = current.clone(); + current_with_snapshot["last-sequence-number"] = serde_json::Value::from(1); + current_with_snapshot["snapshots"] = serde_json::json!([{ + "snapshot-id": 10, + "sequence-number": 1, + "timestamp-ms": 1, + "manifest-list": "s3://warehouse/tables/table-id/metadata/snap-10.avro", + "summary": {"operation": "append"} + }]); + let mut modified_snapshot = current_with_snapshot.clone(); + modified_snapshot["snapshots"][0]["timestamp-ms"] = serde_json::Value::from(2); + let error = validate_table_metadata_transition(¤t_with_snapshot, &modified_snapshot) + .expect_err("published snapshots must be immutable"); + assert_eq!( + error, + TableCatalogStoreError::Invalid("existing snapshot 10 must not be modified".to_string()) + ); + + let mut modified_history = current.clone(); + modified_history["schemas"][0]["fields"][0]["type"] = serde_json::Value::from("string"); + let error = validate_table_metadata_transition(¤t, &modified_history) + .expect_err("published schema definitions must be immutable"); + assert_eq!( + error, + TableCatalogStoreError::Invalid("existing schema 0 must not be modified".to_string()) + ); + + let mut current_with_retired_id = current.clone(); + current_with_retired_id["schemas"] = serde_json::json!([ + { + "type": "struct", + "schema-id": 0, + "fields": [{"id": 1, "name": "id", "required": true, "type": "long"}] + }, + {"type": "struct", "schema-id": 1, "fields": []} + ]); + current_with_retired_id["current-schema-id"] = serde_json::Value::from(1); + let mut reused_id = current_with_retired_id.clone(); + reused_id["schemas"] = serde_json::json!([ + {"type": "struct", "schema-id": 1, "fields": []}, + { + "type": "struct", + "schema-id": 2, + "fields": [{"id": 1, "name": "replacement", "required": true, "type": "long"}] + } + ]); + reused_id["current-schema-id"] = serde_json::Value::from(2); + let error = validate_table_metadata_transition(¤t_with_retired_id, &reused_id) + .expect_err("new schemas must not reuse a previously assigned field ID"); + assert_eq!( + error, + TableCatalogStoreError::Invalid("schema field 1 cannot reuse a previously assigned field id".to_string()) + ); + + let mut valid = current.clone(); + valid["last-column-id"] = serde_json::Value::from(2); + valid["schemas"] + .as_array_mut() + .expect("schemas should be an array") + .push(serde_json::json!({ + "type": "struct", + "schema-id": 1, + "fields": [ + {"id": 1, "name": "renamed_id", "required": true, "type": "long"}, + {"id": 2, "name": "value", "required": false, "type": "string"} + ] + })); + valid["current-schema-id"] = serde_json::Value::from(1); + validate_table_metadata_transition(¤t, &valid) + .expect("renaming an existing field and allocating a new field ID must remain valid"); +} + +#[test] +fn iceberg_metadata_validation_enforces_identifier_field_contracts() { + let mut metadata = table_metadata_json_for_validation(); + metadata["last-column-id"] = serde_json::Value::from(10); + metadata["schemas"][0]["fields"] = serde_json::json!([ + {"id": 1, "name": "id", "required": true, "type": "long"}, + {"id": 2, "name": "optional_id", "required": false, "type": "string"}, + {"id": 3, "name": "float_id", "required": true, "type": "float"}, + { + "id": 4, + "name": "required_parent", + "required": true, + "type": { + "type": "struct", + "fields": [{"id": 5, "name": "nested_id", "required": true, "type": "string"}] + } + }, + { + "id": 6, + "name": "optional_parent", + "required": false, + "type": { + "type": "struct", + "fields": [{"id": 7, "name": "nested_id", "required": true, "type": "long"}] + } + }, + { + "id": 8, + "name": "ids", + "required": true, + "type": {"type": "list", "element-id": 9, "element-required": true, "element": "long"} + } + ]); + metadata["schemas"][0]["identifier-field-ids"] = serde_json::json!([1, 5]); + validate_supported_table_metadata(&metadata).expect("required primitive fields in required structs may identify rows"); + + for invalid_id in [2, 3, 4, 7, 9] { + let mut invalid = metadata.clone(); + invalid["schemas"][0]["identifier-field-ids"] = serde_json::json!([invalid_id]); + validate_supported_table_metadata(&invalid) + .expect_err("optional, floating, complex, collection, and optional-parent fields must not identify rows"); + } +} + +#[test] +fn iceberg_metadata_validation_binds_partition_fields_to_schema_and_field_identity() { + let mut missing_source = table_metadata_json_for_validation(); + missing_source["partition-specs"] = serde_json::json!([{ + "spec-id": 0, + "fields": [{"source-id": 99, "field-id": 1000, "name": "missing", "transform": "identity"}] + }]); + missing_source["last-partition-id"] = serde_json::Value::from(1000); + validate_supported_table_metadata(&missing_source).expect_err("partition source IDs must reference schema fields"); + + let mut reassigned = table_metadata_json_for_validation(); + reassigned["last-column-id"] = serde_json::Value::from(2); + reassigned["schemas"][0]["fields"] = serde_json::json!([ + {"id": 1, "name": "id", "required": true, "type": "long"}, + {"id": 2, "name": "category", "required": false, "type": "string"} + ]); + reassigned["partition-specs"] = serde_json::json!([ + { + "spec-id": 0, + "fields": [{"source-id": 1, "field-id": 1000, "name": "id", "transform": "identity"}] + }, + { + "spec-id": 1, + "fields": [{"source-id": 2, "field-id": 1000, "name": "category", "transform": "identity"}] + } + ]); + reassigned["last-partition-id"] = serde_json::Value::from(1000); + validate_supported_table_metadata(&reassigned) + .expect_err("a partition field ID must not be reassigned to a different source or transform"); + + reassigned["partition-specs"][1]["fields"][0] = + serde_json::json!({"source-id": 1, "field-id": 1000, "name": "renamed_id", "transform": "identity"}); + validate_supported_table_metadata(&reassigned) + .expect("a historical partition field may retain its ID when only its name changes"); + + let mut v1_missing_source = serde_json::json!({ + "format-version": 1, + "table-uuid": "table-uuid", + "location": "s3://warehouse/tables/table-id", + "last-updated-ms": 1, + "last-column-id": 1, + "schema": { + "type": "struct", + "fields": [{"id": 1, "name": "id", "required": true, "type": "long"}] + }, + "partition-spec": [{"source-id": 2, "name": "missing", "transform": "identity"}] + }); + validate_supported_table_metadata(&v1_missing_source) + .expect_err("Iceberg v1 partition source IDs must reference schema fields"); + v1_missing_source["partition-spec"][0]["source-id"] = serde_json::Value::from(1); + v1_missing_source["partition-spec"][0]["field-id"] = serde_json::Value::from(1001); + validate_supported_table_metadata(&v1_missing_source) + .expect_err("explicit Iceberg v1 partition field IDs must retain sequential compatibility IDs"); + + let mut nested_source = table_metadata_json_for_validation(); + nested_source["last-column-id"] = serde_json::Value::from(4); + nested_source["schemas"][0]["fields"] = serde_json::json!([ + { + "id": 1, + "name": "payload", + "required": true, + "type": { + "type": "struct", + "fields": [{"id": 2, "name": "event_date", "required": true, "type": "date"}] + } + }, + { + "id": 3, + "name": "dates", + "required": true, + "type": {"type": "list", "element-id": 4, "element-required": true, "element": "date"} + } + ]); + nested_source["partition-specs"] = serde_json::json!([{ + "spec-id": 0, + "fields": [{"source-id": 2, "field-id": 1000, "name": "event_day", "transform": "day"}] + }]); + nested_source["last-partition-id"] = serde_json::Value::from(1000); + validate_supported_table_metadata(&nested_source).expect("a primitive nested in a struct may be a partition source"); + + nested_source["partition-specs"][0]["fields"][0]["source-id"] = serde_json::Value::from(4); + validate_supported_table_metadata(&nested_source).expect_err("a primitive nested in a list must not be a partition source"); +} + +#[test] +fn iceberg_metadata_validation_binds_defaults_to_current_schema() { + let mut partitioned = table_metadata_json_for_validation(); + partitioned["schemas"] + .as_array_mut() + .expect("schemas should be an array") + .push(serde_json::json!({"type": "struct", "schema-id": 1, "fields": []})); + partitioned["current-schema-id"] = serde_json::Value::from(1); + partitioned["partition-specs"] = serde_json::json!([ + {"spec-id": 0, "fields": []}, + { + "spec-id": 1, + "fields": [{"source-id": 1, "field-id": 1000, "name": "id", "transform": "identity"}] + } + ]); + partitioned["default-spec-id"] = serde_json::Value::from(1); + partitioned["last-partition-id"] = serde_json::Value::from(1000); + validate_supported_table_metadata(&partitioned).expect_err("the default partition spec must bind to the current schema"); + partitioned["partition-specs"][1]["fields"][0]["transform"] = serde_json::Value::from("void"); + validate_supported_table_metadata(&partitioned) + .expect("a void partition field may retain a source removed from the current schema"); + partitioned["partition-specs"][1]["fields"][0]["transform"] = serde_json::Value::from("identity"); + partitioned["default-spec-id"] = serde_json::Value::from(0); + validate_supported_table_metadata(&partitioned) + .expect("a non-default historical partition spec may retain a source removed from the current schema"); + + let mut sorted = table_metadata_json_for_validation(); + sorted["schemas"] + .as_array_mut() + .expect("schemas should be an array") + .push(serde_json::json!({"type": "struct", "schema-id": 1, "fields": []})); + sorted["current-schema-id"] = serde_json::Value::from(1); + sorted["sort-orders"] = serde_json::json!([ + {"order-id": 0, "fields": []}, + { + "order-id": 1, + "fields": [{ + "source-id": 1, + "transform": "identity", + "direction": "asc", + "null-order": "nulls-first" + }] + } + ]); + sorted["default-sort-order-id"] = serde_json::Value::from(1); + validate_supported_table_metadata(&sorted).expect_err("the default sort order must bind to the current schema"); + sorted["default-sort-order-id"] = serde_json::Value::from(0); + validate_supported_table_metadata(&sorted) + .expect("a non-default historical sort order may retain a source removed from the current schema"); +} + +#[test] +fn iceberg_metadata_validation_binds_transforms_to_source_types() { + let mut valid = table_metadata_json_for_validation(); + valid["schemas"][0]["fields"][0]["type"] = serde_json::Value::from("date"); + valid["partition-specs"] = serde_json::json!([{ + "spec-id": 0, + "fields": [{"source-id": 1, "field-id": 1000, "name": "day", "transform": "day"}] + }]); + valid["last-partition-id"] = serde_json::Value::from(1000); + validate_supported_table_metadata(&valid).expect("day transforms may bind to date fields"); + + let mut invalid_source = valid; + invalid_source["schemas"][0]["fields"][0]["type"] = serde_json::Value::from("string"); + validate_supported_table_metadata(&invalid_source).expect_err("day transforms must reject string fields"); + + let mut invalid_width = table_metadata_json_for_validation(); + invalid_width["partition-specs"] = serde_json::json!([{ + "spec-id": 0, + "fields": [{"source-id": 1, "field-id": 1000, "name": "bucket", "transform": "bucket[0]"}] + }]); + invalid_width["last-partition-id"] = serde_json::Value::from(1000); + validate_supported_table_metadata(&invalid_width).expect_err("bucket widths must be positive"); +} + +#[test] +fn iceberg_metadata_validation_enforces_sort_order_fields() { + let mut metadata = table_metadata_json_for_validation(); + metadata["sort-orders"] = serde_json::json!([{ + "order-id": 1, + "fields": [{ + "source-id": 1, + "transform": "identity", + "direction": "asc", + "null-order": "nulls-first" + }] + }]); + metadata["default-sort-order-id"] = serde_json::Value::from(1); + validate_supported_table_metadata(&metadata).expect("complete sort order fields should validate"); + + for (field, invalid_value) in [ + ("source-id", serde_json::Value::from(99)), + ("transform", serde_json::Value::from("")), + ("direction", serde_json::Value::from("ascending")), + ("null-order", serde_json::Value::from("first")), + ] { + let mut invalid = metadata.clone(); + invalid["sort-orders"][0]["fields"][0][field] = invalid_value; + validate_supported_table_metadata(&invalid).expect_err("invalid Iceberg sort fields must be rejected"); + } + + let mut reserved_unsorted = metadata; + reserved_unsorted["sort-orders"][0]["order-id"] = serde_json::Value::from(0); + reserved_unsorted["default-sort-order-id"] = serde_json::Value::from(0); + validate_supported_table_metadata(&reserved_unsorted).expect_err("sort order 0 must remain unsorted"); +} + #[test] fn iceberg_metadata_validation_enforces_snapshot_and_ref_contracts() { let mut metadata = table_metadata_json_for_validation(); @@ -1122,6 +1794,10 @@ fn iceberg_metadata_validation_enforces_snapshot_and_ref_contracts() { empty_operation["snapshots"][0]["summary"]["operation"] = serde_json::Value::from(""); validate_supported_table_metadata(&empty_operation).expect_err("snapshot operation must not be empty"); + let mut non_string_summary = metadata.clone(); + non_string_summary["snapshots"][0]["summary"]["added-records"] = serde_json::Value::from(1); + validate_supported_table_metadata(&non_string_summary).expect_err("snapshot summary values must be strings"); + let mut missing_timestamp = metadata.clone(); missing_timestamp["snapshots"][0] .as_object_mut() @@ -1129,6 +1805,14 @@ fn iceberg_metadata_validation_enforces_snapshot_and_ref_contracts() { .remove("timestamp-ms"); validate_supported_table_metadata(&missing_timestamp).expect_err("snapshot timestamp must be required"); + let mut malformed_snapshot_log = metadata.clone(); + malformed_snapshot_log["snapshot-log"] = serde_json::json!([{"timestamp-ms": 1, "snapshot-id": "10"}]); + validate_supported_table_metadata(&malformed_snapshot_log).expect_err("snapshot log entries must use integer snapshot IDs"); + + let mut malformed_metadata_log = metadata.clone(); + malformed_metadata_log["metadata-log"] = serde_json::json!([{"timestamp-ms": 1, "metadata-file": ""}]); + validate_supported_table_metadata(&malformed_metadata_log).expect_err("metadata log entries must identify a metadata file"); + let mut mismatched_main = metadata.clone(); mismatched_main["snapshots"] .as_array_mut() @@ -1152,6 +1836,309 @@ fn iceberg_metadata_validation_enforces_snapshot_and_ref_contracts() { validate_supported_table_metadata(&invalid_tag).expect_err("tags must reject branch-only retention fields"); } +#[test] +fn iceberg_metadata_validation_rejects_duplicate_snapshot_statistics() { + let mut metadata = table_metadata_json_for_validation(); + metadata["last-sequence-number"] = serde_json::Value::from(1); + metadata["snapshots"] = serde_json::json!([{ + "snapshot-id": 10, + "sequence-number": 1, + "timestamp-ms": 1, + "manifest-list": "s3://warehouse/tables/table-id/metadata/snap-10.avro", + "summary": {"operation": "append"} + }]); + let statistics = serde_json::json!({ + "snapshot-id": 10, + "statistics-path": "s3://warehouse/tables/table-id/metadata/stats.puffin", + "file-size-in-bytes": 1, + "file-footer-size-in-bytes": 0, + "blob-metadata": [] + }); + metadata["statistics"] = serde_json::json!([statistics.clone(), statistics]); + + let error = validate_supported_table_metadata(&metadata).expect_err("a snapshot must not have duplicate statistics entries"); + assert_eq!( + error, + TableCatalogStoreError::Invalid("statistics contains duplicate entries for snapshot 10".to_string()) + ); +} + +#[tokio::test] +async fn snapshot_validation_bounds_changed_statistics_object_fanout() { + let backend = TestCatalogObjectBackend::default(); + let mut metadata = table_metadata_json_for_validation(); + let object_count = TABLE_COMMIT_MAX_STATISTICS_OBJECTS + 1; + metadata["last-sequence-number"] = serde_json::Value::from(object_count); + metadata["snapshots"] = serde_json::Value::Array( + (1..=object_count) + .map(|snapshot_id| { + serde_json::json!({ + "snapshot-id": snapshot_id, + "sequence-number": snapshot_id, + "timestamp-ms": snapshot_id, + "manifest-list": format!("s3://warehouse/tables/table-id/metadata/snap-{snapshot_id}.avro"), + "summary": {"operation": "append"} + }) + }) + .collect(), + ); + metadata["partition-statistics"] = serde_json::Value::Array( + (1..=object_count) + .map(|snapshot_id| { + serde_json::json!({ + "snapshot-id": snapshot_id, + "statistics-path": format!( + "s3://warehouse/tables/table-id/metadata/partition-stats-{snapshot_id}.parquet" + ), + "file-size-in-bytes": 1 + }) + }) + .collect(), + ); + let entry = TableEntry { + version: TABLE_CATALOG_ENTRY_VERSION, + table_bucket: "warehouse".to_string(), + namespace: "analytics".to_string(), + table: "events".to_string(), + table_id: "table-id".to_string(), + table_uuid: "table-uuid".to_string(), + format: "ICEBERG".to_string(), + format_version: 2, + warehouse_location: "s3://warehouse/tables/table-id".to_string(), + metadata_location: "metadata/00001.metadata.json".to_string(), + version_token: "token-v1".to_string(), + generation: 1, + state: TableCatalogEntryState::Active, + properties: BTreeMap::new(), + created_at: None, + updated_at: None, + }; + let context = TableSnapshotGraphValidationContext::new(&backend, "warehouse", &entry); + + let error = validate_table_snapshot_changes(&context, None, &metadata) + .await + .expect_err("statistics object fanout must be bounded before storage lookups"); + assert_eq!( + error, + TableCatalogStoreError::Invalid("statistics object count exceeds the commit limit".to_string()) + ); +} + +#[tokio::test] +async fn snapshot_validation_bounds_changed_statistics_object_bytes() { + let backend = TestCatalogObjectBackend::default(); + let mut metadata = table_metadata_json_for_validation(); + let object_count = TABLE_COMMIT_MAX_STATISTICS_BYTES / TABLE_STATISTICS_FILE_MAX_SIZE + 1; + metadata["last-sequence-number"] = serde_json::Value::from(object_count); + metadata["snapshots"] = serde_json::Value::Array( + (1..=object_count) + .map(|snapshot_id| { + serde_json::json!({ + "snapshot-id": snapshot_id, + "sequence-number": snapshot_id, + "timestamp-ms": snapshot_id, + "manifest-list": format!("s3://warehouse/tables/table-id/metadata/snap-{snapshot_id}.avro"), + "summary": {"operation": "append"} + }) + }) + .collect(), + ); + metadata["partition-statistics"] = serde_json::Value::Array( + (1..=object_count) + .map(|snapshot_id| { + serde_json::json!({ + "snapshot-id": snapshot_id, + "statistics-path": format!( + "s3://warehouse/tables/table-id/metadata/partition-stats-{snapshot_id}.parquet" + ), + "file-size-in-bytes": TABLE_STATISTICS_FILE_MAX_SIZE + }) + }) + .collect(), + ); + let entry = TableEntry { + version: TABLE_CATALOG_ENTRY_VERSION, + table_bucket: "warehouse".to_string(), + namespace: "analytics".to_string(), + table: "events".to_string(), + table_id: "table-id".to_string(), + table_uuid: "table-uuid".to_string(), + format: "ICEBERG".to_string(), + format_version: 2, + warehouse_location: "s3://warehouse/tables/table-id".to_string(), + metadata_location: "metadata/00001.metadata.json".to_string(), + version_token: "token-v1".to_string(), + generation: 1, + state: TableCatalogEntryState::Active, + properties: BTreeMap::new(), + created_at: None, + updated_at: None, + }; + let context = TableSnapshotGraphValidationContext::new(&backend, "warehouse", &entry); + + let error = validate_table_snapshot_changes(&context, None, &metadata) + .await + .expect_err("statistics bytes must be bounded before storage lookups"); + assert_eq!( + error, + TableCatalogStoreError::Invalid("statistics bytes exceed the commit validation limit".to_string()) + ); +} + +#[tokio::test] +async fn snapshot_validation_rechecks_retained_statistics_locations() { + let backend = TestCatalogObjectBackend::default(); + let namespace = Namespace::parse("analytics").expect("namespace should parse"); + let table = IdentifierSegment::parse("events").expect("table should parse"); + let entry = test_table_entry("warehouse", &namespace, &table, "tables/table-id/metadata/v1.metadata.json".to_string()); + let mut current = table_metadata_json_for_validation(); + current["last-sequence-number"] = serde_json::Value::from(1); + current["snapshots"] = serde_json::json!([{ + "snapshot-id": 10, + "sequence-number": 1, + "timestamp-ms": 1, + "manifest-list": "s3://warehouse/tables/table-id/metadata/missing-history.avro", + "summary": {"operation": "append"} + }]); + current["current-snapshot-id"] = serde_json::Value::from(10); + current["refs"] = serde_json::json!({"main": {"type": "branch", "snapshot-id": 10}}); + current["statistics"] = serde_json::json!([{ + "snapshot-id": 10, + "statistics-path": "s3://warehouse/tables/another-table/metadata/stats.puffin", + "file-size-in-bytes": 1, + "file-footer-size-in-bytes": 0, + "blob-metadata": [] + }]); + let target = current.clone(); + let context = TableSnapshotGraphValidationContext::new(&backend, "warehouse", &entry); + + let error = validate_table_snapshot_changes(&context, Some(¤t), &target) + .await + .expect_err("retained statistics must remain inside the table warehouse"); + assert_eq!( + error, + TableCatalogStoreError::Invalid("statistics object is outside the table warehouse".to_string()) + ); +} + +#[tokio::test] +async fn snapshot_validation_rejects_non_puffin_table_statistics() { + let backend = TestCatalogObjectBackend::default(); + let namespace = Namespace::parse("analytics").expect("namespace should parse"); + let table = IdentifierSegment::parse("events").expect("table should parse"); + let entry = test_table_entry("warehouse", &namespace, &table, "tables/table-id/metadata/v1.metadata.json".to_string()); + let mut current = table_metadata_json_for_validation(); + current["last-sequence-number"] = serde_json::Value::from(1); + current["snapshots"] = serde_json::json!([{ + "snapshot-id": 10, + "sequence-number": 1, + "timestamp-ms": 1, + "manifest-list": "s3://warehouse/tables/table-id/metadata/snap-10.avro", + "summary": {"operation": "append"} + }]); + current["current-snapshot-id"] = serde_json::Value::from(10); + current["refs"] = serde_json::json!({"main": {"type": "branch", "snapshot-id": 10}}); + let mut target = current.clone(); + let statistics = b"notpuffin".to_vec(); + target["statistics"] = serde_json::json!([{ + "snapshot-id": 10, + "statistics-path": "s3://warehouse/tables/table-id/metadata/stats.puffin", + "file-size-in-bytes": statistics.len(), + "file-footer-size-in-bytes": 0, + "blob-metadata": [] + }]); + backend + .seed_object("warehouse", "tables/table-id/metadata/stats.puffin", statistics) + .await; + let context = TableSnapshotGraphValidationContext::new(&backend, "warehouse", &entry); + + let error = validate_table_snapshot_changes(&context, Some(¤t), &target) + .await + .expect_err("table statistics must be a Puffin file"); + assert_eq!( + error, + TableCatalogStoreError::Invalid("table statistics object is not a Puffin file".to_string()) + ); +} + +#[tokio::test] +async fn snapshot_validation_rejects_non_parquet_partition_statistics() { + let backend = TestCatalogObjectBackend::default(); + let namespace = Namespace::parse("analytics").expect("namespace should parse"); + let table = IdentifierSegment::parse("events").expect("table should parse"); + let entry = test_table_entry("warehouse", &namespace, &table, "tables/table-id/metadata/v1.metadata.json".to_string()); + let mut current = table_metadata_json_for_validation(); + current["last-sequence-number"] = serde_json::Value::from(1); + current["snapshots"] = serde_json::json!([{ + "snapshot-id": 10, + "sequence-number": 1, + "timestamp-ms": 1, + "manifest-list": "s3://warehouse/tables/table-id/metadata/snap-10.avro", + "summary": {"operation": "append"} + }]); + current["current-snapshot-id"] = serde_json::Value::from(10); + current["refs"] = serde_json::json!({"main": {"type": "branch", "snapshot-id": 10}}); + let mut target = current.clone(); + let statistics = b"notparquet".to_vec(); + target["partition-statistics"] = serde_json::json!([{ + "snapshot-id": 10, + "statistics-path": "s3://warehouse/tables/table-id/metadata/partition-stats.parquet", + "file-size-in-bytes": statistics.len() + }]); + backend + .seed_object("warehouse", "tables/table-id/metadata/partition-stats.parquet", statistics) + .await; + let context = TableSnapshotGraphValidationContext::new(&backend, "warehouse", &entry); + + let error = validate_table_snapshot_changes(&context, Some(¤t), &target) + .await + .expect_err("partition statistics must be a Parquet file"); + assert_eq!( + error, + TableCatalogStoreError::Invalid("partition statistics object is not a Parquet file".to_string()) + ); +} + +#[tokio::test] +async fn snapshot_validation_rejects_statistics_size_mismatch() { + let backend = TestCatalogObjectBackend::default(); + let namespace = Namespace::parse("analytics").expect("namespace should parse"); + let table = IdentifierSegment::parse("events").expect("table should parse"); + let entry = test_table_entry("warehouse", &namespace, &table, "tables/table-id/metadata/v1.metadata.json".to_string()); + let mut current = table_metadata_json_for_validation(); + current["last-sequence-number"] = serde_json::Value::from(1); + current["snapshots"] = serde_json::json!([{ + "snapshot-id": 10, + "sequence-number": 1, + "timestamp-ms": 1, + "manifest-list": "s3://warehouse/tables/table-id/metadata/snap-10.avro", + "summary": {"operation": "append"} + }]); + current["current-snapshot-id"] = serde_json::Value::from(10); + current["refs"] = serde_json::json!({"main": {"type": "branch", "snapshot-id": 10}}); + let mut target = current.clone(); + let statistics = b"PFA1PFA1".to_vec(); + target["statistics"] = serde_json::json!([{ + "snapshot-id": 10, + "statistics-path": "s3://warehouse/tables/table-id/metadata/stats.puffin", + "file-size-in-bytes": statistics.len() + 1, + "file-footer-size-in-bytes": 0, + "blob-metadata": [] + }]); + backend + .seed_object("warehouse", "tables/table-id/metadata/stats.puffin", statistics) + .await; + let context = TableSnapshotGraphValidationContext::new(&backend, "warehouse", &entry); + + let error = validate_table_snapshot_changes(&context, Some(¤t), &target) + .await + .expect_err("statistics lengths must match the published object"); + assert_eq!( + error, + TableCatalogStoreError::Invalid("statistics file-size-in-bytes does not match the object".to_string()) + ); +} + #[test] fn iceberg_metadata_validation_bounds_snapshot_sequence_numbers() { let mut v2 = table_metadata_json_for_validation(); @@ -1170,6 +2157,16 @@ fn iceberg_metadata_validation_bounds_snapshot_sequence_numbers() { "Iceberg v2 snapshot sequence-number must be between zero and last-sequence-number".to_string() ) ); + v2["last-sequence-number"] = serde_json::Value::from(0); + v2["snapshots"][0] + .as_object_mut() + .expect("snapshot should be an object") + .remove("sequence-number"); + let error = validate_supported_table_metadata(&v2).expect_err("Iceberg v2 snapshots must include sequence-number"); + assert_eq!( + error, + TableCatalogStoreError::Invalid("Iceberg v2 snapshot sequence-number is required".to_string()) + ); let mut v1 = serde_json::json!({ "format-version": 1, @@ -1193,6 +2190,11 @@ fn iceberg_metadata_validation_bounds_snapshot_sequence_numbers() { ); v1["snapshots"][0]["sequence-number"] = serde_json::Value::from(0); validate_supported_table_metadata(&v1).expect("a zero v1 compatibility sequence should validate"); + v1["snapshots"][0] + .as_object_mut() + .expect("snapshot should be an object") + .remove("sequence-number"); + validate_supported_table_metadata(&v1).expect("Iceberg v1 snapshots may omit sequence-number"); } #[test] @@ -1261,7 +2263,7 @@ fn iceberg_metadata_version_synchronization_builds_complete_v2_shape() { } #[test] -fn iceberg_manifest_validation_accepts_deflate_and_rejects_unknown_content() { +fn iceberg_manifest_validation_accepts_standard_codecs_and_rejects_unknown_content() { let schema = apache_avro::Schema::parse_str( r#" { @@ -1275,22 +2277,29 @@ fn iceberg_manifest_validation_accepts_deflate_and_rejects_unknown_content() { "#, ) .expect("manifest list schema should parse"); - let mut writer = apache_avro::Writer::with_codec(&schema, Vec::new(), apache_avro::Codec::Deflate(Default::default())) - .expect("compressed manifest list writer should initialize"); - writer - .append_value(apache_avro::types::Value::Record(vec![ - ( - "manifest_path".to_string(), - apache_avro::types::Value::String("s3://warehouse/tables/table-id/metadata/manifest.avro".to_string()), - ), - ("partition_spec_id".to_string(), apache_avro::types::Value::Int(0)), - ])) - .expect("compressed manifest list record should append"); - let compressed = writer.into_inner().expect("compressed manifest list should flush"); - let references = manifest_list_references_from_manifest_list_avro(&compressed) - .expect("deflate-compressed manifest lists should be supported with bounded decoding"); - assert_eq!(references.len(), 1); - assert_eq!(references[0].manifest_path, "s3://warehouse/tables/table-id/metadata/manifest.avro"); + for (label, codec) in [ + ("null", apache_avro::Codec::Null), + ("deflate", apache_avro::Codec::Deflate(Default::default())), + ("snappy", apache_avro::Codec::Snappy), + ("zstandard", apache_avro::Codec::Zstandard(Default::default())), + ] { + let mut writer = apache_avro::Writer::with_codec(&schema, Vec::new(), codec) + .expect("compressed manifest list writer should initialize"); + writer + .append_value(apache_avro::types::Value::Record(vec![ + ( + "manifest_path".to_string(), + apache_avro::types::Value::String("s3://warehouse/tables/table-id/metadata/manifest.avro".to_string()), + ), + ("partition_spec_id".to_string(), apache_avro::types::Value::Int(0)), + ])) + .expect("compressed manifest list record should append"); + let compressed = writer.into_inner().expect("compressed manifest list should flush"); + let references = manifest_list_references_from_manifest_list_avro(&compressed) + .unwrap_or_else(|error| panic!("{label}-compressed manifest lists should be supported: {error}")); + assert_eq!(references.len(), 1); + assert_eq!(references[0].manifest_path, "s3://warehouse/tables/table-id/metadata/manifest.avro"); + } let unknown_content = manifest_avro_bytes_with_status(&[("s3://warehouse/tables/table-id/data/part.parquet", 3, 1)]); let error = data_file_references_from_manifest_avro(&unknown_content) @@ -1366,7 +2375,7 @@ async fn iceberg_snapshot_graph_rejects_unknown_partition_specs() { .seed_object( "warehouse", "tables/table-id/metadata/snap-10.avro", - manifest_list_avro_bytes_with_spec(&[manifest_location], 7), + manifest_list_avro_bytes_with_spec(&[(manifest_location, 1)], 7), ) .await; let mut metadata = table_metadata_json_for_validation(); @@ -1391,6 +2400,118 @@ async fn iceberg_snapshot_graph_rejects_unknown_partition_specs() { ); } +#[tokio::test] +async fn iceberg_snapshot_graph_accepts_null_manifest_list_counts() { + let backend = TestCatalogObjectBackend::default(); + let namespace = Namespace::parse("analytics").expect("namespace should parse"); + let table = IdentifierSegment::parse("events").expect("table should parse"); + let entry = test_table_entry("warehouse", &namespace, &table, "tables/table-id/metadata/v1.metadata.json".to_string()); + let manifest_list_location = "s3://warehouse/tables/table-id/metadata/snap-10.avro"; + let manifest_location = "s3://warehouse/tables/table-id/metadata/manifest-10.avro"; + let manifest = manifest_avro_bytes(&[]); + backend + .seed_object( + "warehouse", + "tables/table-id/metadata/snap-10.avro", + manifest_list_avro_bytes_with_spec_and_null_counts(&[(manifest_location, manifest.len())], 0, true), + ) + .await; + backend + .seed_object("warehouse", "tables/table-id/metadata/manifest-10.avro", manifest) + .await; + let mut metadata = table_metadata_json_for_validation(); + metadata["last-sequence-number"] = serde_json::Value::from(7); + metadata["snapshots"] = serde_json::json!([{ + "snapshot-id": 10, + "sequence-number": 7, + "timestamp-ms": 1, + "manifest-list": manifest_list_location, + "summary": {"operation": "append"} + }]); + metadata["current-snapshot-id"] = serde_json::Value::from(10); + metadata["refs"] = serde_json::json!({"main": {"type": "branch", "snapshot-id": 10}}); + let context = TableSnapshotGraphValidationContext::new(&backend, "warehouse", &entry); + + validate_table_snapshot_changes(&context, None, &metadata) + .await + .expect("nullable v2 manifest-list counts must remain compatible"); +} + +#[tokio::test] +async fn iceberg_snapshot_graph_revalidates_unchanged_snapshots_after_spec_removal() { + let backend = TestCatalogObjectBackend::default(); + let namespace = Namespace::parse("analytics").expect("namespace should parse"); + let table = IdentifierSegment::parse("events").expect("table should parse"); + let entry = test_table_entry("warehouse", &namespace, &table, "tables/table-id/metadata/v1.metadata.json".to_string()); + let manifest_list_location = "s3://warehouse/tables/table-id/metadata/snap-10.avro"; + let manifest_location = "s3://warehouse/tables/table-id/metadata/manifest-10.avro"; + backend + .seed_object( + "warehouse", + "tables/table-id/metadata/snap-10.avro", + manifest_list_avro_bytes_with_spec(&[(manifest_location, 1)], 7), + ) + .await; + let mut current = table_metadata_json_for_validation(); + current["partition-specs"] + .as_array_mut() + .expect("partition specs should be an array") + .push(serde_json::json!({"spec-id": 7, "fields": []})); + current["last-sequence-number"] = serde_json::Value::from(7); + current["snapshots"] = serde_json::json!([{ + "snapshot-id": 10, + "sequence-number": 7, + "timestamp-ms": 1, + "manifest-list": manifest_list_location, + "summary": {"operation": "append"} + }]); + current["current-snapshot-id"] = serde_json::Value::from(10); + current["refs"] = serde_json::json!({"main": {"type": "branch", "snapshot-id": 10}}); + let mut target = current.clone(); + target["partition-specs"] + .as_array_mut() + .expect("partition specs should be an array") + .retain(|spec| spec.get("spec-id").and_then(serde_json::Value::as_i64) != Some(7)); + let context = TableSnapshotGraphValidationContext::new(&backend, "warehouse", &entry); + + let error = validate_table_snapshot_changes(&context, Some(¤t), &target) + .await + .expect_err("removing a spec referenced by an unchanged snapshot must fail"); + assert_eq!( + error, + TableCatalogStoreError::Invalid("snapshot manifest references missing partition spec 7".to_string()) + ); +} + +#[tokio::test] +async fn iceberg_snapshot_graph_skips_unchanged_history_after_spec_addition() { + let backend = TestCatalogObjectBackend::default(); + let namespace = Namespace::parse("analytics").expect("namespace should parse"); + let table = IdentifierSegment::parse("events").expect("table should parse"); + let entry = test_table_entry("warehouse", &namespace, &table, "tables/table-id/metadata/v1.metadata.json".to_string()); + let mut current = table_metadata_json_for_validation(); + current["last-sequence-number"] = serde_json::Value::from(1); + current["snapshots"] = serde_json::json!([{ + "snapshot-id": 10, + "sequence-number": 1, + "timestamp-ms": 1, + "manifest-list": "s3://warehouse/tables/table-id/metadata/missing-history.avro", + "summary": {"operation": "append"} + }]); + current["current-snapshot-id"] = serde_json::Value::from(10); + current["refs"] = serde_json::json!({"main": {"type": "branch", "snapshot-id": 10}}); + let mut target = current.clone(); + target["partition-specs"] + .as_array_mut() + .expect("partition specs should be an array") + .push(serde_json::json!({"spec-id": 7, "fields": []})); + let context = TableSnapshotGraphValidationContext::new(&backend, "warehouse", &entry); + + validate_table_snapshot_changes(&context, Some(¤t), &target) + .await + .expect("adding a partition spec must not reread unchanged snapshot history"); +} + #[tokio::test] async fn iceberg_snapshot_graph_allows_missing_deleted_files() { let backend = TestCatalogObjectBackend::default(); @@ -1400,19 +2521,16 @@ async fn iceberg_snapshot_graph_allows_missing_deleted_files() { let manifest_list_location = "s3://warehouse/tables/table-id/metadata/snap-10.avro"; let manifest_location = "s3://warehouse/tables/table-id/metadata/manifest-10.avro"; let deleted_data_location = "s3://warehouse/tables/table-id/data/deleted.parquet"; + let manifest_bytes = manifest_avro_bytes_with_status(&[(deleted_data_location, 0, 2)]); backend .seed_object( "warehouse", "tables/table-id/metadata/snap-10.avro", - manifest_list_avro_bytes(&[manifest_location]), + manifest_list_avro_bytes(&[(manifest_location, manifest_bytes.len())]), ) .await; backend - .seed_object( - "warehouse", - "tables/table-id/metadata/manifest-10.avro", - manifest_avro_bytes_with_status(&[(deleted_data_location, 0, 2)]), - ) + .seed_object("warehouse", "tables/table-id/metadata/manifest-10.avro", manifest_bytes) .await; let mut metadata = table_metadata_json_for_validation(); metadata["last-sequence-number"] = serde_json::Value::from(7); @@ -1469,19 +2587,16 @@ async fn iceberg_v2_snapshot_graph_accepts_reused_v1_manifests() { let manifest_list_location = "s3://warehouse/tables/table-id/metadata/snap-v1.avro"; let manifest_location = "s3://warehouse/tables/table-id/metadata/manifest-v1.avro"; let data_location = "s3://warehouse/tables/table-id/data/v1.parquet"; + let manifest_bytes = v1_manifest_avro_bytes(data_location); backend .seed_object( "warehouse", "tables/table-id/metadata/snap-v1.avro", - v1_manifest_list_avro_bytes(manifest_location), + v1_manifest_list_avro_bytes(manifest_location, manifest_bytes.len()), ) .await; backend - .seed_object( - "warehouse", - "tables/table-id/metadata/manifest-v1.avro", - v1_manifest_avro_bytes(data_location), - ) + .seed_object("warehouse", "tables/table-id/metadata/manifest-v1.avro", manifest_bytes) .await; backend .seed_object("warehouse", "tables/table-id/data/v1.parquet", vec![1]) @@ -1549,7 +2664,7 @@ async fn iceberg_snapshot_change_validation_skips_unchanged_history() { } #[tokio::test] -async fn iceberg_snapshot_registration_validates_only_active_snapshot_heads() { +async fn iceberg_snapshot_registration_validates_all_retained_snapshots() { let backend = TestCatalogObjectBackend::default(); let namespace = Namespace::parse("analytics").expect("namespace should parse"); let table = IdentifierSegment::parse("events").expect("table should parse"); @@ -1579,9 +2694,13 @@ async fn iceberg_snapshot_registration_validates_only_active_snapshot_heads() { metadata["refs"] = serde_json::json!({"main": {"type": "branch", "snapshot-id": 11}}); let context = TableSnapshotGraphValidationContext::new(&backend, "warehouse", &entry); - validate_table_snapshot_changes(&context, None, &metadata) + let error = validate_table_snapshot_changes(&context, None, &metadata) .await - .expect("registration should validate active snapshot heads without traversing all history"); + .expect_err("registration must validate every retained snapshot"); + assert_eq!( + error, + TableCatalogStoreError::Invalid("snapshot manifest-list object is missing".to_string()) + ); } #[tokio::test] @@ -1593,19 +2712,16 @@ async fn iceberg_snapshot_graph_counts_shared_manifests_once() { let manifest_list = "s3://warehouse/tables/table-id/metadata/shared-list.avro"; let manifest = "s3://warehouse/tables/table-id/metadata/shared-manifest.avro"; let data_file = "s3://warehouse/tables/table-id/data/shared.parquet"; + let manifest_bytes = manifest_avro_bytes(&[(data_file, 0)]); backend .seed_object( "warehouse", "tables/table-id/metadata/shared-list.avro", - manifest_list_avro_bytes(&[manifest]), + manifest_list_avro_bytes(&[(manifest, manifest_bytes.len())]), ) .await; backend - .seed_object( - "warehouse", - "tables/table-id/metadata/shared-manifest.avro", - manifest_avro_bytes(&[(data_file, 0)]), - ) + .seed_object("warehouse", "tables/table-id/metadata/shared-manifest.avro", manifest_bytes) .await; backend .seed_object("warehouse", "tables/table-id/data/shared.parquet", vec![1]) @@ -1637,6 +2753,142 @@ async fn iceberg_snapshot_graph_counts_shared_manifests_once() { .expect("shared manifest objects must consume the commit budget only once"); } +#[tokio::test] +async fn iceberg_snapshot_graph_revalidates_cached_manifest_declarations() { + let backend = TestCatalogObjectBackend::default(); + let namespace = Namespace::parse("analytics").expect("namespace should parse"); + let table = IdentifierSegment::parse("events").expect("table should parse"); + let entry = test_table_entry("warehouse", &namespace, &table, "tables/table-id/metadata/v1.metadata.json".to_string()); + let manifest = "s3://warehouse/tables/table-id/metadata/shared-manifest.avro"; + let data_file = "s3://warehouse/tables/table-id/data/shared.parquet"; + let manifest_bytes = manifest_avro_bytes_with_status_and_partition_spec(&[(data_file, 0, 1)], 0); + let manifest_length = manifest_bytes.len(); + backend + .seed_object( + "warehouse", + "tables/table-id/metadata/list-10.avro", + manifest_list_avro_bytes(&[(manifest, manifest_length)]), + ) + .await; + backend + .seed_object( + "warehouse", + "tables/table-id/metadata/list-11.avro", + manifest_list_avro_bytes(&[(manifest, manifest_length + 1)]), + ) + .await; + backend + .seed_object("warehouse", "tables/table-id/metadata/shared-manifest.avro", manifest_bytes) + .await; + backend + .seed_object("warehouse", "tables/table-id/data/shared.parquet", vec![1]) + .await; + let mut metadata = table_metadata_json_for_validation(); + metadata["last-sequence-number"] = serde_json::Value::from(7); + metadata["snapshots"] = serde_json::json!([ + { + "snapshot-id": 10, + "sequence-number": 7, + "timestamp-ms": 1, + "manifest-list": "s3://warehouse/tables/table-id/metadata/list-10.avro", + "summary": {"operation": "append"} + }, + { + "snapshot-id": 11, + "sequence-number": 7, + "timestamp-ms": 2, + "manifest-list": "s3://warehouse/tables/table-id/metadata/list-11.avro", + "summary": {"operation": "append"} + } + ]); + metadata["current-snapshot-id"] = serde_json::Value::from(11); + metadata["refs"] = serde_json::json!({"main": {"type": "branch", "snapshot-id": 11}}); + let current_metadata = table_metadata_json_for_validation(); + let context = TableSnapshotGraphValidationContext::new(&backend, "warehouse", &entry); + + let error = validate_table_snapshot_changes(&context, Some(¤t_metadata), &metadata) + .await + .expect_err("every manifest-list declaration must match the cached manifest object"); + assert_eq!( + error, + TableCatalogStoreError::Invalid("manifest-list manifest_length does not match the manifest object".to_string()) + ); + + backend + .seed_object( + "warehouse", + "tables/table-id/metadata/list-11.avro", + manifest_list_avro_bytes_with_spec(&[(manifest, manifest_length)], 1), + ) + .await; + metadata["partition-specs"] + .as_array_mut() + .expect("partition specs should be an array") + .push(serde_json::json!({"spec-id": 1, "fields": []})); + + let error = validate_table_snapshot_changes(&context, Some(¤t_metadata), &metadata) + .await + .expect_err("every cached manifest must match each manifest-list partition spec declaration"); + assert_eq!( + error, + TableCatalogStoreError::Invalid("manifest partition-spec-id does not match its manifest-list entry".to_string()) + ); +} + +#[tokio::test] +async fn iceberg_snapshot_graph_bounds_shared_manifest_traversals() { + let backend = TestCatalogObjectBackend::default(); + let namespace = Namespace::parse("analytics").expect("namespace should parse"); + let table = IdentifierSegment::parse("events").expect("table should parse"); + let entry = test_table_entry("warehouse", &namespace, &table, "tables/table-id/metadata/v1.metadata.json".to_string()); + let manifest_list = "s3://warehouse/tables/table-id/metadata/shared-list.avro"; + let manifest = "s3://warehouse/tables/table-id/metadata/shared-manifest.avro"; + let data_file = "s3://warehouse/tables/table-id/data/shared.parquet"; + let manifest_bytes = manifest_avro_bytes(&[(data_file, 0)]); + backend + .seed_object( + "warehouse", + "tables/table-id/metadata/shared-list.avro", + manifest_list_avro_bytes(&[(manifest, manifest_bytes.len())]), + ) + .await; + backend + .seed_object("warehouse", "tables/table-id/metadata/shared-manifest.avro", manifest_bytes) + .await; + backend + .seed_object("warehouse", "tables/table-id/data/shared.parquet", vec![1]) + .await; + let mut metadata = table_metadata_json_for_validation(); + metadata["last-sequence-number"] = serde_json::Value::from(7); + metadata["snapshots"] = serde_json::Value::Array( + (0..=TABLE_COMMIT_MAX_MANIFEST_TRAVERSALS) + .map(|index| { + let snapshot_id = i64::try_from(index + 1).expect("snapshot id should fit in i64"); + serde_json::json!({ + "snapshot-id": snapshot_id, + "sequence-number": 7, + "timestamp-ms": snapshot_id, + "manifest-list": manifest_list, + "summary": {"operation": "append"} + }) + }) + .collect(), + ); + let current_snapshot_id = i64::try_from(TABLE_COMMIT_MAX_MANIFEST_TRAVERSALS + 1).expect("snapshot id should fit in i64"); + metadata["current-snapshot-id"] = serde_json::Value::from(current_snapshot_id); + metadata["refs"] = serde_json::json!({"main": {"type": "branch", "snapshot-id": current_snapshot_id}}); + let current_metadata = table_metadata_json_for_validation(); + let context = TableSnapshotGraphValidationContext::new(&backend, "warehouse", &entry); + + let error = validate_table_snapshot_changes(&context, Some(¤t_metadata), &metadata) + .await + .expect_err("logical manifest traversals must remain bounded across shared snapshots"); + assert_eq!( + error, + TableCatalogStoreError::Invalid("snapshot manifest traversal count exceeds the commit limit".to_string()) + ); +} + #[tokio::test] async fn iceberg_snapshot_graph_accepts_more_than_ten_thousand_live_files() { let backend = TestCatalogObjectBackend::default(); @@ -1645,14 +2897,6 @@ async fn iceberg_snapshot_graph_accepts_more_than_ten_thousand_live_files() { let entry = test_table_entry("warehouse", &namespace, &table, "tables/table-id/metadata/v1.metadata.json".to_string()); let manifest_list = "s3://warehouse/tables/table-id/metadata/boundary-list.avro"; let manifest = "s3://warehouse/tables/table-id/metadata/boundary-manifest.avro"; - backend - .seed_object( - "warehouse", - "tables/table-id/metadata/boundary-list.avro", - manifest_list_avro_bytes(&[manifest]), - ) - .await; - let data_file_count = 10_001; let data_files = (0..data_file_count) .map(|index| format!("s3://warehouse/tables/table-id/data/part-{index:05}.parquet")) @@ -1669,13 +2913,17 @@ async fn iceberg_snapshot_graph_accepts_more_than_ten_thousand_live_files() { .await; } let references = data_files.iter().map(|data_file| (data_file.as_str(), 0)).collect::>(); + let manifest_bytes = manifest_avro_bytes(&references); backend .seed_object( "warehouse", - "tables/table-id/metadata/boundary-manifest.avro", - manifest_avro_bytes(&references), + "tables/table-id/metadata/boundary-list.avro", + manifest_list_avro_bytes(&[(manifest, manifest_bytes.len())]), ) .await; + backend + .seed_object("warehouse", "tables/table-id/metadata/boundary-manifest.avro", manifest_bytes) + .await; let mut metadata = table_metadata_json_for_validation(); metadata["last-sequence-number"] = serde_json::Value::from(7); @@ -1696,7 +2944,7 @@ async fn iceberg_snapshot_graph_accepts_more_than_ten_thousand_live_files() { } #[tokio::test] -async fn iceberg_v2_snapshot_graph_accepts_embedded_v2_manifests() { +async fn iceberg_v2_snapshot_graph_rejects_embedded_v2_manifests() { let backend = TestCatalogObjectBackend::default(); let namespace = Namespace::parse("analytics").expect("namespace should parse"); let table = IdentifierSegment::parse("events").expect("table should parse"); @@ -1726,13 +2974,17 @@ async fn iceberg_v2_snapshot_graph_accepts_embedded_v2_manifests() { metadata["refs"] = serde_json::json!({"main": {"type": "branch", "snapshot-id": 10}}); let context = TableSnapshotGraphValidationContext::new(&backend, "warehouse", &entry); - validate_table_snapshot_changes(&context, None, &metadata) + let error = validate_table_snapshot_changes(&context, None, &metadata) .await - .expect("embedded v2 manifests should use the enclosing snapshot sequence bound"); + .expect_err("new v2 snapshots must use a manifest list"); + assert_eq!( + error, + TableCatalogStoreError::Invalid("new Iceberg v2 snapshots require manifest-list".to_string()) + ); } #[tokio::test] -async fn iceberg_snapshot_graph_accepts_delete_files_in_data_directory() { +async fn iceberg_snapshot_graph_rejects_delete_files_in_data_manifest() { let backend = TestCatalogObjectBackend::default(); let namespace = Namespace::parse("analytics").expect("namespace should parse"); let table = IdentifierSegment::parse("events").expect("table should parse"); @@ -1740,20 +2992,62 @@ async fn iceberg_snapshot_graph_accepts_delete_files_in_data_directory() { let manifest_list = "s3://warehouse/tables/table-id/metadata/list-10.avro"; let manifest = "s3://warehouse/tables/table-id/metadata/snap-delete-manifest.avro"; let delete_file = "s3://warehouse/tables/table-id/data/position-deletes.parquet"; + let manifest_bytes = manifest_avro_bytes(&[(delete_file, 1)]); backend .seed_object( "warehouse", "tables/table-id/metadata/list-10.avro", - manifest_list_avro_bytes(&[manifest]), + manifest_list_avro_bytes(&[(manifest, manifest_bytes.len())]), ) .await; + backend + .seed_object("warehouse", "tables/table-id/metadata/snap-delete-manifest.avro", manifest_bytes) + .await; + backend + .seed_object("warehouse", "tables/table-id/data/position-deletes.parquet", vec![1]) + .await; + let mut metadata = table_metadata_json_for_validation(); + metadata["last-sequence-number"] = serde_json::Value::from(7); + metadata["snapshots"] = serde_json::json!([{ + "snapshot-id": 10, + "sequence-number": 7, + "timestamp-ms": 1, + "manifest-list": manifest_list, + "summary": {"operation": "delete"} + }]); + metadata["current-snapshot-id"] = serde_json::Value::from(10); + metadata["refs"] = serde_json::json!({"main": {"type": "branch", "snapshot-id": 10}}); + let context = TableSnapshotGraphValidationContext::new(&backend, "warehouse", &entry); + + let error = validate_table_snapshot_changes(&context, None, &metadata) + .await + .expect_err("a data manifest must not contain delete files"); + assert_eq!( + error, + TableCatalogStoreError::Invalid("manifest-list content does not match manifest file content".to_string()) + ); +} + +#[tokio::test] +async fn iceberg_snapshot_graph_accepts_delete_files_in_delete_manifest() { + let backend = TestCatalogObjectBackend::default(); + let namespace = Namespace::parse("analytics").expect("namespace should parse"); + let table = IdentifierSegment::parse("events").expect("table should parse"); + let entry = test_table_entry("warehouse", &namespace, &table, "tables/table-id/metadata/v1.metadata.json".to_string()); + let manifest_list = "s3://warehouse/tables/table-id/metadata/list-10.avro"; + let manifest = "s3://warehouse/tables/table-id/metadata/snap-delete-manifest.avro"; + let delete_file = "s3://warehouse/tables/table-id/data/position-deletes.parquet"; + let manifest_bytes = manifest_avro_bytes(&[(delete_file, 1)]); backend .seed_object( "warehouse", - "tables/table-id/metadata/snap-delete-manifest.avro", - manifest_avro_bytes(&[(delete_file, 1)]), + "tables/table-id/metadata/list-10.avro", + manifest_list_avro_bytes_with_spec_and_content(&[(manifest, manifest_bytes.len())], 0, 1), ) .await; + backend + .seed_object("warehouse", "tables/table-id/metadata/snap-delete-manifest.avro", manifest_bytes) + .await; backend .seed_object("warehouse", "tables/table-id/data/position-deletes.parquet", vec![1]) .await; @@ -1772,7 +3066,97 @@ async fn iceberg_snapshot_graph_accepts_delete_files_in_data_directory() { validate_table_snapshot_changes(&context, None, &metadata) .await - .expect("manifest content should identify delete files stored under the data directory"); + .expect("a delete manifest may contain delete files regardless of their object directory"); +} + +#[tokio::test] +async fn iceberg_snapshot_graph_rejects_manifest_length_mismatch() { + let backend = TestCatalogObjectBackend::default(); + let namespace = Namespace::parse("analytics").expect("namespace should parse"); + let table = IdentifierSegment::parse("events").expect("table should parse"); + let entry = test_table_entry("warehouse", &namespace, &table, "tables/table-id/metadata/v1.metadata.json".to_string()); + let manifest_list = "s3://warehouse/tables/table-id/metadata/list-20.avro"; + let manifest = "s3://warehouse/tables/table-id/metadata/manifest-20.avro"; + let data_file = "s3://warehouse/tables/table-id/data/part-20.parquet"; + let manifest_bytes = manifest_avro_bytes(&[(data_file, 0)]); + backend + .seed_object( + "warehouse", + "tables/table-id/metadata/list-20.avro", + manifest_list_avro_bytes(&[(manifest, manifest_bytes.len() + 1)]), + ) + .await; + backend + .seed_object("warehouse", "tables/table-id/metadata/manifest-20.avro", manifest_bytes) + .await; + backend + .seed_object("warehouse", "tables/table-id/data/part-20.parquet", vec![1]) + .await; + let mut metadata = table_metadata_json_for_validation(); + metadata["last-sequence-number"] = serde_json::Value::from(7); + metadata["snapshots"] = serde_json::json!([{ + "snapshot-id": 20, + "sequence-number": 7, + "timestamp-ms": 1, + "manifest-list": manifest_list, + "summary": {"operation": "append"} + }]); + metadata["current-snapshot-id"] = serde_json::Value::from(20); + metadata["refs"] = serde_json::json!({"main": {"type": "branch", "snapshot-id": 20}}); + let context = TableSnapshotGraphValidationContext::new(&backend, "warehouse", &entry); + + let error = validate_table_snapshot_changes(&context, None, &metadata) + .await + .expect_err("manifest-list lengths must match the published manifest object"); + assert_eq!( + error, + TableCatalogStoreError::Invalid("manifest-list manifest_length does not match the manifest object".to_string()) + ); +} + +#[tokio::test] +async fn iceberg_snapshot_graph_rejects_manifest_partition_spec_mismatch() { + let backend = TestCatalogObjectBackend::default(); + let namespace = Namespace::parse("analytics").expect("namespace should parse"); + let table = IdentifierSegment::parse("events").expect("table should parse"); + let entry = test_table_entry("warehouse", &namespace, &table, "tables/table-id/metadata/v1.metadata.json".to_string()); + let manifest_list = "s3://warehouse/tables/table-id/metadata/list-20.avro"; + let manifest = "s3://warehouse/tables/table-id/metadata/manifest-20.avro"; + let data_file = "s3://warehouse/tables/table-id/data/part-20.parquet"; + let manifest_bytes = manifest_avro_bytes_with_status_and_partition_spec(&[(data_file, 0, 1)], 7); + backend + .seed_object( + "warehouse", + "tables/table-id/metadata/list-20.avro", + manifest_list_avro_bytes(&[(manifest, manifest_bytes.len())]), + ) + .await; + backend + .seed_object("warehouse", "tables/table-id/metadata/manifest-20.avro", manifest_bytes) + .await; + backend + .seed_object("warehouse", "tables/table-id/data/part-20.parquet", vec![1]) + .await; + let mut metadata = table_metadata_json_for_validation(); + metadata["last-sequence-number"] = serde_json::Value::from(7); + metadata["snapshots"] = serde_json::json!([{ + "snapshot-id": 20, + "sequence-number": 7, + "timestamp-ms": 1, + "manifest-list": manifest_list, + "summary": {"operation": "append"} + }]); + metadata["current-snapshot-id"] = serde_json::Value::from(20); + metadata["refs"] = serde_json::json!({"main": {"type": "branch", "snapshot-id": 20}}); + let context = TableSnapshotGraphValidationContext::new(&backend, "warehouse", &entry); + + let error = validate_table_snapshot_changes(&context, None, &metadata) + .await + .expect_err("manifest headers must agree with their manifest-list entry"); + assert_eq!( + error, + TableCatalogStoreError::Invalid("manifest partition-spec-id does not match its manifest-list entry".to_string()) + ); } #[tokio::test] @@ -1797,8 +3181,6 @@ fn manifest_avro_bytes(files: &[(&str, i32)]) -> Vec { } fn manifest_avro_bytes_with_status(files: &[(&str, i32, i32)]) -> Vec { - // Historical fixed values of this file's fixtures: snapshot 20, sequence 7 - // (the shared constructor takes snapshot_id fourth, sequence fifth). let files = files .iter() .map(|(path, content, status)| (*path, *content, *status, 20_i64, 7_i64)) @@ -1806,6 +3188,14 @@ fn manifest_avro_bytes_with_status(files: &[(&str, i32, i32)]) -> Vec { crate::table_catalog::test_support::manifest_avro_bytes(&files) } +fn manifest_avro_bytes_with_status_and_partition_spec(files: &[(&str, i32, i32)], partition_spec_id: i32) -> Vec { + let files = files + .iter() + .map(|(path, content, status)| (*path, *content, *status, 20_i64, 7_i64)) + .collect::>(); + crate::table_catalog::test_support::manifest_avro_bytes_with_partition_spec(&files, Some(partition_spec_id)) +} + fn manifest_avro_bytes_with_dt_partition(files: &[(&str, i32, &str)]) -> Vec { let schema = apache_avro::Schema::parse_str( r#" @@ -1839,6 +3229,9 @@ fn manifest_avro_bytes_with_dt_partition(files: &[(&str, i32, &str)]) -> Vec ) .expect("partitioned manifest avro schema should parse"); let mut writer = apache_avro::Writer::new(&schema, Vec::new()).expect("partitioned manifest writer should initialize"); + writer + .add_user_metadata("partition-spec-id".to_string(), "0") + .expect("manifest partition spec metadata should write"); for (file_path, content, partition_value) in files { writer .append_value(apache_avro::types::Value::Record(vec![ @@ -1887,6 +3280,7 @@ fn manifest_avro_bytes_with_sort_order(files: &[(&str, i32, i32)]) -> Vec { "fields": [ {"name": "content", "type": "int"}, {"name": "file_path", "type": "string"}, + {"name": "partition", "type": {"type": "record", "name": "partition", "fields": []}}, {"name": "record_count", "type": "long"}, {"name": "file_size_in_bytes", "type": "long"}, {"name": "sort_order_id", "type": ["null", "int"], "default": null} @@ -1899,6 +3293,9 @@ fn manifest_avro_bytes_with_sort_order(files: &[(&str, i32, i32)]) -> Vec { ) .expect("sort-order manifest avro schema should parse"); let mut writer = apache_avro::Writer::new(&schema, Vec::new()).expect("sorted manifest writer should initialize"); + writer + .add_user_metadata("partition-spec-id".to_string(), "0") + .expect("manifest partition spec metadata should write"); for (file_path, content, sort_order_id) in files { writer .append_value(apache_avro::types::Value::Record(vec![ @@ -1911,6 +3308,7 @@ fn manifest_avro_bytes_with_sort_order(files: &[(&str, i32, i32)]) -> Vec { apache_avro::types::Value::Record(vec![ ("content".to_string(), apache_avro::types::Value::Int(*content)), ("file_path".to_string(), apache_avro::types::Value::String((*file_path).to_string())), + ("partition".to_string(), apache_avro::types::Value::Record(Vec::new())), ("record_count".to_string(), apache_avro::types::Value::Long(1)), ("file_size_in_bytes".to_string(), apache_avro::types::Value::Long(1)), ( @@ -2299,6 +3697,61 @@ async fn object_table_catalog_store_persists_view_entries_and_blocks_non_empty_n store.drop_namespace(bucket, &namespace.public_name()).await.unwrap(); } +#[tokio::test] +async fn object_catalog_view_replacement_recovers_after_committed_write_response_loss() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let view = IdentifierSegment::parse("recent_orders").expect("view should parse"); + let current_metadata = default_view_metadata_file_path(&namespace, &view, "00001.metadata.json"); + let next_metadata = default_view_metadata_file_path(&namespace, &view, "00002.metadata.json"); + store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .unwrap(); + store + .create_view(test_view_entry(bucket, &namespace, &view, current_metadata.clone())) + .await + .unwrap(); + backend + .seed_object( + bucket, + &next_metadata, + serde_json::to_vec(&serde_json::json!({ + "format-version": 1, + "view-uuid": "view-uuid", + "location": format!("s3://{bucket}/views/view-id") + })) + .expect("view metadata should encode"), + ) + .await; + let view_path = store.paths.view_entry_path(bucket, &namespace, &view); + backend.fail_after_next_put(RUSTFS_META_BUCKET, &view_path).await; + + let replaced = store + .replace_view(ViewCommitRequest { + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + view: view.as_str().to_string(), + expected_version_token: "token-v1".to_string(), + expected_metadata_location: current_metadata, + new_metadata_location: next_metadata.clone(), + }) + .await + .expect("an exact persisted replacement must prove the ambiguous write succeeded"); + + assert_eq!(replaced.view.metadata_location, next_metadata); + assert_eq!(replaced.view.generation, 2); + let loaded = store + .load_view(bucket, &namespace.public_name(), view.as_str()) + .await + .expect("view lookup should succeed") + .expect("view should remain present"); + assert_eq!(loaded, replaced.view); +} + #[tokio::test] async fn maintenance_dry_run_keeps_current_metadata() { let backend = TestCatalogObjectBackend::default(); @@ -5206,14 +6659,13 @@ async fn maintenance_worker_preserves_queued_dry_run_after_delete_is_enabled() { let data_file = format!("{table_root}data/part-00001.parquet"); let orphan_data = format!("{table_root}data/orphan.parquet"); let now = OffsetDateTime::UNIX_EPOCH + Duration::seconds(100); + let manifest_bytes = manifest_avro_bytes(&[(&data_file, 0)]); seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; backend - .seed_object(bucket, &manifest_list, manifest_list_avro_bytes(&[&manifest])) - .await; - backend - .seed_object(bucket, &manifest, manifest_avro_bytes(&[(&data_file, 0)])) + .seed_object(bucket, &manifest_list, manifest_list_avro_bytes(&[(&manifest, manifest_bytes.len())])) .await; + backend.seed_object(bucket, &manifest, manifest_bytes).await; backend.seed_object(bucket, &data_file, b"data".to_vec()).await; backend.seed_object(bucket, &orphan_data, b"orphan-data".to_vec()).await; backend @@ -6140,14 +7592,13 @@ async fn maintenance_reachability_expands_manifest_avro_references() { let manifest = format!("{metadata_dir}/manifest-10.avro"); let data_file = format!("{table_root}data/part-00001.parquet"); let delete_file = format!("{table_root}delete/pos-00001.parquet"); + let manifest_bytes = manifest_avro_bytes(&[(&data_file, 0), (&delete_file, 1)]); seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; backend - .seed_object(bucket, &manifest_list, manifest_list_avro_bytes(&[&manifest])) - .await; - backend - .seed_object(bucket, &manifest, manifest_avro_bytes(&[(&data_file, 0), (&delete_file, 1)])) + .seed_object(bucket, &manifest_list, manifest_list_avro_bytes(&[(&manifest, manifest_bytes.len())])) .await; + backend.seed_object(bucket, &manifest, manifest_bytes).await; backend.seed_object(bucket, &data_file, b"data".to_vec()).await; backend.seed_object(bucket, &delete_file, b"delete".to_vec()).await; backend @@ -6281,14 +7732,13 @@ async fn maintenance_reachability_uses_table_warehouse_object_paths() { let manifest = "tables/table-id/metadata/manifest-10.avro".to_string(); let data_file = "tables/table-id/data/part-00001.parquet".to_string(); let orphan_data = "tables/table-id/data/orphan.parquet".to_string(); + let manifest_bytes = manifest_avro_bytes(&[(&data_file, 0)]); seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; backend - .seed_object(bucket, &manifest_list, manifest_list_avro_bytes(&[&manifest])) - .await; - backend - .seed_object(bucket, &manifest, manifest_avro_bytes(&[(&data_file, 0)])) + .seed_object(bucket, &manifest_list, manifest_list_avro_bytes(&[(&manifest, manifest_bytes.len())])) .await; + backend.seed_object(bucket, &manifest, manifest_bytes).await; backend.seed_object(bucket, &data_file, b"data".to_vec()).await; backend.seed_object(bucket, &orphan_data, b"orphan".to_vec()).await; backend @@ -6407,14 +7857,13 @@ async fn maintenance_dry_run_reports_unreachable_manifest_data_and_delete_candid let orphan_manifest = format!("{metadata_dir}/manifest-orphan.avro"); let orphan_data = format!("{table_root}data/orphan.parquet"); let orphan_delete = format!("{table_root}delete/orphan-delete.parquet"); + let manifest_bytes = manifest_avro_bytes(&[(&data_file, 0)]); seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; backend - .seed_object(bucket, &manifest_list, manifest_list_avro_bytes(&[&manifest])) - .await; - backend - .seed_object(bucket, &manifest, manifest_avro_bytes(&[(&data_file, 0)])) + .seed_object(bucket, &manifest_list, manifest_list_avro_bytes(&[(&manifest, manifest_bytes.len())])) .await; + backend.seed_object(bucket, &manifest, manifest_bytes).await; backend.seed_object(bucket, &data_file, b"data".to_vec()).await; backend.seed_object(bucket, &orphan_manifest, manifest_avro_bytes(&[])).await; backend.seed_object(bucket, &orphan_data, b"orphan-data".to_vec()).await; @@ -6485,14 +7934,13 @@ async fn maintenance_delete_removes_only_planned_unreachable_table_objects() { let data_file = format!("{table_root}data/part-00001.parquet"); let orphan_manifest = format!("{metadata_dir}/manifest-orphan.avro"); let orphan_data = format!("{table_root}data/orphan.parquet"); + let manifest_bytes = manifest_avro_bytes(&[(&data_file, 0)]); seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; backend - .seed_object(bucket, &manifest_list, manifest_list_avro_bytes(&[&manifest])) - .await; - backend - .seed_object(bucket, &manifest, manifest_avro_bytes(&[(&data_file, 0)])) + .seed_object(bucket, &manifest_list, manifest_list_avro_bytes(&[(&manifest, manifest_bytes.len())])) .await; + backend.seed_object(bucket, &manifest, manifest_bytes).await; backend.seed_object(bucket, &data_file, b"data".to_vec()).await; backend.seed_object(bucket, &orphan_manifest, manifest_avro_bytes(&[])).await; backend.seed_object(bucket, &orphan_data, b"orphan-data".to_vec()).await; @@ -6947,18 +8395,13 @@ async fn compaction_plan_reports_row_level_delete_files_without_rewrite_candidat let data_file = format!("{data_dir}/part-left.parquet"); let position_delete_file = format!("{delete_dir}/pos-left.parquet"); let equality_delete_file = format!("{delete_dir}/eq-left.parquet"); + let manifest_bytes = manifest_avro_bytes(&[(&data_file, 0), (&position_delete_file, 1), (&equality_delete_file, 2)]); seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; backend - .seed_object(bucket, &manifest_list, manifest_list_avro_bytes(&[&manifest])) - .await; - backend - .seed_object( - bucket, - &manifest, - manifest_avro_bytes(&[(&data_file, 0), (&position_delete_file, 1), (&equality_delete_file, 2)]), - ) + .seed_object(bucket, &manifest_list, manifest_list_avro_bytes(&[(&manifest, manifest_bytes.len())])) .await; + backend.seed_object(bucket, &manifest, manifest_bytes).await; backend.seed_object(bucket, &data_file, parquet_i32_bytes(&[1, 2])).await; backend .seed_object(bucket, &position_delete_file, b"position-delete".to_vec()) @@ -7130,18 +8573,13 @@ async fn compaction_commit_rewrites_small_data_files_and_advances_pointer() { let retained_values = (10..20_000).collect::>(); let retained_parquet = parquet_i32_bytes(&retained_values); let small_file_threshold_bytes = u64::try_from(left_parquet.len().max(right_parquet.len())).unwrap(); + let manifest_bytes = manifest_avro_bytes(&[(&left_data, 0), (&right_data, 0), (&retained_data, 0)]); seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; backend - .seed_object(bucket, &manifest_list, manifest_list_avro_bytes(&[&manifest])) - .await; - backend - .seed_object( - bucket, - &manifest, - manifest_avro_bytes(&[(&left_data, 0), (&right_data, 0), (&retained_data, 0)]), - ) + .seed_object(bucket, &manifest_list, manifest_list_avro_bytes(&[(&manifest, manifest_bytes.len())])) .await; + backend.seed_object(bucket, &manifest, manifest_bytes).await; backend.seed_object(bucket, &left_data, left_parquet).await; backend.seed_object(bucket, &right_data, right_parquet).await; backend.seed_object(bucket, &retained_data, retained_parquet).await; @@ -7309,22 +8747,17 @@ async fn compaction_commit_keeps_partition_rewrite_groups_isolated() { let other_partition_parquet = parquet_i32_bytes(&[5, 6]); let small_file_threshold_bytes = u64::try_from(left_parquet.len().max(right_parquet.len()).max(other_partition_parquet.len())).unwrap(); + let manifest_bytes = manifest_avro_bytes_with_dt_partition(&[ + (&left_data, 0, "2026-06-24"), + (&right_data, 0, "2026-06-24"), + (&other_partition_data, 0, "2026-06-25"), + ]); seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; backend - .seed_object(bucket, &manifest_list, manifest_list_avro_bytes(&[&manifest])) - .await; - backend - .seed_object( - bucket, - &manifest, - manifest_avro_bytes_with_dt_partition(&[ - (&left_data, 0, "2026-06-24"), - (&right_data, 0, "2026-06-24"), - (&other_partition_data, 0, "2026-06-25"), - ]), - ) + .seed_object(bucket, &manifest_list, manifest_list_avro_bytes(&[(&manifest, manifest_bytes.len())])) .await; + backend.seed_object(bucket, &manifest, manifest_bytes).await; backend.seed_object(bucket, &left_data, left_parquet).await; backend.seed_object(bucket, &right_data, right_parquet).await; backend @@ -7495,18 +8928,14 @@ async fn compaction_commit_preserves_sort_order_and_keeps_groups_isolated() { let other_sort_parquet = parquet_i32_bytes(&[5, 6]); let small_file_threshold_bytes = u64::try_from(left_parquet.len().max(right_parquet.len()).max(other_sort_parquet.len())).unwrap(); + let manifest_bytes = + manifest_avro_bytes_with_sort_order(&[(&left_data, 0, 7), (&right_data, 0, 7), (&other_sort_data, 0, 8)]); seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; backend - .seed_object(bucket, &manifest_list, manifest_list_avro_bytes(&[&manifest])) - .await; - backend - .seed_object( - bucket, - &manifest, - manifest_avro_bytes_with_sort_order(&[(&left_data, 0, 7), (&right_data, 0, 7), (&other_sort_data, 0, 8)]), - ) + .seed_object(bucket, &manifest_list, manifest_list_avro_bytes(&[(&manifest, manifest_bytes.len())])) .await; + backend.seed_object(bucket, &manifest, manifest_bytes).await; backend.seed_object(bucket, &left_data, left_parquet).await; backend.seed_object(bucket, &right_data, right_parquet).await; backend.seed_object(bucket, &other_sort_data, other_sort_parquet).await; @@ -7632,14 +9061,13 @@ async fn compaction_commit_rejects_schema_mismatch_without_advancing_pointer() { let manifest = format!("{metadata_dir}/manifest-20.avro"); let left_data = format!("{data_dir}/part-left.parquet"); let right_data = format!("{data_dir}/part-right.parquet"); + let manifest_bytes = manifest_avro_bytes(&[(&left_data, 0), (&right_data, 0)]); seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; backend - .seed_object(bucket, &manifest_list, manifest_list_avro_bytes(&[&manifest])) - .await; - backend - .seed_object(bucket, &manifest, manifest_avro_bytes(&[(&left_data, 0), (&right_data, 0)])) + .seed_object(bucket, &manifest_list, manifest_list_avro_bytes(&[(&manifest, manifest_bytes.len())])) .await; + backend.seed_object(bucket, &manifest, manifest_bytes).await; backend.seed_object(bucket, &left_data, parquet_i32_bytes(&[1, 2])).await; backend.seed_object(bucket, &right_data, parquet_i64_bytes(&[3, 4])).await; backend @@ -7708,18 +9136,13 @@ async fn compaction_commit_rejects_deleted_manifest_entries_without_advancing_po let manifest = format!("{metadata_dir}/manifest-20.avro"); let left_data = format!("{data_dir}/part-left.parquet"); let deleted_data = format!("{data_dir}/part-deleted.parquet"); + let manifest_bytes = manifest_avro_bytes_with_status(&[(&left_data, 0, 1), (&deleted_data, 0, 2)]); seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; backend - .seed_object(bucket, &manifest_list, manifest_list_avro_bytes(&[&manifest])) - .await; - backend - .seed_object( - bucket, - &manifest, - manifest_avro_bytes_with_status(&[(&left_data, 0, 1), (&deleted_data, 0, 2)]), - ) + .seed_object(bucket, &manifest_list, manifest_list_avro_bytes(&[(&manifest, manifest_bytes.len())])) .await; + backend.seed_object(bucket, &manifest, manifest_bytes).await; backend.seed_object(bucket, &left_data, parquet_i32_bytes(&[1, 2])).await; backend.seed_object(bucket, &deleted_data, parquet_i32_bytes(&[3, 4])).await; backend @@ -11643,31 +13066,45 @@ async fn strong_catalog_does_not_guess_view_history_from_a_concurrent_replacemen let first_expected_metadata = initial_metadata.clone(); let first_replace = tokio::spawn(async move { first_replace_store - .replace_view(ViewCommitRequest { - table_bucket: bucket.to_string(), - namespace: first_namespace_name, - view: first_view_name, - expected_version_token: "token-v1".to_string(), - expected_metadata_location: first_expected_metadata, - new_metadata_location: first_metadata, - }) + .replace_view_with_publication( + ViewCommitRequest { + table_bucket: bucket.to_string(), + namespace: first_namespace_name, + view: first_view_name, + expected_version_token: "token-v1".to_string(), + expected_metadata_location: first_expected_metadata, + new_metadata_location: first_metadata, + }, + false, + &UnserializedTestPublication, + ) .await }); - recovery_read.wait_started().await; - second - .replace_view(ViewCommitRequest { - table_bucket: bucket.to_string(), - namespace: namespace.public_name(), - view: view.as_str().to_string(), - expected_version_token: "token-v1".to_string(), - expected_metadata_location: initial_metadata, - new_metadata_location: second_metadata.clone(), - }) + tokio::time::timeout(TABLE_CATALOG_TEST_TIMEOUT, recovery_read.wait_started()) .await - .expect("second writer should publish a different replacement"); + .expect("the first replacement should reach its paused recovery read"); + tokio::time::timeout( + TABLE_CATALOG_TEST_TIMEOUT, + second.replace_view_with_publication( + ViewCommitRequest { + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + view: view.as_str().to_string(), + expected_version_token: "token-v1".to_string(), + expected_metadata_location: initial_metadata, + new_metadata_location: second_metadata.clone(), + }, + false, + &UnserializedTestPublication, + ), + ) + .await + .expect("the independent replacement should not wait for publication serialization") + .expect("second writer should publish a different replacement"); recovery_read.release(); - let error = first_replace + let error = tokio::time::timeout(TABLE_CATALOG_TEST_TIMEOUT, first_replace) .await + .expect("the first replacement should finish after its recovery read is released") .expect("first replacement task should join") .expect_err("a different generation-two view must not prove the first replacement succeeded"); assert_matches!(error, TableCatalogStoreError::Internal(message) if message.contains("injected put failure")); @@ -12407,6 +13844,76 @@ async fn catalog_backings_reject_table_view_identifier_collisions() { } } +#[tokio::test] +async fn catalog_backings_persist_table_format_upgrade_and_replay() { + for mode in [TableCatalogBackingMode::ObjectBacked, TableCatalogBackingMode::DurableStrong] { + let backend = TestCatalogObjectBackend::default(); + let store = ConfiguredTableCatalogStore::new_for_test(backend.clone(), mode); + let bucket = format!("format-upgrade-{mode:?}").to_ascii_lowercase(); + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let next_metadata = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + + store + .put_table_bucket(test_bucket_entry(&bucket)) + .await + .expect("table bucket should be created"); + store + .create_namespace(test_namespace_entry(&bucket, &namespace)) + .await + .expect("namespace should be created"); + let mut entry = test_table_entry(&bucket, &namespace, &table, current_metadata.clone()); + entry.format_version = 1; + store.create_table(entry).await.expect("v1 table should be created"); + backend + .seed_object( + &bucket, + &next_metadata, + serde_json::to_vec(&serde_json::json!({ + "format-version": 2, + "table-uuid": "table-uuid", + "location": format!("s3://{bucket}/tables/table-id") + })) + .expect("target metadata should encode"), + ) + .await; + let request = TableCommitRequest { + table_bucket: bucket.clone(), + namespace: namespace.public_name(), + table: table.as_str().to_string(), + commit_id: "format-upgrade-commit".to_string(), + idempotency_key: Some("format-upgrade-request".to_string()), + operation: "upgrade-format-version".to_string(), + expected_version_token: "token-v1".to_string(), + expected_metadata_location: current_metadata, + new_metadata_location: next_metadata, + requirements: Vec::new(), + writer: Some("iceberg-rest/test".to_string()), + }; + + let committed = store + .commit_table(request.clone()) + .await + .expect("format upgrade should commit"); + assert_eq!(committed.table.format_version, 2); + let replay = store + .commit_table(request) + .await + .expect("exact format upgrade replay should succeed"); + assert_eq!(replay, committed); + + let restarted = ConfiguredTableCatalogStore::new_for_test(backend, mode); + let loaded = restarted + .load_table(&bucket, &namespace.public_name(), table.as_str()) + .await + .expect("restarted catalog should load") + .expect("upgraded table should persist"); + assert_eq!(loaded.format_version, 2); + assert_eq!(loaded.metadata_location, committed.table.metadata_location); + } +} + #[tokio::test] async fn catalog_backings_hide_and_reject_mutation_of_inactive_resources() { for mode in [TableCatalogBackingMode::ObjectBacked, TableCatalogBackingMode::DurableStrong] { diff --git a/scripts/table-catalog/failure_coverage.py b/scripts/table-catalog/failure_coverage.py index aae4f1e93..db1614294 100644 --- a/scripts/table-catalog/failure_coverage.py +++ b/scripts/table-catalog/failure_coverage.py @@ -132,18 +132,8 @@ def failure_probe_plan(warehouse: str, namespace: str, table: str, rest_path: st "expected-version-token": "stale-token-from-previous-load", "expected-metadata-location": "current-metadata-location-from-load-table", "new-metadata-location": f"s3://{warehouse}/tables/table-id/metadata/conflict_probe.metadata.json", - "requirements": [ - { - "type": "assert-current-snapshot-id", - "snapshot-id": 0, - } - ], - "updates": [ - { - "action": "set-current-schema", - "schema-id": 0, - } - ], + "requirements": [], + "updates": [], }, ), probe_step( @@ -157,6 +147,8 @@ def failure_probe_plan(warehouse: str, namespace: str, table: str, rest_path: st "expected-version-token": "current-version-token-from-load-table", "expected-metadata-location": "current-metadata-location-from-load-table", "new-metadata-location": f"s3://{warehouse}/tables/table-id/metadata/does_not_exist.metadata.json", + "requirements": [], + "updates": [], }, ), probe_step( diff --git a/scripts/table-catalog/test_failure_coverage.py b/scripts/table-catalog/test_failure_coverage.py index 164e8f492..60773fa71 100644 --- a/scripts/table-catalog/test_failure_coverage.py +++ b/scripts/table-catalog/test_failure_coverage.py @@ -48,6 +48,8 @@ class FailureCoverageTest(unittest.TestCase): self.assertIn("expected-version-token", by_name["stale-token-commit-conflict"]["body"]) self.assertIn("expected-metadata-location", by_name["stale-token-commit-conflict"]["body"]) self.assertIn("new-metadata-location", by_name["stale-token-commit-conflict"]["body"]) + self.assertEqual(by_name["stale-token-commit-conflict"]["body"]["requirements"], []) + self.assertEqual(by_name["stale-token-commit-conflict"]["body"]["updates"], []) self.assertNotIn("base", by_name["stale-token-commit-conflict"]["body"]) self.assertEqual( by_name["diagnostics-after-finalization-gap"]["path"], @@ -56,6 +58,8 @@ class FailureCoverageTest(unittest.TestCase): self.assertEqual(by_name["diagnostics-after-finalization-gap"]["method"], "GET") self.assertEqual(by_name["recovery-repairs-idempotency-index"]["method"], "POST") self.assertIn("does_not_exist.metadata.json", json.dumps(by_name["missing-metadata-object-rejected"])) + self.assertEqual(by_name["missing-metadata-object-rejected"]["body"]["requirements"], []) + self.assertEqual(by_name["missing-metadata-object-rejected"]["body"]["updates"], []) self.assertNotIn("base", by_name["missing-metadata-object-rejected"]["body"]) def test_cli_prints_failure_matrix_and_probe_plan(self) -> None: